@peerbit/document 1.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +111 -0
- package/lib/esm/document-index.d.ts +126 -0
- package/lib/esm/document-index.js +800 -0
- package/lib/esm/document-index.js.map +1 -0
- package/lib/esm/document-store.d.ts +60 -0
- package/lib/esm/document-store.js +349 -0
- package/lib/esm/document-store.js.map +1 -0
- package/lib/esm/index.d.ts +3 -0
- package/lib/esm/index.js +4 -0
- package/lib/esm/index.js.map +1 -0
- package/lib/esm/package.json +3 -0
- package/lib/esm/query.d.ts +184 -0
- package/lib/esm/query.js +579 -0
- package/lib/esm/query.js.map +1 -0
- package/lib/esm/utils.d.ts +3 -0
- package/lib/esm/utils.js +12 -0
- package/lib/esm/utils.js.map +1 -0
- package/package.json +46 -0
- package/src/document-index.ts +1029 -0
- package/src/document-store.ts +439 -0
- package/src/index.ts +3 -0
- package/src/query.ts +510 -0
- package/src/utils.ts +17 -0
|
@@ -0,0 +1,800 @@
|
|
|
1
|
+
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
2
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
3
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
4
|
+
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
5
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
6
|
+
};
|
|
7
|
+
var __metadata = (this && this.__metadata) || function (k, v) {
|
|
8
|
+
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
9
|
+
};
|
|
10
|
+
import { field, serialize, variant } from "@dao-xyz/borsh";
|
|
11
|
+
import { asString } from "./utils.js";
|
|
12
|
+
import { BORSH_ENCODING } from "@peerbit/log";
|
|
13
|
+
import { equals } from "@peerbit/uint8arrays";
|
|
14
|
+
import { ComposableProgram } from "@peerbit/program";
|
|
15
|
+
import { IntegerCompare, ByteMatchQuery, StringMatch, ResultWithSource, StateFieldQuery, compare, MissingField, StringMatchMethod, LogicalQuery, And, Or, BoolQuery, CollectNextRequest, AbstractSearchRequest, SearchRequest, SortDirection, CloseIteratorRequest, } from "./query.js";
|
|
16
|
+
import { RPC, queryAll, MissingResponsesError, } from "@peerbit/rpc";
|
|
17
|
+
import { Results } from "./query.js";
|
|
18
|
+
import { logger as loggerFn } from "@peerbit/logger";
|
|
19
|
+
import { Cache } from "@peerbit/cache";
|
|
20
|
+
const logger = loggerFn({ module: "document-index" });
|
|
21
|
+
const stringArraysEquals = (a, b) => {
|
|
22
|
+
if (a.length !== b.length) {
|
|
23
|
+
return false;
|
|
24
|
+
}
|
|
25
|
+
for (let i = 0; i < a.length; i++) {
|
|
26
|
+
if (a[i] !== b[i]) {
|
|
27
|
+
return false;
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
return true;
|
|
31
|
+
};
|
|
32
|
+
export let Operation = class Operation {
|
|
33
|
+
};
|
|
34
|
+
Operation = __decorate([
|
|
35
|
+
variant(0)
|
|
36
|
+
], Operation);
|
|
37
|
+
export const BORSH_ENCODING_OPERATION = BORSH_ENCODING(Operation);
|
|
38
|
+
export let PutOperation = class PutOperation extends Operation {
|
|
39
|
+
key;
|
|
40
|
+
data;
|
|
41
|
+
_value;
|
|
42
|
+
constructor(props) {
|
|
43
|
+
super();
|
|
44
|
+
if (props) {
|
|
45
|
+
this.key = props.key;
|
|
46
|
+
this.data = props.data;
|
|
47
|
+
this._value = props.value;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
get value() {
|
|
51
|
+
if (!this._value) {
|
|
52
|
+
throw new Error("Value not decoded, invoke getValue(...) once");
|
|
53
|
+
}
|
|
54
|
+
return this._value;
|
|
55
|
+
}
|
|
56
|
+
getValue(encoding) {
|
|
57
|
+
if (this._value) {
|
|
58
|
+
return this._value;
|
|
59
|
+
}
|
|
60
|
+
this._value = encoding.decoder(this.data);
|
|
61
|
+
return this._value;
|
|
62
|
+
}
|
|
63
|
+
};
|
|
64
|
+
__decorate([
|
|
65
|
+
field({ type: "string" }),
|
|
66
|
+
__metadata("design:type", String)
|
|
67
|
+
], PutOperation.prototype, "key", void 0);
|
|
68
|
+
__decorate([
|
|
69
|
+
field({ type: Uint8Array }),
|
|
70
|
+
__metadata("design:type", Uint8Array)
|
|
71
|
+
], PutOperation.prototype, "data", void 0);
|
|
72
|
+
PutOperation = __decorate([
|
|
73
|
+
variant(0),
|
|
74
|
+
__metadata("design:paramtypes", [Object])
|
|
75
|
+
], PutOperation);
|
|
76
|
+
/* @variant(1)
|
|
77
|
+
export class PutAllOperation<T> extends Operation<T> {
|
|
78
|
+
@field({ type: vec(PutOperation) })
|
|
79
|
+
docs: PutOperation<T>[];
|
|
80
|
+
|
|
81
|
+
constructor(props?: { docs: PutOperation<T>[] }) {
|
|
82
|
+
super();
|
|
83
|
+
if (props) {
|
|
84
|
+
this.docs = props.docs;
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
*/
|
|
89
|
+
export let DeleteOperation = class DeleteOperation extends Operation {
|
|
90
|
+
key;
|
|
91
|
+
constructor(props) {
|
|
92
|
+
super();
|
|
93
|
+
if (props) {
|
|
94
|
+
this.key = props.key;
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
};
|
|
98
|
+
__decorate([
|
|
99
|
+
field({ type: "string" }),
|
|
100
|
+
__metadata("design:type", String)
|
|
101
|
+
], DeleteOperation.prototype, "key", void 0);
|
|
102
|
+
DeleteOperation = __decorate([
|
|
103
|
+
variant(2),
|
|
104
|
+
__metadata("design:paramtypes", [Object])
|
|
105
|
+
], DeleteOperation);
|
|
106
|
+
const extractFieldValue = (doc, path) => {
|
|
107
|
+
for (let i = 0; i < path.length; i++) {
|
|
108
|
+
doc = doc[path[i]];
|
|
109
|
+
}
|
|
110
|
+
return doc;
|
|
111
|
+
};
|
|
112
|
+
const sortCompare = (av, bv) => {
|
|
113
|
+
if (typeof av === "string" && typeof bv === "string") {
|
|
114
|
+
return av.localeCompare(bv);
|
|
115
|
+
}
|
|
116
|
+
if (av < bv) {
|
|
117
|
+
return -1;
|
|
118
|
+
}
|
|
119
|
+
else if (av > bv) {
|
|
120
|
+
return 1;
|
|
121
|
+
}
|
|
122
|
+
return 0;
|
|
123
|
+
};
|
|
124
|
+
const extractSortCompare = (a, b, sorts) => {
|
|
125
|
+
for (const sort of sorts) {
|
|
126
|
+
const av = extractFieldValue(a, sort.key);
|
|
127
|
+
const bv = extractFieldValue(b, sort.key);
|
|
128
|
+
const cmp = sortCompare(av, bv);
|
|
129
|
+
if (cmp != 0) {
|
|
130
|
+
if (sort.direction === SortDirection.ASC) {
|
|
131
|
+
return cmp;
|
|
132
|
+
}
|
|
133
|
+
else {
|
|
134
|
+
return -cmp;
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
return 0;
|
|
139
|
+
};
|
|
140
|
+
const resolvedSort = async (arr, index, sorts) => {
|
|
141
|
+
await Promise.all(arr.map(async (result) => (result[SORT_TMP_KEY] = await index(result.value, result.context))));
|
|
142
|
+
arr.sort((a, b) => extractSortCompare(a[SORT_TMP_KEY], b[SORT_TMP_KEY], sorts));
|
|
143
|
+
return arr;
|
|
144
|
+
};
|
|
145
|
+
/*
|
|
146
|
+
const sortValueWithContext = async<T>(arr: {
|
|
147
|
+
value: T;
|
|
148
|
+
context: Context;
|
|
149
|
+
}[], index: Indexable<T>) => {
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
}
|
|
153
|
+
*/
|
|
154
|
+
const SORT_TMP_KEY = "__sort_ref";
|
|
155
|
+
const introduceEntries = async (responses, type, sync, options) => {
|
|
156
|
+
return Promise.all(responses.map(async (x) => {
|
|
157
|
+
x.response.results.forEach((r) => r.init(type));
|
|
158
|
+
if (typeof options?.remote !== "boolean" && options?.remote?.sync) {
|
|
159
|
+
await sync(x.response);
|
|
160
|
+
}
|
|
161
|
+
if (!x.from) {
|
|
162
|
+
logger.error("Missing from for response");
|
|
163
|
+
}
|
|
164
|
+
options?.onResponse && options.onResponse(x.response, x.from);
|
|
165
|
+
return x;
|
|
166
|
+
}));
|
|
167
|
+
};
|
|
168
|
+
const dedup = (allResult, dedupBy) => {
|
|
169
|
+
const unique = new Set();
|
|
170
|
+
const dedup = [];
|
|
171
|
+
for (const result of allResult) {
|
|
172
|
+
const key = asString(dedupBy(result));
|
|
173
|
+
if (unique.has(key)) {
|
|
174
|
+
continue;
|
|
175
|
+
}
|
|
176
|
+
unique.add(key);
|
|
177
|
+
dedup.push(result);
|
|
178
|
+
}
|
|
179
|
+
return dedup;
|
|
180
|
+
};
|
|
181
|
+
const DEFAULT_INDEX_BY = "id";
|
|
182
|
+
export let DocumentIndex = class DocumentIndex extends ComposableProgram {
|
|
183
|
+
_query;
|
|
184
|
+
type;
|
|
185
|
+
// Index key
|
|
186
|
+
_indexBy;
|
|
187
|
+
_indexByArr;
|
|
188
|
+
// Resolve doc value by index key
|
|
189
|
+
indexByResolver;
|
|
190
|
+
// Indexed (transforms an docuemnt into an obj with fields that ought to be indexed)
|
|
191
|
+
_toIndex;
|
|
192
|
+
_valueEncoding;
|
|
193
|
+
_sync;
|
|
194
|
+
_index;
|
|
195
|
+
_resultsCollectQueue;
|
|
196
|
+
_log;
|
|
197
|
+
constructor(properties) {
|
|
198
|
+
super();
|
|
199
|
+
this._query = properties?.query || new RPC();
|
|
200
|
+
}
|
|
201
|
+
get index() {
|
|
202
|
+
return this._index;
|
|
203
|
+
}
|
|
204
|
+
get valueEncoding() {
|
|
205
|
+
return this._valueEncoding;
|
|
206
|
+
}
|
|
207
|
+
get toIndex() {
|
|
208
|
+
return this._toIndex;
|
|
209
|
+
}
|
|
210
|
+
async open(properties) {
|
|
211
|
+
this._index = new Map();
|
|
212
|
+
this._log = properties.log;
|
|
213
|
+
this.type = properties.type;
|
|
214
|
+
this._sync = properties.sync;
|
|
215
|
+
this._toIndex = properties.fields;
|
|
216
|
+
this._indexBy = properties.indexBy || DEFAULT_INDEX_BY;
|
|
217
|
+
this._indexByArr = Array.isArray(this._indexBy)
|
|
218
|
+
? this._indexBy
|
|
219
|
+
: [this._indexBy];
|
|
220
|
+
this.indexByResolver =
|
|
221
|
+
typeof this._indexBy === "string"
|
|
222
|
+
? (obj) => obj[this._indexBy]
|
|
223
|
+
: (obj) => extractFieldValue(obj, this._indexBy);
|
|
224
|
+
this._valueEncoding = BORSH_ENCODING(this.type);
|
|
225
|
+
this._resultsCollectQueue = new Cache({ max: 10000 }); // TODO choose limit better
|
|
226
|
+
await this._query.open({
|
|
227
|
+
topic: this._log.log.idString + "/document",
|
|
228
|
+
canRead: properties.canRead,
|
|
229
|
+
responseHandler: async (query) => {
|
|
230
|
+
if (query instanceof CloseIteratorRequest) {
|
|
231
|
+
this.processCloseIteratorRequest(query);
|
|
232
|
+
}
|
|
233
|
+
else {
|
|
234
|
+
const results = await this.processFetchRequest(query);
|
|
235
|
+
return new Results({
|
|
236
|
+
// Even if results might have length 0, respond, because then we now at least there are no matching results
|
|
237
|
+
results: results.results.map((r) => new ResultWithSource({
|
|
238
|
+
source: serialize(r.value),
|
|
239
|
+
context: r.context,
|
|
240
|
+
})),
|
|
241
|
+
kept: BigInt(results.kept),
|
|
242
|
+
});
|
|
243
|
+
}
|
|
244
|
+
},
|
|
245
|
+
responseType: Results,
|
|
246
|
+
queryType: AbstractSearchRequest,
|
|
247
|
+
});
|
|
248
|
+
}
|
|
249
|
+
async get(key, options) {
|
|
250
|
+
return (await this.getDetailed(key, options))?.[0]?.results[0]?.value;
|
|
251
|
+
}
|
|
252
|
+
async getDetailed(key, options) {
|
|
253
|
+
let results;
|
|
254
|
+
if (key instanceof Uint8Array) {
|
|
255
|
+
results = await this.queryDetailed(new SearchRequest({
|
|
256
|
+
query: [new ByteMatchQuery({ key: this._indexByArr, value: key })],
|
|
257
|
+
}), options);
|
|
258
|
+
}
|
|
259
|
+
else {
|
|
260
|
+
const stringValue = asString(key);
|
|
261
|
+
results = await this.queryDetailed(new SearchRequest({
|
|
262
|
+
query: [
|
|
263
|
+
new StringMatch({
|
|
264
|
+
key: this._indexByArr,
|
|
265
|
+
value: stringValue,
|
|
266
|
+
}),
|
|
267
|
+
],
|
|
268
|
+
}), options);
|
|
269
|
+
}
|
|
270
|
+
return results;
|
|
271
|
+
}
|
|
272
|
+
get size() {
|
|
273
|
+
return this._index.size;
|
|
274
|
+
}
|
|
275
|
+
async getDocument(value) {
|
|
276
|
+
const payloadValue = await (await this._log.log.get(value.context.head)).getPayloadValue();
|
|
277
|
+
if (payloadValue instanceof PutOperation) {
|
|
278
|
+
return payloadValue.getValue(this.valueEncoding);
|
|
279
|
+
}
|
|
280
|
+
throw new Error("Unexpected");
|
|
281
|
+
}
|
|
282
|
+
async _queryDocuments(filter) {
|
|
283
|
+
// Whether we return the full operation data or just the db value
|
|
284
|
+
const results = [];
|
|
285
|
+
for (const value of this._index.values()) {
|
|
286
|
+
if (filter(value)) {
|
|
287
|
+
results.push({
|
|
288
|
+
context: value.context,
|
|
289
|
+
value: await this.getDocument(value),
|
|
290
|
+
});
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
return results;
|
|
294
|
+
}
|
|
295
|
+
async processFetchRequest(query) {
|
|
296
|
+
// We do special case for querying the id as we can do it faster than iterating
|
|
297
|
+
if (query instanceof SearchRequest) {
|
|
298
|
+
if (query.query.length === 1 &&
|
|
299
|
+
(query.query[0] instanceof ByteMatchQuery ||
|
|
300
|
+
query.query[0] instanceof StringMatch) &&
|
|
301
|
+
stringArraysEquals(query.query[0].key, this._indexByArr)) {
|
|
302
|
+
const firstQuery = query.query[0];
|
|
303
|
+
if (firstQuery instanceof ByteMatchQuery) {
|
|
304
|
+
const doc = this._index.get(asString(firstQuery.value)); // TODO could there be a issue with types here?
|
|
305
|
+
return doc
|
|
306
|
+
? {
|
|
307
|
+
results: [
|
|
308
|
+
{
|
|
309
|
+
value: await this.getDocument(doc),
|
|
310
|
+
context: doc.context,
|
|
311
|
+
},
|
|
312
|
+
],
|
|
313
|
+
kept: 0,
|
|
314
|
+
}
|
|
315
|
+
: { results: [], kept: 0 };
|
|
316
|
+
}
|
|
317
|
+
else if (firstQuery instanceof StringMatch &&
|
|
318
|
+
firstQuery.method === StringMatchMethod.exact &&
|
|
319
|
+
firstQuery.caseInsensitive === false) {
|
|
320
|
+
const doc = this._index.get(firstQuery.value); // TODO could there be a issue with types here?
|
|
321
|
+
return doc
|
|
322
|
+
? {
|
|
323
|
+
results: [
|
|
324
|
+
{
|
|
325
|
+
value: await this.getDocument(doc),
|
|
326
|
+
context: doc.context,
|
|
327
|
+
},
|
|
328
|
+
],
|
|
329
|
+
kept: 0,
|
|
330
|
+
}
|
|
331
|
+
: { results: [], kept: 0 };
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
const results = await this._queryDocuments((doc) => {
|
|
335
|
+
for (const f of query.query) {
|
|
336
|
+
if (!this.handleQueryObject(f, doc)) {
|
|
337
|
+
return false;
|
|
338
|
+
}
|
|
339
|
+
}
|
|
340
|
+
return true;
|
|
341
|
+
});
|
|
342
|
+
// Sort
|
|
343
|
+
await resolvedSort(results, this._toIndex, query.sort);
|
|
344
|
+
const batch = results.splice(0, query.fetch);
|
|
345
|
+
if (results.length > 0) {
|
|
346
|
+
this._resultsCollectQueue.add(query.idString, results); // cache resulst not returned
|
|
347
|
+
}
|
|
348
|
+
return { results: batch, kept: results.length }; // Only return 1 result since we are doing distributed sort, TODO buffer more initially
|
|
349
|
+
}
|
|
350
|
+
else if (query instanceof CollectNextRequest) {
|
|
351
|
+
const results = this._resultsCollectQueue.get(query.idString);
|
|
352
|
+
if (!results) {
|
|
353
|
+
return {
|
|
354
|
+
results: [],
|
|
355
|
+
kept: 0,
|
|
356
|
+
};
|
|
357
|
+
}
|
|
358
|
+
const batch = results.splice(0, query.amount);
|
|
359
|
+
if (results.length === 0) {
|
|
360
|
+
this._resultsCollectQueue.del(query.idString); // TODO add tests for proper cleanup/timeouts
|
|
361
|
+
}
|
|
362
|
+
return { results: batch, kept: results.length };
|
|
363
|
+
}
|
|
364
|
+
throw new Error("Unsupported");
|
|
365
|
+
}
|
|
366
|
+
async processCloseIteratorRequest(query) {
|
|
367
|
+
this._resultsCollectQueue.del(query.idString);
|
|
368
|
+
}
|
|
369
|
+
handleQueryObject(f, doc) {
|
|
370
|
+
if (f instanceof StateFieldQuery) {
|
|
371
|
+
const fv = extractFieldValue(doc.value, f.key);
|
|
372
|
+
if (f instanceof StringMatch) {
|
|
373
|
+
let compare = f.value;
|
|
374
|
+
if (f.caseInsensitive) {
|
|
375
|
+
compare = compare.toLowerCase();
|
|
376
|
+
}
|
|
377
|
+
if (Array.isArray(fv)) {
|
|
378
|
+
for (const string of fv) {
|
|
379
|
+
if (this.handleStringMatch(f, compare, string)) {
|
|
380
|
+
return true;
|
|
381
|
+
}
|
|
382
|
+
}
|
|
383
|
+
return false;
|
|
384
|
+
}
|
|
385
|
+
else {
|
|
386
|
+
if (this.handleStringMatch(f, compare, fv)) {
|
|
387
|
+
return true;
|
|
388
|
+
}
|
|
389
|
+
return false;
|
|
390
|
+
}
|
|
391
|
+
}
|
|
392
|
+
else if (f instanceof ByteMatchQuery) {
|
|
393
|
+
if (fv instanceof Uint8Array === false) {
|
|
394
|
+
if (stringArraysEquals(f.key, this._indexByArr)) {
|
|
395
|
+
return f.valueString === fv;
|
|
396
|
+
}
|
|
397
|
+
return false;
|
|
398
|
+
}
|
|
399
|
+
return equals(fv, f.value);
|
|
400
|
+
}
|
|
401
|
+
else if (f instanceof IntegerCompare) {
|
|
402
|
+
const value = fv;
|
|
403
|
+
if (typeof value !== "bigint" && typeof value !== "number") {
|
|
404
|
+
return false;
|
|
405
|
+
}
|
|
406
|
+
return compare(value, f.compare, f.value.value);
|
|
407
|
+
}
|
|
408
|
+
else if (f instanceof MissingField) {
|
|
409
|
+
return fv == null; // null or undefined
|
|
410
|
+
}
|
|
411
|
+
else if (f instanceof BoolQuery) {
|
|
412
|
+
return fv === f.value; // true/false
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
else if (f instanceof LogicalQuery) {
|
|
416
|
+
if (f instanceof And) {
|
|
417
|
+
for (const and of f.and) {
|
|
418
|
+
if (!this.handleQueryObject(and, doc)) {
|
|
419
|
+
return false;
|
|
420
|
+
}
|
|
421
|
+
}
|
|
422
|
+
return true;
|
|
423
|
+
}
|
|
424
|
+
if (f instanceof Or) {
|
|
425
|
+
for (const or of f.or) {
|
|
426
|
+
if (this.handleQueryObject(or, doc)) {
|
|
427
|
+
return true;
|
|
428
|
+
}
|
|
429
|
+
}
|
|
430
|
+
return false;
|
|
431
|
+
}
|
|
432
|
+
return false;
|
|
433
|
+
}
|
|
434
|
+
logger.info("Unsupported query type: " + f.constructor.name);
|
|
435
|
+
return false;
|
|
436
|
+
}
|
|
437
|
+
handleStringMatch(f, compare, fv) {
|
|
438
|
+
if (typeof fv !== "string") {
|
|
439
|
+
return false;
|
|
440
|
+
}
|
|
441
|
+
if (f.caseInsensitive) {
|
|
442
|
+
fv = fv.toLowerCase();
|
|
443
|
+
}
|
|
444
|
+
if (f.method === StringMatchMethod.exact) {
|
|
445
|
+
return fv === compare;
|
|
446
|
+
}
|
|
447
|
+
if (f.method === StringMatchMethod.prefix) {
|
|
448
|
+
return fv.startsWith(compare);
|
|
449
|
+
}
|
|
450
|
+
if (f.method === StringMatchMethod.contains) {
|
|
451
|
+
return fv.includes(compare);
|
|
452
|
+
}
|
|
453
|
+
throw new Error("Unsupported");
|
|
454
|
+
}
|
|
455
|
+
/**
|
|
456
|
+
* Query and retrieve results with most details
|
|
457
|
+
* @param queryRequest
|
|
458
|
+
* @param options
|
|
459
|
+
* @returns
|
|
460
|
+
*/
|
|
461
|
+
async queryDetailed(queryRequest, options) {
|
|
462
|
+
const local = typeof options?.local == "boolean" ? options?.local : true;
|
|
463
|
+
let remote = undefined;
|
|
464
|
+
if (typeof options?.remote === "boolean") {
|
|
465
|
+
if (options?.remote) {
|
|
466
|
+
remote = {};
|
|
467
|
+
}
|
|
468
|
+
else {
|
|
469
|
+
remote = undefined;
|
|
470
|
+
}
|
|
471
|
+
}
|
|
472
|
+
else {
|
|
473
|
+
remote = options?.remote || {};
|
|
474
|
+
}
|
|
475
|
+
const promises = [];
|
|
476
|
+
if (!local && !remote) {
|
|
477
|
+
throw new Error("Expecting either 'options.remote' or 'options.local' to be true");
|
|
478
|
+
}
|
|
479
|
+
const allResults = [];
|
|
480
|
+
if (local) {
|
|
481
|
+
const results = await this.processFetchRequest(queryRequest);
|
|
482
|
+
if (results.results.length > 0) {
|
|
483
|
+
const resultsObject = new Results({
|
|
484
|
+
results: await Promise.all(results.results.map(async (r) => {
|
|
485
|
+
const payloadValue = await (await this._log.log.get(r.context.head))?.getPayloadValue();
|
|
486
|
+
if (payloadValue instanceof PutOperation) {
|
|
487
|
+
return new ResultWithSource({
|
|
488
|
+
context: r.context,
|
|
489
|
+
value: r.value,
|
|
490
|
+
source: payloadValue.data,
|
|
491
|
+
});
|
|
492
|
+
}
|
|
493
|
+
throw new Error("Unexpected");
|
|
494
|
+
})),
|
|
495
|
+
kept: BigInt(results.kept),
|
|
496
|
+
});
|
|
497
|
+
options?.onResponse &&
|
|
498
|
+
options.onResponse(resultsObject, this.node.identity.publicKey);
|
|
499
|
+
allResults.push(resultsObject);
|
|
500
|
+
}
|
|
501
|
+
}
|
|
502
|
+
if (remote) {
|
|
503
|
+
const replicatorGroups = await this._log.replicators?.();
|
|
504
|
+
if (replicatorGroups) {
|
|
505
|
+
const fn = async () => {
|
|
506
|
+
const rs = [];
|
|
507
|
+
const responseHandler = async (results) => {
|
|
508
|
+
await introduceEntries(results, this.type, this._sync, options).then((x) => x.forEach((y) => rs.push(y.response)));
|
|
509
|
+
};
|
|
510
|
+
try {
|
|
511
|
+
if (queryRequest instanceof CloseIteratorRequest) {
|
|
512
|
+
// don't wait for responses
|
|
513
|
+
await this._query.request(queryRequest, { to: remote.to });
|
|
514
|
+
}
|
|
515
|
+
else {
|
|
516
|
+
await queryAll(this._query, replicatorGroups, queryRequest, responseHandler, remote);
|
|
517
|
+
}
|
|
518
|
+
}
|
|
519
|
+
catch (error) {
|
|
520
|
+
if (error instanceof MissingResponsesError) {
|
|
521
|
+
logger.error("Did not reciveve responses from all shard");
|
|
522
|
+
}
|
|
523
|
+
}
|
|
524
|
+
return rs;
|
|
525
|
+
};
|
|
526
|
+
promises.push(fn());
|
|
527
|
+
}
|
|
528
|
+
else {
|
|
529
|
+
// TODO send without direction out to the world? or just assume we can insert?
|
|
530
|
+
/* promises.push(
|
|
531
|
+
this._query
|
|
532
|
+
.request(queryRequest, remote)
|
|
533
|
+
.then((results) => introduceEntries(results, this.type, this._sync, options).then(x => x.map(y => y.response)))
|
|
534
|
+
); */
|
|
535
|
+
/* throw new Error(
|
|
536
|
+
"Missing remote replicator info for performing distributed document query"
|
|
537
|
+
); */
|
|
538
|
+
}
|
|
539
|
+
}
|
|
540
|
+
const resolved = await Promise.all(promises);
|
|
541
|
+
for (const r of resolved) {
|
|
542
|
+
if (r) {
|
|
543
|
+
if (r instanceof Array) {
|
|
544
|
+
allResults.push(...r);
|
|
545
|
+
}
|
|
546
|
+
else {
|
|
547
|
+
allResults.push(r);
|
|
548
|
+
}
|
|
549
|
+
}
|
|
550
|
+
}
|
|
551
|
+
return allResults;
|
|
552
|
+
}
|
|
553
|
+
/**
|
|
554
|
+
* Query and retrieve results
|
|
555
|
+
* @param queryRequest
|
|
556
|
+
* @param options
|
|
557
|
+
* @returns
|
|
558
|
+
*/
|
|
559
|
+
async search(queryRequest, options) {
|
|
560
|
+
// Set fetch to search size, or max value (default to max u32 (4294967295))
|
|
561
|
+
queryRequest.fetch = options?.size ?? 0xffffffff;
|
|
562
|
+
// So that the iterator is pre-fetching the right amount of entries
|
|
563
|
+
const iterator = this.iterate(queryRequest, options);
|
|
564
|
+
// So that this call will not do any remote requests
|
|
565
|
+
const allResult = await iterator.next(queryRequest.fetch);
|
|
566
|
+
await iterator.close();
|
|
567
|
+
//s Deduplicate and return values directly
|
|
568
|
+
return dedup(allResult, this.indexByResolver);
|
|
569
|
+
}
|
|
570
|
+
/**
|
|
571
|
+
* Query and retrieve documents in a iterator
|
|
572
|
+
* @param queryRequest
|
|
573
|
+
* @param options
|
|
574
|
+
* @returns
|
|
575
|
+
*/
|
|
576
|
+
iterate(queryRequest, options) {
|
|
577
|
+
let fetchPromise = undefined;
|
|
578
|
+
const peerBufferMap = new Map();
|
|
579
|
+
const visited = new Set();
|
|
580
|
+
let done = false;
|
|
581
|
+
let first = false;
|
|
582
|
+
// TODO handle join/leave while iterating
|
|
583
|
+
let stopperFns = [];
|
|
584
|
+
const peerBuffers = () => {
|
|
585
|
+
return [...peerBufferMap.values()].map((x) => x.buffer).flat();
|
|
586
|
+
};
|
|
587
|
+
const fetchFirst = async (n) => {
|
|
588
|
+
done = true; // Assume we are donne
|
|
589
|
+
queryRequest.fetch = n;
|
|
590
|
+
await this.queryDetailed(queryRequest, {
|
|
591
|
+
...options,
|
|
592
|
+
onResponse: (response, from) => {
|
|
593
|
+
if (!from) {
|
|
594
|
+
logger.error("Missing response from");
|
|
595
|
+
return;
|
|
596
|
+
}
|
|
597
|
+
if (response.kept === 0n && response.results.length === 0) {
|
|
598
|
+
return;
|
|
599
|
+
}
|
|
600
|
+
if (response.kept > 0n) {
|
|
601
|
+
done = false; // we have more to do later!
|
|
602
|
+
}
|
|
603
|
+
peerBufferMap.set(from.hashcode(), {
|
|
604
|
+
buffer: response.results
|
|
605
|
+
.filter((x) => !visited.has(asString(this.indexByResolver(x.value))))
|
|
606
|
+
.map((x) => {
|
|
607
|
+
visited.add(asString(this.indexByResolver(x.value)));
|
|
608
|
+
return { from, value: x.value, context: x.context };
|
|
609
|
+
}),
|
|
610
|
+
kept: Number(response.kept),
|
|
611
|
+
});
|
|
612
|
+
},
|
|
613
|
+
});
|
|
614
|
+
return done;
|
|
615
|
+
};
|
|
616
|
+
const fetchAtLeast = async (n) => {
|
|
617
|
+
if (done && first) {
|
|
618
|
+
return;
|
|
619
|
+
}
|
|
620
|
+
await fetchPromise;
|
|
621
|
+
if (!first) {
|
|
622
|
+
first = true;
|
|
623
|
+
fetchPromise = fetchFirst(n);
|
|
624
|
+
return fetchPromise;
|
|
625
|
+
}
|
|
626
|
+
const promises = [];
|
|
627
|
+
stopperFns = [];
|
|
628
|
+
let resultsLeft = 0;
|
|
629
|
+
for (const [peer, buffer] of peerBufferMap) {
|
|
630
|
+
if (buffer.buffer.length < n) {
|
|
631
|
+
if (buffer.kept === 0) {
|
|
632
|
+
if (peerBufferMap.get(peer)?.buffer.length === 0) {
|
|
633
|
+
peerBufferMap.delete(peer); // No more results
|
|
634
|
+
}
|
|
635
|
+
continue;
|
|
636
|
+
}
|
|
637
|
+
// TODO buffer more than deleted?
|
|
638
|
+
// TODO batch to multiple 'to's
|
|
639
|
+
const collectRequest = new CollectNextRequest({
|
|
640
|
+
id: queryRequest.id,
|
|
641
|
+
amount: 10, //n - buffer.buffer.length,
|
|
642
|
+
});
|
|
643
|
+
// Fetch locally?
|
|
644
|
+
if (peer === this.node.identity.publicKey.hashcode()) {
|
|
645
|
+
promises.push(this.processFetchRequest(collectRequest)
|
|
646
|
+
.then((results) => {
|
|
647
|
+
resultsLeft += results.kept;
|
|
648
|
+
if (results.results.length === 0) {
|
|
649
|
+
if (peerBufferMap.get(peer)?.buffer.length === 0) {
|
|
650
|
+
peerBufferMap.delete(peer); // No more results
|
|
651
|
+
}
|
|
652
|
+
}
|
|
653
|
+
else {
|
|
654
|
+
const peerBuffer = peerBufferMap.get(peer);
|
|
655
|
+
if (!peerBuffer) {
|
|
656
|
+
return;
|
|
657
|
+
}
|
|
658
|
+
peerBuffer.kept = results.kept;
|
|
659
|
+
peerBuffer.buffer.push(...results.results
|
|
660
|
+
.filter((x) => !visited.has(asString(this.indexByResolver(x.value))))
|
|
661
|
+
.map((x) => {
|
|
662
|
+
visited.add(asString(this.indexByResolver(x.value)));
|
|
663
|
+
return {
|
|
664
|
+
value: x.value,
|
|
665
|
+
context: x.context,
|
|
666
|
+
from: this.node.identity.publicKey,
|
|
667
|
+
};
|
|
668
|
+
}));
|
|
669
|
+
}
|
|
670
|
+
})
|
|
671
|
+
.catch((e) => {
|
|
672
|
+
logger.error("Failed to collect sorted results self. " + e?.message);
|
|
673
|
+
peerBufferMap.delete(peer);
|
|
674
|
+
}));
|
|
675
|
+
}
|
|
676
|
+
else {
|
|
677
|
+
// Fetch remotely
|
|
678
|
+
promises.push(this._query
|
|
679
|
+
.request(collectRequest, {
|
|
680
|
+
...options,
|
|
681
|
+
stopper: (fn) => stopperFns.push(fn),
|
|
682
|
+
to: [peer],
|
|
683
|
+
})
|
|
684
|
+
.then((response) => introduceEntries(response, this.type, this._sync, options)
|
|
685
|
+
.then((responses) => {
|
|
686
|
+
responses.map((response) => {
|
|
687
|
+
resultsLeft += Number(response.response.kept);
|
|
688
|
+
if (!response.from) {
|
|
689
|
+
logger.error("Missing from for sorted query");
|
|
690
|
+
return;
|
|
691
|
+
}
|
|
692
|
+
if (response.response.results.length === 0) {
|
|
693
|
+
if (peerBufferMap.get(peer)?.buffer.length === 0) {
|
|
694
|
+
peerBufferMap.delete(peer); // No more results
|
|
695
|
+
}
|
|
696
|
+
}
|
|
697
|
+
else {
|
|
698
|
+
const peerBuffer = peerBufferMap.get(peer);
|
|
699
|
+
if (!peerBuffer) {
|
|
700
|
+
return;
|
|
701
|
+
}
|
|
702
|
+
peerBuffer.kept = Number(response.response.kept);
|
|
703
|
+
peerBuffer.buffer.push(...response.response.results
|
|
704
|
+
.filter((x) => !visited.has(asString(this.indexByResolver(x.value))))
|
|
705
|
+
.map((x) => {
|
|
706
|
+
visited.add(asString(this.indexByResolver(x.value)));
|
|
707
|
+
return {
|
|
708
|
+
value: x.value,
|
|
709
|
+
context: x.context,
|
|
710
|
+
from: response.from,
|
|
711
|
+
};
|
|
712
|
+
}));
|
|
713
|
+
}
|
|
714
|
+
});
|
|
715
|
+
})
|
|
716
|
+
.catch((e) => {
|
|
717
|
+
logger.error("Failed to collect sorted results from: " +
|
|
718
|
+
peer +
|
|
719
|
+
". " +
|
|
720
|
+
e?.message);
|
|
721
|
+
peerBufferMap.delete(peer);
|
|
722
|
+
})));
|
|
723
|
+
}
|
|
724
|
+
}
|
|
725
|
+
else {
|
|
726
|
+
resultsLeft += peerBufferMap.get(peer)?.kept || 0;
|
|
727
|
+
}
|
|
728
|
+
}
|
|
729
|
+
return (fetchPromise = Promise.all(promises).then(() => {
|
|
730
|
+
return resultsLeft === 0; // 0 results left to fetch and 0 pending results
|
|
731
|
+
}));
|
|
732
|
+
};
|
|
733
|
+
const next = async (n) => {
|
|
734
|
+
if (n < 0) {
|
|
735
|
+
throw new Error("Expecting to fetch a positive amount of element");
|
|
736
|
+
}
|
|
737
|
+
if (n === 0) {
|
|
738
|
+
return [];
|
|
739
|
+
}
|
|
740
|
+
// TODO everything below is not very optimized
|
|
741
|
+
const fetchedAll = await fetchAtLeast(n);
|
|
742
|
+
// get n next top entries, shift and pull more results
|
|
743
|
+
const results = await resolvedSort(peerBuffers(), this._toIndex, queryRequest.sort);
|
|
744
|
+
const pendingMoreResults = n < results.length;
|
|
745
|
+
const batch = results.splice(0, n);
|
|
746
|
+
for (const result of batch) {
|
|
747
|
+
const arr = peerBufferMap.get(result.from.hashcode());
|
|
748
|
+
if (!arr) {
|
|
749
|
+
logger.error("Unexpected empty result buffer");
|
|
750
|
+
continue;
|
|
751
|
+
}
|
|
752
|
+
const idx = arr.buffer.findIndex((x) => x == result);
|
|
753
|
+
if (idx >= 0) {
|
|
754
|
+
arr.buffer.splice(idx, 1);
|
|
755
|
+
}
|
|
756
|
+
}
|
|
757
|
+
done = fetchedAll && !pendingMoreResults;
|
|
758
|
+
return dedup(batch.map((x) => x.value), this.indexByResolver);
|
|
759
|
+
};
|
|
760
|
+
const close = async () => {
|
|
761
|
+
for (const fn of stopperFns) {
|
|
762
|
+
fn();
|
|
763
|
+
}
|
|
764
|
+
const closeRequest = new CloseIteratorRequest({ id: queryRequest.id });
|
|
765
|
+
const promises = [];
|
|
766
|
+
for (const [peer, buffer] of peerBufferMap) {
|
|
767
|
+
if (buffer.kept === 0) {
|
|
768
|
+
peerBufferMap.delete(peer);
|
|
769
|
+
continue;
|
|
770
|
+
}
|
|
771
|
+
// Fetch locally?
|
|
772
|
+
if (peer === this.node.identity.publicKey.hashcode()) {
|
|
773
|
+
promises.push(this.processCloseIteratorRequest(closeRequest));
|
|
774
|
+
}
|
|
775
|
+
else {
|
|
776
|
+
// Fetch remotely
|
|
777
|
+
promises.push(this._query.send(closeRequest, {
|
|
778
|
+
...options,
|
|
779
|
+
to: [peer],
|
|
780
|
+
}));
|
|
781
|
+
}
|
|
782
|
+
}
|
|
783
|
+
await Promise.all(promises);
|
|
784
|
+
};
|
|
785
|
+
return {
|
|
786
|
+
close,
|
|
787
|
+
next,
|
|
788
|
+
done: () => done,
|
|
789
|
+
};
|
|
790
|
+
}
|
|
791
|
+
};
|
|
792
|
+
__decorate([
|
|
793
|
+
field({ type: RPC }),
|
|
794
|
+
__metadata("design:type", RPC)
|
|
795
|
+
], DocumentIndex.prototype, "_query", void 0);
|
|
796
|
+
DocumentIndex = __decorate([
|
|
797
|
+
variant("documents_index"),
|
|
798
|
+
__metadata("design:paramtypes", [Object])
|
|
799
|
+
], DocumentIndex);
|
|
800
|
+
//# sourceMappingURL=document-index.js.map
|