@enbox/dwn-sdk-js 0.4.16 → 0.4.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser.mjs +1 -1
- package/dist/browser.mjs.map +4 -4
- package/dist/esm/generated/precompiled-validators.js +734 -747
- package/dist/esm/generated/precompiled-validators.js.map +1 -1
- package/dist/esm/json-schemas/definitions.json +5 -0
- package/dist/esm/src/core/dwn-constant.js +2 -2
- package/dist/esm/src/core/dwn-error.js +0 -1
- package/dist/esm/src/core/dwn-error.js.map +1 -1
- package/dist/esm/src/core/protocol-authorization-validation.js +1 -24
- package/dist/esm/src/core/protocol-authorization-validation.js.map +1 -1
- package/dist/esm/src/core/protocol-authorization.js +1 -5
- package/dist/esm/src/core/protocol-authorization.js.map +1 -1
- package/dist/esm/src/core/validation-state-reader.js +2 -3
- package/dist/esm/src/core/validation-state-reader.js.map +1 -1
- package/dist/esm/src/handlers/records-count.js +18 -10
- package/dist/esm/src/handlers/records-count.js.map +1 -1
- package/dist/esm/src/handlers/records-query.js +10 -7
- package/dist/esm/src/handlers/records-query.js.map +1 -1
- package/dist/esm/src/handlers/records-subscribe.js +10 -7
- package/dist/esm/src/handlers/records-subscribe.js.map +1 -1
- package/dist/esm/src/handlers/records-write.js +8 -26
- package/dist/esm/src/handlers/records-write.js.map +1 -1
- package/dist/esm/src/index.js +1 -1
- package/dist/esm/src/index.js.map +1 -1
- package/dist/esm/src/interfaces/protocols-configure.js +4 -9
- package/dist/esm/src/interfaces/protocols-configure.js.map +1 -1
- package/dist/esm/src/store/index-level.js +143 -6
- package/dist/esm/src/store/index-level.js.map +1 -1
- package/dist/esm/src/store/message-store-level.js +158 -21
- package/dist/esm/src/store/message-store-level.js.map +1 -1
- package/dist/esm/src/store/storage-controller.js +1 -3
- package/dist/esm/src/store/storage-controller.js.map +1 -1
- package/dist/esm/src/types/protocols-types.js +0 -10
- package/dist/esm/src/types/protocols-types.js.map +1 -1
- package/dist/esm/src/utils/filter.js +35 -1
- package/dist/esm/src/utils/filter.js.map +1 -1
- package/dist/esm/src/utils/grant-key-coverage.js +4 -6
- package/dist/esm/src/utils/grant-key-coverage.js.map +1 -1
- package/dist/esm/src/utils/messages.js +4 -14
- package/dist/esm/src/utils/messages.js.map +1 -1
- package/dist/esm/src/utils/permission-scope.js +3 -9
- package/dist/esm/src/utils/permission-scope.js.map +1 -1
- package/dist/esm/src/utils/record-limit-occupancy.js +70 -202
- package/dist/esm/src/utils/record-limit-occupancy.js.map +1 -1
- package/dist/esm/src/utils/records.js +9 -9
- package/dist/esm/src/utils/records.js.map +1 -1
- package/dist/esm/tests/core/records-grant-authorization.spec.js +13 -0
- package/dist/esm/tests/core/records-grant-authorization.spec.js.map +1 -1
- package/dist/esm/tests/core/replication-replay-property.spec.js +2 -2
- package/dist/esm/tests/core/replication-replay-property.spec.js.map +1 -1
- package/dist/esm/tests/core/validation-read-closure.spec.js +1 -1
- package/dist/esm/tests/core/validation-read-closure.spec.js.map +1 -1
- package/dist/esm/tests/dwn.spec.js +2 -2
- package/dist/esm/tests/dwn.spec.js.map +1 -1
- package/dist/esm/tests/features/records-delivery.spec.js +2 -5
- package/dist/esm/tests/features/records-delivery.spec.js.map +1 -1
- package/dist/esm/tests/features/records-nested-query-scope.spec.js +138 -73
- package/dist/esm/tests/features/records-nested-query-scope.spec.js.map +1 -1
- package/dist/esm/tests/features/records-record-limit.spec.js +264 -95
- package/dist/esm/tests/features/records-record-limit.spec.js.map +1 -1
- package/dist/esm/tests/fuzz/filter.fuzz.spec.js +19 -4
- package/dist/esm/tests/fuzz/filter.fuzz.spec.js.map +1 -1
- package/dist/esm/tests/handlers/records-write.spec.js +57 -3
- package/dist/esm/tests/handlers/records-write.spec.js.map +1 -1
- package/dist/esm/tests/store/index-level.spec.js +114 -1
- package/dist/esm/tests/store/index-level.spec.js.map +1 -1
- package/dist/esm/tests/store/message-store-cross-context.spec.js +46 -0
- package/dist/esm/tests/store/message-store-cross-context.spec.js.map +1 -0
- package/dist/esm/tests/store/message-store-level.spec.js +6 -0
- package/dist/esm/tests/store/message-store-level.spec.js.map +1 -1
- package/dist/esm/tests/store/message-store.spec.js +81 -0
- package/dist/esm/tests/store/message-store.spec.js.map +1 -1
- package/dist/esm/tests/utils/filters.spec.js +25 -4
- package/dist/esm/tests/utils/filters.spec.js.map +1 -1
- package/dist/esm/tests/utils/permission-scope.spec.js +4 -0
- package/dist/esm/tests/utils/permission-scope.spec.js.map +1 -1
- package/dist/esm/tests/utils/records.spec.js +12 -0
- package/dist/esm/tests/utils/records.spec.js.map +1 -1
- package/dist/esm/tests/validation/json-schemas/records/records-write.spec.js +3 -1
- package/dist/esm/tests/validation/json-schemas/records/records-write.spec.js.map +1 -1
- package/dist/types/generated/precompiled-validators.d.ts.map +1 -1
- package/dist/types/src/core/dwn-constant.d.ts +2 -2
- package/dist/types/src/core/dwn-error.d.ts +0 -1
- package/dist/types/src/core/dwn-error.d.ts.map +1 -1
- package/dist/types/src/core/protocol-authorization-validation.d.ts +0 -10
- package/dist/types/src/core/protocol-authorization-validation.d.ts.map +1 -1
- package/dist/types/src/core/protocol-authorization.d.ts.map +1 -1
- package/dist/types/src/core/validation-state-reader.d.ts.map +1 -1
- package/dist/types/src/handlers/records-count.d.ts +1 -0
- package/dist/types/src/handlers/records-count.d.ts.map +1 -1
- package/dist/types/src/handlers/records-query.d.ts.map +1 -1
- package/dist/types/src/handlers/records-subscribe.d.ts.map +1 -1
- package/dist/types/src/handlers/records-write.d.ts +3 -0
- package/dist/types/src/handlers/records-write.d.ts.map +1 -1
- package/dist/types/src/index.d.ts +3 -3
- package/dist/types/src/index.d.ts.map +1 -1
- package/dist/types/src/interfaces/protocols-configure.d.ts.map +1 -1
- package/dist/types/src/store/index-level.d.ts +37 -0
- package/dist/types/src/store/index-level.d.ts.map +1 -1
- package/dist/types/src/store/message-store-level.d.ts +23 -6
- package/dist/types/src/store/message-store-level.d.ts.map +1 -1
- package/dist/types/src/store/storage-controller.d.ts.map +1 -1
- package/dist/types/src/types/message-store.d.ts +23 -2
- package/dist/types/src/types/message-store.d.ts.map +1 -1
- package/dist/types/src/types/protocols-types.d.ts +7 -24
- package/dist/types/src/types/protocols-types.d.ts.map +1 -1
- package/dist/types/src/types/query-types.d.ts +8 -1
- package/dist/types/src/types/query-types.d.ts.map +1 -1
- package/dist/types/src/types/records-types.d.ts +2 -1
- package/dist/types/src/types/records-types.d.ts.map +1 -1
- package/dist/types/src/utils/filter.d.ts +14 -1
- package/dist/types/src/utils/filter.d.ts.map +1 -1
- package/dist/types/src/utils/grant-key-coverage.d.ts.map +1 -1
- package/dist/types/src/utils/messages.d.ts.map +1 -1
- package/dist/types/src/utils/permission-scope.d.ts +0 -2
- package/dist/types/src/utils/permission-scope.d.ts.map +1 -1
- package/dist/types/src/utils/record-limit-occupancy.d.ts +25 -12
- package/dist/types/src/utils/record-limit-occupancy.d.ts.map +1 -1
- package/dist/types/src/utils/records.d.ts +3 -2
- package/dist/types/src/utils/records.d.ts.map +1 -1
- package/dist/types/tests/features/records-delivery.spec.d.ts.map +1 -1
- package/dist/types/tests/features/records-nested-query-scope.spec.d.ts.map +1 -1
- package/dist/types/tests/features/records-record-limit.spec.d.ts.map +1 -1
- package/dist/types/tests/handlers/records-write.spec.d.ts.map +1 -1
- package/dist/types/tests/store/message-store-cross-context.spec.d.ts +2 -0
- package/dist/types/tests/store/message-store-cross-context.spec.d.ts.map +1 -0
- package/dist/types/tests/store/message-store.spec.d.ts.map +1 -1
- package/package.json +4 -4
- package/src/core/dwn-constant.ts +2 -2
- package/src/core/dwn-error.ts +0 -2
- package/src/core/protocol-authorization-validation.ts +1 -33
- package/src/core/protocol-authorization.ts +0 -7
- package/src/core/validation-state-reader.ts +2 -3
- package/src/handlers/records-count.ts +34 -13
- package/src/handlers/records-query.ts +14 -11
- package/src/handlers/records-subscribe.ts +18 -11
- package/src/handlers/records-write.ts +17 -32
- package/src/index.ts +3 -3
- package/src/interfaces/protocols-configure.ts +4 -13
- package/src/store/index-level.ts +197 -10
- package/src/store/message-store-level.ts +216 -25
- package/src/store/storage-controller.ts +1 -3
- package/src/types/message-store.ts +25 -2
- package/src/types/protocols-types.ts +7 -26
- package/src/types/query-types.ts +10 -2
- package/src/types/records-types.ts +2 -1
- package/src/utils/filter.ts +40 -3
- package/src/utils/grant-key-coverage.ts +4 -7
- package/src/utils/messages.ts +4 -14
- package/src/utils/permission-scope.ts +4 -11
- package/src/utils/record-limit-occupancy.ts +105 -293
- package/src/utils/records.ts +9 -9
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
|
|
2
|
-
import type { CompoundIndexDefinition } from './index-level.js';
|
|
2
|
+
import type { CompoundIndexDefinition, IndexedItem } from './index-level.js';
|
|
3
3
|
import type {
|
|
4
4
|
EventLogEntry,
|
|
5
5
|
EventLogReadOptions,
|
|
@@ -13,22 +13,32 @@ import type {
|
|
|
13
13
|
import type { Filter, KeyValues, PaginationCursor, QueryOptions } from '../types/query-types.js';
|
|
14
14
|
import type { GenericMessage, MessageSort, Pagination } from '../types/message-types.js';
|
|
15
15
|
import type { LevelDatabase, LevelWrapperBatchOperation } from './level-wrapper.js';
|
|
16
|
-
import type {
|
|
16
|
+
import type {
|
|
17
|
+
MessageStore,
|
|
18
|
+
MessageStoreLatestStateTransition,
|
|
19
|
+
MessageStoreOptions,
|
|
20
|
+
MessageStorePutResult,
|
|
21
|
+
MessageStoreQueryOptions,
|
|
22
|
+
RecordLimitOccupancy,
|
|
23
|
+
} from '../types/message-store.js';
|
|
17
24
|
|
|
18
25
|
import * as block from 'multiformats/block';
|
|
19
26
|
import * as cbor from '@ipld/dag-cbor';
|
|
20
27
|
|
|
28
|
+
import { runWithCrossContextLock } from '@enbox/common';
|
|
29
|
+
|
|
21
30
|
import { Cid } from '../utils/cid.js';
|
|
22
31
|
import { CID } from 'multiformats/cid';
|
|
23
32
|
import { executeUnlessAborted } from '../utils/abort.js';
|
|
24
|
-
import { FilterUtility } from '../utils/filter.js';
|
|
25
33
|
import { IndexLevel } from './index-level.js';
|
|
26
34
|
import { Message } from '../core/message.js';
|
|
27
35
|
import { Replication } from '../utils/replication.js';
|
|
28
36
|
import { sha256 } from 'multiformats/hashes/sha2';
|
|
29
37
|
import { SortDirection } from '../types/query-types.js';
|
|
38
|
+
import { assertValidSubtreeFilters, FilterUtility } from '../utils/filter.js';
|
|
30
39
|
import { createLevelDatabase, LevelWrapper } from './level-wrapper.js';
|
|
31
40
|
import { DwnError, DwnErrorCode } from '../core/dwn-error.js';
|
|
41
|
+
import { DwnInterfaceName, DwnMethodName } from '../enums/dwn-interface-method.js';
|
|
32
42
|
|
|
33
43
|
|
|
34
44
|
/**
|
|
@@ -89,6 +99,8 @@ const HEADS_PARTITION = 'heads';
|
|
|
89
99
|
const META_PARTITION = 'meta';
|
|
90
100
|
|
|
91
101
|
const EPOCH_KEY = 'epoch';
|
|
102
|
+
const RECORD_LIMIT_RAW_PAGE_SIZE = 128;
|
|
103
|
+
const RECORD_LIMIT_GROUP_CACHE_SIZE = 128;
|
|
92
104
|
const CURRENT_PARTITIONS = new Set([
|
|
93
105
|
BLOCKS_PARTITION,
|
|
94
106
|
INDEX_PARTITION,
|
|
@@ -136,6 +148,29 @@ type StorePartitions = {
|
|
|
136
148
|
meta: LevelWrapper<string>;
|
|
137
149
|
};
|
|
138
150
|
|
|
151
|
+
type RecordLimitPopulationInput = {
|
|
152
|
+
tenant: string;
|
|
153
|
+
filters: Filter[];
|
|
154
|
+
queryOptions: QueryOptions;
|
|
155
|
+
recordLimit: RecordLimitOccupancy;
|
|
156
|
+
options?: MessageStoreQueryOptions;
|
|
157
|
+
};
|
|
158
|
+
|
|
159
|
+
type RecordLimitOccupantInput = {
|
|
160
|
+
tenant: string;
|
|
161
|
+
index: IndexLevel;
|
|
162
|
+
item: IndexedItem;
|
|
163
|
+
candidateFilter: Filter;
|
|
164
|
+
recordLimit: RecordLimitOccupancy;
|
|
165
|
+
groupCutoffs: Map<string, string>;
|
|
166
|
+
options?: MessageStoreQueryOptions;
|
|
167
|
+
};
|
|
168
|
+
|
|
169
|
+
type RecordLimitPageInput = Omit<RecordLimitOccupantInput, 'item'> & {
|
|
170
|
+
items: IndexedItem[];
|
|
171
|
+
limit?: number;
|
|
172
|
+
};
|
|
173
|
+
|
|
139
174
|
/**
|
|
140
175
|
* A {@link MessageStore} and {@link ReplicationFeedReader} implementation that works in both the
|
|
141
176
|
* browser and server-side, backed by a SINGLE LevelDB root: message blocks, query indexes, the
|
|
@@ -143,8 +178,9 @@ type StorePartitions = {
|
|
|
143
178
|
* the store epoch are all sublevels of one Level instance, so every mutation commits as one fully
|
|
144
179
|
* atomic batch.
|
|
145
180
|
*
|
|
146
|
-
* A per-tenant
|
|
147
|
-
* seq order by construction
|
|
181
|
+
* A per-tenant cross-context write lock spans seq assignment through batch write, so commit order
|
|
182
|
+
* equals seq order by construction even when browser tabs, workers, or service workers share the
|
|
183
|
+
* same IndexedDB database. Seq assignment is gap-free (a failed batch never persists the head),
|
|
148
184
|
* while the readable log stays sparse after compaction and under filters — readers never assume
|
|
149
185
|
* contiguity.
|
|
150
186
|
*/
|
|
@@ -154,7 +190,6 @@ export class MessageStoreLevel implements MessageStore, ReplicationFeedReader {
|
|
|
154
190
|
private readonly wakePublisher?: WakePublisher;
|
|
155
191
|
private partitionsPromise?: Promise<StorePartitions>;
|
|
156
192
|
private epochPromise?: Promise<string>;
|
|
157
|
-
private readonly writeLocks: Map<string, Promise<void>> = new Map();
|
|
158
193
|
|
|
159
194
|
/**
|
|
160
195
|
* @param {MessageStoreLevelConfig} config
|
|
@@ -304,7 +339,7 @@ export class MessageStoreLevel implements MessageStore, ReplicationFeedReader {
|
|
|
304
339
|
filters: Filter[],
|
|
305
340
|
messageSort?: MessageSort,
|
|
306
341
|
pagination?: Pagination,
|
|
307
|
-
options?:
|
|
342
|
+
options?: MessageStoreQueryOptions
|
|
308
343
|
): Promise<{ messages: GenericMessage[], cursor?: PaginationCursor}> {
|
|
309
344
|
options?.signal?.throwIfAborted();
|
|
310
345
|
|
|
@@ -312,7 +347,15 @@ export class MessageStoreLevel implements MessageStore, ReplicationFeedReader {
|
|
|
312
347
|
// this adds 1 to the limit if provided, that way we can check to see if there are additional results and provide a return cursor.
|
|
313
348
|
const queryOptions = MessageStoreLevel.buildQueryOptions(messageSort, pagination);
|
|
314
349
|
const index = await this.index();
|
|
315
|
-
const results =
|
|
350
|
+
const results = options?.recordLimit === undefined
|
|
351
|
+
? await index.query(tenant, filters, queryOptions, options)
|
|
352
|
+
: await this.collectRecordLimitPopulation({
|
|
353
|
+
tenant,
|
|
354
|
+
filters,
|
|
355
|
+
queryOptions,
|
|
356
|
+
recordLimit: options.recordLimit,
|
|
357
|
+
options,
|
|
358
|
+
});
|
|
316
359
|
|
|
317
360
|
let cursor: PaginationCursor | undefined;
|
|
318
361
|
// checks to see if the returned results are greater than the limit, which would indicate additional results.
|
|
@@ -338,13 +381,170 @@ export class MessageStoreLevel implements MessageStore, ReplicationFeedReader {
|
|
|
338
381
|
tenant: string,
|
|
339
382
|
filters: Filter[],
|
|
340
383
|
messageSort?: MessageSort,
|
|
341
|
-
options?:
|
|
384
|
+
options?: MessageStoreQueryOptions
|
|
342
385
|
): Promise<number> {
|
|
343
386
|
options?.signal?.throwIfAborted();
|
|
344
387
|
|
|
345
388
|
const queryOptions = MessageStoreLevel.buildQueryOptions(messageSort);
|
|
346
389
|
const index = await this.index();
|
|
347
|
-
|
|
390
|
+
if (options?.recordLimit === undefined) {
|
|
391
|
+
return index.count(tenant, filters, queryOptions, options);
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
let count = 0;
|
|
395
|
+
for await (const _item of this.iterateRecordLimitPopulation({
|
|
396
|
+
tenant,
|
|
397
|
+
filters,
|
|
398
|
+
queryOptions,
|
|
399
|
+
recordLimit: options.recordLimit,
|
|
400
|
+
options,
|
|
401
|
+
})) {
|
|
402
|
+
count++;
|
|
403
|
+
}
|
|
404
|
+
return count;
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
/**
|
|
408
|
+
* Materializes one bounded page from the shared record-limit population iterator.
|
|
409
|
+
*/
|
|
410
|
+
private async collectRecordLimitPopulation(input: RecordLimitPopulationInput): Promise<IndexedItem[]> {
|
|
411
|
+
const results: IndexedItem[] = [];
|
|
412
|
+
for await (const item of this.iterateRecordLimitPopulation(input)) {
|
|
413
|
+
results.push(item);
|
|
414
|
+
}
|
|
415
|
+
return results;
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
/**
|
|
419
|
+
* Streams caller-filtered records in requested order and admits only records
|
|
420
|
+
* inside their direct-parent group's deterministic rank boundary.
|
|
421
|
+
*
|
|
422
|
+
* Raw candidates are fetched into a small page before any parent lookup is
|
|
423
|
+
* opened. This is deliberate: browser-level/IndexedDB cursors must not be
|
|
424
|
+
* held open while another index transaction is awaited.
|
|
425
|
+
*/
|
|
426
|
+
private async * iterateRecordLimitPopulation(input: RecordLimitPopulationInput): AsyncGenerator<IndexedItem> {
|
|
427
|
+
const { filters, options, queryOptions, recordLimit, tenant } = input;
|
|
428
|
+
const index = await this.index();
|
|
429
|
+
const candidateFilter = MessageStoreLevel.buildRecordLimitCandidateFilter(recordLimit);
|
|
430
|
+
const groupCutoffs = new Map<string, string>();
|
|
431
|
+
const outputLimit = queryOptions.limit;
|
|
432
|
+
let outputCount = 0;
|
|
433
|
+
let rawCursor = queryOptions.cursor;
|
|
434
|
+
|
|
435
|
+
if (outputLimit !== undefined && outputLimit <= 0) {
|
|
436
|
+
return;
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
while (true) {
|
|
440
|
+
const rawItems = await index.queryWithBoundedPaging(tenant, filters, {
|
|
441
|
+
...queryOptions,
|
|
442
|
+
cursor : rawCursor,
|
|
443
|
+
limit : RECORD_LIMIT_RAW_PAGE_SIZE,
|
|
444
|
+
}, options);
|
|
445
|
+
if (rawItems.length === 0) {
|
|
446
|
+
return;
|
|
447
|
+
}
|
|
448
|
+
|
|
449
|
+
const occupants = await this.filterRecordLimitOccupants({
|
|
450
|
+
tenant,
|
|
451
|
+
index,
|
|
452
|
+
items : rawItems,
|
|
453
|
+
candidateFilter,
|
|
454
|
+
recordLimit,
|
|
455
|
+
groupCutoffs,
|
|
456
|
+
options,
|
|
457
|
+
limit : outputLimit === undefined ? undefined : outputLimit - outputCount,
|
|
458
|
+
});
|
|
459
|
+
for (const occupant of occupants) {
|
|
460
|
+
yield occupant;
|
|
461
|
+
outputCount++;
|
|
462
|
+
if (outputLimit !== undefined && outputCount >= outputLimit) {
|
|
463
|
+
return;
|
|
464
|
+
}
|
|
465
|
+
}
|
|
466
|
+
|
|
467
|
+
if (rawItems.length < RECORD_LIMIT_RAW_PAGE_SIZE) {
|
|
468
|
+
return;
|
|
469
|
+
}
|
|
470
|
+
rawCursor = IndexLevel.createCursorFromItem(rawItems.at(-1)!, queryOptions.sortProperty);
|
|
471
|
+
}
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
private async filterRecordLimitOccupants(input: RecordLimitPageInput): Promise<IndexedItem[]> {
|
|
475
|
+
const { items, limit, ...occupantInput } = input;
|
|
476
|
+
const occupants: IndexedItem[] = [];
|
|
477
|
+
for (const item of items) {
|
|
478
|
+
if (await this.isWithinRecordLimit({ ...occupantInput, item })) {
|
|
479
|
+
occupants.push(item);
|
|
480
|
+
if (limit !== undefined && occupants.length >= limit) {
|
|
481
|
+
break;
|
|
482
|
+
}
|
|
483
|
+
}
|
|
484
|
+
}
|
|
485
|
+
return occupants;
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
private async isWithinRecordLimit(input: RecordLimitOccupantInput): Promise<boolean> {
|
|
489
|
+
if (!FilterUtility.matchFilter(input.item.indexes, input.candidateFilter)) {
|
|
490
|
+
return false;
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
const indexedParentId = input.item.indexes.parentId;
|
|
494
|
+
if (indexedParentId !== undefined && typeof indexedParentId !== 'string') {
|
|
495
|
+
throw new TypeError(`MessageStoreLevel: record-limit candidate parentId must be a string.`);
|
|
496
|
+
}
|
|
497
|
+
const parentId = indexedParentId as string | undefined;
|
|
498
|
+
const groupKey = parentId === undefined ? 'root' : `parent:${parentId}`;
|
|
499
|
+
|
|
500
|
+
let cutoff = input.groupCutoffs.get(groupKey);
|
|
501
|
+
if (cutoff !== undefined) {
|
|
502
|
+
// Refresh insertion order so the fixed-size map acts as a small LRU.
|
|
503
|
+
input.groupCutoffs.delete(groupKey);
|
|
504
|
+
input.groupCutoffs.set(groupKey, cutoff);
|
|
505
|
+
} else {
|
|
506
|
+
cutoff = await input.index.findRecordLimitRankCutoff(
|
|
507
|
+
input.tenant,
|
|
508
|
+
input.candidateFilter,
|
|
509
|
+
parentId,
|
|
510
|
+
input.recordLimit.max,
|
|
511
|
+
input.options,
|
|
512
|
+
);
|
|
513
|
+
if (cutoff === undefined) {
|
|
514
|
+
return false;
|
|
515
|
+
}
|
|
516
|
+
MessageStoreLevel.cacheRecordLimitCutoff(input.groupCutoffs, groupKey, cutoff);
|
|
517
|
+
}
|
|
518
|
+
|
|
519
|
+
return IndexLevel.createRecordLimitRankKey(input.item.indexes) <= cutoff;
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
private static cacheRecordLimitCutoff(
|
|
523
|
+
cache: Map<string, string>,
|
|
524
|
+
groupKey: string,
|
|
525
|
+
cutoff: string,
|
|
526
|
+
): void {
|
|
527
|
+
if (cache.size >= RECORD_LIMIT_GROUP_CACHE_SIZE) {
|
|
528
|
+
const oldestKey = cache.keys().next().value as string | undefined;
|
|
529
|
+
if (oldestKey !== undefined) {
|
|
530
|
+
cache.delete(oldestKey);
|
|
531
|
+
}
|
|
532
|
+
}
|
|
533
|
+
cache.set(groupKey, cutoff);
|
|
534
|
+
}
|
|
535
|
+
|
|
536
|
+
private static buildRecordLimitCandidateFilter(recordLimit: RecordLimitOccupancy): Filter {
|
|
537
|
+
const filter: Filter = {
|
|
538
|
+
interface : DwnInterfaceName.Records,
|
|
539
|
+
method : DwnMethodName.Write,
|
|
540
|
+
isLatestBaseState : true,
|
|
541
|
+
protocol : recordLimit.protocol,
|
|
542
|
+
protocolPath : recordLimit.protocolPath,
|
|
543
|
+
};
|
|
544
|
+
if (recordLimit.contextId !== undefined) {
|
|
545
|
+
filter.contextId = { subtree: recordLimit.contextId };
|
|
546
|
+
}
|
|
547
|
+
return filter;
|
|
348
548
|
}
|
|
349
549
|
|
|
350
550
|
/**
|
|
@@ -718,8 +918,11 @@ export class MessageStoreLevel implements MessageStore, ReplicationFeedReader {
|
|
|
718
918
|
async logRead(tenant: string, options: EventLogReadOptions = {}): Promise<EventLogReadResult> {
|
|
719
919
|
const partitions = await this.partitions();
|
|
720
920
|
const { cursor, limit, filters } = options;
|
|
921
|
+
if (filters !== undefined) {
|
|
922
|
+
assertValidSubtreeFilters(filters);
|
|
923
|
+
}
|
|
721
924
|
|
|
722
|
-
// Head-captured-first: the per-tenant write
|
|
925
|
+
// Head-captured-first: the per-tenant write lock serializes commits in position order, so an
|
|
723
926
|
// observed head H is a visibility barrier — every position <= H is already committed when the
|
|
724
927
|
// range scans below run, and anything committed afterward has a position > H and waits for
|
|
725
928
|
// the next page.
|
|
@@ -865,20 +1068,8 @@ export class MessageStoreLevel implements MessageStore, ReplicationFeedReader {
|
|
|
865
1068
|
* a later batch could land first).
|
|
866
1069
|
*/
|
|
867
1070
|
private async withTenantWriteLock<T>(tenant: string, task: () => Promise<T>): Promise<T> {
|
|
868
|
-
const
|
|
869
|
-
|
|
870
|
-
const current = new Promise<void>((resolve) => { release = resolve; });
|
|
871
|
-
this.writeLocks.set(tenant, current);
|
|
872
|
-
|
|
873
|
-
await previous;
|
|
874
|
-
try {
|
|
875
|
-
return await task();
|
|
876
|
-
} finally {
|
|
877
|
-
release();
|
|
878
|
-
if (this.writeLocks.get(tenant) === current) {
|
|
879
|
-
this.writeLocks.delete(tenant);
|
|
880
|
-
}
|
|
881
|
-
}
|
|
1071
|
+
const lockName = `enbox:dwn-message-store:${JSON.stringify([this.config.location!, tenant])}`;
|
|
1072
|
+
return runWithCrossContextLock(lockName, task);
|
|
882
1073
|
}
|
|
883
1074
|
|
|
884
1075
|
/**
|
|
@@ -6,7 +6,6 @@ import type { MessageStore, MessageStorePutResult } from '../types/message-store
|
|
|
6
6
|
import type { RecordsDeleteMessage, RecordsWriteMessage } from '../types/records-types.js';
|
|
7
7
|
|
|
8
8
|
import { DwnConstant } from '../core/dwn-constant.js';
|
|
9
|
-
import { FilterUtility } from '../utils/filter.js';
|
|
10
9
|
import { Message } from '../core/message.js';
|
|
11
10
|
import { Records } from '../utils/records.js';
|
|
12
11
|
import { RecordsDelete } from '../interfaces/records-delete.js';
|
|
@@ -123,8 +122,7 @@ export class StorageController {
|
|
|
123
122
|
// Scope by parent context for nested records
|
|
124
123
|
const parentContextId = Records.getParentContextFromOfContextId(message.contextId);
|
|
125
124
|
if (parentContextId !== undefined && parentContextId !== '') {
|
|
126
|
-
|
|
127
|
-
filter.contextId = prefixFilter;
|
|
125
|
+
filter.contextId = { subtree: parentContextId };
|
|
128
126
|
}
|
|
129
127
|
|
|
130
128
|
// Query for all records at this path and context
|
|
@@ -6,6 +6,29 @@ export interface MessageStoreOptions {
|
|
|
6
6
|
signal?: AbortSignal;
|
|
7
7
|
}
|
|
8
8
|
|
|
9
|
+
/**
|
|
10
|
+
* Read-time `$recordLimit` occupancy policy for one concrete protocol path.
|
|
11
|
+
*
|
|
12
|
+
* Stores partition matching latest RecordsWrites by `parentId`, rank each
|
|
13
|
+
* partition by `dateCreated` then `recordId` ascending, and apply `max`
|
|
14
|
+
* before the caller's filters, sort, and pagination.
|
|
15
|
+
*/
|
|
16
|
+
export type RecordLimitOccupancy = {
|
|
17
|
+
protocol: string;
|
|
18
|
+
protocolPath: string;
|
|
19
|
+
/**
|
|
20
|
+
* Root of the candidate context subtree. An exact-record request supplies
|
|
21
|
+
* its direct-parent context here so sibling candidates still consume slots.
|
|
22
|
+
* Omitted for root protocol paths.
|
|
23
|
+
*/
|
|
24
|
+
contextId?: string;
|
|
25
|
+
max: number;
|
|
26
|
+
};
|
|
27
|
+
|
|
28
|
+
export interface MessageStoreQueryOptions extends MessageStoreOptions {
|
|
29
|
+
recordLimit?: RecordLimitOccupancy;
|
|
30
|
+
}
|
|
31
|
+
|
|
9
32
|
/**
|
|
10
33
|
* Result of a {@link MessageStore.put} operation.
|
|
11
34
|
*/
|
|
@@ -98,7 +121,7 @@ export interface MessageStore {
|
|
|
98
121
|
filters: Filter[],
|
|
99
122
|
messageSort?: MessageSort,
|
|
100
123
|
pagination?: Pagination,
|
|
101
|
-
options?:
|
|
124
|
+
options?: MessageStoreQueryOptions
|
|
102
125
|
): Promise<{ messages: GenericMessage[], cursor?: PaginationCursor}>;
|
|
103
126
|
|
|
104
127
|
/**
|
|
@@ -109,7 +132,7 @@ export interface MessageStore {
|
|
|
109
132
|
tenant: string,
|
|
110
133
|
filters: Filter[],
|
|
111
134
|
messageSort?: MessageSort,
|
|
112
|
-
options?:
|
|
135
|
+
options?: MessageStoreQueryOptions
|
|
113
136
|
): Promise<number>;
|
|
114
137
|
|
|
115
138
|
/**
|
|
@@ -149,33 +149,15 @@ export type ProtocolSizeDefinition = {
|
|
|
149
149
|
};
|
|
150
150
|
|
|
151
151
|
/**
|
|
152
|
-
*
|
|
153
|
-
*/
|
|
154
|
-
export enum ProtocolRecordLimitStrategy {
|
|
155
|
-
/** Reject the incoming write with an error. The author must delete old records first. */
|
|
156
|
-
Reject = 'reject',
|
|
157
|
-
/** Automatically delete the oldest record(s) (by `dateCreated`) to make room for the new one. */
|
|
158
|
-
PurgeOldest = 'purgeOldest',
|
|
159
|
-
}
|
|
160
|
-
|
|
161
|
-
/**
|
|
162
|
-
* Limits the number of records at a given protocol path within the same parent context.
|
|
152
|
+
* Limits the visible records at a given protocol path within the same parent context.
|
|
163
153
|
*
|
|
164
|
-
*
|
|
165
|
-
*
|
|
166
|
-
*
|
|
167
|
-
* Only initial writes (new records) count toward the limit; updates to existing records do not.
|
|
154
|
+
* Root-level records form one group for the protocol path. Nested records are grouped by
|
|
155
|
+
* direct parent context. Occupants are selected deterministically at read time; valid writes
|
|
156
|
+
* outside the visible population remain stored and may become visible later.
|
|
168
157
|
*/
|
|
169
158
|
export type ProtocolRecordLimitDefinition = {
|
|
170
|
-
/** Maximum number of records
|
|
159
|
+
/** Maximum number of visible records at this path within the same parent context. Must be >= 1. */
|
|
171
160
|
max: number;
|
|
172
|
-
|
|
173
|
-
/**
|
|
174
|
-
* Strategy when the limit is reached.
|
|
175
|
-
* Currently only `'reject'` is implemented. Future strategies (e.g. `'purgeOldest'`)
|
|
176
|
-
* are defined in the wire format but not yet enforced.
|
|
177
|
-
*/
|
|
178
|
-
strategy: ProtocolRecordLimitStrategy | `${ProtocolRecordLimitStrategy}` | (string & {});
|
|
179
161
|
};
|
|
180
162
|
|
|
181
163
|
/**
|
|
@@ -287,9 +269,8 @@ export type ProtocolRuleSet = {
|
|
|
287
269
|
$tags?: ProtocolTagsDefinition;
|
|
288
270
|
|
|
289
271
|
/**
|
|
290
|
-
* If
|
|
291
|
-
* parent context
|
|
292
|
-
* whether the write is rejected or the oldest record is purged.
|
|
272
|
+
* If `$recordLimit` is set, reads expose at most `max` records at this protocol path
|
|
273
|
+
* within each direct parent context.
|
|
293
274
|
*/
|
|
294
275
|
$recordLimit?: ProtocolRecordLimitDefinition;
|
|
295
276
|
|
package/src/types/query-types.ts
CHANGED
|
@@ -37,7 +37,15 @@ export type StartsWithFilter = {
|
|
|
37
37
|
startsWith: string;
|
|
38
38
|
};
|
|
39
39
|
|
|
40
|
-
|
|
40
|
+
/**
|
|
41
|
+
* Selects one hierarchical path and every proper `/`-delimited descendant.
|
|
42
|
+
* Supported only for the `contextId` and `protocolPath` indexes.
|
|
43
|
+
*/
|
|
44
|
+
export type SubtreeFilter = {
|
|
45
|
+
subtree: string;
|
|
46
|
+
};
|
|
47
|
+
|
|
48
|
+
export type FilterValue = EqualFilter | OneOfFilter | RangeFilter | SubtreeFilter;
|
|
41
49
|
|
|
42
50
|
export type Filter = {
|
|
43
51
|
[property: string]: FilterValue;
|
|
@@ -58,4 +66,4 @@ export type RangeCriterion = {
|
|
|
58
66
|
export type PaginationCursor = {
|
|
59
67
|
messageCid: string;
|
|
60
68
|
value: string | number;
|
|
61
|
-
};
|
|
69
|
+
};
|
|
@@ -154,7 +154,8 @@ export type RecordsFilter = {
|
|
|
154
154
|
published?: boolean;
|
|
155
155
|
|
|
156
156
|
/**
|
|
157
|
-
*
|
|
157
|
+
* Selects the exact context and its proper `/`-delimited descendants.
|
|
158
|
+
* Prefix-sharing sibling contexts are excluded.
|
|
158
159
|
*/
|
|
159
160
|
contextId?: string;
|
|
160
161
|
schema?: string;
|
package/src/utils/filter.ts
CHANGED
|
@@ -1,4 +1,30 @@
|
|
|
1
|
-
import type { EqualFilter, Filter, FilterValue, KeyValues, OneOfFilter, RangeCriterion, RangeFilter, RangeValue } from '../types/query-types.js';
|
|
1
|
+
import type { EqualFilter, Filter, FilterValue, KeyValues, OneOfFilter, RangeCriterion, RangeFilter, RangeValue, SubtreeFilter } from '../types/query-types.js';
|
|
2
|
+
|
|
3
|
+
const SUBTREE_FILTER_PROPERTIES = new Set(['contextId', 'protocolPath']);
|
|
4
|
+
|
|
5
|
+
/** Returns whether the value is a well-formed hierarchical subtree filter. */
|
|
6
|
+
export function isSubtreeFilter(filter: FilterValue): filter is SubtreeFilter {
|
|
7
|
+
return typeof filter === 'object' &&
|
|
8
|
+
filter !== null &&
|
|
9
|
+
!Array.isArray(filter) &&
|
|
10
|
+
Object.keys(filter).length === 1 &&
|
|
11
|
+
typeof (filter as Partial<SubtreeFilter>).subtree === 'string';
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Rejects subtree filters for indexes without hierarchical path semantics.
|
|
16
|
+
*
|
|
17
|
+
* @throws {TypeError} If a subtree filter targets an index other than `contextId` or `protocolPath`.
|
|
18
|
+
*/
|
|
19
|
+
export function assertValidSubtreeFilters(filters: Filter[]): void {
|
|
20
|
+
for (const filter of filters) {
|
|
21
|
+
for (const property in filter) {
|
|
22
|
+
if (isSubtreeFilter(filter[property]) && !SUBTREE_FILTER_PROPERTIES.has(property)) {
|
|
23
|
+
throw new TypeError(`SubtreeFilter is not supported for index '${property}'.`);
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
}
|
|
2
28
|
|
|
3
29
|
/**
|
|
4
30
|
* A Utility class to help match indexes against filters.
|
|
@@ -84,7 +110,9 @@ export class FilterUtility {
|
|
|
84
110
|
if (this.matchOneOf(filterValue, indexValue)) {
|
|
85
111
|
return true;
|
|
86
112
|
}
|
|
87
|
-
} else if (this.
|
|
113
|
+
} else if (isSubtreeFilter(filterValue) && this.matchesSubtree(filterValue.subtree, indexValue)) {
|
|
114
|
+
return true;
|
|
115
|
+
} else if (this.isRangeFilter(filterValue) && this.matchRange(filterValue, indexValue as RangeValue)) {
|
|
88
116
|
// `filterValue` is a `RangeFilter`
|
|
89
117
|
// range filters cannot range over booleans
|
|
90
118
|
return true;
|
|
@@ -181,6 +209,15 @@ export class FilterUtility {
|
|
|
181
209
|
};
|
|
182
210
|
}
|
|
183
211
|
|
|
212
|
+
/**
|
|
213
|
+
* Returns whether a candidate is the selected hierarchical path or one of
|
|
214
|
+
* its proper `/`-delimited descendants.
|
|
215
|
+
*/
|
|
216
|
+
static matchesSubtree(subtree: string, candidate: unknown): candidate is string {
|
|
217
|
+
return typeof candidate === 'string' &&
|
|
218
|
+
(candidate === subtree || candidate.startsWith(`${subtree}/`));
|
|
219
|
+
}
|
|
220
|
+
|
|
184
221
|
}
|
|
185
222
|
|
|
186
223
|
export class FilterSelector {
|
|
@@ -238,4 +275,4 @@ export class FilterSelector {
|
|
|
238
275
|
singlePropertyFilter[firstRemainingProperty] = filter[firstRemainingProperty];
|
|
239
276
|
return singlePropertyFilter;
|
|
240
277
|
}
|
|
241
|
-
}
|
|
278
|
+
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { PermissionScope, RecordsPermissionScope } from '../types/permission-types.js';
|
|
2
2
|
import type { ProtocolDefinition, ProtocolRuleSet } from '../types/protocols-types.js';
|
|
3
3
|
|
|
4
|
+
import { FilterUtility } from './filter.js';
|
|
4
5
|
import { KeyDerivationScheme } from './hd-key.js';
|
|
5
6
|
import { ProtocolAction } from '../types/protocols-types.js';
|
|
6
7
|
import { DwnInterfaceName, DwnMethodName } from '../enums/dwn-interface-method.js';
|
|
@@ -171,7 +172,7 @@ function readGrantKeyScopeCoversDeliveredScope(input: {
|
|
|
171
172
|
return false;
|
|
172
173
|
}
|
|
173
174
|
|
|
174
|
-
if (
|
|
175
|
+
if (FilterUtility.matchesSubtree(grantProtocolPath, deliveredProtocolPath)) {
|
|
175
176
|
return true;
|
|
176
177
|
}
|
|
177
178
|
|
|
@@ -206,7 +207,7 @@ function writeGrantKeyScopeCoversDeliveredScope(input: {
|
|
|
206
207
|
return true;
|
|
207
208
|
}
|
|
208
209
|
|
|
209
|
-
return
|
|
210
|
+
return FilterUtility.matchesSubtree(grantProtocolPath, deliveredProtocolPath);
|
|
210
211
|
}
|
|
211
212
|
|
|
212
213
|
function getGrantKeyReadRolePathsForScope(protocolDefinition: ProtocolDefinition, scopeProtocolPath: string): string[] {
|
|
@@ -257,7 +258,7 @@ function collectReadRolePaths(protocolDefinition: ProtocolDefinition, ruleSet: P
|
|
|
257
258
|
|
|
258
259
|
function getGrantKeyRolePathsCoveredByScope(protocolDefinition: ProtocolDefinition, scopeProtocolPath?: string): string[] {
|
|
259
260
|
return collectRolePaths(protocolDefinition, protocolDefinition.structure as ProtocolRuleSet)
|
|
260
|
-
.filter((rolePath) => scopeProtocolPath === undefined ||
|
|
261
|
+
.filter((rolePath) => scopeProtocolPath === undefined || FilterUtility.matchesSubtree(scopeProtocolPath, rolePath));
|
|
261
262
|
}
|
|
262
263
|
|
|
263
264
|
function collectRolePaths(protocolDefinition: ProtocolDefinition, ruleSet: ProtocolRuleSet, parentPath: string[] = []): string[] {
|
|
@@ -285,10 +286,6 @@ function isGrantKeyLocalRolePath(protocolDefinition: ProtocolDefinition, protoco
|
|
|
285
286
|
return ruleSet?.$role === true && ruleSet.$keyAgreement !== undefined;
|
|
286
287
|
}
|
|
287
288
|
|
|
288
|
-
function isBoundaryAwareSubtree(scopePath: string, candidatePath: string): boolean {
|
|
289
|
-
return `${candidatePath}/`.startsWith(`${scopePath}/`);
|
|
290
|
-
}
|
|
291
|
-
|
|
292
289
|
function getGrantKeyScopeCacheKey(scope: GrantKeyProtocolPathScope): string {
|
|
293
290
|
return JSON.stringify([scope.protocol, scope.protocolPath]);
|
|
294
291
|
}
|
package/src/utils/messages.ts
CHANGED
|
@@ -196,26 +196,16 @@ export class Messages {
|
|
|
196
196
|
delete filterCopy.dateUpdated;
|
|
197
197
|
}
|
|
198
198
|
|
|
199
|
-
// Convert protocolPathPrefix into a protocolPath
|
|
200
|
-
// The range gte: prefix, lt: prefix + '/\uffff' matches:
|
|
201
|
-
// - exact: 'post' (prefix itself)
|
|
202
|
-
// - children: 'post/attachment', 'post/comment', etc.
|
|
203
|
-
// - NOT siblings: 'poster', 'postfix' (excluded because '/' < any alphanumeric)
|
|
199
|
+
// Convert protocolPathPrefix into a segment-aware protocolPath subtree filter.
|
|
204
200
|
if (protocolPathPrefix !== undefined) {
|
|
205
201
|
delete (filterCopy as any).protocolPathPrefix;
|
|
206
|
-
filterCopy.protocolPath = {
|
|
207
|
-
gte : protocolPathPrefix,
|
|
208
|
-
lt : protocolPathPrefix + '/\uffff',
|
|
209
|
-
};
|
|
202
|
+
filterCopy.protocolPath = { subtree: protocolPathPrefix };
|
|
210
203
|
}
|
|
211
204
|
|
|
212
|
-
// Convert contextIdPrefix into
|
|
205
|
+
// Convert contextIdPrefix into the same segment-aware subtree filter.
|
|
213
206
|
if (contextIdPrefix !== undefined) {
|
|
214
207
|
delete (filterCopy as any).contextIdPrefix;
|
|
215
|
-
filterCopy.contextId = {
|
|
216
|
-
gte : contextIdPrefix,
|
|
217
|
-
lt : contextIdPrefix + '/\uffff',
|
|
218
|
-
};
|
|
208
|
+
filterCopy.contextId = { subtree: contextIdPrefix };
|
|
219
209
|
}
|
|
220
210
|
|
|
221
211
|
return filterCopy;
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { FilterUtility } from './filter.js';
|
|
2
|
+
|
|
1
3
|
/**
|
|
2
4
|
* Minimal protocol-bound scope shape used by permission grant authorization.
|
|
3
5
|
*
|
|
@@ -38,22 +40,13 @@ export class PermissionScopeMatcher {
|
|
|
38
40
|
}
|
|
39
41
|
|
|
40
42
|
if (scope.protocolPath !== undefined) {
|
|
41
|
-
return
|
|
43
|
+
return FilterUtility.matchesSubtree(scope.protocolPath, target.protocolPath);
|
|
42
44
|
}
|
|
43
45
|
|
|
44
46
|
if (scope.contextId !== undefined) {
|
|
45
|
-
return
|
|
47
|
+
return FilterUtility.matchesSubtree(scope.contextId, target.contextId);
|
|
46
48
|
}
|
|
47
49
|
|
|
48
50
|
return true;
|
|
49
51
|
}
|
|
50
|
-
|
|
51
|
-
private static matchesContextId(scopeContextId: string, candidateContextId: unknown): boolean {
|
|
52
|
-
return PermissionScopeMatcher.matchesSubtree(scopeContextId, candidateContextId);
|
|
53
|
-
}
|
|
54
|
-
|
|
55
|
-
private static matchesSubtree(scopeValue: string, candidateValue: unknown): boolean {
|
|
56
|
-
return typeof candidateValue === 'string' &&
|
|
57
|
-
(candidateValue === scopeValue || candidateValue.startsWith(scopeValue + '/'));
|
|
58
|
-
}
|
|
59
52
|
}
|