@kubun/store-graph 0.17.1 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/access-default-records.d.ts +18 -0
- package/lib/access-default-records.js +167 -0
- package/lib/api.d.ts +382 -51
- package/lib/api.js +2397 -186
- package/lib/catalog-merge.d.ts +5 -0
- package/lib/catalog-merge.js +117 -0
- package/lib/catalog-records.d.ts +34 -0
- package/lib/catalog-records.js +527 -0
- package/lib/catalog-scope.d.ts +120 -0
- package/lib/catalog-scope.js +165 -0
- package/lib/catalog-state.d.ts +21 -0
- package/lib/catalog-state.js +122 -0
- package/lib/catalog-types.d.ts +71 -0
- package/lib/catalog-types.js +1 -0
- package/lib/errors.d.ts +15 -0
- package/lib/errors.js +33 -0
- package/lib/identifiers.d.ts +7 -0
- package/lib/identifiers.js +27 -0
- package/lib/index.d.ts +11 -4
- package/lib/index.js +7 -2
- package/lib/migrations.d.ts +8 -10
- package/lib/migrations.js +111 -18
- package/lib/query-builder.js +1 -1
- package/lib/tables.d.ts +194 -39
- package/package.json +14 -11
package/lib/api.js
CHANGED
|
@@ -1,7 +1,17 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { AbstractPostgresAdapter, chunked, IN_LIST_CHUNK } from '@kubun/db-adapter';
|
|
2
|
+
import { CatalogID, DocumentID, DocumentModelID } from '@kubun/id';
|
|
2
3
|
import { winnerStateDigestOf } from '@kubun/mutation';
|
|
4
|
+
import { isDocumentDateField } from '@kubun/protocol';
|
|
5
|
+
import { blake3 } from '@noble/hashes/blake3.js';
|
|
6
|
+
import { bytesToHex } from '@noble/hashes/utils.js';
|
|
7
|
+
import { sql } from 'kysely';
|
|
8
|
+
import { createGraphAccessDefaultRecordMethods } from './access-default-records.js';
|
|
9
|
+
import { createGraphCatalogRecordMethods } from './catalog-records.js';
|
|
10
|
+
import { parseScopeDate, parseScopeDuration, scopeRuleMatches, scopeRuleMatchesFacts, validateScopeCatalog } from './catalog-scope.js';
|
|
11
|
+
import { createCatalogStateMethods } from './catalog-state.js';
|
|
3
12
|
import { serializeCursor } from './cursor.js';
|
|
4
|
-
import { isMissingTableError, ModelNotDeployedError } from './errors.js';
|
|
13
|
+
import { DocumentLockRequiresTransactionError, DocumentNotHoldableError, InvalidModelDateFieldError, isMissingTableError, MissingGraphCommitContextError, ModelNotDeployedError } from './errors.js';
|
|
14
|
+
import { boundedIdentifier } from './identifiers.js';
|
|
5
15
|
import { aggregateFieldName, applyAggregateSelections, applyDocumentFilter, applyDocumentOrderBy, applyGroupBySelections, applyPagination, groupKeyAlias, knownFieldColumn } from './query-builder.js';
|
|
6
16
|
import { resolveSearchableFields, SearchNotEnabledError } from './search.js';
|
|
7
17
|
// --- Helper functions ---
|
|
@@ -263,7 +273,8 @@ export function documentModelFromTable(row) {
|
|
|
263
273
|
interfaces: row.interfaces,
|
|
264
274
|
schema: row.schema,
|
|
265
275
|
fieldsMeta: row.fields_meta,
|
|
266
|
-
providerMergePolicy: row.provider_merge_policy ?? undefined
|
|
276
|
+
providerMergePolicy: row.provider_merge_policy ?? undefined,
|
|
277
|
+
dateField: row.date_field ?? undefined
|
|
267
278
|
};
|
|
268
279
|
if (model.behavior === 'unique') {
|
|
269
280
|
model.uniqueFields = row.unique_fields;
|
|
@@ -280,7 +291,9 @@ export function documentModelToTable(adapter, id, model) {
|
|
|
280
291
|
interfaces: adapter.encodeJSON(model.interfaces),
|
|
281
292
|
schema: adapter.encodeJSON(model.schema),
|
|
282
293
|
fields_meta: adapter.encodeJSON(model.fieldsMeta),
|
|
283
|
-
provider_merge_policy: model.providerMergePolicy ? adapter.encodeJSON(model.providerMergePolicy) : null
|
|
294
|
+
provider_merge_policy: model.providerMergePolicy ? adapter.encodeJSON(model.providerMergePolicy) : null,
|
|
295
|
+
date_field: model.dateField ?? null,
|
|
296
|
+
resolved_date_field: model.dateField ?? null
|
|
284
297
|
};
|
|
285
298
|
}
|
|
286
299
|
function documentNodeFromTable(adapter, doc) {
|
|
@@ -289,8 +302,9 @@ function documentNodeFromTable(adapter, doc) {
|
|
|
289
302
|
model: doc.model,
|
|
290
303
|
owner: doc.owner,
|
|
291
304
|
data: doc.data,
|
|
292
|
-
createdAt: adapter.decodeTimestamp(doc.created_at),
|
|
293
|
-
updatedAt: doc.updated_at ? adapter.decodeTimestamp(doc.updated_at) : null
|
|
305
|
+
createdAt: doc.mutation_created_at == null ? adapter.decodeTimestamp(doc.created_at) : new Date(Number(doc.mutation_created_at)),
|
|
306
|
+
updatedAt: doc.mutation_updated_at == null ? doc.updated_at ? adapter.decodeTimestamp(doc.updated_at) : null : new Date(Number(doc.mutation_updated_at)),
|
|
307
|
+
scopeDate: doc.scope_date == null ? undefined : Number(doc.scope_date)
|
|
294
308
|
};
|
|
295
309
|
}
|
|
296
310
|
function toGlobalInterfaceID(modelID, interfaceID) {
|
|
@@ -310,10 +324,31 @@ function extractFieldValue(data, fieldPath) {
|
|
|
310
324
|
async function addDocumentModel(transaction, adapter, params) {
|
|
311
325
|
const { id, graphModelID, model } = params;
|
|
312
326
|
const t = adapter.types;
|
|
313
|
-
|
|
327
|
+
const inheritedRows = model.interfaces.length === 0 ? [] : await transaction.selectFrom('kubun_graph_document_models').select('resolved_date_field').where('id', 'in', model.interfaces.map((interfaceID)=>toGlobalInterfaceID(id, interfaceID))).execute();
|
|
328
|
+
const inheritedFields = new Set(inheritedRows.map((row)=>row.resolved_date_field).filter((field)=>field != null));
|
|
329
|
+
if (model.dateField == null && inheritedFields.size > 1) {
|
|
330
|
+
throw new InvalidModelDateFieldError('Conflicting inherited date fields');
|
|
331
|
+
}
|
|
332
|
+
const resolvedDateField = model.dateField ?? inheritedFields.values().next().value ?? null;
|
|
333
|
+
if (resolvedDateField != null) {
|
|
334
|
+
if (!isDocumentDateField(model, resolvedDateField)) {
|
|
335
|
+
throw new InvalidModelDateFieldError(`Invalid model date field: ${resolvedDateField}`);
|
|
336
|
+
}
|
|
337
|
+
}
|
|
338
|
+
await transaction.schema.createTable(`k_${id}`).ifNotExists().addColumn('id', t.text, (col)=>col.notNull().primaryKey()).addColumn('owner', t.text, (col)=>col.notNull()).addColumn('model', t.text, (col)=>col.notNull()).addColumn('data', t.json).addColumn('field_hlcs', t.json).addColumn('field_values', t.json).addColumn('unique', t.binary, (col)=>col.notNull()).addColumn('mutation_created_at', t.bigint).addColumn('mutation_updated_at', t.bigint).addColumn('scope_date', t.bigint).addColumn('created_at', t.timestamp, (col)=>{
|
|
314
339
|
return col.defaultTo(adapter.functions.now).notNull();
|
|
315
340
|
}).addColumn('updated_at', t.timestamp).execute();
|
|
316
|
-
await transaction.
|
|
341
|
+
await transaction.schema.createIndex(boundedIdentifier(`idx_scope_date_${id}`)).ifNotExists().on(`k_${id}`).column('scope_date').execute();
|
|
342
|
+
// Document-sync pages walk ids in byte order. SQLite text compares bytewise
|
|
343
|
+
// already; a Postgres database with a linguistic collation needs an index in
|
|
344
|
+
// the "C" collation for those pages to seek instead of sort.
|
|
345
|
+
if (adapter instanceof AbstractPostgresAdapter) {
|
|
346
|
+
await transaction.schema.createIndex(boundedIdentifier(`idx_id_bytes_${id}`)).ifNotExists().on(`k_${id}`).expression(sql`id collate "C"`).execute();
|
|
347
|
+
}
|
|
348
|
+
await transaction.insertInto('kubun_graph_document_models').values({
|
|
349
|
+
...documentModelToTable(adapter, id, model),
|
|
350
|
+
resolved_date_field: resolvedDateField
|
|
351
|
+
}).onConflict((oc)=>oc.doNothing()).execute();
|
|
317
352
|
for (const interfaceID of model.interfaces){
|
|
318
353
|
await transaction.insertInto('kubun_graph_document_model_interfaces').values({
|
|
319
354
|
interface_id: toGlobalInterfaceID(id, interfaceID),
|
|
@@ -325,7 +360,7 @@ async function addDocumentModel(transaction, adapter, params) {
|
|
|
325
360
|
document_model_id: id
|
|
326
361
|
}).onConflict((oc)=>oc.doNothing()).execute();
|
|
327
362
|
}
|
|
328
|
-
// ---
|
|
363
|
+
// --- Transaction-scoped merge-policy winner read ---
|
|
329
364
|
/**
|
|
330
365
|
* Reads a document's `field_values` cell on the given transaction and
|
|
331
366
|
* delegates the merge-policy liveness comparison to `@kubun/mutation`'s pure
|
|
@@ -348,27 +383,320 @@ async function addDocumentModel(transaction, adapter, params) {
|
|
|
348
383
|
const fieldValues = typeof row.field_values === 'string' ? JSON.parse(row.field_values) : row.field_values;
|
|
349
384
|
return winnerStateDigestOf(fieldValues);
|
|
350
385
|
}
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
if (models != null) {
|
|
364
|
-
const catalogModelSet = new Set(models);
|
|
365
|
-
effectiveModelIDs = effectiveModelIDs.filter((id)=>catalogModelSet.has(id));
|
|
386
|
+
/**
|
|
387
|
+
* Batch form of `readWinnerStateDigest`: one chunked `id in` read per model rather than one
|
|
388
|
+
* read per document. A document whose digest cannot be read (bad id, model table not deployed,
|
|
389
|
+
* no row, no digest) is left out of the map.
|
|
390
|
+
*/ export async function readWinnerStateDigests(params) {
|
|
391
|
+
const byModel = new Map();
|
|
392
|
+
for (const documentID of params.documentIDs){
|
|
393
|
+
let modelID;
|
|
394
|
+
try {
|
|
395
|
+
modelID = DocumentID.fromString(documentID).model.toString();
|
|
396
|
+
} catch {
|
|
397
|
+
continue;
|
|
366
398
|
}
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
399
|
+
const ids = byModel.get(modelID);
|
|
400
|
+
if (ids === undefined) byModel.set(modelID, [
|
|
401
|
+
documentID
|
|
402
|
+
]);
|
|
403
|
+
else ids.push(documentID);
|
|
404
|
+
}
|
|
405
|
+
const digests = new Map();
|
|
406
|
+
for (const [modelID, ids] of byModel){
|
|
407
|
+
try {
|
|
408
|
+
for (const chunk of chunked(ids)){
|
|
409
|
+
const rows = await params.tx.selectFrom(`k_${modelID}`).select([
|
|
410
|
+
'id',
|
|
411
|
+
'field_values'
|
|
412
|
+
]).where('id', 'in', chunk).execute();
|
|
413
|
+
for (const row of rows){
|
|
414
|
+
if (row.field_values == null) continue;
|
|
415
|
+
// ParseJSONResultsPlugin may have already parsed the value.
|
|
416
|
+
const fieldValues = typeof row.field_values === 'string' ? JSON.parse(row.field_values) : row.field_values;
|
|
417
|
+
const digest = winnerStateDigestOf(fieldValues);
|
|
418
|
+
if (digest != null) digests.set(String(row.id), digest);
|
|
419
|
+
}
|
|
420
|
+
}
|
|
421
|
+
} catch {}
|
|
422
|
+
}
|
|
423
|
+
return digests;
|
|
424
|
+
}
|
|
425
|
+
/** Whether a mark has waited out the grace: the sweep's own test, shared so a listing agrees. */ export function gcMarkPastGrace(mark, graceMS, nowMS) {
|
|
426
|
+
return mark.markedAt + graceMS <= nowMS;
|
|
427
|
+
}
|
|
428
|
+
/** Hex blake3 over mutation IDs, sorted and joined by newline: the document mutation digest. */ export function mutationIDsDigest(mutationIDs) {
|
|
429
|
+
const sorted = [
|
|
430
|
+
...mutationIDs
|
|
431
|
+
].sort();
|
|
432
|
+
return bytesToHex(blake3(new TextEncoder().encode(sorted.join('\n'))));
|
|
433
|
+
}
|
|
434
|
+
const scopeSchedulers = new WeakMap();
|
|
435
|
+
const allDocumentRefsLockKey = 'graph.document.refs:*';
|
|
436
|
+
// Transactions, by their Kysely handle, that hold the all-documents ref lock exclusively.
|
|
437
|
+
const exclusiveRefLockHolders = new WeakSet();
|
|
438
|
+
function getScopeScheduler(context) {
|
|
439
|
+
let state = scopeSchedulers.get(context.root);
|
|
440
|
+
if (state != null) return state;
|
|
441
|
+
state = {
|
|
442
|
+
pendingScopes: new Map(),
|
|
443
|
+
timer: null,
|
|
444
|
+
running: null,
|
|
445
|
+
failed: false,
|
|
446
|
+
retryDelayMS: 100,
|
|
447
|
+
generation: 0,
|
|
448
|
+
closed: false,
|
|
449
|
+
listeners: new Set()
|
|
450
|
+
};
|
|
451
|
+
scopeSchedulers.set(context.root, state);
|
|
452
|
+
context.onClose?.(async ()=>{
|
|
453
|
+
if (state == null) return;
|
|
454
|
+
state.closed = true;
|
|
455
|
+
state.generation++;
|
|
456
|
+
if (state.timer != null) clearTimeout(state.timer);
|
|
457
|
+
state.timer = null;
|
|
458
|
+
state.pendingScopes.clear();
|
|
459
|
+
await state.running;
|
|
460
|
+
state.listeners.clear();
|
|
461
|
+
});
|
|
462
|
+
return state;
|
|
463
|
+
}
|
|
464
|
+
/**
|
|
465
|
+
* Resolves once the root's scope scheduler has no timer armed, no recompute running and nothing
|
|
466
|
+
* queued. While recomputes keep failing it keeps retrying, so this does not resolve.
|
|
467
|
+
*/ export async function scopeSchedulerIdle(root) {
|
|
468
|
+
const state = scopeSchedulers.get(root);
|
|
469
|
+
if (state == null) return;
|
|
470
|
+
while(!state.closed && (state.timer != null || state.running != null || state.pendingScopes.size > 0)){
|
|
471
|
+
if (state.running != null) await state.running;
|
|
472
|
+
else await new Promise((resolve)=>setTimeout(resolve, 1));
|
|
371
473
|
}
|
|
474
|
+
}
|
|
475
|
+
function requeueScopeTasks(state, tasks) {
|
|
476
|
+
for (const task of tasks){
|
|
477
|
+
const key = JSON.stringify([
|
|
478
|
+
task.modelID,
|
|
479
|
+
task.catalog?.owner,
|
|
480
|
+
task.catalog?.id
|
|
481
|
+
]);
|
|
482
|
+
const previous = state.pendingScopes.get(key);
|
|
483
|
+
state.pendingScopes.set(key, {
|
|
484
|
+
...task,
|
|
485
|
+
owners: previous == null ? task.owners : previous.owners == null || task.owners == null ? undefined : [
|
|
486
|
+
...new Set([
|
|
487
|
+
...previous.owners,
|
|
488
|
+
...task.owners
|
|
489
|
+
])
|
|
490
|
+
]
|
|
491
|
+
});
|
|
492
|
+
}
|
|
493
|
+
}
|
|
494
|
+
// A scope change written outside the scheduler's recomputes, such as a sweep, still reaches
|
|
495
|
+
// whoever emits scope events.
|
|
496
|
+
function notifyScopeListeners(context) {
|
|
497
|
+
const state = getScopeScheduler(context);
|
|
498
|
+
if (state.closed) return;
|
|
499
|
+
for (const listener of state.listeners){
|
|
500
|
+
void Promise.resolve().then(listener).catch((error)=>console.error('catalog scope listener failed', error));
|
|
501
|
+
}
|
|
502
|
+
}
|
|
503
|
+
function armScopeScheduler(context, state, delayMS = 10) {
|
|
504
|
+
if (state.closed || state.timer != null || state.running || state.pendingScopes.size === 0) return;
|
|
505
|
+
const generation = state.generation;
|
|
506
|
+
state.timer = setTimeout(()=>{
|
|
507
|
+
if (generation !== state.generation) return;
|
|
508
|
+
state.timer = null;
|
|
509
|
+
const pending = Array.from(state.pendingScopes.values());
|
|
510
|
+
state.pendingScopes.clear();
|
|
511
|
+
state.running = (async ()=>{
|
|
512
|
+
try {
|
|
513
|
+
const store = await context.root.getStore('graph');
|
|
514
|
+
for(let index = 0; index < pending.length; index++){
|
|
515
|
+
if (generation !== state.generation) break;
|
|
516
|
+
const task = pending[index];
|
|
517
|
+
if (task == null) continue;
|
|
518
|
+
try {
|
|
519
|
+
const nowMS = Date.now();
|
|
520
|
+
const routeRules = await context.getRootCatalogRouteRules(nowMS, task.catalog);
|
|
521
|
+
await store.reconcileCatalogModelRefs({
|
|
522
|
+
modelID: task.modelID,
|
|
523
|
+
routeRules,
|
|
524
|
+
nowMS,
|
|
525
|
+
owners: task.owners,
|
|
526
|
+
catalog: task.catalog
|
|
527
|
+
});
|
|
528
|
+
} catch (error) {
|
|
529
|
+
requeueScopeTasks(state, pending.slice(index));
|
|
530
|
+
throw error;
|
|
531
|
+
}
|
|
532
|
+
}
|
|
533
|
+
state.failed = false;
|
|
534
|
+
state.retryDelayMS = 100;
|
|
535
|
+
return generation === state.generation ? [
|
|
536
|
+
...state.listeners
|
|
537
|
+
] : [];
|
|
538
|
+
} catch (error) {
|
|
539
|
+
if (state.pendingScopes.size === 0) requeueScopeTasks(state, pending);
|
|
540
|
+
state.failed = true;
|
|
541
|
+
console.error('catalog scope recomputation failed', error);
|
|
542
|
+
return [];
|
|
543
|
+
} finally{
|
|
544
|
+
state.running = null;
|
|
545
|
+
if (state.failed) {
|
|
546
|
+
const delay = state.retryDelayMS;
|
|
547
|
+
state.retryDelayMS = Math.min(delay * 2, 30_000);
|
|
548
|
+
armScopeScheduler(context, state, delay);
|
|
549
|
+
} else armScopeScheduler(context, state);
|
|
550
|
+
}
|
|
551
|
+
})().then((listeners)=>{
|
|
552
|
+
for (const listener of listeners){
|
|
553
|
+
void Promise.resolve().then(listener).catch((error)=>console.error('catalog scope listener failed', error));
|
|
554
|
+
}
|
|
555
|
+
});
|
|
556
|
+
}, delayMS);
|
|
557
|
+
}
|
|
558
|
+
const catalogRulePrefix = 'rule:';
|
|
559
|
+
function catalogRefPrefix(catalog) {
|
|
560
|
+
return `${catalogRulePrefix}${catalog.owner}:${catalog.id}:`;
|
|
561
|
+
}
|
|
562
|
+
const custodyPinRefPrefix = 'hold:custody:';
|
|
563
|
+
/** The sticky ref a custodian pins a document with: the catalog ID part holds no `:`. */ export function custodyPinRef(guard) {
|
|
564
|
+
const catalogID = new CatalogID(guard.catalog.owner, guard.catalog.id).toString();
|
|
565
|
+
return `${custodyPinRefPrefix}${guard.custodyOwner}:${catalogID}`;
|
|
566
|
+
}
|
|
567
|
+
export function isCustodyPinRef(ref) {
|
|
568
|
+
return ref.startsWith(custodyPinRefPrefix);
|
|
569
|
+
}
|
|
570
|
+
export function catalogRuleRef(catalog, ruleID) {
|
|
571
|
+
return `${catalogRefPrefix(catalog)}${ruleID}`;
|
|
572
|
+
}
|
|
573
|
+
/**
|
|
574
|
+
* Whether a ref is one of the catalog's rule refs. Catalog IDs may contain `:`, so a prefix
|
|
575
|
+
* alone would also claim the refs of a catalog whose ID extends this one.
|
|
576
|
+
*/ export function isCatalogRuleRef(ref, catalog) {
|
|
577
|
+
const prefix = catalogRefPrefix(catalog);
|
|
578
|
+
return ref.startsWith(prefix) && /^[^:]+$/.test(ref.slice(prefix.length));
|
|
579
|
+
}
|
|
580
|
+
const SWEPT_PAGE = 500;
|
|
581
|
+
const catalogLevelBits = {
|
|
582
|
+
KNOWN: {
|
|
583
|
+
displays: 1,
|
|
584
|
+
retains: 0
|
|
585
|
+
},
|
|
586
|
+
OFFLINE: {
|
|
587
|
+
displays: 1,
|
|
588
|
+
retains: 1
|
|
589
|
+
},
|
|
590
|
+
SERVE_ONLY: {
|
|
591
|
+
displays: 0,
|
|
592
|
+
retains: 1
|
|
593
|
+
}
|
|
594
|
+
};
|
|
595
|
+
function catalogUseFromRow(row) {
|
|
596
|
+
return {
|
|
597
|
+
level: row.level ?? null,
|
|
598
|
+
followed: Number(row.followed) === 1
|
|
599
|
+
};
|
|
600
|
+
}
|
|
601
|
+
function scheduleScopeTasks(context, tasks) {
|
|
602
|
+
const state = getScopeScheduler(context);
|
|
603
|
+
const pending = Array.from(tasks);
|
|
604
|
+
if (pending.length === 0) return;
|
|
605
|
+
context.provider.onCommit(()=>{
|
|
606
|
+
if (state.closed) return;
|
|
607
|
+
requeueScopeTasks(state, pending);
|
|
608
|
+
if (state.timer != null) {
|
|
609
|
+
clearTimeout(state.timer);
|
|
610
|
+
state.timer = null;
|
|
611
|
+
}
|
|
612
|
+
armScopeScheduler(context, state, state.failed ? state.retryDelayMS : undefined);
|
|
613
|
+
});
|
|
614
|
+
}
|
|
615
|
+
function scheduleScopeModels(context, modelIDs) {
|
|
616
|
+
scheduleScopeTasks(context, Array.from(modelIDs, (modelID)=>({
|
|
617
|
+
modelID
|
|
618
|
+
})));
|
|
619
|
+
}
|
|
620
|
+
/**
|
|
621
|
+
* The refs that start with a catalog's rule-ref prefix: a bytewise range rather
|
|
622
|
+
* than LIKE, whose wildcards and escape character catalog IDs may contain.
|
|
623
|
+
* On a Postgres adapter it compares `ref collate "C"`, which the ref index is
|
|
624
|
+
* built on. The caller keeps only the catalog's own refs with
|
|
625
|
+
* `isCatalogRuleRef`. Exported so the plan of the exact statement can be
|
|
626
|
+
* checked.
|
|
627
|
+
*/ export function catalogRefHoldersQuery(db, adapter, catalog) {
|
|
628
|
+
const prefix = catalogRefPrefix(catalog);
|
|
629
|
+
const ref = adapter instanceof AbstractPostgresAdapter ? sql`ref collate "C"` : sql`ref`;
|
|
630
|
+
return db.selectFrom('kubun_graph_document_refs').select([
|
|
631
|
+
'document_id',
|
|
632
|
+
'ref'
|
|
633
|
+
]).where(ref, '>=', prefix).where(ref, '<', `${prefix}\u{10FFFF}`);
|
|
634
|
+
}
|
|
635
|
+
/**
|
|
636
|
+
* The statement one model's keyset leaf page runs: matching documents after
|
|
637
|
+
* `after` (and up to `through`) in bytewise ID order, at most `limit`, each with
|
|
638
|
+
* whether it was created here and joined to its applied mutations. Exported so
|
|
639
|
+
* the plan of the exact statement can be checked.
|
|
640
|
+
*/ export function documentLeafPageStatement(params) {
|
|
641
|
+
const { postgres } = params;
|
|
642
|
+
// Bytewise order on every adapter: SQLite compares text with BINARY,
|
|
643
|
+
// Postgres needs the "C" collation to agree with it.
|
|
644
|
+
const key = postgres ? sql`k.id collate "C"` : sql`k.id`;
|
|
645
|
+
const matches = params.rules.map((rule)=>{
|
|
646
|
+
const conditions = [];
|
|
647
|
+
if (rule.owners != null) {
|
|
648
|
+
conditions.push(rule.owners.length === 0 ? sql`1 = 0` : sql`k.owner in (${sql.join([
|
|
649
|
+
...rule.owners
|
|
650
|
+
])})`);
|
|
651
|
+
}
|
|
652
|
+
if (rule.dateField != null && (rule.from != null || rule.to != null)) {
|
|
653
|
+
const bounds = [
|
|
654
|
+
sql`sd.field = ${rule.dateField}`
|
|
655
|
+
];
|
|
656
|
+
if (rule.from != null) bounds.push(sql`sd.scope_date >= ${rule.from}`);
|
|
657
|
+
if (rule.to != null) bounds.push(sql`sd.scope_date < ${rule.to}`);
|
|
658
|
+
conditions.push(sql`exists (select 1 from kubun_graph_document_scope_dates as sd where sd.document_id = k.id and ${sql.join(bounds, sql` and `)})`);
|
|
659
|
+
} else if (rule.from != null || rule.to != null) {
|
|
660
|
+
// A model without a declared date field windows on its scope date,
|
|
661
|
+
// which is the document's creation time.
|
|
662
|
+
if (rule.from != null) conditions.push(sql`k.scope_date >= ${rule.from}`);
|
|
663
|
+
if (rule.to != null) conditions.push(sql`k.scope_date < ${rule.to}`);
|
|
664
|
+
}
|
|
665
|
+
return conditions.length === 0 ? sql`1 = 1` : sql`(${sql.join(conditions, sql` and `)})`;
|
|
666
|
+
});
|
|
667
|
+
const range = [
|
|
668
|
+
sql`(${sql.join(matches, sql` or `)})`
|
|
669
|
+
];
|
|
670
|
+
if (params.groupReadPredicate != null) range.push(params.groupReadPredicate);
|
|
671
|
+
if (params.after != null) range.push(sql`${key} > ${params.after}`);
|
|
672
|
+
if (params.through != null) range.push(sql`${key} <= ${params.through}`);
|
|
673
|
+
const outerKey = postgres ? sql`d.id collate "C"` : sql`d.id`;
|
|
674
|
+
// SQLite would otherwise pick the status index, which holds nearly every
|
|
675
|
+
// row: the unary plus keeps the join on the document index.
|
|
676
|
+
const applied = postgres ? sql`m.status = 'applied'` : sql`+m.status = 'applied'`;
|
|
677
|
+
return sql`
|
|
678
|
+
select d.id, d.owner, d.authored_here, m.mutation_hash
|
|
679
|
+
from (
|
|
680
|
+
select k.id, k.owner, exists (
|
|
681
|
+
select 1 from kubun_graph_document_refs as r
|
|
682
|
+
where r.document_id = k.id and r.ref = 'local'
|
|
683
|
+
) as authored_here
|
|
684
|
+
from ${sql.table(`k_${params.modelID}`)} as k
|
|
685
|
+
where ${sql.join(range, sql` and `)}
|
|
686
|
+
order by ${key}
|
|
687
|
+
limit ${params.limit}
|
|
688
|
+
) as d
|
|
689
|
+
left join kubun_graph_mutation_log as m
|
|
690
|
+
on m.document_id = d.id and ${applied}
|
|
691
|
+
order by ${outerKey}`;
|
|
692
|
+
}
|
|
693
|
+
// A custody pin, or a ref a SERVE_ONLY catalog could have written: retained, not displayed.
|
|
694
|
+
function mayNeedBytes(row) {
|
|
695
|
+
return isCustodyPinRef(row.ref) || row.ref.startsWith(catalogRulePrefix) && Number(row.displays) === 0 && Number(row.retains) === 1;
|
|
696
|
+
}
|
|
697
|
+
export function createGraphStore(db, adapter, context) {
|
|
698
|
+
if (context == null) throw new MissingGraphCommitContextError();
|
|
699
|
+
const coerce = adapter.coerceFilterValue.bind(adapter);
|
|
372
700
|
// --- Write-path search index maintenance ---
|
|
373
701
|
//
|
|
374
702
|
// Runs inline in the caller's write transaction (`db` here IS that
|
|
@@ -394,7 +722,703 @@ export function createGraphStore(db, adapter) {
|
|
|
394
722
|
if (fields == null) return;
|
|
395
723
|
await adapter.removeSearchEntry(db, model, documentID);
|
|
396
724
|
}
|
|
725
|
+
async function refreshDocumentDates(documentID) {
|
|
726
|
+
const modelID = DocumentID.fromString(documentID).model.toString();
|
|
727
|
+
const row = await db.selectFrom(`k_${modelID}`).selectAll().where('id', '=', documentID).executeTakeFirst();
|
|
728
|
+
if (row == null) return;
|
|
729
|
+
const extrema = await db.selectFrom('kubun_graph_mutation_log').select('hlc').where('document_id', '=', documentID).where('status', '=', 'applied').orderBy('hlc').execute();
|
|
730
|
+
const createdAt = extrema[0] == null ? adapter.decodeTimestamp(row.created_at).getTime() : Date.parse(extrema[0].hlc.slice(0, 24));
|
|
731
|
+
const lastMutation = extrema.at(-1);
|
|
732
|
+
const updatedAt = lastMutation == null ? null : Date.parse(lastMutation.hlc.slice(0, 24));
|
|
733
|
+
const model = await db.selectFrom('kubun_graph_document_models').select([
|
|
734
|
+
'resolved_date_field',
|
|
735
|
+
'schema'
|
|
736
|
+
]).where('id', '=', modelID).executeTakeFirstOrThrow();
|
|
737
|
+
const data = row.data;
|
|
738
|
+
const selectedValue = model.resolved_date_field == null || data == null ? null : extractFieldValue(data, model.resolved_date_field);
|
|
739
|
+
const scopeDate = parseScopeDate(selectedValue) ?? createdAt;
|
|
740
|
+
await db.updateTable(`k_${modelID}`).set({
|
|
741
|
+
mutation_created_at: extrema[0] == null ? null : createdAt,
|
|
742
|
+
mutation_updated_at: updatedAt,
|
|
743
|
+
scope_date: scopeDate
|
|
744
|
+
}).where('id', '=', documentID).execute();
|
|
745
|
+
await db.deleteFrom('kubun_graph_document_scope_dates').where('document_id', '=', documentID).execute();
|
|
746
|
+
const dates = Object.keys(model.schema.properties).filter((field)=>isDocumentDateField(model, field)).map((field)=>({
|
|
747
|
+
document_id: documentID,
|
|
748
|
+
field,
|
|
749
|
+
scope_date: parseScopeDate(data == null ? null : extractFieldValue(data, field)) ?? createdAt
|
|
750
|
+
}));
|
|
751
|
+
if (dates.length > 0) await db.insertInto('kubun_graph_document_scope_dates').values(dates).execute();
|
|
752
|
+
}
|
|
753
|
+
// Same size as a custody scan batch: a batch is one transaction holding the connection.
|
|
754
|
+
const ReconcileBatch = 100;
|
|
755
|
+
const reconcileAll = async (nowMS, routeRules, shouldStop)=>{
|
|
756
|
+
const models = await db.selectFrom('kubun_graph_document_models').select('id').execute();
|
|
757
|
+
const documentIDs = [];
|
|
758
|
+
for (const model of models){
|
|
759
|
+
const docs = await db.selectFrom(`k_${model.id}`).select('id').execute();
|
|
760
|
+
for (const doc of docs)documentIDs.push(doc.id);
|
|
761
|
+
}
|
|
762
|
+
documentIDs.sort();
|
|
763
|
+
if (context.provider !== context.root) {
|
|
764
|
+
// The caller's transaction spans the whole pass, so there is no connection to release.
|
|
765
|
+
if (shouldStop()) return false;
|
|
766
|
+
await lockAllDocumentRefs();
|
|
767
|
+
await recomputeDocumentRefsBatch({
|
|
768
|
+
documentIDs,
|
|
769
|
+
routeRules,
|
|
770
|
+
nowMS
|
|
771
|
+
});
|
|
772
|
+
await api.refreshSweptDocuments({
|
|
773
|
+
nowMS
|
|
774
|
+
});
|
|
775
|
+
return true;
|
|
776
|
+
}
|
|
777
|
+
for(let i = 0; i < documentIDs.length; i += ReconcileBatch){
|
|
778
|
+
if (shouldStop()) return false;
|
|
779
|
+
await api.recomputeDocuments({
|
|
780
|
+
documentIDs: documentIDs.slice(i, i + ReconcileBatch),
|
|
781
|
+
routeRules,
|
|
782
|
+
nowMS
|
|
783
|
+
});
|
|
784
|
+
}
|
|
785
|
+
if (shouldStop()) return false;
|
|
786
|
+
await api.refreshSweptDocuments({
|
|
787
|
+
nowMS
|
|
788
|
+
});
|
|
789
|
+
return true;
|
|
790
|
+
};
|
|
791
|
+
async function tasksForCatalog(ref, rules) {
|
|
792
|
+
const selected = new Map();
|
|
793
|
+
let modelsQuery = db.selectFrom('kubun_graph_document_models').select('id');
|
|
794
|
+
if (!rules.some((rule)=>rule.models == null)) {
|
|
795
|
+
const modelIDs = [
|
|
796
|
+
...new Set(rules.flatMap((rule)=>rule.models ?? []))
|
|
797
|
+
];
|
|
798
|
+
if (modelIDs.length === 0) modelsQuery = modelsQuery.where(sql`1 = 0`);
|
|
799
|
+
else modelsQuery = modelsQuery.where((eb)=>eb.or([
|
|
800
|
+
eb('id', 'in', modelIDs),
|
|
801
|
+
eb('id', 'in', eb.selectFrom('kubun_graph_document_model_interfaces').select('implementation_id').where('interface_id', 'in', modelIDs))
|
|
802
|
+
]));
|
|
803
|
+
}
|
|
804
|
+
const models = await modelsQuery.execute();
|
|
805
|
+
const interfacesOf = memoisedInterfaces();
|
|
806
|
+
for (const model of models){
|
|
807
|
+
const interfaces = await interfacesOf(model.id);
|
|
808
|
+
for (const rule of rules){
|
|
809
|
+
if (rule.models != null && !rule.models.some((id)=>id === model.id || interfaces.includes(id))) continue;
|
|
810
|
+
const previous = selected.get(model.id);
|
|
811
|
+
if (rule.owners == null || previous === null) {
|
|
812
|
+
selected.set(model.id, null);
|
|
813
|
+
} else {
|
|
814
|
+
const owners = previous ?? new Set();
|
|
815
|
+
for (const owner of rule.owners)owners.add(owner);
|
|
816
|
+
selected.set(model.id, owners);
|
|
817
|
+
}
|
|
818
|
+
}
|
|
819
|
+
}
|
|
820
|
+
for (const row of (await catalogRefHolders(ref))){
|
|
821
|
+
const modelID = DocumentID.fromString(row.document_id).model.toString();
|
|
822
|
+
if (!selected.has(modelID)) selected.set(modelID, new Set());
|
|
823
|
+
}
|
|
824
|
+
return Array.from(selected, ([modelID, owners])=>({
|
|
825
|
+
modelID,
|
|
826
|
+
owners: owners == null ? undefined : Array.from(owners),
|
|
827
|
+
catalog: ref
|
|
828
|
+
}));
|
|
829
|
+
}
|
|
830
|
+
async function catalogRefHolders(catalog) {
|
|
831
|
+
const rows = await catalogRefHoldersQuery(db, adapter, catalog).execute();
|
|
832
|
+
return rows.filter((row)=>isCatalogRuleRef(row.ref, catalog));
|
|
833
|
+
}
|
|
834
|
+
function visibleScopePredicate(modelID, refs) {
|
|
835
|
+
const refCondition = refs == null ? sql`` : sql`and scope_ref.ref in (${sql.join(refs)})`;
|
|
836
|
+
return sql`exists (
|
|
837
|
+
select 1 from kubun_graph_document_refs as scope_ref
|
|
838
|
+
where scope_ref.document_id = ${sql.ref(`k_${modelID}.id`)}
|
|
839
|
+
and scope_ref.displays = 1 ${refCondition}
|
|
840
|
+
)`;
|
|
841
|
+
}
|
|
842
|
+
/**
|
|
843
|
+
* Settle scope work queued before a synchronous mutation, skipping the catalog the mutation
|
|
844
|
+
* reconciles itself. Entries leave the queue only when the transaction commits.
|
|
845
|
+
*/ const settlePendingScopes = async (nowMS, except)=>{
|
|
846
|
+
const scheduler = scopeSchedulers.get(context.root);
|
|
847
|
+
for (const [key, task] of scheduler?.pendingScopes ?? []){
|
|
848
|
+
if (except == null || task.catalog?.owner !== except.owner || task.catalog.id !== except.id) {
|
|
849
|
+
const pendingRules = await context.getCatalogRouteRules(nowMS, task.catalog);
|
|
850
|
+
await api.reconcileCatalogModelRefs({
|
|
851
|
+
modelID: task.modelID,
|
|
852
|
+
routeRules: pendingRules,
|
|
853
|
+
nowMS,
|
|
854
|
+
owners: task.owners,
|
|
855
|
+
catalog: task.catalog
|
|
856
|
+
});
|
|
857
|
+
}
|
|
858
|
+
context.provider.onCommit(()=>{
|
|
859
|
+
if (scheduler?.pendingScopes.get(key) === task) scheduler.pendingScopes.delete(key);
|
|
860
|
+
});
|
|
861
|
+
}
|
|
862
|
+
};
|
|
863
|
+
async function visibleCatalogRefs(ref) {
|
|
864
|
+
const catalog = await db.selectFrom('kubun_graph_catalog_use').select([
|
|
865
|
+
'rules',
|
|
866
|
+
'level'
|
|
867
|
+
]).where('owner_did', '=', ref.owner).where('catalog_id', '=', ref.id).executeTakeFirst();
|
|
868
|
+
if (catalog?.level == null || catalogLevelBits[catalog.level].displays !== 1) return [];
|
|
869
|
+
const rules = typeof catalog.rules === 'string' ? JSON.parse(catalog.rules) : catalog.rules;
|
|
870
|
+
return rules.map((rule)=>catalogRuleRef(ref, rule.id));
|
|
871
|
+
}
|
|
872
|
+
function custodyReceiptFromRow(row) {
|
|
873
|
+
return {
|
|
874
|
+
documentID: row.document_id,
|
|
875
|
+
custodian: row.custodian,
|
|
876
|
+
groupID: row.group_id,
|
|
877
|
+
custodyOwner: row.custody_owner,
|
|
878
|
+
catalog: {
|
|
879
|
+
owner: row.catalog_owner,
|
|
880
|
+
id: row.catalog_id
|
|
881
|
+
},
|
|
882
|
+
assignmentDigest: row.assignment_digest,
|
|
883
|
+
headDigest: row.head_digest,
|
|
884
|
+
mutationDigest: row.mutation_digest,
|
|
885
|
+
receivedAt: Number(row.received_at)
|
|
886
|
+
};
|
|
887
|
+
}
|
|
888
|
+
function gcMarkFromRow(row) {
|
|
889
|
+
return {
|
|
890
|
+
documentID: row.document_id,
|
|
891
|
+
digest: row.digest,
|
|
892
|
+
markedAt: Number(row.marked_at)
|
|
893
|
+
};
|
|
894
|
+
}
|
|
895
|
+
function holdFromRow(row) {
|
|
896
|
+
return {
|
|
897
|
+
id: row.hold_id,
|
|
898
|
+
documentID: row.document_id,
|
|
899
|
+
createdBy: row.created_by,
|
|
900
|
+
reason: row.reason,
|
|
901
|
+
expiresAt: row.expires_at == null ? null : Number(row.expires_at)
|
|
902
|
+
};
|
|
903
|
+
}
|
|
904
|
+
const readScopeReader = async ()=>await db.selectFrom('kubun_graph_scope_identity').select([
|
|
905
|
+
'reader_did',
|
|
906
|
+
'default_read_level'
|
|
907
|
+
]).where('id', '=', 1).executeTakeFirst();
|
|
908
|
+
// Whether the scope reader may read a document of this owner and model with this read rule;
|
|
909
|
+
// true with no reader, or when the reader owns it.
|
|
910
|
+
const scopeReaderMayRead = async (owner, model, override)=>await readerMayRead(await readScopeReader(), owner, model, override);
|
|
911
|
+
// One pass's read checks: nothing a ref recompute or swept refresh writes changes an answer.
|
|
912
|
+
const memoisedReadCheck = ()=>{
|
|
913
|
+
let reader;
|
|
914
|
+
const answers = new Map();
|
|
915
|
+
return async (owner, model, override)=>{
|
|
916
|
+
const key = JSON.stringify([
|
|
917
|
+
owner,
|
|
918
|
+
model,
|
|
919
|
+
override
|
|
920
|
+
]);
|
|
921
|
+
let answer = answers.get(key);
|
|
922
|
+
if (answer == null) {
|
|
923
|
+
reader ??= readScopeReader();
|
|
924
|
+
answer = reader.then((row)=>readerMayRead(row, owner, model, override));
|
|
925
|
+
answers.set(key, answer);
|
|
926
|
+
}
|
|
927
|
+
return await answer;
|
|
928
|
+
};
|
|
929
|
+
};
|
|
930
|
+
// One pass's model interfaces, from one read on first use.
|
|
931
|
+
const memoisedInterfaces = ()=>{
|
|
932
|
+
let all;
|
|
933
|
+
return async (modelID)=>{
|
|
934
|
+
all ??= api.listModelInterfaces();
|
|
935
|
+
return (await all).get(modelID) ?? [];
|
|
936
|
+
};
|
|
937
|
+
};
|
|
938
|
+
const readerMayRead = async (reader, owner, model, override)=>{
|
|
939
|
+
if (reader == null || reader.reader_did === owner) return true;
|
|
940
|
+
const grant = override == null ? await api.getUserModelAccessDefault(owner, model, 'read') : null;
|
|
941
|
+
const access = override ?? grant;
|
|
942
|
+
const membership = context.provider.hasStore('p2p') ? await context.provider.getStore('p2p') : undefined;
|
|
943
|
+
return access == null ? !await api.hasControllerAccessDefaults(owner) && reader.default_read_level === 'anyone' : access.level === 'anyone' || access.level === 'restricted' && (access.allowedDIDs?.includes(reader.reader_did) === true || membership != null && (await membership.isMemberOfAnyCircle(reader.reader_did, access.allowedCircles ?? []) || await membership.isMemberOfAnyGroup(reader.reader_did, access.allowedGroups ?? [])));
|
|
944
|
+
};
|
|
945
|
+
function sweptFromRow(row) {
|
|
946
|
+
const parse = (value)=>typeof value === 'string' ? JSON.parse(value) : value;
|
|
947
|
+
return {
|
|
948
|
+
documentID: row.document_id,
|
|
949
|
+
modelID: row.model_id,
|
|
950
|
+
owner: row.owner,
|
|
951
|
+
createdAt: Number(row.created_at),
|
|
952
|
+
scopeDate: row.scope_date == null ? null : Number(row.scope_date),
|
|
953
|
+
fieldDates: parse(row.field_dates),
|
|
954
|
+
readAccess: row.read_access == null ? null : parse(row.read_access),
|
|
955
|
+
mutationDigest: row.mutation_digest,
|
|
956
|
+
sweptAt: Number(row.swept_at)
|
|
957
|
+
};
|
|
958
|
+
}
|
|
959
|
+
// Catalogs at a retaining level, and the pull route rules, as a ref recompute reads them.
|
|
960
|
+
const retainingRules = async (nowMS)=>{
|
|
961
|
+
const uses = [];
|
|
962
|
+
for (const use of (await db.selectFrom('kubun_graph_catalog_use').selectAll().execute())){
|
|
963
|
+
if (use.level == null || catalogLevelBits[use.level].retains !== 1) continue;
|
|
964
|
+
uses.push({
|
|
965
|
+
catalog: {
|
|
966
|
+
owner: use.owner_did,
|
|
967
|
+
id: use.catalog_id
|
|
968
|
+
},
|
|
969
|
+
rules: typeof use.rules === 'string' ? JSON.parse(use.rules) : use.rules
|
|
970
|
+
});
|
|
971
|
+
}
|
|
972
|
+
return {
|
|
973
|
+
uses,
|
|
974
|
+
routeRules: await context.getCatalogRouteRules(nowMS)
|
|
975
|
+
};
|
|
976
|
+
};
|
|
977
|
+
// Every rule of each catalog, whatever its level, keyed by `JSON.stringify([owner, id])`.
|
|
978
|
+
const catalogRulesByRef = async (catalogs)=>{
|
|
979
|
+
const byKey = new Map();
|
|
980
|
+
for (const catalog of catalogs)byKey.set(JSON.stringify([
|
|
981
|
+
catalog.owner,
|
|
982
|
+
catalog.id
|
|
983
|
+
]), catalog);
|
|
984
|
+
const found = new Map();
|
|
985
|
+
// Two bound parameters per catalog.
|
|
986
|
+
for (const chunk of chunked(byKey.values(), IN_LIST_CHUNK / 2)){
|
|
987
|
+
const rows = await db.selectFrom('kubun_graph_catalog_use').select([
|
|
988
|
+
'owner_did',
|
|
989
|
+
'catalog_id',
|
|
990
|
+
'rules'
|
|
991
|
+
]).where((eb)=>eb.or(chunk.map((catalog)=>eb.and([
|
|
992
|
+
eb('owner_did', '=', catalog.owner),
|
|
993
|
+
eb('catalog_id', '=', catalog.id)
|
|
994
|
+
])))).execute();
|
|
995
|
+
for (const row of rows){
|
|
996
|
+
found.set(JSON.stringify([
|
|
997
|
+
row.owner_did,
|
|
998
|
+
row.catalog_id
|
|
999
|
+
]), typeof row.rules === 'string' ? JSON.parse(row.rules) : row.rules);
|
|
1000
|
+
}
|
|
1001
|
+
}
|
|
1002
|
+
return found;
|
|
1003
|
+
};
|
|
1004
|
+
/**
|
|
1005
|
+
* The catalogs whose rules would retain the swept document were it here: the same catalog
|
|
1006
|
+
* levels, route rules and read check a ref recompute applies, over the facts kept at sweep.
|
|
1007
|
+
*/ async function sweptRetainingCatalogs(params) {
|
|
1008
|
+
const { entry, nowMS, retaining } = params;
|
|
1009
|
+
const checks = params.checks ?? {
|
|
1010
|
+
mayRead: scopeReaderMayRead,
|
|
1011
|
+
interfaces: (modelID)=>api.getModelInterfaces(modelID)
|
|
1012
|
+
};
|
|
1013
|
+
if (!await checks.mayRead(entry.owner, entry.modelID, entry.readAccess)) return [];
|
|
1014
|
+
const interfaces = await checks.interfaces(entry.modelID);
|
|
1015
|
+
const facts = {
|
|
1016
|
+
model: entry.modelID,
|
|
1017
|
+
owner: entry.owner,
|
|
1018
|
+
dateOf: (field)=>field == null ? entry.scopeDate ?? entry.createdAt : entry.fieldDates[field] ?? entry.createdAt
|
|
1019
|
+
};
|
|
1020
|
+
const matches = (rules)=>rules.some((rule)=>scopeRuleMatchesFacts(rule, facts, nowMS, interfaces));
|
|
1021
|
+
const found = new Map();
|
|
1022
|
+
for (const use of retaining.uses)if (matches(use.rules)) found.set(catalogRefPrefix(use.catalog), use.catalog);
|
|
1023
|
+
for (const route of retaining.routeRules)if (matches(route.rules)) found.set(catalogRefPrefix(route.catalog), route.catalog);
|
|
1024
|
+
return Array.from(found.values());
|
|
1025
|
+
}
|
|
1026
|
+
async function writeSweptDocument(doc, digest, nowMS) {
|
|
1027
|
+
const fieldDates = {};
|
|
1028
|
+
for (const row of (await db.selectFrom('kubun_graph_document_scope_dates').select([
|
|
1029
|
+
'field',
|
|
1030
|
+
'scope_date'
|
|
1031
|
+
]).where('document_id', '=', doc.id).execute()))fieldDates[row.field] = Number(row.scope_date);
|
|
1032
|
+
const read = doc.data?.accessPermissions?.read;
|
|
1033
|
+
const values = {
|
|
1034
|
+
model_id: doc.model,
|
|
1035
|
+
owner: doc.owner,
|
|
1036
|
+
created_at: doc.createdAt.getTime(),
|
|
1037
|
+
scope_date: doc.scopeDate ?? null,
|
|
1038
|
+
field_dates: JSON.stringify(fieldDates),
|
|
1039
|
+
read_access: read == null ? null : JSON.stringify(read),
|
|
1040
|
+
mutation_digest: digest,
|
|
1041
|
+
swept_at: nowMS,
|
|
1042
|
+
in_scope: 0,
|
|
1043
|
+
scope_left_at: nowMS
|
|
1044
|
+
};
|
|
1045
|
+
await db.insertInto('kubun_graph_swept_documents').values({
|
|
1046
|
+
document_id: doc.id,
|
|
1047
|
+
...values
|
|
1048
|
+
}).onConflict((oc)=>oc.column('document_id').doUpdateSet(values)).execute();
|
|
1049
|
+
}
|
|
1050
|
+
async function recordScopeChange(documentID) {
|
|
1051
|
+
await recordScopeChanges([
|
|
1052
|
+
documentID
|
|
1053
|
+
]);
|
|
1054
|
+
}
|
|
1055
|
+
// Deduplicated per statement: Postgres refuses an upsert that updates one row twice.
|
|
1056
|
+
async function recordScopeChanges(documentIDs) {
|
|
1057
|
+
// Two bound parameters per row.
|
|
1058
|
+
for (const chunk of chunked(documentIDs, IN_LIST_CHUNK / 2)){
|
|
1059
|
+
await db.insertInto('kubun_graph_scope_changes').values(chunk.map((documentID)=>({
|
|
1060
|
+
document_id: documentID,
|
|
1061
|
+
version: 1
|
|
1062
|
+
}))).onConflict((oc)=>oc.column('document_id').doUpdateSet({
|
|
1063
|
+
version: sql`kubun_graph_scope_changes.version + 1`
|
|
1064
|
+
})).execute();
|
|
1065
|
+
}
|
|
1066
|
+
}
|
|
1067
|
+
// Ref writers decide from rows another transaction may be replacing, so each document's refs
|
|
1068
|
+
// take one writer at a time. A single-document writer takes the all-documents key shared, then
|
|
1069
|
+
// its document's key. A bulk writer takes only the all-documents key, exclusive: one lock
|
|
1070
|
+
// however many documents it rewrites, where a lock per document held to commit would exhaust
|
|
1071
|
+
// Postgres's shared lock table on a large graph.
|
|
1072
|
+
// SQLite serializes write transactions already; outside a transaction the lock would end
|
|
1073
|
+
// with its own statement.
|
|
1074
|
+
function refLocksApply() {
|
|
1075
|
+
return adapter instanceof AbstractPostgresAdapter && db.isTransaction;
|
|
1076
|
+
}
|
|
1077
|
+
async function lockAllDocumentRefs() {
|
|
1078
|
+
if (!refLocksApply()) return;
|
|
1079
|
+
await sql`select pg_advisory_xact_lock(hashtextextended(${allDocumentRefsLockKey}, 0))`.execute(db);
|
|
1080
|
+
exclusiveRefLockHolders.add(db);
|
|
1081
|
+
}
|
|
1082
|
+
// Bits alone cannot tell a SERVE_ONLY ref from a pull route's, so candidates are matched
|
|
1083
|
+
// against the catalogs this device serves.
|
|
1084
|
+
async function refsNeedBytes(rows, servedCatalogs = readServedCatalogs) {
|
|
1085
|
+
const candidates = rows.filter(mayNeedBytes);
|
|
1086
|
+
if (candidates.some((row)=>isCustodyPinRef(row.ref))) return true;
|
|
1087
|
+
if (candidates.length === 0) return false;
|
|
1088
|
+
const served = await servedCatalogs();
|
|
1089
|
+
return served.some((catalog)=>candidates.some((row)=>isCatalogRuleRef(row.ref, catalog)));
|
|
1090
|
+
}
|
|
1091
|
+
async function readServedCatalogs() {
|
|
1092
|
+
const served = await db.selectFrom('kubun_graph_catalog_use').select([
|
|
1093
|
+
'owner_did',
|
|
1094
|
+
'catalog_id'
|
|
1095
|
+
]).where('level', '=', 'SERVE_ONLY').execute();
|
|
1096
|
+
return served.map((use)=>({
|
|
1097
|
+
owner: use.owner_did,
|
|
1098
|
+
id: use.catalog_id
|
|
1099
|
+
}));
|
|
1100
|
+
}
|
|
1101
|
+
async function readCatalogUses(catalog) {
|
|
1102
|
+
let query = db.selectFrom('kubun_graph_catalog_use').selectAll();
|
|
1103
|
+
if (catalog != null) query = query.where('owner_did', '=', catalog.owner).where('catalog_id', '=', catalog.id);
|
|
1104
|
+
return (await query.execute()).map((use)=>({
|
|
1105
|
+
catalog: {
|
|
1106
|
+
owner: use.owner_did,
|
|
1107
|
+
id: use.catalog_id
|
|
1108
|
+
},
|
|
1109
|
+
level: use.level,
|
|
1110
|
+
rules: typeof use.rules === 'string' ? JSON.parse(use.rules) : use.rules
|
|
1111
|
+
}));
|
|
1112
|
+
}
|
|
1113
|
+
// The reads of a single-document recompute, each as it is asked for.
|
|
1114
|
+
const directRecomputeInputs = (nowMS, catalog)=>({
|
|
1115
|
+
document: async (documentID)=>await api.getDocument(documentID),
|
|
1116
|
+
priorRefs: async (documentID)=>await db.selectFrom('kubun_graph_document_refs').select([
|
|
1117
|
+
'ref',
|
|
1118
|
+
'displays',
|
|
1119
|
+
'retains'
|
|
1120
|
+
]).where('document_id', '=', documentID).execute(),
|
|
1121
|
+
liveHoldRefs: async (documentID)=>new Set((await api.listDocumentHolds(documentID, nowMS)).map((hold)=>`hold:${hold.id}`)),
|
|
1122
|
+
interfaces: async (modelID)=>await api.getModelInterfaces(modelID),
|
|
1123
|
+
guardedRules: catalogRulesByRef,
|
|
1124
|
+
mayRead: scopeReaderMayRead,
|
|
1125
|
+
catalogs: async ()=>await readCatalogUses(catalog),
|
|
1126
|
+
servedCatalogs: readServedCatalogs,
|
|
1127
|
+
present: false
|
|
1128
|
+
});
|
|
1129
|
+
// Documents read by model, absent ones left out; an undeployed model throws as `getDocument`.
|
|
1130
|
+
async function readDocumentsByID(documentIDs) {
|
|
1131
|
+
const byModel = new Map();
|
|
1132
|
+
for (const documentID of documentIDs){
|
|
1133
|
+
const modelID = DocumentID.fromString(documentID).model.toString();
|
|
1134
|
+
const ids = byModel.get(modelID);
|
|
1135
|
+
if (ids == null) byModel.set(modelID, [
|
|
1136
|
+
documentID
|
|
1137
|
+
]);
|
|
1138
|
+
else ids.push(documentID);
|
|
1139
|
+
}
|
|
1140
|
+
const found = new Map();
|
|
1141
|
+
for (const [modelID, ids] of byModel){
|
|
1142
|
+
const rows = await db.selectFrom(`k_${modelID}`).selectAll().where('id', 'in', ids).execute().catch((err)=>{
|
|
1143
|
+
if (isMissingTableError(err, `k_${modelID}`)) throw new ModelNotDeployedError(modelID, {
|
|
1144
|
+
cause: err
|
|
1145
|
+
});
|
|
1146
|
+
throw err;
|
|
1147
|
+
});
|
|
1148
|
+
for (const row of rows){
|
|
1149
|
+
const node = documentNodeFromTable(adapter, row);
|
|
1150
|
+
found.set(node.id, node);
|
|
1151
|
+
}
|
|
1152
|
+
}
|
|
1153
|
+
return found;
|
|
1154
|
+
}
|
|
1155
|
+
/**
|
|
1156
|
+
* A bulk writer's recompute: each document as `recomputeDocumentRefs` rewrites it, in the given
|
|
1157
|
+
* order, under the caller's ref lock. A document's recompute writes only its own refs, mark,
|
|
1158
|
+
* scope change and byte jobs, none of which another document's recompute reads, so the rest
|
|
1159
|
+
* is read once per chunk of documents or once per pass.
|
|
1160
|
+
*/ const recomputeDocumentRefsBatch = async (params)=>{
|
|
1161
|
+
const { documentIDs, routeRules, nowMS, catalog, releaseCustodyPins: release } = params;
|
|
1162
|
+
if (documentIDs.length === 0) return;
|
|
1163
|
+
if (context.provider === context.root) {
|
|
1164
|
+
// Outside a transaction each document's recompute commits on its own.
|
|
1165
|
+
for (const documentID of documentIDs)await api.recomputeDocumentRefs({
|
|
1166
|
+
documentID,
|
|
1167
|
+
routeRules,
|
|
1168
|
+
nowMS,
|
|
1169
|
+
catalog,
|
|
1170
|
+
releaseCustodyPins: release
|
|
1171
|
+
});
|
|
1172
|
+
return;
|
|
1173
|
+
}
|
|
1174
|
+
await lockDocumentRefs(documentIDs);
|
|
1175
|
+
const effectiveRouteRules = routeRules.length === 0 ? await context.getCatalogRouteRules(nowMS, catalog) : routeRules;
|
|
1176
|
+
const interfaces = memoisedInterfaces();
|
|
1177
|
+
const mayRead = memoisedReadCheck();
|
|
1178
|
+
let catalogs;
|
|
1179
|
+
let served;
|
|
1180
|
+
const guardRules = new Map();
|
|
1181
|
+
for (const chunk of chunked(documentIDs)){
|
|
1182
|
+
const documents = await readDocumentsByID(chunk);
|
|
1183
|
+
const prior = new Map();
|
|
1184
|
+
for (const row of (await db.selectFrom('kubun_graph_document_refs').select([
|
|
1185
|
+
'document_id',
|
|
1186
|
+
'ref',
|
|
1187
|
+
'displays',
|
|
1188
|
+
'retains'
|
|
1189
|
+
]).where('document_id', 'in', chunk).execute())){
|
|
1190
|
+
const rows = prior.get(row.document_id) ?? [];
|
|
1191
|
+
rows.push({
|
|
1192
|
+
ref: row.ref,
|
|
1193
|
+
displays: row.displays,
|
|
1194
|
+
retains: row.retains
|
|
1195
|
+
});
|
|
1196
|
+
prior.set(row.document_id, rows);
|
|
1197
|
+
}
|
|
1198
|
+
const holds = new Map();
|
|
1199
|
+
for (const row of (await db.selectFrom('kubun_graph_document_holds').select([
|
|
1200
|
+
'document_id',
|
|
1201
|
+
'hold_id'
|
|
1202
|
+
]).where('document_id', 'in', chunk).where('released_at', 'is', null).where((eb)=>eb.or([
|
|
1203
|
+
eb('expires_at', 'is', null),
|
|
1204
|
+
eb('expires_at', '>', nowMS)
|
|
1205
|
+
])).execute())){
|
|
1206
|
+
const refs = holds.get(row.document_id) ?? new Set();
|
|
1207
|
+
refs.add(`hold:${row.hold_id}`);
|
|
1208
|
+
holds.set(row.document_id, refs);
|
|
1209
|
+
}
|
|
1210
|
+
const inputs = {
|
|
1211
|
+
document: async (documentID)=>documents.get(documentID) ?? null,
|
|
1212
|
+
priorRefs: async (documentID)=>prior.get(documentID) ?? [],
|
|
1213
|
+
liveHoldRefs: async (documentID)=>holds.get(documentID) ?? new Set(),
|
|
1214
|
+
interfaces,
|
|
1215
|
+
guardedRules: async (wanted)=>{
|
|
1216
|
+
const missing = wanted.filter((catalog)=>!guardRules.has(JSON.stringify([
|
|
1217
|
+
catalog.owner,
|
|
1218
|
+
catalog.id
|
|
1219
|
+
])));
|
|
1220
|
+
if (missing.length > 0) {
|
|
1221
|
+
const found = await catalogRulesByRef(missing);
|
|
1222
|
+
for (const catalog of missing){
|
|
1223
|
+
const key = JSON.stringify([
|
|
1224
|
+
catalog.owner,
|
|
1225
|
+
catalog.id
|
|
1226
|
+
]);
|
|
1227
|
+
guardRules.set(key, found.get(key) ?? []);
|
|
1228
|
+
}
|
|
1229
|
+
}
|
|
1230
|
+
return guardRules;
|
|
1231
|
+
},
|
|
1232
|
+
mayRead,
|
|
1233
|
+
catalogs: async ()=>{
|
|
1234
|
+
catalogs ??= readCatalogUses(catalog);
|
|
1235
|
+
return await catalogs;
|
|
1236
|
+
},
|
|
1237
|
+
servedCatalogs: async ()=>{
|
|
1238
|
+
served ??= readServedCatalogs();
|
|
1239
|
+
return await served;
|
|
1240
|
+
},
|
|
1241
|
+
// Nothing in the pass removes a document.
|
|
1242
|
+
present: true
|
|
1243
|
+
};
|
|
1244
|
+
for (const documentID of chunk){
|
|
1245
|
+
const released = new Set(release != null && release.refs.length > 0 && await release.confirm() ? release.refs : []);
|
|
1246
|
+
await recomputeRefsWith({
|
|
1247
|
+
documentID,
|
|
1248
|
+
routeRules: effectiveRouteRules,
|
|
1249
|
+
nowMS,
|
|
1250
|
+
catalog,
|
|
1251
|
+
released
|
|
1252
|
+
}, inputs);
|
|
1253
|
+
}
|
|
1254
|
+
}
|
|
1255
|
+
};
|
|
1256
|
+
// Rewrites one document's refs and mark from what `inputs` reads: false when it is absent.
|
|
1257
|
+
const recomputeRefsWith = async (step, inputs)=>{
|
|
1258
|
+
const { documentID, routeRules, nowMS, catalog, released } = step;
|
|
1259
|
+
const doc = await inputs.document(documentID);
|
|
1260
|
+
if (doc == null) return false;
|
|
1261
|
+
const prior = await inputs.priorRefs(documentID);
|
|
1262
|
+
const liveHoldRefs = await inputs.liveHoldRefs(documentID);
|
|
1263
|
+
const refs = new Map();
|
|
1264
|
+
for (const row of prior){
|
|
1265
|
+
if (row.ref === 'local' || // Custody pins are sticky: scope exit, read loss and a lost assignment keep them. Null
|
|
1266
|
+
// data is only ever a tombstone winning, which releases them.
|
|
1267
|
+
isCustodyPinRef(row.ref) && doc.data != null && !released.has(row.ref) || // A hold with no row (never written, or released) has nothing keeping the ref alive.
|
|
1268
|
+
liveHoldRefs.has(row.ref) || catalog != null && !row.ref.startsWith('hold:') && !isCatalogRuleRef(row.ref, catalog)) refs.set(row.ref, {
|
|
1269
|
+
displays: Number(row.displays),
|
|
1270
|
+
retains: Number(row.retains)
|
|
1271
|
+
});
|
|
1272
|
+
}
|
|
1273
|
+
const interfaces = await inputs.interfaces(doc.model);
|
|
1274
|
+
// Computed before the read check: a custodian keeps what it guards whatever it may read.
|
|
1275
|
+
const guards = doc.data == null ? [] : await context.custody?.guards() ?? [];
|
|
1276
|
+
const guardedRules = await inputs.guardedRules(guards.filter((guard)=>!refs.has(custodyPinRef(guard))).map((guard)=>guard.catalog));
|
|
1277
|
+
for (const guard of guards){
|
|
1278
|
+
const ref = custodyPinRef(guard);
|
|
1279
|
+
if (refs.has(ref)) continue;
|
|
1280
|
+
// Any rule of the guarded catalog, whatever level this device keeps it at.
|
|
1281
|
+
const rules = guardedRules.get(JSON.stringify([
|
|
1282
|
+
guard.catalog.owner,
|
|
1283
|
+
guard.catalog.id
|
|
1284
|
+
])) ?? [];
|
|
1285
|
+
if (!rules.some((rule)=>scopeRuleMatches(rule, doc, nowMS, interfaces))) continue;
|
|
1286
|
+
if (!await context.custody?.keepsOwner(guard.custodyOwner, doc.owner)) continue;
|
|
1287
|
+
refs.set(ref, {
|
|
1288
|
+
displays: 0,
|
|
1289
|
+
retains: 1
|
|
1290
|
+
});
|
|
1291
|
+
}
|
|
1292
|
+
const finish = async ()=>{
|
|
1293
|
+
await replaceDocumentRefs({
|
|
1294
|
+
documentID,
|
|
1295
|
+
prior,
|
|
1296
|
+
refs,
|
|
1297
|
+
catalog,
|
|
1298
|
+
nowMS,
|
|
1299
|
+
present: inputs.present
|
|
1300
|
+
});
|
|
1301
|
+
// In this transaction, so kept bytes are queued, or released ones dropped, before the
|
|
1302
|
+
// document can change again. Prior refs are read without their catalog's current level:
|
|
1303
|
+
// a level change commits before the recompute that drops the catalog's refs.
|
|
1304
|
+
if (doc.data != null && await refsNeedBytes(Array.from(refs, ([ref, bits])=>({
|
|
1305
|
+
ref,
|
|
1306
|
+
...bits
|
|
1307
|
+
})), inputs.servedCatalogs)) await context.documentBytes.needed(documentID);
|
|
1308
|
+
else if (prior.some(mayNeedBytes)) await context.documentBytes.released(documentID);
|
|
1309
|
+
return true;
|
|
1310
|
+
};
|
|
1311
|
+
const data = doc.data;
|
|
1312
|
+
if (!await inputs.mayRead(doc.owner, doc.model, data?.accessPermissions?.read ?? null)) return await finish();
|
|
1313
|
+
for (const use of (await inputs.catalogs())){
|
|
1314
|
+
if (use.level == null) continue;
|
|
1315
|
+
const bits = catalogLevelBits[use.level];
|
|
1316
|
+
for (const rule of use.rules){
|
|
1317
|
+
if (!scopeRuleMatches(rule, doc, nowMS, interfaces)) continue;
|
|
1318
|
+
refs.set(catalogRuleRef(use.catalog, rule.id), bits);
|
|
1319
|
+
}
|
|
1320
|
+
}
|
|
1321
|
+
for (const route of routeRules){
|
|
1322
|
+
if (catalog != null && (route.catalog.owner !== catalog.owner || route.catalog.id !== catalog.id)) continue;
|
|
1323
|
+
for (const rule of route.rules){
|
|
1324
|
+
if (!scopeRuleMatches(rule, doc, nowMS, interfaces)) continue;
|
|
1325
|
+
const ref = catalogRuleRef(route.catalog, rule.id);
|
|
1326
|
+
refs.set(ref, {
|
|
1327
|
+
displays: refs.get(ref)?.displays ?? 0,
|
|
1328
|
+
retains: 1
|
|
1329
|
+
});
|
|
1330
|
+
}
|
|
1331
|
+
}
|
|
1332
|
+
return await finish();
|
|
1333
|
+
};
|
|
1334
|
+
async function lockDocumentRefs(documentIDs) {
|
|
1335
|
+
if (!refLocksApply()) return;
|
|
1336
|
+
if (exclusiveRefLockHolders.has(db)) {
|
|
1337
|
+
// Taken again rather than trusted, as a rolled-back savepoint releases its locks. A held
|
|
1338
|
+
// lock is taken again without another lock table entry.
|
|
1339
|
+
await lockAllDocumentRefs();
|
|
1340
|
+
return;
|
|
1341
|
+
}
|
|
1342
|
+
await sql`select pg_advisory_xact_lock_shared(hashtextextended(${allDocumentRefsLockKey}, 0))`.execute(db);
|
|
1343
|
+
for (const documentID of [
|
|
1344
|
+
...new Set(documentIDs)
|
|
1345
|
+
].sort()){
|
|
1346
|
+
await sql`select pg_advisory_xact_lock(hashtextextended(${`graph.document.refs:${documentID}`}, 0))`.execute(db);
|
|
1347
|
+
}
|
|
1348
|
+
}
|
|
1349
|
+
async function mutationDigest(documentID) {
|
|
1350
|
+
const rows = await db.selectFrom('kubun_graph_mutation_log').select('mutation_hash').where('document_id', '=', documentID).where('status', '=', 'applied').execute();
|
|
1351
|
+
return mutationIDsDigest(rows.map((row)=>row.mutation_hash));
|
|
1352
|
+
}
|
|
1353
|
+
// Runs in the ref writer's locked transaction, after its write, so a mark never disagrees with
|
|
1354
|
+
// the refs a concurrent reader can see. A mark restarts only when the document's mutations
|
|
1355
|
+
// changed, so grace time is kept across repeated recomputes.
|
|
1356
|
+
async function syncGCMark(documentID, nowMS, present = false) {
|
|
1357
|
+
const retaining = await db.selectFrom('kubun_graph_document_refs').select('ref').where('document_id', '=', documentID).where('retains', '>', 0).limit(1).executeTakeFirst();
|
|
1358
|
+
if (retaining != null || !present && await api.getDocument(documentID) == null) {
|
|
1359
|
+
await db.deleteFrom('kubun_graph_gc_marks').where('document_id', '=', documentID).execute();
|
|
1360
|
+
return;
|
|
1361
|
+
}
|
|
1362
|
+
const digest = await mutationDigest(documentID);
|
|
1363
|
+
await db.insertInto('kubun_graph_gc_marks').values({
|
|
1364
|
+
document_id: documentID,
|
|
1365
|
+
digest,
|
|
1366
|
+
marked_at: nowMS
|
|
1367
|
+
}).onConflict((oc)=>oc.column('document_id').doUpdateSet({
|
|
1368
|
+
digest,
|
|
1369
|
+
marked_at: nowMS
|
|
1370
|
+
}).where('kubun_graph_gc_marks.digest', '!=', digest)).execute();
|
|
1371
|
+
}
|
|
1372
|
+
async function replaceDocumentRefs(params) {
|
|
1373
|
+
await writeDocumentRefs(params);
|
|
1374
|
+
await syncGCMark(params.documentID, params.nowMS, params.present);
|
|
1375
|
+
}
|
|
1376
|
+
async function writeDocumentRefs(params) {
|
|
1377
|
+
const { documentID, prior, refs, catalog } = params;
|
|
1378
|
+
// Hold refs are managed on every path so a stale one is removed even in a catalog-scoped pass.
|
|
1379
|
+
const managed = (ref)=>catalog == null || isCatalogRuleRef(ref, catalog) || ref.startsWith('hold:');
|
|
1380
|
+
const previous = prior.filter((row)=>managed(row.ref));
|
|
1381
|
+
const selected = new Map([
|
|
1382
|
+
...refs
|
|
1383
|
+
].filter(([ref])=>managed(ref)));
|
|
1384
|
+
if (previous.length === selected.size && previous.every((row)=>{
|
|
1385
|
+
const bits = selected.get(row.ref);
|
|
1386
|
+
return bits != null && bits.displays === Number(row.displays) && bits.retains === Number(row.retains);
|
|
1387
|
+
})) return;
|
|
1388
|
+
const removal = db.deleteFrom('kubun_graph_document_refs').where('document_id', '=', documentID);
|
|
1389
|
+
if (catalog == null) await removal.execute();
|
|
1390
|
+
else if (previous.length > 0) await removal.where('ref', 'in', previous.map((row)=>row.ref)).execute();
|
|
1391
|
+
if (selected.size > 0) {
|
|
1392
|
+
await db.insertInto('kubun_graph_document_refs').values(Array.from(selected, ([ref, bits])=>({
|
|
1393
|
+
document_id: documentID,
|
|
1394
|
+
ref,
|
|
1395
|
+
displays: bits.displays,
|
|
1396
|
+
retains: bits.retains
|
|
1397
|
+
}))).execute();
|
|
1398
|
+
}
|
|
1399
|
+
await recordScopeChange(documentID);
|
|
1400
|
+
}
|
|
1401
|
+
const pendingLifetimeMS = 24 * 60 * 60 * 1000;
|
|
1402
|
+
async function expirePushedMutations(now, groupID) {
|
|
1403
|
+
// Guard deletion and parking must share the same lock, including an upsert racing deletion.
|
|
1404
|
+
let guards = db.selectFrom('kubun_graph_pending_groups').select('group_id');
|
|
1405
|
+
if (groupID != null) guards = guards.where('group_id', '=', groupID);
|
|
1406
|
+
const groups = await (adapter instanceof AbstractPostgresAdapter ? guards.orderBy('group_id').forUpdate() : guards).execute();
|
|
1407
|
+
let query = db.deleteFrom('kubun_graph_mutation_log').where('status', '=', 'pending').where('arrival_group_id', 'is not', null).where('parked_at', '<=', now - pendingLifetimeMS);
|
|
1408
|
+
if (groupID != null) query = query.where('arrival_group_id', '=', groupID);
|
|
1409
|
+
await query.execute();
|
|
1410
|
+
// A parking transaction still needs its guard until it has inserted the new row.
|
|
1411
|
+
if (groupID != null || groups.length === 0) return;
|
|
1412
|
+
await db.deleteFrom('kubun_graph_pending_groups').where('group_id', 'in', groups.map((group)=>group.group_id)).where((eb)=>eb.not(eb.exists(eb.selectFrom('kubun_graph_mutation_log').select('mutation_hash').where('status', '=', 'pending').whereRef('arrival_group_id', '=', 'kubun_graph_pending_groups.group_id')))).execute();
|
|
1413
|
+
}
|
|
397
1414
|
const api = {
|
|
1415
|
+
...createGraphAccessDefaultRecordMethods(db, adapter),
|
|
1416
|
+
...createGraphCatalogRecordMethods(db, adapter, {
|
|
1417
|
+
onClose: context.onClose,
|
|
1418
|
+
// A custody change can complete or stop a pin handoff.
|
|
1419
|
+
onCustodyRecordStored: ()=>context.provider.onCommit(()=>context.custodyRecordsChanged?.())
|
|
1420
|
+
}),
|
|
1421
|
+
...createCatalogStateMethods(db),
|
|
398
1422
|
// --- Document operations ---
|
|
399
1423
|
async getDocument (id) {
|
|
400
1424
|
const parsedID = DocumentID.from(id);
|
|
@@ -436,7 +1460,11 @@ export function createGraphStore(db, adapter) {
|
|
|
436
1460
|
// duplicate id does not depend on which query returned first.
|
|
437
1461
|
const rowsByModel = await Promise.all(Array.from(idsByModel, async ([modelID, documentIDs])=>{
|
|
438
1462
|
try {
|
|
439
|
-
|
|
1463
|
+
const rows = [];
|
|
1464
|
+
for (const chunk of chunked(documentIDs)){
|
|
1465
|
+
rows.push(...await db.selectFrom(`k_${modelID}`).selectAll().where('id', 'in', chunk).execute());
|
|
1466
|
+
}
|
|
1467
|
+
return rows;
|
|
440
1468
|
} catch (err) {
|
|
441
1469
|
// A batch spans models, so one undeployed model contributes nothing
|
|
442
1470
|
// rather than failing the read for every other id asked for. The
|
|
@@ -462,6 +1490,27 @@ export function createGraphStore(db, adapter) {
|
|
|
462
1490
|
const rows = await db.selectFrom('kubun_graph_document_model_interfaces').select('interface_id').where('implementation_id', '=', modelID).execute();
|
|
463
1491
|
return rows.map((row)=>row.interface_id);
|
|
464
1492
|
},
|
|
1493
|
+
async listModelInterfaces () {
|
|
1494
|
+
const links = await db.selectFrom('kubun_graph_document_model_interfaces').select([
|
|
1495
|
+
'implementation_id',
|
|
1496
|
+
'interface_id'
|
|
1497
|
+
]).execute();
|
|
1498
|
+
const interfaces = new Map();
|
|
1499
|
+
for (const link of links){
|
|
1500
|
+
const list = interfaces.get(link.implementation_id) ?? [];
|
|
1501
|
+
list.push(link.interface_id);
|
|
1502
|
+
interfaces.set(link.implementation_id, list);
|
|
1503
|
+
}
|
|
1504
|
+
return interfaces;
|
|
1505
|
+
},
|
|
1506
|
+
async getDocumentModelsByID (ids) {
|
|
1507
|
+
const models = new Map();
|
|
1508
|
+
for (const chunk of chunked(new Set(ids))){
|
|
1509
|
+
const rows = await db.selectFrom('kubun_graph_document_models').selectAll().where('id', 'in', chunk).execute();
|
|
1510
|
+
for (const row of rows)models.set(row.id, documentModelFromTable(row));
|
|
1511
|
+
}
|
|
1512
|
+
return models;
|
|
1513
|
+
},
|
|
465
1514
|
async createDocument (params) {
|
|
466
1515
|
const id = params.id.toString();
|
|
467
1516
|
const model = params.id.model.toString();
|
|
@@ -474,10 +1523,23 @@ export function createGraphStore(db, adapter) {
|
|
|
474
1523
|
data: params.data ? adapter.encodeJSON(params.data) : null,
|
|
475
1524
|
unique: adapter.encodeBinary(params.unique)
|
|
476
1525
|
}).returningAll().executeTakeFirstOrThrow(()=>new Error(`createDocument failed to return row for ${id}`));
|
|
1526
|
+
// Re-acquired, by an answer or a pushed version: nothing left to restore.
|
|
1527
|
+
await db.deleteFrom('kubun_graph_swept_documents').where('document_id', '=', id).execute();
|
|
477
1528
|
if (params.data) {
|
|
478
1529
|
await reindexDocument(model, id, params.data);
|
|
479
1530
|
}
|
|
480
|
-
|
|
1531
|
+
await refreshDocumentDates(id);
|
|
1532
|
+
if (params.authoredHere !== false) await api.markDocumentLocal(id);
|
|
1533
|
+
await api.recomputeDocumentRefs({
|
|
1534
|
+
documentID: id,
|
|
1535
|
+
routeRules: []
|
|
1536
|
+
});
|
|
1537
|
+
// markDocumentLocal already told the listeners for an authored create.
|
|
1538
|
+
if (params.authoredHere === false) await context.documentApplied({
|
|
1539
|
+
documentID: id,
|
|
1540
|
+
authored: false
|
|
1541
|
+
});
|
|
1542
|
+
return await api.getDocument(id) ?? documentNodeFromTable(adapter, doc);
|
|
481
1543
|
},
|
|
482
1544
|
async saveDocument (params) {
|
|
483
1545
|
const id = params.id.toString();
|
|
@@ -491,22 +1553,955 @@ export function createGraphStore(db, adapter) {
|
|
|
491
1553
|
} else {
|
|
492
1554
|
await unindexDocument(model, id);
|
|
493
1555
|
}
|
|
494
|
-
|
|
1556
|
+
await refreshDocumentDates(id);
|
|
1557
|
+
await api.recomputeDocumentRefs({
|
|
1558
|
+
documentID: id,
|
|
1559
|
+
routeRules: []
|
|
1560
|
+
});
|
|
1561
|
+
// Not authored: a local apply calls markDocumentLocal after this, which says so.
|
|
1562
|
+
await context.documentApplied({
|
|
1563
|
+
documentID: id,
|
|
1564
|
+
authored: false
|
|
1565
|
+
});
|
|
1566
|
+
return await api.getDocument(id) ?? documentNodeFromTable(adapter, doc);
|
|
1567
|
+
},
|
|
1568
|
+
async getDocumentDates (documentID) {
|
|
1569
|
+
const modelID = DocumentID.fromString(documentID).model.toString();
|
|
1570
|
+
const row = await db.selectFrom(`k_${modelID}`).select([
|
|
1571
|
+
'mutation_created_at',
|
|
1572
|
+
'mutation_updated_at',
|
|
1573
|
+
'scope_date'
|
|
1574
|
+
]).where('id', '=', documentID).executeTakeFirst();
|
|
1575
|
+
if (row == null) return null;
|
|
1576
|
+
return {
|
|
1577
|
+
createdAt: row.mutation_created_at == null ? null : Number(row.mutation_created_at),
|
|
1578
|
+
updatedAt: row.mutation_updated_at == null ? null : Number(row.mutation_updated_at),
|
|
1579
|
+
scopeDate: row.scope_date == null ? null : Number(row.scope_date)
|
|
1580
|
+
};
|
|
1581
|
+
},
|
|
1582
|
+
async setCatalogDefinition (ref, catalog) {
|
|
1583
|
+
validateScopeCatalog(catalog);
|
|
1584
|
+
const dateModelIDs = new Set();
|
|
1585
|
+
for (const rule of catalog.rules){
|
|
1586
|
+
if (rule.dateField == null) continue;
|
|
1587
|
+
for (const modelID of rule.models ?? [])dateModelIDs.add(modelID);
|
|
1588
|
+
}
|
|
1589
|
+
const dateModels = new Map();
|
|
1590
|
+
for (const chunk of chunked(dateModelIDs)){
|
|
1591
|
+
const rows = await db.selectFrom('kubun_graph_document_models').select([
|
|
1592
|
+
'id',
|
|
1593
|
+
'schema'
|
|
1594
|
+
]).where('id', 'in', chunk).execute();
|
|
1595
|
+
for (const row of rows)dateModels.set(row.id, row);
|
|
1596
|
+
}
|
|
1597
|
+
for (const rule of catalog.rules){
|
|
1598
|
+
if (rule.dateField == null) continue;
|
|
1599
|
+
for (const modelID of rule.models ?? []){
|
|
1600
|
+
const model = dateModels.get(modelID);
|
|
1601
|
+
if (model == null || !isDocumentDateField(model, rule.dateField)) {
|
|
1602
|
+
throw new InvalidModelDateFieldError(`Invalid date-field override: ${rule.dateField}`);
|
|
1603
|
+
}
|
|
1604
|
+
}
|
|
1605
|
+
}
|
|
1606
|
+
const previous = await db.selectFrom('kubun_graph_catalog_use').select([
|
|
1607
|
+
'rules',
|
|
1608
|
+
'level'
|
|
1609
|
+
]).where('owner_did', '=', ref.owner).where('catalog_id', '=', ref.id).executeTakeFirst();
|
|
1610
|
+
const previousRules = previous == null ? [] : typeof previous.rules === 'string' ? JSON.parse(previous.rules) : previous.rules;
|
|
1611
|
+
const affectedTasks = previous?.level != null ? await tasksForCatalog(ref, [
|
|
1612
|
+
...previousRules,
|
|
1613
|
+
...catalog.rules
|
|
1614
|
+
]) : [];
|
|
1615
|
+
await db.insertInto('kubun_graph_catalog_use').values({
|
|
1616
|
+
owner_did: ref.owner,
|
|
1617
|
+
catalog_id: ref.id,
|
|
1618
|
+
name: catalog.name,
|
|
1619
|
+
rules: adapter.encodeJSON(catalog.rules),
|
|
1620
|
+
level: null,
|
|
1621
|
+
followed: 0
|
|
1622
|
+
}).onConflict((oc)=>oc.columns([
|
|
1623
|
+
'owner_did',
|
|
1624
|
+
'catalog_id'
|
|
1625
|
+
]).doUpdateSet({
|
|
1626
|
+
name: catalog.name,
|
|
1627
|
+
rules: adapter.encodeJSON(catalog.rules)
|
|
1628
|
+
})).execute();
|
|
1629
|
+
scheduleScopeTasks(context, affectedTasks);
|
|
1630
|
+
},
|
|
1631
|
+
async setScopeReaderDID (did, defaultReadLevel = 'only_owner') {
|
|
1632
|
+
const current = await db.selectFrom('kubun_graph_scope_identity').select([
|
|
1633
|
+
'reader_did',
|
|
1634
|
+
'default_read_level'
|
|
1635
|
+
]).where('id', '=', 1).executeTakeFirst();
|
|
1636
|
+
if (current?.reader_did === did && current.default_read_level === defaultReadLevel) return;
|
|
1637
|
+
await db.insertInto('kubun_graph_scope_identity').values({
|
|
1638
|
+
id: 1,
|
|
1639
|
+
reader_did: did,
|
|
1640
|
+
default_read_level: defaultReadLevel
|
|
1641
|
+
}).onConflict((oc)=>oc.column('id').doUpdateSet({
|
|
1642
|
+
reader_did: did,
|
|
1643
|
+
default_read_level: defaultReadLevel
|
|
1644
|
+
})).execute();
|
|
1645
|
+
const models = await db.selectFrom('kubun_graph_document_models').select('id').execute();
|
|
1646
|
+
scheduleScopeModels(context, models.map((model)=>model.id));
|
|
1647
|
+
},
|
|
1648
|
+
async setCatalogUse (ref, use) {
|
|
1649
|
+
const catalog = await db.selectFrom('kubun_graph_catalog_use').select([
|
|
1650
|
+
'rules',
|
|
1651
|
+
'level'
|
|
1652
|
+
]).where('owner_did', '=', ref.owner).where('catalog_id', '=', ref.id).executeTakeFirst();
|
|
1653
|
+
const rules = catalog == null ? [] : typeof catalog.rules === 'string' ? JSON.parse(catalog.rules) : catalog.rules;
|
|
1654
|
+
const affectedTasks = catalog != null && (catalog.level ?? null) !== use.level ? await tasksForCatalog(ref, rules) : [];
|
|
1655
|
+
await db.updateTable('kubun_graph_catalog_use').set({
|
|
1656
|
+
level: use.level,
|
|
1657
|
+
followed: Number(use.followed)
|
|
1658
|
+
}).where('owner_did', '=', ref.owner).where('catalog_id', '=', ref.id).execute();
|
|
1659
|
+
scheduleScopeTasks(context, affectedTasks);
|
|
1660
|
+
},
|
|
1661
|
+
async getCatalogUse (ref) {
|
|
1662
|
+
const row = await db.selectFrom('kubun_graph_catalog_use').select([
|
|
1663
|
+
'level',
|
|
1664
|
+
'followed'
|
|
1665
|
+
]).where('owner_did', '=', ref.owner).where('catalog_id', '=', ref.id).executeTakeFirst();
|
|
1666
|
+
return row == null ? null : catalogUseFromRow(row);
|
|
1667
|
+
},
|
|
1668
|
+
async getCatalogUses (refs) {
|
|
1669
|
+
if (refs.length === 0) return [];
|
|
1670
|
+
// The exact keys as one row-value list, as the batched record decisions read them.
|
|
1671
|
+
const unique = new Map(refs.map((ref)=>[
|
|
1672
|
+
JSON.stringify([
|
|
1673
|
+
ref.owner,
|
|
1674
|
+
ref.id
|
|
1675
|
+
]),
|
|
1676
|
+
ref
|
|
1677
|
+
]));
|
|
1678
|
+
const wanted = sql.join(Array.from(unique.values(), (ref)=>sql`(${ref.owner}, ${ref.id})`));
|
|
1679
|
+
const rows = await db.selectFrom('kubun_graph_catalog_use').select([
|
|
1680
|
+
'owner_did',
|
|
1681
|
+
'catalog_id',
|
|
1682
|
+
'level',
|
|
1683
|
+
'followed'
|
|
1684
|
+
]).where(sql`(${sql.ref('owner_did')}, ${sql.ref('catalog_id')}) in (values ${wanted})`).execute();
|
|
1685
|
+
const uses = new Map(rows.map((row)=>[
|
|
1686
|
+
JSON.stringify([
|
|
1687
|
+
row.owner_did,
|
|
1688
|
+
row.catalog_id
|
|
1689
|
+
]),
|
|
1690
|
+
catalogUseFromRow(row)
|
|
1691
|
+
]));
|
|
1692
|
+
return refs.map((ref)=>uses.get(JSON.stringify([
|
|
1693
|
+
ref.owner,
|
|
1694
|
+
ref.id
|
|
1695
|
+
])) ?? null);
|
|
1696
|
+
},
|
|
1697
|
+
async deleteCatalogUse (ref) {
|
|
1698
|
+
if (context.provider === context.root) {
|
|
1699
|
+
await context.root.withTransaction(async (tx)=>{
|
|
1700
|
+
await (await tx.getStore('graph')).deleteCatalogUse(ref);
|
|
1701
|
+
});
|
|
1702
|
+
return;
|
|
1703
|
+
}
|
|
1704
|
+
const existing = await db.selectFrom('kubun_graph_catalog_use').select([
|
|
1705
|
+
'rules',
|
|
1706
|
+
'level'
|
|
1707
|
+
]).where('owner_did', '=', ref.owner).where('catalog_id', '=', ref.id).executeTakeFirst();
|
|
1708
|
+
if (context.provider.hasStore('p2p')) {
|
|
1709
|
+
const p2p = await context.provider.getStore('p2p');
|
|
1710
|
+
await p2p.deleteCatalogRoutes(ref.owner, ref.id);
|
|
1711
|
+
await p2p.removeCatalogOffersForCatalog(ref.owner, ref.id);
|
|
1712
|
+
}
|
|
1713
|
+
if (existing === undefined) return;
|
|
1714
|
+
const rules = typeof existing.rules === 'string' ? JSON.parse(existing.rules) : existing.rules;
|
|
1715
|
+
const affectedTasks = existing.level != null ? await tasksForCatalog(ref, rules) : [];
|
|
1716
|
+
await db.deleteFrom('kubun_graph_catalog_use').where('owner_did', '=', ref.owner).where('catalog_id', '=', ref.id).execute();
|
|
1717
|
+
scheduleScopeTasks(context, affectedTasks);
|
|
1718
|
+
await api.reconcileCatalogRefs(Date.now(), [], ref);
|
|
1719
|
+
},
|
|
1720
|
+
async markDocumentLocal (documentID) {
|
|
1721
|
+
if (context.provider === context.root) {
|
|
1722
|
+
await context.root.withTransaction(async (tx)=>{
|
|
1723
|
+
await (await tx.getStore('graph')).markDocumentLocal(documentID);
|
|
1724
|
+
});
|
|
1725
|
+
return;
|
|
1726
|
+
}
|
|
1727
|
+
await lockDocumentRefs([
|
|
1728
|
+
documentID
|
|
1729
|
+
]);
|
|
1730
|
+
const inserted = await db.insertInto('kubun_graph_document_refs').values({
|
|
1731
|
+
document_id: documentID,
|
|
1732
|
+
ref: 'local',
|
|
1733
|
+
displays: 1,
|
|
1734
|
+
retains: 0
|
|
1735
|
+
}).onConflict((oc)=>oc.doNothing()).returning('ref').execute();
|
|
1736
|
+
if (inserted.length > 0) await recordScopeChange(documentID);
|
|
1737
|
+
await syncGCMark(documentID, Date.now());
|
|
1738
|
+
// Called once per apply authored here, after its data is final.
|
|
1739
|
+
await context.documentApplied({
|
|
1740
|
+
documentID,
|
|
1741
|
+
authored: true
|
|
1742
|
+
});
|
|
1743
|
+
},
|
|
1744
|
+
async placeDocumentHold (params) {
|
|
1745
|
+
if (context.provider === context.root) {
|
|
1746
|
+
return await context.root.withTransaction(async (tx)=>(await tx.getStore('graph')).placeDocumentHold(params));
|
|
1747
|
+
}
|
|
1748
|
+
const { holdID, documentID, createdBy, reason, expiresAt } = params;
|
|
1749
|
+
await lockDocumentRefs([
|
|
1750
|
+
documentID
|
|
1751
|
+
]);
|
|
1752
|
+
// A sweep deletes the row under this lock: one committed since the caller looked leaves
|
|
1753
|
+
// nothing to hold.
|
|
1754
|
+
if ((await api.getDocument(documentID))?.data == null) throw new DocumentNotHoldableError();
|
|
1755
|
+
await db.insertInto('kubun_graph_document_holds').values({
|
|
1756
|
+
hold_id: holdID,
|
|
1757
|
+
document_id: documentID,
|
|
1758
|
+
created_by: createdBy,
|
|
1759
|
+
reason: reason ?? null,
|
|
1760
|
+
expires_at: expiresAt ?? null,
|
|
1761
|
+
released_at: null
|
|
1762
|
+
}).execute();
|
|
1763
|
+
await db.insertInto('kubun_graph_document_refs').values({
|
|
1764
|
+
document_id: documentID,
|
|
1765
|
+
ref: `hold:${holdID}`,
|
|
1766
|
+
displays: 0,
|
|
1767
|
+
retains: 1
|
|
1768
|
+
}).onConflict((oc)=>oc.doNothing()).execute();
|
|
1769
|
+
await recordScopeChange(documentID);
|
|
1770
|
+
await syncGCMark(documentID, Date.now());
|
|
1771
|
+
return {
|
|
1772
|
+
id: holdID,
|
|
1773
|
+
documentID,
|
|
1774
|
+
createdBy,
|
|
1775
|
+
reason: reason ?? null,
|
|
1776
|
+
expiresAt: expiresAt ?? null
|
|
1777
|
+
};
|
|
1778
|
+
},
|
|
1779
|
+
async releaseDocumentHold (holdID, nowMS) {
|
|
1780
|
+
if (context.provider === context.root) {
|
|
1781
|
+
return await context.root.withTransaction(async (tx)=>(await tx.getStore('graph')).releaseDocumentHold(holdID, nowMS));
|
|
1782
|
+
}
|
|
1783
|
+
// A hold never changes document, so its document can be locked before the release decides.
|
|
1784
|
+
const target = await db.selectFrom('kubun_graph_document_holds').select('document_id').where('hold_id', '=', holdID).executeTakeFirst();
|
|
1785
|
+
if (target == null) return null;
|
|
1786
|
+
await lockDocumentRefs([
|
|
1787
|
+
target.document_id
|
|
1788
|
+
]);
|
|
1789
|
+
const released = await db.updateTable('kubun_graph_document_holds').set({
|
|
1790
|
+
released_at: nowMS
|
|
1791
|
+
}).where('hold_id', '=', holdID).where('released_at', 'is', null).returningAll().executeTakeFirst();
|
|
1792
|
+
if (released == null) return null;
|
|
1793
|
+
await db.deleteFrom('kubun_graph_document_refs').where('document_id', '=', released.document_id).where('ref', '=', `hold:${holdID}`).execute();
|
|
1794
|
+
await recordScopeChange(released.document_id);
|
|
1795
|
+
await syncGCMark(released.document_id, nowMS);
|
|
1796
|
+
return holdFromRow(released);
|
|
1797
|
+
},
|
|
1798
|
+
documentMutationDigest: mutationDigest,
|
|
1799
|
+
async hasAppliedMutations (documentID, mutationIDs) {
|
|
1800
|
+
for (const chunk of chunked(mutationIDs)){
|
|
1801
|
+
const row = await db.selectFrom('kubun_graph_mutation_log').select((eb)=>eb.fn.countAll().as('count')).where('document_id', '=', documentID).where('status', '=', 'applied').where('mutation_hash', 'in', chunk).executeTakeFirst();
|
|
1802
|
+
if (Number(row?.count ?? 0) !== chunk.length) return false;
|
|
1803
|
+
}
|
|
1804
|
+
return true;
|
|
1805
|
+
},
|
|
1806
|
+
async putCustodyNonce (nonce) {
|
|
1807
|
+
await db.deleteFrom('kubun_graph_custody_nonces').where('expires_at', '<=', Date.now()).execute();
|
|
1808
|
+
await db.insertInto('kubun_graph_custody_nonces').values({
|
|
1809
|
+
nonce: nonce.nonce,
|
|
1810
|
+
custodian: nonce.custodian,
|
|
1811
|
+
assignment_digest: nonce.assignmentDigest,
|
|
1812
|
+
expires_at: nonce.expiresAt
|
|
1813
|
+
}).execute();
|
|
1814
|
+
},
|
|
1815
|
+
async consumeCustodyNonce (params) {
|
|
1816
|
+
// One statement, so two concurrent consumers cannot both see the row.
|
|
1817
|
+
const result = await db.deleteFrom('kubun_graph_custody_nonces').where('nonce', '=', params.nonce).where('custodian', '=', params.custodian).where('assignment_digest', '=', params.assignmentDigest).where('expires_at', '>', params.nowMS).executeTakeFirst();
|
|
1818
|
+
return Number(result.numDeletedRows) === 1;
|
|
1819
|
+
},
|
|
1820
|
+
async putCustodyReceipts (receipts) {
|
|
1821
|
+
for (const receipt of receipts){
|
|
1822
|
+
const values = {
|
|
1823
|
+
group_id: receipt.groupID,
|
|
1824
|
+
custody_owner: receipt.custodyOwner,
|
|
1825
|
+
catalog_owner: receipt.catalog.owner,
|
|
1826
|
+
catalog_id: receipt.catalog.id,
|
|
1827
|
+
assignment_digest: receipt.assignmentDigest,
|
|
1828
|
+
head_digest: receipt.headDigest,
|
|
1829
|
+
mutation_digest: receipt.mutationDigest,
|
|
1830
|
+
received_at: receipt.receivedAt
|
|
1831
|
+
};
|
|
1832
|
+
await db.insertInto('kubun_graph_custody_receipts').values({
|
|
1833
|
+
document_id: receipt.documentID,
|
|
1834
|
+
custodian: receipt.custodian,
|
|
1835
|
+
...values
|
|
1836
|
+
}).onConflict((oc)=>oc.columns([
|
|
1837
|
+
'document_id',
|
|
1838
|
+
'custodian'
|
|
1839
|
+
]).doUpdateSet(values)).execute();
|
|
1840
|
+
}
|
|
1841
|
+
},
|
|
1842
|
+
async listCustodyPinnedDocuments (params) {
|
|
1843
|
+
let query = db.selectFrom('kubun_graph_document_refs').select('document_id').where('ref', '=', params.ref);
|
|
1844
|
+
if (params.after != null) query = query.where('document_id', '>', params.after);
|
|
1845
|
+
const rows = await query.orderBy('document_id').limit(params.limit).execute();
|
|
1846
|
+
return rows.map((row)=>row.document_id);
|
|
1847
|
+
},
|
|
1848
|
+
async listCustodyReceipts (documentID) {
|
|
1849
|
+
const rows = await db.selectFrom('kubun_graph_custody_receipts').selectAll().where('document_id', '=', documentID).orderBy('custodian').execute();
|
|
1850
|
+
return rows.map(custodyReceiptFromRow);
|
|
1851
|
+
},
|
|
1852
|
+
async getGCMark (documentID) {
|
|
1853
|
+
const row = await db.selectFrom('kubun_graph_gc_marks').selectAll().where('document_id', '=', documentID).executeTakeFirst();
|
|
1854
|
+
return row == null ? null : gcMarkFromRow(row);
|
|
1855
|
+
},
|
|
1856
|
+
async dropGCMark (params) {
|
|
1857
|
+
await db.deleteFrom('kubun_graph_gc_marks').where('document_id', '=', params.documentID).where('digest', '=', params.digest).execute();
|
|
1858
|
+
},
|
|
1859
|
+
async listGCMarks (params) {
|
|
1860
|
+
let query = db.selectFrom('kubun_graph_gc_marks').selectAll();
|
|
1861
|
+
// Bytewise order and cursor, so paging agrees across adapters.
|
|
1862
|
+
const postgres = adapter instanceof AbstractPostgresAdapter;
|
|
1863
|
+
if (params.after != null) query = query.where(postgres ? sql`document_id collate "C" > ${params.after}` : sql`document_id > ${params.after}`);
|
|
1864
|
+
const rows = await (postgres ? query.orderBy(sql`document_id collate "C"`) : query.orderBy('document_id')).limit(params.limit).execute();
|
|
1865
|
+
return rows.map(gcMarkFromRow);
|
|
1866
|
+
},
|
|
1867
|
+
async getDocumentHold (holdID) {
|
|
1868
|
+
const row = await db.selectFrom('kubun_graph_document_holds').selectAll().where('hold_id', '=', holdID).executeTakeFirst();
|
|
1869
|
+
return row == null ? null : holdFromRow(row);
|
|
1870
|
+
},
|
|
1871
|
+
async listDocumentHolds (documentID, nowMS) {
|
|
1872
|
+
const rows = await db.selectFrom('kubun_graph_document_holds').selectAll().where('document_id', '=', documentID).where('released_at', 'is', null).where((eb)=>eb.or([
|
|
1873
|
+
eb('expires_at', 'is', null),
|
|
1874
|
+
eb('expires_at', '>', nowMS)
|
|
1875
|
+
])).orderBy('hold_id').execute();
|
|
1876
|
+
return rows.map(holdFromRow);
|
|
1877
|
+
},
|
|
1878
|
+
async nextDocumentHoldExpiry () {
|
|
1879
|
+
const tick = await db.selectFrom('kubun_graph_catalog_scope_tick').select('last_at').where('id', '=', 1).executeTakeFirst();
|
|
1880
|
+
let query = db.selectFrom('kubun_graph_document_holds').select((eb)=>eb.fn.min('expires_at').as('expires_at')).where('released_at', 'is', null).where('expires_at', 'is not', null);
|
|
1881
|
+
if (tick != null) query = query.where('expires_at', '>', Number(tick.last_at));
|
|
1882
|
+
const row = await query.executeTakeFirst();
|
|
1883
|
+
return row?.expires_at == null ? null : Number(row.expires_at);
|
|
1884
|
+
},
|
|
1885
|
+
async listPendingScopeChanges () {
|
|
1886
|
+
const rows = await db.selectFrom('kubun_graph_scope_changes').select([
|
|
1887
|
+
'document_id',
|
|
1888
|
+
'version'
|
|
1889
|
+
]).orderBy('document_id').execute();
|
|
1890
|
+
return rows.map((row)=>({
|
|
1891
|
+
documentID: row.document_id,
|
|
1892
|
+
version: row.version
|
|
1893
|
+
}));
|
|
1894
|
+
},
|
|
1895
|
+
async ackDocumentScopeChange (documentID, version) {
|
|
1896
|
+
await db.deleteFrom('kubun_graph_scope_changes').where('document_id', '=', documentID).where('version', '=', version).execute();
|
|
1897
|
+
},
|
|
1898
|
+
async getDocumentScope (documentID) {
|
|
1899
|
+
const rows = await db.selectFrom('kubun_graph_document_refs').select([
|
|
1900
|
+
'ref',
|
|
1901
|
+
'displays',
|
|
1902
|
+
'retains'
|
|
1903
|
+
]).where('document_id', '=', documentID).orderBy('ref').execute();
|
|
1904
|
+
const retained = rows.some((row)=>Number(row.retains) === 1);
|
|
1905
|
+
return {
|
|
1906
|
+
scope: rows.some((row)=>Number(row.displays) === 1) ? 'visible' : retained ? 'retained' : 'outOfScope',
|
|
1907
|
+
retained,
|
|
1908
|
+
refs: rows.map((row)=>row.ref)
|
|
1909
|
+
};
|
|
1910
|
+
},
|
|
1911
|
+
async getDocumentAvailability (documentID, nowMS) {
|
|
1912
|
+
const doc = await api.getDocument(documentID);
|
|
1913
|
+
if (doc != null) return doc.data == null ? null : 'LOCAL';
|
|
1914
|
+
const entry = await api.getSweptDocument(documentID);
|
|
1915
|
+
if (entry == null) return null;
|
|
1916
|
+
const catalogs = await sweptRetainingCatalogs({
|
|
1917
|
+
entry,
|
|
1918
|
+
nowMS,
|
|
1919
|
+
retaining: await retainingRules(nowMS)
|
|
1920
|
+
});
|
|
1921
|
+
return catalogs.length > 0 ? 'UNAVAILABLE' : null;
|
|
1922
|
+
},
|
|
1923
|
+
async nextSweptDocumentTransition (nowMS) {
|
|
1924
|
+
const rows = await db.selectFrom('kubun_graph_swept_documents').selectAll().execute();
|
|
1925
|
+
if (rows.length === 0) return null;
|
|
1926
|
+
const retaining = await retainingRules(nowMS);
|
|
1927
|
+
const windowed = [
|
|
1928
|
+
...retaining.uses.flatMap((use)=>use.rules),
|
|
1929
|
+
...retaining.routeRules.flatMap((route)=>route.rules)
|
|
1930
|
+
].filter((rule)=>rule.maxAge != null || rule.maxAhead != null);
|
|
1931
|
+
if (windowed.length === 0) return null;
|
|
1932
|
+
const interfaces = new Map();
|
|
1933
|
+
let next = null;
|
|
1934
|
+
for (const row of rows){
|
|
1935
|
+
const entry = sweptFromRow(row);
|
|
1936
|
+
let modelInterfaces = interfaces.get(entry.modelID);
|
|
1937
|
+
if (modelInterfaces == null) {
|
|
1938
|
+
modelInterfaces = await api.getModelInterfaces(entry.modelID);
|
|
1939
|
+
interfaces.set(entry.modelID, modelInterfaces);
|
|
1940
|
+
}
|
|
1941
|
+
for (const rule of windowed){
|
|
1942
|
+
// The rule without its window: only the window moves with time.
|
|
1943
|
+
const unbounded = {
|
|
1944
|
+
...rule,
|
|
1945
|
+
maxAge: undefined,
|
|
1946
|
+
maxAhead: undefined
|
|
1947
|
+
};
|
|
1948
|
+
const facts = {
|
|
1949
|
+
model: entry.modelID,
|
|
1950
|
+
owner: entry.owner,
|
|
1951
|
+
dateOf: ()=>0
|
|
1952
|
+
};
|
|
1953
|
+
if (!scopeRuleMatchesFacts(unbounded, facts, nowMS, modelInterfaces)) continue;
|
|
1954
|
+
const date = rule.dateField == null ? entry.scopeDate ?? entry.createdAt : entry.fieldDates[rule.dateField] ?? entry.createdAt;
|
|
1955
|
+
// `date < now + maxAhead` starts holding just after `date - maxAhead`;
|
|
1956
|
+
// `date >= now - maxAge` stops holding just after `date + maxAge`.
|
|
1957
|
+
const times = [
|
|
1958
|
+
rule.maxAhead == null ? null : date - parseScopeDuration(rule.maxAhead) + 1,
|
|
1959
|
+
rule.maxAge == null ? null : date + parseScopeDuration(rule.maxAge) + 1
|
|
1960
|
+
];
|
|
1961
|
+
for (const time of times)if (time != null && time > nowMS && (next == null || time < next)) next = time;
|
|
1962
|
+
}
|
|
1963
|
+
}
|
|
1964
|
+
return next;
|
|
1965
|
+
},
|
|
1966
|
+
async getSweptDocument (documentID) {
|
|
1967
|
+
const row = await db.selectFrom('kubun_graph_swept_documents').selectAll().where('document_id', '=', documentID).executeTakeFirst();
|
|
1968
|
+
return row == null ? null : sweptFromRow(row);
|
|
1969
|
+
},
|
|
1970
|
+
async listUnavailableDocuments (params) {
|
|
1971
|
+
const retaining = await retainingRules(params.nowMS);
|
|
1972
|
+
const checks = {
|
|
1973
|
+
mayRead: memoisedReadCheck(),
|
|
1974
|
+
interfaces: memoisedInterfaces()
|
|
1975
|
+
};
|
|
1976
|
+
const found = [];
|
|
1977
|
+
let after = params.after;
|
|
1978
|
+
for(;;){
|
|
1979
|
+
let query = db.selectFrom('kubun_graph_swept_documents').selectAll().orderBy('document_id').limit(SWEPT_PAGE);
|
|
1980
|
+
if (after != null) query = query.where('document_id', '>', after);
|
|
1981
|
+
const rows = await query.execute();
|
|
1982
|
+
for (const row of rows){
|
|
1983
|
+
const entry = sweptFromRow(row);
|
|
1984
|
+
const catalogs = await sweptRetainingCatalogs({
|
|
1985
|
+
entry,
|
|
1986
|
+
nowMS: params.nowMS,
|
|
1987
|
+
retaining,
|
|
1988
|
+
checks
|
|
1989
|
+
});
|
|
1990
|
+
if (catalogs.length === 0) continue;
|
|
1991
|
+
found.push({
|
|
1992
|
+
documentID: entry.documentID,
|
|
1993
|
+
owner: entry.owner,
|
|
1994
|
+
catalogs
|
|
1995
|
+
});
|
|
1996
|
+
if (found.length >= params.limit) return found;
|
|
1997
|
+
}
|
|
1998
|
+
if (rows.length < SWEPT_PAGE) return found;
|
|
1999
|
+
after = rows.at(-1)?.document_id;
|
|
2000
|
+
}
|
|
2001
|
+
},
|
|
2002
|
+
async refreshSweptDocuments (params) {
|
|
2003
|
+
if (context.provider === context.root) {
|
|
2004
|
+
await context.root.withTransaction(async (tx)=>{
|
|
2005
|
+
const graph = await tx.getStore('graph');
|
|
2006
|
+
await graph.refreshSweptDocuments(params);
|
|
2007
|
+
});
|
|
2008
|
+
return;
|
|
2009
|
+
}
|
|
2010
|
+
const { nowMS, ttlMS } = params;
|
|
2011
|
+
let query = db.selectFrom('kubun_graph_swept_documents').selectAll().orderBy('document_id');
|
|
2012
|
+
if (params.modelID != null) query = query.where('model_id', '=', params.modelID);
|
|
2013
|
+
const rows = await query.execute();
|
|
2014
|
+
if (rows.length === 0) return;
|
|
2015
|
+
const retaining = await retainingRules(nowMS);
|
|
2016
|
+
const checks = {
|
|
2017
|
+
mayRead: memoisedReadCheck(),
|
|
2018
|
+
interfaces: memoisedInterfaces()
|
|
2019
|
+
};
|
|
2020
|
+
const entered = [];
|
|
2021
|
+
const left = [];
|
|
2022
|
+
const expired = [];
|
|
2023
|
+
for (const row of rows){
|
|
2024
|
+
const matching = (await sweptRetainingCatalogs({
|
|
2025
|
+
entry: sweptFromRow(row),
|
|
2026
|
+
nowMS,
|
|
2027
|
+
retaining,
|
|
2028
|
+
checks
|
|
2029
|
+
})).length > 0;
|
|
2030
|
+
const wasMatching = Number(row.in_scope) === 1;
|
|
2031
|
+
if (matching !== wasMatching) (matching ? entered : left).push(row.document_id);
|
|
2032
|
+
else if (!matching && ttlMS != null && Number(row.scope_left_at) + ttlMS <= nowMS) expired.push(row.document_id);
|
|
2033
|
+
}
|
|
2034
|
+
for (const chunk of chunked(entered))await db.updateTable('kubun_graph_swept_documents').set({
|
|
2035
|
+
in_scope: 1
|
|
2036
|
+
}).where('document_id', 'in', chunk).execute();
|
|
2037
|
+
for (const chunk of chunked(left))await db.updateTable('kubun_graph_swept_documents').set({
|
|
2038
|
+
in_scope: 0,
|
|
2039
|
+
scope_left_at: nowMS
|
|
2040
|
+
}).where('document_id', 'in', chunk).execute();
|
|
2041
|
+
for (const chunk of chunked(expired))await db.deleteFrom('kubun_graph_swept_documents').where('document_id', 'in', chunk).execute();
|
|
2042
|
+
// Read back as an availability change by whoever emits scope events.
|
|
2043
|
+
await recordScopeChanges([
|
|
2044
|
+
...entered,
|
|
2045
|
+
...left
|
|
2046
|
+
]);
|
|
2047
|
+
if (entered.length + left.length > 0) context.provider.onCommit(()=>notifyScopeListeners(context));
|
|
2048
|
+
},
|
|
2049
|
+
async documentNeedsBytes (documentID) {
|
|
2050
|
+
const doc = await api.getDocument(documentID);
|
|
2051
|
+
if (doc?.data == null) return false;
|
|
2052
|
+
const rows = await db.selectFrom('kubun_graph_document_refs').select([
|
|
2053
|
+
'ref',
|
|
2054
|
+
'displays',
|
|
2055
|
+
'retains'
|
|
2056
|
+
]).where('document_id', '=', documentID).execute();
|
|
2057
|
+
return await refsNeedBytes(rows);
|
|
2058
|
+
},
|
|
2059
|
+
async countCatalogDocuments (ref) {
|
|
2060
|
+
const holders = await catalogRefHolders(ref);
|
|
2061
|
+
return new Set(holders.map((holder)=>holder.document_id)).size;
|
|
2062
|
+
},
|
|
2063
|
+
async recomputeDocumentRefs (params) {
|
|
2064
|
+
const { documentID, routeRules, nowMS = Date.now(), catalog } = params;
|
|
2065
|
+
if (context.provider === context.root) {
|
|
2066
|
+
return await context.root.withTransaction(async (tx)=>(await tx.getStore('graph')).recomputeDocumentRefs({
|
|
2067
|
+
...params,
|
|
2068
|
+
nowMS
|
|
2069
|
+
}));
|
|
2070
|
+
}
|
|
2071
|
+
await lockDocumentRefs([
|
|
2072
|
+
documentID
|
|
2073
|
+
]);
|
|
2074
|
+
const release = params.releaseCustodyPins;
|
|
2075
|
+
const released = new Set(release != null && release.refs.length > 0 && await release.confirm() ? release.refs : []);
|
|
2076
|
+
const effectiveRouteRules = routeRules.length === 0 ? await context.getCatalogRouteRules(nowMS, catalog) : routeRules;
|
|
2077
|
+
const present = await recomputeRefsWith({
|
|
2078
|
+
documentID,
|
|
2079
|
+
routeRules: effectiveRouteRules,
|
|
2080
|
+
nowMS,
|
|
2081
|
+
catalog,
|
|
2082
|
+
released
|
|
2083
|
+
}, directRecomputeInputs(nowMS, catalog));
|
|
2084
|
+
if (!present) return {
|
|
2085
|
+
scope: 'outOfScope',
|
|
2086
|
+
retained: false,
|
|
2087
|
+
refs: []
|
|
2088
|
+
};
|
|
2089
|
+
return await api.getDocumentScope(documentID);
|
|
2090
|
+
},
|
|
2091
|
+
async sweepDocument (params) {
|
|
2092
|
+
if (context.provider === context.root) {
|
|
2093
|
+
return await context.root.withTransaction(async (tx)=>(await tx.getStore('graph')).sweepDocument(params));
|
|
2094
|
+
}
|
|
2095
|
+
const { documentID, nowMS } = params;
|
|
2096
|
+
const modelID = DocumentID.fromString(documentID).model.toString();
|
|
2097
|
+
// The row first, as an apply takes it before the ref lock.
|
|
2098
|
+
if (!await api.lockDocumentRow(documentID)) return {
|
|
2099
|
+
swept: false,
|
|
2100
|
+
refusal: 'absent'
|
|
2101
|
+
};
|
|
2102
|
+
await lockDocumentRefs([
|
|
2103
|
+
documentID
|
|
2104
|
+
]);
|
|
2105
|
+
// Stored refs can be stale (an expired hold, a rule now matching): decide from fresh ones.
|
|
2106
|
+
const scope = await api.recomputeDocumentRefs({
|
|
2107
|
+
documentID,
|
|
2108
|
+
routeRules: [],
|
|
2109
|
+
nowMS
|
|
2110
|
+
});
|
|
2111
|
+
if (scope.retained) return {
|
|
2112
|
+
swept: false,
|
|
2113
|
+
refusal: 'retained'
|
|
2114
|
+
};
|
|
2115
|
+
if ((await api.listDocumentHolds(documentID, nowMS)).length > 0) return {
|
|
2116
|
+
swept: false,
|
|
2117
|
+
refusal: 'held'
|
|
2118
|
+
};
|
|
2119
|
+
const mark = await api.getGCMark(documentID);
|
|
2120
|
+
if (mark == null) return {
|
|
2121
|
+
swept: false,
|
|
2122
|
+
refusal: 'unmarked'
|
|
2123
|
+
};
|
|
2124
|
+
if (mark.digest !== await mutationDigest(documentID)) return {
|
|
2125
|
+
swept: false,
|
|
2126
|
+
refusal: 'changed'
|
|
2127
|
+
};
|
|
2128
|
+
if (!gcMarkPastGrace(mark, params.graceMS, nowMS)) return {
|
|
2129
|
+
swept: false,
|
|
2130
|
+
refusal: 'grace'
|
|
2131
|
+
};
|
|
2132
|
+
// Inbound history no receipt covered.
|
|
2133
|
+
const pending = await db.selectFrom('kubun_graph_mutation_log').select('mutation_hash').where('document_id', '=', documentID).where('status', '=', 'pending').limit(1).executeTakeFirst();
|
|
2134
|
+
if (pending != null) return {
|
|
2135
|
+
swept: false,
|
|
2136
|
+
refusal: 'pending'
|
|
2137
|
+
};
|
|
2138
|
+
if (!await params.confirm(context.provider)) return {
|
|
2139
|
+
swept: false,
|
|
2140
|
+
refusal: 'unconfirmed'
|
|
2141
|
+
};
|
|
2142
|
+
const swept = await api.getDocument(documentID);
|
|
2143
|
+
// A tombstone leaves nothing to restore.
|
|
2144
|
+
if (swept?.data != null) await writeSweptDocument(swept, mark.digest, nowMS);
|
|
2145
|
+
await unindexDocument(modelID, documentID);
|
|
2146
|
+
await db.deleteFrom(`k_${modelID}`).where('id', '=', documentID).execute();
|
|
2147
|
+
for (const table of [
|
|
2148
|
+
'kubun_graph_mutation_log',
|
|
2149
|
+
'kubun_graph_document_refs',
|
|
2150
|
+
'kubun_graph_document_scope_dates',
|
|
2151
|
+
'kubun_graph_gc_marks',
|
|
2152
|
+
'kubun_graph_custody_receipts',
|
|
2153
|
+
// No live hold is left: it would have retained the document.
|
|
2154
|
+
'kubun_graph_document_holds'
|
|
2155
|
+
]){
|
|
2156
|
+
await db.deleteFrom(table).where('document_id', '=', documentID).execute();
|
|
2157
|
+
}
|
|
2158
|
+
// Bumped, not removed: a copy still displayed leaves the view through the scope event.
|
|
2159
|
+
await recordScopeChange(documentID);
|
|
2160
|
+
context.provider.onCommit(()=>notifyScopeListeners(context));
|
|
2161
|
+
return {
|
|
2162
|
+
swept: true
|
|
2163
|
+
};
|
|
2164
|
+
},
|
|
2165
|
+
async releaseCustodyPinRef (params) {
|
|
2166
|
+
if (context.provider === context.root) {
|
|
2167
|
+
return await context.root.withTransaction(async (tx)=>(await tx.getStore('graph')).releaseCustodyPinRef(params));
|
|
2168
|
+
}
|
|
2169
|
+
if (!isCustodyPinRef(params.ref)) return 0;
|
|
2170
|
+
// One exclusive key, as bulk ref writers take it: no per-document lock entry per pin.
|
|
2171
|
+
await lockAllDocumentRefs();
|
|
2172
|
+
const documentIDs = await db.selectFrom('kubun_graph_document_refs').select('document_id').where('ref', '=', params.ref).orderBy('document_id').execute();
|
|
2173
|
+
await recomputeDocumentRefsBatch({
|
|
2174
|
+
documentIDs: documentIDs.map((row)=>row.document_id),
|
|
2175
|
+
routeRules: [],
|
|
2176
|
+
nowMS: params.nowMS,
|
|
2177
|
+
releaseCustodyPins: {
|
|
2178
|
+
refs: [
|
|
2179
|
+
params.ref
|
|
2180
|
+
],
|
|
2181
|
+
confirm: async ()=>true
|
|
2182
|
+
}
|
|
2183
|
+
});
|
|
2184
|
+
return documentIDs.length;
|
|
2185
|
+
},
|
|
2186
|
+
async catalogRulesMatchDocument (params) {
|
|
2187
|
+
const use = await db.selectFrom('kubun_graph_catalog_use').select('rules').where('owner_did', '=', params.catalog.owner).where('catalog_id', '=', params.catalog.id).executeTakeFirst();
|
|
2188
|
+
if (use == null) return false;
|
|
2189
|
+
const rules = typeof use.rules === 'string' ? JSON.parse(use.rules) : use.rules;
|
|
2190
|
+
if (rules.length === 0) return false;
|
|
2191
|
+
const doc = await api.getDocument(params.documentID);
|
|
2192
|
+
if (doc == null) return false;
|
|
2193
|
+
const interfaces = await api.getModelInterfaces(doc.model);
|
|
2194
|
+
return rules.some((rule)=>scopeRuleMatches(rule, doc, params.nowMS, interfaces));
|
|
2195
|
+
},
|
|
2196
|
+
async lockDocumentRow (documentID) {
|
|
2197
|
+
if (context.provider === context.root) throw new DocumentLockRequiresTransactionError();
|
|
2198
|
+
const modelID = DocumentID.fromString(documentID).model.toString();
|
|
2199
|
+
const row = db.selectFrom(`k_${modelID}`).select('id').where('id', '=', documentID);
|
|
2200
|
+
const locked = await (adapter instanceof AbstractPostgresAdapter ? row.forUpdate() : row).executeTakeFirst().catch((err)=>{
|
|
2201
|
+
if (isMissingTableError(err, `k_${modelID}`)) return undefined;
|
|
2202
|
+
throw err;
|
|
2203
|
+
});
|
|
2204
|
+
return locked != null;
|
|
2205
|
+
},
|
|
2206
|
+
async settlePendingCatalogRefs (nowMS) {
|
|
2207
|
+
if (context.provider === context.root) {
|
|
2208
|
+
await context.root.withTransaction(async (tx)=>{
|
|
2209
|
+
await (await tx.getStore('graph')).settlePendingCatalogRefs(nowMS);
|
|
2210
|
+
});
|
|
2211
|
+
return;
|
|
2212
|
+
}
|
|
2213
|
+
await settlePendingScopes(nowMS);
|
|
2214
|
+
},
|
|
2215
|
+
async scheduleCatalogRefReconcile () {
|
|
2216
|
+
scheduleScopeModels(context, await api.listDocumentModelIDs());
|
|
2217
|
+
},
|
|
2218
|
+
async recomputeDocuments (params) {
|
|
2219
|
+
if (context.provider === context.root) {
|
|
2220
|
+
await context.root.withTransaction(async (tx)=>{
|
|
2221
|
+
await (await tx.getStore('graph')).recomputeDocuments(params);
|
|
2222
|
+
});
|
|
2223
|
+
return;
|
|
2224
|
+
}
|
|
2225
|
+
if (params.documentIDs.length === 0) return;
|
|
2226
|
+
await lockAllDocumentRefs();
|
|
2227
|
+
await recomputeDocumentRefsBatch({
|
|
2228
|
+
documentIDs: [
|
|
2229
|
+
...params.documentIDs
|
|
2230
|
+
],
|
|
2231
|
+
routeRules: params.routeRules,
|
|
2232
|
+
nowMS: params.nowMS
|
|
2233
|
+
});
|
|
2234
|
+
},
|
|
2235
|
+
async reconcileCatalogRefs (nowMS, routeRules, catalog, options = {}) {
|
|
2236
|
+
if (catalog != null && context.provider === context.root) {
|
|
2237
|
+
await context.root.withTransaction(async (tx)=>{
|
|
2238
|
+
await (await tx.getStore('graph')).reconcileCatalogRefs(nowMS, routeRules, catalog);
|
|
2239
|
+
});
|
|
2240
|
+
return;
|
|
2241
|
+
}
|
|
2242
|
+
const effectiveRouteRules = routeRules.length === 0 ? await context.getCatalogRouteRules(nowMS, catalog) : routeRules;
|
|
2243
|
+
if (catalog != null) {
|
|
2244
|
+
await settlePendingScopes(nowMS, catalog);
|
|
2245
|
+
const use = await db.selectFrom('kubun_graph_catalog_use').select([
|
|
2246
|
+
'rules',
|
|
2247
|
+
'level'
|
|
2248
|
+
]).where('owner_did', '=', catalog.owner).where('catalog_id', '=', catalog.id).executeTakeFirst();
|
|
2249
|
+
const localRules = use?.level == null ? [] : typeof use.rules === 'string' ? JSON.parse(use.rules) : use.rules;
|
|
2250
|
+
const rules = [
|
|
2251
|
+
...localRules,
|
|
2252
|
+
...effectiveRouteRules.filter((entry)=>entry.catalog.owner === catalog.owner && entry.catalog.id === catalog.id).flatMap((entry)=>entry.rules)
|
|
2253
|
+
];
|
|
2254
|
+
for (const task of (await tasksForCatalog(catalog, rules))){
|
|
2255
|
+
await api.reconcileCatalogModelRefs({
|
|
2256
|
+
modelID: task.modelID,
|
|
2257
|
+
routeRules: effectiveRouteRules,
|
|
2258
|
+
nowMS,
|
|
2259
|
+
owners: task.owners,
|
|
2260
|
+
catalog
|
|
2261
|
+
});
|
|
2262
|
+
}
|
|
2263
|
+
return;
|
|
2264
|
+
}
|
|
2265
|
+
const scheduler = scopeSchedulers.get(context.root);
|
|
2266
|
+
if (scheduler != null) {
|
|
2267
|
+
scheduler.generation++;
|
|
2268
|
+
if (scheduler.timer != null) clearTimeout(scheduler.timer);
|
|
2269
|
+
scheduler.timer = null;
|
|
2270
|
+
scheduler.pendingScopes.clear();
|
|
2271
|
+
}
|
|
2272
|
+
if (!await reconcileAll(nowMS, effectiveRouteRules, options.shouldStop ?? (()=>false))) return;
|
|
2273
|
+
await db.insertInto('kubun_graph_catalog_scope_tick').values({
|
|
2274
|
+
id: 1,
|
|
2275
|
+
last_at: nowMS
|
|
2276
|
+
}).onConflict((oc)=>oc.column('id').doUpdateSet({
|
|
2277
|
+
last_at: nowMS
|
|
2278
|
+
})).execute();
|
|
2279
|
+
if (scheduler != null) scheduler.failed = false;
|
|
2280
|
+
if (scheduler != null) scheduler.retryDelayMS = 100;
|
|
2281
|
+
},
|
|
2282
|
+
async reconcileCatalogModelRefs (params) {
|
|
2283
|
+
const { modelID, routeRules, nowMS, owners, catalog } = params;
|
|
2284
|
+
if (context.provider === context.root) {
|
|
2285
|
+
await context.root.withTransaction(async (tx)=>{
|
|
2286
|
+
await (await tx.getStore('graph')).reconcileCatalogModelRefs(params);
|
|
2287
|
+
});
|
|
2288
|
+
return;
|
|
2289
|
+
}
|
|
2290
|
+
const documentIDs = new Set();
|
|
2291
|
+
if (owners == null || owners.length > 0) {
|
|
2292
|
+
let query = db.selectFrom(`k_${modelID}`).select('id');
|
|
2293
|
+
if (owners != null) query = query.where('owner', 'in', owners);
|
|
2294
|
+
const docs = await query.execute().catch((error)=>{
|
|
2295
|
+
if (isMissingTableError(error, `k_${modelID}`)) return [];
|
|
2296
|
+
throw error;
|
|
2297
|
+
});
|
|
2298
|
+
for (const doc of docs)documentIDs.add(doc.id);
|
|
2299
|
+
}
|
|
2300
|
+
if (catalog != null) {
|
|
2301
|
+
for (const holder of (await catalogRefHolders(catalog))){
|
|
2302
|
+
if (DocumentID.fromString(holder.document_id).model.toString() === modelID) {
|
|
2303
|
+
documentIDs.add(holder.document_id);
|
|
2304
|
+
}
|
|
2305
|
+
}
|
|
2306
|
+
}
|
|
2307
|
+
await lockAllDocumentRefs();
|
|
2308
|
+
await recomputeDocumentRefsBatch({
|
|
2309
|
+
documentIDs: [
|
|
2310
|
+
...documentIDs
|
|
2311
|
+
].sort(),
|
|
2312
|
+
routeRules,
|
|
2313
|
+
nowMS,
|
|
2314
|
+
catalog
|
|
2315
|
+
});
|
|
2316
|
+
// A swept document has no row to recompute: its kept facts are matched instead.
|
|
2317
|
+
await api.refreshSweptDocuments({
|
|
2318
|
+
nowMS,
|
|
2319
|
+
modelID
|
|
2320
|
+
});
|
|
2321
|
+
},
|
|
2322
|
+
async scheduleCustodyScan (catalog) {
|
|
2323
|
+
await db.insertInto('kubun_graph_custody_scans').values({
|
|
2324
|
+
owner_did: catalog.owner,
|
|
2325
|
+
catalog_id: catalog.id,
|
|
2326
|
+
after_document_id: null
|
|
2327
|
+
}).onConflict((oc)=>oc.columns([
|
|
2328
|
+
'owner_did',
|
|
2329
|
+
'catalog_id'
|
|
2330
|
+
]).doUpdateSet({
|
|
2331
|
+
after_document_id: null
|
|
2332
|
+
})).execute();
|
|
2333
|
+
context.provider.onCommit(()=>context.custody?.scansQueued());
|
|
2334
|
+
},
|
|
2335
|
+
async advanceCustodyScan (params) {
|
|
2336
|
+
if (context.provider === context.root) {
|
|
2337
|
+
return await context.root.withTransaction(async (tx)=>(await tx.getStore('graph')).advanceCustodyScan(params));
|
|
2338
|
+
}
|
|
2339
|
+
const query = db.selectFrom('kubun_graph_custody_scans').selectAll().orderBy('owner_did').orderBy('catalog_id').limit(1);
|
|
2340
|
+
// Two runners of one scan would each recompute the same batch.
|
|
2341
|
+
const scan = await (adapter instanceof AbstractPostgresAdapter ? query.forUpdate() : query).executeTakeFirst();
|
|
2342
|
+
if (scan == null) return false;
|
|
2343
|
+
const ref = {
|
|
2344
|
+
owner: scan.owner_did,
|
|
2345
|
+
id: scan.catalog_id
|
|
2346
|
+
};
|
|
2347
|
+
const use = await db.selectFrom('kubun_graph_catalog_use').select('rules').where('owner_did', '=', ref.owner).where('catalog_id', '=', ref.id).executeTakeFirst();
|
|
2348
|
+
const rules = use == null ? [] : typeof use.rules === 'string' ? JSON.parse(use.rules) : use.rules;
|
|
2349
|
+
const postgres = adapter instanceof AbstractPostgresAdapter;
|
|
2350
|
+
const candidates = [];
|
|
2351
|
+
for (const task of rules.length === 0 ? [] : await tasksForCatalog(ref, rules)){
|
|
2352
|
+
if (task.owners != null && task.owners.length === 0) continue;
|
|
2353
|
+
let docs = db.selectFrom(`k_${task.modelID}`).select('id');
|
|
2354
|
+
if (task.owners != null) docs = docs.where('owner', 'in', task.owners);
|
|
2355
|
+
// Bytewise, so the cursor agrees with the JavaScript order below on both adapters.
|
|
2356
|
+
if (scan.after_document_id != null) docs = docs.where(postgres ? sql`id collate "C" > ${scan.after_document_id}` : sql`id > ${scan.after_document_id}`);
|
|
2357
|
+
const rows = await (postgres ? docs.orderBy(sql`id collate "C"`) : docs.orderBy('id')).limit(params.batchSize).execute().catch((error)=>{
|
|
2358
|
+
if (isMissingTableError(error, `k_${task.modelID}`)) return [];
|
|
2359
|
+
throw error;
|
|
2360
|
+
});
|
|
2361
|
+
for (const row of rows)candidates.push(row.id);
|
|
2362
|
+
}
|
|
2363
|
+
const batch = [
|
|
2364
|
+
...new Set(candidates)
|
|
2365
|
+
].sort((a, b)=>a < b ? -1 : a > b ? 1 : 0).slice(0, params.batchSize);
|
|
2366
|
+
await recomputeDocumentRefsBatch({
|
|
2367
|
+
documentIDs: batch,
|
|
2368
|
+
routeRules: [],
|
|
2369
|
+
nowMS: params.nowMS,
|
|
2370
|
+
catalog: ref
|
|
2371
|
+
});
|
|
2372
|
+
const last = batch.at(-1);
|
|
2373
|
+
const scans = db.updateTable('kubun_graph_custody_scans').where('owner_did', '=', ref.owner).where('catalog_id', '=', ref.id);
|
|
2374
|
+
if (batch.length < params.batchSize || last == null) await db.deleteFrom('kubun_graph_custody_scans').where('owner_did', '=', ref.owner).where('catalog_id', '=', ref.id).execute();
|
|
2375
|
+
else await scans.set({
|
|
2376
|
+
after_document_id: last
|
|
2377
|
+
}).execute();
|
|
2378
|
+
const remaining = await db.selectFrom('kubun_graph_custody_scans').select('owner_did').limit(1).executeTakeFirst();
|
|
2379
|
+
return remaining != null;
|
|
2380
|
+
},
|
|
2381
|
+
async runCustodyScans (options = {}) {
|
|
2382
|
+
const batchSize = options.batchSize ?? 100;
|
|
2383
|
+
let batches = 0;
|
|
2384
|
+
while(options.maxBatches == null || batches < options.maxBatches){
|
|
2385
|
+
batches++;
|
|
2386
|
+
const more = await context.root.withTransaction(async (tx)=>(await tx.getStore('graph')).advanceCustodyScan({
|
|
2387
|
+
nowMS: options.nowMS ?? Date.now(),
|
|
2388
|
+
batchSize
|
|
2389
|
+
}));
|
|
2390
|
+
if (!more) return true;
|
|
2391
|
+
}
|
|
2392
|
+
return false;
|
|
2393
|
+
},
|
|
2394
|
+
onScopeRecomputed (listener) {
|
|
2395
|
+
const scheduler = getScopeScheduler(context);
|
|
2396
|
+
scheduler.listeners.add(listener);
|
|
2397
|
+
return ()=>scheduler.listeners.delete(listener);
|
|
2398
|
+
},
|
|
2399
|
+
async advanceCatalogAging (nowMS, routeRules) {
|
|
2400
|
+
if (scopeSchedulers.get(context.root)?.failed === true) {
|
|
2401
|
+
await api.reconcileCatalogRefs(nowMS, routeRules);
|
|
2402
|
+
return;
|
|
2403
|
+
}
|
|
2404
|
+
const tick = await db.selectFrom('kubun_graph_catalog_scope_tick').select('last_at').where('id', '=', 1).executeTakeFirst();
|
|
2405
|
+
if (tick == null) {
|
|
2406
|
+
await api.reconcileCatalogRefs(nowMS, routeRules);
|
|
2407
|
+
return;
|
|
2408
|
+
}
|
|
2409
|
+
const previousMS = Number(tick.last_at);
|
|
2410
|
+
if (previousMS === nowMS) return;
|
|
2411
|
+
const catalogs = await db.selectFrom('kubun_graph_catalog_use').select([
|
|
2412
|
+
'rules'
|
|
2413
|
+
]).where('level', 'is not', null).execute();
|
|
2414
|
+
const rules = catalogs.flatMap((catalog)=>typeof catalog.rules === 'string' ? JSON.parse(catalog.rules) : catalog.rules);
|
|
2415
|
+
rules.push(...routeRules.flatMap((route)=>route.rules));
|
|
2416
|
+
const windowedRules = rules.filter((rule)=>rule.maxAge != null || rule.maxAhead != null);
|
|
2417
|
+
const modelRows = await db.selectFrom('kubun_graph_document_models').select('id').execute();
|
|
2418
|
+
const interfacesOf = memoisedInterfaces();
|
|
2419
|
+
const affected = new Set();
|
|
2420
|
+
for (const rule of windowedRules){
|
|
2421
|
+
const selectedModels = [];
|
|
2422
|
+
for (const model of modelRows){
|
|
2423
|
+
const interfaces = await interfacesOf(model.id);
|
|
2424
|
+
if (rule.models == null || rule.models.some((id)=>id === model.id || interfaces.includes(id))) selectedModels.push(model.id);
|
|
2425
|
+
}
|
|
2426
|
+
if (selectedModels.length === 0) continue;
|
|
2427
|
+
const collectRange = async (oldBound, newBound)=>{
|
|
2428
|
+
const from = Math.min(oldBound, newBound);
|
|
2429
|
+
const to = Math.max(oldBound, newBound);
|
|
2430
|
+
if (from === to) return;
|
|
2431
|
+
if (rule.dateField != null) {
|
|
2432
|
+
const rows = await db.selectFrom('kubun_graph_document_scope_dates').select('document_id').where('field', '=', rule.dateField).where('scope_date', '>=', from).where('scope_date', '<', to).execute();
|
|
2433
|
+
const selected = new Set(selectedModels);
|
|
2434
|
+
for (const row of rows){
|
|
2435
|
+
if (selected.has(DocumentID.fromString(row.document_id).model.toString())) {
|
|
2436
|
+
affected.add(row.document_id);
|
|
2437
|
+
}
|
|
2438
|
+
}
|
|
2439
|
+
} else {
|
|
2440
|
+
for (const modelID of selectedModels){
|
|
2441
|
+
const rows = await db.selectFrom(`k_${modelID}`).select('id').where('scope_date', '>=', from).where('scope_date', '<', to).execute();
|
|
2442
|
+
for (const row of rows)affected.add(row.id);
|
|
2443
|
+
}
|
|
2444
|
+
}
|
|
2445
|
+
};
|
|
2446
|
+
if (rule.maxAge != null) {
|
|
2447
|
+
const duration = parseScopeDuration(rule.maxAge);
|
|
2448
|
+
await collectRange(previousMS - duration, nowMS - duration);
|
|
2449
|
+
}
|
|
2450
|
+
if (rule.maxAhead != null) {
|
|
2451
|
+
const duration = parseScopeDuration(rule.maxAhead);
|
|
2452
|
+
await collectRange(previousMS + duration, nowMS + duration);
|
|
2453
|
+
}
|
|
2454
|
+
}
|
|
2455
|
+
// A hold that expired keeps its ref until its document is recomputed.
|
|
2456
|
+
const expiredHolds = await db.selectFrom('kubun_graph_document_holds').select('document_id').where('released_at', 'is', null).where('expires_at', '>', previousMS).where('expires_at', '<=', nowMS).execute();
|
|
2457
|
+
for (const row of expiredHolds)affected.add(row.document_id);
|
|
2458
|
+
const ordered = [
|
|
2459
|
+
...affected
|
|
2460
|
+
].sort();
|
|
2461
|
+
if (ordered.length > 0) await lockAllDocumentRefs();
|
|
2462
|
+
await recomputeDocumentRefsBatch({
|
|
2463
|
+
documentIDs: ordered,
|
|
2464
|
+
routeRules,
|
|
2465
|
+
nowMS
|
|
2466
|
+
});
|
|
2467
|
+
// A swept document has no row to recompute: a window reaching it is found from its facts.
|
|
2468
|
+
await api.refreshSweptDocuments({
|
|
2469
|
+
nowMS
|
|
2470
|
+
});
|
|
2471
|
+
await db.updateTable('kubun_graph_catalog_scope_tick').set({
|
|
2472
|
+
last_at: nowMS
|
|
2473
|
+
}).where('id', '=', 1).execute();
|
|
495
2474
|
},
|
|
496
2475
|
async queryDocuments (params) {
|
|
497
|
-
const {
|
|
2476
|
+
const { catalogRef, filter, modelIDs, orderBy, owner, ...pagination } = params;
|
|
498
2477
|
const isReverse = params.last != null;
|
|
499
|
-
const
|
|
2478
|
+
const reader = await db.selectFrom('kubun_graph_scope_identity').select('id').executeTakeFirst();
|
|
2479
|
+
const scopeRefs = catalogRef == null ? null : await visibleCatalogRefs(catalogRef);
|
|
2480
|
+
if (catalogRef != null && scopeRefs?.length === 0) {
|
|
2481
|
+
return {
|
|
2482
|
+
entries: [],
|
|
2483
|
+
hasMore: false
|
|
2484
|
+
};
|
|
2485
|
+
}
|
|
2486
|
+
const filterScope = reader != null || catalogRef != null;
|
|
2487
|
+
const effectiveModelIDs = modelIDs;
|
|
2488
|
+
const effectiveOwners = undefined;
|
|
500
2489
|
if (effectiveModelIDs.length === 0) {
|
|
501
2490
|
return {
|
|
502
2491
|
entries: [],
|
|
503
2492
|
hasMore: false
|
|
504
2493
|
};
|
|
505
2494
|
}
|
|
506
|
-
const
|
|
2495
|
+
const first = effectiveModelIDs[0];
|
|
2496
|
+
if (first == null) return {
|
|
2497
|
+
entries: [],
|
|
2498
|
+
hasMore: false
|
|
2499
|
+
};
|
|
2500
|
+
const rest = effectiveModelIDs.slice(1);
|
|
507
2501
|
// Tombstoned documents (data IS NULL) stay on disk for sync/CRDT but must
|
|
508
2502
|
// never surface in read queries.
|
|
509
2503
|
let query = db.selectFrom(`k_${first}`).selectAll().where('data', 'is not', null);
|
|
2504
|
+
if (filterScope) query = query.where(visibleScopePredicate(first, scopeRefs));
|
|
510
2505
|
if (owner != null) {
|
|
511
2506
|
query = query.where('owner', '=', owner);
|
|
512
2507
|
}
|
|
@@ -519,6 +2514,7 @@ export function createGraphStore(db, adapter) {
|
|
|
519
2514
|
for (const model of rest){
|
|
520
2515
|
query = query.unionAll((eb)=>{
|
|
521
2516
|
let q = eb.selectFrom(`k_${model}`).selectAll().where('data', 'is not', null);
|
|
2517
|
+
if (filterScope) q = q.where(visibleScopePredicate(model, scopeRefs));
|
|
522
2518
|
if (owner != null) {
|
|
523
2519
|
q = q.where('owner', '=', owner);
|
|
524
2520
|
}
|
|
@@ -531,7 +2527,18 @@ export function createGraphStore(db, adapter) {
|
|
|
531
2527
|
return q;
|
|
532
2528
|
});
|
|
533
2529
|
}
|
|
534
|
-
|
|
2530
|
+
// A compound SELECT only accepts result-column references in its ORDER BY (both
|
|
2531
|
+
// SQLite and Postgres reject the JSON-path expressions a data-field order emits),
|
|
2532
|
+
// and a WHERE added after `unionAll` binds to the first branch alone, so a keyset
|
|
2533
|
+
// cursor would only filter that one table. Reading the union as a derived table
|
|
2534
|
+
// lets ordering and the keyset predicate apply the single-model path's expressions
|
|
2535
|
+
// to every row. This relies on every branch selecting the same column set: all
|
|
2536
|
+
// model tables come from the one `createTable` in `addDocumentModel`, so the
|
|
2537
|
+
// derived table has exactly the per-table columns. A field typed differently
|
|
2538
|
+
// across models orders per adapter, as the JSON-extracted value: SQLite sorts
|
|
2539
|
+
// numbers before text, Postgres compares the text form.
|
|
2540
|
+
const source = rest.length > 0 ? db.selectFrom(query.as('k_document')).selectAll() : query;
|
|
2541
|
+
const [orderedQuery, orderedFieldsPaths] = applyDocumentOrderBy(source, adapter, orderBy, isReverse);
|
|
535
2542
|
const [paginatedQuery, limit] = applyPagination(orderedQuery, pagination, orderBy, coerce);
|
|
536
2543
|
const results = await paginatedQuery.execute();
|
|
537
2544
|
const docs = isReverse ? results.slice(0, limit).reverse() : results.slice(0, limit);
|
|
@@ -591,12 +2598,17 @@ export function createGraphStore(db, adapter) {
|
|
|
591
2598
|
};
|
|
592
2599
|
},
|
|
593
2600
|
async aggregateDocuments (params) {
|
|
594
|
-
const { aggregates,
|
|
2601
|
+
const { aggregates, catalogRef, count, filter, modelIDs, owner, readAccess } = params;
|
|
595
2602
|
const empty = emptyAggregateValues(count, aggregates);
|
|
596
2603
|
// No count and no aggregate ops requested: there is nothing to select, so
|
|
597
2604
|
// skip the query entirely rather than build an invalid empty SELECT.
|
|
598
2605
|
if (!count && aggregates.length === 0) return empty;
|
|
599
|
-
const
|
|
2606
|
+
const scopeRefs = catalogRef == null ? null : await visibleCatalogRefs(catalogRef);
|
|
2607
|
+
if (catalogRef != null && scopeRefs?.length === 0) return empty;
|
|
2608
|
+
const reader = await db.selectFrom('kubun_graph_scope_identity').select('id').executeTakeFirst();
|
|
2609
|
+
const filterScope = reader != null || catalogRef != null;
|
|
2610
|
+
const effectiveModelIDs = modelIDs;
|
|
2611
|
+
const effectiveOwners = undefined;
|
|
600
2612
|
if (effectiveModelIDs.length === 0) return empty;
|
|
601
2613
|
// Aggregate each model in its own table, then combine at the raw sum/count/min/max
|
|
602
2614
|
// level so interface-connection results stay correct: averages combine as Σsum /
|
|
@@ -610,6 +2622,7 @@ export function createGraphStore(db, adapter) {
|
|
|
610
2622
|
return selections;
|
|
611
2623
|
});
|
|
612
2624
|
query = query.where('data', 'is not', null);
|
|
2625
|
+
if (filterScope) query = query.where(visibleScopePredicate(modelID, scopeRefs));
|
|
613
2626
|
if (owner != null) {
|
|
614
2627
|
query = query.where('owner', '=', owner);
|
|
615
2628
|
}
|
|
@@ -626,8 +2639,7 @@ export function createGraphStore(db, adapter) {
|
|
|
626
2639
|
return await query.executeTakeFirst();
|
|
627
2640
|
} catch (err) {
|
|
628
2641
|
// A model whose table was never materialised contributes nothing, matching
|
|
629
|
-
// the not-yet-materialised-table tolerance used by
|
|
630
|
-
// and getDocumentIDsForScope in this file.
|
|
2642
|
+
// the not-yet-materialised-table tolerance used by getDocumentIDsForScope.
|
|
631
2643
|
if (isMissingTableError(err, `k_${modelID}`)) return undefined;
|
|
632
2644
|
throw err;
|
|
633
2645
|
}
|
|
@@ -643,9 +2655,14 @@ export function createGraphStore(db, adapter) {
|
|
|
643
2655
|
return finalizeAggregateValues(count, aggregates, acc);
|
|
644
2656
|
},
|
|
645
2657
|
async groupedAggregateDocuments (params) {
|
|
646
|
-
const { aggregates,
|
|
647
|
-
const
|
|
2658
|
+
const { aggregates, catalogRef, count, filter, groupBy, having, limit, modelIDs, orderBy, owner, readAccess } = params;
|
|
2659
|
+
const effectiveModelIDs = modelIDs;
|
|
2660
|
+
const effectiveOwners = undefined;
|
|
648
2661
|
if (effectiveModelIDs.length === 0) return [];
|
|
2662
|
+
const scopeRefs = catalogRef == null ? null : await visibleCatalogRefs(catalogRef);
|
|
2663
|
+
if (catalogRef != null && scopeRefs?.length === 0) return [];
|
|
2664
|
+
const reader = await db.selectFrom('kubun_graph_scope_identity').select('id').executeTakeFirst();
|
|
2665
|
+
const filterScope = reader != null || catalogRef != null;
|
|
649
2666
|
// Run the same GROUP BY query per model table in parallel, tolerating tables
|
|
650
2667
|
// that were never materialised (they contribute nothing), mirroring the
|
|
651
2668
|
// multi-model fan-out in aggregateDocuments.
|
|
@@ -664,6 +2681,7 @@ export function createGraphStore(db, adapter) {
|
|
|
664
2681
|
});
|
|
665
2682
|
query = query.groupBy((eb)=>applyGroupBySelections(eb, groupBy).expressions);
|
|
666
2683
|
query = query.where('data', 'is not', null);
|
|
2684
|
+
if (filterScope) query = query.where(visibleScopePredicate(modelID, scopeRefs));
|
|
667
2685
|
if (owner != null) {
|
|
668
2686
|
query = query.where('owner', '=', owner);
|
|
669
2687
|
}
|
|
@@ -743,31 +2761,169 @@ export function createGraphStore(db, adapter) {
|
|
|
743
2761
|
const models = await db.selectFrom('kubun_graph_document_models').select('id').execute();
|
|
744
2762
|
return models.map((m)=>m.id);
|
|
745
2763
|
},
|
|
746
|
-
async getDistinctOwnersForModel (modelID) {
|
|
747
|
-
try {
|
|
748
|
-
const result = await db.selectFrom(`k_${modelID}`).select('owner').distinct().execute();
|
|
749
|
-
return result.map((r)=>r.owner);
|
|
750
|
-
} catch (err) {
|
|
751
|
-
if (isMissingTableError(err, `k_${modelID}`)) return [];
|
|
752
|
-
throw err;
|
|
753
|
-
}
|
|
754
|
-
},
|
|
755
2764
|
async queryDocumentsByOwner (modelID, ownerDID) {
|
|
756
2765
|
const docs = await db.selectFrom(`k_${modelID}`).selectAll().where('owner', '=', ownerDID).execute();
|
|
757
2766
|
return docs.map((doc)=>documentNodeFromTable(adapter, doc));
|
|
758
2767
|
},
|
|
759
2768
|
// --- Mutation log ---
|
|
760
2769
|
async insertMutationLogEntry (entry) {
|
|
761
|
-
|
|
2770
|
+
if (!context.provider.isTransactionBound) {
|
|
2771
|
+
await context.provider.withTransaction(async (tx)=>{
|
|
2772
|
+
const graph = await tx.getStore('graph');
|
|
2773
|
+
await graph.insertMutationLogEntry(entry);
|
|
2774
|
+
});
|
|
2775
|
+
return;
|
|
2776
|
+
}
|
|
2777
|
+
const inserted = await db.insertInto('kubun_graph_mutation_log').values(entry).onConflict((oc)=>entry.status === 'applied' ? oc.column('mutation_hash').doUpdateSet({
|
|
2778
|
+
status: 'applied',
|
|
2779
|
+
next_attempt_at: null
|
|
2780
|
+
}).where((eb)=>eb(eb.ref('kubun_graph_mutation_log.status'), '!=', 'applied')) : oc.doNothing()).returning('document_id').execute();
|
|
2781
|
+
if (inserted[0] != null && entry.status === 'applied') {
|
|
2782
|
+
await refreshDocumentDates(inserted[0].document_id);
|
|
2783
|
+
await api.recomputeDocumentRefs({
|
|
2784
|
+
documentID: inserted[0].document_id,
|
|
2785
|
+
routeRules: []
|
|
2786
|
+
});
|
|
2787
|
+
}
|
|
2788
|
+
},
|
|
2789
|
+
async parkPushedMutation (entry) {
|
|
2790
|
+
if (!context.provider.isTransactionBound) {
|
|
2791
|
+
return await context.provider.withTransaction(async (tx)=>{
|
|
2792
|
+
const graph = await tx.getStore('graph');
|
|
2793
|
+
return await graph.parkPushedMutation(entry);
|
|
2794
|
+
});
|
|
2795
|
+
}
|
|
2796
|
+
await db.insertInto('kubun_graph_pending_groups').values({
|
|
2797
|
+
group_id: entry.arrival_group_id
|
|
2798
|
+
}).onConflict((oc)=>oc.column('group_id').doUpdateSet({
|
|
2799
|
+
group_id: entry.arrival_group_id
|
|
2800
|
+
})).execute();
|
|
2801
|
+
const parkedAt = Date.now();
|
|
2802
|
+
await expirePushedMutations(parkedAt, entry.arrival_group_id);
|
|
2803
|
+
const inserted = await db.insertInto('kubun_graph_mutation_log').columns([
|
|
2804
|
+
'mutation_hash',
|
|
2805
|
+
'model_id',
|
|
2806
|
+
'document_id',
|
|
2807
|
+
'author_did',
|
|
2808
|
+
'hlc',
|
|
2809
|
+
'mutation_jwt',
|
|
2810
|
+
'status',
|
|
2811
|
+
'arrival_group_id',
|
|
2812
|
+
'sender_peer_did',
|
|
2813
|
+
'parked_at'
|
|
2814
|
+
]).expression(db.selectNoFrom((eb)=>[
|
|
2815
|
+
eb.val(entry.mutation_hash).as('mutation_hash'),
|
|
2816
|
+
eb.val(entry.model_id).as('model_id'),
|
|
2817
|
+
eb.val(entry.document_id).as('document_id'),
|
|
2818
|
+
eb.val(entry.author_did).as('author_did'),
|
|
2819
|
+
eb.val(entry.hlc).as('hlc'),
|
|
2820
|
+
eb.val(entry.mutation_jwt).as('mutation_jwt'),
|
|
2821
|
+
eb.val('pending').as('status'),
|
|
2822
|
+
eb.val(entry.arrival_group_id).as('arrival_group_id'),
|
|
2823
|
+
eb.val(entry.sender_peer_did).as('sender_peer_did'),
|
|
2824
|
+
eb.val(parkedAt).as('parked_at')
|
|
2825
|
+
]).where((eb)=>eb(eb.selectFrom('kubun_graph_mutation_log').select((count)=>count.fn.countAll().as('count')).where('status', '=', 'pending').where('arrival_group_id', '=', entry.arrival_group_id), '<', 1000)).where((eb)=>eb(eb.selectFrom('kubun_graph_mutation_log').select((count)=>count.fn.countAll().as('count')).where('status', '=', 'pending').where('arrival_group_id', '=', entry.arrival_group_id).where('sender_peer_did', '=', entry.sender_peer_did), '<', 100))).onConflict((oc)=>oc.doNothing()).returning('mutation_hash').execute();
|
|
2826
|
+
if (inserted.length > 0) return 'parked';
|
|
2827
|
+
return await api.hasMutationHash(entry.mutation_hash) ? 'duplicate' : 'full';
|
|
762
2828
|
},
|
|
763
2829
|
async hasMutationHash (hash) {
|
|
764
2830
|
const row = await db.selectFrom('kubun_graph_mutation_log').select('mutation_hash').where('mutation_hash', '=', hash).executeTakeFirst();
|
|
765
2831
|
return row != null;
|
|
766
2832
|
},
|
|
767
2833
|
async updateMutationStatus (mutationHash, status) {
|
|
768
|
-
|
|
2834
|
+
if (context.provider === context.root) {
|
|
2835
|
+
await context.root.withTransaction(async (tx)=>{
|
|
2836
|
+
await (await tx.getStore('graph')).updateMutationStatus(mutationHash, status);
|
|
2837
|
+
});
|
|
2838
|
+
return;
|
|
2839
|
+
}
|
|
2840
|
+
const updated = await db.updateTable('kubun_graph_mutation_log').set({
|
|
769
2841
|
status
|
|
770
|
-
}).where('mutation_hash', '=', mutationHash).execute();
|
|
2842
|
+
}).where('mutation_hash', '=', mutationHash).returning('document_id').execute();
|
|
2843
|
+
if (updated[0] != null) {
|
|
2844
|
+
await refreshDocumentDates(updated[0].document_id);
|
|
2845
|
+
await api.recomputeDocumentRefs({
|
|
2846
|
+
documentID: updated[0].document_id,
|
|
2847
|
+
routeRules: []
|
|
2848
|
+
});
|
|
2849
|
+
}
|
|
2850
|
+
},
|
|
2851
|
+
async getDuePendingMutations (now) {
|
|
2852
|
+
if (!context.provider.isTransactionBound) {
|
|
2853
|
+
return await context.provider.withTransaction(async (tx)=>{
|
|
2854
|
+
const graph = await tx.getStore('graph');
|
|
2855
|
+
return await graph.getDuePendingMutations(now);
|
|
2856
|
+
});
|
|
2857
|
+
}
|
|
2858
|
+
await expirePushedMutations(now);
|
|
2859
|
+
const models = await db.selectFrom('kubun_graph_document_models').select('id').execute();
|
|
2860
|
+
if (models.length === 0) return [];
|
|
2861
|
+
return await db.selectFrom('kubun_graph_mutation_log').selectAll().where('status', '=', 'pending').where((eb)=>eb.or([
|
|
2862
|
+
eb('next_attempt_at', 'is', null),
|
|
2863
|
+
eb('next_attempt_at', '<=', now)
|
|
2864
|
+
])).where((eb)=>eb.or(models.map(({ id })=>eb.and([
|
|
2865
|
+
eb('model_id', '=', id),
|
|
2866
|
+
eb.exists(eb.selectFrom(`k_${id} as k_`).select('id').whereRef('k_.id', '=', 'kubun_graph_mutation_log.document_id'))
|
|
2867
|
+
])))).orderBy('hlc', 'asc').limit(100).execute();
|
|
2868
|
+
},
|
|
2869
|
+
async getNextPendingAttemptAt () {
|
|
2870
|
+
if (!context.provider.isTransactionBound) {
|
|
2871
|
+
return await context.provider.withTransaction(async (tx)=>{
|
|
2872
|
+
const graph = await tx.getStore('graph');
|
|
2873
|
+
return await graph.getNextPendingAttemptAt();
|
|
2874
|
+
});
|
|
2875
|
+
}
|
|
2876
|
+
const now = Date.now();
|
|
2877
|
+
await expirePushedMutations(now);
|
|
2878
|
+
const expiry = await db.selectFrom('kubun_graph_mutation_log').select('parked_at').where('status', '=', 'pending').where('arrival_group_id', 'is not', null).where('parked_at', 'is not', null).orderBy('parked_at', 'asc').limit(1).executeTakeFirst();
|
|
2879
|
+
const expiresAt = expiry?.parked_at == null ? null : Number(expiry.parked_at) + pendingLifetimeMS;
|
|
2880
|
+
const models = await db.selectFrom('kubun_graph_document_models').select('id').execute();
|
|
2881
|
+
if (models.length === 0) return expiresAt;
|
|
2882
|
+
const row = await db.selectFrom('kubun_graph_mutation_log').select('next_attempt_at').where('status', '=', 'pending').where('next_attempt_at', 'is not', null).where((eb)=>eb.or(models.map(({ id })=>eb.and([
|
|
2883
|
+
eb('model_id', '=', id),
|
|
2884
|
+
eb.exists(eb.selectFrom(`k_${id} as k_`).select('id').whereRef('k_.id', '=', 'kubun_graph_mutation_log.document_id'))
|
|
2885
|
+
])))).orderBy('next_attempt_at', 'asc').limit(1).executeTakeFirst();
|
|
2886
|
+
const nextAttemptAt = row?.next_attempt_at == null ? null : Number(row.next_attempt_at);
|
|
2887
|
+
if (nextAttemptAt == null) return expiresAt;
|
|
2888
|
+
return expiresAt == null ? nextAttemptAt : Math.min(nextAttemptAt, expiresAt);
|
|
2889
|
+
},
|
|
2890
|
+
async recordPendingMutationAttempt (params) {
|
|
2891
|
+
const { mutationHash, outcome, now, jitter } = params;
|
|
2892
|
+
if (context.provider === context.root) {
|
|
2893
|
+
await context.root.withTransaction(async (tx)=>{
|
|
2894
|
+
const graph = await tx.getStore('graph');
|
|
2895
|
+
await graph.recordPendingMutationAttempt(params);
|
|
2896
|
+
});
|
|
2897
|
+
return;
|
|
2898
|
+
}
|
|
2899
|
+
// Successful apply already promoted the row; stale failures may only touch pending rows.
|
|
2900
|
+
const statuses = outcome === 'applied' ? [
|
|
2901
|
+
'pending',
|
|
2902
|
+
'applied'
|
|
2903
|
+
] : [
|
|
2904
|
+
'pending'
|
|
2905
|
+
];
|
|
2906
|
+
const row = await db.selectFrom('kubun_graph_mutation_log').select([
|
|
2907
|
+
'attempt_count',
|
|
2908
|
+
'document_id',
|
|
2909
|
+
'status'
|
|
2910
|
+
]).where('mutation_hash', '=', mutationHash).where('status', 'in', statuses).executeTakeFirst();
|
|
2911
|
+
if (row == null) return;
|
|
2912
|
+
const count = row.attempt_count + 1;
|
|
2913
|
+
const delay = Math.min(60_000, 100 * 2 ** Math.min(count - 1, 10) * (1 + 0.25 * jitter));
|
|
2914
|
+
const updated = await db.updateTable('kubun_graph_mutation_log').set({
|
|
2915
|
+
attempt_count: count,
|
|
2916
|
+
last_attempt_at: now,
|
|
2917
|
+
next_attempt_at: outcome === 'transient' ? now + Math.round(delay) : null,
|
|
2918
|
+
status: outcome === 'transient' ? 'pending' : outcome
|
|
2919
|
+
}).where('mutation_hash', '=', mutationHash).where('status', 'in', statuses).returning('document_id').execute();
|
|
2920
|
+
if (outcome === 'applied' && row.status !== 'applied' && updated.length > 0) {
|
|
2921
|
+
await refreshDocumentDates(row.document_id);
|
|
2922
|
+
await api.recomputeDocumentRefs({
|
|
2923
|
+
documentID: row.document_id,
|
|
2924
|
+
routeRules: []
|
|
2925
|
+
});
|
|
2926
|
+
}
|
|
771
2927
|
},
|
|
772
2928
|
async getMutationLogForDocuments (documentIDs) {
|
|
773
2929
|
if (documentIDs.length === 0) return [];
|
|
@@ -803,6 +2959,31 @@ export function createGraphStore(db, adapter) {
|
|
|
803
2959
|
}
|
|
804
2960
|
return values;
|
|
805
2961
|
},
|
|
2962
|
+
async getFieldValuesMany (documentIDs) {
|
|
2963
|
+
const byModel = new Map();
|
|
2964
|
+
for (const documentID of documentIDs){
|
|
2965
|
+
const modelID = DocumentID.fromString(documentID).model.toString();
|
|
2966
|
+
const ids = byModel.get(modelID) ?? new Set();
|
|
2967
|
+
ids.add(documentID);
|
|
2968
|
+
byModel.set(modelID, ids);
|
|
2969
|
+
}
|
|
2970
|
+
const found = new Map();
|
|
2971
|
+
for (const [modelID, ids] of byModel){
|
|
2972
|
+
for (const chunk of chunked(ids)){
|
|
2973
|
+
const rows = await db.selectFrom(`k_${modelID}`).select([
|
|
2974
|
+
'id',
|
|
2975
|
+
'field_values'
|
|
2976
|
+
]).where('id', 'in', chunk).execute();
|
|
2977
|
+
for (const row of rows){
|
|
2978
|
+
if (row.field_values == null) continue;
|
|
2979
|
+
const values = row.field_values;
|
|
2980
|
+
// ParseJSONResultsPlugin may have already parsed the value
|
|
2981
|
+
found.set(row.id, typeof values === 'string' ? JSON.parse(values) : values);
|
|
2982
|
+
}
|
|
2983
|
+
}
|
|
2984
|
+
}
|
|
2985
|
+
return found;
|
|
2986
|
+
},
|
|
806
2987
|
async updateFieldHLCs (docID, fieldHLCs, fieldValues) {
|
|
807
2988
|
const modelID = docID.model.toString();
|
|
808
2989
|
await db.updateTable(`k_${modelID}`).set({
|
|
@@ -817,6 +2998,8 @@ export function createGraphStore(db, adapter) {
|
|
|
817
2998
|
async createGraph (params) {
|
|
818
2999
|
const graphModelID = params.id;
|
|
819
3000
|
const name = params.name ?? graphModelID;
|
|
3001
|
+
const recordModelIDs = Object.keys(params.record);
|
|
3002
|
+
const existingModels = recordModelIDs.length === 0 ? [] : await db.selectFrom('kubun_graph_document_models').select('id').where('id', 'in', recordModelIDs).execute();
|
|
820
3003
|
await db.transaction().setIsolationLevel('read committed').execute(async (trx)=>{
|
|
821
3004
|
const pluginConfigValue = params.pluginConfig ? adapter.encodeJSON(params.pluginConfig) : null;
|
|
822
3005
|
await trx.insertInto('kubun_graph_models').values({
|
|
@@ -937,6 +3120,7 @@ export function createGraphStore(db, adapter) {
|
|
|
937
3120
|
}))).execute();
|
|
938
3121
|
}
|
|
939
3122
|
});
|
|
3123
|
+
scheduleScopeModels(context, existingModels.map((model)=>model.id));
|
|
940
3124
|
return graphModelID;
|
|
941
3125
|
},
|
|
942
3126
|
async getGraph (id) {
|
|
@@ -992,7 +3176,8 @@ export function createGraphStore(db, adapter) {
|
|
|
992
3176
|
async dropSearchIndex (modelID) {
|
|
993
3177
|
await adapter.dropSearchIndex(db, modelID);
|
|
994
3178
|
},
|
|
995
|
-
async updateSearchEntry (
|
|
3179
|
+
async updateSearchEntry (params) {
|
|
3180
|
+
const { modelID, documentID, data, fields } = params;
|
|
996
3181
|
const fieldValues = {};
|
|
997
3182
|
for (const field of fields){
|
|
998
3183
|
const value = extractFieldValue(data, field);
|
|
@@ -1006,6 +3191,10 @@ export function createGraphStore(db, adapter) {
|
|
|
1006
3191
|
await adapter.removeSearchEntry(db, modelID, documentID);
|
|
1007
3192
|
},
|
|
1008
3193
|
async searchDocuments (params) {
|
|
3194
|
+
const reader = await db.selectFrom('kubun_graph_scope_identity').select('id').executeTakeFirst();
|
|
3195
|
+
const scopeRefs = params.catalogRef == null ? null : await visibleCatalogRefs(params.catalogRef);
|
|
3196
|
+
if (params.catalogRef != null && scopeRefs?.length === 0) return [];
|
|
3197
|
+
const filterScope = reader != null || params.catalogRef != null;
|
|
1009
3198
|
// Guard against querying a model with no `kubun_graph_search_models`
|
|
1010
3199
|
// registry row: without this, a missing `fts_<model>` table surfaces
|
|
1011
3200
|
// as a raw adapter error (e.g. "no such table") instead of a clean,
|
|
@@ -1023,17 +3212,32 @@ export function createGraphStore(db, adapter) {
|
|
|
1023
3212
|
const perModelLimit = params.first != null ? offset + params.first : undefined;
|
|
1024
3213
|
const allResults = [];
|
|
1025
3214
|
for (const modelID of params.modelIDs){
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
});
|
|
1031
|
-
for (const hit of hits){
|
|
1032
|
-
allResults.push({
|
|
1033
|
-
documentID: hit.documentID,
|
|
3215
|
+
let limit = Math.max(perModelLimit ?? 50, 50);
|
|
3216
|
+
while(true){
|
|
3217
|
+
const hits = await adapter.searchIndex(db, {
|
|
3218
|
+
query: params.query,
|
|
1034
3219
|
modelID,
|
|
1035
|
-
|
|
3220
|
+
limit
|
|
1036
3221
|
});
|
|
3222
|
+
let visibleIDs = null;
|
|
3223
|
+
if (filterScope && hits.length > 0) {
|
|
3224
|
+
let refsQuery = db.selectFrom('kubun_graph_document_refs').select('document_id').where('document_id', 'in', hits.map((hit)=>hit.documentID)).where('displays', '=', 1);
|
|
3225
|
+
if (scopeRefs != null) refsQuery = refsQuery.where('ref', 'in', scopeRefs);
|
|
3226
|
+
const rows = await refsQuery.execute();
|
|
3227
|
+
visibleIDs = new Set(rows.map((row)=>row.document_id));
|
|
3228
|
+
}
|
|
3229
|
+
const visibleHits = visibleIDs == null ? hits : hits.filter((hit)=>visibleIDs.has(hit.documentID));
|
|
3230
|
+
if (visibleHits.length >= (perModelLimit ?? 50) || hits.length < limit) {
|
|
3231
|
+
for (const hit of visibleHits){
|
|
3232
|
+
allResults.push({
|
|
3233
|
+
documentID: hit.documentID,
|
|
3234
|
+
modelID,
|
|
3235
|
+
rank: hit.rank
|
|
3236
|
+
});
|
|
3237
|
+
}
|
|
3238
|
+
break;
|
|
3239
|
+
}
|
|
3240
|
+
limit *= 2;
|
|
1037
3241
|
}
|
|
1038
3242
|
}
|
|
1039
3243
|
// Sort by absolute rank descending (SQLite bm25 returns negative, PG ts_rank positive)
|
|
@@ -1043,121 +3247,6 @@ export function createGraphStore(db, adapter) {
|
|
|
1043
3247
|
}
|
|
1044
3248
|
return allResults;
|
|
1045
3249
|
},
|
|
1046
|
-
// --- Catalog operations ---
|
|
1047
|
-
async createCatalog (catalog) {
|
|
1048
|
-
await db.insertInto('kubun_graph_catalogs').values({
|
|
1049
|
-
id: catalog.id,
|
|
1050
|
-
owner_did: catalog.owner_did,
|
|
1051
|
-
name: catalog.name,
|
|
1052
|
-
description: catalog.description,
|
|
1053
|
-
filter_criteria: adapter.encodeJSON(catalog.filter_criteria),
|
|
1054
|
-
hlc: catalog.hlc,
|
|
1055
|
-
// Own catalogs scope sync immediately; no discovery provenance.
|
|
1056
|
-
active: 1,
|
|
1057
|
-
source_group_id: null,
|
|
1058
|
-
source_circle_id: null,
|
|
1059
|
-
signed_token: catalog.signed_token ?? null
|
|
1060
|
-
}).execute();
|
|
1061
|
-
},
|
|
1062
|
-
async upsertDiscoveredCatalog ({ catalog, sourceGroupID, sourceCircleID }) {
|
|
1063
|
-
await db.insertInto('kubun_graph_catalogs').values({
|
|
1064
|
-
id: catalog.id,
|
|
1065
|
-
owner_did: catalog.owner_did,
|
|
1066
|
-
name: catalog.name,
|
|
1067
|
-
description: catalog.description,
|
|
1068
|
-
filter_criteria: adapter.encodeJSON(catalog.filter_criteria),
|
|
1069
|
-
hlc: catalog.hlc,
|
|
1070
|
-
// Discovery applies no sync — the catalog arrives known, awaiting an
|
|
1071
|
-
// explicit local activation decision.
|
|
1072
|
-
active: 0,
|
|
1073
|
-
source_group_id: sourceGroupID,
|
|
1074
|
-
source_circle_id: sourceCircleID,
|
|
1075
|
-
signed_token: catalog.signed_token ?? null
|
|
1076
|
-
})// First-discovery-wins: an existing row (already known or locally
|
|
1077
|
-
// activated) keeps its `active` flag and source columns untouched.
|
|
1078
|
-
.onConflict((oc)=>oc.column('id').doNothing()).execute();
|
|
1079
|
-
},
|
|
1080
|
-
async setCatalogActive (id, active) {
|
|
1081
|
-
await db.updateTable('kubun_graph_catalogs').set({
|
|
1082
|
-
active: active ? 1 : 0,
|
|
1083
|
-
updated_at: adapter.encodeTimestamp(new Date())
|
|
1084
|
-
}).where('id', '=', id).execute();
|
|
1085
|
-
},
|
|
1086
|
-
async setCatalogsActive (ids, active) {
|
|
1087
|
-
if (ids.length === 0) {
|
|
1088
|
-
return;
|
|
1089
|
-
}
|
|
1090
|
-
await db.updateTable('kubun_graph_catalogs').set({
|
|
1091
|
-
active: active ? 1 : 0,
|
|
1092
|
-
updated_at: adapter.encodeTimestamp(new Date())
|
|
1093
|
-
}).where('id', 'in', ids).execute();
|
|
1094
|
-
},
|
|
1095
|
-
async listStoredCatalogs (filter) {
|
|
1096
|
-
let query = db.selectFrom('kubun_graph_catalogs').selectAll();
|
|
1097
|
-
if (filter?.active === true) {
|
|
1098
|
-
query = query.where('active', '=', 1);
|
|
1099
|
-
} else if (filter?.active === false) {
|
|
1100
|
-
query = query.where((eb)=>eb.or([
|
|
1101
|
-
eb('active', 'is', null),
|
|
1102
|
-
eb('active', '=', 0)
|
|
1103
|
-
]));
|
|
1104
|
-
}
|
|
1105
|
-
return await query.execute();
|
|
1106
|
-
},
|
|
1107
|
-
async getCatalog (id) {
|
|
1108
|
-
return await db.selectFrom('kubun_graph_catalogs').selectAll().where('id', '=', id).executeTakeFirst();
|
|
1109
|
-
},
|
|
1110
|
-
async getCatalogs (ids) {
|
|
1111
|
-
const result = new Map();
|
|
1112
|
-
if (ids.length === 0) return result;
|
|
1113
|
-
const rows = await db.selectFrom('kubun_graph_catalogs').selectAll().where('id', 'in', ids).execute();
|
|
1114
|
-
for (const row of rows){
|
|
1115
|
-
result.set(row.id, row);
|
|
1116
|
-
}
|
|
1117
|
-
return result;
|
|
1118
|
-
},
|
|
1119
|
-
async updateCatalog (id, update) {
|
|
1120
|
-
const values = {};
|
|
1121
|
-
if (update.name != null) values.name = update.name;
|
|
1122
|
-
if (update.description != null) values.description = update.description;
|
|
1123
|
-
if (update.filter_criteria != null) {
|
|
1124
|
-
values.filter_criteria = adapter.encodeJSON(update.filter_criteria);
|
|
1125
|
-
}
|
|
1126
|
-
if (update.hlc != null) values.hlc = update.hlc;
|
|
1127
|
-
if (update.signed_token != null) values.signed_token = update.signed_token;
|
|
1128
|
-
values.updated_at = adapter.encodeTimestamp(new Date());
|
|
1129
|
-
await db.updateTable('kubun_graph_catalogs').set(values).where('id', '=', id).execute();
|
|
1130
|
-
},
|
|
1131
|
-
async deleteCatalog (id) {
|
|
1132
|
-
await db.deleteFrom('kubun_graph_catalogs').where('id', '=', id).execute();
|
|
1133
|
-
},
|
|
1134
|
-
async listCatalogs (ownerDID) {
|
|
1135
|
-
return await db.selectFrom('kubun_graph_catalogs').selectAll().where('owner_did', '=', ownerDID).execute();
|
|
1136
|
-
},
|
|
1137
|
-
async resolveCatalogScope (catalogID) {
|
|
1138
|
-
const catalog = await api.getCatalog(catalogID);
|
|
1139
|
-
if (catalog == null) {
|
|
1140
|
-
return {
|
|
1141
|
-
models: undefined,
|
|
1142
|
-
owners: undefined
|
|
1143
|
-
};
|
|
1144
|
-
}
|
|
1145
|
-
const criteria = catalog.filter_criteria;
|
|
1146
|
-
const models = criteria.models != null && criteria.models.length > 0 ? criteria.models : undefined;
|
|
1147
|
-
let owners;
|
|
1148
|
-
if (criteria.owners != null && criteria.owners.length > 0) {
|
|
1149
|
-
owners = [
|
|
1150
|
-
...criteria.owners
|
|
1151
|
-
];
|
|
1152
|
-
}
|
|
1153
|
-
// Note: circle-based owner resolution is not included here because
|
|
1154
|
-
// circle tables are in a separate store. Consumers should resolve
|
|
1155
|
-
// circle members externally if needed.
|
|
1156
|
-
return {
|
|
1157
|
-
models,
|
|
1158
|
-
owners
|
|
1159
|
-
};
|
|
1160
|
-
},
|
|
1161
3250
|
// --- Cluster registration ---
|
|
1162
3251
|
async registerClusterModels (_clusterID, entries) {
|
|
1163
3252
|
if (entries.length === 0) return;
|
|
@@ -1167,6 +3256,35 @@ export function createGraphStore(db, adapter) {
|
|
|
1167
3256
|
}))).execute();
|
|
1168
3257
|
},
|
|
1169
3258
|
// --- User model access defaults ---
|
|
3259
|
+
async hasControllerAccessDefaults (ownerDID, modelID) {
|
|
3260
|
+
let query = db.selectFrom('kubun_graph_access_default_records').select('owner').where('type', '=', 'access.default').where('owner', '=', ownerDID);
|
|
3261
|
+
if (modelID != null) {
|
|
3262
|
+
// Prefix ranges depend on collation ordering; type and owner narrow the primary key scan.
|
|
3263
|
+
const prefix = `access.default:[${JSON.stringify(modelID)},`;
|
|
3264
|
+
query = query.where(sql`substr(id, 1, ${Array.from(prefix).length}) = ${prefix}`);
|
|
3265
|
+
}
|
|
3266
|
+
return await query.limit(1).executeTakeFirst() != null;
|
|
3267
|
+
},
|
|
3268
|
+
async invalidateAccessDefaultProjections (ownerDID) {
|
|
3269
|
+
if (context.provider === context.root) {
|
|
3270
|
+
return context.root.withTransaction(async (tx)=>(await tx.getStore('graph')).invalidateAccessDefaultProjections(ownerDID));
|
|
3271
|
+
}
|
|
3272
|
+
let ownersQuery = db.selectFrom('kubun_graph_access_default_records').where('type', '=', 'access.default').select('owner').distinct();
|
|
3273
|
+
if (ownerDID != null) ownersQuery = ownersQuery.where('owner', '=', ownerDID);
|
|
3274
|
+
const owners = (await ownersQuery.execute()).map(({ owner })=>owner);
|
|
3275
|
+
if (owners.length === 0) return;
|
|
3276
|
+
await api.accessDefaultRecords().lock?.(owners.map((owner)=>({
|
|
3277
|
+
type: 'access.default',
|
|
3278
|
+
key: `!projection:${owner}`
|
|
3279
|
+
})));
|
|
3280
|
+
const rows = await db.selectFrom('kubun_graph_user_model_access_defaults').select([
|
|
3281
|
+
'model_id',
|
|
3282
|
+
'permission_type'
|
|
3283
|
+
]).where('owner_did', 'in', owners).execute();
|
|
3284
|
+
// Signed slots survive invalidation so a successful repair can restore the winner.
|
|
3285
|
+
await db.deleteFrom('kubun_graph_user_model_access_defaults').where('owner_did', 'in', owners).execute();
|
|
3286
|
+
scheduleScopeModels(context, rows.filter((row)=>row.permission_type === 'read').map((row)=>row.model_id));
|
|
3287
|
+
},
|
|
1170
3288
|
async getUserModelAccessDefault (ownerDID, modelID, permissionType) {
|
|
1171
3289
|
const result = await db.selectFrom('kubun_graph_user_model_access_defaults').select([
|
|
1172
3290
|
'access_level',
|
|
@@ -1244,8 +3362,14 @@ export function createGraphStore(db, adapter) {
|
|
|
1244
3362
|
'permission_type'
|
|
1245
3363
|
]).doUpdateSet(updateValues);
|
|
1246
3364
|
}).execute();
|
|
3365
|
+
if (params.permissionType === 'read') {
|
|
3366
|
+
scheduleScopeModels(context, [
|
|
3367
|
+
params.modelID
|
|
3368
|
+
]);
|
|
3369
|
+
}
|
|
1247
3370
|
},
|
|
1248
|
-
async removeUserModelAccessDefaults (
|
|
3371
|
+
async removeUserModelAccessDefaults (params) {
|
|
3372
|
+
const { ownerDID, modelID, permissionTypes, hlc } = params;
|
|
1249
3373
|
// Tombstone rather than delete: retain the row with rule fields nulled,
|
|
1250
3374
|
// `removed` set, and `hlc` stamped so it stays the LWW anchor. Deleting
|
|
1251
3375
|
// would erase the anchor and let a stale set revive the rule.
|
|
@@ -1287,9 +3411,15 @@ export function createGraphStore(db, adapter) {
|
|
|
1287
3411
|
removed: 1,
|
|
1288
3412
|
updated_at: updatedAt
|
|
1289
3413
|
})).execute();
|
|
3414
|
+
if (permissionTypes.includes('read')) {
|
|
3415
|
+
scheduleScopeModels(context, [
|
|
3416
|
+
modelID
|
|
3417
|
+
]);
|
|
3418
|
+
}
|
|
1290
3419
|
},
|
|
1291
3420
|
// --- Document scope / metadata helpers ---
|
|
1292
|
-
async getDocumentIDsForScope (
|
|
3421
|
+
async getDocumentIDsForScope (params) {
|
|
3422
|
+
const { scopes, excludedDocumentIDs = [], limit, groupRead } = params;
|
|
1293
3423
|
// Group by model so each model's table is hit once with every owner it was
|
|
1294
3424
|
// asked for, rather than once per (model, owner) pair.
|
|
1295
3425
|
const ownersByModel = new Map();
|
|
@@ -1303,12 +3433,12 @@ export function createGraphStore(db, adapter) {
|
|
|
1303
3433
|
owners.add(scope.ownerDID);
|
|
1304
3434
|
}
|
|
1305
3435
|
}
|
|
1306
|
-
|
|
1307
|
-
// `ownersByModel` order rather than completion order — the returned array
|
|
1308
|
-
// feeds merkle-tree construction, which must not vary run to run.
|
|
1309
|
-
const idsByModel = await Promise.all(Array.from(ownersByModel, async ([modelID, owners])=>{
|
|
3436
|
+
const readModel = async (modelID, owners, modelLimit)=>{
|
|
1310
3437
|
try {
|
|
1311
|
-
|
|
3438
|
+
let query = db.selectFrom(`k_${modelID}`).select('id').where('owner', 'in', Array.from(owners));
|
|
3439
|
+
if (groupRead != null) query = query.where(adapter.groupReadPredicate(sql.ref('field_values'), groupRead));
|
|
3440
|
+
if (modelLimit != null) query = query.limit(modelLimit);
|
|
3441
|
+
const docs = await query.execute();
|
|
1312
3442
|
return docs.map((d)=>d.id);
|
|
1313
3443
|
} catch (err) {
|
|
1314
3444
|
// A model with no table contributes nothing — one missing table
|
|
@@ -1316,7 +3446,24 @@ export function createGraphStore(db, adapter) {
|
|
|
1316
3446
|
if (!isMissingTableError(err, `k_${modelID}`)) throw err;
|
|
1317
3447
|
return [];
|
|
1318
3448
|
}
|
|
1319
|
-
}
|
|
3449
|
+
};
|
|
3450
|
+
let idsByModel;
|
|
3451
|
+
if (limit == null) {
|
|
3452
|
+
// Disjoint tables, so the reads go out together. Flattened in
|
|
3453
|
+
// `ownersByModel` order rather than completion order — the returned array
|
|
3454
|
+
// feeds merkle-tree construction, which must not vary run to run.
|
|
3455
|
+
idsByModel = await Promise.all(Array.from(ownersByModel, ([modelID, owners])=>readModel(modelID, owners)));
|
|
3456
|
+
} else {
|
|
3457
|
+
// A total limit: each model reads only what is left of it.
|
|
3458
|
+
idsByModel = [];
|
|
3459
|
+
let remaining = limit;
|
|
3460
|
+
for (const [modelID, owners] of ownersByModel){
|
|
3461
|
+
if (remaining <= 0) break;
|
|
3462
|
+
const ids = await readModel(modelID, owners, remaining);
|
|
3463
|
+
idsByModel.push(ids);
|
|
3464
|
+
remaining -= ids.length;
|
|
3465
|
+
}
|
|
3466
|
+
}
|
|
1320
3467
|
const docIDs = idsByModel.flat();
|
|
1321
3468
|
if (excludedDocumentIDs.length > 0) {
|
|
1322
3469
|
const excluded = new Set(excludedDocumentIDs);
|
|
@@ -1324,6 +3471,70 @@ export function createGraphStore(db, adapter) {
|
|
|
1324
3471
|
}
|
|
1325
3472
|
return docIDs;
|
|
1326
3473
|
},
|
|
3474
|
+
async listDocumentLeafPage (params) {
|
|
3475
|
+
if (params.limit <= 0 || params.rules.length === 0) return [];
|
|
3476
|
+
const rulesByModel = new Map();
|
|
3477
|
+
for (const rule of params.rules){
|
|
3478
|
+
const rules = rulesByModel.get(rule.modelID);
|
|
3479
|
+
if (rules == null) rulesByModel.set(rule.modelID, [
|
|
3480
|
+
rule
|
|
3481
|
+
]);
|
|
3482
|
+
else rules.push(rule);
|
|
3483
|
+
}
|
|
3484
|
+
const postgres = adapter instanceof AbstractPostgresAdapter;
|
|
3485
|
+
const leaves = [];
|
|
3486
|
+
for (const [modelID, rules] of rulesByModel){
|
|
3487
|
+
let rows;
|
|
3488
|
+
try {
|
|
3489
|
+
rows = (await documentLeafPageStatement({
|
|
3490
|
+
modelID,
|
|
3491
|
+
rules,
|
|
3492
|
+
after: params.after,
|
|
3493
|
+
through: params.through ?? null,
|
|
3494
|
+
limit: params.limit,
|
|
3495
|
+
postgres,
|
|
3496
|
+
...params.groupRead == null ? {} : {
|
|
3497
|
+
groupReadPredicate: adapter.groupReadPredicate(sql.ref('k.field_values'), params.groupRead)
|
|
3498
|
+
}
|
|
3499
|
+
}).execute(db)).rows;
|
|
3500
|
+
} catch (err) {
|
|
3501
|
+
if (!isMissingTableError(err, `k_${modelID}`)) throw err;
|
|
3502
|
+
continue;
|
|
3503
|
+
}
|
|
3504
|
+
let current;
|
|
3505
|
+
for (const row of rows){
|
|
3506
|
+
if (current == null || current.documentID !== row.id) {
|
|
3507
|
+
current = {
|
|
3508
|
+
documentID: row.id,
|
|
3509
|
+
owner: row.owner,
|
|
3510
|
+
authoredHere: row.authored_here === true || Number(row.authored_here) === 1,
|
|
3511
|
+
acceptedMutationIDs: []
|
|
3512
|
+
};
|
|
3513
|
+
leaves.push(current);
|
|
3514
|
+
}
|
|
3515
|
+
if (row.mutation_hash != null) current.acceptedMutationIDs.push(row.mutation_hash);
|
|
3516
|
+
}
|
|
3517
|
+
}
|
|
3518
|
+
leaves.sort((a, b)=>a.documentID < b.documentID ? -1 : a.documentID > b.documentID ? 1 : 0);
|
|
3519
|
+
const page = leaves.slice(0, params.limit);
|
|
3520
|
+
for (const leaf of page)leaf.acceptedMutationIDs.sort();
|
|
3521
|
+
return page;
|
|
3522
|
+
},
|
|
3523
|
+
async getAppliedMutationsForDocuments (documentIDs) {
|
|
3524
|
+
const found = new Map();
|
|
3525
|
+
for (const chunk of chunked(documentIDs)){
|
|
3526
|
+
const rows = await db.selectFrom('kubun_graph_mutation_log').selectAll().where('document_id', 'in', chunk).where('status', '=', 'applied').orderBy('hlc', 'asc').execute();
|
|
3527
|
+
for (const row of rows){
|
|
3528
|
+
const entries = found.get(row.document_id) ?? [];
|
|
3529
|
+
entries.push(row);
|
|
3530
|
+
found.set(row.document_id, entries);
|
|
3531
|
+
}
|
|
3532
|
+
}
|
|
3533
|
+
return found;
|
|
3534
|
+
},
|
|
3535
|
+
async getAppliedMutations (documentID) {
|
|
3536
|
+
return await db.selectFrom('kubun_graph_mutation_log').selectAll().where('document_id', '=', documentID).where('status', '=', 'applied').orderBy('hlc', 'asc').execute();
|
|
3537
|
+
},
|
|
1327
3538
|
async getDocumentMetadataForSync (modelID, documentIDs) {
|
|
1328
3539
|
if (documentIDs.length === 0) return [];
|
|
1329
3540
|
const rows = await db.selectFrom(`k_${modelID}`).select([
|