@framers/agentos-ext-account-sources 0.7.3 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -9
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/sync/AccountSourceSync.d.ts +12 -7
- package/dist/sync/AccountSourceSync.d.ts.map +1 -1
- package/dist/sync/AccountSourceSync.js +26 -7
- package/dist/sync/AccountSourceSync.js.map +1 -1
- package/dist/sync/SourceRevocations.d.ts +37 -3
- package/dist/sync/SourceRevocations.d.ts.map +1 -1
- package/dist/sync/SourceRevocations.js +86 -2
- package/dist/sync/SourceRevocations.js.map +1 -1
- package/dist/sync/SqlAccountSourceSyncStore.d.ts +111 -24
- package/dist/sync/SqlAccountSourceSyncStore.d.ts.map +1 -1
- package/dist/sync/SqlAccountSourceSyncStore.js +273 -52
- package/dist/sync/SqlAccountSourceSyncStore.js.map +1 -1
- package/dist/sync/state.d.ts +28 -0
- package/dist/sync/state.d.ts.map +1 -1
- package/dist/sync/state.js +48 -0
- package/dist/sync/state.js.map +1 -1
- package/dist/sync/types.d.ts +13 -1
- package/dist/sync/types.d.ts.map +1 -1
- package/manifest.json +1 -1
- package/package.json +1 -1
|
@@ -6,8 +6,21 @@
|
|
|
6
6
|
*/
|
|
7
7
|
import { createHash, randomUUID } from 'node:crypto';
|
|
8
8
|
import { SqlSourceDerivationStore, createSourceDerivation, } from '@framers/agentos/cognition/library';
|
|
9
|
-
import { createSourceSyncState, transitionSourceSync, sourceSyncDue, openSourceSyncCycle, advanceSourceSyncPage, committableSourceCursor, assertSourceSyncPolicy, SourceSyncFault, } from './state.js';
|
|
9
|
+
import { createSourceSyncState, transitionSourceSync, sourceSyncDue, openSourceSyncCycle, advanceSourceSyncPage, committableSourceCursor, assertSourceSyncPolicy, SourceSyncFault, SOURCE_SYNC_OPERATING_BOUNDS, sourceSyncContentChanges, SourceSyncPolicyTuneError, } from './state.js';
|
|
10
10
|
import { sourceSyncTables } from './schema.js';
|
|
11
|
+
/**
|
|
12
|
+
* Thrown by `expire` when it fails after removing at least one source. Each removal committed in a transaction of its
|
|
13
|
+
* own, so `partial` counts them in `removed`, with the failure itself in `failed`; `cause` holds the failure, whose
|
|
14
|
+
* message this error carries.
|
|
15
|
+
*/
|
|
16
|
+
export class SourceSyncExpiryError extends Error {
|
|
17
|
+
partial;
|
|
18
|
+
constructor(removed, cause) {
|
|
19
|
+
super(cause instanceof Error ? cause.message : String(cause), { cause });
|
|
20
|
+
this.name = 'SourceSyncExpiryError';
|
|
21
|
+
this.partial = { removed, failed: 1 };
|
|
22
|
+
}
|
|
23
|
+
}
|
|
11
24
|
class StaleSync extends Error {
|
|
12
25
|
constructor() { super('stale_source_sync'); }
|
|
13
26
|
}
|
|
@@ -34,6 +47,14 @@ const sameText = (stored, read) => (stored.textHash !== undefined ? stored.textH
|
|
|
34
47
|
&& stored.characters === read.characters && stored.activityAt === read.activityAt
|
|
35
48
|
&& stored.messages.length === read.messages.length
|
|
36
49
|
&& stored.messages.every((message, index) => sameSpan(message, read.messages[index]));
|
|
50
|
+
/**
|
|
51
|
+
* Whether a read begins with a retained copy's text, by the hash the store keeps of it, and holds every message of
|
|
52
|
+
* the copy at the same span, as a thread's read does after a reply: everything the read adds follows the copy's
|
|
53
|
+
* text. A copy an earlier release wrote, which has no text hash, never qualifies.
|
|
54
|
+
*/
|
|
55
|
+
const beginsWith = (stored, read, text) => stored.textHash !== undefined && stored.characters <= read.characters
|
|
56
|
+
&& stored.textHash === textHashOf(text.slice(0, stored.characters))
|
|
57
|
+
&& stored.messages.every((message) => read.messages.some((span) => sameSpan(message, span)));
|
|
37
58
|
/**
|
|
38
59
|
* Whether a read holds exactly the retained copy: its text (`sameText`) at the same version and content hash. Only the
|
|
39
60
|
* expiry and the message windows, which each read decides afresh, may differ.
|
|
@@ -45,9 +66,9 @@ const sameCopy = (stored, read) => sameText(stored, read) && stored.dependency.v
|
|
|
45
66
|
* that owner's fence (`fence`), and a change made with a handle checks its lease, generation and host liveness in
|
|
46
67
|
* the same transaction. Applying a changed source purges its old copy and every derived result that depends on it
|
|
47
68
|
* before the replacement is written. A read that repeats the text of a copy the store holds is never charged again,
|
|
48
|
-
*
|
|
49
|
-
*
|
|
50
|
-
* `sourceDerivationSchema` first.
|
|
69
|
+
* one that begins with that text is charged only for what follows it, and with the host's `refresh` binding a read
|
|
70
|
+
* of that very copy keeps it in place. Provider reads, embedding and model calls stay outside the store. The
|
|
71
|
+
* constructor runs no DDL: the host's migration runs `sourceSyncSchema` and core's `sourceDerivationSchema` first.
|
|
51
72
|
*
|
|
52
73
|
* @typeParam Prepared - The host's in-memory value for one document, passed through to its `write` binding.
|
|
53
74
|
*/
|
|
@@ -495,17 +516,19 @@ export class SqlAccountSourceSyncStore {
|
|
|
495
516
|
}
|
|
496
517
|
/**
|
|
497
518
|
* Applies one read result for a work item taken in this cycle, under a live lease and the owner fence. A retry
|
|
498
|
-
* keeps the work unresolved. A document is weighed first: excluded, outside the retained limits or
|
|
499
|
-
*
|
|
500
|
-
*
|
|
501
|
-
*
|
|
502
|
-
*
|
|
503
|
-
*
|
|
504
|
-
*
|
|
505
|
-
* read
|
|
506
|
-
*
|
|
507
|
-
*
|
|
508
|
-
*
|
|
519
|
+
* keeps the work unresolved. A document is weighed first: excluded, outside the retained limits or over the
|
|
520
|
+
* indexing allowance with the characters it adds to index, it is recorded as a skip with its reason. A document
|
|
521
|
+
* that fits replaces the source's old copy in the same transaction (host write, indexing charge and provenance),
|
|
522
|
+
* after the old copy and every result derived from it are purged, evicting this connection's older sources the
|
|
523
|
+
* limits no longer admit. A read that repeats the text and message spans of a copy the store still holds, within
|
|
524
|
+
* its expiry and window, adds nothing to index and is never charged; one that begins with that copy's text, every
|
|
525
|
+
* message of the copy at the same span, adds only the text that follows and is charged that alone. When a
|
|
526
|
+
* repeating read also repeats the copy's version and content hash, the host's `refresh` binding keeps the copy,
|
|
527
|
+
* with every result derived from it, and gives it the read's expiry and message windows; otherwise the copy is
|
|
528
|
+
* written again from a fresh preparation. A skip and every other result purge the old copy, except an
|
|
529
|
+
* `indexing_limit` skip that finds the copy still held, still admitted as it is by the retained limits and the
|
|
530
|
+
* read beginning with its text, every message of it at the same span: that copy, admitted and charged under an
|
|
531
|
+
* earlier read, stays with the shorter of its own and the read's expiry.
|
|
509
532
|
*
|
|
510
533
|
* @param prepared - The host's prepared value for a document result; null for any other result, and for a document
|
|
511
534
|
* weighed before its preparation.
|
|
@@ -540,11 +563,11 @@ export class SqlAccountSourceSyncStore {
|
|
|
540
563
|
: { kind: 'pending' };
|
|
541
564
|
let victims = [];
|
|
542
565
|
// The old copy stays when the read is that copy and the host refreshes it in place (`refreshed`), or when an
|
|
543
|
-
// indexing skip keeps it (`kept`, with `shortened` its provenance); a read
|
|
544
|
-
//
|
|
566
|
+
// indexing skip keeps it (`kept`, with `shortened` its provenance); a written read is charged the characters it
|
|
567
|
+
// adds to index (`charge`).
|
|
545
568
|
let refreshed = false;
|
|
546
569
|
let kept = false;
|
|
547
|
-
let charge =
|
|
570
|
+
let charge = 0;
|
|
548
571
|
let shortened = null;
|
|
549
572
|
const excluded = await tx.get(`SELECT 1 FROM ${this.t.exclusions}
|
|
550
573
|
WHERE tenant_id=$1 AND owner_id=$2 AND connection_id=$3 AND source_id=$4`, [...params(s), work.sourceId]);
|
|
@@ -560,23 +583,26 @@ export class SqlAccountSourceSyncStore {
|
|
|
560
583
|
const now = this.now();
|
|
561
584
|
const selected = this.choose([...all.filter((d) => d !== old), read], row.policy, maximumDocuments, maximumCharacters, now);
|
|
562
585
|
// A copy still within its expiry and window was charged when it was admitted: a read that repeats its text
|
|
563
|
-
// adds nothing to index, and one that also repeats its version and content hash is that copy. A
|
|
564
|
-
//
|
|
586
|
+
// adds nothing to index, and one that also repeats its version and content hash is that copy. A read that
|
|
587
|
+
// begins with its text adds only what follows it, so a reply to a long thread is charged once, not with the
|
|
588
|
+
// whole thread again. A copy past its expiry or window is no longer held, whether or not `expire` has removed
|
|
589
|
+
// it yet.
|
|
565
590
|
const holding = old !== undefined && retainable(old, row.policy, now) ? old : undefined;
|
|
566
591
|
const repeats = holding !== undefined && sameText(holding, read);
|
|
592
|
+
const text = result.document.read.content.text;
|
|
593
|
+
const appends = holding !== undefined && !repeats && beginsWith(holding, read, text);
|
|
594
|
+
const indexed = repeats ? 0 : appends ? read.characters - holding.characters : read.characters;
|
|
567
595
|
if (!selected.includes(read)) {
|
|
568
596
|
const reason = selected.length >= Math.min(row.policy.maxRetainedDocuments, maximumDocuments)
|
|
569
597
|
? 'document_limit' : 'held_limit';
|
|
570
598
|
disposition = { kind: 'skipped', reason };
|
|
571
599
|
}
|
|
572
|
-
else if (
|
|
600
|
+
else if (indexed > Math.min(row.policy.maxIndexedCharacters, allowance.additionalIndexedCharacters)) {
|
|
573
601
|
disposition = { kind: 'skipped', reason: 'indexing_limit' };
|
|
574
602
|
// The copy stays where the read begins with its text and the retained limits still admit it as it is, so an
|
|
575
603
|
// edit of a message it holds, or a lowered ceiling, removes it.
|
|
576
604
|
kept = holding !== undefined && holding.dependency.generation === read.dependency.generation
|
|
577
|
-
&& holding
|
|
578
|
-
&& holding.textHash === textHashOf(result.document.read.content.text.slice(0, holding.characters))
|
|
579
|
-
&& holding.messages.every((message) => read.messages.some((span) => sameSpan(message, span)))
|
|
605
|
+
&& beginsWith(holding, read, text)
|
|
580
606
|
&& this.choose(all, row.policy, maximumDocuments, maximumCharacters, now).includes(holding);
|
|
581
607
|
if (kept && holding !== undefined) {
|
|
582
608
|
// The kept copy takes the shorter of its own and the read's expiry and message windows, never a longer one.
|
|
@@ -596,7 +622,7 @@ export class SqlAccountSourceSyncStore {
|
|
|
596
622
|
return { handle: h, disposition: pending };
|
|
597
623
|
}
|
|
598
624
|
normalized = read;
|
|
599
|
-
charge =
|
|
625
|
+
charge = indexed;
|
|
600
626
|
victims = all.filter((d) => d !== old && !selected.includes(d)).map((d) => d.dependency.sourceId);
|
|
601
627
|
disposition = { kind: 'applied', version: read.dependency.version };
|
|
602
628
|
}
|
|
@@ -623,8 +649,8 @@ export class SqlAccountSourceSyncStore {
|
|
|
623
649
|
}
|
|
624
650
|
else if (normalized && result.kind === 'document' && prepared !== null) {
|
|
625
651
|
await this.options.bindings.write(tx, s, normalized.dependency, result.document, prepared);
|
|
626
|
-
if (charge)
|
|
627
|
-
await this.options.bindings.chargeIndexed(tx, s,
|
|
652
|
+
if (charge > 0)
|
|
653
|
+
await this.options.bindings.chargeIndexed(tx, s, charge);
|
|
628
654
|
await tx.run(`INSERT INTO ${this.t.documents}(tenant_id,owner_id,connection_id,source_id,document,expires_at)
|
|
629
655
|
VALUES($1,$2,$3,$4,$5::jsonb,$6)`, [...params(s), work.sourceId,
|
|
630
656
|
JSON.stringify(normalized), normalized.dependency.expiresAt]);
|
|
@@ -871,15 +897,34 @@ export class SqlAccountSourceSyncStore {
|
|
|
871
897
|
* the new policy, prunes retained sources to it and shortens their expiry to its window; the other controls rebind
|
|
872
898
|
* content to the new generation without changing its version or expiry.
|
|
873
899
|
*
|
|
874
|
-
*
|
|
875
|
-
*
|
|
876
|
-
*
|
|
900
|
+
* A tune (`policy_tuned`), which takes the new policy too, does none of that. It stores a policy that differs from the
|
|
901
|
+
* stored one only in `SOURCE_SYNC_OPERATING_BOUNDS`, in one transaction under the owner fence's row lock and the
|
|
902
|
+
* connection's: no barrier is recorded and neither the generation nor the owner's epoch moves, so every fence ticket
|
|
903
|
+
* and derived result stays valid, and the cursor, revalidation, progress, schedule and retained sources stay, in any
|
|
904
|
+
* state. The new bounds apply from the next lease. While a lease is live a tune that would change the policy changes
|
|
905
|
+
* nothing and answers null, so no sync sees its policy change; a lease past its time by this store's clock ends with
|
|
906
|
+
* the tune, as a claim would replace it. A cycle left open between leases goes on under the new bounds, except that
|
|
907
|
+
* its scan ends there, as a bounded scan ends, when it has already reached a lower `maxScanPages` or `maxCandidates`,
|
|
908
|
+
* or when the new `scanPageSize` is below the size its next page token was read under (the checkpoint's
|
|
909
|
+
* `pageTokenSize`). A tune stores the whole policy it is given, so of two tunes made from one snapshot the later one's
|
|
910
|
+
* policy stands. A tune to the stored policy answers the snapshot and changes nothing, under a live lease too.
|
|
911
|
+
*
|
|
912
|
+
* @returns The snapshot after the barrier is complete or the tune is stored; null for a tune that would change the
|
|
913
|
+
* policy while a lease is live.
|
|
914
|
+
* @throws {SourceSyncPolicyTuneError} for a tune whose policy differs from the stored one outside the operating
|
|
915
|
+
* bounds; nothing changes.
|
|
916
|
+
* @throws {Error} `source_policy_control_required` unless a policy comes exactly with `policy_changed` or
|
|
917
|
+
* `policy_tuned`, and `stale_source_sync` when the host's `active` binding refuses anything but disconnecting, a
|
|
918
|
+
* barrier is still set (a tune completes none) or the connection does not exist.
|
|
877
919
|
*/
|
|
878
920
|
async control(scope, event, policy) {
|
|
879
921
|
if (policy)
|
|
880
922
|
assertSourceSyncPolicy(policy);
|
|
881
|
-
|
|
923
|
+
const takesPolicy = event.type === 'policy_changed' || event.type === 'policy_tuned';
|
|
924
|
+
if (takesPolicy !== (policy !== undefined))
|
|
882
925
|
throw new Error('source_policy_control_required');
|
|
926
|
+
if (event.type === 'policy_tuned')
|
|
927
|
+
return this.tune(scope, policy);
|
|
883
928
|
await this.recoverControl(scope);
|
|
884
929
|
await this.fence.blockOwner(scope, async (tx) => {
|
|
885
930
|
if (!await this.options.bindings.active(tx, scope) && event.type !== 'disconnect')
|
|
@@ -901,6 +946,63 @@ export class SqlAccountSourceSyncStore {
|
|
|
901
946
|
await this.recoverControl(scope);
|
|
902
947
|
return this.snapshot(scope);
|
|
903
948
|
}
|
|
949
|
+
/**
|
|
950
|
+
* Stores a tuned policy as `control` describes for `policy_tuned`.
|
|
951
|
+
*
|
|
952
|
+
* @returns The snapshot, or null when the policy would change while a lease is live.
|
|
953
|
+
*/
|
|
954
|
+
async tune(scope, policy) {
|
|
955
|
+
const stored = await this.db.transaction(async (tx) => {
|
|
956
|
+
// The owner fence's row lock first, as every store transaction takes it; an authorization that answers false
|
|
957
|
+
// changes nothing, so the owner's epoch, and every ticket and handle captured at it, stays.
|
|
958
|
+
await this.fenceOn(tx).activateOwner(scope, async () => false);
|
|
959
|
+
if (!await this.options.bindings.active(tx, scope))
|
|
960
|
+
throw new StaleSync();
|
|
961
|
+
const row = await this.row(tx, scope);
|
|
962
|
+
if (row.barrier)
|
|
963
|
+
throw new StaleSync();
|
|
964
|
+
const changed = sourceSyncContentChanges(row.policy, policy);
|
|
965
|
+
if (changed.length > 0)
|
|
966
|
+
throw new SourceSyncPolicyTuneError(changed);
|
|
967
|
+
if (SOURCE_SYNC_OPERATING_BOUNDS.every((bound) => row.policy[bound] === policy[bound]))
|
|
968
|
+
return true;
|
|
969
|
+
const now = this.now();
|
|
970
|
+
// A sync runs under the policy it claimed its lease with; a provider may check its requests against the stored one.
|
|
971
|
+
if (row.lease_until !== null && Number(row.lease_until) > now)
|
|
972
|
+
return false;
|
|
973
|
+
const previous = row.policy;
|
|
974
|
+
row.state = transitionSourceSync(row.state, { type: 'policy_tuned' }, now);
|
|
975
|
+
row.policy = policy;
|
|
976
|
+
// A lease past its time by this store's clock ends here, as a claim would replace it: a worker whose clock runs
|
|
977
|
+
// behind then holds a handle no later call accepts. Its next call, and the end of a request it already has in
|
|
978
|
+
// flight, are refused, so neither can be handed on as a lost grant once a provider sees the new policy.
|
|
979
|
+
row.lease_token = null;
|
|
980
|
+
row.lease_until = null;
|
|
981
|
+
const cp = row.checkpoint;
|
|
982
|
+
// The page size the open scan's next token was read under; a checkpoint an earlier release wrote does not keep
|
|
983
|
+
// it, and its token came from the stored policy's size.
|
|
984
|
+
const issued = cp?.pageTokenSize ?? previous.scanPageSize;
|
|
985
|
+
if (cp?.phase === 'scan' && (cp.scanPages >= policy.maxScanPages || cp.candidates >= policy.maxCandidates
|
|
986
|
+
|| (cp.nextPageToken !== null && policy.scanPageSize < issued))) {
|
|
987
|
+
// The open scan has already reached a lower bound, whose next scan page the new bounds would refuse, or holds a
|
|
988
|
+
// page token read under a larger page size, which a provider may answer at that size: it ends as a bounded
|
|
989
|
+
// scan ends, and its next page is history.
|
|
990
|
+
if (!validNumber(cp.revision + 1))
|
|
991
|
+
throw new Error('source_sync_counter_exhausted');
|
|
992
|
+
row.checkpoint = { ...cp, revision: cp.revision + 1, phase: 'history', nextPageToken: null,
|
|
993
|
+
pageTokenSize: undefined, targetCursor: null, bounded: true };
|
|
994
|
+
row.last_page = null;
|
|
995
|
+
}
|
|
996
|
+
else if (cp?.phase === 'scan' && cp.nextPageToken !== null && cp.pageTokenSize === undefined) {
|
|
997
|
+
// The scan stays open on a token whose size its checkpoint does not keep: it keeps that size now, so a later
|
|
998
|
+
// tune compares with it rather than with the policy this tune stores.
|
|
999
|
+
row.checkpoint = { ...cp, pageTokenSize: issued };
|
|
1000
|
+
}
|
|
1001
|
+
await this.save(tx, scope, row);
|
|
1002
|
+
return true;
|
|
1003
|
+
});
|
|
1004
|
+
return stored ? this.snapshot(scope) : null;
|
|
1005
|
+
}
|
|
904
1006
|
async rebind(tx, scope, generation) {
|
|
905
1007
|
await this.options.bindings.rebind(tx, scope, generation);
|
|
906
1008
|
for (const doc of await this.documentRows(tx, scope)) {
|
|
@@ -1020,12 +1122,15 @@ export class SqlAccountSourceSyncStore {
|
|
|
1020
1122
|
}
|
|
1021
1123
|
/**
|
|
1022
1124
|
* Removes retained sources whose expiry has passed, with every result derived from them, also while the connection
|
|
1023
|
-
* is paused, then expires the owner's derived results.
|
|
1024
|
-
* reads refuse an expired source regardless.
|
|
1125
|
+
* is paused or the owner's fence is inactive, then expires the owner's derived results. An inactive owner, such as a
|
|
1126
|
+
* suspended account or one a control's barrier holds, stays inactive; reads refuse an expired source regardless.
|
|
1127
|
+
* Each source goes in a transaction of its own, so a later failure leaves the removals before it in place.
|
|
1025
1128
|
*
|
|
1026
1129
|
* @param limit - At most this many sources, an integer from 1 to 1,000.
|
|
1027
1130
|
* @returns How many sources were removed.
|
|
1028
|
-
* @throws {Error} `invalid_sync_batch` for another limit
|
|
1131
|
+
* @throws {Error} `invalid_sync_batch` for another limit, and the failure itself when a removal or the results'
|
|
1132
|
+
* expiry fails: once a source was removed, a `SourceSyncExpiryError` with the failure's message, the failure as its
|
|
1133
|
+
* `cause` and the removals made before it in `partial`.
|
|
1029
1134
|
*/
|
|
1030
1135
|
async expire(scope, limit = 100) {
|
|
1031
1136
|
if (!Number.isInteger(limit) || limit < 1 || limit > 1000)
|
|
@@ -1034,29 +1139,145 @@ export class SqlAccountSourceSyncStore {
|
|
|
1034
1139
|
WHERE tenant_id=$1 AND owner_id=$2 AND connection_id=$3 AND expires_at <= $4
|
|
1035
1140
|
ORDER BY expires_at,source_id LIMIT $5`, [...params(scope), this.now(), limit]);
|
|
1036
1141
|
let removed = 0;
|
|
1037
|
-
|
|
1038
|
-
const
|
|
1039
|
-
|
|
1142
|
+
try {
|
|
1143
|
+
for (const row of rows)
|
|
1144
|
+
if (await this.expireSource(scope, row.source_id))
|
|
1145
|
+
removed++;
|
|
1146
|
+
await this.fence.expireResults(scope, limit);
|
|
1147
|
+
}
|
|
1148
|
+
catch (error) {
|
|
1149
|
+
// The removals before the failure have committed, so their count goes to the caller with it.
|
|
1150
|
+
throw removed > 0 ? new SourceSyncExpiryError(removed, error) : error;
|
|
1151
|
+
}
|
|
1152
|
+
return removed;
|
|
1153
|
+
}
|
|
1154
|
+
/**
|
|
1155
|
+
* Removes one retained source whose expiry has passed, with every result derived from it, under its owner's row
|
|
1156
|
+
* lock, whether or not the owner's fence is active: the fence's invalidation advances the owner's epoch, so no handle
|
|
1157
|
+
* from before it applies anything, and leaves an inactive owner inactive.
|
|
1158
|
+
*
|
|
1159
|
+
* @returns Whether it was removed: false when it is gone or its expiry has not passed, as after a later read moved it.
|
|
1160
|
+
*/
|
|
1161
|
+
async expireSource(scope, sourceId) {
|
|
1162
|
+
return this.db.transaction(async (tx) => {
|
|
1163
|
+
const fence = this.fenceOn(tx);
|
|
1164
|
+
// The owner fence's row lock first, as every store transaction takes it; an answer of false changes nothing.
|
|
1165
|
+
await fence.activateOwner(scope, async () => false);
|
|
1166
|
+
const current = await tx.get(`SELECT expires_at FROM ${this.t.documents}
|
|
1167
|
+
WHERE tenant_id=$1 AND owner_id=$2 AND connection_id=$3 AND source_id=$4`, [...params(scope), sourceId]);
|
|
1168
|
+
if (!current || Number(current.expires_at) > this.now())
|
|
1169
|
+
return false;
|
|
1170
|
+
await fence.invalidate(scope, { kind: 'source', connectionId: scope.connectionId, sourceId }, async () => { });
|
|
1171
|
+
return true;
|
|
1172
|
+
});
|
|
1173
|
+
}
|
|
1174
|
+
/**
|
|
1175
|
+
* Removes retained sources whose expiry has passed from every connection, or from `owner`'s alone, the earliest
|
|
1176
|
+
* expiry first, each with every result derived from it as `expire` does for one connection, however far a
|
|
1177
|
+
* maintenance walk is from them and whether or not their owner's fence is active; then expires the derived results
|
|
1178
|
+
* of each owner it removed a source of. The store's own expiry decides, which a policy change's prune and an
|
|
1179
|
+
* indexing skip that keeps a copy may have shortened. An owner whose removal fails, as when its host purge keeps
|
|
1180
|
+
* failing, is counted and left out of the call's later reads, so a call with up to `limit` such owners never holds
|
|
1181
|
+
* back another owner's copies. An owner whose derived results then fail to expire is counted too, once; the call
|
|
1182
|
+
* still answers every source it removed, and the other owners' results still expire.
|
|
1183
|
+
*
|
|
1184
|
+
* @param limit - At most this many sources, an integer from 1 to 1,000.
|
|
1185
|
+
* @param owner - The owner whose sources alone are swept, or null for every owner.
|
|
1186
|
+
* @returns The sources removed, and the owners whose removal or whose derived results' expiry failed.
|
|
1187
|
+
* @throws {Error} `invalid_sync_batch` for another limit.
|
|
1188
|
+
*/
|
|
1189
|
+
async sweepExpired(limit = 100, owner = null) {
|
|
1190
|
+
if (!Number.isInteger(limit) || limit < 1 || limit > 1000)
|
|
1191
|
+
throw new Error('invalid_sync_batch');
|
|
1192
|
+
let removed = 0;
|
|
1193
|
+
let failed = 0;
|
|
1194
|
+
const swept = new Map();
|
|
1195
|
+
// The owners whose removal failed, which each later read leaves out.
|
|
1196
|
+
const passed = new Set();
|
|
1197
|
+
const passedTenants = [];
|
|
1198
|
+
const passedOwners = [];
|
|
1199
|
+
// Each read after the first follows a newly failed owner, and the call ends once more than `limit` owners have
|
|
1200
|
+
// failed, so at most `limit` + 2 reads run.
|
|
1201
|
+
for (;;) {
|
|
1202
|
+
const rows = await this.db.all(`SELECT d.tenant_id,d.owner_id,d.connection_id,d.source_id FROM ${this.t.documents} d
|
|
1203
|
+
WHERE d.expires_at <= $1 AND ($2::text IS NULL OR (d.tenant_id=$2 AND d.owner_id=$3))
|
|
1204
|
+
AND NOT EXISTS(SELECT 1 FROM unnest($5::text[],$6::text[]) AS p(tenant_id,owner_id)
|
|
1205
|
+
WHERE p.tenant_id=d.tenant_id AND p.owner_id=d.owner_id)
|
|
1206
|
+
ORDER BY d.expires_at,d.tenant_id,d.owner_id,d.connection_id,d.source_id LIMIT $4`, [this.now(), owner?.tenantId ?? null, owner?.ownerId ?? null, limit - removed, passedTenants, passedOwners]);
|
|
1207
|
+
let passedNow = false;
|
|
1208
|
+
for (const row of rows) {
|
|
1209
|
+
const ownerKey = JSON.stringify([row.tenant_id, row.owner_id]);
|
|
1210
|
+
if (passed.has(ownerKey))
|
|
1211
|
+
continue;
|
|
1212
|
+
try {
|
|
1213
|
+
if (await this.expireSource({ tenantId: row.tenant_id, ownerId: row.owner_id, connectionId: row.connection_id }, row.source_id)) {
|
|
1214
|
+
removed++;
|
|
1215
|
+
swept.set(ownerKey, { tenantId: row.tenant_id, ownerId: row.owner_id });
|
|
1216
|
+
}
|
|
1217
|
+
}
|
|
1218
|
+
catch {
|
|
1219
|
+
failed++;
|
|
1220
|
+
passed.add(ownerKey);
|
|
1221
|
+
passedTenants.push(row.tenant_id);
|
|
1222
|
+
passedOwners.push(row.owner_id);
|
|
1223
|
+
passedNow = true;
|
|
1224
|
+
}
|
|
1225
|
+
}
|
|
1226
|
+
if (!passedNow || removed >= limit || passed.size > limit)
|
|
1040
1227
|
break;
|
|
1228
|
+
}
|
|
1229
|
+
// An owner whose derived results cannot be expired counts once, as a failed removal does; the copies already
|
|
1230
|
+
// removed stay counted, and every other owner's results still expire.
|
|
1231
|
+
for (const [ownerKey, sweptOwner] of swept) {
|
|
1041
1232
|
try {
|
|
1042
|
-
|
|
1043
|
-
const current = await tx.get(`SELECT expires_at FROM ${this.t.documents}
|
|
1044
|
-
WHERE tenant_id=$1 AND owner_id=$2 AND connection_id=$3 AND source_id=$4`, [...params(scope), row.source_id]);
|
|
1045
|
-
if (!current || Number(current.expires_at) > this.now())
|
|
1046
|
-
return false;
|
|
1047
|
-
await this.fenceOn(tx).invalidate(scope, { kind: 'source', connectionId: scope.connectionId, sourceId: row.source_id }, async () => { }, { expected: ticket });
|
|
1048
|
-
return true;
|
|
1049
|
-
});
|
|
1050
|
-
if (result.kind === 'applied' && result.value)
|
|
1051
|
-
removed++;
|
|
1233
|
+
await this.fence.expireResults(sweptOwner, limit);
|
|
1052
1234
|
}
|
|
1053
|
-
catch
|
|
1054
|
-
if (!(
|
|
1055
|
-
|
|
1235
|
+
catch {
|
|
1236
|
+
if (!passed.has(ownerKey))
|
|
1237
|
+
failed++;
|
|
1056
1238
|
}
|
|
1057
1239
|
}
|
|
1058
|
-
|
|
1059
|
-
|
|
1240
|
+
return { removed, failed };
|
|
1241
|
+
}
|
|
1242
|
+
/**
|
|
1243
|
+
* Removes a stopped owner's rows from the store in one transaction: every connection of the owner, with its work
|
|
1244
|
+
* rows, retained-source provenance and exclusions, once each connection is disconnected with no barrier, lease, work
|
|
1245
|
+
* or retained source. `remove` runs first in the same transaction, after the owner fence's row lock and the
|
|
1246
|
+
* connections' row locks, for a lifecycle's own rows; `SourceRevocations.removeOwner` passes its own. The core
|
|
1247
|
+
* fence's owner row and its deletion records stay: they hold no content, keep the owner's epoch ahead of any old
|
|
1248
|
+
* handle, and a restore replays the deletion from them.
|
|
1249
|
+
*
|
|
1250
|
+
* @param remove - The lifecycle's removal in the same transaction; answering false rolls the whole call back.
|
|
1251
|
+
* @returns Whether the owner's rows were removed: false, changing nothing, while a connection of the owner has not
|
|
1252
|
+
* stopped or when `remove` answers false.
|
|
1253
|
+
*/
|
|
1254
|
+
async removeOwner(owner, remove = async () => true) {
|
|
1255
|
+
const where = [owner.tenantId, owner.ownerId];
|
|
1256
|
+
try {
|
|
1257
|
+
await this.db.transaction(async (tx) => {
|
|
1258
|
+
// The owner fence's row lock first, as every store transaction takes it; an answer of false changes nothing.
|
|
1259
|
+
await this.fenceOn(tx).activateOwner(owner, async () => false);
|
|
1260
|
+
const rows = await tx.all(`SELECT * FROM ${this.t.connections}
|
|
1261
|
+
WHERE tenant_id=$1 AND owner_id=$2 ORDER BY connection_id FOR UPDATE`, where);
|
|
1262
|
+
if (rows.some((row) => row.barrier !== null || row.lease_token !== null
|
|
1263
|
+
|| row.state.status !== 'disconnected'))
|
|
1264
|
+
throw new StaleSync();
|
|
1265
|
+
if (await tx.get(`SELECT 1 FROM ${this.t.documents} WHERE tenant_id=$1 AND owner_id=$2 LIMIT 1`, where)
|
|
1266
|
+
|| await tx.get(`SELECT 1 FROM ${this.t.work} WHERE tenant_id=$1 AND owner_id=$2 LIMIT 1`, where)) {
|
|
1267
|
+
throw new StaleSync();
|
|
1268
|
+
}
|
|
1269
|
+
if (!await remove(tx))
|
|
1270
|
+
throw new StaleSync();
|
|
1271
|
+
// The work rows, provenance and exclusions go with their connection (ON DELETE CASCADE).
|
|
1272
|
+
await tx.run(`DELETE FROM ${this.t.connections} WHERE tenant_id=$1 AND owner_id=$2`, where);
|
|
1273
|
+
});
|
|
1274
|
+
return true;
|
|
1275
|
+
}
|
|
1276
|
+
catch (error) {
|
|
1277
|
+
if (error instanceof StaleSync)
|
|
1278
|
+
return false;
|
|
1279
|
+
throw error;
|
|
1280
|
+
}
|
|
1060
1281
|
}
|
|
1061
1282
|
}
|
|
1062
1283
|
//# sourceMappingURL=SqlAccountSourceSyncStore.js.map
|