@nathapp/nax-agent 0.91.7 → 0.91.8-canary.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +8 -0
- package/dist/config/catalog-overrides.d.ts +13 -0
- package/dist/index.d.ts +3 -1
- package/dist/index.js +1 -0
- package/dist/internal/atomic-write.d.ts +22 -7
- package/dist/internal/atomic-write.js +107 -12
- package/dist/internal/pinned-directory-worker.d.ts +52 -0
- package/dist/internal/pinned-directory-worker.js +105 -0
- package/dist/internal/pinned-directory.d.ts +48 -0
- package/dist/internal/pinned-directory.js +168 -0
- package/dist/internal.d.ts +2 -0
- package/dist/internal.js +2 -0
- package/dist/memory/delivery.d.ts +38 -0
- package/dist/memory/delivery.js +28 -0
- package/dist/memory/index-render.d.ts +55 -0
- package/dist/memory/index-render.js +225 -0
- package/dist/memory/index.d.ts +12 -0
- package/dist/memory/index.js +11 -0
- package/dist/memory/local-provider.d.ts +3 -0
- package/dist/memory/local-provider.js +127 -0
- package/dist/memory/mirror-lock.d.ts +109 -0
- package/dist/memory/mirror-lock.js +334 -0
- package/dist/memory/note.d.ts +15 -0
- package/dist/memory/note.js +131 -0
- package/dist/memory/operations.d.ts +123 -0
- package/dist/memory/operations.js +471 -0
- package/dist/memory/outbox.d.ts +35 -0
- package/dist/memory/outbox.js +289 -0
- package/dist/memory/provider-types.d.ts +50 -0
- package/dist/memory/provider-types.js +1 -0
- package/dist/memory/records.d.ts +41 -0
- package/dist/memory/records.js +93 -0
- package/dist/memory/refusal.d.ts +36 -0
- package/dist/memory/refusal.js +25 -0
- package/dist/memory/search.d.ts +17 -0
- package/dist/memory/search.js +112 -0
- package/dist/memory/store.d.ts +106 -0
- package/dist/memory/store.js +459 -0
- package/dist/memory/support.d.ts +63 -0
- package/dist/memory/support.js +98 -0
- package/dist/memory/sync-flight.d.ts +25 -0
- package/dist/memory/sync-flight.js +65 -0
- package/dist/memory/sync.d.ts +82 -0
- package/dist/memory/sync.js +344 -0
- package/dist/memory/types.d.ts +108 -0
- package/dist/memory/types.js +51 -0
- package/dist/memory/validate.d.ts +31 -0
- package/dist/memory/validate.js +104 -0
- package/dist/native/adapter-deps.d.ts +9 -0
- package/dist/native/adapter-deps.js +9 -0
- package/dist/native/models.js +3 -0
- package/dist/native/session/compaction.d.ts +15 -1
- package/dist/native/session/compaction.js +90 -23
- package/dist/native/session/manual-compaction.d.ts +57 -0
- package/dist/native/session/manual-compaction.js +72 -0
- package/dist/native/session/session-summarizer.d.ts +75 -0
- package/dist/native/session/session-summarizer.js +73 -0
- package/dist/native/session/transcript-identity.d.ts +12 -0
- package/dist/native/session/transcript-identity.js +16 -0
- package/dist/native/session/turn-compaction-step.d.ts +30 -2
- package/dist/native/session/turn-compaction-step.js +7 -1
- package/dist/native/session/turn-loop-round-trip.js +71 -17
- package/dist/native/session/turn-loop.js +2 -6
- package/dist/native/session-adapter.d.ts +19 -0
- package/dist/native/session-adapter.js +42 -64
- package/dist/sandbox/srt-backend.js +9 -16
- package/dist/session/agent-session-options.js +1 -0
- package/dist/session/agent-session-turn.d.ts +6 -0
- package/dist/session/agent-session-turn.js +5 -4
- package/dist/session/agent-session-types.d.ts +38 -0
- package/dist/session/agent-session.d.ts +8 -0
- package/dist/session/agent-session.js +163 -20
- package/dist/session/native-backend.js +8 -0
- package/dist/session/session-backend.d.ts +19 -1
- package/dist/session/session-types.d.ts +6 -0
- package/dist/session/steer-channel.d.ts +36 -0
- package/dist/session/steer-channel.js +39 -0
- package/dist/session-store/index.d.ts +1 -1
- package/dist/session-store/index.js +1 -1
- package/dist/session-store/session-lock.d.ts +2 -2
- package/dist/session-store/session-lock.js +4 -2
- package/dist/tools/ask-request.d.ts +8 -0
- package/dist/tools/ask-request.js +34 -1
- package/dist/tools/provider-adapt.js +5 -1
- package/dist/tools/provider-types.d.ts +7 -0
- package/dist/tools/types.d.ts +16 -0
- package/package.json +4 -4
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
import { errorMessage } from "#src/infra/errors";
|
|
2
|
+
import { NaxError } from "#src/infra/nax-error";
|
|
3
|
+
import { MEMORY_PROBE_TIMEOUT_MS } from "./types.js";
|
|
4
|
+
/** The stage every memory-origin log line carries. */
|
|
5
|
+
const ERROR_STAGE = "memory";
|
|
6
|
+
function isArmedTimer(id) {
|
|
7
|
+
return typeof id === "object" && id !== null && "handle" in id;
|
|
8
|
+
}
|
|
9
|
+
/**
|
|
10
|
+
* The timers an operation races against when the caller injects none: the process
|
|
11
|
+
* globals, read at call time so a substituted global is still observed. The handle
|
|
12
|
+
* is handed back boxed — `MemoryTimers` keeps the timer id opaque precisely so a
|
|
13
|
+
* fake clock can stand in for the real one, and a box clears the handle this race
|
|
14
|
+
* armed without ever casting one.
|
|
15
|
+
*/
|
|
16
|
+
export const GLOBAL_MEMORY_TIMERS = {
|
|
17
|
+
setTimeout: (callback, ms) => ({ handle: globalThis.setTimeout(callback, ms) }),
|
|
18
|
+
clearTimeout: (id) => {
|
|
19
|
+
if (isArmedTimer(id))
|
|
20
|
+
globalThis.clearTimeout(id.handle);
|
|
21
|
+
},
|
|
22
|
+
};
|
|
23
|
+
function timedOut(ms) {
|
|
24
|
+
return new NaxError(`memory operation did not settle within ${ms}ms`, "MEMORY_TIMEOUT", {
|
|
25
|
+
stage: ERROR_STAGE,
|
|
26
|
+
timeoutMs: ms,
|
|
27
|
+
});
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* `promise`'s value if it settles within `ms`, and a `MEMORY_TIMEOUT` rejection if it does not.
|
|
31
|
+
*
|
|
32
|
+
* The timer is armed when the race starts and cleared the moment either side
|
|
33
|
+
* settles, so a race that finished leaves nothing armed behind it — the property
|
|
34
|
+
* a fake clock reads as `pending() === 0`. `setTimeout` is the cancellable kind
|
|
35
|
+
* here on purpose (the `forbidden-patterns-source` exception): `Bun.sleep` cannot
|
|
36
|
+
* be cancelled, and a sleep that outlives the answer is exactly the leak this
|
|
37
|
+
* clears.
|
|
38
|
+
*
|
|
39
|
+
* A timer race does not cancel the work it raced: the provider call keeps running
|
|
40
|
+
* after a timeout, because nothing on that side of the seam accepts a signal. Its
|
|
41
|
+
* late rejection is therefore claimed here rather than left to surface as an
|
|
42
|
+
* unhandled rejection in whatever process this library runs in.
|
|
43
|
+
*/
|
|
44
|
+
export async function raceTimeout(promise, ms, timers) {
|
|
45
|
+
promise.catch(() => undefined);
|
|
46
|
+
const deadline = Promise.withResolvers();
|
|
47
|
+
const armed = timers.setTimeout(() => deadline.reject(timedOut(ms)), ms);
|
|
48
|
+
try {
|
|
49
|
+
return await Promise.race([promise, deadline.promise]);
|
|
50
|
+
}
|
|
51
|
+
finally {
|
|
52
|
+
timers.clearTimeout(armed);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
/** Ask the provider whether it answers within the probe deadline: a throw and a timeout are `false`. */
|
|
56
|
+
export async function providerAnswers(provider, timers) {
|
|
57
|
+
try {
|
|
58
|
+
return await raceTimeout(provider.reachable(), MEMORY_PROBE_TIMEOUT_MS, timers);
|
|
59
|
+
}
|
|
60
|
+
catch {
|
|
61
|
+
// A provider that threw and one that never answered are the same fact here:
|
|
62
|
+
// neither can be delivered to.
|
|
63
|
+
return false;
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* Report a failure without letting the reporting fail the caller.
|
|
68
|
+
*
|
|
69
|
+
* The logger is an input, not a guarantee: a host logger that throws would
|
|
70
|
+
* otherwise turn the never-throws contract into a rejection at exactly the
|
|
71
|
+
* moment a result is owed, so a logger failure is discarded on purpose. Shared
|
|
72
|
+
* with the save path, whose rollback is also reported rather than thrown.
|
|
73
|
+
*/
|
|
74
|
+
export function warnQuietly(logger, message, data) {
|
|
75
|
+
try {
|
|
76
|
+
logger.warn(ERROR_STAGE, message, data);
|
|
77
|
+
}
|
|
78
|
+
catch {
|
|
79
|
+
// Nothing left to report it to: the outcome the caller is owed still returns.
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* Run one bookkeeping step whose failure must not become the caller's answer.
|
|
84
|
+
*
|
|
85
|
+
* A provider's refusal is the error the caller is owed, and what follows it — taking the
|
|
86
|
+
* refused write back off the queue, putting back the note the mirror held — is a consequence
|
|
87
|
+
* of that error, not an alternative to it. A step that throws is reported and the rest still
|
|
88
|
+
* runs: a mirror left missing a note it holds, because a queue file could not be removed, is
|
|
89
|
+
* worse than an entry surviving to be refused again.
|
|
90
|
+
*/
|
|
91
|
+
export async function bookkeeping(logger, message, data, step) {
|
|
92
|
+
try {
|
|
93
|
+
await step();
|
|
94
|
+
}
|
|
95
|
+
catch (error) {
|
|
96
|
+
warnQuietly(logger, message, { ...data, error: errorMessage(error) });
|
|
97
|
+
}
|
|
98
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Claim the mirror at `lockPath` for a sync in this process: `false` when one already
|
|
3
|
+
* holds it, in which case the caller has nothing to do.
|
|
4
|
+
*/
|
|
5
|
+
export declare function claimSync(lockPath: string): boolean;
|
|
6
|
+
/** Give the claim back. A claim left behind is a mirror this process can never sync again. */
|
|
7
|
+
export declare function releaseSync(lockPath: string): void;
|
|
8
|
+
/** Whether this process has a sync in flight for the mirror at `lockPath`. */
|
|
9
|
+
export declare function syncInFlight(lockPath: string): boolean;
|
|
10
|
+
/**
|
|
11
|
+
* Claim the mirror at `lockPath` for as long as a write operation of this process owns its
|
|
12
|
+
* lock file. Claims nest rather than contend — the operations of one mirror are ordered by
|
|
13
|
+
* the per-store chain before they reach a lock — so this records ownership, it does not win it.
|
|
14
|
+
*/
|
|
15
|
+
export declare function claimOperation(lockPath: string): void;
|
|
16
|
+
/** Give the claim back, whether the operation delivered, refused or threw. */
|
|
17
|
+
export declare function releaseOperation(lockPath: string): void;
|
|
18
|
+
/**
|
|
19
|
+
* Whether a write operation of this process owns the mirror at `lockPath` right now.
|
|
20
|
+
*
|
|
21
|
+
* A sync reads this before it asks the lock protocol for the file: the protocol would see our
|
|
22
|
+
* own pid, call it `own`, and rename a running approval's lock aside in the middle of its
|
|
23
|
+
* read-modify-write.
|
|
24
|
+
*/
|
|
25
|
+
export declare function operationInFlight(lockPath: string): boolean;
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Which mirrors this process owns right now, by lock path — and by which kind of owner.
|
|
3
|
+
*
|
|
4
|
+
* `acquireSessionLock` is a cross-process protocol, and it classifies the caller's own pid
|
|
5
|
+
* as `own` rather than `live` — so it cannot tell a second caller in this process from the
|
|
6
|
+
* first, and a second acquisition takes the file over from the first. Two callers in one
|
|
7
|
+
* process therefore need a guard of their own, and this module is it: a sync claims its mirror
|
|
8
|
+
* before its first `await` (a claim taken after a yield is one two callers both pass) and gives
|
|
9
|
+
* it back in its `finally`, and a write operation claims it for as long as it holds the lock
|
|
10
|
+
* file.
|
|
11
|
+
*
|
|
12
|
+
* The two claims are kept apart because they answer different questions, and both readers ask
|
|
13
|
+
* both. A lock file naming this very pid is normally a sync that is still writing the mirror,
|
|
14
|
+
* and a save must not take it over; a file naming this pid with no claim here is a release that
|
|
15
|
+
* failed, which the next save reclaims so a leaked lock cannot wedge every later save. That is
|
|
16
|
+
* why the sync claim is read by the operations. The reverse matters just as much: an approval
|
|
17
|
+
* holds the lock file for a read-modify-write, and a sync that starts while it does would be
|
|
18
|
+
* handed that same file — `own` pid — so the operation claim is what the sync reads to refuse.
|
|
19
|
+
* Without it, two writers of this process run on one mirror at once.
|
|
20
|
+
*
|
|
21
|
+
* Both facts belong to one registry, which is why it lives beside neither caller.
|
|
22
|
+
*/
|
|
23
|
+
const syncs = new Set();
|
|
24
|
+
/** The write operations of this process — saves, forgets and approvals — holding their mirror. */
|
|
25
|
+
const operations = new Set();
|
|
26
|
+
/**
|
|
27
|
+
* Claim the mirror at `lockPath` for a sync in this process: `false` when one already
|
|
28
|
+
* holds it, in which case the caller has nothing to do.
|
|
29
|
+
*/
|
|
30
|
+
export function claimSync(lockPath) {
|
|
31
|
+
if (syncs.has(lockPath))
|
|
32
|
+
return false;
|
|
33
|
+
syncs.add(lockPath);
|
|
34
|
+
return true;
|
|
35
|
+
}
|
|
36
|
+
/** Give the claim back. A claim left behind is a mirror this process can never sync again. */
|
|
37
|
+
export function releaseSync(lockPath) {
|
|
38
|
+
syncs.delete(lockPath);
|
|
39
|
+
}
|
|
40
|
+
/** Whether this process has a sync in flight for the mirror at `lockPath`. */
|
|
41
|
+
export function syncInFlight(lockPath) {
|
|
42
|
+
return syncs.has(lockPath);
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Claim the mirror at `lockPath` for as long as a write operation of this process owns its
|
|
46
|
+
* lock file. Claims nest rather than contend — the operations of one mirror are ordered by
|
|
47
|
+
* the per-store chain before they reach a lock — so this records ownership, it does not win it.
|
|
48
|
+
*/
|
|
49
|
+
export function claimOperation(lockPath) {
|
|
50
|
+
operations.add(lockPath);
|
|
51
|
+
}
|
|
52
|
+
/** Give the claim back, whether the operation delivered, refused or threw. */
|
|
53
|
+
export function releaseOperation(lockPath) {
|
|
54
|
+
operations.delete(lockPath);
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* Whether a write operation of this process owns the mirror at `lockPath` right now.
|
|
58
|
+
*
|
|
59
|
+
* A sync reads this before it asks the lock protocol for the file: the protocol would see our
|
|
60
|
+
* own pid, call it `own`, and rename a running approval's lock aside in the middle of its
|
|
61
|
+
* read-modify-write.
|
|
62
|
+
*/
|
|
63
|
+
export function operationInFlight(lockPath) {
|
|
64
|
+
return operations.has(lockPath);
|
|
65
|
+
}
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* One sync: the guarded lifecycle around the mirror's exchange with a provider.
|
|
3
|
+
*
|
|
4
|
+
* Two guards, because ownership has two shapes. The lock files are the cross-process ones —
|
|
5
|
+
* one for the project mirror and, when the store has a user directory, one for the user mirror
|
|
6
|
+
* every project shares. `acquireSessionLock` classifies the caller's own pid as `own` — it
|
|
7
|
+
* cannot tell a second caller in this process from the first — so a module-level in-flight set
|
|
8
|
+
* keyed by the lock paths is what stops the same process syncing twice at once. Both are taken
|
|
9
|
+
* before the first `await`: a check that yields first is a check two callers pass.
|
|
10
|
+
*
|
|
11
|
+
* `syncMemory` never throws and never rejects. A live holder is `locked`, an
|
|
12
|
+
* unreachable provider is `unreachable`, anything unexpected is `failed` with
|
|
13
|
+
* its message — and the lock and the in-flight key are released on every path,
|
|
14
|
+
* including the failing ones, because a guard left behind is a store that can
|
|
15
|
+
* never sync again. A cleanup that fails is logged, never thrown: it must not
|
|
16
|
+
* turn a reportable outcome into a rejection.
|
|
17
|
+
*
|
|
18
|
+
* Step 2 asks the provider whether it answers, and asks it through
|
|
19
|
+
* `raceTimeout`: an answer that never arrives must not hold the run open, so the
|
|
20
|
+
* probe is bounded by `MEMORY_PROBE_TIMEOUT_MS` on the injected timers. `false`,
|
|
21
|
+
* a throw and a timeout are one outcome — `unreachable` — and it is recorded in
|
|
22
|
+
* `sync.json` over the cursors a failed probe learned nothing about.
|
|
23
|
+
*
|
|
24
|
+
* Step 3 delivers what the agent already wrote. Each `queued` entry is pushed in
|
|
25
|
+
* `seq` order through the same kind of race, and the provider's answer is applied
|
|
26
|
+
* to the mirror before the entry is removed: a `save` carries the returned id and
|
|
27
|
+
* status into its note, a `forget` removes the note only when the provider says it
|
|
28
|
+
* is `deleted`. The two failures are not the same fact. `MEMORY_PUSH_REJECTED` is
|
|
29
|
+
* permanent — retrying would repeat it — so the entry is marked with the reason and
|
|
30
|
+
* the drain continues. Anything else, a timeout included, is transient: the drain
|
|
31
|
+
* stops, the entry stays `queued` for the next run, `sync.json` records why, and
|
|
32
|
+
* the run ends `failed` — after the index is refreshed, so a forget still waiting
|
|
33
|
+
* is visible as `(forget pending)`.
|
|
34
|
+
*
|
|
35
|
+
* Step 4 asks the provider for what changed elsewhere, one page at a time. The cursor
|
|
36
|
+
* each page hands back is checkpointed before the next page is asked for, so the disk
|
|
37
|
+
* always names the last page that arrived: a page that throws, times out or never comes
|
|
38
|
+
* leaves the run resuming there rather than from the beginning. `done` is the provider's
|
|
39
|
+
* answer about the sequence, not a budget, so a sequence that ends on the last page one
|
|
40
|
+
* run may drain has finished; only one with more behind it is a page-limit failure.
|
|
41
|
+
*
|
|
42
|
+
* What a page *means* is decided one entry at a time. A note the schema refuses — a slug
|
|
43
|
+
* outside `MEMORY_SLUG_RE`, a field missing or over its model limit — is skipped and
|
|
44
|
+
* reported once, never thrown: one unreadable entry must not cost the page its cursor. A
|
|
45
|
+
* key the outbox still holds is left exactly as the local mirror has it and is not counted,
|
|
46
|
+
* because the local write is the newer fact. A tombstone deletes. `decisions` replace
|
|
47
|
+
* `decisions.jsonl` on a project page only. The held set is read after each page, and the
|
|
48
|
+
* page's scope — not the payload's `scope` field — decides where a note lands.
|
|
49
|
+
*/
|
|
50
|
+
import type { AgentLogger } from "#src/infra/agent-logger";
|
|
51
|
+
import type { MemoryOutbox } from "./outbox.ts";
|
|
52
|
+
import type { MemoryProvider } from "./provider-types.ts";
|
|
53
|
+
import type { MemoryStore } from "./store.ts";
|
|
54
|
+
import { type MemoryTimers } from "./types.ts";
|
|
55
|
+
/** Everything one sync needs: the mirror, the queue and the provider it talks to. */
|
|
56
|
+
export interface MemorySyncInput {
|
|
57
|
+
readonly store: MemoryStore;
|
|
58
|
+
readonly outbox: MemoryOutbox;
|
|
59
|
+
readonly provider: MemoryProvider;
|
|
60
|
+
readonly logger: AgentLogger;
|
|
61
|
+
/** The clock the lock protocol stamps with; default `() => new Date()`. */
|
|
62
|
+
readonly now?: () => Date;
|
|
63
|
+
/** The timer seam the step timeouts race against; default the globals. */
|
|
64
|
+
readonly timers?: MemoryTimers;
|
|
65
|
+
/** The pid this process claims in the lock; default `process.pid`. */
|
|
66
|
+
readonly pid?: number;
|
|
67
|
+
/** The liveness probe a lock holder is judged by; default `isProcessAlive`. */
|
|
68
|
+
readonly isAlive?: (pid: number) => boolean;
|
|
69
|
+
/** How many notes one pull asks for; default `MEMORY_PULL_PAGE_SIZE`. */
|
|
70
|
+
readonly pageSize?: number;
|
|
71
|
+
}
|
|
72
|
+
/** What one run did, and how it ended. */
|
|
73
|
+
export interface MemorySyncResult {
|
|
74
|
+
readonly outcome: "synced" | "unreachable" | "locked" | "failed";
|
|
75
|
+
readonly pushed: number;
|
|
76
|
+
readonly rejected: number;
|
|
77
|
+
readonly pulled: number;
|
|
78
|
+
readonly skipped: number;
|
|
79
|
+
/** The message of the failure that ended the run, when it ended in one. */
|
|
80
|
+
readonly error?: string;
|
|
81
|
+
}
|
|
82
|
+
export declare function syncMemory(input: MemorySyncInput): Promise<MemorySyncResult>;
|
|
@@ -0,0 +1,344 @@
|
|
|
1
|
+
import { errorMessage } from "#src/infra/errors";
|
|
2
|
+
import { applyPushed } from "./delivery.js";
|
|
3
|
+
import { refreshMemoryIndex } from "./index-render.js";
|
|
4
|
+
import { acquireMirrorLocks, storeLockTargets } from "./mirror-lock.js";
|
|
5
|
+
import { isMemoryPushRejected, parseMemoryNoteRecord } from "./note.js";
|
|
6
|
+
import { GLOBAL_MEMORY_TIMERS, providerAnswers, raceTimeout, warnQuietly } from "./support.js";
|
|
7
|
+
import { claimSync, releaseSync } from "./sync-flight.js";
|
|
8
|
+
import { MEMORY_OP_TIMEOUT_MS, MEMORY_PULL_MAX_PAGES, MEMORY_PULL_PAGE_SIZE, MEMORY_SLUG_RE, } from "./types.js";
|
|
9
|
+
/** What `sync.json` records when the provider did not answer this run. */
|
|
10
|
+
const UNREACHABLE = "unreachable";
|
|
11
|
+
/** What `sync.json` records when a provider had more pages than one run may drain. */
|
|
12
|
+
const PAGE_LIMIT_REACHED = "pull page limit reached";
|
|
13
|
+
/** What a run holds when it took no lock: releasing nothing is releasing successfully. */
|
|
14
|
+
const NO_RELEASE = async () => undefined;
|
|
15
|
+
function noCounts() {
|
|
16
|
+
return { pushed: 0, rejected: 0, pulled: 0, skipped: 0 };
|
|
17
|
+
}
|
|
18
|
+
/** A run that stopped before it moved anything. */
|
|
19
|
+
function stopped(outcome, error) {
|
|
20
|
+
return { outcome, ...noCounts(), ...(error === undefined ? {} : { error }) };
|
|
21
|
+
}
|
|
22
|
+
/** The clock a run stamps with: the caller's, or the process clock. */
|
|
23
|
+
function clockOf(input) {
|
|
24
|
+
return input.now ?? (() => new Date());
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* Take every lock this store's mirrors need, or `null` when another writer holds one.
|
|
28
|
+
*
|
|
29
|
+
* The set is what a store with a user directory needs: the project's lock alone leaves two
|
|
30
|
+
* projects staging the same user note at once. `acquireMirrorLocks` takes them in the one
|
|
31
|
+
* canonical order, maps a live holder to `null` and hands back what it already took.
|
|
32
|
+
*/
|
|
33
|
+
async function takeLock(input) {
|
|
34
|
+
return await acquireMirrorLocks(input, storeLockTargets(input.store));
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Step 2's failing arm: record why this run could not reach the provider, and stop.
|
|
38
|
+
*
|
|
39
|
+
* Only `lastError` changes — a probe that failed learned nothing about the cursors
|
|
40
|
+
* or the last pull, so whatever the previous sync left carries over untouched — and
|
|
41
|
+
* the outbox is not read, so the queued writes stay exactly as they were.
|
|
42
|
+
*/
|
|
43
|
+
async function recordUnreachable(input) {
|
|
44
|
+
await recordLastError(input, UNREACHABLE);
|
|
45
|
+
return stopped("unreachable");
|
|
46
|
+
}
|
|
47
|
+
/** Note why a run could not finish, leaving every other field of the state as it was. */
|
|
48
|
+
async function recordLastError(input, message) {
|
|
49
|
+
const state = await input.store.readSyncState();
|
|
50
|
+
await input.store.writeSyncState({ ...state, lastError: message });
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* The write an entry stands for, as the provider takes it.
|
|
54
|
+
*
|
|
55
|
+
* The entry carries the queue's own bookkeeping — its sequence, when it was taken,
|
|
56
|
+
* whether it is still queued — and none of that is the provider's to read, so the
|
|
57
|
+
* request is projected from the fields the two shapes share rather than forwarded.
|
|
58
|
+
*/
|
|
59
|
+
function requestOf(entry) {
|
|
60
|
+
return {
|
|
61
|
+
kind: entry.kind,
|
|
62
|
+
clientId: entry.clientId,
|
|
63
|
+
scope: entry.scope,
|
|
64
|
+
slug: entry.slug,
|
|
65
|
+
writeMode: entry.writeMode,
|
|
66
|
+
...(entry.note === undefined ? {} : { note: entry.note }),
|
|
67
|
+
...(entry.reason === undefined ? {} : { reason: entry.reason }),
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* One entry, pushed through the operation race.
|
|
72
|
+
*
|
|
73
|
+
* `MEMORY_PUSH_REJECTED` is the provider saying it will never accept this write, so
|
|
74
|
+
* it is separated here from every other throw and from the `MEMORY_TIMEOUT` the race
|
|
75
|
+
* raises: only the first is worth keeping on the entry, and only the others are worth
|
|
76
|
+
* another attempt.
|
|
77
|
+
*/
|
|
78
|
+
async function pushEntry(input, entry, timers) {
|
|
79
|
+
try {
|
|
80
|
+
const pushed = input.provider.push(requestOf(entry));
|
|
81
|
+
return { kind: "pushed", result: await raceTimeout(pushed, MEMORY_OP_TIMEOUT_MS, timers) };
|
|
82
|
+
}
|
|
83
|
+
catch (error) {
|
|
84
|
+
const reason = errorMessage(error);
|
|
85
|
+
return isMemoryPushRejected(error) ? { kind: "rejected", reason } : { kind: "failed", reason };
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* Step 3: deliver every queued write, oldest first.
|
|
90
|
+
*
|
|
91
|
+
* `list()` answers with the rejected entries too — they are the record of a refusal —
|
|
92
|
+
* so only `queued` entries are pushed: nothing here retries a permanent one.
|
|
93
|
+
*/
|
|
94
|
+
async function drainOutbox(input) {
|
|
95
|
+
const timers = input.timers ?? GLOBAL_MEMORY_TIMERS;
|
|
96
|
+
let pushed = 0;
|
|
97
|
+
let rejected = 0;
|
|
98
|
+
for (const entry of await input.outbox.list()) {
|
|
99
|
+
if (entry.state !== "queued")
|
|
100
|
+
continue;
|
|
101
|
+
const outcome = await pushEntry(input, entry, timers);
|
|
102
|
+
if (outcome.kind === "failed")
|
|
103
|
+
return { pushed, rejected, failure: outcome.reason };
|
|
104
|
+
if (outcome.kind === "rejected") {
|
|
105
|
+
await input.outbox.markRejected(entry.clientId, outcome.reason);
|
|
106
|
+
rejected += 1;
|
|
107
|
+
continue;
|
|
108
|
+
}
|
|
109
|
+
await applyPushed(input, entry, outcome.result);
|
|
110
|
+
await input.outbox.remove(entry.clientId);
|
|
111
|
+
pushed += 1;
|
|
112
|
+
}
|
|
113
|
+
return { pushed, rejected };
|
|
114
|
+
}
|
|
115
|
+
/** The cursor pair with one scope moved: the other scope's cursor is carried over untouched. */
|
|
116
|
+
function cursorWith(cursor, scope, next) {
|
|
117
|
+
return scope === "user" ? { ...cursor, user: next } : { ...cursor, project: next };
|
|
118
|
+
}
|
|
119
|
+
/**
|
|
120
|
+
* After each page: record where this scope's pull got to.
|
|
121
|
+
*
|
|
122
|
+
* The page is the checkpoint, so it is written before the next one is asked for. A page
|
|
123
|
+
* that throws, times out or never arrives therefore leaves the disk naming the last page
|
|
124
|
+
* that did arrive, and the next run resumes there instead of from the beginning. The
|
|
125
|
+
* `lastError` a previous run left is cleared here: this pull reached the provider.
|
|
126
|
+
*/
|
|
127
|
+
async function checkpointPull(input, scope, cursor) {
|
|
128
|
+
const state = await input.store.readSyncState();
|
|
129
|
+
await input.store.writeSyncState({
|
|
130
|
+
provider: input.provider.id,
|
|
131
|
+
cursor: cursorWith(state.cursor, scope, cursor),
|
|
132
|
+
lastPullAt: clockOf(input)().toISOString(),
|
|
133
|
+
lastError: null,
|
|
134
|
+
});
|
|
135
|
+
}
|
|
136
|
+
/** The statuses a pull applies as a deletion: the note is retired wherever it came from. */
|
|
137
|
+
const TOMBSTONES = new Set(["superseded", "rejected", "deleted"]);
|
|
138
|
+
/**
|
|
139
|
+
* One entry of a page read as a note the mirror may hold.
|
|
140
|
+
*
|
|
141
|
+
* The slug is checked first, so `Bad Slug` is refused as a slug and not as an opaque shape
|
|
142
|
+
* error; `parseMemoryNoteRecord` is then the schema a mirror read applies, so a pulled note
|
|
143
|
+
* and a stored one are held to one contract. Both refusals are a skip, never a throw.
|
|
144
|
+
*/
|
|
145
|
+
function readPulledEntry(entry) {
|
|
146
|
+
const slug = typeof entry === "object" && entry !== null && "slug" in entry ? entry.slug : undefined;
|
|
147
|
+
if (typeof slug !== "string" || !MEMORY_SLUG_RE.test(slug)) {
|
|
148
|
+
const reason = typeof slug === "string" ? `slug "${slug}" is not a valid note slug` : "slug is missing";
|
|
149
|
+
return { kind: "invalid", ...(typeof slug === "string" ? { slug } : {}), reason };
|
|
150
|
+
}
|
|
151
|
+
try {
|
|
152
|
+
return { kind: "note", note: parseMemoryNoteRecord(entry) };
|
|
153
|
+
}
|
|
154
|
+
catch (error) {
|
|
155
|
+
return { kind: "invalid", slug, reason: errorMessage(error) };
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
/** One entry the mirror could not take: reported once, at warn, naming the scope and the entry. */
|
|
159
|
+
function warnInvalidEntry(logger, scope, entry) {
|
|
160
|
+
warnQuietly(logger, "skipping a pulled note that is not a valid memory note", {
|
|
161
|
+
scope,
|
|
162
|
+
...(entry.slug === undefined ? {} : { slug: entry.slug }),
|
|
163
|
+
error: entry.reason,
|
|
164
|
+
});
|
|
165
|
+
}
|
|
166
|
+
/**
|
|
167
|
+
* Step 4's mirror half: one page applied to its scope's mirror.
|
|
168
|
+
*
|
|
169
|
+
* The held set is read here, after the page arrived, not once per run: a write enqueued
|
|
170
|
+
* while the page was in flight is one this page must not overwrite, and the local copy is
|
|
171
|
+
* the newer fact — so a held key is left exactly as the mirror has it, and is not counted,
|
|
172
|
+
* because nothing was skipped: the write is still on its way. The page's scope, not the
|
|
173
|
+
* payload's own `scope` field, says where a note lands. A tombstone deletes rather than
|
|
174
|
+
* counts: a retired note is not a pull. Decisions and the cursor checkpoint follow both.
|
|
175
|
+
*/
|
|
176
|
+
async function applyPulledPage(input, scope, answer) {
|
|
177
|
+
const held = await input.outbox.heldKeys();
|
|
178
|
+
let pulled = 0;
|
|
179
|
+
let skipped = 0;
|
|
180
|
+
for (const entry of answer.notes) {
|
|
181
|
+
const parsed = readPulledEntry(entry);
|
|
182
|
+
if (parsed.kind === "invalid") {
|
|
183
|
+
skipped += 1;
|
|
184
|
+
warnInvalidEntry(input.logger, scope, parsed);
|
|
185
|
+
continue;
|
|
186
|
+
}
|
|
187
|
+
const { note } = parsed;
|
|
188
|
+
if (held.has(`${scope}/${note.slug}`))
|
|
189
|
+
continue;
|
|
190
|
+
if (TOMBSTONES.has(note.status)) {
|
|
191
|
+
await input.store.deleteNote(scope, note.slug);
|
|
192
|
+
continue;
|
|
193
|
+
}
|
|
194
|
+
await input.store.writeNote({ ...note, scope });
|
|
195
|
+
pulled += 1;
|
|
196
|
+
}
|
|
197
|
+
// The project mirror owns `decisions.jsonl`: a user page's decisions are not the mirror's to write.
|
|
198
|
+
if (scope === "project" && answer.decisions !== undefined)
|
|
199
|
+
await input.store.writeDecisions(answer.decisions);
|
|
200
|
+
return { pulled, skipped };
|
|
201
|
+
}
|
|
202
|
+
/**
|
|
203
|
+
* One scope's pages, one at a time, each checkpointed before the next is asked for.
|
|
204
|
+
*
|
|
205
|
+
* A page fails the pull wherever it goes wrong — the provider not answering, timing out, or
|
|
206
|
+
* the mirror refusing what it carried — and all three leave the cursor at the last page that
|
|
207
|
+
* was applied. Applying a page sits in the same catch as the race so its reason reaches
|
|
208
|
+
* `lastError`, and not only the run's never-throws shell. The limit is a budget on this run,
|
|
209
|
+
* not a claim about the sequence: a page that says `done` finished whatever page it is, so
|
|
210
|
+
* reaching the limit is a failure only when the provider still has more to give.
|
|
211
|
+
*/
|
|
212
|
+
async function pullScope(input, scope, timers) {
|
|
213
|
+
const take = input.pageSize ?? MEMORY_PULL_PAGE_SIZE;
|
|
214
|
+
let cursor = (await input.store.readSyncState()).cursor[scope];
|
|
215
|
+
let pulled = 0;
|
|
216
|
+
let skipped = 0;
|
|
217
|
+
for (let page = 0; page < MEMORY_PULL_MAX_PAGES; page += 1) {
|
|
218
|
+
try {
|
|
219
|
+
const answer = await raceTimeout(input.provider.pull({ scope, cursor, take }), MEMORY_OP_TIMEOUT_MS, timers);
|
|
220
|
+
const applied = await applyPulledPage(input, scope, answer);
|
|
221
|
+
pulled += applied.pulled;
|
|
222
|
+
skipped += applied.skipped;
|
|
223
|
+
cursor = answer.nextCursor ?? cursor;
|
|
224
|
+
await checkpointPull(input, scope, cursor);
|
|
225
|
+
if (answer.done)
|
|
226
|
+
return { pulled, skipped };
|
|
227
|
+
}
|
|
228
|
+
catch (error) {
|
|
229
|
+
return { pulled, skipped, failure: errorMessage(error) };
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
return { pulled, skipped, failure: PAGE_LIMIT_REACHED };
|
|
233
|
+
}
|
|
234
|
+
/** The scopes a run pulls, in order: the project mirror, then the user's when the store has one. */
|
|
235
|
+
function pullScopes(store) {
|
|
236
|
+
return store.paths.userDir === null ? ["project"] : ["project", "user"];
|
|
237
|
+
}
|
|
238
|
+
/**
|
|
239
|
+
* Step 4: ask the provider for every page of every scope.
|
|
240
|
+
*
|
|
241
|
+
* The first failure ends the pull and the other scope is not asked: a cursor that did
|
|
242
|
+
* not advance is where the next run resumes, and reading past it would record progress
|
|
243
|
+
* this run never made.
|
|
244
|
+
*/
|
|
245
|
+
async function pullAll(input) {
|
|
246
|
+
const timers = input.timers ?? GLOBAL_MEMORY_TIMERS;
|
|
247
|
+
let pulled = 0;
|
|
248
|
+
let skipped = 0;
|
|
249
|
+
for (const scope of pullScopes(input.store)) {
|
|
250
|
+
const result = await pullScope(input, scope, timers);
|
|
251
|
+
pulled += result.pulled;
|
|
252
|
+
skipped += result.skipped;
|
|
253
|
+
if (result.failure !== undefined)
|
|
254
|
+
return { pulled, skipped, failure: result.failure };
|
|
255
|
+
}
|
|
256
|
+
return { pulled, skipped };
|
|
257
|
+
}
|
|
258
|
+
/**
|
|
259
|
+
* A run that stopped after it moved something: record why, refresh the index, report failed.
|
|
260
|
+
*
|
|
261
|
+
* The index is still refreshed, because what the run did move is what the next session
|
|
262
|
+
* has to see — a forget left queued renders as `(forget pending)` — and the counters are
|
|
263
|
+
* the work that was done before the step that failed.
|
|
264
|
+
*/
|
|
265
|
+
async function failRun(input, counts, failure) {
|
|
266
|
+
await recordLastError(input, failure);
|
|
267
|
+
return await finish(input, counts, failure);
|
|
268
|
+
}
|
|
269
|
+
/**
|
|
270
|
+
* Step 5: regenerate the index the next session reads, and report what the run did.
|
|
271
|
+
*
|
|
272
|
+
* `failure` is the step that stopped early, if any: the counters are what did move
|
|
273
|
+
* before it stopped.
|
|
274
|
+
*/
|
|
275
|
+
async function finish(input, counts, failure) {
|
|
276
|
+
const heldForgets = await input.outbox.heldKeys("forget");
|
|
277
|
+
await refreshMemoryIndex(input.store, { heldForgets });
|
|
278
|
+
return failure === undefined ? { outcome: "synced", ...counts } : { outcome: "failed", ...counts, error: failure };
|
|
279
|
+
}
|
|
280
|
+
/** Hand the lock back if this run took it; a failed release is logged, never thrown. */
|
|
281
|
+
async function releaseQuietly(input, release) {
|
|
282
|
+
try {
|
|
283
|
+
await release();
|
|
284
|
+
}
|
|
285
|
+
catch (error) {
|
|
286
|
+
warnQuietly(input.logger, "memory sync could not release the lock", {
|
|
287
|
+
path: input.store.paths.lock,
|
|
288
|
+
error: errorMessage(error),
|
|
289
|
+
});
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
export async function syncMemory(input) {
|
|
293
|
+
const lock = input.store.paths.lock;
|
|
294
|
+
// Every lock this run takes is claimed here, synchronously and before the first await: a
|
|
295
|
+
// guard taken after a yield is one two callers both pass. Nothing has touched the
|
|
296
|
+
// filesystem on this path. Only the claims *this* run won are recorded, so a run that loses
|
|
297
|
+
// contention on the second mirror gives back the first — and never the claim of the sync
|
|
298
|
+
// that is already running, which would let a third caller in behind it.
|
|
299
|
+
const claimed = [];
|
|
300
|
+
const won = storeLockTargets(input.store).every((target) => {
|
|
301
|
+
if (!claimSync(target.path))
|
|
302
|
+
return false;
|
|
303
|
+
claimed.push(target.path);
|
|
304
|
+
return true;
|
|
305
|
+
});
|
|
306
|
+
if (!won) {
|
|
307
|
+
for (const path of claimed)
|
|
308
|
+
releaseSync(path);
|
|
309
|
+
return stopped("locked");
|
|
310
|
+
}
|
|
311
|
+
let release = NO_RELEASE;
|
|
312
|
+
try {
|
|
313
|
+
const acquired = await takeLock(input);
|
|
314
|
+
if (acquired === null)
|
|
315
|
+
return stopped("locked");
|
|
316
|
+
release = acquired;
|
|
317
|
+
const timers = input.timers ?? GLOBAL_MEMORY_TIMERS;
|
|
318
|
+
if (!(await providerAnswers(input.provider, timers)))
|
|
319
|
+
return await recordUnreachable(input);
|
|
320
|
+
const drain = await drainOutbox(input);
|
|
321
|
+
const counts = { ...noCounts(), pushed: drain.pushed, rejected: drain.rejected };
|
|
322
|
+
// A drain that stopped early says nothing about the provider's own changes: no pull.
|
|
323
|
+
if (drain.failure !== undefined)
|
|
324
|
+
return await failRun(input, counts, drain.failure);
|
|
325
|
+
const pull = await pullAll(input);
|
|
326
|
+
counts.pulled = pull.pulled;
|
|
327
|
+
counts.skipped = pull.skipped;
|
|
328
|
+
if (pull.failure !== undefined)
|
|
329
|
+
return await failRun(input, counts, pull.failure);
|
|
330
|
+
return await finish(input, counts);
|
|
331
|
+
}
|
|
332
|
+
catch (error) {
|
|
333
|
+
const message = errorMessage(error);
|
|
334
|
+
warnQuietly(input.logger, "memory sync failed", { path: lock, error: message });
|
|
335
|
+
return stopped("failed", message);
|
|
336
|
+
}
|
|
337
|
+
finally {
|
|
338
|
+
await releaseQuietly(input, release);
|
|
339
|
+
// Cleared whatever the cleanup did: a lock we could not release ages into
|
|
340
|
+
// stale recovery, but this process must not be wedged until then.
|
|
341
|
+
for (const path of claimed)
|
|
342
|
+
releaseSync(path);
|
|
343
|
+
}
|
|
344
|
+
}
|