@duetso/agent 0.1.99 → 0.1.102
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/package.json +1 -1
- package/dist/src/memory/context-pack.d.ts +2 -2
- package/dist/src/memory/context-pack.d.ts.map +1 -1
- package/dist/src/memory/context-pack.js +21 -16
- package/dist/src/memory/context-pack.js.map +1 -1
- package/dist/src/memory/embedding-worker.d.ts +46 -3
- package/dist/src/memory/embedding-worker.d.ts.map +1 -1
- package/dist/src/memory/embedding-worker.js +139 -37
- package/dist/src/memory/embedding-worker.js.map +1 -1
- package/dist/src/memory/observational.d.ts +8 -3
- package/dist/src/memory/observational.d.ts.map +1 -1
- package/dist/src/memory/observational.js +10 -10
- package/dist/src/memory/observational.js.map +1 -1
- package/dist/src/memory/pglite.d.ts +57 -17
- package/dist/src/memory/pglite.d.ts.map +1 -1
- package/dist/src/memory/pglite.js +153 -140
- package/dist/src/memory/pglite.js.map +1 -1
- package/dist/src/memory/recall.d.ts +9 -2
- package/dist/src/memory/recall.d.ts.map +1 -1
- package/dist/src/memory/recall.js +23 -20
- package/dist/src/memory/recall.js.map +1 -1
- package/dist/src/memory/session.d.ts +78 -0
- package/dist/src/memory/session.d.ts.map +1 -0
- package/dist/src/memory/session.js +198 -0
- package/dist/src/memory/session.js.map +1 -0
- package/dist/src/memory/storage.d.ts +29 -15
- package/dist/src/memory/storage.d.ts.map +1 -1
- package/dist/src/memory/storage.js +80 -54
- package/dist/src/memory/storage.js.map +1 -1
- package/dist/src/turn-runner/tools.d.ts +8 -6
- package/dist/src/turn-runner/tools.d.ts.map +1 -1
- package/dist/src/turn-runner/tools.js +3 -3
- package/dist/src/turn-runner/tools.js.map +1 -1
- package/dist/src/turn-runner/turn-runner.d.ts.map +1 -1
- package/dist/src/turn-runner/turn-runner.js +14 -6
- package/dist/src/turn-runner/turn-runner.js.map +1 -1
- package/package.json +1 -1
package/dist/package.json
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import type { PGlite } from "@electric-sql/pglite";
|
|
2
1
|
import type { ObservationalMemorySettings } from "../types/memory.js";
|
|
2
|
+
import type { MemorySession } from "./session.js";
|
|
3
3
|
import type { MemoryContextCache } from "./store.js";
|
|
4
4
|
/**
|
|
5
5
|
* Compaction trigger: rebuild the frozen memory pack rendered above
|
|
@@ -21,7 +21,7 @@ import type { MemoryContextCache } from "./store.js";
|
|
|
21
21
|
* memory bookkeeping.
|
|
22
22
|
*/
|
|
23
23
|
export declare function rebuildMemoryContextPack(options: {
|
|
24
|
-
|
|
24
|
+
session: MemorySession | undefined;
|
|
25
25
|
cache: MemoryContextCache;
|
|
26
26
|
settings: ObservationalMemorySettings;
|
|
27
27
|
sessionId?: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"context-pack.d.ts","sourceRoot":"","sources":["../../../src/memory/context-pack.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,
|
|
1
|
+
{"version":3,"file":"context-pack.d.ts","sourceRoot":"","sources":["../../../src/memory/context-pack.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,2BAA2B,EAAE,MAAM,oBAAoB,CAAC;AAEtE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAClD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAErD;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAsB,wBAAwB,CAAC,OAAO,EAAE;IACtD,OAAO,EAAE,aAAa,GAAG,SAAS,CAAC;IACnC,KAAK,EAAE,kBAAkB,CAAC;IAC1B,QAAQ,EAAE,2BAA2B,CAAC;IACtC,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,GAAG,OAAO,CAAC,IAAI,CAAC,CAwBhB"}
|
|
@@ -19,22 +19,27 @@ import { loadGlobalPack, loadLocalPack } from "./loader.js";
|
|
|
19
19
|
* memory bookkeeping.
|
|
20
20
|
*/
|
|
21
21
|
export async function rebuildMemoryContextPack(options) {
|
|
22
|
-
if (!options.
|
|
22
|
+
if (!options.session)
|
|
23
23
|
return;
|
|
24
|
-
//
|
|
25
|
-
//
|
|
26
|
-
//
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
24
|
+
// One withDb pins the open across the global+local pack queries so the
|
|
25
|
+
// cross-process lock is held just once for the rebuild, then released
|
|
26
|
+
// a couple seconds later when the idle-close timer fires.
|
|
27
|
+
await options.session.withDb(async (db) => {
|
|
28
|
+
// Local layer skipped when the runner has no session id (one-shot
|
|
29
|
+
// tools, tests). Global layer always runs because the loader's
|
|
30
|
+
// `excludeSessionId` is optional and meaningful as undefined.
|
|
31
|
+
const [globalPack, localPack] = await Promise.all([
|
|
32
|
+
loadGlobalPack(db, {
|
|
33
|
+
...(options.sessionId !== undefined ? { excludeSessionId: options.sessionId } : {}),
|
|
34
|
+
tokenBudget: options.settings.globalContextTokenBudget,
|
|
35
|
+
recencyHalfLifeMs: options.settings.recencyHalfLifeMs,
|
|
36
|
+
reflectionBias: options.settings.reflectionBias,
|
|
37
|
+
}),
|
|
38
|
+
options.sessionId !== undefined
|
|
39
|
+
? loadLocalPack(db, { sessionId: options.sessionId })
|
|
40
|
+
: Promise.resolve([]),
|
|
41
|
+
]);
|
|
42
|
+
options.cache.setContextPack({ global: globalPack, local: localPack });
|
|
43
|
+
});
|
|
39
44
|
}
|
|
40
45
|
//# sourceMappingURL=context-pack.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"context-pack.js","sourceRoot":"","sources":["../../../src/memory/context-pack.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"context-pack.js","sourceRoot":"","sources":["../../../src/memory/context-pack.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAI5D;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAAC,OAK9C;IACC,IAAI,CAAC,OAAO,CAAC,OAAO;QAAE,OAAO;IAE7B,uEAAuE;IACvE,sEAAsE;IACtE,0DAA0D;IAC1D,MAAM,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE;QACxC,kEAAkE;QAClE,+DAA+D;QAC/D,8DAA8D;QAC9D,MAAM,CAAC,UAAU,EAAE,SAAS,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAChD,cAAc,CAAC,EAAE,EAAE;gBACjB,GAAG,CAAC,OAAO,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACnF,WAAW,EAAE,OAAO,CAAC,QAAQ,CAAC,wBAAwB;gBACtD,iBAAiB,EAAE,OAAO,CAAC,QAAQ,CAAC,iBAAiB;gBACrD,cAAc,EAAE,OAAO,CAAC,QAAQ,CAAC,cAAc;aAChD,CAAC;YACF,OAAO,CAAC,SAAS,KAAK,SAAS;gBAC7B,CAAC,CAAC,aAAa,CAAC,EAAE,EAAE,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,CAAC;gBACrD,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;SACxB,CAAC,CAAC;QAEH,OAAO,CAAC,KAAK,CAAC,cAAc,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC;IACzE,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -1,23 +1,66 @@
|
|
|
1
|
-
import type { PGlite } from "@electric-sql/pglite";
|
|
2
1
|
import type { EmbedFn } from "./embedding.js";
|
|
2
|
+
import type { MemorySession } from "./session.js";
|
|
3
3
|
export interface EmbeddingBackfillWorkerOptions {
|
|
4
|
-
|
|
4
|
+
/** Memory session owning the PGlite handle and cross-process lock. */
|
|
5
|
+
session: MemorySession;
|
|
5
6
|
/** Embedding callable. Defaults to the gateway client; tests inject a stub. */
|
|
6
7
|
embed: EmbedFn;
|
|
7
8
|
/** Path to append progress lines to. Optional; when omitted the worker logs nothing. */
|
|
8
9
|
logPath?: string;
|
|
10
|
+
/**
|
|
11
|
+
* Idle delay between outer drain ticks in milliseconds. Defaults to
|
|
12
|
+
* 10s, which gives a peer duet CLI room to acquire the lock between
|
|
13
|
+
* drains. Tests override this to drive several ticks per second.
|
|
14
|
+
*/
|
|
15
|
+
idleSleepMs?: number;
|
|
16
|
+
/**
|
|
17
|
+
* Backoff after a failed drain in milliseconds. Defaults to 60s.
|
|
18
|
+
* Tests override this so a transient-error case does not block the
|
|
19
|
+
* suite for a full minute.
|
|
20
|
+
*/
|
|
21
|
+
errorSleepMs?: number;
|
|
22
|
+
/**
|
|
23
|
+
* Minimum time between re-embed attempts for the same observation
|
|
24
|
+
* id, in milliseconds. Defaults to 5 minutes. Caps unbounded
|
|
25
|
+
* re-embedding when something (typically the reflector's
|
|
26
|
+
* delete-and-reinsert cycle) keeps wiping a row's embedding behind
|
|
27
|
+
* the worker's back.
|
|
28
|
+
*/
|
|
29
|
+
attemptCooldownMs?: number;
|
|
30
|
+
/**
|
|
31
|
+
* Maximum size of the live log file in bytes before it is rotated
|
|
32
|
+
* to `<logPath>.1` (replacing any prior rotation). Defaults to 1 MB.
|
|
33
|
+
* Each call to `log` checks the current size first and rotates
|
|
34
|
+
* before appending, so a chronic failure cannot grow the log without
|
|
35
|
+
* bound.
|
|
36
|
+
*/
|
|
37
|
+
logMaxBytes?: number;
|
|
9
38
|
}
|
|
10
39
|
export declare class EmbeddingBackfillWorker {
|
|
11
40
|
private readonly options;
|
|
41
|
+
private readonly idleSleepMs;
|
|
42
|
+
private readonly errorSleepMs;
|
|
43
|
+
private readonly attemptCooldownMs;
|
|
44
|
+
private readonly logMaxBytes;
|
|
45
|
+
private readonly recentAttempts;
|
|
12
46
|
private abortController?;
|
|
13
47
|
private runningPromise?;
|
|
14
48
|
constructor(options: EmbeddingBackfillWorkerOptions);
|
|
15
49
|
/** Start the background loop. Idempotent — a second call is a no-op. */
|
|
16
50
|
start(): void;
|
|
17
|
-
/**
|
|
51
|
+
/**
|
|
52
|
+
* Stop the loop and wait for the in-flight tick (if any) to settle.
|
|
53
|
+
* Because each tick awaits a `session.withDb`, this transitively waits
|
|
54
|
+
* for any open the worker holds to drain and the lock to release.
|
|
55
|
+
*/
|
|
18
56
|
stop(): Promise<void>;
|
|
19
57
|
private run;
|
|
20
58
|
private selectBatch;
|
|
59
|
+
/**
|
|
60
|
+
* Return the ids currently inside the attempt-cooldown window and
|
|
61
|
+
* forget anything older so the map cannot grow unbounded.
|
|
62
|
+
*/
|
|
63
|
+
private collectCooledIds;
|
|
21
64
|
private persistBatch;
|
|
22
65
|
private log;
|
|
23
66
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"embedding-worker.d.ts","sourceRoot":"","sources":["../../../src/memory/embedding-worker.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"embedding-worker.d.ts","sourceRoot":"","sources":["../../../src/memory/embedding-worker.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAuClD,MAAM,WAAW,8BAA8B;IAC7C,sEAAsE;IACtE,OAAO,EAAE,aAAa,CAAC;IACvB,+EAA+E;IAC/E,KAAK,EAAE,OAAO,CAAC;IACf,wFAAwF;IACxF,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;;OAIG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;;;;;OAMG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B;;;;;;OAMG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAID,qBAAa,uBAAuB;IAClC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAiC;IACzD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IACtC,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAS;IAC3C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IAKrC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAA6B;IAC5D,OAAO,CAAC,eAAe,CAAC,CAAkB;IAC1C,OAAO,CAAC,cAAc,CAAC,CAAgB;gBAE3B,OAAO,EAAE,8BAA8B;IAQnD,wEAAwE;IACxE,KAAK,IAAI,IAAI;IAMb;;;;OAIG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;YAOb,GAAG;YAsCH,WAAW;IAyBzB;;;OAGG;IACH,OAAO,CAAC,gBAAgB;YAcV,YAAY;IAuC1B,OAAO,CAAC,GAAG;CA+BZ"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { appendFileSync, mkdirSync } from "node:fs";
|
|
1
|
+
import { appendFileSync, mkdirSync, renameSync, statSync } from "node:fs";
|
|
2
2
|
import { dirname } from "node:path";
|
|
3
3
|
/**
|
|
4
4
|
* Background worker that fills missing embeddings while the CLI is
|
|
@@ -7,29 +7,55 @@ import { dirname } from "node:path";
|
|
|
7
7
|
* to the user immediately; the embedding lands within a few seconds in
|
|
8
8
|
* the background, after which the row becomes hybrid-retrievable.
|
|
9
9
|
*
|
|
10
|
-
*
|
|
11
|
-
* 1.
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
10
|
+
* Tick shape:
|
|
11
|
+
* 1. Acquire the memory session via one `withDb`. Inside that open,
|
|
12
|
+
* drain unembedded rows in `BATCH_SIZE` chunks until a select
|
|
13
|
+
* returns zero rows, then exit `withDb` so the cross-process lock
|
|
14
|
+
* releases shortly after via the session's idle-close timer.
|
|
15
|
+
* 2. Sleep `IDLE_SLEEP_MS` between ticks. Long enough that a peer
|
|
16
|
+
* duet CLI can acquire the lock between drains, short enough that
|
|
17
|
+
* a newly-written observation becomes searchable within seconds.
|
|
18
|
+
* 3. On error (rate limit, transient network), back off for
|
|
19
|
+
* `ERROR_SLEEP_MS` before retrying.
|
|
19
20
|
*
|
|
20
21
|
* Failure modes are local: any error inside the loop logs and resumes
|
|
21
22
|
* after a backoff. The worker never throws to the caller.
|
|
22
23
|
*/
|
|
23
24
|
const BATCH_SIZE = 50;
|
|
24
|
-
const
|
|
25
|
-
const
|
|
26
|
-
|
|
25
|
+
const DEFAULT_IDLE_SLEEP_MS = 10_000;
|
|
26
|
+
const DEFAULT_ERROR_SLEEP_MS = 60_000;
|
|
27
|
+
// Cap how often a single observation may be re-embedded. The production
|
|
28
|
+
// memory store can churn rows (the reflector deletes and re-inserts a
|
|
29
|
+
// session's observations, which cascade-deletes their embeddings). If
|
|
30
|
+
// the embedding row vanishes between drains we should re-embed at
|
|
31
|
+
// most once per cooldown rather than running an unbounded hot loop
|
|
32
|
+
// over the same ids.
|
|
33
|
+
const DEFAULT_ATTEMPT_COOLDOWN_MS = 5 * 60_000;
|
|
34
|
+
// Snippet length for embedding-failure log lines. The remote endpoint
|
|
35
|
+
// occasionally responds with a full HTML page when misrouted (404 from
|
|
36
|
+
// the marketing site) and dumping the whole body fills the log with
|
|
37
|
+
// tens of kilobytes per failure.
|
|
38
|
+
const ERROR_BODY_LOG_MAX = 200;
|
|
39
|
+
const DEFAULT_LOG_MAX_BYTES = 1024 * 1024;
|
|
27
40
|
export class EmbeddingBackfillWorker {
|
|
28
41
|
options;
|
|
42
|
+
idleSleepMs;
|
|
43
|
+
errorSleepMs;
|
|
44
|
+
attemptCooldownMs;
|
|
45
|
+
logMaxBytes;
|
|
46
|
+
// Per-id record of the last attempt timestamp. selectBatch filters
|
|
47
|
+
// out ids attempted within the cooldown, breaking the hot loop when
|
|
48
|
+
// a row's embedding keeps disappearing between drains. Entries are
|
|
49
|
+
// purged lazily as they expire.
|
|
50
|
+
recentAttempts = new Map();
|
|
29
51
|
abortController;
|
|
30
52
|
runningPromise;
|
|
31
53
|
constructor(options) {
|
|
32
54
|
this.options = options;
|
|
55
|
+
this.idleSleepMs = options.idleSleepMs ?? DEFAULT_IDLE_SLEEP_MS;
|
|
56
|
+
this.errorSleepMs = options.errorSleepMs ?? DEFAULT_ERROR_SLEEP_MS;
|
|
57
|
+
this.attemptCooldownMs = options.attemptCooldownMs ?? DEFAULT_ATTEMPT_COOLDOWN_MS;
|
|
58
|
+
this.logMaxBytes = options.logMaxBytes ?? DEFAULT_LOG_MAX_BYTES;
|
|
33
59
|
}
|
|
34
60
|
/** Start the background loop. Idempotent — a second call is a no-op. */
|
|
35
61
|
start() {
|
|
@@ -38,7 +64,11 @@ export class EmbeddingBackfillWorker {
|
|
|
38
64
|
this.abortController = new AbortController();
|
|
39
65
|
this.runningPromise = this.run(this.abortController.signal);
|
|
40
66
|
}
|
|
41
|
-
/**
|
|
67
|
+
/**
|
|
68
|
+
* Stop the loop and wait for the in-flight tick (if any) to settle.
|
|
69
|
+
* Because each tick awaits a `session.withDb`, this transitively waits
|
|
70
|
+
* for any open the worker holds to drain and the lock to release.
|
|
71
|
+
*/
|
|
42
72
|
async stop() {
|
|
43
73
|
this.abortController?.abort();
|
|
44
74
|
await this.runningPromise;
|
|
@@ -48,44 +78,82 @@ export class EmbeddingBackfillWorker {
|
|
|
48
78
|
async run(signal) {
|
|
49
79
|
while (!signal.aborted) {
|
|
50
80
|
try {
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
81
|
+
// One withDb per tick: drain everything that's currently
|
|
82
|
+
// outstanding while we hold the lock, then exit so the
|
|
83
|
+
// session can idle-close and another duet CLI can take a
|
|
84
|
+
// turn. A returned `undefined` means the lock could not be
|
|
85
|
+
// acquired in time — fall through to the idle sleep and
|
|
86
|
+
// retry the same tick later.
|
|
87
|
+
await this.options.session.withDb(async (db) => {
|
|
88
|
+
while (!signal.aborted) {
|
|
89
|
+
const batch = await this.selectBatch(db);
|
|
90
|
+
if (batch.length === 0)
|
|
91
|
+
return;
|
|
92
|
+
// Stamp the attempt before we await the embedding so a
|
|
93
|
+
// failure mid-call still counts as an attempt and the same
|
|
94
|
+
// ids do not get retried in a tight loop on the next drain.
|
|
95
|
+
const attemptAt = Date.now();
|
|
96
|
+
for (const row of batch)
|
|
97
|
+
this.recentAttempts.set(row.id, attemptAt);
|
|
98
|
+
const result = await this.options.embed(batch.map((row) => row.content));
|
|
99
|
+
if (result.embeddings.length !== batch.length) {
|
|
100
|
+
throw new Error(`Embedding response length (${result.embeddings.length}) did not match batch size (${batch.length})`);
|
|
101
|
+
}
|
|
102
|
+
await this.persistBatch(db, batch, result.embeddings, result.model);
|
|
103
|
+
this.log(`Embedded ${batch.length} observations`);
|
|
104
|
+
}
|
|
105
|
+
});
|
|
106
|
+
if (signal.aborted)
|
|
107
|
+
return;
|
|
108
|
+
await sleep(this.idleSleepMs, signal);
|
|
63
109
|
}
|
|
64
110
|
catch (error) {
|
|
65
111
|
if (signal.aborted)
|
|
66
112
|
return;
|
|
67
|
-
|
|
68
|
-
this.
|
|
69
|
-
await sleep(ERROR_SLEEP_MS, signal);
|
|
113
|
+
this.log(`Embedding batch failed: ${summarizeError(error)}`);
|
|
114
|
+
await sleep(this.errorSleepMs, signal);
|
|
70
115
|
}
|
|
71
116
|
}
|
|
72
117
|
}
|
|
73
|
-
async selectBatch() {
|
|
118
|
+
async selectBatch(db) {
|
|
74
119
|
// LEFT JOIN over the embeddings table is the cheapest way to pick
|
|
75
120
|
// rows with no embedding yet. The composite index on
|
|
76
121
|
// (kind, priority, created_at DESC) carries the ORDER BY without a
|
|
77
122
|
// separate sort.
|
|
78
|
-
|
|
123
|
+
//
|
|
124
|
+
// The exclusion list comes from `recentAttempts` and caps how
|
|
125
|
+
// often the worker may retry the same id, even if its embedding
|
|
126
|
+
// row keeps being cascade-deleted between drains.
|
|
127
|
+
const excludedIds = this.collectCooledIds();
|
|
128
|
+
const result = await db.query(`SELECT o.id, o.content
|
|
79
129
|
FROM observations o
|
|
80
130
|
LEFT JOIN observation_embeddings e ON e.observation_id = o.id
|
|
81
131
|
WHERE e.observation_id IS NULL
|
|
132
|
+
AND NOT (o.id = ANY($2::text[]))
|
|
82
133
|
ORDER BY
|
|
83
134
|
CASE o.priority WHEN 'high' THEN 3 WHEN 'medium' THEN 2 ELSE 1 END DESC,
|
|
84
135
|
o.created_at DESC
|
|
85
|
-
LIMIT $1`, [BATCH_SIZE]);
|
|
136
|
+
LIMIT $1`, [BATCH_SIZE, excludedIds]);
|
|
86
137
|
return result.rows;
|
|
87
138
|
}
|
|
88
|
-
|
|
139
|
+
/**
|
|
140
|
+
* Return the ids currently inside the attempt-cooldown window and
|
|
141
|
+
* forget anything older so the map cannot grow unbounded.
|
|
142
|
+
*/
|
|
143
|
+
collectCooledIds() {
|
|
144
|
+
const now = Date.now();
|
|
145
|
+
const cutoff = now - this.attemptCooldownMs;
|
|
146
|
+
const active = [];
|
|
147
|
+
for (const [id, at] of this.recentAttempts) {
|
|
148
|
+
if (at < cutoff) {
|
|
149
|
+
this.recentAttempts.delete(id);
|
|
150
|
+
continue;
|
|
151
|
+
}
|
|
152
|
+
active.push(id);
|
|
153
|
+
}
|
|
154
|
+
return active;
|
|
155
|
+
}
|
|
156
|
+
async persistBatch(db, batch, vectors, model) {
|
|
89
157
|
// One transaction so a partial network or write failure does not
|
|
90
158
|
// leave the embeddings table half-populated relative to the
|
|
91
159
|
// candidate set we just queried. `model` is the identifier the
|
|
@@ -100,7 +168,7 @@ export class EmbeddingBackfillWorker {
|
|
|
100
168
|
// a foreign-key violation, losing every embedding in this batch.
|
|
101
169
|
// Skipping the missing parent inserts 0 rows for that observation
|
|
102
170
|
// and lets the surviving rows commit.
|
|
103
|
-
await
|
|
171
|
+
await db.transaction(async (tx) => {
|
|
104
172
|
const now = Date.now();
|
|
105
173
|
for (let index = 0; index < batch.length; index++) {
|
|
106
174
|
const row = batch[index];
|
|
@@ -116,11 +184,29 @@ export class EmbeddingBackfillWorker {
|
|
|
116
184
|
});
|
|
117
185
|
}
|
|
118
186
|
log(message) {
|
|
119
|
-
|
|
187
|
+
const logPath = this.options.logPath;
|
|
188
|
+
if (!logPath)
|
|
120
189
|
return;
|
|
121
190
|
try {
|
|
122
|
-
mkdirSync(dirname(
|
|
123
|
-
|
|
191
|
+
mkdirSync(dirname(logPath), { recursive: true });
|
|
192
|
+
// Rotate before appending so the live file never exceeds the
|
|
193
|
+
// cap by more than one line. The single `.1` rotation keeps the
|
|
194
|
+
// most recent overflow available for post-mortem without
|
|
195
|
+
// accumulating an unbounded ring of files.
|
|
196
|
+
try {
|
|
197
|
+
if (statSync(logPath).size >= this.logMaxBytes) {
|
|
198
|
+
renameSync(logPath, `${logPath}.1`);
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
catch {
|
|
202
|
+
// No existing file or stat failed; appendFileSync will create.
|
|
203
|
+
}
|
|
204
|
+
// Include the pid so multi-CLI runs (which share the log file
|
|
205
|
+
// via the same HOME-backed `.duet/logs/` path) can be
|
|
206
|
+
// disambiguated. Each appendFileSync call writes a single line
|
|
207
|
+
// well under PIPE_BUF, so O_APPEND keeps lines from different
|
|
208
|
+
// processes intact even when they interleave.
|
|
209
|
+
appendFileSync(logPath, `[${new Date().toISOString()} pid=${process.pid}] ${message}\n`, "utf8");
|
|
124
210
|
}
|
|
125
211
|
catch {
|
|
126
212
|
// Logging is best-effort; failures here must not bring the worker
|
|
@@ -136,6 +222,22 @@ export class EmbeddingBackfillWorker {
|
|
|
136
222
|
function formatVector(values) {
|
|
137
223
|
return `[${values.join(",")}]`;
|
|
138
224
|
}
|
|
225
|
+
/**
|
|
226
|
+
* Render an error for the worker log without dumping multi-kilobyte
|
|
227
|
+
* HTML bodies. The embedding endpoint can respond with a full Next.js
|
|
228
|
+
* 404 page (over 50KB) when misrouted, and the raw message would
|
|
229
|
+
* otherwise be appended verbatim on every retry.
|
|
230
|
+
*/
|
|
231
|
+
function summarizeError(error) {
|
|
232
|
+
const raw = error instanceof Error ? error.message : String(error);
|
|
233
|
+
const collapsed = raw
|
|
234
|
+
.replace(/<[^>]+>/g, " ")
|
|
235
|
+
.replace(/\s+/g, " ")
|
|
236
|
+
.trim();
|
|
237
|
+
if (collapsed.length <= ERROR_BODY_LOG_MAX)
|
|
238
|
+
return collapsed;
|
|
239
|
+
return `${collapsed.slice(0, ERROR_BODY_LOG_MAX)}… (truncated ${collapsed.length - ERROR_BODY_LOG_MAX} chars)`;
|
|
240
|
+
}
|
|
139
241
|
function sleep(ms, signal) {
|
|
140
242
|
return new Promise((resolve) => {
|
|
141
243
|
if (signal.aborted) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"embedding-worker.js","sourceRoot":"","sources":["../../../src/memory/embedding-worker.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"embedding-worker.js","sourceRoot":"","sources":["../../../src/memory/embedding-worker.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAE,SAAS,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAC1E,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAIpC;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,MAAM,UAAU,GAAG,EAAE,CAAC;AACtB,MAAM,qBAAqB,GAAG,MAAM,CAAC;AACrC,MAAM,sBAAsB,GAAG,MAAM,CAAC;AACtC,wEAAwE;AACxE,sEAAsE;AACtE,sEAAsE;AACtE,kEAAkE;AAClE,mEAAmE;AACnE,qBAAqB;AACrB,MAAM,2BAA2B,GAAG,CAAC,GAAG,MAAM,CAAC;AAC/C,sEAAsE;AACtE,uEAAuE;AACvE,oEAAoE;AACpE,iCAAiC;AACjC,MAAM,kBAAkB,GAAG,GAAG,CAAC;AAuC/B,MAAM,qBAAqB,GAAG,IAAI,GAAG,IAAI,CAAC;AAE1C,MAAM,OAAO,uBAAuB;IACjB,OAAO,CAAiC;IACxC,WAAW,CAAS;IACpB,YAAY,CAAS;IACrB,iBAAiB,CAAS;IAC1B,WAAW,CAAS;IACrC,mEAAmE;IACnE,oEAAoE;IACpE,mEAAmE;IACnE,gCAAgC;IACf,cAAc,GAAG,IAAI,GAAG,EAAkB,CAAC;IACpD,eAAe,CAAmB;IAClC,cAAc,CAAiB;IAEvC,YAAY,OAAuC;QACjD,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,qBAAqB,CAAC;QAChE,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,sBAAsB,CAAC;QACnE,IAAI,CAAC,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,IAAI,2BAA2B,CAAC;QAClF,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,qBAAqB,CAAC;IAClE,CAAC;IAED,wEAAwE;IACxE,KAAK;QACH,IAAI,IAAI,CAAC,cAAc;YAAE,OAAO;QAChC,IAAI,CAAC,eAAe,GAAG,IAAI,eAAe,EAAE,CAAC;QAC7C,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;IAC9D,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,IAAI;QACR,IAAI,CAAC,eAAe,EAAE,KAAK,EAAE,CAAC;QAC9B,MAAM,IAAI,CAAC,cAAc,CAAC;QAC1B,IAAI,CAAC,eAAe,GAAG,SAAS,CAAC;QACjC,IAAI,CAAC,cAAc,GAAG,SAAS,CAAC;IAClC,CAAC;IAEO,KAAK,CAAC,GAAG,CAAC,MAAmB;QACnC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACvB,IAAI,CAAC;gBACH,yDAAyD;gBACzD,uDAAuD;gBACvD,yDAAyD;gBACzD,2DAA2D;gBAC3D,wDAAwD;gBACxD,6BAA6B;gBAC7B,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE;oBAC7C,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;wBACvB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;wBACzC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;4BAAE,OAAO;wBAC/B,uDAAuD;wBACvD,2DAA2D;wBAC3D,4DAA4D;wBAC5D,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;wBAC7B,KAAK,MAAM,GAAG,IAAI,KAAK;4BAAE,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,SAAS,CAAC,CAAC;wBACpE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC;wBACzE,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC;4BAC9C,MAAM,IAAI,KAAK,CACb,8BAA8B,MAAM,CAAC,UAAU,CAAC,MAAM,+BAA+B,KAAK,CAAC,MAAM,GAAG,CACrG,CAAC;wBACJ,CAAC;wBACD,MAAM,IAAI,CAAC,YAAY,CAAC,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;wBACpE,IAAI,CAAC,GAAG,CAAC,YAAY,KAAK,CAAC,MAAM,eAAe,CAAC,CAAC;oBACpD,CAAC;gBACH,CAAC,CAAC,CAAC;gBACH,IAAI,MAAM,CAAC,OAAO;oBAAE,OAAO;gBAC3B,MAAM,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;YACxC,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,MAAM,CAAC,OAAO;oBAAE,OAAO;gBAC3B,IAAI,CAAC,GAAG,CAAC,2BAA2B,cAAc,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;gBAC7D,MAAM,KAAK,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;YACzC,CAAC;QACH,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,EAAU;QAClC,kEAAkE;QAClE,qDAAqD;QACrD,mEAAmE;QACnE,iBAAiB;QACjB,EAAE;QACF,8DAA8D;QAC9D,gEAAgE;QAChE,kDAAkD;QAClD,MAAM,WAAW,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAC5C,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,KAAK,CAC3B;;;;;;;;gBAQU,EACV,CAAC,UAAU,EAAE,WAAW,CAAC,CAC1B,CAAC;QACF,OAAO,MAAM,CAAC,IAAI,CAAC;IACrB,CAAC;IAED;;;OAGG;IACK,gBAAgB;QACtB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,MAAM,GAAG,GAAG,GAAG,IAAI,CAAC,iBAAiB,CAAC;QAC5C,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,KAAK,MAAM,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YAC3C,IAAI,EAAE,GAAG,MAAM,EAAE,CAAC;gBAChB,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;gBAC/B,SAAS;YACX,CAAC;YACD,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAClB,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,YAAY,CACxB,EAAU,EACV,KAAwC,EACxC,OAAmB,EACnB,KAAa;QAEb,iEAAiE;QACjE,4DAA4D;QAC5D,+DAA+D;QAC/D,6DAA6D;QAC7D,gEAAgE;QAChE,0BAA0B;QAC1B,EAAE;QACF,oEAAoE;QACpE,kEAAkE;QAClE,gEAAgE;QAChE,mEAAmE;QACnE,iEAAiE;QACjE,kEAAkE;QAClE,sCAAsC;QACtC,MAAM,EAAE,CAAC,WAAW,CAAC,KAAK,EAAE,EAAe,EAAE,EAAE;YAC7C,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACvB,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;gBAClD,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,CAAE,CAAC;gBAC1B,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAE,CAAC;gBAC/B,MAAM,EAAE,CAAC,KAAK,CACZ;;;;;;8CAMoC,EACpC,CAAC,GAAG,CAAC,EAAE,EAAE,KAAK,EAAE,YAAY,CAAC,MAAM,CAAC,EAAE,GAAG,CAAC,CAC3C,CAAC;YACJ,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,GAAG,CAAC,OAAe;QACzB,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC;QACrC,IAAI,CAAC,OAAO;YAAE,OAAO;QACrB,IAAI,CAAC;YACH,SAAS,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YACjD,6DAA6D;YAC7D,gEAAgE;YAChE,yDAAyD;YACzD,2CAA2C;YAC3C,IAAI,CAAC;gBACH,IAAI,QAAQ,CAAC,OAAO,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;oBAC/C,UAAU,CAAC,OAAO,EAAE,GAAG,OAAO,IAAI,CAAC,CAAC;gBACtC,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,+DAA+D;YACjE,CAAC;YACD,8DAA8D;YAC9D,sDAAsD;YACtD,+DAA+D;YAC/D,8DAA8D;YAC9D,8CAA8C;YAC9C,cAAc,CACZ,OAAO,EACP,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,QAAQ,OAAO,CAAC,GAAG,KAAK,OAAO,IAAI,EAC/D,MAAM,CACP,CAAC;QACJ,CAAC;QAAC,MAAM,CAAC;YACP,kEAAkE;YAClE,+DAA+D;QACjE,CAAC;IACH,CAAC;CACF;AAED;;;;GAIG;AACH,SAAS,YAAY,CAAC,MAAgB;IACpC,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;AACjC,CAAC;AAED;;;;;GAKG;AACH,SAAS,cAAc,CAAC,KAAc;IACpC,MAAM,GAAG,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACnE,MAAM,SAAS,GAAG,GAAG;SAClB,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC;SACxB,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;SACpB,IAAI,EAAE,CAAC;IACV,IAAI,SAAS,CAAC,MAAM,IAAI,kBAAkB;QAAE,OAAO,SAAS,CAAC;IAC7D,OAAO,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,kBAAkB,CAAC,gBAAgB,SAAS,CAAC,MAAM,GAAG,kBAAkB,SAAS,CAAC;AACjH,CAAC;AAED,SAAS,KAAK,CAAC,EAAU,EAAE,MAAmB;IAC5C,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QAC7B,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,OAAO,EAAE,CAAC;YACV,OAAO;QACT,CAAC;QACD,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YAC5B,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YAC7C,OAAO,EAAE,CAAC;QACZ,CAAC,EAAE,EAAE,CAAC,CAAC;QACP,MAAM,OAAO,GAAG,GAAS,EAAE;YACzB,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YAC7C,OAAO,EAAE,CAAC;QACZ,CAAC,CAAC;QACF,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;IAC5D,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -2,7 +2,7 @@ import type { AgentMessage } from "@earendil-works/pi-agent-core";
|
|
|
2
2
|
import type { Model, Usage } from "@earendil-works/pi-ai";
|
|
3
3
|
import { type WireGuardHorizon } from "../turn-runner/wire-shaping.js";
|
|
4
4
|
import type { MemoryContextCache } from "./store.js";
|
|
5
|
-
import {
|
|
5
|
+
import type { MemorySession } from "./session.js";
|
|
6
6
|
import type { Observation, ObservationalMemoryActivityEvent, ObservationalMemorySettings, ObservationalMemorySettingsInput } from "../types/memory.js";
|
|
7
7
|
import { type RawMemoryMessage } from "./observational-prompts.js";
|
|
8
8
|
/**
|
|
@@ -167,8 +167,13 @@ export interface ObservationalContextTransformOptions {
|
|
|
167
167
|
horizon: WireGuardHorizon;
|
|
168
168
|
}
|
|
169
169
|
export interface ObservationalMemoryUpdateOptions {
|
|
170
|
-
/**
|
|
171
|
-
|
|
170
|
+
/**
|
|
171
|
+
* Memory session that owns the durable memory rows. The observer and
|
|
172
|
+
* reflector wrap each storage call in `session.withDb`, so the
|
|
173
|
+
* cross-process lock is held only for the duration of each query and
|
|
174
|
+
* a peer duet CLI can step in between writes.
|
|
175
|
+
*/
|
|
176
|
+
session: MemorySession;
|
|
172
177
|
/** Frozen context-pack cache; queried for the global memories the observer can attribute usage against. */
|
|
173
178
|
memory: MemoryContextCache;
|
|
174
179
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"observational.d.ts","sourceRoot":"","sources":["../../../src/memory/observational.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,+BAA+B,CAAC;AAClE,OAAO,KAAK,EAAgB,KAAK,EAAe,KAAK,EAAE,MAAM,uBAAuB,CAAC;AAIrF,OAAO,EAML,KAAK,gBAAgB,EACtB,MAAM,gCAAgC,CAAC;AACxC,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AACrD,OAAO,
|
|
1
|
+
{"version":3,"file":"observational.d.ts","sourceRoot":"","sources":["../../../src/memory/observational.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,+BAA+B,CAAC;AAClE,OAAO,KAAK,EAAgB,KAAK,EAAe,KAAK,EAAE,MAAM,uBAAuB,CAAC;AAIrF,OAAO,EAML,KAAK,gBAAgB,EACtB,MAAM,gCAAgC,CAAC;AACxC,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AACrD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAOlD,OAAO,KAAK,EACV,WAAW,EAEX,gCAAgC,EAChC,2BAA2B,EAC3B,gCAAgC,EACjC,MAAM,oBAAoB,CAAC;AAQ5B,OAAO,EAQL,KAAK,gBAAgB,EACtB,MAAM,4BAA4B,CAAC;AAEpC;;;;;;GAMG;AACH,eAAO,MAAM,yBAAyB,SAAU,CAAC;AAEjD;;;;;;;;GAQG;AACH,eAAO,MAAM,oBAAoB;IAC/B,iDAAiD;;IAEjD,2DAA2D;;IAE3D,yCAAyC;;CAEjC,CAAC;AAEX;;;;;;;GAOG;AACH,eAAO,MAAM,YAAY,MAAM,CAAC;AAEhC;;;;;GAKG;AACH,eAAO,MAAM,sBAAsB;IACjC;;;;;;;;;;;;;;OAcG;;IAEH;;;;;;;;;;;OAWG;;IAEH;;;;OAIG;;CAEK,CAAC;AAeX,MAAM,WAAW,oBAAoB;IACnC,WAAW,EAAE;QACX,aAAa,EAAE,MAAM,CAAC;QACtB,mBAAmB,EAAE,MAAM,CAAC;QAC5B,uBAAuB,EAAE,MAAM,CAAC;QAChC,gBAAgB,EAAE,MAAM,CAAC;QACzB,sBAAsB,EAAE,MAAM,CAAC;KAChC,CAAC;IACF,UAAU,EAAE;QACV,iBAAiB,EAAE,MAAM,CAAC;QAC1B,gBAAgB,EAAE,MAAM,CAAC;KAC1B,CAAC;IACF,wBAAwB,EAAE,MAAM,CAAC;CAClC;AAED;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,gBAAgB,EAAE,MAAM,GAAG,oBAAoB,CAqBlF;AAMD,MAAM,WAAW,cAAc;IAC7B,8EAA8E;IAC9E,SAAS,EAAE,OAAO,CAAC;IACnB,4DAA4D;IAC5D,YAAY,EAAE,MAAM,CAAC;IACrB;;;;;;OAMG;IACH,kBAAkB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC9B,gFAAgF;IAChF,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,4EAA4E;IAC5E,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,kFAAkF;IAClF,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAMD,MAAM,WAAW,eAAe;IAC9B,qEAAqE;IACrE,YAAY,EAAE,MAAM,CAAC;IACrB,2EAA2E;IAC3E,qBAAqB,CAAC,EAAE,MAAM,CAAC;CAChC;AAoDD,MAAM,WAAW,wBAAwB;IACvC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;CAClC;AAED,qBAAa,kBAAkB;IAC7B,OAAO,CAAC,QAAQ,CAAC,UAAU,CAA8C;gBAE7D,MAAM,EAAE,wBAAwB;IAkB5C,OAAO,CAAC,WAAW,EAAE,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC;IAYxC,aAAa,IAAI,MAAM,EAAE;CAG1B;AAED,MAAM,WAAW,oCAAoC;IACnD,MAAM,EAAE,kBAAkB,CAAC;IAC3B;;;OAGG;IACH,gBAAgB,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,EAAE,gCAAgC,CAAC;IAC5C;;;;;;OAMG;IACH,YAAY,CAAC,EAAE,MAAM,IAAI,CAAC;IAC1B;;;;;;;;;OASG;IACH,OAAO,EAAE,gBAAgB,CAAC;CAC3B;AAED,MAAM,WAAW,gCAAgC;IAC/C;;;;;OAKG;IACH,OAAO,EAAE,aAAa,CAAC;IACvB,2GAA2G;IAC3G,MAAM,EAAE,kBAAkB,CAAC;IAC3B;;;;;OAKG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;OAGG;IACH,gBAAgB,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,EAAE,gCAAgC,CAAC;IAC5C,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,YAAY,EAAE,CAAC;IACzB,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,IAAI,CAAC;IACjC,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,gCAAgC,KAAK,IAAI,CAAC;CAChE;AAED,MAAM,WAAW,+BAA+B;IAC9C,YAAY,EAAE,WAAW,EAAE,CAAC;IAC5B,WAAW,EAAE,WAAW,EAAE,CAAC;CAC5B;AAED;;;;;GAKG;AACH,wBAAgB,kCAAkC,CAChD,gBAAgB,EAAE,MAAM,EACxB,KAAK,CAAC,EAAE,gCAAgC,GACvC,2BAA2B,CAwB7B;AAED;;;;;;;GAOG;AACH,wBAAgB,mCAAmC,CAAC,QAAQ,EAAE,2BAA2B,GAAG,IAAI,CAwB/F;AAED,wBAAgB,mCAAmC,CAAC,OAAO,EAAE,oCAAoC,IAIjF,UAAU,YAAY,EAAE,KAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CAmEjE;AA8BD,wBAAsB,yBAAyB,CAC7C,OAAO,EAAE,gCAAgC,GACxC,OAAO,CAAC,+BAA+B,CAAC,CAyE1C;AAED,wBAAgB,8BAA8B,CAAC,YAAY,EAAE,MAAM,GAAG,MAAM,CAS3E;AAsCD;;;;;GAKG;AACH,wBAAgB,iCAAiC,CAAC,QAAQ,EAAE,YAAY,EAAE,GAAG,YAAY,EAAE,CAE1F;AAgRD,wBAAsB,6BAA6B,CAAC,OAAO,EAAE;IAC3D,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,CAAC,YAAY,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;CAClD,GAAG,OAAO,CAAC,MAAM,CAAC,CAclB;AAED,wBAAgB,gCAAgC,CAAC,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,GAAG,MAAM,CAY3F;AAMD;;;;;;;;;;;;GAYG;AACH,wBAAgB,8BAA8B,CAC5C,QAAQ,EAAE,gBAAgB,EAAE,EAC5B,gBAAgB,EAAE,MAAM,GACvB,gBAAgB,EAAE,CAmBpB;AAqBD,wBAAgB,wBAAwB,CACtC,QAAQ,EAAE,gBAAgB,EAAE,EAC5B,YAAY,EAAE,WAAW,EAAE,GAC1B,gBAAgB,EAAE,CAMpB;AA0BD,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,YAAY,EAAE,GAAG,gBAAgB,EAAE,CAI/E;AAED,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,YAAY,GAAG,gBAAgB,GAAG,SAAS,CAgBrF;AA6OD,wBAAgB,wBAAwB,CAAC,YAAY,EAAE,MAAM,GAAG,MAAM,CAWrE;AAcD;;;;;;GAMG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAEnD"}
|
|
@@ -332,7 +332,7 @@ export async function updateObservationalMemory(options) {
|
|
|
332
332
|
// so the model can attribute usage back to specific cross-session
|
|
333
333
|
// memories.
|
|
334
334
|
const localSnapshot = options.sessionId
|
|
335
|
-
? await readSessionObservations(options.
|
|
335
|
+
? await readSessionObservations(options.session, options.sessionId)
|
|
336
336
|
: { observations: [], estimatedObservationTokens: 0 };
|
|
337
337
|
const globalPack = options.memory.getContextPack().global;
|
|
338
338
|
const unobservedMessages = getUnobservedMessageTail(rawMessages, localSnapshot.observations);
|
|
@@ -344,7 +344,7 @@ export async function updateObservationalMemory(options) {
|
|
|
344
344
|
message: "Observing conversation into memory...",
|
|
345
345
|
});
|
|
346
346
|
const { observation, usageBumped } = await activateObservations({
|
|
347
|
-
|
|
347
|
+
session: options.session,
|
|
348
348
|
messages: unobservedMessages,
|
|
349
349
|
previousLocalObservations: localSnapshot.observations,
|
|
350
350
|
attributableMemories: globalPack,
|
|
@@ -365,7 +365,7 @@ export async function updateObservationalMemory(options) {
|
|
|
365
365
|
});
|
|
366
366
|
}
|
|
367
367
|
if (options.sessionId) {
|
|
368
|
-
const refreshed = await readSessionObservations(options.
|
|
368
|
+
const refreshed = await readSessionObservations(options.session, options.sessionId);
|
|
369
369
|
if (refreshed.estimatedObservationTokens >= settings.reflection.observationTokens) {
|
|
370
370
|
emitMemoryActivity(options.onActivity, {
|
|
371
371
|
phase: "reflection",
|
|
@@ -373,7 +373,7 @@ export async function updateObservationalMemory(options) {
|
|
|
373
373
|
message: "Reflecting memory observations...",
|
|
374
374
|
});
|
|
375
375
|
const reflections = await reflectObservations({
|
|
376
|
-
|
|
376
|
+
session: options.session,
|
|
377
377
|
sessionObservations: refreshed.observations,
|
|
378
378
|
settings,
|
|
379
379
|
sessionId: options.sessionId,
|
|
@@ -455,14 +455,14 @@ async function activateObservations(args) {
|
|
|
455
455
|
// Usage attribution applies whether or not the observer had new
|
|
456
456
|
// memory worth recording — a turn can lean on a prior memory even
|
|
457
457
|
// when nothing new is durable.
|
|
458
|
-
const usageBumped = await applyUsageBumps(args.
|
|
458
|
+
const usageBumped = await applyUsageBumps(args.session, args.attributableMemories, observations.usedObservationIds);
|
|
459
459
|
if (!observations.hasMemory || !observations.observations.trim()) {
|
|
460
460
|
// The same low-signal messages may become useful context for a
|
|
461
461
|
// later suffix, so do not record an empty checkpoint.
|
|
462
462
|
return { usageBumped };
|
|
463
463
|
}
|
|
464
464
|
const range = `${args.messages[0]?.id ?? "unknown"}:${args.messages[args.messages.length - 1]?.id ?? "unknown"}`;
|
|
465
|
-
const observation = await appendObservation(args.
|
|
465
|
+
const observation = await appendObservation(args.session, {
|
|
466
466
|
kind: "observation",
|
|
467
467
|
...(args.sessionId !== undefined ? { sessionId: args.sessionId } : {}),
|
|
468
468
|
observedDate: new Date().toISOString().slice(0, 10),
|
|
@@ -482,7 +482,7 @@ async function activateObservations(args) {
|
|
|
482
482
|
* ids unconditionally so a strict guard here is the only line of
|
|
483
483
|
* defense.
|
|
484
484
|
*/
|
|
485
|
-
async function applyUsageBumps(
|
|
485
|
+
async function applyUsageBumps(session, candidates, usedIds) {
|
|
486
486
|
if (!usedIds || usedIds.length === 0)
|
|
487
487
|
return [];
|
|
488
488
|
const candidatesById = new Map(candidates.map((c) => [c.id, c]));
|
|
@@ -491,11 +491,11 @@ async function applyUsageBumps(db, candidates, usedIds) {
|
|
|
491
491
|
.filter((observation) => Boolean(observation));
|
|
492
492
|
if (validated.length === 0)
|
|
493
493
|
return [];
|
|
494
|
-
await bumpLastUsed(
|
|
494
|
+
await bumpLastUsed(session, validated.map((observation) => observation.id), Date.now());
|
|
495
495
|
return validated;
|
|
496
496
|
}
|
|
497
497
|
async function reflectObservations(args) {
|
|
498
|
-
const {
|
|
498
|
+
const { session, sessionObservations, settings, sessionId, model, onUsage } = args;
|
|
499
499
|
const source = sessionObservations.map((observation) => observation.content).join("\n\n");
|
|
500
500
|
const rendered = renderObservationGroupsForReflection(source) ?? source;
|
|
501
501
|
const targetTokens = settings.reflection.bufferActivation;
|
|
@@ -542,7 +542,7 @@ async function reflectObservations(args) {
|
|
|
542
542
|
// replaced by the new reflection. Other sessions' rows in the
|
|
543
543
|
// global pool are untouched, which is what makes cross-session
|
|
544
544
|
// memory durable past a reflection event.
|
|
545
|
-
await replaceSessionObservations(
|
|
545
|
+
await replaceSessionObservations(session, sessionId, [reflected]);
|
|
546
546
|
return [reflected];
|
|
547
547
|
}
|
|
548
548
|
async function observe(messages, previousLocalObservations, attributableMemories, settings, model, onUsage) {
|