@loreai/core 0.18.0 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bun/curator.d.ts.map +1 -1
- package/dist/bun/db.d.ts +73 -0
- package/dist/bun/db.d.ts.map +1 -1
- package/dist/bun/distillation.d.ts +2 -13
- package/dist/bun/distillation.d.ts.map +1 -1
- package/dist/bun/embedding.d.ts +5 -1
- package/dist/bun/embedding.d.ts.map +1 -1
- package/dist/bun/gradient.d.ts +9 -0
- package/dist/bun/gradient.d.ts.map +1 -1
- package/dist/bun/index.d.ts +2 -2
- package/dist/bun/index.d.ts.map +1 -1
- package/dist/bun/index.js +817 -99
- package/dist/bun/index.js.map +4 -4
- package/dist/bun/ltm.d.ts +99 -5
- package/dist/bun/ltm.d.ts.map +1 -1
- package/dist/bun/session-limiter.d.ts +26 -0
- package/dist/bun/session-limiter.d.ts.map +1 -0
- package/dist/bun/temporal.d.ts +2 -0
- package/dist/bun/temporal.d.ts.map +1 -1
- package/dist/node/curator.d.ts.map +1 -1
- package/dist/node/db.d.ts +73 -0
- package/dist/node/db.d.ts.map +1 -1
- package/dist/node/distillation.d.ts +2 -13
- package/dist/node/distillation.d.ts.map +1 -1
- package/dist/node/embedding.d.ts +5 -1
- package/dist/node/embedding.d.ts.map +1 -1
- package/dist/node/gradient.d.ts +9 -0
- package/dist/node/gradient.d.ts.map +1 -1
- package/dist/node/index.d.ts +2 -2
- package/dist/node/index.d.ts.map +1 -1
- package/dist/node/index.js +817 -99
- package/dist/node/index.js.map +4 -4
- package/dist/node/ltm.d.ts +99 -5
- package/dist/node/ltm.d.ts.map +1 -1
- package/dist/node/session-limiter.d.ts +26 -0
- package/dist/node/session-limiter.d.ts.map +1 -0
- package/dist/node/temporal.d.ts +2 -0
- package/dist/node/temporal.d.ts.map +1 -1
- package/dist/types/curator.d.ts.map +1 -1
- package/dist/types/db.d.ts +73 -0
- package/dist/types/db.d.ts.map +1 -1
- package/dist/types/distillation.d.ts +2 -13
- package/dist/types/distillation.d.ts.map +1 -1
- package/dist/types/embedding.d.ts +5 -1
- package/dist/types/embedding.d.ts.map +1 -1
- package/dist/types/gradient.d.ts +9 -0
- package/dist/types/gradient.d.ts.map +1 -1
- package/dist/types/index.d.ts +2 -2
- package/dist/types/index.d.ts.map +1 -1
- package/dist/types/ltm.d.ts +99 -5
- package/dist/types/ltm.d.ts.map +1 -1
- package/dist/types/session-limiter.d.ts +26 -0
- package/dist/types/session-limiter.d.ts.map +1 -0
- package/dist/types/temporal.d.ts +2 -0
- package/dist/types/temporal.d.ts.map +1 -1
- package/package.json +2 -1
- package/src/curator.ts +54 -2
- package/src/db.ts +347 -0
- package/src/distillation.ts +55 -14
- package/src/embedding.ts +28 -3
- package/src/gradient.ts +183 -74
- package/src/index.ts +8 -0
- package/src/ltm.ts +480 -45
- package/src/session-limiter.ts +47 -0
- package/src/temporal.ts +10 -0
package/dist/types/ltm.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ltm.d.ts","sourceRoot":"","sources":["../../src/ltm.ts"],"names":[],"mappings":"AAaA,MAAM,MAAM,cAAc,GAAG;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;CACzB,CAAC;AAWF,wBAAgB,MAAM,CAAC,KAAK,EAAE;IAC5B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,SAAS,GAAG,QAAQ,CAAC;IAC5B,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,+FAA+F;IAC/F,EAAE,CAAC,EAAE,MAAM,CAAC;CACb,GAAG,MAAM,CAuFT;AAED,wBAAgB,MAAM,CACpB,EAAE,EAAE,MAAM,EACV,KAAK,EAAE;IAAE,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,UAAU,CAAC,EAAE,MAAM,CAAA;CAAE,QA4BjD;AAED,wBAAgB,MAAM,CAAC,EAAE,EAAE,MAAM,QAEhC;AAuCD;;;;;;;;GAQG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,EAAE;IACxC,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,GAAG;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CA0CvC;AAED,wBAAgB,UAAU,CACxB,WAAW,EAAE,MAAM,EACnB,YAAY,UAAO,GAClB,cAAc,EAAE,CAoBlB;AAqED
|
|
1
|
+
{"version":3,"file":"ltm.d.ts","sourceRoot":"","sources":["../../src/ltm.ts"],"names":[],"mappings":"AAaA,MAAM,MAAM,cAAc,GAAG;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;CACzB,CAAC;AAWF,wBAAgB,MAAM,CAAC,KAAK,EAAE;IAC5B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,SAAS,GAAG,QAAQ,CAAC;IAC5B,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,+FAA+F;IAC/F,EAAE,CAAC,EAAE,MAAM,CAAC;CACb,GAAG,MAAM,CAuFT;AAED,wBAAgB,MAAM,CACpB,EAAE,EAAE,MAAM,EACV,KAAK,EAAE;IAAE,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,UAAU,CAAC,EAAE,MAAM,CAAA;CAAE,QA4BjD;AAED,wBAAgB,MAAM,CAAC,EAAE,EAAE,MAAM,QAEhC;AAuCD;;;;;;;;GAQG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,EAAE;IACxC,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,GAAG;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CA0CvC;AAED,wBAAgB,UAAU,CACxB,WAAW,EAAE,MAAM,EACnB,YAAY,UAAO,GAClB,cAAc,EAAE,CAoBlB;AAqED;;;GAGG;AACH,MAAM,MAAM,iBAAiB,GAAG,UAAU,GAAG,SAAS,GAAG,YAAY,GAAG,cAAc,GAAG,QAAQ,CAAC;AAElG,6DAA6D;AAC7D,MAAM,MAAM,iBAAiB,GAAG;IAC9B;gEAC4D;IAC5D,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,wEAAwE;IACxE,UAAU,CAAC,EAAE,CAAC,iBAAiB,GAAG,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,EAAE,CAAC;IACnD;;;uCAGmC;IACnC,iBAAiB,CAAC,EAAE,CAAC,iBAAiB,GAAG,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,EAAE,CAAC;CAC3D,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAsB,UAAU,CAC9B,WAAW,EAAE,MAAM,EACnB,SAAS,EAAE,MAAM,GAAG,SAAS,EAC7B,SAAS,EAAE,MAAM,EACjB,OAAO,CAAC,EAAE,iBAAiB,GAC1B,OAAO,CAAC,cAAc,EAAE,CAAC,CAoM3B;AAgCD,wBAAgB,GAAG,IAAI,cAAc,EAAE,CAMtC;AAED,0EAA0E;AAC1E,wBAAgB,YAAY,IAAI,cAAc,EAAE,CAQ/C;AAgCD,wBAAgB,MAAM,CAAC,KAAK,EAAE;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,GAAG,cAAc,EAAE,CAkCnB;AAED,MAAM,MAAM,oBAAoB,GAAG,cAAc,GAAG;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC;AAErE;;;GAGG;AACH,wBAAgB,YAAY,CAAC,KAAK,EAAE;IAClC,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,GAAG,oBAAoB,EAAE,CA6BzB;AAED;;;;;GAKG;AACH,wBAAgB,yBAAyB,CAAC,KAAK,EAAE;IAC/C,KAAK,EAAE,MAAM,CAAC;IACd,kBAAkB,EAAE,MAAM,CAAC;IAC3B,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,GAAG,oBAAoB,EAAE,CA0BzB;AAED,wBAAgB,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,cAAc,GAAG,IAAI,CAIrD;AAED;;;;;;;;;GASG;AACH,wBAAgB,cAAc,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAQxD;AASD;;;;;GAKG;AACH,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAQrD;AAED;;;GAGG;AACH,wBAAgB,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAQrD;AAED;;;GAGG;AACH,wBAAgB,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAwBhD;AAED;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAqBtE;AAED;;;;;GAKG;AACH,wBAAgB,aAAa,IAAI,MAAM,CAsCtC;AAMD,MAAM,MAAM,cAAc,GAAG;IAC3B,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,WAAW,GAAG,YAAY,GAAG,WAAW,GAAG,OAAO,CAAC;IACzD,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB,CAAC;AAEF;;;;;;;GAOG;AACH,wBAAgB,KAAK,CAAC,WAAW,EAAE,MAAM,GAAG,cAAc,EAAE,CA8E3D;AAMD,MAAM,MAAM,YAAY,GAAG;IACzB,SAAS,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IACzC,MAAM,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CAC9C,CAAC;AAEF,+EAA+E;AAC/E,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,CAE7D;AAED,MAAM,MAAM,WAAW,GAAG;IACxB,QAAQ,EAAE,YAAY,EAAE,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,2EAA2E;IAC3E,gBAAgB,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACtC,iFAAiF;IACjF,WAAW,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAClC,CAAC;AA8KF,wBAAsB,WAAW,CAC/B,WAAW,EAAE,MAAM,EACnB,IAAI,CAAC,EAAE;IAAE,MAAM,CAAC,EAAE,OAAO,CAAA;CAAE,GAC1B,OAAO,CAAC,WAAW,CAAC,CAKtB;AAED,0EAA0E;AAC1E,wBAAsB,iBAAiB,CACrC,IAAI,CAAC,EAAE;IAAE,MAAM,CAAC,EAAE,OAAO,CAAA;CAAE,GAC1B,OAAO,CAAC,WAAW,CAAC,CAWtB;AAMD,MAAM,MAAM,mBAAmB,GAAG,YAAY,GAAG,SAAS,GAAG,iBAAiB,CAAC;AAS/E,0CAA0C;AAC1C,wBAAgB,mBAAmB,CAAC,KAAK,EAAE;IACzC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,OAAO,CAAC;IAClB,MAAM,EAAE,mBAAmB,CAAC;CAC7B,GAAG,IAAI,CAgBP;AAED;;;;GAIG;AACH,wBAAgB,yBAAyB,CACvC,SAAS,EAAE,MAAM,GAAG,IAAI,EACxB,MAAM,EAAE,WAAW,EACnB,QAAQ,EAAE,OAAO,EACjB,MAAM,EAAE,mBAAmB,GAC1B,IAAI,CAiBN;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,iBAAiB,CAC/B,SAAS,EAAE,MAAM,GAAG,IAAI,EACxB,MAAM,EAAE,WAAW,GAClB,IAAI,CAuDN;AAED,wDAAwD;AACxD,wBAAgB,gBAAgB,CAC9B,SAAS,EAAE,MAAM,GAAG,IAAI,GACvB,KAAK,CAAC;IAAE,UAAU,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,OAAO,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC,CAelE;AAED,kDAAkD;AAClD,wBAAgB,qBAAqB,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,GAAG,MAAM,CAWtE;AAKD;;;;GAIG;AACH,wBAAgB,kBAAkB,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,GAAG,IAAI,CAwBjE;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAgB,uBAAuB,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,GAAG,MAAM,GAAG,IAAI,CA2C/E;AAED,sDAAsD;AACtD,wBAAgB,uBAAuB,CACrC,SAAS,EAAE,MAAM,GAAG,IAAI,EACxB,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,MAAM,GACjB,IAAI,CAGN;AAED,8EAA8E;AAC9E,wBAAgB,uBAAuB,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,GAAG,MAAM,GAAG,IAAI,CAU/E"}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Per-key concurrency limiter using p-limit.
|
|
3
|
+
*
|
|
4
|
+
* Each key (typically a session ID) gets its own p-limit(1) instance,
|
|
5
|
+
* serializing async operations on the same key while allowing different
|
|
6
|
+
* keys to run fully in parallel.
|
|
7
|
+
*
|
|
8
|
+
* Two independent limiter pools are provided — one for distillation and
|
|
9
|
+
* one for curation — so they don't block each other.
|
|
10
|
+
*/
|
|
11
|
+
import pLimit from "p-limit";
|
|
12
|
+
type LimitFunction = ReturnType<typeof pLimit>;
|
|
13
|
+
/** Serializes distillation.run() and metaDistill() per session. */
|
|
14
|
+
export declare const distillLimiter: {
|
|
15
|
+
get: (key: string) => LimitFunction;
|
|
16
|
+
isBusy: (key: string) => boolean;
|
|
17
|
+
clear: () => void;
|
|
18
|
+
};
|
|
19
|
+
/** Serializes curator.run() per session with skip-if-busy semantics. */
|
|
20
|
+
export declare const curatorLimiter: {
|
|
21
|
+
get: (key: string) => LimitFunction;
|
|
22
|
+
isBusy: (key: string) => boolean;
|
|
23
|
+
clear: () => void;
|
|
24
|
+
};
|
|
25
|
+
export {};
|
|
26
|
+
//# sourceMappingURL=session-limiter.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"session-limiter.d.ts","sourceRoot":"","sources":["../../src/session-limiter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,MAAM,MAAM,SAAS,CAAC;AAE7B,KAAK,aAAa,GAAG,UAAU,CAAC,OAAO,MAAM,CAAC,CAAC;AA6B/C,mEAAmE;AACnE,eAAO,MAAM,cAAc;eAxBP,MAAM,KAAG,aAAa;kBAUnB,MAAM,KAAG,OAAO;iBAMnB,IAAI;CAQyB,CAAC;AAElD,wEAAwE;AACxE,eAAO,MAAM,cAAc;eA3BP,MAAM,KAAG,aAAa;kBAUnB,MAAM,KAAG,OAAO;iBAMnB,IAAI;CAWyB,CAAC"}
|
package/dist/types/temporal.d.ts
CHANGED
|
@@ -84,6 +84,8 @@ export declare function searchScored(input: {
|
|
|
84
84
|
*/
|
|
85
85
|
export declare function temporalCnorm(timestamps: number[], now?: number): number;
|
|
86
86
|
export declare function count(projectPath: string, sessionID?: string): number;
|
|
87
|
+
/** Quick existence check — true if any temporal messages exist for this session. */
|
|
88
|
+
export declare function hasMessages(projectPath: string, sessionID: string): boolean;
|
|
87
89
|
export declare function undistilledCount(projectPath: string, sessionID?: string): number;
|
|
88
90
|
/** Sum of estimated tokens across undistilled messages for a project/session. */
|
|
89
91
|
export declare function undistilledTokens(projectPath: string, sessionID?: string): number;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"temporal.d.ts","sourceRoot":"","sources":["../../src/temporal.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAQrD;;;;;;;;;;;;;;;;;;GAkBG;AACH,eAAO,MAAM,gBAAgB,WAAS,CAAC;AAEvC;;;;;;;;;GASG;AACH,wBAAgB,WAAW,CAAC,KAAK,EAAE,QAAQ,EAAE,GAAG,MAAM,CAarD;AAiBD,wBAAgB,KAAK,CAAC,KAAK,EAAE;IAC3B,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,WAAW,CAAC;IAClB,KAAK,EAAE,QAAQ,EAAE,CAAC;CACnB,QA8CA;AAED,MAAM,MAAM,eAAe,GAAG;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,wBAAgB,WAAW,CACzB,WAAW,EAAE,MAAM,EACnB,SAAS,CAAC,EAAE,MAAM,GACjB,eAAe,EAAE,CASnB;AAED,wBAAgB,SAAS,CACvB,WAAW,EAAE,MAAM,EACnB,SAAS,EAAE,MAAM,GAChB,eAAe,EAAE,CAOnB;AAED,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,QAQ1C;AA6BD,wBAAgB,MAAM,CAAC,KAAK,EAAE;IAC5B,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,GAAG,eAAe,EAAE,CA8BpB;AAED,MAAM,MAAM,qBAAqB,GAAG,eAAe,GAAG;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC;AAEvE;;;GAGG;AACH,wBAAgB,YAAY,CAAC,KAAK,EAAE;IAClC,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,GAAG,qBAAqB,EAAE,CAwB1B;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,aAAa,CAC3B,UAAU,EAAE,MAAM,EAAE,EACpB,GAAG,GAAE,MAAmB,GACvB,MAAM,CAoBR;AAED,wBAAgB,KAAK,CAAC,WAAW,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,CAWrE;AAED,wBAAgB,gBAAgB,CAC9B,WAAW,EAAE,MAAM,EACnB,SAAS,CAAC,EAAE,MAAM,GACjB,MAAM,CAWR;AAED,iFAAiF;AACjF,wBAAgB,iBAAiB,CAC/B,WAAW,EAAE,MAAM,EACnB,SAAS,CAAC,EAAE,MAAM,GACjB,MAAM,CAWR;AAED,MAAM,MAAM,WAAW,GAAG;IACxB,kFAAkF;IAClF,UAAU,EAAE,MAAM,CAAC;IACnB,8FAA8F;IAC9F,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF;;;;;;;;;;;;GAYG;AACH,wBAAgB,KAAK,CAAC,KAAK,EAAE;IAC3B,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;CACtB,GAAG,WAAW,CA0Ed"}
|
|
1
|
+
{"version":3,"file":"temporal.d.ts","sourceRoot":"","sources":["../../src/temporal.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAQrD;;;;;;;;;;;;;;;;;;GAkBG;AACH,eAAO,MAAM,gBAAgB,WAAS,CAAC;AAEvC;;;;;;;;;GASG;AACH,wBAAgB,WAAW,CAAC,KAAK,EAAE,QAAQ,EAAE,GAAG,MAAM,CAarD;AAiBD,wBAAgB,KAAK,CAAC,KAAK,EAAE;IAC3B,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,WAAW,CAAC;IAClB,KAAK,EAAE,QAAQ,EAAE,CAAC;CACnB,QA8CA;AAED,MAAM,MAAM,eAAe,GAAG;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,wBAAgB,WAAW,CACzB,WAAW,EAAE,MAAM,EACnB,SAAS,CAAC,EAAE,MAAM,GACjB,eAAe,EAAE,CASnB;AAED,wBAAgB,SAAS,CACvB,WAAW,EAAE,MAAM,EACnB,SAAS,EAAE,MAAM,GAChB,eAAe,EAAE,CAOnB;AAED,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,QAQ1C;AA6BD,wBAAgB,MAAM,CAAC,KAAK,EAAE;IAC5B,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,GAAG,eAAe,EAAE,CA8BpB;AAED,MAAM,MAAM,qBAAqB,GAAG,eAAe,GAAG;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC;AAEvE;;;GAGG;AACH,wBAAgB,YAAY,CAAC,KAAK,EAAE;IAClC,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,GAAG,qBAAqB,EAAE,CAwB1B;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,aAAa,CAC3B,UAAU,EAAE,MAAM,EAAE,EACpB,GAAG,GAAE,MAAmB,GACvB,MAAM,CAoBR;AAED,wBAAgB,KAAK,CAAC,WAAW,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,CAWrE;AAED,oFAAoF;AACpF,wBAAgB,WAAW,CAAC,WAAW,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAO3E;AAED,wBAAgB,gBAAgB,CAC9B,WAAW,EAAE,MAAM,EACnB,SAAS,CAAC,EAAE,MAAM,GACjB,MAAM,CAWR;AAED,iFAAiF;AACjF,wBAAgB,iBAAiB,CAC/B,WAAW,EAAE,MAAM,EACnB,SAAS,CAAC,EAAE,MAAM,GACjB,MAAM,CAWR;AAED,MAAM,MAAM,WAAW,GAAG;IACxB,kFAAkF;IAClF,UAAU,EAAE,MAAM,CAAC;IACnB,8FAA8F;IAC9F,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF;;;;;;;;;;;;GAYG;AACH,wBAAgB,KAAK,CAAC,KAAK,EAAE;IAC3B,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;CACtB,GAAG,WAAW,CA0Ed"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@loreai/core",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.19.0",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"license": "FSL-1.1-Apache-2.0",
|
|
6
6
|
"description": "Shared memory engine for Lore — three-tier storage, distillation, gradient context management",
|
|
@@ -26,6 +26,7 @@
|
|
|
26
26
|
"@huggingface/hub": "2.11.0",
|
|
27
27
|
"@huggingface/transformers": "^3.7.1",
|
|
28
28
|
"micromark": "^4.0.0",
|
|
29
|
+
"p-limit": "7",
|
|
29
30
|
"remark": "^15.0.1",
|
|
30
31
|
"uuidv7": "^1.1.0",
|
|
31
32
|
"zod": "^4.3.6"
|
package/src/curator.ts
CHANGED
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
import { config } from "./config";
|
|
2
|
+
import { saveSessionTracking, loadSessionTracking, ensureProject } from "./db";
|
|
2
3
|
import * as temporal from "./temporal";
|
|
3
4
|
import * as ltm from "./ltm";
|
|
4
5
|
import * as log from "./log";
|
|
5
6
|
import { CURATOR_SYSTEM, curatorUser, CONSOLIDATION_SYSTEM, consolidationUser } from "./prompt";
|
|
6
7
|
import { detectAndFormat } from "./instruction-detect";
|
|
8
|
+
import { curatorLimiter } from "./session-limiter";
|
|
7
9
|
import type { LLMClient } from "./types";
|
|
8
10
|
|
|
9
11
|
/**
|
|
@@ -121,8 +123,20 @@ export function applyOps(
|
|
|
121
123
|
// Track which messages we've already curated — per session to prevent
|
|
122
124
|
// cross-session leaking (curation on session A advancing the timestamp
|
|
123
125
|
// past session B's messages, causing B's curation to find < 3 recent).
|
|
126
|
+
// In-memory cache backed by session_state DB table so it survives restarts.
|
|
124
127
|
const lastCuratedAt = new Map<string, number>();
|
|
125
128
|
|
|
129
|
+
/** Get the last-curated timestamp for a session, loading from DB if needed. */
|
|
130
|
+
function getLastCuratedAt(sessionID: string): number {
|
|
131
|
+
const cached = lastCuratedAt.get(sessionID);
|
|
132
|
+
if (cached !== undefined) return cached;
|
|
133
|
+
// Load from DB on first access
|
|
134
|
+
const persisted = loadSessionTracking(sessionID);
|
|
135
|
+
const ts = persisted?.lastCuratedAt ?? 0;
|
|
136
|
+
lastCuratedAt.set(sessionID, ts);
|
|
137
|
+
return ts;
|
|
138
|
+
}
|
|
139
|
+
|
|
126
140
|
export async function run(input: {
|
|
127
141
|
llm: LLMClient;
|
|
128
142
|
projectPath: string;
|
|
@@ -132,9 +146,33 @@ export async function run(input: {
|
|
|
132
146
|
const cfg = config();
|
|
133
147
|
if (!cfg.curator.enabled) return { created: 0, updated: 0, deleted: 0 };
|
|
134
148
|
|
|
149
|
+
// Skip-if-busy: curation is periodic, not accumulative. If a curation is
|
|
150
|
+
// already running for this session, skip — the next trigger will pick up
|
|
151
|
+
// any new messages. Serializing would waste an LLM call.
|
|
152
|
+
//
|
|
153
|
+
// The isBusy() check and get()() enqueue are both synchronous — in Node's
|
|
154
|
+
// single-threaded event loop no microtask can interleave between them, so
|
|
155
|
+
// there is no TOCTOU race. The p-limit(1) serialization is a safety net
|
|
156
|
+
// if this invariant is ever violated.
|
|
157
|
+
if (curatorLimiter.isBusy(input.sessionID)) {
|
|
158
|
+
log.info(`curation skipped: already running for session ${input.sessionID.slice(0, 16)}`);
|
|
159
|
+
return { created: 0, updated: 0, deleted: 0 };
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
return curatorLimiter.get(input.sessionID)(() => runInner(input));
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
async function runInner(input: {
|
|
166
|
+
llm: LLMClient;
|
|
167
|
+
projectPath: string;
|
|
168
|
+
sessionID: string;
|
|
169
|
+
model?: { providerID: string; modelID: string };
|
|
170
|
+
}): Promise<{ created: number; updated: number; deleted: number }> {
|
|
171
|
+
const cfg = config();
|
|
172
|
+
|
|
135
173
|
// Get recent messages since last curation
|
|
136
174
|
const all = temporal.bySession(input.projectPath, input.sessionID);
|
|
137
|
-
const sessionCuratedAt =
|
|
175
|
+
const sessionCuratedAt = getLastCuratedAt(input.sessionID);
|
|
138
176
|
const recent = all.filter((m) => m.created_at > sessionCuratedAt);
|
|
139
177
|
if (recent.length < 3) return { created: 0, updated: 0, deleted: 0 };
|
|
140
178
|
|
|
@@ -190,12 +228,26 @@ export async function run(input: {
|
|
|
190
228
|
log.info(`post-curation dedup: merged ${dupes.totalRemoved} duplicate entries`);
|
|
191
229
|
result.deleted += dupes.totalRemoved;
|
|
192
230
|
}
|
|
231
|
+
// Record auto-signals for adaptive threshold calibration.
|
|
232
|
+
// Merged pairs → accept; non-merged high-similarity pairs → reject.
|
|
233
|
+
if (dupes.pairSimilarities.size > 0) {
|
|
234
|
+
const pid = ensureProject(input.projectPath);
|
|
235
|
+
ltm.recordAutoSignals(pid, dupes);
|
|
236
|
+
// Recalibrate if enough data has accumulated
|
|
237
|
+
const newThreshold = ltm.calibrateDedupThreshold(pid);
|
|
238
|
+
if (newThreshold !== null) {
|
|
239
|
+
const count = ltm.getDedupFeedbackCount(pid);
|
|
240
|
+
ltm.saveCalibratedThreshold(pid, newThreshold, count);
|
|
241
|
+
}
|
|
242
|
+
}
|
|
193
243
|
} catch (err) {
|
|
194
244
|
log.warn("post-curation dedup failed (non-fatal):", err);
|
|
195
245
|
}
|
|
196
246
|
}
|
|
197
247
|
|
|
198
|
-
|
|
248
|
+
const now = Date.now();
|
|
249
|
+
lastCuratedAt.set(input.sessionID, now);
|
|
250
|
+
saveSessionTracking(input.sessionID, { lastCuratedAt: now });
|
|
199
251
|
return result;
|
|
200
252
|
}
|
|
201
253
|
|
package/src/db.ts
CHANGED
|
@@ -515,6 +515,55 @@ const MIGRATIONS: string[] = [
|
|
|
515
515
|
AND ih.source_id = '__declined__'
|
|
516
516
|
);
|
|
517
517
|
`,
|
|
518
|
+
`
|
|
519
|
+
-- Version 23: Persist volatile session tracking state across restarts.
|
|
520
|
+
-- Previously these were in-memory only, causing duplicate processing,
|
|
521
|
+
-- false compaction detection, and expensive prompt cache busts on restart.
|
|
522
|
+
ALTER TABLE session_state ADD COLUMN last_curated_at INTEGER NOT NULL DEFAULT 0;
|
|
523
|
+
ALTER TABLE session_state ADD COLUMN message_count INTEGER NOT NULL DEFAULT 0;
|
|
524
|
+
ALTER TABLE session_state ADD COLUMN turns_since_curation INTEGER NOT NULL DEFAULT 0;
|
|
525
|
+
ALTER TABLE session_state ADD COLUMN ltm_cache_text TEXT;
|
|
526
|
+
ALTER TABLE session_state ADD COLUMN ltm_cache_tokens INTEGER;
|
|
527
|
+
ALTER TABLE session_state ADD COLUMN ltm_pin_text TEXT;
|
|
528
|
+
ALTER TABLE session_state ADD COLUMN ltm_pin_tokens INTEGER;
|
|
529
|
+
ALTER TABLE session_state ADD COLUMN consecutive_text_only_turns INTEGER NOT NULL DEFAULT 0;
|
|
530
|
+
`,
|
|
531
|
+
`
|
|
532
|
+
-- Version 24: Persist remaining volatile session state across restarts.
|
|
533
|
+
-- Session identity (Tier 1/2/3 session correlation)
|
|
534
|
+
ALTER TABLE session_state ADD COLUMN fingerprint TEXT NOT NULL DEFAULT '';
|
|
535
|
+
ALTER TABLE session_state ADD COLUMN header_session_id TEXT;
|
|
536
|
+
ALTER TABLE session_state ADD COLUMN header_name TEXT;
|
|
537
|
+
-- Cache warming state
|
|
538
|
+
ALTER TABLE session_state ADD COLUMN resolved_conversation_ttl TEXT NOT NULL DEFAULT '5m';
|
|
539
|
+
ALTER TABLE session_state ADD COLUMN warmup_state TEXT;
|
|
540
|
+
-- Gradient calibration state (survives restarts to avoid uncalibrated busts)
|
|
541
|
+
ALTER TABLE session_state ADD COLUMN dynamic_context_cap REAL NOT NULL DEFAULT 0;
|
|
542
|
+
ALTER TABLE session_state ADD COLUMN bust_rate_ema REAL NOT NULL DEFAULT -1;
|
|
543
|
+
ALTER TABLE session_state ADD COLUMN inter_bust_interval_ema REAL NOT NULL DEFAULT -1;
|
|
544
|
+
ALTER TABLE session_state ADD COLUMN last_layer INTEGER NOT NULL DEFAULT 0;
|
|
545
|
+
ALTER TABLE session_state ADD COLUMN last_known_input INTEGER NOT NULL DEFAULT 0;
|
|
546
|
+
ALTER TABLE session_state ADD COLUMN last_turn_at INTEGER NOT NULL DEFAULT 0;
|
|
547
|
+
ALTER TABLE session_state ADD COLUMN last_bust_at INTEGER NOT NULL DEFAULT 0;
|
|
548
|
+
`,
|
|
549
|
+
`
|
|
550
|
+
-- Version 25: Adaptive dedup threshold — store accept/reject feedback
|
|
551
|
+
-- on embedding-based duplicate pairs for per-project threshold calibration.
|
|
552
|
+
-- Titles stored instead of FK IDs because entries are deleted during dedup;
|
|
553
|
+
-- the similarity float is the actual calibration input.
|
|
554
|
+
CREATE TABLE IF NOT EXISTS dedup_feedback (
|
|
555
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
556
|
+
project_id TEXT,
|
|
557
|
+
entry_a_title TEXT NOT NULL,
|
|
558
|
+
entry_b_title TEXT NOT NULL,
|
|
559
|
+
similarity REAL NOT NULL,
|
|
560
|
+
accepted INTEGER NOT NULL,
|
|
561
|
+
source TEXT NOT NULL DEFAULT 'manual',
|
|
562
|
+
created_at INTEGER NOT NULL
|
|
563
|
+
);
|
|
564
|
+
CREATE INDEX IF NOT EXISTS idx_dedup_feedback_project
|
|
565
|
+
ON dedup_feedback(project_id);
|
|
566
|
+
`,
|
|
518
567
|
];
|
|
519
568
|
|
|
520
569
|
/** Return the resolved path of the SQLite database file. */
|
|
@@ -765,6 +814,21 @@ export function close() {
|
|
|
765
814
|
* git_remote was not yet populated (pre-v14 rows), it is backfilled lazily.
|
|
766
815
|
*/
|
|
767
816
|
export function ensureProject(path: string, name?: string): string {
|
|
817
|
+
// Guard: reject synthetic test paths when targeting the production DB.
|
|
818
|
+
// Test paths like "/test/ltm/project" are absolute paths that don't exist
|
|
819
|
+
// on any real filesystem — they're only valid in test suites running against
|
|
820
|
+
// a temp DB (LORE_DB_PATH set by test preload). If we see such a path
|
|
821
|
+
// without LORE_DB_PATH being set, a test is likely hitting the production DB.
|
|
822
|
+
// Note: LORE_DB_PATH unset is used as a proxy for "production DB". This
|
|
823
|
+
// wouldn't catch the unlikely case of someone explicitly setting LORE_DB_PATH
|
|
824
|
+
// to the default production path, but that's not a realistic scenario.
|
|
825
|
+
if (!process.env.LORE_DB_PATH && /^\/test\//.test(path)) {
|
|
826
|
+
throw new Error(
|
|
827
|
+
`Refusing to create project with test path "${path}" in the production DB. ` +
|
|
828
|
+
`Set LORE_DB_PATH to a temp path, or run tests via \`bun test\` from the repo root.`,
|
|
829
|
+
);
|
|
830
|
+
}
|
|
831
|
+
|
|
768
832
|
// 1. Exact path match (fast path)
|
|
769
833
|
const existing = db()
|
|
770
834
|
.query("SELECT id, git_remote FROM projects WHERE path = ?")
|
|
@@ -1068,6 +1132,289 @@ export function loadAllSessionCosts(): Map<string, SessionCostSnapshot> {
|
|
|
1068
1132
|
return result;
|
|
1069
1133
|
}
|
|
1070
1134
|
|
|
1135
|
+
// ---------------------------------------------------------------------------
|
|
1136
|
+
// Session tracking state (session_state table, v23 columns)
|
|
1137
|
+
// ---------------------------------------------------------------------------
|
|
1138
|
+
|
|
1139
|
+
/** Fields that can be persisted for session tracking state. */
|
|
1140
|
+
export type SessionTrackingState = {
|
|
1141
|
+
lastCuratedAt?: number;
|
|
1142
|
+
messageCount?: number;
|
|
1143
|
+
turnsSinceCuration?: number;
|
|
1144
|
+
consecutiveTextOnlyTurns?: number;
|
|
1145
|
+
ltmCacheText?: string | null;
|
|
1146
|
+
ltmCacheTokens?: number | null;
|
|
1147
|
+
ltmPinText?: string | null;
|
|
1148
|
+
ltmPinTokens?: number | null;
|
|
1149
|
+
// v24: session identity
|
|
1150
|
+
fingerprint?: string;
|
|
1151
|
+
headerSessionId?: string | null;
|
|
1152
|
+
headerName?: string | null;
|
|
1153
|
+
// v24: cache warming
|
|
1154
|
+
resolvedConversationTTL?: string;
|
|
1155
|
+
warmupState?: string | null; // JSON blob
|
|
1156
|
+
// v24: gradient calibration
|
|
1157
|
+
dynamicContextCap?: number;
|
|
1158
|
+
bustRateEMA?: number;
|
|
1159
|
+
interBustIntervalEMA?: number;
|
|
1160
|
+
lastLayer?: number;
|
|
1161
|
+
lastKnownInput?: number;
|
|
1162
|
+
lastTurnAt?: number;
|
|
1163
|
+
lastBustAt?: number;
|
|
1164
|
+
};
|
|
1165
|
+
|
|
1166
|
+
/**
|
|
1167
|
+
* Persist session tracking state. Ensures the row exists, then updates
|
|
1168
|
+
* only the fields that are explicitly provided (not undefined).
|
|
1169
|
+
*/
|
|
1170
|
+
export function saveSessionTracking(sessionID: string, state: SessionTrackingState): void {
|
|
1171
|
+
const now = Date.now();
|
|
1172
|
+
|
|
1173
|
+
// Ensure row exists (no-op if it already does)
|
|
1174
|
+
db()
|
|
1175
|
+
.query(
|
|
1176
|
+
"INSERT OR IGNORE INTO session_state (session_id, force_min_layer, updated_at) VALUES (?, 0, ?)",
|
|
1177
|
+
)
|
|
1178
|
+
.run(sessionID, now);
|
|
1179
|
+
|
|
1180
|
+
// Build SET clauses for only the provided fields
|
|
1181
|
+
const sets: string[] = ["updated_at = ?"];
|
|
1182
|
+
const vals: (string | number | null)[] = [now];
|
|
1183
|
+
|
|
1184
|
+
if (state.lastCuratedAt !== undefined) {
|
|
1185
|
+
sets.push("last_curated_at = ?");
|
|
1186
|
+
vals.push(state.lastCuratedAt);
|
|
1187
|
+
}
|
|
1188
|
+
if (state.messageCount !== undefined) {
|
|
1189
|
+
sets.push("message_count = ?");
|
|
1190
|
+
vals.push(state.messageCount);
|
|
1191
|
+
}
|
|
1192
|
+
if (state.turnsSinceCuration !== undefined) {
|
|
1193
|
+
sets.push("turns_since_curation = ?");
|
|
1194
|
+
vals.push(state.turnsSinceCuration);
|
|
1195
|
+
}
|
|
1196
|
+
if (state.consecutiveTextOnlyTurns !== undefined) {
|
|
1197
|
+
sets.push("consecutive_text_only_turns = ?");
|
|
1198
|
+
vals.push(state.consecutiveTextOnlyTurns);
|
|
1199
|
+
}
|
|
1200
|
+
if (state.ltmCacheText !== undefined) {
|
|
1201
|
+
sets.push("ltm_cache_text = ?");
|
|
1202
|
+
vals.push(state.ltmCacheText);
|
|
1203
|
+
}
|
|
1204
|
+
if (state.ltmCacheTokens !== undefined) {
|
|
1205
|
+
sets.push("ltm_cache_tokens = ?");
|
|
1206
|
+
vals.push(state.ltmCacheTokens);
|
|
1207
|
+
}
|
|
1208
|
+
if (state.ltmPinText !== undefined) {
|
|
1209
|
+
sets.push("ltm_pin_text = ?");
|
|
1210
|
+
vals.push(state.ltmPinText);
|
|
1211
|
+
}
|
|
1212
|
+
if (state.ltmPinTokens !== undefined) {
|
|
1213
|
+
sets.push("ltm_pin_tokens = ?");
|
|
1214
|
+
vals.push(state.ltmPinTokens);
|
|
1215
|
+
}
|
|
1216
|
+
// v24: session identity
|
|
1217
|
+
if (state.fingerprint !== undefined) {
|
|
1218
|
+
sets.push("fingerprint = ?");
|
|
1219
|
+
vals.push(state.fingerprint);
|
|
1220
|
+
}
|
|
1221
|
+
if (state.headerSessionId !== undefined) {
|
|
1222
|
+
sets.push("header_session_id = ?");
|
|
1223
|
+
vals.push(state.headerSessionId);
|
|
1224
|
+
}
|
|
1225
|
+
if (state.headerName !== undefined) {
|
|
1226
|
+
sets.push("header_name = ?");
|
|
1227
|
+
vals.push(state.headerName);
|
|
1228
|
+
}
|
|
1229
|
+
// v24: cache warming
|
|
1230
|
+
if (state.resolvedConversationTTL !== undefined) {
|
|
1231
|
+
sets.push("resolved_conversation_ttl = ?");
|
|
1232
|
+
vals.push(state.resolvedConversationTTL);
|
|
1233
|
+
}
|
|
1234
|
+
if (state.warmupState !== undefined) {
|
|
1235
|
+
sets.push("warmup_state = ?");
|
|
1236
|
+
vals.push(state.warmupState);
|
|
1237
|
+
}
|
|
1238
|
+
// v24: gradient calibration
|
|
1239
|
+
if (state.dynamicContextCap !== undefined) {
|
|
1240
|
+
sets.push("dynamic_context_cap = ?");
|
|
1241
|
+
vals.push(state.dynamicContextCap);
|
|
1242
|
+
}
|
|
1243
|
+
if (state.bustRateEMA !== undefined) {
|
|
1244
|
+
sets.push("bust_rate_ema = ?");
|
|
1245
|
+
vals.push(state.bustRateEMA);
|
|
1246
|
+
}
|
|
1247
|
+
if (state.interBustIntervalEMA !== undefined) {
|
|
1248
|
+
sets.push("inter_bust_interval_ema = ?");
|
|
1249
|
+
vals.push(state.interBustIntervalEMA);
|
|
1250
|
+
}
|
|
1251
|
+
if (state.lastLayer !== undefined) {
|
|
1252
|
+
sets.push("last_layer = ?");
|
|
1253
|
+
vals.push(state.lastLayer);
|
|
1254
|
+
}
|
|
1255
|
+
if (state.lastKnownInput !== undefined) {
|
|
1256
|
+
sets.push("last_known_input = ?");
|
|
1257
|
+
vals.push(state.lastKnownInput);
|
|
1258
|
+
}
|
|
1259
|
+
if (state.lastTurnAt !== undefined) {
|
|
1260
|
+
sets.push("last_turn_at = ?");
|
|
1261
|
+
vals.push(state.lastTurnAt);
|
|
1262
|
+
}
|
|
1263
|
+
if (state.lastBustAt !== undefined) {
|
|
1264
|
+
sets.push("last_bust_at = ?");
|
|
1265
|
+
vals.push(state.lastBustAt);
|
|
1266
|
+
}
|
|
1267
|
+
|
|
1268
|
+
// Update only the specified columns
|
|
1269
|
+
db()
|
|
1270
|
+
.query(
|
|
1271
|
+
"UPDATE session_state SET " + sets.join(", ") + " WHERE session_id = ?",
|
|
1272
|
+
)
|
|
1273
|
+
.run(...vals, sessionID);
|
|
1274
|
+
}
|
|
1275
|
+
|
|
1276
|
+
/** Loaded session tracking state. */
|
|
1277
|
+
export type LoadedSessionTracking = {
|
|
1278
|
+
lastCuratedAt: number;
|
|
1279
|
+
messageCount: number;
|
|
1280
|
+
turnsSinceCuration: number;
|
|
1281
|
+
consecutiveTextOnlyTurns: number;
|
|
1282
|
+
ltmCacheText: string | null;
|
|
1283
|
+
ltmCacheTokens: number | null;
|
|
1284
|
+
ltmPinText: string | null;
|
|
1285
|
+
ltmPinTokens: number | null;
|
|
1286
|
+
// v24: session identity
|
|
1287
|
+
fingerprint: string;
|
|
1288
|
+
headerSessionId: string | null;
|
|
1289
|
+
headerName: string | null;
|
|
1290
|
+
// v24: cache warming
|
|
1291
|
+
resolvedConversationTTL: string;
|
|
1292
|
+
warmupState: string | null;
|
|
1293
|
+
// v24: gradient calibration
|
|
1294
|
+
dynamicContextCap: number;
|
|
1295
|
+
bustRateEMA: number;
|
|
1296
|
+
interBustIntervalEMA: number;
|
|
1297
|
+
lastLayer: number;
|
|
1298
|
+
lastKnownInput: number;
|
|
1299
|
+
lastTurnAt: number;
|
|
1300
|
+
lastBustAt: number;
|
|
1301
|
+
};
|
|
1302
|
+
|
|
1303
|
+
/**
|
|
1304
|
+
* Load persisted session tracking state. Returns null if no row exists.
|
|
1305
|
+
*/
|
|
1306
|
+
export function loadSessionTracking(sessionID: string): LoadedSessionTracking | null {
|
|
1307
|
+
const row = db()
|
|
1308
|
+
.query(
|
|
1309
|
+
`SELECT last_curated_at, message_count, turns_since_curation,
|
|
1310
|
+
consecutive_text_only_turns,
|
|
1311
|
+
ltm_cache_text, ltm_cache_tokens, ltm_pin_text, ltm_pin_tokens,
|
|
1312
|
+
fingerprint, header_session_id, header_name,
|
|
1313
|
+
resolved_conversation_ttl, warmup_state,
|
|
1314
|
+
dynamic_context_cap, bust_rate_ema, inter_bust_interval_ema,
|
|
1315
|
+
last_layer, last_known_input, last_turn_at, last_bust_at
|
|
1316
|
+
FROM session_state WHERE session_id = ?`,
|
|
1317
|
+
)
|
|
1318
|
+
.get(sessionID) as {
|
|
1319
|
+
last_curated_at: number;
|
|
1320
|
+
message_count: number;
|
|
1321
|
+
turns_since_curation: number;
|
|
1322
|
+
consecutive_text_only_turns: number;
|
|
1323
|
+
ltm_cache_text: string | null;
|
|
1324
|
+
ltm_cache_tokens: number | null;
|
|
1325
|
+
ltm_pin_text: string | null;
|
|
1326
|
+
ltm_pin_tokens: number | null;
|
|
1327
|
+
fingerprint: string;
|
|
1328
|
+
header_session_id: string | null;
|
|
1329
|
+
header_name: string | null;
|
|
1330
|
+
resolved_conversation_ttl: string;
|
|
1331
|
+
warmup_state: string | null;
|
|
1332
|
+
dynamic_context_cap: number;
|
|
1333
|
+
bust_rate_ema: number;
|
|
1334
|
+
inter_bust_interval_ema: number;
|
|
1335
|
+
last_layer: number;
|
|
1336
|
+
last_known_input: number;
|
|
1337
|
+
last_turn_at: number;
|
|
1338
|
+
last_bust_at: number;
|
|
1339
|
+
} | null;
|
|
1340
|
+
if (!row) return null;
|
|
1341
|
+
return {
|
|
1342
|
+
lastCuratedAt: row.last_curated_at,
|
|
1343
|
+
messageCount: row.message_count,
|
|
1344
|
+
turnsSinceCuration: row.turns_since_curation,
|
|
1345
|
+
consecutiveTextOnlyTurns: row.consecutive_text_only_turns,
|
|
1346
|
+
ltmCacheText: row.ltm_cache_text,
|
|
1347
|
+
ltmCacheTokens: row.ltm_cache_tokens,
|
|
1348
|
+
ltmPinText: row.ltm_pin_text,
|
|
1349
|
+
ltmPinTokens: row.ltm_pin_tokens,
|
|
1350
|
+
fingerprint: row.fingerprint,
|
|
1351
|
+
headerSessionId: row.header_session_id,
|
|
1352
|
+
headerName: row.header_name,
|
|
1353
|
+
resolvedConversationTTL: row.resolved_conversation_ttl,
|
|
1354
|
+
warmupState: row.warmup_state,
|
|
1355
|
+
dynamicContextCap: row.dynamic_context_cap,
|
|
1356
|
+
bustRateEMA: row.bust_rate_ema,
|
|
1357
|
+
interBustIntervalEMA: row.inter_bust_interval_ema,
|
|
1358
|
+
lastLayer: row.last_layer,
|
|
1359
|
+
lastKnownInput: row.last_known_input,
|
|
1360
|
+
lastTurnAt: row.last_turn_at,
|
|
1361
|
+
lastBustAt: row.last_bust_at,
|
|
1362
|
+
};
|
|
1363
|
+
}
|
|
1364
|
+
|
|
1365
|
+
/**
|
|
1366
|
+
* Load all persisted header → session ID mappings from the session_state table.
|
|
1367
|
+
*
|
|
1368
|
+
* Used on gateway startup (in initIfNeeded) to pre-populate the in-memory
|
|
1369
|
+
* headerSessionIndex so Tier 1 session identification works immediately
|
|
1370
|
+
* after a process restart — without this, the first post-restart request
|
|
1371
|
+
* with a known session header would generate a new session ID and orphan
|
|
1372
|
+
* the old session's persisted state.
|
|
1373
|
+
*/
|
|
1374
|
+
export function loadHeaderSessionIndex(): Array<{
|
|
1375
|
+
sessionId: string;
|
|
1376
|
+
headerSessionId: string;
|
|
1377
|
+
headerName: string;
|
|
1378
|
+
}> {
|
|
1379
|
+
const rows = db()
|
|
1380
|
+
.query(
|
|
1381
|
+
`SELECT session_id, header_session_id, header_name
|
|
1382
|
+
FROM session_state
|
|
1383
|
+
WHERE header_session_id IS NOT NULL AND header_name IS NOT NULL`,
|
|
1384
|
+
)
|
|
1385
|
+
.all() as Array<{
|
|
1386
|
+
session_id: string;
|
|
1387
|
+
header_session_id: string;
|
|
1388
|
+
header_name: string;
|
|
1389
|
+
}>;
|
|
1390
|
+
return rows.map((row) => ({
|
|
1391
|
+
sessionId: row.session_id,
|
|
1392
|
+
headerSessionId: row.header_session_id,
|
|
1393
|
+
headerName: row.header_name,
|
|
1394
|
+
}));
|
|
1395
|
+
}
|
|
1396
|
+
|
|
1397
|
+
// ---------------------------------------------------------------------------
|
|
1398
|
+
// Key-value store (kv_meta table)
|
|
1399
|
+
// ---------------------------------------------------------------------------
|
|
1400
|
+
|
|
1401
|
+
/** Get a kv_meta value by key. Returns null if not found. */
|
|
1402
|
+
export function getKV(key: string): string | null {
|
|
1403
|
+
const row = db()
|
|
1404
|
+
.query("SELECT value FROM kv_meta WHERE key = ?")
|
|
1405
|
+
.get(key) as { value: string } | null;
|
|
1406
|
+
return row?.value ?? null;
|
|
1407
|
+
}
|
|
1408
|
+
|
|
1409
|
+
/** Set a kv_meta value (upsert). */
|
|
1410
|
+
export function setKV(key: string, value: string): void {
|
|
1411
|
+
db()
|
|
1412
|
+
.query(
|
|
1413
|
+
"INSERT INTO kv_meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = ?",
|
|
1414
|
+
)
|
|
1415
|
+
.run(key, value, value);
|
|
1416
|
+
}
|
|
1417
|
+
|
|
1071
1418
|
// ---------------------------------------------------------------------------
|
|
1072
1419
|
// Installation metadata (metadata table)
|
|
1073
1420
|
// ---------------------------------------------------------------------------
|
package/src/distillation.ts
CHANGED
|
@@ -14,6 +14,7 @@ import {
|
|
|
14
14
|
} from "./prompt";
|
|
15
15
|
import { toolStripAnnotation } from "./gradient";
|
|
16
16
|
import { workerSessionIDs } from "./worker";
|
|
17
|
+
import { distillLimiter } from "./session-limiter";
|
|
17
18
|
import type { LLMClient } from "./types";
|
|
18
19
|
|
|
19
20
|
// Re-export for backwards compat — index.ts and others may still import from here.
|
|
@@ -610,8 +611,23 @@ function resetOrphans(projectPath: string, sessionID: string): number {
|
|
|
610
611
|
return orphans.length;
|
|
611
612
|
}
|
|
612
613
|
|
|
613
|
-
// Main distillation entry point — called on session.idle or when urgent
|
|
614
|
+
// Main distillation entry point — called on session.idle or when urgent.
|
|
615
|
+
// Serialized per session via p-limit(1) to prevent concurrent runs from
|
|
616
|
+
// reading the same undistilled messages and producing duplicate rows.
|
|
614
617
|
export async function run(input: {
|
|
618
|
+
llm: LLMClient;
|
|
619
|
+
projectPath: string;
|
|
620
|
+
sessionID: string;
|
|
621
|
+
model?: { providerID: string; modelID: string };
|
|
622
|
+
force?: boolean;
|
|
623
|
+
skipMeta?: boolean;
|
|
624
|
+
urgent?: boolean;
|
|
625
|
+
callType?: "batch" | "direct";
|
|
626
|
+
}): Promise<{ rounds: number; distilled: number }> {
|
|
627
|
+
return distillLimiter.get(input.sessionID)(() => runInner(input));
|
|
628
|
+
}
|
|
629
|
+
|
|
630
|
+
async function runInner(input: {
|
|
615
631
|
llm: LLMClient;
|
|
616
632
|
projectPath: string;
|
|
617
633
|
sessionID: string;
|
|
@@ -697,7 +713,8 @@ export async function run(input: {
|
|
|
697
713
|
gen0Count(input.projectPath, input.sessionID) >=
|
|
698
714
|
cfg.distillation.metaThreshold
|
|
699
715
|
) {
|
|
700
|
-
|
|
716
|
+
// Call inner directly — we're already under the per-session limiter.
|
|
717
|
+
await metaDistillInner({
|
|
701
718
|
llm: input.llm,
|
|
702
719
|
projectPath: input.projectPath,
|
|
703
720
|
sessionID: input.sessionID,
|
|
@@ -776,17 +793,29 @@ async function distillSegment(input: {
|
|
|
776
793
|
return null;
|
|
777
794
|
}
|
|
778
795
|
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
796
|
+
// Atomic: store distillation + mark source messages as distilled in one
|
|
797
|
+
// transaction. Without this, a crash between the two statements would leave
|
|
798
|
+
// messages undistilled but with an existing distillation row, causing
|
|
799
|
+
// re-processing on restart and duplicate distillation content.
|
|
800
|
+
let distillId: string;
|
|
801
|
+
db().exec("BEGIN IMMEDIATE");
|
|
802
|
+
try {
|
|
803
|
+
distillId = storeDistillation({
|
|
804
|
+
projectPath: input.projectPath,
|
|
805
|
+
sessionID: input.sessionID,
|
|
806
|
+
observations: result.observations,
|
|
807
|
+
sourceIDs: input.messages.map((m) => m.id),
|
|
808
|
+
generation: 0,
|
|
809
|
+
rCompression: rComp,
|
|
810
|
+
cNorm,
|
|
811
|
+
callType: input.callType,
|
|
812
|
+
});
|
|
813
|
+
temporal.markDistilled(input.messages.map((m) => m.id));
|
|
814
|
+
db().exec("COMMIT");
|
|
815
|
+
} catch (e) {
|
|
816
|
+
db().exec("ROLLBACK");
|
|
817
|
+
throw e;
|
|
818
|
+
}
|
|
790
819
|
|
|
791
820
|
log.info(
|
|
792
821
|
`distill segment: ${input.messages.length} msgs, ` +
|
|
@@ -840,7 +869,8 @@ async function distillSegment(input: {
|
|
|
840
869
|
* via `<previous-meta-summary>` so the LLM updates in place rather than
|
|
841
870
|
* re-deriving from scratch.
|
|
842
871
|
*
|
|
843
|
-
*
|
|
872
|
+
* Serialized per session via the same p-limit(1) as `run()`. Exported for
|
|
873
|
+
* the idle handler which calls metaDistill() independently of run().
|
|
844
874
|
*/
|
|
845
875
|
export async function metaDistill(input: {
|
|
846
876
|
llm: LLMClient;
|
|
@@ -849,6 +879,17 @@ export async function metaDistill(input: {
|
|
|
849
879
|
model?: { providerID: string; modelID: string };
|
|
850
880
|
urgent?: boolean;
|
|
851
881
|
callType?: "batch" | "direct";
|
|
882
|
+
}): Promise<DistillationResult | null> {
|
|
883
|
+
return distillLimiter.get(input.sessionID)(() => metaDistillInner(input));
|
|
884
|
+
}
|
|
885
|
+
|
|
886
|
+
async function metaDistillInner(input: {
|
|
887
|
+
llm: LLMClient;
|
|
888
|
+
projectPath: string;
|
|
889
|
+
sessionID: string;
|
|
890
|
+
model?: { providerID: string; modelID: string };
|
|
891
|
+
urgent?: boolean;
|
|
892
|
+
callType?: "batch" | "direct";
|
|
852
893
|
}): Promise<DistillationResult | null> {
|
|
853
894
|
const existing = loadGen0(input.projectPath, input.sessionID);
|
|
854
895
|
|