@loreai/core 0.11.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bun/db.d.ts.map +1 -1
- package/dist/bun/distillation.d.ts +26 -0
- package/dist/bun/distillation.d.ts.map +1 -1
- package/dist/bun/gradient.d.ts.map +1 -1
- package/dist/bun/index.js +132 -27
- package/dist/bun/index.js.map +3 -3
- package/dist/bun/recall.d.ts.map +1 -1
- package/dist/bun/temporal.d.ts +15 -0
- package/dist/bun/temporal.d.ts.map +1 -1
- package/dist/node/db.d.ts.map +1 -1
- package/dist/node/distillation.d.ts +26 -0
- package/dist/node/distillation.d.ts.map +1 -1
- package/dist/node/gradient.d.ts.map +1 -1
- package/dist/node/index.js +132 -27
- package/dist/node/index.js.map +3 -3
- package/dist/node/recall.d.ts.map +1 -1
- package/dist/node/temporal.d.ts +15 -0
- package/dist/node/temporal.d.ts.map +1 -1
- package/dist/types/db.d.ts.map +1 -1
- package/dist/types/distillation.d.ts +26 -0
- package/dist/types/distillation.d.ts.map +1 -1
- package/dist/types/gradient.d.ts.map +1 -1
- package/dist/types/recall.d.ts.map +1 -1
- package/dist/types/temporal.d.ts +15 -0
- package/dist/types/temporal.d.ts.map +1 -1
- package/package.json +1 -1
- package/src/db.ts +76 -8
- package/src/distillation.ts +125 -19
- package/src/gradient.ts +43 -9
- package/src/recall.ts +18 -0
- package/src/temporal.ts +39 -0
package/dist/bun/db.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"db.d.ts","sourceRoot":"","sources":["../../src/db.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAyVtC,wBAAgB,EAAE,IAAI,QAAQ,
|
|
1
|
+
{"version":3,"file":"db.d.ts","sourceRoot":"","sources":["../../src/db.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAyVtC,wBAAgB,EAAE,IAAI,QAAQ,CAkC7B;AAmGD,wBAAgB,KAAK,SAKpB;AAGD,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,GAAG,MAAM,CAYjE;AAED,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAK1D;AAED,2DAA2D;AAC3D,wBAAgB,WAAW,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAKrD;AAED;;;GAGG;AACH,wBAAgB,UAAU,IAAI,OAAO,CAKpC;AAMD;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAK3D;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAYxE"}
|
|
@@ -3,6 +3,32 @@ import { workerSessionIDs } from "./worker";
|
|
|
3
3
|
import type { LLMClient } from "./types";
|
|
4
4
|
export { workerSessionIDs };
|
|
5
5
|
type TemporalMessage = temporal.TemporalMessage;
|
|
6
|
+
/**
|
|
7
|
+
* Compression health ratio: k / √N.
|
|
8
|
+
*
|
|
9
|
+
* k = distilled token count, N = source token count.
|
|
10
|
+
* Values < 1.0 signal likely lossy compression (below the square-root
|
|
11
|
+
* boundary). Values > 1.0 signal relatively faithful compression.
|
|
12
|
+
*
|
|
13
|
+
* Based on the "LLM Context Square Root Theory" heuristic from
|
|
14
|
+
* D7x7z49/llm-context-idea. The specific threshold is unvalidated —
|
|
15
|
+
* use as a diagnostic signal, not a hard gate.
|
|
16
|
+
*/
|
|
17
|
+
export declare function compressionRatio(distilledTokens: number, sourceTokens: number): number;
|
|
18
|
+
/**
|
|
19
|
+
* Segment detection: group related messages into distillation-sized chunks.
|
|
20
|
+
*
|
|
21
|
+
* When the message count exceeds `maxSegment`, prefers splitting at the
|
|
22
|
+
* largest inter-message time gap (if it's ≥ 3× the median gap) to respect
|
|
23
|
+
* natural conversation boundaries. Falls back to count-based splitting at
|
|
24
|
+
* `maxSegment` when timestamps are uniform.
|
|
25
|
+
*
|
|
26
|
+
* Trailing segments with < 3 messages are merged into the previous segment
|
|
27
|
+
* to avoid tiny distillation inputs with too little context.
|
|
28
|
+
*
|
|
29
|
+
* Exported for testing; `run()` is the production caller.
|
|
30
|
+
*/
|
|
31
|
+
export declare function detectSegments(messages: TemporalMessage[], maxSegment: number): TemporalMessage[][];
|
|
6
32
|
/**
|
|
7
33
|
* Truncate tool outputs within a `TemporalMessage.content` string (produced
|
|
8
34
|
* by `temporal.partsToText`). Plain text and `[reasoning]` chunks pass
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"distillation.d.ts","sourceRoot":"","sources":["../../src/distillation.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,QAAQ,MAAM,YAAY,CAAC;AAWvC,OAAO,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAC5C,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAGzC,OAAO,EAAE,gBAAgB,EAAE,CAAC;AAE5B,KAAK,eAAe,GAAG,QAAQ,CAAC,eAAe,CAAC;
|
|
1
|
+
{"version":3,"file":"distillation.d.ts","sourceRoot":"","sources":["../../src/distillation.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,QAAQ,MAAM,YAAY,CAAC;AAWvC,OAAO,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAC5C,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAGzC,OAAO,EAAE,gBAAgB,EAAE,CAAC;AAE5B,KAAK,eAAe,GAAG,QAAQ,CAAC,eAAe,CAAC;AAEhD;;;;;;;;;;GAUG;AACH,wBAAgB,gBAAgB,CAC9B,eAAe,EAAE,MAAM,EACvB,YAAY,EAAE,MAAM,GACnB,MAAM,CAGR;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,cAAc,CAC5B,QAAQ,EAAE,eAAe,EAAE,EAC3B,UAAU,EAAE,MAAM,GACjB,eAAe,EAAE,EAAE,CAGrB;AAwGD;;;;;;;;;;;;GAYG;AACH,wBAAgB,4BAA4B,CAC1C,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,GACf,MAAM,CAwBR;AAgBD;;;;;;;GAOG;AACH,wBAAgB,cAAc,CAC5B,QAAQ,EAAE,eAAe,EAAE,EAC3B,kBAAkB,CAAC,EAAE,MAAM,GAC1B,MAAM,CAUR;AAED,KAAK,kBAAkB,GAAG;IACxB,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAwBF;;;;;;;;;;;;GAYG;AACH,wBAAgB,sBAAsB,CACpC,WAAW,EAAE,MAAM,EACnB,SAAS,EAAE,MAAM,GAChB,MAAM,GAAG,SAAS,CAEpB;AAwBD,+EAA+E;AAC/E,wBAAgB,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,EAAE,CAQpD;AAED,MAAM,MAAM,YAAY,GAAG;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF;;;;;;;;;;;;;GAaG;AACH,wBAAgB,cAAc,CAC5B,WAAW,EAAE,MAAM,EACnB,SAAS,EAAE,MAAM,EACjB,eAAe,UAAQ,GACtB,YAAY,EAAE,CAqBhB;AA0ID,wBAAsB,GAAG,CAAC,KAAK,EAAE;IAC/B,GAAG,EAAE,SAAS,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC;IAChD,qEAAqE;IACrE,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB,GAAG,OAAO,CAAC;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,CAAC,CA4DjD;AAmED;;;;;;;GAOG;AACH,wBAAsB,WAAW,CAAC,KAAK,EAAE;IACvC,GAAG,EAAE,SAAS,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC;CACjD,GAAG,OAAO,CAAC,kBAAkB,GAAG,IAAI,CAAC,CA+ErC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"gradient.d.ts","sourceRoot":"","sources":["../../src/gradient.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAyB,oBAAoB,EAAqE,MAAM,SAAS,CAAC;AAQ9I,KAAK,gBAAgB,GAAG,oBAAoB,CAAC;AAmJ7C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,wBAAgB,YAAY,CAC1B,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,MAAM,EACnB,GAAG,GAAE,MAAmB,GACvB;IAAE,SAAS,EAAE,KAAK,CAAA;CAAE,GAAG;IAAE,SAAS,EAAE,IAAI,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAU5D;AAED;;;;GAIG;AACH,wBAAgB,oBAAoB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAK/D;AAMD,wBAAgB,cAAc,CAAC,MAAM,EAAE;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,QASzE;AAED;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,MAAM,QAEhD;AAED,sFAAsF;AACtF,wBAAgB,gBAAgB,CAC9B,iBAAiB,EAAE,MAAM,EACzB,qBAAqB,EAAE,MAAM,GAC5B,MAAM,CAIR;AAED,0EAA0E;AAC1E,wBAAgB,YAAY,CAAC,MAAM,EAAE,MAAM,QAE1C;AAED,uEAAuE;AACvE,wBAAgB,YAAY,IAAI,MAAM,CAErC;AAED;;;;;GAKG;AACH,wBAAgB,YAAY,CAAC,WAAW,EAAE,MAAM,GAAG,MAAM,CAIxD;AAWD,wBAAgB,SAAS,CACvB,WAAW,EAAE,MAAM,EACnB,SAAS,CAAC,EAAE,MAAM,EAClB,YAAY,CAAC,EAAE,MAAM,QA2BtB;AAED,wBAAgB,WAAW,IAAI,MAAM,CAEpC;AAED;;;GAGG;AACH,wBAAgB,uBAAuB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAEjE;AAED,0EAA0E;AAC1E,wBAAgB,wBAAwB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAElE;AAED,+EAA+E;AAC/E,wBAAgB,YAAY,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,WAAW,CAK5D;AAED;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,WAAW,EAAE,SAAS,CAAC,EAAE,MAAM,QAWtE;AAGD,wBAAgB,gBAAgB,CAAC,SAAS,CAAC,EAAE,MAAM,QAWlD;AAED;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,SAAS,EAAE,MAAM,GAAG;IACtD,cAAc,EAAE,OAAO,CAAC;IACxB,iBAAiB,EAAE,OAAO,CAAC;IAC3B,aAAa,EAAE,OAAO,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;CACpB,GAAG,IAAI,CASP;AAED;;;;GAIG;AACH,wBAAgB,oBAAoB,CAAC,SAAS,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,IAAI,CAExE;AAwFD,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAwB5E;AA+CD;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,sBAAsB,CACpC,QAAQ,EAAE,gBAAgB,EAAE,EAC5B,cAAc,EAAE,MAAM,GACrB,gBAAgB,EAAE,CA0EpB;AA4YD,wBAAgB,gBAAgB,CAAC,SAAS,CAAC,EAAE,MAAM,QAOlD;AA0BD,wBAAgB,mBAAmB,CAAC,SAAS,CAAC,EAAE,MAAM,QAOrD;AAuFD,MAAM,MAAM,WAAW,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;AAE5C,MAAM,MAAM,eAAe,GAAG;IAC5B,QAAQ,EAAE,gBAAgB,EAAE,CAAC;IAC7B,KAAK,EAAE,WAAW,CAAC;IACnB,eAAe,EAAE,MAAM,CAAC;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IAEpB,MAAM,EAAE,MAAM,CAAC;IACf,eAAe,EAAE,MAAM,CAAC;IACxB,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAIF,wBAAgB,uBAAuB,IAAI,OAAO,CAIjD;
|
|
1
|
+
{"version":3,"file":"gradient.d.ts","sourceRoot":"","sources":["../../src/gradient.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAyB,oBAAoB,EAAqE,MAAM,SAAS,CAAC;AAQ9I,KAAK,gBAAgB,GAAG,oBAAoB,CAAC;AAmJ7C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,wBAAgB,YAAY,CAC1B,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,MAAM,EACnB,GAAG,GAAE,MAAmB,GACvB;IAAE,SAAS,EAAE,KAAK,CAAA;CAAE,GAAG;IAAE,SAAS,EAAE,IAAI,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAU5D;AAED;;;;GAIG;AACH,wBAAgB,oBAAoB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAK/D;AAMD,wBAAgB,cAAc,CAAC,MAAM,EAAE;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,QASzE;AAED;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,MAAM,QAEhD;AAED,sFAAsF;AACtF,wBAAgB,gBAAgB,CAC9B,iBAAiB,EAAE,MAAM,EACzB,qBAAqB,EAAE,MAAM,GAC5B,MAAM,CAIR;AAED,0EAA0E;AAC1E,wBAAgB,YAAY,CAAC,MAAM,EAAE,MAAM,QAE1C;AAED,uEAAuE;AACvE,wBAAgB,YAAY,IAAI,MAAM,CAErC;AAED;;;;;GAKG;AACH,wBAAgB,YAAY,CAAC,WAAW,EAAE,MAAM,GAAG,MAAM,CAIxD;AAWD,wBAAgB,SAAS,CACvB,WAAW,EAAE,MAAM,EACnB,SAAS,CAAC,EAAE,MAAM,EAClB,YAAY,CAAC,EAAE,MAAM,QA2BtB;AAED,wBAAgB,WAAW,IAAI,MAAM,CAEpC;AAED;;;GAGG;AACH,wBAAgB,uBAAuB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAEjE;AAED,0EAA0E;AAC1E,wBAAgB,wBAAwB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAElE;AAED,+EAA+E;AAC/E,wBAAgB,YAAY,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,WAAW,CAK5D;AAED;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,WAAW,EAAE,SAAS,CAAC,EAAE,MAAM,QAWtE;AAGD,wBAAgB,gBAAgB,CAAC,SAAS,CAAC,EAAE,MAAM,QAWlD;AAED;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,SAAS,EAAE,MAAM,GAAG;IACtD,cAAc,EAAE,OAAO,CAAC;IACxB,iBAAiB,EAAE,OAAO,CAAC;IAC3B,aAAa,EAAE,OAAO,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;CACpB,GAAG,IAAI,CASP;AAED;;;;GAIG;AACH,wBAAgB,oBAAoB,CAAC,SAAS,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,IAAI,CAExE;AAwFD,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAwB5E;AA+CD;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,sBAAsB,CACpC,QAAQ,EAAE,gBAAgB,EAAE,EAC5B,cAAc,EAAE,MAAM,GACrB,gBAAgB,EAAE,CA0EpB;AA4YD,wBAAgB,gBAAgB,CAAC,SAAS,CAAC,EAAE,MAAM,QAOlD;AA0BD,wBAAgB,mBAAmB,CAAC,SAAS,CAAC,EAAE,MAAM,QAOrD;AAuFD,MAAM,MAAM,WAAW,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;AAE5C,MAAM,MAAM,eAAe,GAAG;IAC5B,QAAQ,EAAE,gBAAgB,EAAE,CAAC;IAC7B,KAAK,EAAE,WAAW,CAAC;IACnB,eAAe,EAAE,MAAM,CAAC;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IAEpB,MAAM,EAAE,MAAM,CAAC;IACf,eAAe,EAAE,MAAM,CAAC;IACxB,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAIF,wBAAgB,uBAAuB,IAAI,OAAO,CAIjD;AAgSD,wBAAgB,SAAS,CAAC,KAAK,EAAE;IAC/B,QAAQ,EAAE,gBAAgB,EAAE,CAAC;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,GAAG,eAAe,CAwDlB;AAGD,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,gBAAgB,EAAE,GAAG,MAAM,CAErE"}
|
package/dist/bun/index.js
CHANGED
|
@@ -131,6 +131,7 @@ __export(temporal_exports, {
|
|
|
131
131
|
search: () => search2,
|
|
132
132
|
searchScored: () => searchScored,
|
|
133
133
|
store: () => store,
|
|
134
|
+
temporalCnorm: () => temporalCnorm,
|
|
134
135
|
undistilled: () => undistilled,
|
|
135
136
|
undistilledCount: () => undistilledCount
|
|
136
137
|
});
|
|
@@ -493,12 +494,13 @@ function db() {
|
|
|
493
494
|
mkdirSync(dir, { recursive: true });
|
|
494
495
|
path = join(dir, "lore.db");
|
|
495
496
|
}
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
migrate(
|
|
497
|
+
const database = new Database(path);
|
|
498
|
+
database.exec("PRAGMA journal_mode = WAL");
|
|
499
|
+
database.exec("PRAGMA foreign_keys = ON");
|
|
500
|
+
database.exec("PRAGMA busy_timeout = 5000");
|
|
501
|
+
database.exec("PRAGMA auto_vacuum = INCREMENTAL");
|
|
502
|
+
migrate(database);
|
|
503
|
+
instance = database;
|
|
502
504
|
return instance;
|
|
503
505
|
}
|
|
504
506
|
var VACUUM_MIGRATION_INDEX = 2;
|
|
@@ -507,16 +509,47 @@ function migrate(database) {
|
|
|
507
509
|
"SELECT name FROM sqlite_master WHERE type='table' AND name='schema_version'"
|
|
508
510
|
).get();
|
|
509
511
|
const current2 = row ? database.query("SELECT version FROM schema_version").get()?.version ?? 0 : 0;
|
|
510
|
-
if (current2 >= MIGRATIONS.length)
|
|
512
|
+
if (current2 >= MIGRATIONS.length) {
|
|
513
|
+
recoverMissingObjects(database);
|
|
514
|
+
return;
|
|
515
|
+
}
|
|
511
516
|
for (let i = current2; i < MIGRATIONS.length; i++) {
|
|
512
517
|
if (i === VACUUM_MIGRATION_INDEX) {
|
|
513
518
|
database.exec("PRAGMA auto_vacuum = INCREMENTAL");
|
|
514
519
|
database.exec("VACUUM");
|
|
515
520
|
} else {
|
|
516
|
-
|
|
521
|
+
try {
|
|
522
|
+
database.exec(MIGRATIONS[i]);
|
|
523
|
+
} catch (e) {
|
|
524
|
+
if (e instanceof Error && /duplicate column name/i.test(e.message)) {
|
|
525
|
+
const stripped = stripAppliedAlters(MIGRATIONS[i], database);
|
|
526
|
+
if (stripped.trim()) database.exec(stripped);
|
|
527
|
+
} else {
|
|
528
|
+
throw e;
|
|
529
|
+
}
|
|
530
|
+
}
|
|
517
531
|
}
|
|
518
532
|
}
|
|
519
533
|
database.exec(`UPDATE schema_version SET version = ${MIGRATIONS.length}`);
|
|
534
|
+
recoverMissingObjects(database);
|
|
535
|
+
}
|
|
536
|
+
function stripAppliedAlters(migration, database) {
|
|
537
|
+
return migration.replace(
|
|
538
|
+
/ALTER\s+TABLE\s+(\w+)\s+ADD\s+COLUMN\s+(\w+)\b[^;]*;/gi,
|
|
539
|
+
(match, table, column) => {
|
|
540
|
+
const cols = database.query(`PRAGMA table_info(${table})`).all();
|
|
541
|
+
if (cols.some((c) => c.name === column)) return "";
|
|
542
|
+
return match;
|
|
543
|
+
}
|
|
544
|
+
);
|
|
545
|
+
}
|
|
546
|
+
function recoverMissingObjects(database) {
|
|
547
|
+
database.exec(`
|
|
548
|
+
CREATE TABLE IF NOT EXISTS kv_meta (
|
|
549
|
+
key TEXT PRIMARY KEY,
|
|
550
|
+
value TEXT NOT NULL
|
|
551
|
+
);
|
|
552
|
+
`);
|
|
520
553
|
}
|
|
521
554
|
function close() {
|
|
522
555
|
if (instance) {
|
|
@@ -11422,6 +11455,18 @@ function searchScored(input) {
|
|
|
11422
11455
|
return [];
|
|
11423
11456
|
}
|
|
11424
11457
|
}
|
|
11458
|
+
function temporalCnorm(timestamps, now = Date.now()) {
|
|
11459
|
+
const n = timestamps.length;
|
|
11460
|
+
if (n < 2) return 0;
|
|
11461
|
+
const durations = timestamps.map((t2) => now - t2);
|
|
11462
|
+
const totalDuration = durations.reduce((a, b) => a + b, 0);
|
|
11463
|
+
if (totalDuration <= 0) return 0;
|
|
11464
|
+
const weights = durations.map((d) => d / totalDuration);
|
|
11465
|
+
const uniform = 1 / n;
|
|
11466
|
+
const variance = weights.reduce((sum, w) => sum + (w - uniform) ** 2, 0) / n;
|
|
11467
|
+
const maxVariance = (n - 1) / (n * n);
|
|
11468
|
+
return maxVariance === 0 ? 0 : variance / maxVariance;
|
|
11469
|
+
}
|
|
11425
11470
|
function count(projectPath, sessionID) {
|
|
11426
11471
|
const pid = ensureProject(projectPath);
|
|
11427
11472
|
const query = sessionID ? "SELECT COUNT(*) as count FROM temporal_messages WHERE project_id = ? AND session_id = ?" : "SELECT COUNT(*) as count FROM temporal_messages WHERE project_id = ?";
|
|
@@ -26739,6 +26784,8 @@ function check2(projectPath) {
|
|
|
26739
26784
|
// src/distillation.ts
|
|
26740
26785
|
var distillation_exports = {};
|
|
26741
26786
|
__export(distillation_exports, {
|
|
26787
|
+
compressionRatio: () => compressionRatio,
|
|
26788
|
+
detectSegments: () => detectSegments,
|
|
26742
26789
|
latestMetaObservations: () => latestMetaObservations,
|
|
26743
26790
|
loadForSession: () => loadForSession,
|
|
26744
26791
|
messagesToText: () => messagesToText,
|
|
@@ -27519,15 +27566,29 @@ function transform2(input) {
|
|
|
27519
27566
|
return result;
|
|
27520
27567
|
}
|
|
27521
27568
|
function currentTurnStart(messages) {
|
|
27522
|
-
|
|
27569
|
+
if (messages.length === 0) return 0;
|
|
27570
|
+
let boundary = messages.length;
|
|
27523
27571
|
for (let i = messages.length - 1; i >= 0; i--) {
|
|
27524
27572
|
if (messages[i].info.role === "user") {
|
|
27525
|
-
|
|
27573
|
+
boundary = i;
|
|
27526
27574
|
break;
|
|
27527
27575
|
}
|
|
27528
27576
|
}
|
|
27529
|
-
if (
|
|
27530
|
-
|
|
27577
|
+
if (boundary === messages.length) return 0;
|
|
27578
|
+
for (let i = boundary - 1; i >= 0; i--) {
|
|
27579
|
+
const msg = messages[i];
|
|
27580
|
+
const hasToolParts = msg.parts.some(isToolPart);
|
|
27581
|
+
if (hasToolParts) {
|
|
27582
|
+
boundary = i;
|
|
27583
|
+
continue;
|
|
27584
|
+
}
|
|
27585
|
+
if (msg.info.role === "user") {
|
|
27586
|
+
boundary = i;
|
|
27587
|
+
continue;
|
|
27588
|
+
}
|
|
27589
|
+
break;
|
|
27590
|
+
}
|
|
27591
|
+
return boundary;
|
|
27531
27592
|
}
|
|
27532
27593
|
function tryFit(input) {
|
|
27533
27594
|
if (input.prefixTokens > input.distilledBudget && input.prefix.length > 0)
|
|
@@ -27582,25 +27643,50 @@ function isWorkerSession(sessionID) {
|
|
|
27582
27643
|
}
|
|
27583
27644
|
|
|
27584
27645
|
// src/distillation.ts
|
|
27646
|
+
function compressionRatio(distilledTokens, sourceTokens) {
|
|
27647
|
+
if (sourceTokens <= 0) return 0;
|
|
27648
|
+
return distilledTokens / Math.sqrt(sourceTokens);
|
|
27649
|
+
}
|
|
27585
27650
|
function detectSegments(messages, maxSegment) {
|
|
27586
27651
|
if (messages.length <= maxSegment) return [messages];
|
|
27587
|
-
|
|
27588
|
-
|
|
27589
|
-
|
|
27590
|
-
|
|
27591
|
-
|
|
27592
|
-
|
|
27593
|
-
|
|
27594
|
-
|
|
27595
|
-
|
|
27596
|
-
|
|
27597
|
-
|
|
27598
|
-
|
|
27599
|
-
|
|
27600
|
-
|
|
27652
|
+
return splitSegments(messages, maxSegment);
|
|
27653
|
+
}
|
|
27654
|
+
var MIN_SEGMENT = 3;
|
|
27655
|
+
var GAP_THRESHOLD_MULTIPLIER = 3;
|
|
27656
|
+
function splitSegments(messages, maxSegment) {
|
|
27657
|
+
if (messages.length <= maxSegment) return [messages];
|
|
27658
|
+
const splitIdx = findSplitIndex(messages, maxSegment);
|
|
27659
|
+
const left = messages.slice(0, splitIdx);
|
|
27660
|
+
const right = messages.slice(splitIdx);
|
|
27661
|
+
const result = splitSegments(left, maxSegment);
|
|
27662
|
+
if (right.length < MIN_SEGMENT) {
|
|
27663
|
+
result[result.length - 1].push(...right);
|
|
27664
|
+
} else {
|
|
27665
|
+
result.push(...splitSegments(right, maxSegment));
|
|
27666
|
+
}
|
|
27667
|
+
return result;
|
|
27668
|
+
}
|
|
27669
|
+
function findSplitIndex(messages, maxSegment) {
|
|
27670
|
+
const gaps = [];
|
|
27671
|
+
for (let i = 1; i < messages.length; i++) {
|
|
27672
|
+
gaps.push({
|
|
27673
|
+
index: i,
|
|
27674
|
+
gap: messages[i].created_at - messages[i - 1].created_at
|
|
27675
|
+
});
|
|
27676
|
+
}
|
|
27677
|
+
if (gaps.length === 0) return maxSegment;
|
|
27678
|
+
const sortedGaps = gaps.map((g) => g.gap).sort((a, b) => a - b);
|
|
27679
|
+
const medianGap = sortedGaps[Math.floor(sortedGaps.length / 2)];
|
|
27680
|
+
let bestGap = { index: -1, gap: 0 };
|
|
27681
|
+
for (const g of gaps) {
|
|
27682
|
+
if (g.gap > bestGap.gap && g.index >= MIN_SEGMENT && messages.length - g.index >= MIN_SEGMENT) {
|
|
27683
|
+
bestGap = g;
|
|
27601
27684
|
}
|
|
27602
27685
|
}
|
|
27603
|
-
|
|
27686
|
+
if (bestGap.index > 0 && bestGap.gap >= medianGap * GAP_THRESHOLD_MULTIPLIER) {
|
|
27687
|
+
return bestGap.index;
|
|
27688
|
+
}
|
|
27689
|
+
return maxSegment;
|
|
27604
27690
|
}
|
|
27605
27691
|
function formatTime(ms) {
|
|
27606
27692
|
const d = new Date(ms);
|
|
@@ -27838,6 +27924,13 @@ async function distillSegment(input) {
|
|
|
27838
27924
|
generation: 0
|
|
27839
27925
|
});
|
|
27840
27926
|
markDistilled(input.messages.map((m) => m.id));
|
|
27927
|
+
const distilledTokens = Math.ceil(result.observations.length / 3);
|
|
27928
|
+
const sourceTokens = input.messages.reduce((sum, m) => sum + m.tokens, 0);
|
|
27929
|
+
const rComp = compressionRatio(distilledTokens, sourceTokens);
|
|
27930
|
+
const cNorm = temporalCnorm(input.messages.map((m) => m.created_at));
|
|
27931
|
+
info(
|
|
27932
|
+
`distill segment: ${input.messages.length} msgs, ${sourceTokens}\u2192${distilledTokens} tokens, R=${rComp.toFixed(2)}, C_norm=${cNorm.toFixed(3)}`
|
|
27933
|
+
);
|
|
27841
27934
|
if (isAvailable()) {
|
|
27842
27935
|
embedDistillation(distillId, result.observations);
|
|
27843
27936
|
}
|
|
@@ -28202,6 +28295,18 @@ async function runRecall(input) {
|
|
|
28202
28295
|
key: (r) => `t:${r.item.id}`
|
|
28203
28296
|
}
|
|
28204
28297
|
);
|
|
28298
|
+
if (temporalResults.length > 0) {
|
|
28299
|
+
const recencySorted = [...temporalResults].sort(
|
|
28300
|
+
(a, b) => b.created_at - a.created_at
|
|
28301
|
+
);
|
|
28302
|
+
allRrfLists.push({
|
|
28303
|
+
items: recencySorted.map((item) => ({
|
|
28304
|
+
source: "temporal",
|
|
28305
|
+
item
|
|
28306
|
+
})),
|
|
28307
|
+
key: (r) => `t:${r.item.id}`
|
|
28308
|
+
});
|
|
28309
|
+
}
|
|
28205
28310
|
}
|
|
28206
28311
|
if (isAvailable() && scope !== "session") {
|
|
28207
28312
|
try {
|