@loreai/core 0.17.1 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bun/agents-file.d.ts +4 -0
- package/dist/bun/agents-file.d.ts.map +1 -1
- package/dist/bun/config.d.ts +2 -0
- package/dist/bun/config.d.ts.map +1 -1
- package/dist/bun/curator.d.ts +45 -0
- package/dist/bun/curator.d.ts.map +1 -1
- package/dist/bun/data-dir.d.ts +18 -0
- package/dist/bun/data-dir.d.ts.map +1 -0
- package/dist/bun/db.d.ts +85 -0
- package/dist/bun/db.d.ts.map +1 -1
- package/dist/bun/distillation.d.ts +2 -13
- package/dist/bun/distillation.d.ts.map +1 -1
- package/dist/bun/embedding-vendor.d.ts +22 -38
- package/dist/bun/embedding-vendor.d.ts.map +1 -1
- package/dist/bun/embedding-worker-types.d.ts +17 -12
- package/dist/bun/embedding-worker-types.d.ts.map +1 -1
- package/dist/bun/embedding-worker.d.ts +9 -2
- package/dist/bun/embedding-worker.d.ts.map +1 -1
- package/dist/bun/embedding-worker.js +38864 -33
- package/dist/bun/embedding-worker.js.map +4 -4
- package/dist/bun/embedding.d.ts +35 -23
- package/dist/bun/embedding.d.ts.map +1 -1
- package/dist/bun/gradient.d.ts +17 -1
- package/dist/bun/gradient.d.ts.map +1 -1
- package/dist/bun/import/detect.d.ts +14 -0
- package/dist/bun/import/detect.d.ts.map +1 -0
- package/dist/bun/import/extract.d.ts +43 -0
- package/dist/bun/import/extract.d.ts.map +1 -0
- package/dist/bun/import/history.d.ts +40 -0
- package/dist/bun/import/history.d.ts.map +1 -0
- package/dist/bun/import/index.d.ts +17 -0
- package/dist/bun/import/index.d.ts.map +1 -0
- package/dist/bun/import/providers/aider.d.ts +2 -0
- package/dist/bun/import/providers/aider.d.ts.map +1 -0
- package/dist/bun/import/providers/claude-code.d.ts +2 -0
- package/dist/bun/import/providers/claude-code.d.ts.map +1 -0
- package/dist/bun/import/providers/cline.d.ts +2 -0
- package/dist/bun/import/providers/cline.d.ts.map +1 -0
- package/dist/bun/import/providers/codex.d.ts +2 -0
- package/dist/bun/import/providers/codex.d.ts.map +1 -0
- package/dist/bun/import/providers/continue.d.ts +2 -0
- package/dist/bun/import/providers/continue.d.ts.map +1 -0
- package/dist/bun/import/providers/index.d.ts +19 -0
- package/dist/bun/import/providers/index.d.ts.map +1 -0
- package/dist/bun/import/providers/opencode.d.ts +2 -0
- package/dist/bun/import/providers/opencode.d.ts.map +1 -0
- package/dist/bun/import/providers/pi.d.ts +2 -0
- package/dist/bun/import/providers/pi.d.ts.map +1 -0
- package/dist/bun/import/types.d.ts +82 -0
- package/dist/bun/import/types.d.ts.map +1 -0
- package/dist/bun/index.d.ts +5 -2
- package/dist/bun/index.d.ts.map +1 -1
- package/dist/bun/index.js +3150 -439
- package/dist/bun/index.js.map +4 -4
- package/dist/bun/instruction-detect.d.ts +66 -0
- package/dist/bun/instruction-detect.d.ts.map +1 -0
- package/dist/bun/log.d.ts +9 -0
- package/dist/bun/log.d.ts.map +1 -1
- package/dist/bun/ltm.d.ts +139 -5
- package/dist/bun/ltm.d.ts.map +1 -1
- package/dist/bun/pattern-extract.d.ts +7 -0
- package/dist/bun/pattern-extract.d.ts.map +1 -1
- package/dist/bun/prompt.d.ts +1 -1
- package/dist/bun/prompt.d.ts.map +1 -1
- package/dist/bun/recall.d.ts.map +1 -1
- package/dist/bun/search.d.ts +5 -3
- package/dist/bun/search.d.ts.map +1 -1
- package/dist/bun/session-limiter.d.ts +26 -0
- package/dist/bun/session-limiter.d.ts.map +1 -0
- package/dist/bun/temporal.d.ts +2 -0
- package/dist/bun/temporal.d.ts.map +1 -1
- package/dist/bun/types.d.ts +1 -1
- package/dist/node/agents-file.d.ts +4 -0
- package/dist/node/agents-file.d.ts.map +1 -1
- package/dist/node/config.d.ts +2 -0
- package/dist/node/config.d.ts.map +1 -1
- package/dist/node/curator.d.ts +45 -0
- package/dist/node/curator.d.ts.map +1 -1
- package/dist/node/data-dir.d.ts +18 -0
- package/dist/node/data-dir.d.ts.map +1 -0
- package/dist/node/db.d.ts +85 -0
- package/dist/node/db.d.ts.map +1 -1
- package/dist/node/distillation.d.ts +2 -13
- package/dist/node/distillation.d.ts.map +1 -1
- package/dist/node/embedding-vendor.d.ts +22 -38
- package/dist/node/embedding-vendor.d.ts.map +1 -1
- package/dist/node/embedding-worker-types.d.ts +17 -12
- package/dist/node/embedding-worker-types.d.ts.map +1 -1
- package/dist/node/embedding-worker.d.ts +9 -2
- package/dist/node/embedding-worker.d.ts.map +1 -1
- package/dist/node/embedding-worker.js +38864 -33
- package/dist/node/embedding-worker.js.map +4 -4
- package/dist/node/embedding.d.ts +35 -23
- package/dist/node/embedding.d.ts.map +1 -1
- package/dist/node/gradient.d.ts +17 -1
- package/dist/node/gradient.d.ts.map +1 -1
- package/dist/node/import/detect.d.ts +14 -0
- package/dist/node/import/detect.d.ts.map +1 -0
- package/dist/node/import/extract.d.ts +43 -0
- package/dist/node/import/extract.d.ts.map +1 -0
- package/dist/node/import/history.d.ts +40 -0
- package/dist/node/import/history.d.ts.map +1 -0
- package/dist/node/import/index.d.ts +17 -0
- package/dist/node/import/index.d.ts.map +1 -0
- package/dist/node/import/providers/aider.d.ts +2 -0
- package/dist/node/import/providers/aider.d.ts.map +1 -0
- package/dist/node/import/providers/claude-code.d.ts +2 -0
- package/dist/node/import/providers/claude-code.d.ts.map +1 -0
- package/dist/node/import/providers/cline.d.ts +2 -0
- package/dist/node/import/providers/cline.d.ts.map +1 -0
- package/dist/node/import/providers/codex.d.ts +2 -0
- package/dist/node/import/providers/codex.d.ts.map +1 -0
- package/dist/node/import/providers/continue.d.ts +2 -0
- package/dist/node/import/providers/continue.d.ts.map +1 -0
- package/dist/node/import/providers/index.d.ts +19 -0
- package/dist/node/import/providers/index.d.ts.map +1 -0
- package/dist/node/import/providers/opencode.d.ts +2 -0
- package/dist/node/import/providers/opencode.d.ts.map +1 -0
- package/dist/node/import/providers/pi.d.ts +2 -0
- package/dist/node/import/providers/pi.d.ts.map +1 -0
- package/dist/node/import/types.d.ts +82 -0
- package/dist/node/import/types.d.ts.map +1 -0
- package/dist/node/index.d.ts +5 -2
- package/dist/node/index.d.ts.map +1 -1
- package/dist/node/index.js +3150 -439
- package/dist/node/index.js.map +4 -4
- package/dist/node/instruction-detect.d.ts +66 -0
- package/dist/node/instruction-detect.d.ts.map +1 -0
- package/dist/node/log.d.ts +9 -0
- package/dist/node/log.d.ts.map +1 -1
- package/dist/node/ltm.d.ts +139 -5
- package/dist/node/ltm.d.ts.map +1 -1
- package/dist/node/pattern-extract.d.ts +7 -0
- package/dist/node/pattern-extract.d.ts.map +1 -1
- package/dist/node/prompt.d.ts +1 -1
- package/dist/node/prompt.d.ts.map +1 -1
- package/dist/node/recall.d.ts.map +1 -1
- package/dist/node/search.d.ts +5 -3
- package/dist/node/search.d.ts.map +1 -1
- package/dist/node/session-limiter.d.ts +26 -0
- package/dist/node/session-limiter.d.ts.map +1 -0
- package/dist/node/temporal.d.ts +2 -0
- package/dist/node/temporal.d.ts.map +1 -1
- package/dist/node/types.d.ts +1 -1
- package/dist/types/agents-file.d.ts +4 -0
- package/dist/types/agents-file.d.ts.map +1 -1
- package/dist/types/config.d.ts +2 -0
- package/dist/types/config.d.ts.map +1 -1
- package/dist/types/curator.d.ts +45 -0
- package/dist/types/curator.d.ts.map +1 -1
- package/dist/types/data-dir.d.ts +18 -0
- package/dist/types/data-dir.d.ts.map +1 -0
- package/dist/types/db.d.ts +85 -0
- package/dist/types/db.d.ts.map +1 -1
- package/dist/types/distillation.d.ts +2 -13
- package/dist/types/distillation.d.ts.map +1 -1
- package/dist/types/embedding-vendor.d.ts +22 -38
- package/dist/types/embedding-vendor.d.ts.map +1 -1
- package/dist/types/embedding-worker-types.d.ts +17 -12
- package/dist/types/embedding-worker-types.d.ts.map +1 -1
- package/dist/types/embedding-worker.d.ts +9 -2
- package/dist/types/embedding-worker.d.ts.map +1 -1
- package/dist/types/embedding.d.ts +35 -23
- package/dist/types/embedding.d.ts.map +1 -1
- package/dist/types/gradient.d.ts +17 -1
- package/dist/types/gradient.d.ts.map +1 -1
- package/dist/types/import/detect.d.ts +14 -0
- package/dist/types/import/detect.d.ts.map +1 -0
- package/dist/types/import/extract.d.ts +43 -0
- package/dist/types/import/extract.d.ts.map +1 -0
- package/dist/types/import/history.d.ts +40 -0
- package/dist/types/import/history.d.ts.map +1 -0
- package/dist/types/import/index.d.ts +17 -0
- package/dist/types/import/index.d.ts.map +1 -0
- package/dist/types/import/providers/aider.d.ts +2 -0
- package/dist/types/import/providers/aider.d.ts.map +1 -0
- package/dist/types/import/providers/claude-code.d.ts +2 -0
- package/dist/types/import/providers/claude-code.d.ts.map +1 -0
- package/dist/types/import/providers/cline.d.ts +2 -0
- package/dist/types/import/providers/cline.d.ts.map +1 -0
- package/dist/types/import/providers/codex.d.ts +2 -0
- package/dist/types/import/providers/codex.d.ts.map +1 -0
- package/dist/types/import/providers/continue.d.ts +2 -0
- package/dist/types/import/providers/continue.d.ts.map +1 -0
- package/dist/types/import/providers/index.d.ts +19 -0
- package/dist/types/import/providers/index.d.ts.map +1 -0
- package/dist/types/import/providers/opencode.d.ts +2 -0
- package/dist/types/import/providers/opencode.d.ts.map +1 -0
- package/dist/types/import/providers/pi.d.ts +2 -0
- package/dist/types/import/providers/pi.d.ts.map +1 -0
- package/dist/types/import/types.d.ts +82 -0
- package/dist/types/import/types.d.ts.map +1 -0
- package/dist/types/index.d.ts +5 -2
- package/dist/types/index.d.ts.map +1 -1
- package/dist/types/instruction-detect.d.ts +66 -0
- package/dist/types/instruction-detect.d.ts.map +1 -0
- package/dist/types/log.d.ts +9 -0
- package/dist/types/log.d.ts.map +1 -1
- package/dist/types/ltm.d.ts +139 -5
- package/dist/types/ltm.d.ts.map +1 -1
- package/dist/types/pattern-extract.d.ts +7 -0
- package/dist/types/pattern-extract.d.ts.map +1 -1
- package/dist/types/prompt.d.ts +1 -1
- package/dist/types/prompt.d.ts.map +1 -1
- package/dist/types/recall.d.ts.map +1 -1
- package/dist/types/search.d.ts +5 -3
- package/dist/types/search.d.ts.map +1 -1
- package/dist/types/session-limiter.d.ts +26 -0
- package/dist/types/session-limiter.d.ts.map +1 -0
- package/dist/types/temporal.d.ts +2 -0
- package/dist/types/temporal.d.ts.map +1 -1
- package/dist/types/types.d.ts +1 -1
- package/package.json +3 -4
- package/src/agents-file.ts +41 -13
- package/src/config.ts +31 -18
- package/src/curator.ts +163 -75
- package/src/data-dir.ts +76 -0
- package/src/db.ts +457 -11
- package/src/distillation.ts +65 -16
- package/src/embedding-vendor.ts +23 -40
- package/src/embedding-worker-types.ts +19 -11
- package/src/embedding-worker.ts +111 -47
- package/src/embedding.ts +224 -174
- package/src/gradient.ts +192 -75
- package/src/import/detect.ts +37 -0
- package/src/import/extract.ts +137 -0
- package/src/import/history.ts +99 -0
- package/src/import/index.ts +45 -0
- package/src/import/providers/aider.ts +207 -0
- package/src/import/providers/claude-code.ts +339 -0
- package/src/import/providers/cline.ts +324 -0
- package/src/import/providers/codex.ts +369 -0
- package/src/import/providers/continue.ts +304 -0
- package/src/import/providers/index.ts +32 -0
- package/src/import/providers/opencode.ts +272 -0
- package/src/import/providers/pi.ts +332 -0
- package/src/import/types.ts +91 -0
- package/src/index.ts +13 -0
- package/src/instruction-detect.ts +275 -0
- package/src/log.ts +91 -3
- package/src/ltm.ts +789 -41
- package/src/pattern-extract.ts +41 -0
- package/src/prompt.ts +7 -1
- package/src/recall.ts +43 -5
- package/src/search.ts +7 -5
- package/src/session-limiter.ts +47 -0
- package/src/temporal.ts +18 -6
- package/src/types.ts +1 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"recall.d.ts","sourceRoot":"","sources":["../../src/recall.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AACH,OAAO,KAAK,SAAS,MAAM,cAAc,CAAC;AAC1C,OAAO,KAAK,GAAG,MAAM,OAAO,CAAC;AAC7B,OAAO,KAAK,QAAQ,MAAM,YAAY,CAAC;AAIvC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AAC3C,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAgBzC,KAAK,YAAY,GAAG;IAClB,EAAE,EAAE,MAAM,CAAC;IACX,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,kBAAkB,GAAG,YAAY,GAAG;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC;AAEjE,MAAM,MAAM,WAAW,GAAG,KAAK,GAAG,SAAS,GAAG,SAAS,GAAG,WAAW,CAAC;AAEtE,MAAM,MAAM,WAAW,GAAG;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,sDAAsD;IACtD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,iGAAiG;IACjG,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,gDAAgD;IAChD,WAAW,EAAE,MAAM,CAAC;IACpB,gEAAgE;IAChE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,sEAAsE;IACtE,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,mFAAmF;IACnF,GAAG,CAAC,EAAE,SAAS,CAAC;IAChB,6EAA6E;IAC7E,YAAY,CAAC,EAAE,UAAU,CAAC,QAAQ,CAAC,CAAC;CACrC,CAAC;AAEF,2EAA2E;AAC3E,MAAM,MAAM,YAAY,GAAG,MAAM,CAAC;AAElC,MAAM,MAAM,YAAY,GACpB;IAAE,MAAM,EAAE,WAAW,CAAC;IAAC,IAAI,EAAE,GAAG,CAAC,oBAAoB,CAAA;CAAE,GACvD;IACE,MAAM,EAAE,iBAAiB,CAAC;IAC1B,IAAI,EAAE,GAAG,CAAC,oBAAoB,CAAC;IAC/B,YAAY,EAAE,MAAM,CAAC;CACtB,GACD;IAAE,MAAM,EAAE,cAAc,CAAC;IAAC,IAAI,EAAE,kBAAkB,CAAA;CAAE,GACpD;IAAE,MAAM,EAAE,UAAU,CAAC;IAAC,IAAI,EAAE,QAAQ,CAAC,qBAAqB,CAAA;CAAE,GAC5D;IAAE,MAAM,EAAE,aAAa,CAAC;IAAC,IAAI,EAAE,SAAS,CAAC,gBAAgB,CAAA;CAAE,CAAC;AAEhE,MAAM,MAAM,kBAAkB,GAAG;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC;AA0WvE;;;;;GAKG;AACH,wBAAsB,YAAY,CAChC,KAAK,EAAE,WAAW,GACjB,OAAO,CAAC,kBAAkB,EAAE,CAAC,
|
|
1
|
+
{"version":3,"file":"recall.d.ts","sourceRoot":"","sources":["../../src/recall.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AACH,OAAO,KAAK,SAAS,MAAM,cAAc,CAAC;AAC1C,OAAO,KAAK,GAAG,MAAM,OAAO,CAAC;AAC7B,OAAO,KAAK,QAAQ,MAAM,YAAY,CAAC;AAIvC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AAC3C,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAgBzC,KAAK,YAAY,GAAG;IAClB,EAAE,EAAE,MAAM,CAAC;IACX,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,kBAAkB,GAAG,YAAY,GAAG;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC;AAEjE,MAAM,MAAM,WAAW,GAAG,KAAK,GAAG,SAAS,GAAG,SAAS,GAAG,WAAW,CAAC;AAEtE,MAAM,MAAM,WAAW,GAAG;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,sDAAsD;IACtD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,iGAAiG;IACjG,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,gDAAgD;IAChD,WAAW,EAAE,MAAM,CAAC;IACpB,gEAAgE;IAChE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,sEAAsE;IACtE,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,mFAAmF;IACnF,GAAG,CAAC,EAAE,SAAS,CAAC;IAChB,6EAA6E;IAC7E,YAAY,CAAC,EAAE,UAAU,CAAC,QAAQ,CAAC,CAAC;CACrC,CAAC;AAEF,2EAA2E;AAC3E,MAAM,MAAM,YAAY,GAAG,MAAM,CAAC;AAElC,MAAM,MAAM,YAAY,GACpB;IAAE,MAAM,EAAE,WAAW,CAAC;IAAC,IAAI,EAAE,GAAG,CAAC,oBAAoB,CAAA;CAAE,GACvD;IACE,MAAM,EAAE,iBAAiB,CAAC;IAC1B,IAAI,EAAE,GAAG,CAAC,oBAAoB,CAAC;IAC/B,YAAY,EAAE,MAAM,CAAC;CACtB,GACD;IAAE,MAAM,EAAE,cAAc,CAAC;IAAC,IAAI,EAAE,kBAAkB,CAAA;CAAE,GACpD;IAAE,MAAM,EAAE,UAAU,CAAC;IAAC,IAAI,EAAE,QAAQ,CAAC,qBAAqB,CAAA;CAAE,GAC5D;IAAE,MAAM,EAAE,aAAa,CAAC;IAAC,IAAI,EAAE,SAAS,CAAC,gBAAgB,CAAA;CAAE,CAAC;AAEhE,MAAM,MAAM,kBAAkB,GAAG;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC;AA0WvE;;;;;GAKG;AACH,wBAAsB,YAAY,CAChC,KAAK,EAAE,WAAW,GACjB,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAuY/B;AAMD;;;;;;GAMG;AACH,wBAAgB,UAAU,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,CAoE7C;AAED,wFAAwF;AACxF,wBAAsB,SAAS,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,CAAC,CAczE;AAED,sEAAsE;AACtE,eAAO,MAAM,uBAAuB,wiBAC8f,CAAC;AAEniB,mEAAmE;AACnE,eAAO,MAAM,yBAAyB;;;;CAKrC,CAAC"}
|
package/dist/types/search.d.ts
CHANGED
|
@@ -104,14 +104,15 @@ export declare function normalizeRank(rank: number, minRank: number, maxRank: nu
|
|
|
104
104
|
/**
|
|
105
105
|
* Reciprocal Rank Fusion: merge multiple ranked lists into a single ranked list.
|
|
106
106
|
*
|
|
107
|
-
* RRF score = Σ(
|
|
107
|
+
* RRF score = Σ(weight / (k + rank_i)) for each list where the item appears.
|
|
108
108
|
* k = 60 is standard (from Cormack et al., 2009; also used by QMD).
|
|
109
109
|
*
|
|
110
110
|
* RRF is rank-based, not score-based — raw score magnitude differences across
|
|
111
111
|
* different FTS5 tables don't matter. Only relative ordering within each list.
|
|
112
112
|
*
|
|
113
|
-
* @param lists Each list provides items (in ranked order)
|
|
114
|
-
* for deduplication
|
|
113
|
+
* @param lists Each list provides items (in ranked order), a key function
|
|
114
|
+
* for deduplication, and an optional weight (default 1).
|
|
115
|
+
* Items at the front of the array are rank 0.
|
|
115
116
|
* @param k Smoothing constant. Default 60.
|
|
116
117
|
* @returns Fused list sorted by RRF score descending. When items appear
|
|
117
118
|
* in multiple lists, the first occurrence's item is kept.
|
|
@@ -119,6 +120,7 @@ export declare function normalizeRank(rank: number, minRank: number, maxRank: nu
|
|
|
119
120
|
export declare function reciprocalRankFusion<T>(lists: Array<{
|
|
120
121
|
items: T[];
|
|
121
122
|
key: (item: T) => string;
|
|
123
|
+
weight?: number;
|
|
122
124
|
}>, k?: number): Array<{
|
|
123
125
|
item: T;
|
|
124
126
|
score: number;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"search.d.ts","sourceRoot":"","sources":["../../src/search.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,SAAS,EAAE,WAAW,CAAC,MAAM,CAmGxC,CAAC;AAEH;;;;GAIG;AACH,eAAO,MAAM,WAAW,SAAO,CAAC;AAEhC;;;;;;;;;;;;;GAaG;AACH,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,EAAE,CASjD;AAED;;;;;GAKG;AACH,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAI5C;AAED;;;;GAIG;AACH,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAI9C;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAgB,eAAe,CAAC,GAAG,EAAE,MAAM,EAAE,QAAQ,SAAI,GAAG,MAAM,EAAE,CAmBnE;AAED;;;;;;;;GAQG;AACH,wBAAgB,gBAAgB,CAAC,CAAC,EAChC,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,CAAC,SAAS,EAAE,MAAM,KAAK,CAAC,EAAE,GACjC,CAAC,EAAE,CAiBL;AAMD;;;;;;;;;;GAUG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,SAAK,GAAG,MAAM,EAAE,CAelE;AAMD;;;;;;;GAOG;AACH,wBAAgB,aAAa,CAC3B,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,MAAM,EACf,OAAO,EAAE,MAAM,GACd,MAAM,CAMR;AAED
|
|
1
|
+
{"version":3,"file":"search.d.ts","sourceRoot":"","sources":["../../src/search.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,SAAS,EAAE,WAAW,CAAC,MAAM,CAmGxC,CAAC;AAEH;;;;GAIG;AACH,eAAO,MAAM,WAAW,SAAO,CAAC;AAEhC;;;;;;;;;;;;;GAaG;AACH,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,EAAE,CASjD;AAED;;;;;GAKG;AACH,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAI5C;AAED;;;;GAIG;AACH,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAI9C;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAgB,eAAe,CAAC,GAAG,EAAE,MAAM,EAAE,QAAQ,SAAI,GAAG,MAAM,EAAE,CAmBnE;AAED;;;;;;;;GAQG;AACH,wBAAgB,gBAAgB,CAAC,CAAC,EAChC,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,CAAC,SAAS,EAAE,MAAM,KAAK,CAAC,EAAE,GACjC,CAAC,EAAE,CAiBL;AAMD;;;;;;;;;;GAUG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,SAAK,GAAG,MAAM,EAAE,CAelE;AAMD;;;;;;;GAOG;AACH,wBAAgB,aAAa,CAC3B,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,MAAM,EACf,OAAO,EAAE,MAAM,GACd,MAAM,CAMR;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,oBAAoB,CAAC,CAAC,EACpC,KAAK,EAAE,KAAK,CAAC;IAAE,KAAK,EAAE,CAAC,EAAE,CAAC;IAAC,GAAG,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,EACvE,CAAC,SAAK,GACL,KAAK,CAAC;IAAE,IAAI,EAAE,CAAC,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CAmBnC;AAMD;;;;;;;;;;GAUG;AACH,wBAAgB,kBAAkB,CAAC,CAAC,EAClC,KAAK,EAAE,CAAC,EAAE,EACV,OAAO,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,MAAM,EAC5B,KAAK,EAAE,MAAM,GACZ,CAAC,EAAE,CAcL;AAQD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAEzC;;;;;;;;;;GAUG;AACH,wBAAsB,WAAW,CAC/B,GAAG,EAAE,SAAS,EACd,KAAK,EAAE,MAAM,EACb,KAAK,CAAC,EAAE;IAAE,UAAU,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,EAC/C,SAAS,CAAC,EAAE,MAAM,GACjB,OAAO,CAAC,MAAM,EAAE,CAAC,CAqCnB"}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Per-key concurrency limiter using p-limit.
|
|
3
|
+
*
|
|
4
|
+
* Each key (typically a session ID) gets its own p-limit(1) instance,
|
|
5
|
+
* serializing async operations on the same key while allowing different
|
|
6
|
+
* keys to run fully in parallel.
|
|
7
|
+
*
|
|
8
|
+
* Two independent limiter pools are provided — one for distillation and
|
|
9
|
+
* one for curation — so they don't block each other.
|
|
10
|
+
*/
|
|
11
|
+
import pLimit from "p-limit";
|
|
12
|
+
type LimitFunction = ReturnType<typeof pLimit>;
|
|
13
|
+
/** Serializes distillation.run() and metaDistill() per session. */
|
|
14
|
+
export declare const distillLimiter: {
|
|
15
|
+
get: (key: string) => LimitFunction;
|
|
16
|
+
isBusy: (key: string) => boolean;
|
|
17
|
+
clear: () => void;
|
|
18
|
+
};
|
|
19
|
+
/** Serializes curator.run() per session with skip-if-busy semantics. */
|
|
20
|
+
export declare const curatorLimiter: {
|
|
21
|
+
get: (key: string) => LimitFunction;
|
|
22
|
+
isBusy: (key: string) => boolean;
|
|
23
|
+
clear: () => void;
|
|
24
|
+
};
|
|
25
|
+
export {};
|
|
26
|
+
//# sourceMappingURL=session-limiter.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"session-limiter.d.ts","sourceRoot":"","sources":["../../src/session-limiter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,MAAM,MAAM,SAAS,CAAC;AAE7B,KAAK,aAAa,GAAG,UAAU,CAAC,OAAO,MAAM,CAAC,CAAC;AA6B/C,mEAAmE;AACnE,eAAO,MAAM,cAAc;eAxBP,MAAM,KAAG,aAAa;kBAUnB,MAAM,KAAG,OAAO;iBAMnB,IAAI;CAQyB,CAAC;AAElD,wEAAwE;AACxE,eAAO,MAAM,cAAc;eA3BP,MAAM,KAAG,aAAa;kBAUnB,MAAM,KAAG,OAAO;iBAMnB,IAAI;CAWyB,CAAC"}
|
package/dist/types/temporal.d.ts
CHANGED
|
@@ -84,6 +84,8 @@ export declare function searchScored(input: {
|
|
|
84
84
|
*/
|
|
85
85
|
export declare function temporalCnorm(timestamps: number[], now?: number): number;
|
|
86
86
|
export declare function count(projectPath: string, sessionID?: string): number;
|
|
87
|
+
/** Quick existence check — true if any temporal messages exist for this session. */
|
|
88
|
+
export declare function hasMessages(projectPath: string, sessionID: string): boolean;
|
|
87
89
|
export declare function undistilledCount(projectPath: string, sessionID?: string): number;
|
|
88
90
|
/** Sum of estimated tokens across undistilled messages for a project/session. */
|
|
89
91
|
export declare function undistilledTokens(projectPath: string, sessionID?: string): number;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"temporal.d.ts","sourceRoot":"","sources":["../../src/temporal.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAQrD;;;;;;;;;;;;;;;;;;GAkBG;AACH,eAAO,MAAM,gBAAgB,WAAS,CAAC;AAEvC;;;;;;;;;GASG;AACH,wBAAgB,WAAW,CAAC,KAAK,EAAE,QAAQ,EAAE,GAAG,MAAM,CAarD;AAiBD,wBAAgB,KAAK,CAAC,KAAK,EAAE;IAC3B,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,WAAW,CAAC;IAClB,KAAK,EAAE,QAAQ,EAAE,CAAC;CACnB,QA8CA;AAED,MAAM,MAAM,eAAe,GAAG;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,wBAAgB,WAAW,CACzB,WAAW,EAAE,MAAM,EACnB,SAAS,CAAC,EAAE,MAAM,GACjB,eAAe,EAAE,CASnB;AAED,wBAAgB,SAAS,CACvB,WAAW,EAAE,MAAM,EACnB,SAAS,EAAE,MAAM,GAChB,eAAe,EAAE,CAOnB;AAED,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,QAQ1C;
|
|
1
|
+
{"version":3,"file":"temporal.d.ts","sourceRoot":"","sources":["../../src/temporal.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAQrD;;;;;;;;;;;;;;;;;;GAkBG;AACH,eAAO,MAAM,gBAAgB,WAAS,CAAC;AAEvC;;;;;;;;;GASG;AACH,wBAAgB,WAAW,CAAC,KAAK,EAAE,QAAQ,EAAE,GAAG,MAAM,CAarD;AAiBD,wBAAgB,KAAK,CAAC,KAAK,EAAE;IAC3B,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,WAAW,CAAC;IAClB,KAAK,EAAE,QAAQ,EAAE,CAAC;CACnB,QA8CA;AAED,MAAM,MAAM,eAAe,GAAG;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,wBAAgB,WAAW,CACzB,WAAW,EAAE,MAAM,EACnB,SAAS,CAAC,EAAE,MAAM,GACjB,eAAe,EAAE,CASnB;AAED,wBAAgB,SAAS,CACvB,WAAW,EAAE,MAAM,EACnB,SAAS,EAAE,MAAM,GAChB,eAAe,EAAE,CAOnB;AAED,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,QAQ1C;AA6BD,wBAAgB,MAAM,CAAC,KAAK,EAAE;IAC5B,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,GAAG,eAAe,EAAE,CA8BpB;AAED,MAAM,MAAM,qBAAqB,GAAG,eAAe,GAAG;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC;AAEvE;;;GAGG;AACH,wBAAgB,YAAY,CAAC,KAAK,EAAE;IAClC,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,GAAG,qBAAqB,EAAE,CAwB1B;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,aAAa,CAC3B,UAAU,EAAE,MAAM,EAAE,EACpB,GAAG,GAAE,MAAmB,GACvB,MAAM,CAoBR;AAED,wBAAgB,KAAK,CAAC,WAAW,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,CAWrE;AAED,oFAAoF;AACpF,wBAAgB,WAAW,CAAC,WAAW,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAO3E;AAED,wBAAgB,gBAAgB,CAC9B,WAAW,EAAE,MAAM,EACnB,SAAS,CAAC,EAAE,MAAM,GACjB,MAAM,CAWR;AAED,iFAAiF;AACjF,wBAAgB,iBAAiB,CAC/B,WAAW,EAAE,MAAM,EACnB,SAAS,CAAC,EAAE,MAAM,GACjB,MAAM,CAWR;AAED,MAAM,MAAM,WAAW,GAAG;IACxB,kFAAkF;IAClF,UAAU,EAAE,MAAM,CAAC;IACnB,8FAA8F;IAC9F,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF;;;;;;;;;;;;GAYG;AACH,wBAAgB,KAAK,CAAC,KAAK,EAAE;IAC3B,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;CACtB,GAAG,WAAW,CA0Ed"}
|
package/dist/types/types.d.ts
CHANGED
|
@@ -164,7 +164,7 @@ export type LoreMessageWithParts = {
|
|
|
164
164
|
* Host adapters implement this:
|
|
165
165
|
* - OpenCode: wraps `client.session.create()` + `client.session.prompt()`
|
|
166
166
|
* - Pi: wraps `complete()` from `@mariozechner/pi-ai`
|
|
167
|
-
* -
|
|
167
|
+
* - Gateway: direct `fetch()` to provider APIs
|
|
168
168
|
*/
|
|
169
169
|
export interface LLMClient {
|
|
170
170
|
/**
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@loreai/core",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.19.0",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"license": "FSL-1.1-Apache-2.0",
|
|
6
6
|
"description": "Shared memory engine for Lore — three-tier storage, distillation, gradient context management",
|
|
@@ -24,14 +24,13 @@
|
|
|
24
24
|
},
|
|
25
25
|
"dependencies": {
|
|
26
26
|
"@huggingface/hub": "2.11.0",
|
|
27
|
+
"@huggingface/transformers": "^3.7.1",
|
|
27
28
|
"micromark": "^4.0.0",
|
|
29
|
+
"p-limit": "7",
|
|
28
30
|
"remark": "^15.0.1",
|
|
29
31
|
"uuidv7": "^1.1.0",
|
|
30
32
|
"zod": "^4.3.6"
|
|
31
33
|
},
|
|
32
|
-
"optionalDependencies": {
|
|
33
|
-
"fastembed": "^2.1.0"
|
|
34
|
-
},
|
|
35
34
|
"devDependencies": {
|
|
36
35
|
"@types/mdast": "^4.0.4"
|
|
37
36
|
},
|
package/src/agents-file.ts
CHANGED
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
|
|
11
11
|
import { existsSync, readFileSync, writeFileSync, mkdirSync, statSync } from "fs";
|
|
12
12
|
import { dirname, join } from "path";
|
|
13
|
-
import { db } from "./db";
|
|
13
|
+
import { db, ensureProject } from "./db";
|
|
14
14
|
import * as ltm from "./ltm";
|
|
15
15
|
import { serialize, inline, h, ul, liph, strong, t, root, unescapeMarkdown } from "./markdown";
|
|
16
16
|
|
|
@@ -425,21 +425,35 @@ function _importEntries(
|
|
|
425
425
|
ltm.update(entry.id, { content: entry.content });
|
|
426
426
|
}
|
|
427
427
|
} else {
|
|
428
|
-
// Unknown UUID — entry came from another machine
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
id
|
|
437
|
-
|
|
428
|
+
// Unknown UUID — entry came from another machine.
|
|
429
|
+
// Check for a fuzzy title match before creating — prevents duplicates
|
|
430
|
+
// when two machines independently create entries for the same concept
|
|
431
|
+
// with different UUIDs but similar titles.
|
|
432
|
+
const pid = ensureProject(projectPath);
|
|
433
|
+
const fuzzyMatch = ltm.findFuzzyDuplicate({ title: entry.title, projectId: pid });
|
|
434
|
+
if (fuzzyMatch) {
|
|
435
|
+
// Title-similar entry exists locally — update it, discard foreign UUID
|
|
436
|
+
if (fuzzyMatch.title !== entry.title || ltm.get(fuzzyMatch.id)?.content !== entry.content) {
|
|
437
|
+
ltm.update(fuzzyMatch.id, { content: entry.content });
|
|
438
|
+
}
|
|
439
|
+
} else {
|
|
440
|
+
ltm.create({
|
|
441
|
+
projectPath,
|
|
442
|
+
category: entry.category,
|
|
443
|
+
title: entry.title,
|
|
444
|
+
content: entry.content,
|
|
445
|
+
scope: "project",
|
|
446
|
+
crossProject: false,
|
|
447
|
+
id: entry.id,
|
|
448
|
+
});
|
|
449
|
+
}
|
|
438
450
|
}
|
|
439
451
|
} else {
|
|
440
452
|
// Hand-written entry — create with a new UUIDv7
|
|
441
|
-
// Check for a near-duplicate by title to avoid double-import on re-runs
|
|
442
|
-
|
|
453
|
+
// Check for a near-duplicate by title to avoid double-import on re-runs.
|
|
454
|
+
// Scope to project-only entries (false) — cross-project entries from other
|
|
455
|
+
// projects should not silently suppress a hand-written entry in this project.
|
|
456
|
+
const existing = ltm.forProject(projectPath, false);
|
|
443
457
|
const titleMatch = existing.find(
|
|
444
458
|
(e) => e.title.toLowerCase() === entry.title.toLowerCase(),
|
|
445
459
|
);
|
|
@@ -559,6 +573,10 @@ export function shouldImportLoreFile(projectPath: string): boolean {
|
|
|
559
573
|
/**
|
|
560
574
|
* Import knowledge entries from `.lore.md` into the local DB.
|
|
561
575
|
* Parses the full file content (no section markers to split on).
|
|
576
|
+
*
|
|
577
|
+
* After a successful import, updates the file cache so that
|
|
578
|
+
* `shouldImportLoreFile()` fast-paths on the next check — the file
|
|
579
|
+
* content hasn't changed, only the DB was updated to match it.
|
|
562
580
|
*/
|
|
563
581
|
export function importLoreFile(projectPath: string): void {
|
|
564
582
|
const fp = join(projectPath, LORE_FILE);
|
|
@@ -569,4 +587,14 @@ export function importLoreFile(projectPath: string): void {
|
|
|
569
587
|
if (!fileEntries.length) return;
|
|
570
588
|
|
|
571
589
|
_importEntries(fileEntries, projectPath);
|
|
590
|
+
|
|
591
|
+
// Update cache: DB now matches the file, so shouldImportLoreFile() can
|
|
592
|
+
// fast-path on the next check. We re-stat after import because the file
|
|
593
|
+
// hasn't changed — only the DB was updated to match it.
|
|
594
|
+
try {
|
|
595
|
+
const { mtimeMs } = statSync(fp);
|
|
596
|
+
setCache(fp, { mtimeMs, hash: hashSection(fileContent) });
|
|
597
|
+
} catch {
|
|
598
|
+
// stat failure is non-fatal — worst case we re-import next time
|
|
599
|
+
}
|
|
572
600
|
}
|
package/src/config.ts
CHANGED
|
@@ -135,15 +135,25 @@ export const LoreConfig = z.object({
|
|
|
135
135
|
.default({ title: 6.0, content: 2.0, category: 3.0 }),
|
|
136
136
|
/** Max results per source in recall tool before fusion. Default: 10. */
|
|
137
137
|
recallLimit: z.number().min(1).max(50).default(10),
|
|
138
|
-
/** Enable LLM-based query expansion for the recall tool. Default:
|
|
139
|
-
*
|
|
140
|
-
*
|
|
141
|
-
|
|
138
|
+
/** Enable LLM-based query expansion for the recall tool. Default: true.
|
|
139
|
+
* The configured model generates 2–3 alternative query phrasings before
|
|
140
|
+
* search, improving recall for ambiguous queries. Guarded by a 3-second
|
|
141
|
+
* timeout — if expansion fails or times out, the original query is used. */
|
|
142
|
+
queryExpansion: z.boolean().default(true),
|
|
143
|
+
/** RRF weight multiplier for vector search lists. Applied when the query
|
|
144
|
+
* has >= `vectorBoostMinTerms` meaningful terms (after stopword removal).
|
|
145
|
+
* Boosts semantic/vector results relative to keyword-based BM25 lists.
|
|
146
|
+
* Default: 1.5. Set to 1.0 to disable. */
|
|
147
|
+
vectorBoostWeight: z.number().min(1).max(5).default(1.5),
|
|
148
|
+
/** Minimum meaningful query terms (after stopword removal) to activate
|
|
149
|
+
* vector boost. Short keyword queries (1-2 terms) are left unweighted
|
|
150
|
+
* since BM25 excels there. Default: 3. */
|
|
151
|
+
vectorBoostMinTerms: z.number().min(1).max(10).default(3),
|
|
142
152
|
/** Vector embedding search.
|
|
143
153
|
* Supports multiple providers:
|
|
144
|
-
* - "local" (default):
|
|
145
|
-
*
|
|
146
|
-
* cached
|
|
154
|
+
* - "local" (default): @huggingface/transformers + nomic-embed-text-v1.5, no API key needed.
|
|
155
|
+
* 768 dims (Matryoshka-capable: 64–768). Model downloaded on first use (~137MB INT8),
|
|
156
|
+
* cached locally. Uses task instruction prefixes (search_document: / search_query:).
|
|
147
157
|
* - "voyage": Voyage AI (VOYAGE_API_KEY, voyage-code-3, 1024 dims)
|
|
148
158
|
* - "openai": OpenAI (OPENAI_API_KEY, text-embedding-3-small, 1536 dims)
|
|
149
159
|
* Set enabled: false to explicitly disable even with a provider available. */
|
|
@@ -153,20 +163,21 @@ export const LoreConfig = z.object({
|
|
|
153
163
|
* Set to false to explicitly disable. */
|
|
154
164
|
enabled: z.boolean().default(true),
|
|
155
165
|
/** Embedding provider. Default: "local".
|
|
156
|
-
* - "local":
|
|
166
|
+
* - "local": @huggingface/transformers, no API key (default model: nomic-embed-text-v1.5, 768 dims)
|
|
157
167
|
* - "voyage": VOYAGE_API_KEY (default model: voyage-code-3, 1024 dims)
|
|
158
168
|
* - "openai": OPENAI_API_KEY (default model: text-embedding-3-small, 1536 dims) */
|
|
159
169
|
provider: z.enum(["local", "voyage", "openai"]).default("local"),
|
|
160
170
|
/** Model ID for the embedding provider. Default depends on provider. */
|
|
161
|
-
model: z.string().default("
|
|
162
|
-
/** Embedding dimensions. Default:
|
|
163
|
-
|
|
171
|
+
model: z.string().default("nomic-ai/nomic-embed-text-v1.5"),
|
|
172
|
+
/** Embedding dimensions. Default: 768 (local) / 1024 (voyage) / 1536 (openai).
|
|
173
|
+
* For the local Nomic v1.5 model, supports Matryoshka dimensions: 64, 128, 256, 512, 768. */
|
|
174
|
+
dimensions: z.number().min(64).max(2048).default(768),
|
|
164
175
|
})
|
|
165
176
|
.default({
|
|
166
177
|
enabled: true,
|
|
167
178
|
provider: "local",
|
|
168
|
-
model: "
|
|
169
|
-
dimensions:
|
|
179
|
+
model: "nomic-ai/nomic-embed-text-v1.5",
|
|
180
|
+
dimensions: 768,
|
|
170
181
|
}),
|
|
171
182
|
/** Recall output formatting — controls how search results are presented to the agent. */
|
|
172
183
|
recall: z
|
|
@@ -186,8 +197,10 @@ export const LoreConfig = z.object({
|
|
|
186
197
|
.default({
|
|
187
198
|
ftsWeights: { title: 6.0, content: 2.0, category: 3.0 },
|
|
188
199
|
recallLimit: 10,
|
|
189
|
-
queryExpansion:
|
|
190
|
-
|
|
200
|
+
queryExpansion: true,
|
|
201
|
+
vectorBoostWeight: 1.5,
|
|
202
|
+
vectorBoostMinTerms: 3,
|
|
203
|
+
embeddings: { enabled: true, provider: "local" as const, model: "nomic-ai/nomic-embed-text-v1.5", dimensions: 768 },
|
|
191
204
|
recall: { charBudget: 8000, relevanceFloor: 0.15, maxResults: 15 },
|
|
192
205
|
}),
|
|
193
206
|
cache: z
|
|
@@ -207,9 +220,9 @@ export const LoreConfig = z.object({
|
|
|
207
220
|
.object({
|
|
208
221
|
/** Enable cache warming. Default: true. */
|
|
209
222
|
enabled: z.boolean().default(true),
|
|
210
|
-
/** Override the
|
|
211
|
-
* skipped. Default: auto-derived from
|
|
212
|
-
* (~0.
|
|
223
|
+
/** Override the return probability threshold below which warming is
|
|
224
|
+
* skipped. Default: auto-derived from corrected cost ratio
|
|
225
|
+
* read/(write-read) (~0.087 for 5m TTL, ~0.042 for 1h TTL). */
|
|
213
226
|
minReturnProbability: z.number().min(0).max(1).optional(),
|
|
214
227
|
})
|
|
215
228
|
.default({ enabled: true }),
|
package/src/curator.ts
CHANGED
|
@@ -1,8 +1,11 @@
|
|
|
1
1
|
import { config } from "./config";
|
|
2
|
+
import { saveSessionTracking, loadSessionTracking, ensureProject } from "./db";
|
|
2
3
|
import * as temporal from "./temporal";
|
|
3
4
|
import * as ltm from "./ltm";
|
|
4
5
|
import * as log from "./log";
|
|
5
6
|
import { CURATOR_SYSTEM, curatorUser, CONSOLIDATION_SYSTEM, consolidationUser } from "./prompt";
|
|
7
|
+
import { detectAndFormat } from "./instruction-detect";
|
|
8
|
+
import { curatorLimiter } from "./session-limiter";
|
|
6
9
|
import type { LLMClient } from "./types";
|
|
7
10
|
|
|
8
11
|
/**
|
|
@@ -11,9 +14,9 @@ import type { LLMClient } from "./types";
|
|
|
11
14
|
* The curator prompt also instructs the model to stay within this limit,
|
|
12
15
|
* so truncation is a last-resort safety net.
|
|
13
16
|
*/
|
|
14
|
-
const MAX_ENTRY_CONTENT_LENGTH = 1200;
|
|
17
|
+
export const MAX_ENTRY_CONTENT_LENGTH = 1200;
|
|
15
18
|
|
|
16
|
-
type CuratorOp =
|
|
19
|
+
export type CuratorOp =
|
|
17
20
|
| {
|
|
18
21
|
op: "create";
|
|
19
22
|
category: string;
|
|
@@ -25,7 +28,11 @@ type CuratorOp =
|
|
|
25
28
|
| { op: "update"; id: string; content?: string; confidence?: number }
|
|
26
29
|
| { op: "delete"; id: string; reason: string };
|
|
27
30
|
|
|
28
|
-
|
|
31
|
+
/**
|
|
32
|
+
* Parse the LLM's JSON response into typed curator ops.
|
|
33
|
+
* Handles markdown fences and filters invalid entries.
|
|
34
|
+
*/
|
|
35
|
+
export function parseOps(text: string): CuratorOp[] {
|
|
29
36
|
const cleaned = text
|
|
30
37
|
.trim()
|
|
31
38
|
.replace(/^```json?\s*/i, "")
|
|
@@ -45,58 +52,29 @@ function parseOps(text: string): CuratorOp[] {
|
|
|
45
52
|
}
|
|
46
53
|
}
|
|
47
54
|
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
const all = temporal.bySession(input.projectPath, input.sessionID);
|
|
64
|
-
const sessionCuratedAt = lastCuratedAt.get(input.sessionID) ?? 0;
|
|
65
|
-
const recent = all.filter((m) => m.created_at > sessionCuratedAt);
|
|
66
|
-
if (recent.length < 3) return { created: 0, updated: 0, deleted: 0 };
|
|
67
|
-
|
|
68
|
-
const text = recent.map((m) => `[${m.role}] ${m.content}`).join("\n\n");
|
|
69
|
-
const existing = ltm.forProject(input.projectPath, false);
|
|
70
|
-
const existingForPrompt = existing.map((e) => ({
|
|
71
|
-
id: e.id,
|
|
72
|
-
category: e.category,
|
|
73
|
-
title: e.title,
|
|
74
|
-
content: e.content,
|
|
75
|
-
}));
|
|
76
|
-
|
|
77
|
-
const userContent = curatorUser({
|
|
78
|
-
messages: text,
|
|
79
|
-
existing: existingForPrompt,
|
|
80
|
-
});
|
|
81
|
-
const model = input.model ?? cfg.model;
|
|
82
|
-
const responseText = await input.llm.prompt(
|
|
83
|
-
CURATOR_SYSTEM,
|
|
84
|
-
userContent,
|
|
85
|
-
{ model, workerID: "lore-curator", thinking: false, sessionID: input.sessionID, maxTokens: 2048 },
|
|
86
|
-
);
|
|
87
|
-
if (!responseText) return { created: 0, updated: 0, deleted: 0 };
|
|
88
|
-
|
|
89
|
-
const ops = parseOps(responseText);
|
|
55
|
+
/**
|
|
56
|
+
* Apply a list of curator ops (create/update/delete) to the knowledge DB.
|
|
57
|
+
* Shared by both the live curator and the conversation import system.
|
|
58
|
+
*
|
|
59
|
+
* @returns Counts of applied operations.
|
|
60
|
+
*/
|
|
61
|
+
export function applyOps(
|
|
62
|
+
ops: CuratorOp[],
|
|
63
|
+
input: {
|
|
64
|
+
projectPath?: string;
|
|
65
|
+
sessionID?: string;
|
|
66
|
+
/** If true, skip "create" ops (used by consolidation). */
|
|
67
|
+
skipCreate?: boolean;
|
|
68
|
+
},
|
|
69
|
+
): { created: number; updated: number; deleted: number } {
|
|
90
70
|
let created = 0;
|
|
91
71
|
let updated = 0;
|
|
92
72
|
let deleted = 0;
|
|
93
|
-
|
|
94
73
|
const idsToSync: string[] = [];
|
|
95
74
|
|
|
96
75
|
for (const op of ops) {
|
|
97
76
|
if (op.op === "create") {
|
|
98
|
-
|
|
99
|
-
// 500-word limit, but enforce it here as a hard safety net.
|
|
77
|
+
if (input.skipCreate) continue;
|
|
100
78
|
const content =
|
|
101
79
|
op.content.length > MAX_ENTRY_CONTENT_LENGTH
|
|
102
80
|
? op.content.slice(0, MAX_ENTRY_CONTENT_LENGTH) +
|
|
@@ -139,10 +117,140 @@ export async function run(input: {
|
|
|
139
117
|
ltm.syncRefs(id);
|
|
140
118
|
}
|
|
141
119
|
|
|
142
|
-
lastCuratedAt.set(input.sessionID, Date.now());
|
|
143
120
|
return { created, updated, deleted };
|
|
144
121
|
}
|
|
145
122
|
|
|
123
|
+
// Track which messages we've already curated — per session to prevent
|
|
124
|
+
// cross-session leaking (curation on session A advancing the timestamp
|
|
125
|
+
// past session B's messages, causing B's curation to find < 3 recent).
|
|
126
|
+
// In-memory cache backed by session_state DB table so it survives restarts.
|
|
127
|
+
const lastCuratedAt = new Map<string, number>();
|
|
128
|
+
|
|
129
|
+
/** Get the last-curated timestamp for a session, loading from DB if needed. */
|
|
130
|
+
function getLastCuratedAt(sessionID: string): number {
|
|
131
|
+
const cached = lastCuratedAt.get(sessionID);
|
|
132
|
+
if (cached !== undefined) return cached;
|
|
133
|
+
// Load from DB on first access
|
|
134
|
+
const persisted = loadSessionTracking(sessionID);
|
|
135
|
+
const ts = persisted?.lastCuratedAt ?? 0;
|
|
136
|
+
lastCuratedAt.set(sessionID, ts);
|
|
137
|
+
return ts;
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
export async function run(input: {
|
|
141
|
+
llm: LLMClient;
|
|
142
|
+
projectPath: string;
|
|
143
|
+
sessionID: string;
|
|
144
|
+
model?: { providerID: string; modelID: string };
|
|
145
|
+
}): Promise<{ created: number; updated: number; deleted: number }> {
|
|
146
|
+
const cfg = config();
|
|
147
|
+
if (!cfg.curator.enabled) return { created: 0, updated: 0, deleted: 0 };
|
|
148
|
+
|
|
149
|
+
// Skip-if-busy: curation is periodic, not accumulative. If a curation is
|
|
150
|
+
// already running for this session, skip — the next trigger will pick up
|
|
151
|
+
// any new messages. Serializing would waste an LLM call.
|
|
152
|
+
//
|
|
153
|
+
// The isBusy() check and get()() enqueue are both synchronous — in Node's
|
|
154
|
+
// single-threaded event loop no microtask can interleave between them, so
|
|
155
|
+
// there is no TOCTOU race. The p-limit(1) serialization is a safety net
|
|
156
|
+
// if this invariant is ever violated.
|
|
157
|
+
if (curatorLimiter.isBusy(input.sessionID)) {
|
|
158
|
+
log.info(`curation skipped: already running for session ${input.sessionID.slice(0, 16)}`);
|
|
159
|
+
return { created: 0, updated: 0, deleted: 0 };
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
return curatorLimiter.get(input.sessionID)(() => runInner(input));
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
async function runInner(input: {
|
|
166
|
+
llm: LLMClient;
|
|
167
|
+
projectPath: string;
|
|
168
|
+
sessionID: string;
|
|
169
|
+
model?: { providerID: string; modelID: string };
|
|
170
|
+
}): Promise<{ created: number; updated: number; deleted: number }> {
|
|
171
|
+
const cfg = config();
|
|
172
|
+
|
|
173
|
+
// Get recent messages since last curation
|
|
174
|
+
const all = temporal.bySession(input.projectPath, input.sessionID);
|
|
175
|
+
const sessionCuratedAt = getLastCuratedAt(input.sessionID);
|
|
176
|
+
const recent = all.filter((m) => m.created_at > sessionCuratedAt);
|
|
177
|
+
if (recent.length < 3) return { created: 0, updated: 0, deleted: 0 };
|
|
178
|
+
|
|
179
|
+
const text = recent.map((m) => `[${m.role}] ${m.content}`).join("\n\n");
|
|
180
|
+
const existing = ltm.forProject(input.projectPath, false);
|
|
181
|
+
const existingForPrompt = existing.map((e) => ({
|
|
182
|
+
id: e.id,
|
|
183
|
+
category: e.category,
|
|
184
|
+
title: e.title,
|
|
185
|
+
content: e.content,
|
|
186
|
+
}));
|
|
187
|
+
|
|
188
|
+
const baseUserContent = curatorUser({
|
|
189
|
+
messages: text,
|
|
190
|
+
existing: existingForPrompt,
|
|
191
|
+
});
|
|
192
|
+
|
|
193
|
+
// Detect repeated instructions across prior sessions and append as
|
|
194
|
+
// additional context for the curator. This is async (may embed candidates)
|
|
195
|
+
// but fast — typically <250ms for 5 candidates with local embeddings.
|
|
196
|
+
let crossSessionContext = "";
|
|
197
|
+
try {
|
|
198
|
+
crossSessionContext = await detectAndFormat({
|
|
199
|
+
projectPath: input.projectPath,
|
|
200
|
+
sessionID: input.sessionID,
|
|
201
|
+
});
|
|
202
|
+
} catch (err) {
|
|
203
|
+
log.warn("instruction-detect failed (non-fatal):", err);
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
const userContent = baseUserContent + crossSessionContext;
|
|
207
|
+
const model = input.model ?? cfg.model;
|
|
208
|
+
const responseText = await input.llm.prompt(
|
|
209
|
+
CURATOR_SYSTEM,
|
|
210
|
+
userContent,
|
|
211
|
+
{ model, workerID: "lore-curator", thinking: false, sessionID: input.sessionID, maxTokens: 2048 },
|
|
212
|
+
);
|
|
213
|
+
if (!responseText) return { created: 0, updated: 0, deleted: 0 };
|
|
214
|
+
|
|
215
|
+
const ops = parseOps(responseText);
|
|
216
|
+
const result = applyOps(ops, {
|
|
217
|
+
projectPath: input.projectPath,
|
|
218
|
+
sessionID: input.sessionID,
|
|
219
|
+
});
|
|
220
|
+
|
|
221
|
+
// Post-curation dedup sweep: if the curator created new entries, check for
|
|
222
|
+
// and auto-merge any semantic duplicates it introduced. Uses embedding-based
|
|
223
|
+
// similarity when available, falls back to word-overlap.
|
|
224
|
+
if (result.created > 0) {
|
|
225
|
+
try {
|
|
226
|
+
const dupes = await ltm.deduplicate(input.projectPath, { dryRun: false });
|
|
227
|
+
if (dupes.totalRemoved > 0) {
|
|
228
|
+
log.info(`post-curation dedup: merged ${dupes.totalRemoved} duplicate entries`);
|
|
229
|
+
result.deleted += dupes.totalRemoved;
|
|
230
|
+
}
|
|
231
|
+
// Record auto-signals for adaptive threshold calibration.
|
|
232
|
+
// Merged pairs → accept; non-merged high-similarity pairs → reject.
|
|
233
|
+
if (dupes.pairSimilarities.size > 0) {
|
|
234
|
+
const pid = ensureProject(input.projectPath);
|
|
235
|
+
ltm.recordAutoSignals(pid, dupes);
|
|
236
|
+
// Recalibrate if enough data has accumulated
|
|
237
|
+
const newThreshold = ltm.calibrateDedupThreshold(pid);
|
|
238
|
+
if (newThreshold !== null) {
|
|
239
|
+
const count = ltm.getDedupFeedbackCount(pid);
|
|
240
|
+
ltm.saveCalibratedThreshold(pid, newThreshold, count);
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
} catch (err) {
|
|
244
|
+
log.warn("post-curation dedup failed (non-fatal):", err);
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
const now = Date.now();
|
|
249
|
+
lastCuratedAt.set(input.sessionID, now);
|
|
250
|
+
saveSessionTracking(input.sessionID, { lastCuratedAt: now });
|
|
251
|
+
return result;
|
|
252
|
+
}
|
|
253
|
+
|
|
146
254
|
export function resetCurationTracker(sessionID?: string) {
|
|
147
255
|
if (sessionID) {
|
|
148
256
|
lastCuratedAt.delete(sessionID);
|
|
@@ -190,31 +298,11 @@ export async function consolidate(input: {
|
|
|
190
298
|
if (!responseText) return { updated: 0, deleted: 0 };
|
|
191
299
|
|
|
192
300
|
const ops = parseOps(responseText);
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
if (op.op === "update") {
|
|
199
|
-
const entry = ltm.get(op.id);
|
|
200
|
-
if (entry) {
|
|
201
|
-
const content =
|
|
202
|
-
op.content !== undefined && op.content.length > MAX_ENTRY_CONTENT_LENGTH
|
|
203
|
-
? op.content.slice(0, MAX_ENTRY_CONTENT_LENGTH) +
|
|
204
|
-
" [truncated — entry too long]"
|
|
205
|
-
: op.content;
|
|
206
|
-
ltm.update(op.id, { content, confidence: op.confidence });
|
|
207
|
-
updated++;
|
|
208
|
-
}
|
|
209
|
-
} else if (op.op === "delete") {
|
|
210
|
-
const entry = ltm.get(op.id);
|
|
211
|
-
if (entry) {
|
|
212
|
-
ltm.remove(op.id);
|
|
213
|
-
deleted++;
|
|
214
|
-
}
|
|
215
|
-
}
|
|
216
|
-
// "create" ops are silently ignored — consolidation must not add entries.
|
|
217
|
-
}
|
|
301
|
+
const result = applyOps(ops, {
|
|
302
|
+
projectPath: input.projectPath,
|
|
303
|
+
sessionID: input.sessionID,
|
|
304
|
+
skipCreate: true, // Consolidation must not add entries.
|
|
305
|
+
});
|
|
218
306
|
|
|
219
|
-
return { updated, deleted };
|
|
307
|
+
return { updated: result.updated, deleted: result.deleted };
|
|
220
308
|
}
|