@loreai/core 0.16.0 → 0.17.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -0
- package/dist/bun/agents-file.d.ts +13 -1
- package/dist/bun/agents-file.d.ts.map +1 -1
- package/dist/bun/config.d.ts +20 -1
- package/dist/bun/config.d.ts.map +1 -1
- package/dist/bun/data.d.ts +174 -0
- package/dist/bun/data.d.ts.map +1 -0
- package/dist/bun/db.d.ts +65 -0
- package/dist/bun/db.d.ts.map +1 -1
- package/dist/bun/distillation.d.ts +49 -6
- package/dist/bun/distillation.d.ts.map +1 -1
- package/dist/bun/embedding-vendor.d.ts +66 -0
- package/dist/bun/embedding-vendor.d.ts.map +1 -0
- package/dist/bun/embedding-worker-types.d.ts +66 -0
- package/dist/bun/embedding-worker-types.d.ts.map +1 -0
- package/dist/bun/embedding-worker.d.ts +16 -0
- package/dist/bun/embedding-worker.d.ts.map +1 -0
- package/dist/bun/embedding-worker.js +100 -0
- package/dist/bun/embedding-worker.js.map +7 -0
- package/dist/bun/embedding.d.ts +91 -8
- package/dist/bun/embedding.d.ts.map +1 -1
- package/dist/bun/git.d.ts +47 -0
- package/dist/bun/git.d.ts.map +1 -0
- package/dist/bun/gradient.d.ts +19 -1
- package/dist/bun/gradient.d.ts.map +1 -1
- package/dist/bun/index.d.ts +9 -6
- package/dist/bun/index.d.ts.map +1 -1
- package/dist/bun/index.js +13029 -10885
- package/dist/bun/index.js.map +4 -4
- package/dist/bun/lat-reader.d.ts +1 -1
- package/dist/bun/lat-reader.d.ts.map +1 -1
- package/dist/bun/ltm.d.ts.map +1 -1
- package/dist/bun/markdown.d.ts +11 -0
- package/dist/bun/markdown.d.ts.map +1 -1
- package/dist/bun/prompt.d.ts +1 -1
- package/dist/bun/prompt.d.ts.map +1 -1
- package/dist/bun/recall.d.ts +53 -0
- package/dist/bun/recall.d.ts.map +1 -1
- package/dist/bun/search.d.ts +29 -0
- package/dist/bun/search.d.ts.map +1 -1
- package/dist/bun/temporal.d.ts +2 -0
- package/dist/bun/temporal.d.ts.map +1 -1
- package/dist/bun/types.d.ts +15 -0
- package/dist/bun/types.d.ts.map +1 -1
- package/dist/bun/worker-model.d.ts +12 -9
- package/dist/bun/worker-model.d.ts.map +1 -1
- package/dist/node/agents-file.d.ts +13 -1
- package/dist/node/agents-file.d.ts.map +1 -1
- package/dist/node/config.d.ts +20 -1
- package/dist/node/config.d.ts.map +1 -1
- package/dist/node/data.d.ts +174 -0
- package/dist/node/data.d.ts.map +1 -0
- package/dist/node/db.d.ts +65 -0
- package/dist/node/db.d.ts.map +1 -1
- package/dist/node/distillation.d.ts +49 -6
- package/dist/node/distillation.d.ts.map +1 -1
- package/dist/node/embedding-vendor.d.ts +66 -0
- package/dist/node/embedding-vendor.d.ts.map +1 -0
- package/dist/node/embedding-worker-types.d.ts +66 -0
- package/dist/node/embedding-worker-types.d.ts.map +1 -0
- package/dist/node/embedding-worker.d.ts +16 -0
- package/dist/node/embedding-worker.d.ts.map +1 -0
- package/dist/node/embedding-worker.js +100 -0
- package/dist/node/embedding-worker.js.map +7 -0
- package/dist/node/embedding.d.ts +91 -8
- package/dist/node/embedding.d.ts.map +1 -1
- package/dist/node/git.d.ts +47 -0
- package/dist/node/git.d.ts.map +1 -0
- package/dist/node/gradient.d.ts +19 -1
- package/dist/node/gradient.d.ts.map +1 -1
- package/dist/node/index.d.ts +9 -6
- package/dist/node/index.d.ts.map +1 -1
- package/dist/node/index.js +13029 -10885
- package/dist/node/index.js.map +4 -4
- package/dist/node/lat-reader.d.ts +1 -1
- package/dist/node/lat-reader.d.ts.map +1 -1
- package/dist/node/ltm.d.ts.map +1 -1
- package/dist/node/markdown.d.ts +11 -0
- package/dist/node/markdown.d.ts.map +1 -1
- package/dist/node/prompt.d.ts +1 -1
- package/dist/node/prompt.d.ts.map +1 -1
- package/dist/node/recall.d.ts +53 -0
- package/dist/node/recall.d.ts.map +1 -1
- package/dist/node/search.d.ts +29 -0
- package/dist/node/search.d.ts.map +1 -1
- package/dist/node/temporal.d.ts +2 -0
- package/dist/node/temporal.d.ts.map +1 -1
- package/dist/node/types.d.ts +15 -0
- package/dist/node/types.d.ts.map +1 -1
- package/dist/node/worker-model.d.ts +12 -9
- package/dist/node/worker-model.d.ts.map +1 -1
- package/dist/types/agents-file.d.ts +13 -1
- package/dist/types/agents-file.d.ts.map +1 -1
- package/dist/types/config.d.ts +20 -1
- package/dist/types/config.d.ts.map +1 -1
- package/dist/types/data.d.ts +174 -0
- package/dist/types/data.d.ts.map +1 -0
- package/dist/types/db.d.ts +65 -0
- package/dist/types/db.d.ts.map +1 -1
- package/dist/types/distillation.d.ts +49 -6
- package/dist/types/distillation.d.ts.map +1 -1
- package/dist/types/embedding-vendor.d.ts +66 -0
- package/dist/types/embedding-vendor.d.ts.map +1 -0
- package/dist/types/embedding-worker-types.d.ts +66 -0
- package/dist/types/embedding-worker-types.d.ts.map +1 -0
- package/dist/types/embedding-worker.d.ts +16 -0
- package/dist/types/embedding-worker.d.ts.map +1 -0
- package/dist/types/embedding.d.ts +91 -8
- package/dist/types/embedding.d.ts.map +1 -1
- package/dist/types/git.d.ts +47 -0
- package/dist/types/git.d.ts.map +1 -0
- package/dist/types/gradient.d.ts +19 -1
- package/dist/types/gradient.d.ts.map +1 -1
- package/dist/types/index.d.ts +9 -6
- package/dist/types/index.d.ts.map +1 -1
- package/dist/types/lat-reader.d.ts +1 -1
- package/dist/types/lat-reader.d.ts.map +1 -1
- package/dist/types/ltm.d.ts.map +1 -1
- package/dist/types/markdown.d.ts +11 -0
- package/dist/types/markdown.d.ts.map +1 -1
- package/dist/types/prompt.d.ts +1 -1
- package/dist/types/prompt.d.ts.map +1 -1
- package/dist/types/recall.d.ts +53 -0
- package/dist/types/recall.d.ts.map +1 -1
- package/dist/types/search.d.ts +29 -0
- package/dist/types/search.d.ts.map +1 -1
- package/dist/types/temporal.d.ts +2 -0
- package/dist/types/temporal.d.ts.map +1 -1
- package/dist/types/types.d.ts +15 -0
- package/dist/types/types.d.ts.map +1 -1
- package/dist/types/worker-model.d.ts +12 -9
- package/dist/types/worker-model.d.ts.map +1 -1
- package/package.json +5 -2
- package/src/agents-file.ts +87 -4
- package/src/config.ts +68 -5
- package/src/curator.ts +2 -2
- package/src/data.ts +768 -0
- package/src/db.ts +386 -7
- package/src/distillation.ts +178 -35
- package/src/embedding-vendor.ts +102 -0
- package/src/embedding-worker-types.ts +82 -0
- package/src/embedding-worker.ts +185 -0
- package/src/embedding.ts +607 -61
- package/src/git.ts +144 -0
- package/src/gradient.ts +174 -17
- package/src/index.ts +20 -0
- package/src/lat-reader.ts +5 -11
- package/src/ltm.ts +17 -44
- package/src/markdown.ts +15 -0
- package/src/prompt.ts +1 -2
- package/src/recall.ts +401 -70
- package/src/search.ts +71 -1
- package/src/temporal.ts +42 -35
- package/src/types.ts +15 -0
- package/src/worker-model.ts +14 -9
package/README.md
CHANGED
|
@@ -19,6 +19,17 @@ bun add @loreai/core
|
|
|
19
19
|
|
|
20
20
|
You only need to install this directly if you're building a new adapter. End users install one of the host packages above.
|
|
21
21
|
|
|
22
|
+
### Optional dependency: `fastembed`
|
|
23
|
+
|
|
24
|
+
`fastembed` is declared as an `optionalDependencies` because its native `onnxruntime-node` bindings can fail to build on some hosts (e.g. CUDA 13 on Linux/x64 — [microsoft/onnxruntime#26586](https://github.com/microsoft/onnxruntime/discussions/26586)). Install always succeeds, and `embed()` resolves a provider in this order:
|
|
25
|
+
|
|
26
|
+
1. **Vendored** (standalone `lore` binary only) — fastembed and its native bindings are bundled directly into the binary at compile time via `bun build --compile`. The bge-small INT8 model files and the side-load `libonnxruntime` shared library ride along as Bun assets and are materialized to `~/.lore/embeddings-vendored/v{version}-{target}/` on first call. Supported targets: `darwin-arm64`, `linux-arm64`, `linux-x64`, `windows-x64`. (`darwin-x64` is unsupported — Apple Silicon-only.)
|
|
27
|
+
2. **npm-installed** — `import("fastembed")` resolves to the user's `node_modules`, including the optional-dep install.
|
|
28
|
+
3. **Remote auto-fallback** — when the local probe fails AND `VOYAGE_API_KEY` or `OPENAI_API_KEY` is set, `embed()` swaps to that provider for the rest of the process. Voyage wins ties.
|
|
29
|
+
4. **FTS-only** — if none of the above resolve, `recall.runRecall()` and `vectorSearch()` return zero hits and callers continue with full-text search only.
|
|
30
|
+
|
|
31
|
+
To force the optional install on a CUDA-13 host, run with `ONNXRUNTIME_NODE_INSTALL_CUDA=skip` — the bundled CPU EP is sufficient for `bge-small-en-v1.5`.
|
|
32
|
+
|
|
22
33
|
## Documentation
|
|
23
34
|
|
|
24
35
|
Full architecture, benchmarks, and rationale: **[github.com/BYK/loreai](https://github.com/BYK/loreai)**
|
|
@@ -15,6 +15,11 @@ export declare const LORE_SECTION_END = "<!-- End lore-managed section -->";
|
|
|
15
15
|
* by lore — no section markers needed, no non-lore content to preserve.
|
|
16
16
|
*/
|
|
17
17
|
export declare const LORE_FILE = ".lore.md";
|
|
18
|
+
/**
|
|
19
|
+
* Clear the cached mtime/hash for a project's `.lore.md`.
|
|
20
|
+
* Useful in tests or after data wipes to force a full re-check.
|
|
21
|
+
*/
|
|
22
|
+
export declare function clearLoreFileCache(projectPath: string): void;
|
|
18
23
|
export type ParsedFileEntry = {
|
|
19
24
|
/** UUID from `<!-- lore:UUID -->` marker, or null for hand-written entries. */
|
|
20
25
|
id: string | null;
|
|
@@ -65,11 +70,18 @@ export declare function loreFileExists(projectPath: string): boolean;
|
|
|
65
70
|
/**
|
|
66
71
|
* Export current knowledge entries to `.lore.md` in the project root.
|
|
67
72
|
* The entire file is lore-owned — no section markers, no content to preserve.
|
|
73
|
+
*
|
|
74
|
+
* Skips the write if the content hash matches the cached hash (DB state
|
|
75
|
+
* unchanged since last export), avoiding unnecessary filesystem writes
|
|
76
|
+
* and mtime bumps.
|
|
68
77
|
*/
|
|
69
78
|
export declare function exportLoreFile(projectPath: string): void;
|
|
70
79
|
/**
|
|
71
80
|
* Returns true if `.lore.md` needs to be imported:
|
|
72
|
-
* - File exists and its content differs from what lore would currently produce
|
|
81
|
+
* - File exists and its content differs from what lore would currently produce.
|
|
82
|
+
*
|
|
83
|
+
* Uses an mtime + content-hash cache to skip the expensive buildSection()
|
|
84
|
+
* call when the file hasn't been touched since we last processed it.
|
|
73
85
|
*/
|
|
74
86
|
export declare function shouldImportLoreFile(projectPath: string): boolean;
|
|
75
87
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"agents-file.d.ts","sourceRoot":"","sources":["../../src/agents-file.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;
|
|
1
|
+
{"version":3,"file":"agents-file.d.ts","sourceRoot":"","sources":["../../src/agents-file.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAYH,eAAO,MAAM,kBAAkB,qGACqE,CAAC;AACrG,eAAO,MAAM,gBAAgB,sCAAsC,CAAC;AAepE;;;;GAIG;AACH,eAAO,MAAM,SAAS,aAAa,CAAC;AA8CpC;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAI5D;AAMD,MAAM,MAAM,eAAe,GAAG;IAC5B,+EAA+E;IAC/E,EAAE,EAAE,MAAM,GAAG,IAAI,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AA+EF;;;;;;;;GAQG;AACH,wBAAgB,uBAAuB,CAAC,OAAO,EAAE,MAAM,GAAG,eAAe,EAAE,CAgD1E;AAmFD;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,KAAK,EAAE;IAClC,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;CAClB,GAAG,IAAI,CA8BP;AAMD;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,KAAK,EAAE;IAClC,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;CAClB,GAAG,OAAO,CAcV;AAsED;;;GAGG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE;IACpC,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;CAClB,GAAG,IAAI,CAeP;AAMD;;GAEG;AACH,wBAAgB,cAAc,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAE3D;AAED;;;;;;;GAOG;AACH,wBAAgB,cAAc,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAiBxD;AAED;;;;;;GAMG;AACH,wBAAgB,oBAAoB,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAwBjE;AAED;;;GAGG;AACH,wBAAgB,cAAc,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CASxD"}
|
package/dist/bun/config.d.ts
CHANGED
|
@@ -15,11 +15,14 @@ export declare const LoreConfig: z.ZodObject<{
|
|
|
15
15
|
ltm: z.ZodDefault<z.ZodNumber>;
|
|
16
16
|
targetCacheReadCostPerTurn: z.ZodDefault<z.ZodNumber>;
|
|
17
17
|
maxLayer0Tokens: z.ZodOptional<z.ZodNumber>;
|
|
18
|
+
targetBustCost: z.ZodDefault<z.ZodNumber>;
|
|
19
|
+
maxContextTokens: z.ZodOptional<z.ZodNumber>;
|
|
18
20
|
}, z.core.$strip>>;
|
|
19
21
|
idleResumeMinutes: z.ZodDefault<z.ZodNumber>;
|
|
20
22
|
distillation: z.ZodDefault<z.ZodObject<{
|
|
21
23
|
minMessages: z.ZodDefault<z.ZodNumber>;
|
|
22
|
-
|
|
24
|
+
minSegmentTokens: z.ZodDefault<z.ZodNumber>;
|
|
25
|
+
maxSegmentTokens: z.ZodDefault<z.ZodNumber>;
|
|
23
26
|
metaThreshold: z.ZodDefault<z.ZodNumber>;
|
|
24
27
|
toolOutputMaxChars: z.ZodDefault<z.ZodNumber>;
|
|
25
28
|
}, z.core.$strip>>;
|
|
@@ -54,6 +57,22 @@ export declare const LoreConfig: z.ZodObject<{
|
|
|
54
57
|
model: z.ZodDefault<z.ZodString>;
|
|
55
58
|
dimensions: z.ZodDefault<z.ZodNumber>;
|
|
56
59
|
}, z.core.$strip>>;
|
|
60
|
+
recall: z.ZodDefault<z.ZodObject<{
|
|
61
|
+
charBudget: z.ZodDefault<z.ZodNumber>;
|
|
62
|
+
relevanceFloor: z.ZodDefault<z.ZodNumber>;
|
|
63
|
+
maxResults: z.ZodDefault<z.ZodNumber>;
|
|
64
|
+
}, z.core.$strip>>;
|
|
65
|
+
}, z.core.$strip>>;
|
|
66
|
+
cache: z.ZodDefault<z.ZodObject<{
|
|
67
|
+
conversationTTL: z.ZodDefault<z.ZodEnum<{
|
|
68
|
+
"5m": "5m";
|
|
69
|
+
"1h": "1h";
|
|
70
|
+
auto: "auto";
|
|
71
|
+
}>>;
|
|
72
|
+
warming: z.ZodDefault<z.ZodObject<{
|
|
73
|
+
enabled: z.ZodDefault<z.ZodBoolean>;
|
|
74
|
+
minReturnProbability: z.ZodOptional<z.ZodNumber>;
|
|
75
|
+
}, z.core.$strip>>;
|
|
57
76
|
}, z.core.$strip>>;
|
|
58
77
|
crossProject: z.ZodDefault<z.ZodBoolean>;
|
|
59
78
|
agentsFile: z.ZodDefault<z.ZodObject<{
|
package/dist/bun/config.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAIxB,eAAO,MAAM,UAAU
|
|
1
|
+
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAIxB,eAAO,MAAM,UAAU;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAiOrB,CAAC;AAEH,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,UAAU,CAAC,CAAC;AAIpD,wBAAgB,MAAM,IAAI,UAAU,CAEnC;AAED,wBAAsB,IAAI,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CASjE"}
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* data.ts — Data listing, inspection, and deletion for Lore.
|
|
3
|
+
*
|
|
4
|
+
* Provides a unified API for both the CLI (`lore data`) and the web UI
|
|
5
|
+
* (`/ui/`) to browse, search, and delete stored data across all tables.
|
|
6
|
+
*
|
|
7
|
+
* Cross-cutting concerns (e.g. `clearProject` touches knowledge, temporal,
|
|
8
|
+
* distillations, and session_state in one transaction) live here instead of
|
|
9
|
+
* being spread across ltm/temporal/distillation modules.
|
|
10
|
+
*/
|
|
11
|
+
export type ProjectSummary = {
|
|
12
|
+
id: string;
|
|
13
|
+
path: string;
|
|
14
|
+
name: string | null;
|
|
15
|
+
git_remote: string | null;
|
|
16
|
+
created_at: number;
|
|
17
|
+
knowledge_count: number;
|
|
18
|
+
session_count: number;
|
|
19
|
+
message_count: number;
|
|
20
|
+
distillation_count: number;
|
|
21
|
+
};
|
|
22
|
+
export type SessionSummary = {
|
|
23
|
+
session_id: string;
|
|
24
|
+
message_count: number;
|
|
25
|
+
first_message_at: number;
|
|
26
|
+
last_message_at: number;
|
|
27
|
+
distilled_count: number;
|
|
28
|
+
undistilled_count: number;
|
|
29
|
+
distillation_count: number;
|
|
30
|
+
};
|
|
31
|
+
export type DistillationSummary = {
|
|
32
|
+
id: string;
|
|
33
|
+
session_id: string;
|
|
34
|
+
generation: number;
|
|
35
|
+
token_count: number;
|
|
36
|
+
r_compression: number | null;
|
|
37
|
+
c_norm: number | null;
|
|
38
|
+
archived: number;
|
|
39
|
+
created_at: number;
|
|
40
|
+
call_type: string | null;
|
|
41
|
+
};
|
|
42
|
+
export type DistillationDetail = DistillationSummary & {
|
|
43
|
+
project_id: string;
|
|
44
|
+
observations: string;
|
|
45
|
+
source_ids: string;
|
|
46
|
+
};
|
|
47
|
+
export type ClearResult = {
|
|
48
|
+
knowledge_deleted: number;
|
|
49
|
+
temporal_deleted: number;
|
|
50
|
+
distillations_deleted: number;
|
|
51
|
+
sessions_cleared: number;
|
|
52
|
+
};
|
|
53
|
+
export type GlobalStats = {
|
|
54
|
+
project_count: number;
|
|
55
|
+
knowledge_count: number;
|
|
56
|
+
session_count: number;
|
|
57
|
+
message_count: number;
|
|
58
|
+
distillation_count: number;
|
|
59
|
+
db_size_bytes: number;
|
|
60
|
+
};
|
|
61
|
+
/** List all projects with summary counts. */
|
|
62
|
+
export declare function listProjects(): ProjectSummary[];
|
|
63
|
+
/** List distinct sessions for a project, with message/distillation counts. */
|
|
64
|
+
export declare function listSessions(projectPath: string, limit?: number): SessionSummary[];
|
|
65
|
+
/** List distillations for a project (optionally filtered by session). */
|
|
66
|
+
export declare function listDistillations(projectPath: string, opts?: {
|
|
67
|
+
sessionId?: string;
|
|
68
|
+
limit?: number;
|
|
69
|
+
}): DistillationSummary[];
|
|
70
|
+
/** Get a single distillation by ID (or resolved prefix). */
|
|
71
|
+
export declare function getDistillation(id: string): DistillationDetail | null;
|
|
72
|
+
/**
|
|
73
|
+
* Resolve a partial ID prefix to a full ID for a given table.
|
|
74
|
+
* Returns null if 0 or 2+ matches (ambiguous prefix).
|
|
75
|
+
*/
|
|
76
|
+
export declare function resolveId(table: "knowledge" | "distillations", prefix: string): string | null;
|
|
77
|
+
/** Global stats for the dashboard. */
|
|
78
|
+
export declare function globalStats(): GlobalStats;
|
|
79
|
+
/**
|
|
80
|
+
* Count rows that will be affected, for confirmation prompts.
|
|
81
|
+
*/
|
|
82
|
+
export declare function countForProject(projectPath: string): {
|
|
83
|
+
knowledge: number;
|
|
84
|
+
messages: number;
|
|
85
|
+
distillations: number;
|
|
86
|
+
sessions: number;
|
|
87
|
+
};
|
|
88
|
+
/**
|
|
89
|
+
* Clear all data for a project.
|
|
90
|
+
* Deletes: knowledge, temporal_messages, distillations, session_state.
|
|
91
|
+
* Does NOT delete the project row itself (preserves path->id mapping).
|
|
92
|
+
* Regenerates `.lore.md` if the project path exists on disk.
|
|
93
|
+
*/
|
|
94
|
+
export declare function clearProject(projectPath: string): ClearResult;
|
|
95
|
+
/**
|
|
96
|
+
* Fully delete a project: all associated data AND the project row itself.
|
|
97
|
+
* Also removes path aliases pointing to this project.
|
|
98
|
+
*
|
|
99
|
+
* Unlike clearProject(), this does NOT call ensureProject() (avoids
|
|
100
|
+
* re-creating the project) and does NOT regenerate .lore.md.
|
|
101
|
+
*
|
|
102
|
+
* Returns deletion counts, or null if the project ID doesn't exist.
|
|
103
|
+
*/
|
|
104
|
+
export declare function deleteProject(projectId: string): ClearResult | null;
|
|
105
|
+
/** Rename a project. Returns true if the project exists and was renamed. */
|
|
106
|
+
export declare function renameProject(projectId: string, newName: string): boolean;
|
|
107
|
+
/** Clear only knowledge entries for a project. Regenerates .lore.md. */
|
|
108
|
+
export declare function clearKnowledge(projectPath: string): number;
|
|
109
|
+
/** Clear only temporal messages for a project. */
|
|
110
|
+
export declare function clearTemporal(projectPath: string): number;
|
|
111
|
+
/** Clear only distillations for a project. */
|
|
112
|
+
export declare function clearDistillations(projectPath: string): number;
|
|
113
|
+
/** Delete a single knowledge entry. Returns true if found and deleted. */
|
|
114
|
+
export declare function deleteKnowledge(id: string): boolean;
|
|
115
|
+
/** Delete a single distillation. Returns true if found and deleted. */
|
|
116
|
+
export declare function deleteDistillation(id: string): boolean;
|
|
117
|
+
/**
|
|
118
|
+
* Delete all data for a specific session (messages + distillations + session_state).
|
|
119
|
+
*/
|
|
120
|
+
export declare function deleteSession(projectPath: string, sessionId: string): {
|
|
121
|
+
messages_deleted: number;
|
|
122
|
+
distillations_deleted: number;
|
|
123
|
+
};
|
|
124
|
+
/**
|
|
125
|
+
* Nuclear option: close the DB, delete the file, re-initialize.
|
|
126
|
+
* Returns the path of the deleted DB file.
|
|
127
|
+
*/
|
|
128
|
+
export declare function wipeDatabase(): string;
|
|
129
|
+
export type MergeResult = {
|
|
130
|
+
knowledge_moved: number;
|
|
131
|
+
messages_moved: number;
|
|
132
|
+
distillations_moved: number;
|
|
133
|
+
};
|
|
134
|
+
/**
|
|
135
|
+
* Merge a source project into a target project.
|
|
136
|
+
*
|
|
137
|
+
* Moves all data (knowledge, messages, distillations, LAT sections, path
|
|
138
|
+
* aliases) from source to target, then deletes the source project row.
|
|
139
|
+
* The source project's path is registered as an alias of the target.
|
|
140
|
+
*
|
|
141
|
+
* Returns counts of moved rows for reporting.
|
|
142
|
+
*/
|
|
143
|
+
export declare function mergeProjects(sourceId: string, targetId: string): MergeResult;
|
|
144
|
+
/**
|
|
145
|
+
* Backfill git_remote for existing projects, merge duplicates, and
|
|
146
|
+
* update project names from git remote repo names where still using
|
|
147
|
+
* the directory-basename default.
|
|
148
|
+
*
|
|
149
|
+
* Iterates all projects that lack a git_remote value, runs `git remote -v`
|
|
150
|
+
* on their stored path, and:
|
|
151
|
+
* - If no other project shares that remote: sets git_remote on the row.
|
|
152
|
+
* - If another project already has that remote: merges this project into
|
|
153
|
+
* the existing one (consolidating fragmented data).
|
|
154
|
+
*
|
|
155
|
+
* Also backfills project names: if a project's name matches the directory
|
|
156
|
+
* basename (the old default) or is null, and a git remote is available,
|
|
157
|
+
* the name is updated to the repo name from the remote URL.
|
|
158
|
+
*
|
|
159
|
+
* Skips projects whose path no longer exists on disk or is not a git repo.
|
|
160
|
+
*
|
|
161
|
+
* Returns counts for reporting.
|
|
162
|
+
*/
|
|
163
|
+
export declare function backfillGitRemotes(): {
|
|
164
|
+
updated: number;
|
|
165
|
+
merged: number;
|
|
166
|
+
namesBackfilled: number;
|
|
167
|
+
mergeDetails: Array<{
|
|
168
|
+
sourcePath: string;
|
|
169
|
+
targetPath: string;
|
|
170
|
+
gitRemote: string;
|
|
171
|
+
result: MergeResult;
|
|
172
|
+
}>;
|
|
173
|
+
};
|
|
174
|
+
//# sourceMappingURL=data.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../src/data.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAoBH,MAAM,MAAM,cAAc,GAAG;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,eAAe,EAAE,MAAM,CAAC;IACxB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,kBAAkB,EAAE,MAAM,CAAC;CAC5B,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG;IAC3B,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,gBAAgB,EAAE,MAAM,CAAC;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,eAAe,EAAE,MAAM,CAAC;IACxB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,kBAAkB,EAAE,MAAM,CAAC;CAC5B,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG;IAChC,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CAC1B,CAAC;AAEF,MAAM,MAAM,kBAAkB,GAAG,mBAAmB,GAAG;IACrD,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG;IACxB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,gBAAgB,EAAE,MAAM,CAAC;IACzB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,gBAAgB,EAAE,MAAM,CAAC;CAC1B,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG;IACxB,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,CAAC;IACxB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,aAAa,EAAE,MAAM,CAAC;CACvB,CAAC;AAMF,6CAA6C;AAC7C,wBAAgB,YAAY,IAAI,cAAc,EAAE,CAW/C;AAED,8EAA8E;AAC9E,wBAAgB,YAAY,CAC1B,WAAW,EAAE,MAAM,EACnB,KAAK,SAAK,GACT,cAAc,EAAE,CAqBlB;AAED,yEAAyE;AACzE,wBAAgB,iBAAiB,CAC/B,WAAW,EAAE,MAAM,EACnB,IAAI,CAAC,EAAE;IAAE,SAAS,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,GAC5C,mBAAmB,EAAE,CAuBvB;AAED,4DAA4D;AAC5D,wBAAgB,eAAe,CAAC,EAAE,EAAE,MAAM,GAAG,kBAAkB,GAAG,IAAI,CAQrE;AAED;;;GAGG;AACH,wBAAgB,SAAS,CACvB,KAAK,EAAE,WAAW,GAAG,eAAe,EACpC,MAAM,EAAE,MAAM,GACb,MAAM,GAAG,IAAI,CAKf;AAMD,sCAAsC;AACtC,wBAAgB,WAAW,IAAI,WAAW,CA0BzC;AAMD;;GAEG;AACH,wBAAgB,eAAe,CAAC,WAAW,EAAE,MAAM,GAAG;IACpD,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;CAClB,CAqBA;AAED;;;;;GAKG;AACH,wBAAgB,YAAY,CAAC,WAAW,EAAE,MAAM,GAAG,WAAW,CA+E7D;AAED;;;;;;;;GAQG;AACH,wBAAgB,aAAa,CAAC,SAAS,EAAE,MAAM,GAAG,WAAW,GAAG,IAAI,CAsGnE;AAED,4EAA4E;AAC5E,wBAAgB,aAAa,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAKzE;AAED,wEAAwE;AACxE,wBAAgB,cAAc,CAAC,WAAW,EAAE,MAAM,GAAG,MAAM,CAsB1D;AAED,kDAAkD;AAClD,wBAAgB,aAAa,CAAC,WAAW,EAAE,MAAM,GAAG,MAAM,CAezD;AAED,8CAA8C;AAC9C,wBAAgB,kBAAkB,CAAC,WAAW,EAAE,MAAM,GAAG,MAAM,CAe9D;AAED,0EAA0E;AAC1E,wBAAgB,eAAe,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAKnD;AAED,uEAAuE;AACvE,wBAAgB,kBAAkB,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAKtD;AAED;;GAEG;AACH,wBAAgB,aAAa,CAC3B,WAAW,EAAE,MAAM,EACnB,SAAS,EAAE,MAAM,GAChB;IAAE,gBAAgB,EAAE,MAAM,CAAC;IAAC,qBAAqB,EAAE,MAAM,CAAA;CAAE,CAmC7D;AAED;;;GAGG;AACH,wBAAgB,YAAY,IAAI,MAAM,CAmBrC;AAMD,MAAM,MAAM,WAAW,GAAG;IACxB,eAAe,EAAE,MAAM,CAAC;IACxB,cAAc,EAAE,MAAM,CAAC;IACvB,mBAAmB,EAAE,MAAM,CAAC;CAC7B,CAAC;AAEF;;;;;;;;GAQG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,WAAW,CAmC7E;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAgB,kBAAkB,IAAI;IACpC,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,eAAe,EAAE,MAAM,CAAC;IACxB,YAAY,EAAE,KAAK,CAAC;QAClB,UAAU,EAAE,MAAM,CAAC;QACnB,UAAU,EAAE,MAAM,CAAC;QACnB,SAAS,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,WAAW,CAAC;KACrB,CAAC,CAAC;CACJ,CA6EA"}
|
package/dist/bun/db.d.ts
CHANGED
|
@@ -1,6 +1,43 @@
|
|
|
1
1
|
import { Database } from "#db/driver";
|
|
2
|
+
/**
|
|
3
|
+
* Extract the repository name from a normalized git remote URL.
|
|
4
|
+
*
|
|
5
|
+
* Examples:
|
|
6
|
+
* "github.com/BYK/LoreAI" → "LoreAI"
|
|
7
|
+
* "github.com/org/repo" → "repo"
|
|
8
|
+
* "github.com" → null (no path components)
|
|
9
|
+
* null → null
|
|
10
|
+
*/
|
|
11
|
+
export declare function repoNameFromRemote(remote: string | null): string | null;
|
|
12
|
+
/** Return the resolved path of the SQLite database file. */
|
|
13
|
+
export declare function dbPath(): string;
|
|
2
14
|
export declare function db(): Database;
|
|
15
|
+
/**
|
|
16
|
+
* Merge all data from `sourceId` project into `targetId` project.
|
|
17
|
+
*
|
|
18
|
+
* Moves knowledge, temporal messages, distillations, LAT sections, and
|
|
19
|
+
* path aliases from source to target. Registers the source project's path
|
|
20
|
+
* as an alias of the target. Deletes the source project row.
|
|
21
|
+
*
|
|
22
|
+
* Used internally during lazy git-remote backfill when two path-only
|
|
23
|
+
* projects are discovered to share the same git remote.
|
|
24
|
+
*/
|
|
25
|
+
export declare function mergeProjectInternal(sourceId: string, targetId: string): void;
|
|
3
26
|
export declare function close(): void;
|
|
27
|
+
/**
|
|
28
|
+
* Look up or create a project by filesystem path, with git-remote awareness.
|
|
29
|
+
*
|
|
30
|
+
* Resolution order:
|
|
31
|
+
* 1. Exact path match in `projects` table (fast path, O(1) index scan)
|
|
32
|
+
* 2. Path alias match in `project_path_aliases` (worktree/clone re-visits)
|
|
33
|
+
* 3. Git remote match — runs `git remote -v` (once per unique path, cached),
|
|
34
|
+
* finds an existing project with the same normalized remote URL
|
|
35
|
+
* 4. Create a new project row
|
|
36
|
+
*
|
|
37
|
+
* When a git-remote match is found (step 3), the new path is registered as
|
|
38
|
+
* an alias so subsequent calls skip the subprocess. If the matched project's
|
|
39
|
+
* git_remote was not yet populated (pre-v14 rows), it is backfilled lazily.
|
|
40
|
+
*/
|
|
4
41
|
export declare function ensureProject(path: string, name?: string): string;
|
|
5
42
|
export declare function projectId(path: string): string | undefined;
|
|
6
43
|
/** Look up a project's display name by its internal ID. */
|
|
@@ -19,6 +56,34 @@ export declare function loadForceMinLayer(sessionID: string): number;
|
|
|
19
56
|
* (consumed) to avoid unbounded growth.
|
|
20
57
|
*/
|
|
21
58
|
export declare function saveForceMinLayer(sessionID: string, layer: number): void;
|
|
59
|
+
/** Persisted cost snapshot for a session. */
|
|
60
|
+
export type SessionCostSnapshot = {
|
|
61
|
+
conversationCost: number;
|
|
62
|
+
workerCost: number;
|
|
63
|
+
conversationTurns: number;
|
|
64
|
+
cacheReadTokens: number;
|
|
65
|
+
cacheWriteTokens: number;
|
|
66
|
+
warmupSavings: number;
|
|
67
|
+
warmupHits: number;
|
|
68
|
+
ttlSavings: number;
|
|
69
|
+
ttlHits: number;
|
|
70
|
+
batchSavings: number;
|
|
71
|
+
};
|
|
72
|
+
/**
|
|
73
|
+
* Persist a session's cost snapshot. Uses INSERT OR REPLACE so it works
|
|
74
|
+
* whether or not a row already exists (forceMinLayer may have created one).
|
|
75
|
+
*/
|
|
76
|
+
export declare function saveSessionCosts(sessionID: string, costs: SessionCostSnapshot): void;
|
|
77
|
+
/**
|
|
78
|
+
* Load persisted cost snapshot for a session. Returns null if not stored
|
|
79
|
+
* or if all cost columns are zero (pre-migration row from forceMinLayer only).
|
|
80
|
+
*/
|
|
81
|
+
export declare function loadSessionCosts(sessionID: string): SessionCostSnapshot | null;
|
|
82
|
+
/**
|
|
83
|
+
* Load cost snapshots for all sessions that have non-zero cost data.
|
|
84
|
+
* Returns a map of sessionID → SessionCostSnapshot.
|
|
85
|
+
*/
|
|
86
|
+
export declare function loadAllSessionCosts(): Map<string, SessionCostSnapshot>;
|
|
22
87
|
/** Get a metadata value by key. Returns null if not found. */
|
|
23
88
|
export declare function getMeta(key: string): string | null;
|
|
24
89
|
/** Set a metadata value (upsert). */
|
package/dist/bun/db.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"db.d.ts","sourceRoot":"","sources":["../../src/db.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"db.d.ts","sourceRoot":"","sources":["../../src/db.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAMtC;;;;;;;;GAQG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,GAAG,MAAM,GAAG,IAAI,CAMvE;AA0bD,4DAA4D;AAC5D,wBAAgB,MAAM,IAAI,MAAM,CAI/B;AAID,wBAAgB,EAAE,IAAI,QAAQ,CA8C7B;AAsHD;;;;;;;;;GASG;AACH,wBAAgB,oBAAoB,CAClC,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,GACf,IAAI,CAoCN;AAED,wBAAgB,KAAK,SAKpB;AAID;;;;;;;;;;;;;GAaG;AACH,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,GAAG,MAAM,CAiEjE;AAED,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAW1D;AAED,2DAA2D;AAC3D,wBAAgB,WAAW,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAKrD;AAED;;;GAGG;AACH,wBAAgB,UAAU,IAAI,OAAO,CAKpC;AAMD;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAK3D;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAYxE;AAED,6CAA6C;AAC7C,MAAM,MAAM,mBAAmB,GAAG;IAChC,gBAAgB,EAAE,MAAM,CAAC;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,eAAe,EAAE,MAAM,CAAC;IACxB,gBAAgB,EAAE,MAAM,CAAC;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,mBAAmB,GAAG,IAAI,CA2BpF;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,SAAS,EAAE,MAAM,GAAG,mBAAmB,GAAG,IAAI,CAiC9E;AAED;;;GAGG;AACH,wBAAgB,mBAAmB,IAAI,GAAG,CAAC,MAAM,EAAE,mBAAmB,CAAC,CAsCtE;AAMD,8DAA8D;AAC9D,wBAAgB,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAKlD;AAED,qCAAqC;AACrC,wBAAgB,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAMxD;AAED;;;;;GAKG;AACH,wBAAgB,aAAa,IAAI,MAAM,CAMtC"}
|
|
@@ -15,20 +15,58 @@ type TemporalMessage = temporal.TemporalMessage;
|
|
|
15
15
|
* use as a diagnostic signal, not a hard gate.
|
|
16
16
|
*/
|
|
17
17
|
export declare function compressionRatio(distilledTokens: number, sourceTokens: number): number;
|
|
18
|
+
/**
|
|
19
|
+
* Maximum allowed expansion for distillation output.
|
|
20
|
+
*
|
|
21
|
+
* Tiny segments can't meaningfully compress — distillation adds metadata
|
|
22
|
+
* (timestamps, importance markers, cross-references) that necessarily
|
|
23
|
+
* exceeds the source. Allow generous expansion for small segments while
|
|
24
|
+
* still enforcing compression on large ones.
|
|
25
|
+
*
|
|
26
|
+
* @returns Maximum allowed distilled tokens for a given source token count.
|
|
27
|
+
*/
|
|
28
|
+
export declare function maxAllowedExpansion(sourceTokens: number): number;
|
|
18
29
|
/**
|
|
19
30
|
* Segment detection: group related messages into distillation-sized chunks.
|
|
20
31
|
*
|
|
21
|
-
* When the
|
|
32
|
+
* When the total token count exceeds `maxTokens`, prefers splitting at the
|
|
22
33
|
* largest inter-message time gap (if it's ≥ 3× the median gap) to respect
|
|
23
|
-
* natural conversation boundaries. Falls back to
|
|
24
|
-
*
|
|
34
|
+
* natural conversation boundaries. Falls back to token-boundary splitting
|
|
35
|
+
* when timestamps are uniform.
|
|
25
36
|
*
|
|
26
|
-
* Trailing segments
|
|
27
|
-
* to avoid tiny distillation inputs
|
|
37
|
+
* Trailing segments whose token sum is below {@link MIN_SEGMENT_TOKENS}
|
|
38
|
+
* are merged into the previous segment to avoid tiny distillation inputs
|
|
39
|
+
* with too little context.
|
|
28
40
|
*
|
|
29
41
|
* Exported for testing; `run()` is the production caller.
|
|
30
42
|
*/
|
|
31
|
-
export declare function detectSegments(messages: TemporalMessage[],
|
|
43
|
+
export declare function detectSegments(messages: TemporalMessage[], maxTokens: number): TemporalMessage[][];
|
|
44
|
+
/**
|
|
45
|
+
* Compute the max_tokens budget for a worker LLM call.
|
|
46
|
+
*
|
|
47
|
+
* @param inputTokens Estimated source token count
|
|
48
|
+
* @param ratio Compression ratio (0.0–1.0) — output ≈ ratio × input
|
|
49
|
+
* @param floor Minimum output tokens
|
|
50
|
+
* @param cap Maximum output tokens
|
|
51
|
+
*/
|
|
52
|
+
export declare function workerTokenBudget(inputTokens: number, ratio: number, floor: number, cap: number): number;
|
|
53
|
+
/**
|
|
54
|
+
* Compute the max_tokens budget for gen-0 distillation of raw messages.
|
|
55
|
+
*
|
|
56
|
+
* Uses a √N-based formula (8 × √N) instead of a linear ratio so that the
|
|
57
|
+
* budget grows sub-linearly with input size. This naturally constrains the
|
|
58
|
+
* LLM to produce output at ~R ≈ 2–4 (the square-root boundary) and avoids
|
|
59
|
+
* expansion on small segments where a linear 0.25 ratio + 1024 floor gave
|
|
60
|
+
* the model far too much room.
|
|
61
|
+
*
|
|
62
|
+
* The multiplier (8) gives ~4× headroom above the R=2.0 target, accounting
|
|
63
|
+
* for the detailed observation format (emoji markers, timestamps, entity
|
|
64
|
+
* tags, exact numbers) required by the distillation prompt.
|
|
65
|
+
*
|
|
66
|
+
* @param sourceTokens Estimated source token count from raw messages
|
|
67
|
+
* @returns Token budget clamped to [256, 4096]
|
|
68
|
+
*/
|
|
69
|
+
export declare function distillTokenBudget(sourceTokens: number): number;
|
|
32
70
|
/**
|
|
33
71
|
* Truncate tool outputs within a `TemporalMessage.content` string (produced
|
|
34
72
|
* by `temporal.partsToText`). Plain text and `[reasoning]` chunks pass
|
|
@@ -100,6 +138,7 @@ export type Distillation = {
|
|
|
100
138
|
* rare callers that explicitly want all rows.
|
|
101
139
|
*/
|
|
102
140
|
export declare function loadForSession(projectPath: string, sessionID: string, includeArchived?: boolean): Distillation[];
|
|
141
|
+
export declare function gen0Count(projectPath: string, sessionID: string): number;
|
|
103
142
|
export declare function run(input: {
|
|
104
143
|
llm: LLMClient;
|
|
105
144
|
projectPath: string;
|
|
@@ -121,6 +160,9 @@ export declare function run(input: {
|
|
|
121
160
|
* where the caller is blocking on the result. Background/idle distillation
|
|
122
161
|
* should leave this false to benefit from batch API 50% cost savings. */
|
|
123
162
|
urgent?: boolean;
|
|
163
|
+
/** Whether the LLM call will use batch or direct pricing. Recorded on the
|
|
164
|
+
* distillation row for accurate historical cost estimates. */
|
|
165
|
+
callType?: "batch" | "direct";
|
|
124
166
|
}): Promise<{
|
|
125
167
|
rounds: number;
|
|
126
168
|
distilled: number;
|
|
@@ -142,6 +184,7 @@ export declare function metaDistill(input: {
|
|
|
142
184
|
modelID: string;
|
|
143
185
|
};
|
|
144
186
|
urgent?: boolean;
|
|
187
|
+
callType?: "batch" | "direct";
|
|
145
188
|
}): Promise<DistillationResult | null>;
|
|
146
189
|
/**
|
|
147
190
|
* Backfill `r_compression` and `c_norm` for distillations that were created
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"distillation.d.ts","sourceRoot":"","sources":["../../src/distillation.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,QAAQ,MAAM,YAAY,CAAC;AAavC,OAAO,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAC5C,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAGzC,OAAO,EAAE,gBAAgB,EAAE,CAAC;AAE5B,KAAK,eAAe,GAAG,QAAQ,CAAC,eAAe,CAAC;AAEhD;;;;;;;;;;GAUG;AACH,wBAAgB,gBAAgB,CAC9B,eAAe,EAAE,MAAM,EACvB,YAAY,EAAE,MAAM,GACnB,MAAM,CAGR;AAED
|
|
1
|
+
{"version":3,"file":"distillation.d.ts","sourceRoot":"","sources":["../../src/distillation.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,QAAQ,MAAM,YAAY,CAAC;AAavC,OAAO,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAC5C,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAGzC,OAAO,EAAE,gBAAgB,EAAE,CAAC;AAE5B,KAAK,eAAe,GAAG,QAAQ,CAAC,eAAe,CAAC;AAEhD;;;;;;;;;;GAUG;AACH,wBAAgB,gBAAgB,CAC9B,eAAe,EAAE,MAAM,EACvB,YAAY,EAAE,MAAM,GACnB,MAAM,CAGR;AAED;;;;;;;;;GASG;AACH,wBAAgB,mBAAmB,CAAC,YAAY,EAAE,MAAM,GAAG,MAAM,CAIhE;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,cAAc,CAC5B,QAAQ,EAAE,eAAe,EAAE,EAC3B,SAAS,EAAE,MAAM,GAChB,eAAe,EAAE,EAAE,CAIrB;AAED;;;;;;;GAOG;AACH,wBAAgB,iBAAiB,CAC/B,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,EACb,GAAG,EAAE,MAAM,GACV,MAAM,CAER;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,kBAAkB,CAAC,YAAY,EAAE,MAAM,GAAG,MAAM,CAK/D;AAsID;;;;;;;;;;;;GAYG;AACH,wBAAgB,4BAA4B,CAC1C,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,GACf,MAAM,CAwBR;AAgBD;;;;;;;GAOG;AACH,wBAAgB,cAAc,CAC5B,QAAQ,EAAE,eAAe,EAAE,EAC3B,kBAAkB,CAAC,EAAE,MAAM,GAC1B,MAAM,CAUR;AAED,KAAK,kBAAkB,GAAG;IACxB,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAwBF;;;;;;;;;;;;GAYG;AACH,wBAAgB,sBAAsB,CACpC,WAAW,EAAE,MAAM,EACnB,SAAS,EAAE,MAAM,GAChB,MAAM,GAAG,SAAS,CAEpB;AAwBD,+EAA+E;AAC/E,wBAAgB,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,EAAE,CAQpD;AAED,MAAM,MAAM,YAAY,GAAG;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,2EAA2E;IAC3E,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,8EAA8E;IAC9E,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;CACvB,CAAC;AAEF;;;;;;;;;;;;;GAaG;AACH,wBAAgB,cAAc,CAC5B,WAAW,EAAE,MAAM,EACnB,SAAS,EAAE,MAAM,EACjB,eAAe,UAAQ,GACtB,YAAY,EAAE,CAuBhB;AAyCD,wBAAgB,SAAS,CAAC,WAAW,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,CASxE;AAgGD,wBAAsB,GAAG,CAAC,KAAK,EAAE;IAC/B,GAAG,EAAE,SAAS,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC;IAChD,qEAAqE;IACrE,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB;;;;mEAI+D;IAC/D,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB;;;8EAG0E;IAC1E,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB;mEAC+D;IAC/D,QAAQ,CAAC,EAAE,OAAO,GAAG,QAAQ,CAAC;CAC/B,GAAG,OAAO,CAAC;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,CAAC,CAmFjD;AAmHD;;;;;;;GAOG;AACH,wBAAsB,WAAW,CAAC,KAAK,EAAE;IACvC,GAAG,EAAE,SAAS,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC;IAChD,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,QAAQ,CAAC,EAAE,OAAO,GAAG,QAAQ,CAAC;CAC/B,GAAG,OAAO,CAAC,kBAAkB,GAAG,IAAI,CAAC,CAoGrC;AAMD;;;;;;;;;;;GAWG;AACH,wBAAgB,eAAe,IAAI,MAAM,CAkDxC"}
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Vendored bge-small registration for the standalone Lore binary.
|
|
3
|
+
*
|
|
4
|
+
* The Bun-compiled `lore` binary uses `bun build --compile` to bundle
|
|
5
|
+
* `fastembed` + `onnxruntime-node` + `@anush008/tokenizers-<platform>`
|
|
6
|
+
* directly into the executable — including the platform-specific
|
|
7
|
+
* `.node` addons which Bun embeds and dlopens from `$bunfs` at runtime.
|
|
8
|
+
*
|
|
9
|
+
* Two pieces don't fit into Bun's automatic bundling and need our help:
|
|
10
|
+
*
|
|
11
|
+
* 1. **Side-load shared libraries**. `onnxruntime_binding.node` does a
|
|
12
|
+
* runtime `dlopen("libonnxruntime.so.1")` (or the .dylib / .dll
|
|
13
|
+
* equivalent) for the actual ONNX Runtime computation library. Bun
|
|
14
|
+
* doesn't follow this kind of dependency. The binary's wrapper
|
|
15
|
+
* pre-loads these libs via `bun:ffi` *before* fastembed evaluates,
|
|
16
|
+
* so when the addon's dlopen fires it finds the cached handle.
|
|
17
|
+
*
|
|
18
|
+
* 2. **Model weights + tokenizer**. fastembed downloads from the HF
|
|
19
|
+
* Hub on first use; we want zero network on first run. The wrapper
|
|
20
|
+
* embeds the bge-small INT8 files as Bun assets, writes them to a
|
|
21
|
+
* real disk dir on first run, and sets `globalThis.__LORE_VENDOR_MODEL__`
|
|
22
|
+
* to that path. This module exposes that registration to the
|
|
23
|
+
* LocalProvider so it can hand the path to fastembed's CUSTOM-mode
|
|
24
|
+
* init (`modelAbsoluteDirPath` + `modelName`).
|
|
25
|
+
*
|
|
26
|
+
* In npm-mode usage from `@loreai/opencode` / `@loreai/pi` the global
|
|
27
|
+
* is unset and `vendorModelInfo()` returns `null`, so the LocalProvider
|
|
28
|
+
* falls through to fastembed's default Qdrant repo + cache.
|
|
29
|
+
*/
|
|
30
|
+
/** What the binary wrapper writes to globalThis after extracting model files. */
|
|
31
|
+
export interface VendorRegistration {
|
|
32
|
+
/** Absolute path to the dir containing the bge-small files
|
|
33
|
+
* (config.json, tokenizer.json, model_quantized.onnx, …). Pass to
|
|
34
|
+
* fastembed as `modelAbsoluteDirPath` in CUSTOM init. */
|
|
35
|
+
modelAbsoluteDirPath: string;
|
|
36
|
+
/** Filename of the ONNX weights inside that dir. Pass to fastembed
|
|
37
|
+
* as `modelName` in CUSTOM init. */
|
|
38
|
+
modelName: string;
|
|
39
|
+
/** Target identifier the binary was built for, e.g. "linux-x64".
|
|
40
|
+
* Diagnostic only — the runtime doesn't branch on it. */
|
|
41
|
+
target: string;
|
|
42
|
+
/** Lore CLI version that produced the binary. Diagnostic only. */
|
|
43
|
+
version: string;
|
|
44
|
+
}
|
|
45
|
+
/** Test-only: programmatically set/clear the registration to exercise
|
|
46
|
+
* both binary-mode and npm-mode code paths without spinning up a real
|
|
47
|
+
* compiled binary. */
|
|
48
|
+
export declare function _setVendorRegistration(reg: VendorRegistration | null): void;
|
|
49
|
+
/** Subset of the registration fastembed needs. Stripped of the
|
|
50
|
+
* diagnostic fields so the LocalProvider has exactly what it should
|
|
51
|
+
* hand to `FlagEmbedding.init`. */
|
|
52
|
+
export interface VendorModelInfo {
|
|
53
|
+
modelAbsoluteDirPath: string;
|
|
54
|
+
modelName: string;
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* Resolve the bundled-model arguments for fastembed CUSTOM init. Returns
|
|
58
|
+
* `null` when no vendor is registered (npm-mode), so the caller can fall
|
|
59
|
+
* through to fastembed's default cacheDir/HF Hub flow.
|
|
60
|
+
*/
|
|
61
|
+
export declare function vendorModelInfo(): VendorModelInfo | null;
|
|
62
|
+
/** True iff this process is running inside a vendored Lore binary. */
|
|
63
|
+
export declare function isVendoredBinary(): boolean;
|
|
64
|
+
/** The full registration, for diagnostics (`lore --print-vendor-info`). */
|
|
65
|
+
export declare function vendorRegistration(): VendorRegistration | null;
|
|
66
|
+
//# sourceMappingURL=embedding-vendor.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"embedding-vendor.d.ts","sourceRoot":"","sources":["../../src/embedding-vendor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAMH,iFAAiF;AACjF,MAAM,WAAW,kBAAkB;IACjC;;8DAE0D;IAC1D,oBAAoB,EAAE,MAAM,CAAC;IAC7B;yCACqC;IACrC,SAAS,EAAE,MAAM,CAAC;IAClB;8DAC0D;IAC1D,MAAM,EAAE,MAAM,CAAC;IACf,kEAAkE;IAClE,OAAO,EAAE,MAAM,CAAC;CACjB;AAUD;;uBAEuB;AACvB,wBAAgB,sBAAsB,CAAC,GAAG,EAAE,kBAAkB,GAAG,IAAI,GAAG,IAAI,CAI3E;AAMD;;oCAEoC;AACpC,MAAM,WAAW,eAAe;IAC9B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;;;GAIG;AACH,wBAAgB,eAAe,IAAI,eAAe,GAAG,IAAI,CAOxD;AAED,sEAAsE;AACtE,wBAAgB,gBAAgB,IAAI,OAAO,CAE1C;AAED,2EAA2E;AAC3E,wBAAgB,kBAAkB,IAAI,kBAAkB,GAAG,IAAI,CAE9D"}
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared message types for the embedding worker thread.
|
|
3
|
+
*
|
|
4
|
+
* The embedding worker (`embedding-worker.ts`) runs fastembed/ONNX inference
|
|
5
|
+
* in a separate `node:worker_threads` Worker so the main thread's event loop
|
|
6
|
+
* stays free during inference. This file defines the message protocol between
|
|
7
|
+
* the main thread (`LocalProvider` in `embedding.ts`) and the worker.
|
|
8
|
+
*
|
|
9
|
+
* Imported by both sides — keep this file free of runtime dependencies.
|
|
10
|
+
*/
|
|
11
|
+
/** Request an embedding batch. */
|
|
12
|
+
export interface EmbedRequest {
|
|
13
|
+
type: "embed";
|
|
14
|
+
/** Monotonic request ID for correlating responses. */
|
|
15
|
+
id: number;
|
|
16
|
+
/** Texts to embed. */
|
|
17
|
+
texts: string[];
|
|
18
|
+
/** "document" for storage, "query" for search. */
|
|
19
|
+
inputType: "document" | "query";
|
|
20
|
+
/** "high" = recall queries (jump the queue), "normal" = backfill. */
|
|
21
|
+
priority: "high" | "normal";
|
|
22
|
+
}
|
|
23
|
+
/** Ask the worker to exit cleanly. */
|
|
24
|
+
export interface ShutdownRequest {
|
|
25
|
+
type: "shutdown";
|
|
26
|
+
}
|
|
27
|
+
export type WorkerInbound = EmbedRequest | ShutdownRequest;
|
|
28
|
+
/** Embedding result — vectors are Float32Array[], sent via structured clone. */
|
|
29
|
+
export interface EmbedResult {
|
|
30
|
+
type: "result";
|
|
31
|
+
/** Matches the request ID. */
|
|
32
|
+
id: number;
|
|
33
|
+
/** One Float32Array per input text. Sent via structured clone
|
|
34
|
+
* (Bun preserves Float32Array identity across threads). */
|
|
35
|
+
vectors: Float32Array[];
|
|
36
|
+
}
|
|
37
|
+
/** A single embed request failed (ONNX error, etc.). */
|
|
38
|
+
export interface EmbedError {
|
|
39
|
+
type: "error";
|
|
40
|
+
/** Matches the request ID. */
|
|
41
|
+
id: number;
|
|
42
|
+
/** Human-readable error message. */
|
|
43
|
+
error: string;
|
|
44
|
+
}
|
|
45
|
+
/** Model initialization failed inside the worker. All pending and future
|
|
46
|
+
* requests should be rejected — the worker is unusable. */
|
|
47
|
+
export interface InitError {
|
|
48
|
+
type: "init-error";
|
|
49
|
+
/** Human-readable error message. */
|
|
50
|
+
error: string;
|
|
51
|
+
}
|
|
52
|
+
export type WorkerOutbound = EmbedResult | EmbedError | InitError;
|
|
53
|
+
/** Passed to the worker via `workerData` at construction time. */
|
|
54
|
+
export interface WorkerInitData {
|
|
55
|
+
/** fastembed model name, e.g. "BGESmallENV15". */
|
|
56
|
+
modelName: string;
|
|
57
|
+
/** Vendored model info for binary mode, or null for npm mode.
|
|
58
|
+
* Mirrors the `globalThis.__LORE_VENDOR_MODEL__` registration which
|
|
59
|
+
* only exists on the main thread — passed explicitly so the worker
|
|
60
|
+
* can hand it to `FlagEmbedding.init()`. */
|
|
61
|
+
vendorModel: {
|
|
62
|
+
modelAbsoluteDirPath: string;
|
|
63
|
+
modelName: string;
|
|
64
|
+
} | null;
|
|
65
|
+
}
|
|
66
|
+
//# sourceMappingURL=embedding-worker-types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"embedding-worker-types.d.ts","sourceRoot":"","sources":["../../src/embedding-worker-types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAMH,kCAAkC;AAClC,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,OAAO,CAAC;IACd,sDAAsD;IACtD,EAAE,EAAE,MAAM,CAAC;IACX,sBAAsB;IACtB,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,kDAAkD;IAClD,SAAS,EAAE,UAAU,GAAG,OAAO,CAAC;IAChC,qEAAqE;IACrE,QAAQ,EAAE,MAAM,GAAG,QAAQ,CAAC;CAC7B;AAED,sCAAsC;AACtC,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,UAAU,CAAC;CAClB;AAED,MAAM,MAAM,aAAa,GAAG,YAAY,GAAG,eAAe,CAAC;AAM3D,gFAAgF;AAChF,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,QAAQ,CAAC;IACf,8BAA8B;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX;gEAC4D;IAC5D,OAAO,EAAE,YAAY,EAAE,CAAC;CACzB;AAED,wDAAwD;AACxD,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,OAAO,CAAC;IACd,8BAA8B;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX,oCAAoC;IACpC,KAAK,EAAE,MAAM,CAAC;CACf;AAED;4DAC4D;AAC5D,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,YAAY,CAAC;IACnB,oCAAoC;IACpC,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,MAAM,cAAc,GAAG,WAAW,GAAG,UAAU,GAAG,SAAS,CAAC;AAMlE,kEAAkE;AAClE,MAAM,WAAW,cAAc;IAC7B,kDAAkD;IAClD,SAAS,EAAE,MAAM,CAAC;IAClB;;;iDAG6C;IAC7C,WAAW,EAAE;QAAE,oBAAoB,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;CACzE"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Embedding worker thread — runs fastembed/ONNX inference off the main thread.
|
|
3
|
+
*
|
|
4
|
+
* This file is the entry point for a `node:worker_threads` Worker spawned by
|
|
5
|
+
* `LocalProvider` in `embedding.ts`. It owns the `FlagEmbedding` ONNX model
|
|
6
|
+
* and processes embed requests sequentially from a priority queue. Moving
|
|
7
|
+
* inference here keeps the main thread's event loop free — HTTP requests,
|
|
8
|
+
* SSE streams, and session APIs are no longer blocked during embedding.
|
|
9
|
+
*
|
|
10
|
+
* Communication uses `parentPort` message passing with structured clone.
|
|
11
|
+
* Float32Array vectors are sent back directly (Bun preserves identity).
|
|
12
|
+
*
|
|
13
|
+
* @see embedding-worker-types.ts for the message protocol.
|
|
14
|
+
*/
|
|
15
|
+
export {};
|
|
16
|
+
//# sourceMappingURL=embedding-worker.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"embedding-worker.d.ts","sourceRoot":"","sources":["../../src/embedding-worker.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG"}
|