@luckydraw/cumulus 0.7.3 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +4 -4
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -3
- package/dist/index.js.map +1 -1
- package/dist/lib/config.d.ts +11 -0
- package/dist/lib/config.d.ts.map +1 -1
- package/dist/lib/config.js +31 -0
- package/dist/lib/config.js.map +1 -1
- package/dist/lib/content-store.d.ts +10 -7
- package/dist/lib/content-store.d.ts.map +1 -1
- package/dist/lib/content-store.js +111 -52
- package/dist/lib/content-store.js.map +1 -1
- package/dist/lib/embeddings.d.ts +36 -2
- package/dist/lib/embeddings.d.ts.map +1 -1
- package/dist/lib/embeddings.js +306 -34
- package/dist/lib/embeddings.js.map +1 -1
- package/dist/lib/history.d.ts +10 -0
- package/dist/lib/history.d.ts.map +1 -1
- package/dist/lib/history.js +92 -0
- package/dist/lib/history.js.map +1 -1
- package/dist/lib/retriever.d.ts.map +1 -1
- package/dist/lib/retriever.js +23 -5
- package/dist/lib/retriever.js.map +1 -1
- package/package.json +1 -1
package/dist/index.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export { HistoryStore } from './lib/history.js';
|
|
1
|
+
export { HistoryStore, setOnAppendHook } from './lib/history.js';
|
|
2
2
|
export type { Attachment, Message, HistoryStats } from './lib/history.js';
|
|
3
3
|
export { ContentStore } from './lib/content-store.js';
|
|
4
4
|
export type { ContentType, ContentSourceType, ContentChunk, StoredContentMeta, StoredContent, StoreOptions, SearchOptions, ContentSearchResult, } from './lib/content-store.js';
|
|
@@ -8,7 +8,7 @@ export { estimateTokens, ContextBudget } from './lib/context-budget.js';
|
|
|
8
8
|
export type { ContextBudgetConfig } from './lib/context-budget.js';
|
|
9
9
|
export { AdaptiveContextBudget, getAdaptiveStatePath, loadAdaptiveState, recordAndSaveAdaptiveTurn, } from './lib/adaptive-context-budget.js';
|
|
10
10
|
export type { AdaptiveContextBudgetConfig, AdaptiveState, AdaptiveTurnRecord, RecordTurnInput, } from './lib/adaptive-context-budget.js';
|
|
11
|
-
export { cosineSimilarity, embeddingsAvailable, generateMissingEmbeddings, getEmbeddingProvider, getEmbeddingsPath, loadEmbeddings, saveEmbeddings, } from './lib/embeddings.js';
|
|
11
|
+
export { cosineSimilarity, embeddingsAvailable, embedSingleMessage, generateMissingEmbeddings, getEmbeddingProvider, getEmbeddingsPath, loadEmbeddings, saveEmbeddings, } from './lib/embeddings.js';
|
|
12
12
|
export type { EmbeddingConfig, EmbeddingEntry, EmbeddingProvider, EmbeddingsFile, } from './lib/embeddings.js';
|
|
13
13
|
export { SessionManager } from './lib/session.js';
|
|
14
14
|
export type { SessionChunk, SessionSearchResult } from './lib/session.js';
|
|
@@ -16,8 +16,8 @@ export { retrieve, RETRIEVAL_BUDGET_FRACTION } from './lib/retriever.js';
|
|
|
16
16
|
export type { RetrievalResult, RetrieverOptions } from './lib/retriever.js';
|
|
17
17
|
export { StreamProcessor, shouldExternalizeUserInput, externalizeUserInput, } from './lib/stream-processor.js';
|
|
18
18
|
export type { StreamEventType, StreamMessage, ProcessedLine, StreamProcessorConfig, } from './lib/stream-processor.js';
|
|
19
|
-
export { loadGlobalConfig, loadThreadConfig, mergeConfigs, resolveFilePath, readAlwaysIncludeFiles, saveConfig, addAlwaysIncludeFile, removeAlwaysIncludeFile, listAlwaysIncludeFiles, } from './lib/config.js';
|
|
20
|
-
export type { CumulusConfig, AlwaysIncludeFile, AlwaysIncludeResult } from './lib/config.js';
|
|
19
|
+
export { loadGlobalConfig, loadThreadConfig, mergeConfigs, resolveFilePath, readAlwaysIncludeFiles, saveConfig, addAlwaysIncludeFile, removeAlwaysIncludeFile, listAlwaysIncludeFiles, resolveRagBudget, } from './lib/config.js';
|
|
20
|
+
export type { CumulusConfig, RagBudgetPreset, AlwaysIncludeFile, AlwaysIncludeResult } from './lib/config.js';
|
|
21
21
|
export { processImagesInMessage, readClipboardImage } from './lib/image-utils.js';
|
|
22
22
|
export type { DetectedImage, ImageProcessingResult } from './lib/image-utils.js';
|
|
23
23
|
export { isGitRepo, captureSnapshot, restoreSnapshot } from './lib/snapshots.js';
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACjE,YAAY,EAAE,UAAU,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAG1E,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AACtD,YAAY,EACV,WAAW,EACX,iBAAiB,EACjB,YAAY,EACZ,iBAAiB,EACjB,aAAa,EACb,YAAY,EACZ,aAAa,EACb,mBAAmB,GACpB,MAAM,wBAAwB,CAAC;AAGhC,OAAO,EACL,cAAc,EACd,wBAAwB,EACxB,kBAAkB,EAClB,eAAe,EACf,iBAAiB,GAClB,MAAM,2BAA2B,CAAC;AACnC,YAAY,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAGjF,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAC;AACxE,YAAY,EAAE,mBAAmB,EAAE,MAAM,yBAAyB,CAAC;AAGnE,OAAO,EACL,qBAAqB,EACrB,oBAAoB,EACpB,iBAAiB,EACjB,yBAAyB,GAC1B,MAAM,kCAAkC,CAAC;AAC1C,YAAY,EACV,2BAA2B,EAC3B,aAAa,EACb,kBAAkB,EAClB,eAAe,GAChB,MAAM,kCAAkC,CAAC;AAG1C,OAAO,EACL,gBAAgB,EAChB,mBAAmB,EACnB,kBAAkB,EAClB,yBAAyB,EACzB,oBAAoB,EACpB,iBAAiB,EACjB,cAAc,EACd,cAAc,GACf,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EACV,eAAe,EACf,cAAc,EACd,iBAAiB,EACjB,cAAc,GACf,MAAM,qBAAqB,CAAC;AAG7B,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAClD,YAAY,EAAE,YAAY,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAC;AAG1E,OAAO,EAAE,QAAQ,EAAE,yBAAyB,EAAE,MAAM,oBAAoB,CAAC;AACzE,YAAY,EAAE,eAAe,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAG5E,OAAO,EACL,eAAe,EACf,0BAA0B,EAC1B,oBAAoB,GACrB,MAAM,2BAA2B,CAAC;AACnC,YAAY,EACV,eAAe,EACf,aAAa,EACb,aAAa,EACb,qBAAqB,GACtB,MAAM,2BAA2B,CAAC;AAGnC,OAAO,EACL,gBAAgB,EAChB,gBAAgB,EAChB,YAAY,EACZ,eAAe,EACf,sBAAsB,EACtB,UAAU,EACV,oBAAoB,EACpB,uBAAuB,EACvB,sBAAsB,EACtB,gBAAgB,GACjB,MAAM,iBAAiB,CAAC;AACzB,YAAY,EAAE,aAAa,EAAE,eAAe,EAAE,iBAAiB,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAG9G,OAAO,EAAE,sBAAsB,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAClF,YAAY,EAAE,aAAa,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAGjF,OAAO,EAAE,SAAS,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACjF,YAAY,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAGrE,OAAO,EACL,UAAU,EACV,cAAc,EACd,aAAa,EACb,gBAAgB,EAChB,aAAa,EACb,eAAe,EACf,oBAAoB,EACpB,sBAAsB,GACvB,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EAAE,gBAAgB,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAGlF,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAChD,YAAY,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAGnE,OAAO,EAAE,oBAAoB,EAAE,MAAM,wBAAwB,CAAC;AAC9D,YAAY,EAAE,sBAAsB,EAAE,qBAAqB,EAAE,MAAM,wBAAwB,CAAC;AAQ5F,qFAAqF;AACrF,wBAAgB,YAAY,IAAI,MAAM,CAErC;AAED,0EAA0E;AAC1E,wBAAgB,gBAAgB,IAAI,MAAM,CAEzC"}
|
package/dist/index.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// Cumulus - RLM-based CLI chat wrapper for Claude
|
|
2
2
|
// Exports for library usage
|
|
3
3
|
// History
|
|
4
|
-
export { HistoryStore } from './lib/history.js';
|
|
4
|
+
export { HistoryStore, setOnAppendHook } from './lib/history.js';
|
|
5
5
|
// Content Store
|
|
6
6
|
export { ContentStore } from './lib/content-store.js';
|
|
7
7
|
// Content Detector
|
|
@@ -11,7 +11,7 @@ export { estimateTokens, ContextBudget } from './lib/context-budget.js';
|
|
|
11
11
|
// Adaptive Context Budget
|
|
12
12
|
export { AdaptiveContextBudget, getAdaptiveStatePath, loadAdaptiveState, recordAndSaveAdaptiveTurn, } from './lib/adaptive-context-budget.js';
|
|
13
13
|
// Embeddings
|
|
14
|
-
export { cosineSimilarity, embeddingsAvailable, generateMissingEmbeddings, getEmbeddingProvider, getEmbeddingsPath, loadEmbeddings, saveEmbeddings, } from './lib/embeddings.js';
|
|
14
|
+
export { cosineSimilarity, embeddingsAvailable, embedSingleMessage, generateMissingEmbeddings, getEmbeddingProvider, getEmbeddingsPath, loadEmbeddings, saveEmbeddings, } from './lib/embeddings.js';
|
|
15
15
|
// Session
|
|
16
16
|
export { SessionManager } from './lib/session.js';
|
|
17
17
|
// Retriever
|
|
@@ -19,7 +19,7 @@ export { retrieve, RETRIEVAL_BUDGET_FRACTION } from './lib/retriever.js';
|
|
|
19
19
|
// Stream Processor
|
|
20
20
|
export { StreamProcessor, shouldExternalizeUserInput, externalizeUserInput, } from './lib/stream-processor.js';
|
|
21
21
|
// Config
|
|
22
|
-
export { loadGlobalConfig, loadThreadConfig, mergeConfigs, resolveFilePath, readAlwaysIncludeFiles, saveConfig, addAlwaysIncludeFile, removeAlwaysIncludeFile, listAlwaysIncludeFiles, } from './lib/config.js';
|
|
22
|
+
export { loadGlobalConfig, loadThreadConfig, mergeConfigs, resolveFilePath, readAlwaysIncludeFiles, saveConfig, addAlwaysIncludeFile, removeAlwaysIncludeFile, listAlwaysIncludeFiles, resolveRagBudget, } from './lib/config.js';
|
|
23
23
|
// Image Utils
|
|
24
24
|
export { processImagesInMessage, readClipboardImage } from './lib/image-utils.js';
|
|
25
25
|
// Snapshots
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,kDAAkD;AAClD,4BAA4B;AAE5B,UAAU;AACV,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,kDAAkD;AAClD,4BAA4B;AAE5B,UAAU;AACV,OAAO,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAGjE,gBAAgB;AAChB,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAYtD,mBAAmB;AACnB,OAAO,EACL,cAAc,EACd,wBAAwB,EACxB,kBAAkB,EAClB,eAAe,EACf,iBAAiB,GAClB,MAAM,2BAA2B,CAAC;AAGnC,iBAAiB;AACjB,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAC;AAGxE,0BAA0B;AAC1B,OAAO,EACL,qBAAqB,EACrB,oBAAoB,EACpB,iBAAiB,EACjB,yBAAyB,GAC1B,MAAM,kCAAkC,CAAC;AAQ1C,aAAa;AACb,OAAO,EACL,gBAAgB,EAChB,mBAAmB,EACnB,kBAAkB,EAClB,yBAAyB,EACzB,oBAAoB,EACpB,iBAAiB,EACjB,cAAc,EACd,cAAc,GACf,MAAM,qBAAqB,CAAC;AAQ7B,UAAU;AACV,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAGlD,YAAY;AACZ,OAAO,EAAE,QAAQ,EAAE,yBAAyB,EAAE,MAAM,oBAAoB,CAAC;AAGzE,mBAAmB;AACnB,OAAO,EACL,eAAe,EACf,0BAA0B,EAC1B,oBAAoB,GACrB,MAAM,2BAA2B,CAAC;AAQnC,SAAS;AACT,OAAO,EACL,gBAAgB,EAChB,gBAAgB,EAChB,YAAY,EACZ,eAAe,EACf,sBAAsB,EACtB,UAAU,EACV,oBAAoB,EACpB,uBAAuB,EACvB,sBAAsB,EACtB,gBAAgB,GACjB,MAAM,iBAAiB,CAAC;AAGzB,cAAc;AACd,OAAO,EAAE,sBAAsB,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAGlF,YAAY;AACZ,OAAO,EAAE,SAAS,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAGjF,aAAa;AACb,OAAO,EACL,UAAU,EACV,cAAc,EACd,aAAa,EACb,gBAAgB,EAChB,aAAa,EACb,eAAe,EACf,oBAAoB,EACpB,sBAAsB,GACvB,MAAM,qBAAqB,CAAC;AAG7B,SAAS;AACT,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAGhD,6CAA6C;AAC7C,OAAO,EAAE,oBAAoB,EAAE,MAAM,wBAAwB,CAAC;AAG9D,yDAAyD;AACzD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AACrC,OAAO,EAAE,aAAa,EAAE,MAAM,KAAK,CAAC;AAEpC,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAE9D,qFAAqF;AACrF,MAAM,UAAU,YAAY;IAC1B,OAAO,IAAI,CAAC,aAAa,EAAE,KAAK,EAAE,UAAU,CAAC,CAAC;AAChD,CAAC;AAED,0EAA0E;AAC1E,MAAM,UAAU,gBAAgB;IAC9B,OAAO,IAAI,CAAC,aAAa,EAAE,KAAK,EAAE,UAAU,CAAC,CAAC;AAChD,CAAC"}
|
package/dist/lib/config.d.ts
CHANGED
|
@@ -4,13 +4,24 @@
|
|
|
4
4
|
* Handles global and per-thread configuration, including reading
|
|
5
5
|
* files that should always be included in the context window.
|
|
6
6
|
*/
|
|
7
|
+
/** RAG budget preset for testing TTFT vs context size tradeoffs */
|
|
8
|
+
export type RagBudgetPreset = 'small' | 'medium' | 'large' | 'adaptive';
|
|
7
9
|
/** Configuration structure for Cumulus */
|
|
8
10
|
export interface CumulusConfig {
|
|
9
11
|
/** File paths to always include in context */
|
|
10
12
|
alwaysInclude?: string[];
|
|
11
13
|
/** Project directory for this thread (used by callers like Janus to set cwd) */
|
|
12
14
|
projectDir?: string;
|
|
15
|
+
/** RAG budget preset — overrides adaptive budget when set to small/medium/large */
|
|
16
|
+
ragBudgetPreset?: RagBudgetPreset;
|
|
13
17
|
}
|
|
18
|
+
/**
|
|
19
|
+
* Resolve the RAG budget based on config preset and adaptive fallback.
|
|
20
|
+
* @param config - Merged config (may have ragBudgetPreset)
|
|
21
|
+
* @param adaptiveBudget - The budget from AdaptiveContextBudget (used when preset is 'adaptive' or unset)
|
|
22
|
+
* @returns Token budget for RAG retrieval
|
|
23
|
+
*/
|
|
24
|
+
export declare function resolveRagBudget(config: CumulusConfig, adaptiveBudget: number): number;
|
|
14
25
|
/** Information about a single always-include file */
|
|
15
26
|
export interface AlwaysIncludeFile {
|
|
16
27
|
/** Original path from config */
|
package/dist/lib/config.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/lib/config.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAWH,0CAA0C;AAC1C,MAAM,WAAW,aAAa;IAC5B,8CAA8C;IAC9C,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;IACzB,gFAAgF;IAChF,UAAU,CAAC,EAAE,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/lib/config.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAWH,mEAAmE;AACnE,MAAM,MAAM,eAAe,GAAG,OAAO,GAAG,QAAQ,GAAG,OAAO,GAAG,UAAU,CAAC;AASxE,0CAA0C;AAC1C,MAAM,WAAW,aAAa;IAC5B,8CAA8C;IAC9C,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;IACzB,gFAAgF;IAChF,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,mFAAmF;IACnF,eAAe,CAAC,EAAE,eAAe,CAAC;CACnC;AAED;;;;;GAKG;AACH,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,GAAG,MAAM,CAItF;AAED,qDAAqD;AACrD,MAAM,WAAW,iBAAiB;IAChC,gCAAgC;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,6BAA6B;IAC7B,YAAY,EAAE,MAAM,CAAC;IACrB,uCAAuC;IACvC,OAAO,EAAE,MAAM,CAAC;IAChB,6BAA6B;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,oCAAoC;IACpC,SAAS,EAAE,OAAO,CAAC;IACnB,6CAA6C;IAC7C,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,iDAAiD;AACjD,MAAM,WAAW,mBAAmB;IAClC,kCAAkC;IAClC,KAAK,EAAE,iBAAiB,EAAE,CAAC;IAC3B,oCAAoC;IACpC,WAAW,EAAE,MAAM,CAAC;IACpB,iDAAiD;IACjD,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAED;;GAEG;AACH,wBAAsB,gBAAgB,IAAI,OAAO,CAAC,aAAa,CAAC,CAG/D;AAED;;GAEG;AACH,wBAAsB,gBAAgB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,CAGjF;AA2DD;;GAEG;AACH,wBAAgB,YAAY,CAAC,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,aAAa,GAAG,aAAa,CAgCxF;AAED;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,CAgBrE;AAED;;;;GAIG;AACH,wBAAsB,sBAAsB,CAC1C,MAAM,EAAE,aAAa,EACrB,GAAG,EAAE,MAAM,GACV,OAAO,CAAC,mBAAmB,CAAC,CAiD9B;AAyBD;;GAEG;AACH,wBAAsB,UAAU,CAAC,MAAM,EAAE,aAAa,EAAE,UAAU,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAY1F;AAED;;;;GAIG;AACH,wBAAsB,oBAAoB,CAAC,QAAQ,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAU/F;AAED;;;;GAIG;AACH,wBAAsB,uBAAuB,CAC3C,QAAQ,EAAE,MAAM,EAChB,UAAU,CAAC,EAAE,MAAM,GAClB,OAAO,CAAC,IAAI,CAAC,CAWf;AAED;;GAEG;AACH,wBAAsB,sBAAsB,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;IACzE,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB,CAAC,CAUD"}
|
package/dist/lib/config.js
CHANGED
|
@@ -10,6 +10,24 @@ import * as path from 'path';
|
|
|
10
10
|
import { estimateTokens } from './context-budget.js';
|
|
11
11
|
const CUMULUS_DIR = path.join(os.homedir(), '.cumulus');
|
|
12
12
|
const THREADS_DIR = path.join(CUMULUS_DIR, 'threads');
|
|
13
|
+
/** Fixed token budgets for each preset (large gaps for clear TTFT differentiation) */
|
|
14
|
+
const RAG_BUDGET_PRESETS = {
|
|
15
|
+
small: 10_000,
|
|
16
|
+
medium: 50_000,
|
|
17
|
+
large: 150_000,
|
|
18
|
+
};
|
|
19
|
+
/**
|
|
20
|
+
* Resolve the RAG budget based on config preset and adaptive fallback.
|
|
21
|
+
* @param config - Merged config (may have ragBudgetPreset)
|
|
22
|
+
* @param adaptiveBudget - The budget from AdaptiveContextBudget (used when preset is 'adaptive' or unset)
|
|
23
|
+
* @returns Token budget for RAG retrieval
|
|
24
|
+
*/
|
|
25
|
+
export function resolveRagBudget(config, adaptiveBudget) {
|
|
26
|
+
const preset = config.ragBudgetPreset ?? 'adaptive';
|
|
27
|
+
if (preset === 'adaptive')
|
|
28
|
+
return adaptiveBudget;
|
|
29
|
+
return RAG_BUDGET_PRESETS[preset];
|
|
30
|
+
}
|
|
13
31
|
/**
|
|
14
32
|
* Load the global configuration from ~/.cumulus/config.json
|
|
15
33
|
*/
|
|
@@ -59,6 +77,14 @@ async function loadConfigFile(configPath) {
|
|
|
59
77
|
delete config.projectDir;
|
|
60
78
|
}
|
|
61
79
|
}
|
|
80
|
+
// Validate ragBudgetPreset if present
|
|
81
|
+
if (config.ragBudgetPreset !== undefined) {
|
|
82
|
+
const valid = ['small', 'medium', 'large', 'adaptive'];
|
|
83
|
+
if (!valid.includes(config.ragBudgetPreset)) {
|
|
84
|
+
console.warn(`[Config] ragBudgetPreset must be one of ${valid.join('/')} in ${configPath}`);
|
|
85
|
+
delete config.ragBudgetPreset;
|
|
86
|
+
}
|
|
87
|
+
}
|
|
62
88
|
return config;
|
|
63
89
|
}
|
|
64
90
|
catch (err) {
|
|
@@ -91,6 +117,11 @@ export function mergeConfigs(global, thread) {
|
|
|
91
117
|
if (projectDir) {
|
|
92
118
|
merged.projectDir = projectDir;
|
|
93
119
|
}
|
|
120
|
+
// ragBudgetPreset: thread overrides global
|
|
121
|
+
const ragBudgetPreset = thread.ragBudgetPreset ?? global.ragBudgetPreset;
|
|
122
|
+
if (ragBudgetPreset) {
|
|
123
|
+
merged.ragBudgetPreset = ragBudgetPreset;
|
|
124
|
+
}
|
|
94
125
|
return merged;
|
|
95
126
|
}
|
|
96
127
|
/**
|
package/dist/lib/config.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config.js","sourceRoot":"","sources":["../../src/lib/config.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAE7B,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAErD,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,UAAU,CAAC,CAAC;AACxD,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"config.js","sourceRoot":"","sources":["../../src/lib/config.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAE7B,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAErD,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,UAAU,CAAC,CAAC;AACxD,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;AAKtD,sFAAsF;AACtF,MAAM,kBAAkB,GAAyD;IAC/E,KAAK,EAAE,MAAM;IACb,MAAM,EAAE,MAAM;IACd,KAAK,EAAE,OAAO;CACf,CAAC;AAYF;;;;;GAKG;AACH,MAAM,UAAU,gBAAgB,CAAC,MAAqB,EAAE,cAAsB;IAC5E,MAAM,MAAM,GAAG,MAAM,CAAC,eAAe,IAAI,UAAU,CAAC;IACpD,IAAI,MAAM,KAAK,UAAU;QAAE,OAAO,cAAc,CAAC;IACjD,OAAO,kBAAkB,CAAC,MAAM,CAAC,CAAC;AACpC,CAAC;AA4BD;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB;IACpC,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,aAAa,CAAC,CAAC;IACzD,OAAO,cAAc,CAAC,UAAU,CAAC,CAAC;AACpC,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,UAAkB;IACvD,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,GAAG,UAAU,cAAc,CAAC,CAAC;IACvE,OAAO,cAAc,CAAC,UAAU,CAAC,CAAC;AACpC,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,cAAc,CAAC,UAAkB;IAC9C,IAAI,CAAC;QACH,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YAC/B,OAAO,EAAE,CAAC;QACZ,CAAC;QACD,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QACrD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAEnC,qBAAqB;QACrB,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;YAClD,OAAO,CAAC,IAAI,CAAC,qCAAqC,UAAU,sBAAsB,CAAC,CAAC;YACpF,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,oCAAoC;QACpC,IAAI,MAAM,CAAC,aAAa,KAAK,SAAS,EAAE,CAAC;YACvC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC;gBACzC,OAAO,CAAC,IAAI,CAAC,8CAA8C,UAAU,EAAE,CAAC,CAAC;gBACzE,OAAO,EAAE,CAAC;YACZ,CAAC;YACD,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,aAAa,EAAE,CAAC;gBACxC,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;oBAC7B,OAAO,CAAC,IAAI,CAAC,mDAAmD,UAAU,EAAE,CAAC,CAAC;oBAC9E,OAAO,EAAE,CAAC;gBACZ,CAAC;YACH,CAAC;QACH,CAAC;QAED,iCAAiC;QACjC,IAAI,MAAM,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;YACpC,IAAI,OAAO,MAAM,CAAC,UAAU,KAAK,QAAQ,EAAE,CAAC;gBAC1C,OAAO,CAAC,IAAI,CAAC,2CAA2C,UAAU,EAAE,CAAC,CAAC;gBACtE,OAAO,MAAM,CAAC,UAAU,CAAC;YAC3B,CAAC;QACH,CAAC;QAED,sCAAsC;QACtC,IAAI,MAAM,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;YACzC,MAAM,KAAK,GAAsB,CAAC,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC;YAC1E,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,eAAe,CAAC,EAAE,CAAC;gBAC5C,OAAO,CAAC,IAAI,CAAC,2CAA2C,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,UAAU,EAAE,CAAC,CAAC;gBAC5F,OAAO,MAAM,CAAC,eAAe,CAAC;YAChC,CAAC;QACH,CAAC;QAED,OAAO,MAAuB,CAAC;IACjC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,GAAG,YAAY,WAAW,EAAE,CAAC;YAC/B,OAAO,CAAC,IAAI,CAAC,4BAA4B,UAAU,KAAK,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;QACzE,CAAC;QACD,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,YAAY,CAAC,MAAqB,EAAE,MAAqB;IACvE,MAAM,MAAM,GAAkB,EAAE,CAAC;IAEjC,gFAAgF;IAChF,MAAM,cAAc,GAAG,MAAM,CAAC,aAAa,IAAI,EAAE,CAAC;IAClD,MAAM,cAAc,GAAG,MAAM,CAAC,aAAa,IAAI,EAAE,CAAC;IAElD,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3D,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/B,MAAM,CAAC,aAAa,GAAG,EAAE,CAAC;QAE1B,KAAK,MAAM,IAAI,IAAI,CAAC,GAAG,cAAc,EAAE,GAAG,cAAc,CAAC,EAAE,CAAC;YAC1D,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBACpB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBACf,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAClC,CAAC;QACH,CAAC;IACH,CAAC;IAED,yEAAyE;IACzE,MAAM,UAAU,GAAG,MAAM,CAAC,UAAU,IAAI,MAAM,CAAC,UAAU,CAAC;IAC1D,IAAI,UAAU,EAAE,CAAC;QACf,MAAM,CAAC,UAAU,GAAG,UAAU,CAAC;IACjC,CAAC;IAED,2CAA2C;IAC3C,MAAM,eAAe,GAAG,MAAM,CAAC,eAAe,IAAI,MAAM,CAAC,eAAe,CAAC;IACzE,IAAI,eAAe,EAAE,CAAC;QACpB,MAAM,CAAC,eAAe,GAAG,eAAe,CAAC;IAC3C,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAAC,QAAgB,EAAE,GAAW;IAC3D,8BAA8B;IAC9B,IAAI,QAAQ,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QAC9B,OAAO,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IACpD,CAAC;IACD,IAAI,QAAQ,KAAK,GAAG,EAAE,CAAC;QACrB,OAAO,EAAE,CAAC,OAAO,EAAE,CAAC;IACtB,CAAC;IAED,wBAAwB;IACxB,IAAI,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC9B,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,mCAAmC;IACnC,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;AACrC,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,MAAqB,EACrB,GAAW;IAEX,MAAM,KAAK,GAAwB,EAAE,CAAC;IACtC,IAAI,WAAW,GAAG,CAAC,CAAC;IAEpB,MAAM,KAAK,GAAG,MAAM,CAAC,aAAa,IAAI,EAAE,CAAC;IAEzC,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE,CAAC;QAC7B,MAAM,YAAY,GAAG,eAAe,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QACpD,MAAM,QAAQ,GAAsB;YAClC,IAAI,EAAE,QAAQ;YACd,YAAY;YACZ,OAAO,EAAE,EAAE;YACX,MAAM,EAAE,CAAC;YACT,SAAS,EAAE,KAAK;SACjB,CAAC;QAEF,IAAI,CAAC;YACH,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;gBACjC,QAAQ,CAAC,KAAK,GAAG,gBAAgB,CAAC;gBAClC,OAAO,CAAC,IAAI,CAAC,2CAA2C,YAAY,EAAE,CAAC,CAAC;YAC1E,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,GAAG,EAAE,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;gBACvC,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;oBACvB,QAAQ,CAAC,KAAK,GAAG,qBAAqB,CAAC;oBACvC,OAAO,CAAC,IAAI,CAAC,gDAAgD,YAAY,EAAE,CAAC,CAAC;gBAC/E,CAAC;qBAAM,CAAC;oBACN,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;oBACvD,QAAQ,CAAC,OAAO,GAAG,OAAO,CAAC;oBAC3B,QAAQ,CAAC,MAAM,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;oBAC1C,WAAW,IAAI,QAAQ,CAAC,MAAM,CAAC;gBACjC,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,QAAQ,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;YACtE,QAAQ,CAAC,KAAK,GAAG,QAAQ,CAAC;YAC1B,OAAO,CAAC,IAAI,CAAC,+CAA+C,YAAY,KAAK,QAAQ,EAAE,CAAC,CAAC;QAC3F,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACvB,CAAC;IAED,4BAA4B;IAC5B,MAAM,gBAAgB,GAAG,0BAA0B,CAAC,KAAK,CAAC,CAAC;IAE3D,OAAO;QACL,KAAK;QACL,WAAW;QACX,gBAAgB;KACjB,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,SAAS,0BAA0B,CAAC,KAA0B;IAC5D,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC;IAE5D,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,KAAK,GAAa,CAAC,4CAA4C,EAAE,KAAK,CAAC,CAAC;IAE9E,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC9B,KAAK,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC;QAC7B,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACzB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAEpB,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,MAAqB,EAAE,UAAmB;IACzE,MAAM,UAAU,GAAG,UAAU;QAC3B,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,GAAG,UAAU,cAAc,CAAC;QACrD,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,aAAa,CAAC,CAAC;IAE1C,0BAA0B;IAC1B,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IACrC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QACxB,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACzC,CAAC;IAED,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;AACvE,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,QAAgB,EAAE,UAAmB;IAC9E,MAAM,MAAM,GAAG,UAAU,CAAC,CAAC,CAAC,MAAM,gBAAgB,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,MAAM,gBAAgB,EAAE,CAAC;IAE1F,MAAM,aAAa,GAAG,MAAM,CAAC,aAAa,IAAI,EAAE,CAAC;IAEjD,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QACtC,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC7B,MAAM,CAAC,aAAa,GAAG,aAAa,CAAC;QACrC,MAAM,UAAU,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IACvC,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC3C,QAAgB,EAChB,UAAmB;IAEnB,MAAM,MAAM,GAAG,UAAU,CAAC,CAAC,CAAC,MAAM,gBAAgB,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,MAAM,gBAAgB,EAAE,CAAC;IAE1F,MAAM,aAAa,GAAG,MAAM,CAAC,aAAa,IAAI,EAAE,CAAC;IACjD,MAAM,KAAK,GAAG,aAAa,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAE9C,IAAI,KAAK,KAAK,CAAC,CAAC,EAAE,CAAC;QACjB,aAAa,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QAC/B,MAAM,CAAC,aAAa,GAAG,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS,CAAC;QAC5E,MAAM,UAAU,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IACvC,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAAC,UAAmB;IAK9D,MAAM,YAAY,GAAG,MAAM,gBAAgB,EAAE,CAAC;IAC9C,MAAM,YAAY,GAAG,UAAU,CAAC,CAAC,CAAC,MAAM,gBAAgB,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAC1E,MAAM,YAAY,GAAG,YAAY,CAAC,YAAY,EAAE,YAAY,CAAC,CAAC;IAE9D,OAAO;QACL,MAAM,EAAE,YAAY,CAAC,aAAa,IAAI,EAAE;QACxC,MAAM,EAAE,YAAY,CAAC,aAAa,IAAI,EAAE;QACxC,MAAM,EAAE,YAAY,CAAC,aAAa,IAAI,EAAE;KACzC,CAAC;AACJ,CAAC"}
|
|
@@ -103,6 +103,10 @@ export declare class ContentStore {
|
|
|
103
103
|
readonly basePath: string;
|
|
104
104
|
private indexPath;
|
|
105
105
|
private chunksDir;
|
|
106
|
+
/** In-memory cache for the content index. null = not yet loaded. */
|
|
107
|
+
private cachedIndex;
|
|
108
|
+
/** File size at the time the index cache was last populated. -1 = unknown. */
|
|
109
|
+
private cachedIndexSize;
|
|
106
110
|
constructor(basePath: string);
|
|
107
111
|
/**
|
|
108
112
|
* Ensure storage directories exist.
|
|
@@ -171,7 +175,8 @@ export declare class ContentStore {
|
|
|
171
175
|
*/
|
|
172
176
|
findByHash(hash: string): Promise<StoredContentMeta | null>;
|
|
173
177
|
/**
|
|
174
|
-
* Load the content index.
|
|
178
|
+
* Load the content index with in-memory caching.
|
|
179
|
+
* Uses incremental read when the file has grown since last cache.
|
|
175
180
|
*/
|
|
176
181
|
loadIndex(): Promise<StoredContentMeta[]>;
|
|
177
182
|
/**
|
|
@@ -209,16 +214,14 @@ export declare class ContentStore {
|
|
|
209
214
|
delete(id: string): Promise<boolean>;
|
|
210
215
|
/**
|
|
211
216
|
* Delete embeddings for a content ID.
|
|
217
|
+
* Rebuilds the binary embeddings file without the deleted entries.
|
|
212
218
|
*/
|
|
213
219
|
private deleteEmbeddings;
|
|
214
220
|
/**
|
|
215
221
|
* Save the index file.
|
|
222
|
+
* Called after delete operations — invalidates the incremental cache.
|
|
216
223
|
*/
|
|
217
224
|
private saveIndex;
|
|
218
|
-
/**
|
|
219
|
-
* Get path to embeddings file.
|
|
220
|
-
*/
|
|
221
|
-
private getEmbeddingsPath;
|
|
222
225
|
/**
|
|
223
226
|
* Get embeddings for all chunks of a specific content item.
|
|
224
227
|
* Generates missing embeddings first if needed.
|
|
@@ -226,11 +229,11 @@ export declare class ContentStore {
|
|
|
226
229
|
*/
|
|
227
230
|
getEmbeddingsForContent(contentId: string): Promise<Map<number, number[]> | null>;
|
|
228
231
|
/**
|
|
229
|
-
* Load content embeddings.
|
|
232
|
+
* Load content embeddings using the cached binary-format loader.
|
|
230
233
|
*/
|
|
231
234
|
private loadContentEmbeddings;
|
|
232
235
|
/**
|
|
233
|
-
* Save content embeddings.
|
|
236
|
+
* Save content embeddings using the cached binary-format saver.
|
|
234
237
|
*/
|
|
235
238
|
private saveContentEmbeddings;
|
|
236
239
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"content-store.d.ts","sourceRoot":"","sources":["../../src/lib/content-store.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"content-store.d.ts","sourceRoot":"","sources":["../../src/lib/content-store.ts"],"names":[],"mappings":"AAiBA;;GAEG;AACH,MAAM,MAAM,WAAW,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,MAAM,GAAG,OAAO,GAAG,aAAa,CAAC;AAEvF;;GAEG;AACH,MAAM,MAAM,iBAAiB,GACzB,WAAW,GACX,UAAU,GACV,aAAa,GACb,WAAW,GACX,YAAY,GACZ,aAAa,GACb,gBAAgB,CAAC;AAErB;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,4BAA4B;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,wBAAwB;IACxB,OAAO,EAAE,MAAM,CAAC;IAChB,oCAAoC;IACpC,aAAa,EAAE,MAAM,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,kCAAkC;IAClC,EAAE,EAAE,MAAM,CAAC;IACX,8BAA8B;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,kCAAkC;IAClC,UAAU,EAAE,iBAAiB,CAAC;IAC9B,6CAA6C;IAC7C,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,0CAA0C;IAC1C,YAAY,EAAE,MAAM,CAAC;IACrB,4BAA4B;IAC5B,aAAa,EAAE,MAAM,CAAC;IACtB,4BAA4B;IAC5B,WAAW,EAAE,WAAW,CAAC;IACzB,mCAAmC;IACnC,OAAO,EAAE,MAAM,CAAC;IAChB,uBAAuB;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,qCAAqC;IACrC,WAAW,EAAE,MAAM,CAAC;IACpB,oDAAoD;IACpD,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACnC;AAED;;GAEG;AACH,MAAM,WAAW,aAAc,SAAQ,iBAAiB;IACtD,mCAAmC;IACnC,MAAM,EAAE,YAAY,EAAE,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,kBAAkB;IAClB,UAAU,EAAE,iBAAiB,CAAC;IAC9B,yBAAyB;IACzB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,mDAAmD;IACnD,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,+CAA+C;IAC/C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,0BAA0B;IAC1B,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,6BAA6B;IAC7B,WAAW,CAAC,EAAE,iBAAiB,EAAE,CAAC;IAClC,8BAA8B;IAC9B,YAAY,CAAC,EAAE,WAAW,EAAE,CAAC;IAC7B,sBAAsB;IACtB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,kBAAkB;IAClB,IAAI,CAAC,EAAE,SAAS,GAAG,UAAU,GAAG,QAAQ,CAAC;IACzC,oEAAoE;IACpE,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;CAC1B;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,uBAAuB;IACvB,IAAI,EAAE,iBAAiB,CAAC;IACxB,4BAA4B;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,uBAAuB;IACvB,OAAO,EAAE,MAAM,CAAC;CACjB;AAyBD;;;;;;GAMG;AACH,qBAAa,YAAY;IACvB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,SAAS,CAAS;IAE1B,oEAAoE;IACpE,OAAO,CAAC,WAAW,CAAoC;IACvD,8EAA8E;IAC9E,OAAO,CAAC,eAAe,CAAc;gBAEzB,QAAQ,EAAE,MAAM;IAM5B;;OAEG;YACW,iBAAiB;IAI/B;;OAEG;IACH,YAAY,IAAI,MAAM;IAItB;;OAEG;IACG,eAAe,IAAI,OAAO,CAAC,MAAM,CAAC;IAMxC;;OAEG;IACH,OAAO,CAAC,WAAW;IAInB;;OAEG;IACH,OAAO,CAAC,YAAY;IAsCpB;;;OAGG;IACH,OAAO,CAAC,qBAAqB;IA0F7B;;;;OAIG;IACH,OAAO,CAAC,sBAAsB;IAkG9B;;OAEG;IACH,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,mBAAmB,CAezC;IAEF;;OAEG;IACH,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,cAAc,CACwB;IAE9D;;OAEG;IACH,OAAO,CAAC,eAAe;IAIvB;;;OAGG;IACH,OAAO,CAAC,gBAAgB;IA+FxB;;;OAGG;IACH,OAAO,CAAC,qBAAqB;IAI7B;;;OAGG;IACG,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,iBAAiB,CAAC;IAsE/E;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAoDzB;;OAEG;IACG,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,iBAAiB,GAAG,IAAI,CAAC;IAKjE;;;OAGG;IACG,SAAS,IAAI,OAAO,CAAC,iBAAiB,EAAE,CAAC;IA+E/C;;;;;OAKG;IACG,QAAQ,CAAC,EAAE,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IAkCvE;;OAEG;IACG,OAAO,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,iBAAiB,GAAG,IAAI,CAAC;IAK5D;;;OAGG;IACG,gBAAgB,CACpB,EAAE,EAAE,MAAM,GACT,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,EAAE,GAAG,IAAI,CAAC;IAkCvE;;OAEG;IACG,IAAI,CACR,OAAO,GAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,iBAAiB,EAAE,CAAA;KAAO,GAClE,OAAO,CAAC,iBAAiB,EAAE,CAAC;IAiB/B;;;;OAIG;IACG,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IA8B1C;;;OAGG;YACW,gBAAgB;IAiC9B;;;OAGG;YACW,SAAS;IASvB;;;;OAIG;IACG,uBAAuB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,GAAG,IAAI,CAAC;IAmBvF;;OAEG;YACW,qBAAqB;IAInC;;OAEG;YACW,qBAAqB;IAQnC;;OAEG;IACG,yBAAyB,IAAI,OAAO,CAAC,MAAM,CAAC;IAuClD;;OAEG;IACG,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC;IAmLxF;;OAEG;IACG,QAAQ,IAAI,OAAO,CAAC;QACxB,UAAU,EAAE,MAAM,CAAC;QACnB,SAAS,EAAE,MAAM,CAAC;QAClB,WAAW,EAAE,MAAM,CAAC;QACpB,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACrC,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KACvC,CAAC;CAoBH"}
|
|
@@ -4,7 +4,7 @@ import * as path from 'path';
|
|
|
4
4
|
import { nanoid } from 'nanoid';
|
|
5
5
|
import { generateHeuristicSummary } from './content-detector.js';
|
|
6
6
|
import { estimateTokens } from './context-budget.js';
|
|
7
|
-
import { cosineSimilarity, embeddingsAvailable, getEmbeddingProvider } from './embeddings.js';
|
|
7
|
+
import { cosineSimilarity, embeddingsAvailable, getEmbeddingProvider, loadContentEmbeddingsCached, saveContentEmbeddingsCached, invalidateContentEmbeddingsCache, } from './embeddings.js';
|
|
8
8
|
/** Default chunk size in tokens */
|
|
9
9
|
const DEFAULT_CHUNK_SIZE = 800;
|
|
10
10
|
/** Minimum structural unit size before merging with sibling */
|
|
@@ -24,6 +24,10 @@ export class ContentStore {
|
|
|
24
24
|
basePath;
|
|
25
25
|
indexPath;
|
|
26
26
|
chunksDir;
|
|
27
|
+
/** In-memory cache for the content index. null = not yet loaded. */
|
|
28
|
+
cachedIndex = null;
|
|
29
|
+
/** File size at the time the index cache was last populated. -1 = unknown. */
|
|
30
|
+
cachedIndexSize = -1;
|
|
27
31
|
constructor(basePath) {
|
|
28
32
|
this.basePath = basePath;
|
|
29
33
|
this.indexPath = path.join(basePath, 'index.jsonl');
|
|
@@ -442,6 +446,13 @@ export class ContentStore {
|
|
|
442
446
|
// Append metadata to index
|
|
443
447
|
const indexLine = JSON.stringify(meta) + '\n';
|
|
444
448
|
await fs.appendFile(this.indexPath, indexLine, 'utf-8');
|
|
449
|
+
// Update index cache
|
|
450
|
+
if (this.cachedIndex !== null) {
|
|
451
|
+
this.cachedIndex.push(meta);
|
|
452
|
+
}
|
|
453
|
+
if (this.cachedIndexSize >= 0) {
|
|
454
|
+
this.cachedIndexSize += Buffer.byteLength(indexLine, 'utf-8');
|
|
455
|
+
}
|
|
445
456
|
return meta;
|
|
446
457
|
}
|
|
447
458
|
/**
|
|
@@ -502,22 +513,78 @@ export class ContentStore {
|
|
|
502
513
|
return index.find(meta => meta.contentHash === hash) ?? null;
|
|
503
514
|
}
|
|
504
515
|
/**
|
|
505
|
-
* Load the content index.
|
|
516
|
+
* Load the content index with in-memory caching.
|
|
517
|
+
* Uses incremental read when the file has grown since last cache.
|
|
506
518
|
*/
|
|
507
519
|
async loadIndex() {
|
|
520
|
+
// Get current file size
|
|
521
|
+
let currentSize = -1;
|
|
522
|
+
try {
|
|
523
|
+
const stat = await fs.stat(this.indexPath);
|
|
524
|
+
currentSize = stat.size;
|
|
525
|
+
}
|
|
526
|
+
catch (error) {
|
|
527
|
+
if (error.code === 'ENOENT') {
|
|
528
|
+
this.cachedIndex = [];
|
|
529
|
+
this.cachedIndexSize = 0;
|
|
530
|
+
return [];
|
|
531
|
+
}
|
|
532
|
+
throw error;
|
|
533
|
+
}
|
|
534
|
+
// Cache hit: file unchanged
|
|
535
|
+
if (this.cachedIndex !== null && currentSize === this.cachedIndexSize) {
|
|
536
|
+
return this.cachedIndex;
|
|
537
|
+
}
|
|
538
|
+
// Incremental read: file grew
|
|
539
|
+
if (this.cachedIndex !== null &&
|
|
540
|
+
this.cachedIndexSize >= 0 &&
|
|
541
|
+
currentSize > this.cachedIndexSize) {
|
|
542
|
+
try {
|
|
543
|
+
const bytesToRead = currentSize - this.cachedIndexSize;
|
|
544
|
+
const buf = Buffer.allocUnsafe(bytesToRead);
|
|
545
|
+
const fh = await fs.open(this.indexPath, 'r');
|
|
546
|
+
try {
|
|
547
|
+
await fh.read(buf, 0, bytesToRead, this.cachedIndexSize);
|
|
548
|
+
}
|
|
549
|
+
finally {
|
|
550
|
+
await fh.close();
|
|
551
|
+
}
|
|
552
|
+
const newContent = buf.toString('utf-8');
|
|
553
|
+
const newEntries = newContent
|
|
554
|
+
.split('\n')
|
|
555
|
+
.filter(line => line.trim())
|
|
556
|
+
.map(line => JSON.parse(line));
|
|
557
|
+
this.cachedIndex.push(...newEntries);
|
|
558
|
+
this.cachedIndexSize = currentSize;
|
|
559
|
+
return this.cachedIndex;
|
|
560
|
+
}
|
|
561
|
+
catch {
|
|
562
|
+
// Fall through to full read
|
|
563
|
+
this.cachedIndex = null;
|
|
564
|
+
this.cachedIndexSize = -1;
|
|
565
|
+
}
|
|
566
|
+
}
|
|
567
|
+
// Full read
|
|
508
568
|
try {
|
|
509
569
|
const content = await fs.readFile(this.indexPath, 'utf-8');
|
|
510
570
|
if (!content.trim()) {
|
|
571
|
+
this.cachedIndex = [];
|
|
572
|
+
this.cachedIndexSize = currentSize;
|
|
511
573
|
return [];
|
|
512
574
|
}
|
|
513
|
-
|
|
575
|
+
const entries = content
|
|
514
576
|
.trim()
|
|
515
577
|
.split('\n')
|
|
516
578
|
.filter(line => line.trim())
|
|
517
579
|
.map(line => JSON.parse(line));
|
|
580
|
+
this.cachedIndex = entries;
|
|
581
|
+
this.cachedIndexSize = currentSize;
|
|
582
|
+
return entries;
|
|
518
583
|
}
|
|
519
584
|
catch (error) {
|
|
520
585
|
if (error.code === 'ENOENT') {
|
|
586
|
+
this.cachedIndex = [];
|
|
587
|
+
this.cachedIndexSize = 0;
|
|
521
588
|
return [];
|
|
522
589
|
}
|
|
523
590
|
throw error;
|
|
@@ -648,33 +715,54 @@ export class ContentStore {
|
|
|
648
715
|
}
|
|
649
716
|
/**
|
|
650
717
|
* Delete embeddings for a content ID.
|
|
718
|
+
* Rebuilds the binary embeddings file without the deleted entries.
|
|
651
719
|
*/
|
|
652
720
|
async deleteEmbeddings(contentId) {
|
|
653
|
-
const embeddingsPath = this.getEmbeddingsPath();
|
|
654
721
|
try {
|
|
655
|
-
const
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
722
|
+
const currentMap = await loadContentEmbeddingsCached(this.basePath);
|
|
723
|
+
// Remove all entries for this contentId (key format: "contentId:chunkIndex")
|
|
724
|
+
const prefix = `${contentId}:`;
|
|
725
|
+
let changed = false;
|
|
726
|
+
for (const key of Array.from(currentMap.keys())) {
|
|
727
|
+
if (key.startsWith(prefix)) {
|
|
728
|
+
currentMap.delete(key);
|
|
729
|
+
changed = true;
|
|
730
|
+
}
|
|
731
|
+
}
|
|
732
|
+
if (!changed)
|
|
733
|
+
return;
|
|
734
|
+
// Invalidate cache so we can write a clean state
|
|
735
|
+
invalidateContentEmbeddingsCache(this.basePath);
|
|
736
|
+
// Delete existing bin and json files so saveContentEmbeddingsCached starts clean
|
|
737
|
+
const { getContentEmbeddingsBinPath, getContentEmbeddingsJsonPath } = await import('./embeddings.js');
|
|
738
|
+
try {
|
|
739
|
+
await fs.unlink(getContentEmbeddingsBinPath(this.basePath));
|
|
740
|
+
}
|
|
741
|
+
catch { /* ignore */ }
|
|
742
|
+
try {
|
|
743
|
+
await fs.unlink(getContentEmbeddingsJsonPath(this.basePath));
|
|
744
|
+
}
|
|
745
|
+
catch { /* ignore */ }
|
|
746
|
+
// Write the filtered map back (merges into empty = filtered map)
|
|
747
|
+
if (currentMap.size > 0) {
|
|
748
|
+
await saveContentEmbeddingsCached(this.basePath, currentMap);
|
|
749
|
+
}
|
|
660
750
|
}
|
|
661
751
|
catch {
|
|
662
|
-
// No embeddings
|
|
752
|
+
// No embeddings or error — nothing to delete
|
|
663
753
|
}
|
|
664
754
|
}
|
|
665
755
|
/**
|
|
666
756
|
* Save the index file.
|
|
757
|
+
* Called after delete operations — invalidates the incremental cache.
|
|
667
758
|
*/
|
|
668
759
|
async saveIndex(index) {
|
|
669
760
|
await this.ensureDirectories();
|
|
670
761
|
const content = index.map(m => JSON.stringify(m)).join('\n');
|
|
671
762
|
await fs.writeFile(this.indexPath, content + (content ? '\n' : ''), 'utf-8');
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
*/
|
|
676
|
-
getEmbeddingsPath() {
|
|
677
|
-
return path.join(this.basePath, 'embeddings.json');
|
|
763
|
+
// Invalidate cache: file was fully rewritten (not append-only)
|
|
764
|
+
this.cachedIndex = null;
|
|
765
|
+
this.cachedIndexSize = -1;
|
|
678
766
|
}
|
|
679
767
|
/**
|
|
680
768
|
* Get embeddings for all chunks of a specific content item.
|
|
@@ -697,49 +785,20 @@ export class ContentStore {
|
|
|
697
785
|
return result;
|
|
698
786
|
}
|
|
699
787
|
/**
|
|
700
|
-
* Load content embeddings.
|
|
788
|
+
* Load content embeddings using the cached binary-format loader.
|
|
701
789
|
*/
|
|
702
790
|
async loadContentEmbeddings() {
|
|
703
|
-
|
|
704
|
-
try {
|
|
705
|
-
const content = await fs.readFile(embeddingsPath, 'utf-8');
|
|
706
|
-
const data = JSON.parse(content);
|
|
707
|
-
const map = new Map();
|
|
708
|
-
for (const entry of data.entries) {
|
|
709
|
-
// Key is contentId:chunkIndex
|
|
710
|
-
map.set(`${entry.contentId}:${entry.chunkIndex}`, entry.embedding);
|
|
711
|
-
}
|
|
712
|
-
return map;
|
|
713
|
-
}
|
|
714
|
-
catch (error) {
|
|
715
|
-
if (error.code === 'ENOENT') {
|
|
716
|
-
return new Map();
|
|
717
|
-
}
|
|
718
|
-
throw error;
|
|
719
|
-
}
|
|
791
|
+
return loadContentEmbeddingsCached(this.basePath);
|
|
720
792
|
}
|
|
721
793
|
/**
|
|
722
|
-
* Save content embeddings.
|
|
794
|
+
* Save content embeddings using the cached binary-format saver.
|
|
723
795
|
*/
|
|
724
796
|
async saveContentEmbeddings(entries) {
|
|
725
|
-
const
|
|
726
|
-
const
|
|
727
|
-
|
|
728
|
-
try {
|
|
729
|
-
const content = await fs.readFile(embeddingsPath, 'utf-8');
|
|
730
|
-
existingData = JSON.parse(content);
|
|
797
|
+
const newEntries = new Map();
|
|
798
|
+
for (const entry of entries) {
|
|
799
|
+
newEntries.set(`${entry.contentId}:${entry.chunkIndex}`, entry.embedding);
|
|
731
800
|
}
|
|
732
|
-
|
|
733
|
-
existingData = {
|
|
734
|
-
version: 1,
|
|
735
|
-
model: provider.name,
|
|
736
|
-
dimensions: provider.dimensions,
|
|
737
|
-
entries: [],
|
|
738
|
-
};
|
|
739
|
-
}
|
|
740
|
-
existingData.entries.push(...entries);
|
|
741
|
-
await fs.mkdir(this.basePath, { recursive: true });
|
|
742
|
-
await fs.writeFile(embeddingsPath, JSON.stringify(existingData), 'utf-8');
|
|
801
|
+
await saveContentEmbeddingsCached(this.basePath, newEntries);
|
|
743
802
|
}
|
|
744
803
|
/**
|
|
745
804
|
* Generate embeddings for content that doesn't have them yet.
|