@luckydraw/cumulus 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +148 -0
- package/dist/cli/cumulus.d.ts +3 -0
- package/dist/cli/cumulus.d.ts.map +1 -0
- package/dist/cli/cumulus.js +233 -0
- package/dist/cli/cumulus.js.map +1 -0
- package/dist/index.d.ts +33 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +43 -0
- package/dist/index.js.map +1 -0
- package/dist/lib/config.d.ts +86 -0
- package/dist/lib/config.d.ts.map +1 -0
- package/dist/lib/config.js +241 -0
- package/dist/lib/config.js.map +1 -0
- package/dist/lib/content-detector.d.ts +46 -0
- package/dist/lib/content-detector.d.ts.map +1 -0
- package/dist/lib/content-detector.js +359 -0
- package/dist/lib/content-detector.js.map +1 -0
- package/dist/lib/content-store.d.ts +255 -0
- package/dist/lib/content-store.d.ts.map +1 -0
- package/dist/lib/content-store.js +955 -0
- package/dist/lib/content-store.js.map +1 -0
- package/dist/lib/context-budget.d.ts +83 -0
- package/dist/lib/context-budget.d.ts.map +1 -0
- package/dist/lib/context-budget.js +101 -0
- package/dist/lib/context-budget.js.map +1 -0
- package/dist/lib/embeddings.d.ts +64 -0
- package/dist/lib/embeddings.d.ts.map +1 -0
- package/dist/lib/embeddings.js +176 -0
- package/dist/lib/embeddings.js.map +1 -0
- package/dist/lib/history.d.ts +120 -0
- package/dist/lib/history.d.ts.map +1 -0
- package/dist/lib/history.js +205 -0
- package/dist/lib/history.js.map +1 -0
- package/dist/lib/image-utils.d.ts +41 -0
- package/dist/lib/image-utils.d.ts.map +1 -0
- package/dist/lib/image-utils.js +288 -0
- package/dist/lib/image-utils.js.map +1 -0
- package/dist/lib/migrate.d.ts +35 -0
- package/dist/lib/migrate.d.ts.map +1 -0
- package/dist/lib/migrate.js +196 -0
- package/dist/lib/migrate.js.map +1 -0
- package/dist/lib/retriever.d.ts +56 -0
- package/dist/lib/retriever.d.ts.map +1 -0
- package/dist/lib/retriever.js +644 -0
- package/dist/lib/retriever.js.map +1 -0
- package/dist/lib/revert.d.ts +23 -0
- package/dist/lib/revert.d.ts.map +1 -0
- package/dist/lib/revert.js +75 -0
- package/dist/lib/revert.js.map +1 -0
- package/dist/lib/session.d.ts +65 -0
- package/dist/lib/session.d.ts.map +1 -0
- package/dist/lib/session.js +289 -0
- package/dist/lib/session.js.map +1 -0
- package/dist/lib/snapshots.d.ts +39 -0
- package/dist/lib/snapshots.d.ts.map +1 -0
- package/dist/lib/snapshots.js +99 -0
- package/dist/lib/snapshots.js.map +1 -0
- package/dist/lib/stream-processor.d.ts +149 -0
- package/dist/lib/stream-processor.d.ts.map +1 -0
- package/dist/lib/stream-processor.js +389 -0
- package/dist/lib/stream-processor.js.map +1 -0
- package/dist/lib/summarizer.d.ts +67 -0
- package/dist/lib/summarizer.d.ts.map +1 -0
- package/dist/lib/summarizer.js +213 -0
- package/dist/lib/summarizer.js.map +1 -0
- package/dist/mcp/index.d.ts +3 -0
- package/dist/mcp/index.d.ts.map +1 -0
- package/dist/mcp/index.js +16 -0
- package/dist/mcp/index.js.map +1 -0
- package/dist/mcp/proxy.d.ts +19 -0
- package/dist/mcp/proxy.d.ts.map +1 -0
- package/dist/mcp/proxy.js +120 -0
- package/dist/mcp/proxy.js.map +1 -0
- package/dist/mcp/server.d.ts +6 -0
- package/dist/mcp/server.d.ts.map +1 -0
- package/dist/mcp/server.js +29 -0
- package/dist/mcp/server.js.map +1 -0
- package/dist/mcp/shared-server.d.ts +21 -0
- package/dist/mcp/shared-server.d.ts.map +1 -0
- package/dist/mcp/shared-server.js +210 -0
- package/dist/mcp/shared-server.js.map +1 -0
- package/dist/mcp/tool-handler.d.ts +20 -0
- package/dist/mcp/tool-handler.d.ts.map +1 -0
- package/dist/mcp/tool-handler.js +1405 -0
- package/dist/mcp/tool-handler.js.map +1 -0
- package/dist/tui/components/App.d.ts +11 -0
- package/dist/tui/components/App.d.ts.map +1 -0
- package/dist/tui/components/App.js +607 -0
- package/dist/tui/components/App.js.map +1 -0
- package/dist/tui/components/DebugContextView.d.ts +13 -0
- package/dist/tui/components/DebugContextView.d.ts.map +1 -0
- package/dist/tui/components/DebugContextView.js +78 -0
- package/dist/tui/components/DebugContextView.js.map +1 -0
- package/dist/tui/components/IncludeMenu.d.ts +12 -0
- package/dist/tui/components/IncludeMenu.d.ts.map +1 -0
- package/dist/tui/components/IncludeMenu.js +127 -0
- package/dist/tui/components/IncludeMenu.js.map +1 -0
- package/dist/tui/components/InputArea.d.ts +27 -0
- package/dist/tui/components/InputArea.d.ts.map +1 -0
- package/dist/tui/components/InputArea.js +366 -0
- package/dist/tui/components/InputArea.js.map +1 -0
- package/dist/tui/components/MarkdownText.d.ts +38 -0
- package/dist/tui/components/MarkdownText.d.ts.map +1 -0
- package/dist/tui/components/MarkdownText.js +234 -0
- package/dist/tui/components/MarkdownText.js.map +1 -0
- package/dist/tui/components/MessageBubble.d.ts +11 -0
- package/dist/tui/components/MessageBubble.d.ts.map +1 -0
- package/dist/tui/components/MessageBubble.js +16 -0
- package/dist/tui/components/MessageBubble.js.map +1 -0
- package/dist/tui/components/MessageHistory.d.ts +11 -0
- package/dist/tui/components/MessageHistory.d.ts.map +1 -0
- package/dist/tui/components/MessageHistory.js +12 -0
- package/dist/tui/components/MessageHistory.js.map +1 -0
- package/dist/tui/components/RevertMenu.d.ts +17 -0
- package/dist/tui/components/RevertMenu.d.ts.map +1 -0
- package/dist/tui/components/RevertMenu.js +144 -0
- package/dist/tui/components/RevertMenu.js.map +1 -0
- package/dist/tui/components/StatusBar.d.ts +14 -0
- package/dist/tui/components/StatusBar.d.ts.map +1 -0
- package/dist/tui/components/StatusBar.js +13 -0
- package/dist/tui/components/StatusBar.js.map +1 -0
- package/dist/tui/components/StreamingResponse.d.ts +15 -0
- package/dist/tui/components/StreamingResponse.d.ts.map +1 -0
- package/dist/tui/components/StreamingResponse.js +52 -0
- package/dist/tui/components/StreamingResponse.js.map +1 -0
- package/dist/tui/hooks/useAppState.d.ts +147 -0
- package/dist/tui/hooks/useAppState.d.ts.map +1 -0
- package/dist/tui/hooks/useAppState.js +110 -0
- package/dist/tui/hooks/useAppState.js.map +1 -0
- package/dist/tui/hooks/useClaudeProcess.d.ts +19 -0
- package/dist/tui/hooks/useClaudeProcess.d.ts.map +1 -0
- package/dist/tui/hooks/useClaudeProcess.js +185 -0
- package/dist/tui/hooks/useClaudeProcess.js.map +1 -0
- package/dist/tui/index.d.ts +10 -0
- package/dist/tui/index.d.ts.map +1 -0
- package/dist/tui/index.js +11 -0
- package/dist/tui/index.js.map +1 -0
- package/dist/tui/utils/streamParser.d.ts +31 -0
- package/dist/tui/utils/streamParser.d.ts.map +1 -0
- package/dist/tui/utils/streamParser.js +63 -0
- package/dist/tui/utils/streamParser.js.map +1 -0
- package/package.json +94 -0
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
import { ContentStore } from './content-store.js';
|
|
2
|
+
import { ContextBudget } from './context-budget.js';
|
|
3
|
+
/**
|
|
4
|
+
* Types of stream-json events we care about.
|
|
5
|
+
*/
|
|
6
|
+
export type StreamEventType = 'assistant' | 'user' | 'tool_use' | 'tool_result' | 'system' | 'error' | 'unknown';
|
|
7
|
+
/**
|
|
8
|
+
* Parsed stream-json message structure.
|
|
9
|
+
*/
|
|
10
|
+
export interface StreamMessage {
|
|
11
|
+
type?: string;
|
|
12
|
+
subtype?: string;
|
|
13
|
+
message?: {
|
|
14
|
+
content?: Array<{
|
|
15
|
+
type: string;
|
|
16
|
+
text?: string;
|
|
17
|
+
content?: string;
|
|
18
|
+
tool_use_id?: string;
|
|
19
|
+
name?: string;
|
|
20
|
+
input?: Record<string, unknown>;
|
|
21
|
+
is_error?: boolean;
|
|
22
|
+
}>;
|
|
23
|
+
};
|
|
24
|
+
result?: string;
|
|
25
|
+
content?: string;
|
|
26
|
+
tool_use_id?: string;
|
|
27
|
+
name?: string;
|
|
28
|
+
output?: string;
|
|
29
|
+
tool_use_result?: {
|
|
30
|
+
stdout?: string;
|
|
31
|
+
stderr?: string;
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Result of processing a stream line.
|
|
36
|
+
*/
|
|
37
|
+
export interface ProcessedLine {
|
|
38
|
+
/** Original or modified JSON line */
|
|
39
|
+
line: string;
|
|
40
|
+
/** Whether the line was modified */
|
|
41
|
+
modified: boolean;
|
|
42
|
+
/** If content was externalized, the stored content ID */
|
|
43
|
+
storedContentId?: string;
|
|
44
|
+
/** Event type detected */
|
|
45
|
+
eventType: StreamEventType;
|
|
46
|
+
/** Error message if parsing failed */
|
|
47
|
+
error?: string;
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Configuration for StreamProcessor.
|
|
51
|
+
*/
|
|
52
|
+
export interface StreamProcessorConfig {
|
|
53
|
+
/** Context budget for tracking token usage */
|
|
54
|
+
budget: ContextBudget;
|
|
55
|
+
/** Content store for externalizing large content */
|
|
56
|
+
contentStore: ContentStore;
|
|
57
|
+
/** Callback when content is externalized */
|
|
58
|
+
onExternalize?: (contentId: string, originalSize: number, summary: string) => void;
|
|
59
|
+
/**
|
|
60
|
+
* Minimum tokens for automatic externalization (default: 100).
|
|
61
|
+
* Tool results above this threshold are always externalized regardless of budget.
|
|
62
|
+
* Set to Infinity to disable always-externalize (revert to budget-only behavior).
|
|
63
|
+
*/
|
|
64
|
+
minExternalizeTokens?: number;
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* StreamProcessor intercepts stream-json output and externalizes large content.
|
|
68
|
+
*
|
|
69
|
+
* It tracks:
|
|
70
|
+
* - Current tool invocations (to associate results with tools)
|
|
71
|
+
* - Token budget consumption
|
|
72
|
+
* - Content externalization
|
|
73
|
+
*
|
|
74
|
+
* Usage:
|
|
75
|
+
* ```
|
|
76
|
+
* const processor = new StreamProcessor({
|
|
77
|
+
* budget: new ContextBudget(),
|
|
78
|
+
* contentStore: new ContentStore(basePath),
|
|
79
|
+
* });
|
|
80
|
+
*
|
|
81
|
+
* for (const line of streamLines) {
|
|
82
|
+
* const result = await processor.processLine(line);
|
|
83
|
+
* // Use result.line (original or modified)
|
|
84
|
+
* }
|
|
85
|
+
* ```
|
|
86
|
+
*/
|
|
87
|
+
export declare class StreamProcessor {
|
|
88
|
+
private budget;
|
|
89
|
+
private contentStore;
|
|
90
|
+
private onExternalize?;
|
|
91
|
+
private minExternalizeTokens;
|
|
92
|
+
/** Track active tool calls by tool_use_id */
|
|
93
|
+
private activeTools;
|
|
94
|
+
/** Track the last tool_use event for associating with results */
|
|
95
|
+
private lastToolUse;
|
|
96
|
+
constructor(config: StreamProcessorConfig);
|
|
97
|
+
/**
|
|
98
|
+
* Process a single stream-json line.
|
|
99
|
+
* Returns the original or modified line.
|
|
100
|
+
*/
|
|
101
|
+
processLine(jsonLine: string): Promise<ProcessedLine>;
|
|
102
|
+
/**
|
|
103
|
+
* Detect the type of stream event.
|
|
104
|
+
*/
|
|
105
|
+
private detectEventType;
|
|
106
|
+
/**
|
|
107
|
+
* Track a tool_use event.
|
|
108
|
+
*/
|
|
109
|
+
private handleToolUse;
|
|
110
|
+
/**
|
|
111
|
+
* Process a tool_result event.
|
|
112
|
+
* ALWAYS stores content to ContentStore for RAG indexing.
|
|
113
|
+
* Only replaces inline content with [STORED:xxx] if it exceeds budget.
|
|
114
|
+
*/
|
|
115
|
+
private handleToolResult;
|
|
116
|
+
/**
|
|
117
|
+
* Create the replacement content string for externalized content.
|
|
118
|
+
*/
|
|
119
|
+
private createReplacementContent;
|
|
120
|
+
/**
|
|
121
|
+
* Create a modified message with replacement content.
|
|
122
|
+
*/
|
|
123
|
+
private createModifiedMessage;
|
|
124
|
+
/**
|
|
125
|
+
* Reset processor state for a new turn.
|
|
126
|
+
*/
|
|
127
|
+
reset(): void;
|
|
128
|
+
/**
|
|
129
|
+
* Get current budget stats.
|
|
130
|
+
*/
|
|
131
|
+
getBudgetStats(): {
|
|
132
|
+
consumed: number;
|
|
133
|
+
limit: number;
|
|
134
|
+
remaining: number;
|
|
135
|
+
utilizationPercent: number;
|
|
136
|
+
};
|
|
137
|
+
}
|
|
138
|
+
/**
|
|
139
|
+
* Check if a user input should be externalized.
|
|
140
|
+
*/
|
|
141
|
+
export declare function shouldExternalizeUserInput(content: string, budget: ContextBudget): boolean;
|
|
142
|
+
/**
|
|
143
|
+
* Externalize user input and return replacement message.
|
|
144
|
+
*/
|
|
145
|
+
export declare function externalizeUserInput(content: string, contentStore: ContentStore): Promise<{
|
|
146
|
+
replacement: string;
|
|
147
|
+
contentId: string;
|
|
148
|
+
}>;
|
|
149
|
+
//# sourceMappingURL=stream-processor.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"stream-processor.d.ts","sourceRoot":"","sources":["../../src/lib/stream-processor.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAmC,MAAM,oBAAoB,CAAC;AACnF,OAAO,EAAE,aAAa,EAAkB,MAAM,qBAAqB,CAAC;AAYpE;;GAEG;AACH,MAAM,MAAM,eAAe,GACvB,WAAW,GACX,MAAM,GACN,UAAU,GACV,aAAa,GACb,QAAQ,GACR,OAAO,GACP,SAAS,CAAC;AAEd;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE;QACR,OAAO,CAAC,EAAE,KAAK,CAAC;YACd,IAAI,EAAE,MAAM,CAAC;YACb,IAAI,CAAC,EAAE,MAAM,CAAC;YACd,OAAO,CAAC,EAAE,MAAM,CAAC;YACjB,WAAW,CAAC,EAAE,MAAM,CAAC;YACrB,IAAI,CAAC,EAAE,MAAM,CAAC;YACd,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;YAChC,QAAQ,CAAC,EAAE,OAAO,CAAC;SACpB,CAAC,CAAC;KACJ,CAAC;IACF,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,eAAe,CAAC,EAAE;QAEhB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,MAAM,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,qCAAqC;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,oCAAoC;IACpC,QAAQ,EAAE,OAAO,CAAC;IAClB,yDAAyD;IACzD,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,0BAA0B;IAC1B,SAAS,EAAE,eAAe,CAAC;IAC3B,sCAAsC;IACtC,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAkBD;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,8CAA8C;IAC9C,MAAM,EAAE,aAAa,CAAC;IACtB,oDAAoD;IACpD,YAAY,EAAE,YAAY,CAAC;IAC3B,4CAA4C;IAC5C,aAAa,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;IACnF;;;;OAIG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;CAC/B;AAiCD;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,qBAAa,eAAe;IAC1B,OAAO,CAAC,MAAM,CAAgB;IAC9B,OAAO,CAAC,YAAY,CAAe;IACnC,OAAO,CAAC,aAAa,CAAC,CAAqE;IAC3F,OAAO,CAAC,oBAAoB,CAAS;IAErC,6CAA6C;IAC7C,OAAO,CAAC,WAAW,CAAuC;IAE1D,iEAAiE;IACjE,OAAO,CAAC,WAAW,CAA4B;gBAEnC,MAAM,EAAE,qBAAqB;IAOzC;;;OAGG;IACG,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC;IAwD3D;;OAEG;IACH,OAAO,CAAC,eAAe;IAuBvB;;OAEG;IACH,OAAO,CAAC,aAAa;IAgBrB;;;;OAIG;YACW,gBAAgB;IA0I9B;;OAEG;IACH,OAAO,CAAC,wBAAwB;IAkBhC;;OAEG;IACH,OAAO,CAAC,qBAAqB;IA8B7B;;OAEG;IACH,KAAK,IAAI,IAAI;IAMb;;OAEG;IACH,cAAc,IAAI;QAChB,QAAQ,EAAE,MAAM,CAAC;QACjB,KAAK,EAAE,MAAM,CAAC;QACd,SAAS,EAAE,MAAM,CAAC;QAClB,kBAAkB,EAAE,MAAM,CAAC;KAC5B;CAGF;AAED;;GAEG;AACH,wBAAgB,0BAA0B,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,aAAa,GAAG,OAAO,CAG1F;AAED;;GAEG;AACH,wBAAsB,oBAAoB,CACxC,OAAO,EAAE,MAAM,EACf,YAAY,EAAE,YAAY,GACzB,OAAO,CAAC;IAAE,WAAW,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,CAAC,CA0BrD"}
|
|
@@ -0,0 +1,389 @@
|
|
|
1
|
+
import { analyzeContent, generateHeuristicSummary } from './content-detector.js';
|
|
2
|
+
import { estimateTokens } from './context-budget.js';
|
|
3
|
+
/**
|
|
4
|
+
* Strip system reminder tags from content before indexing.
|
|
5
|
+
* These are injected by Claude and shouldn't pollute RAG search results.
|
|
6
|
+
*/
|
|
7
|
+
function stripSystemReminders(content) {
|
|
8
|
+
// Remove <system-reminder>...</system-reminder> blocks (including multiline)
|
|
9
|
+
// Don't trim - preserve original whitespace in content
|
|
10
|
+
return content.replace(/<system-reminder>[\s\S]*?<\/system-reminder>/gi, '');
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* Minimum token count for always-externalize.
|
|
14
|
+
* Content below this threshold stays inline (error messages, status lines).
|
|
15
|
+
* Content above is always replaced with a [STORED:xxx] reference.
|
|
16
|
+
*/
|
|
17
|
+
const DEFAULT_MIN_EXTERNALIZE_TOKENS = 100;
|
|
18
|
+
/**
|
|
19
|
+
* Map tool names to content source types.
|
|
20
|
+
*/
|
|
21
|
+
function toolToSourceType(toolName) {
|
|
22
|
+
const name = toolName.toLowerCase();
|
|
23
|
+
if (name === 'read' || name.includes('read'))
|
|
24
|
+
return 'file_read';
|
|
25
|
+
if (name === 'bash' || name.includes('bash') || name.includes('shell'))
|
|
26
|
+
return 'bash_output';
|
|
27
|
+
if (name.includes('web') || name.includes('fetch') || name.includes('http'))
|
|
28
|
+
return 'web_fetch';
|
|
29
|
+
return 'tool_result';
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Extract metadata from tool context.
|
|
33
|
+
*/
|
|
34
|
+
function extractToolMetadata(toolName, input) {
|
|
35
|
+
const metadata = { toolName };
|
|
36
|
+
if (input) {
|
|
37
|
+
// Common fields across tools
|
|
38
|
+
if (input.file_path)
|
|
39
|
+
metadata.path = input.file_path;
|
|
40
|
+
if (input.command)
|
|
41
|
+
metadata.command = input.command;
|
|
42
|
+
if (input.url)
|
|
43
|
+
metadata.url = input.url;
|
|
44
|
+
if (input.pattern)
|
|
45
|
+
metadata.pattern = input.pattern;
|
|
46
|
+
}
|
|
47
|
+
return metadata;
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* StreamProcessor intercepts stream-json output and externalizes large content.
|
|
51
|
+
*
|
|
52
|
+
* It tracks:
|
|
53
|
+
* - Current tool invocations (to associate results with tools)
|
|
54
|
+
* - Token budget consumption
|
|
55
|
+
* - Content externalization
|
|
56
|
+
*
|
|
57
|
+
* Usage:
|
|
58
|
+
* ```
|
|
59
|
+
* const processor = new StreamProcessor({
|
|
60
|
+
* budget: new ContextBudget(),
|
|
61
|
+
* contentStore: new ContentStore(basePath),
|
|
62
|
+
* });
|
|
63
|
+
*
|
|
64
|
+
* for (const line of streamLines) {
|
|
65
|
+
* const result = await processor.processLine(line);
|
|
66
|
+
* // Use result.line (original or modified)
|
|
67
|
+
* }
|
|
68
|
+
* ```
|
|
69
|
+
*/
|
|
70
|
+
export class StreamProcessor {
|
|
71
|
+
budget;
|
|
72
|
+
contentStore;
|
|
73
|
+
onExternalize;
|
|
74
|
+
minExternalizeTokens;
|
|
75
|
+
/** Track active tool calls by tool_use_id */
|
|
76
|
+
activeTools = new Map();
|
|
77
|
+
/** Track the last tool_use event for associating with results */
|
|
78
|
+
lastToolUse = null;
|
|
79
|
+
constructor(config) {
|
|
80
|
+
this.budget = config.budget;
|
|
81
|
+
this.contentStore = config.contentStore;
|
|
82
|
+
this.onExternalize = config.onExternalize;
|
|
83
|
+
this.minExternalizeTokens = config.minExternalizeTokens ?? DEFAULT_MIN_EXTERNALIZE_TOKENS;
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* Process a single stream-json line.
|
|
87
|
+
* Returns the original or modified line.
|
|
88
|
+
*/
|
|
89
|
+
async processLine(jsonLine) {
|
|
90
|
+
const trimmed = jsonLine.trim();
|
|
91
|
+
if (!trimmed) {
|
|
92
|
+
return {
|
|
93
|
+
line: jsonLine,
|
|
94
|
+
modified: false,
|
|
95
|
+
eventType: 'unknown',
|
|
96
|
+
};
|
|
97
|
+
}
|
|
98
|
+
let parsed;
|
|
99
|
+
try {
|
|
100
|
+
parsed = JSON.parse(trimmed);
|
|
101
|
+
}
|
|
102
|
+
catch {
|
|
103
|
+
return {
|
|
104
|
+
line: jsonLine,
|
|
105
|
+
modified: false,
|
|
106
|
+
eventType: 'unknown',
|
|
107
|
+
error: 'Failed to parse JSON',
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
const eventType = this.detectEventType(parsed);
|
|
111
|
+
// Track tool_use events
|
|
112
|
+
if (eventType === 'tool_use') {
|
|
113
|
+
this.handleToolUse(parsed);
|
|
114
|
+
return {
|
|
115
|
+
line: jsonLine,
|
|
116
|
+
modified: false,
|
|
117
|
+
eventType,
|
|
118
|
+
};
|
|
119
|
+
}
|
|
120
|
+
// Process tool_result events
|
|
121
|
+
if (eventType === 'tool_result') {
|
|
122
|
+
return this.handleToolResult(parsed, jsonLine);
|
|
123
|
+
}
|
|
124
|
+
// For other events, just track token usage for text content
|
|
125
|
+
if (eventType === 'assistant' && parsed.message?.content) {
|
|
126
|
+
for (const block of parsed.message.content) {
|
|
127
|
+
if (block.type === 'text' && block.text) {
|
|
128
|
+
const tokens = estimateTokens(block.text);
|
|
129
|
+
this.budget.consume(tokens);
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
return {
|
|
134
|
+
line: jsonLine,
|
|
135
|
+
modified: false,
|
|
136
|
+
eventType,
|
|
137
|
+
};
|
|
138
|
+
}
|
|
139
|
+
/**
|
|
140
|
+
* Detect the type of stream event.
|
|
141
|
+
*/
|
|
142
|
+
detectEventType(parsed) {
|
|
143
|
+
// Check for tool events in message content first (takes priority)
|
|
144
|
+
if (parsed.message?.content) {
|
|
145
|
+
for (const block of parsed.message.content) {
|
|
146
|
+
if (block.type === 'tool_use')
|
|
147
|
+
return 'tool_use';
|
|
148
|
+
if (block.type === 'tool_result')
|
|
149
|
+
return 'tool_result';
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
// Check for top-level tool result
|
|
153
|
+
if (parsed.output !== undefined || parsed.result !== undefined) {
|
|
154
|
+
return 'tool_result';
|
|
155
|
+
}
|
|
156
|
+
// Fall back to top-level type
|
|
157
|
+
if (parsed.type === 'assistant')
|
|
158
|
+
return 'assistant';
|
|
159
|
+
if (parsed.type === 'user')
|
|
160
|
+
return 'user';
|
|
161
|
+
if (parsed.type === 'system')
|
|
162
|
+
return 'system';
|
|
163
|
+
if (parsed.type === 'error')
|
|
164
|
+
return 'error';
|
|
165
|
+
return 'unknown';
|
|
166
|
+
}
|
|
167
|
+
/**
|
|
168
|
+
* Track a tool_use event.
|
|
169
|
+
*/
|
|
170
|
+
handleToolUse(parsed) {
|
|
171
|
+
if (parsed.message?.content) {
|
|
172
|
+
for (const block of parsed.message.content) {
|
|
173
|
+
if (block.type === 'tool_use' && block.tool_use_id && block.name) {
|
|
174
|
+
const context = {
|
|
175
|
+
name: block.name,
|
|
176
|
+
toolUseId: block.tool_use_id,
|
|
177
|
+
startTime: Date.now(),
|
|
178
|
+
};
|
|
179
|
+
this.activeTools.set(block.tool_use_id, context);
|
|
180
|
+
this.lastToolUse = context;
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
/**
|
|
186
|
+
* Process a tool_result event.
|
|
187
|
+
* ALWAYS stores content to ContentStore for RAG indexing.
|
|
188
|
+
* Only replaces inline content with [STORED:xxx] if it exceeds budget.
|
|
189
|
+
*/
|
|
190
|
+
async handleToolResult(parsed, originalLine) {
|
|
191
|
+
// Extract content from tool result
|
|
192
|
+
let content;
|
|
193
|
+
let toolUseId;
|
|
194
|
+
// Try different formats
|
|
195
|
+
if (parsed.output) {
|
|
196
|
+
content = parsed.output;
|
|
197
|
+
toolUseId = parsed.tool_use_id;
|
|
198
|
+
}
|
|
199
|
+
else if (parsed.result) {
|
|
200
|
+
content = parsed.result;
|
|
201
|
+
toolUseId = parsed.tool_use_id;
|
|
202
|
+
}
|
|
203
|
+
else if (parsed.message?.content) {
|
|
204
|
+
for (const block of parsed.message.content) {
|
|
205
|
+
if (block.type === 'tool_result') {
|
|
206
|
+
// tool_result blocks use 'content' field, not 'text'
|
|
207
|
+
content = block.content ?? block.text;
|
|
208
|
+
toolUseId = block.tool_use_id;
|
|
209
|
+
break;
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
if (!content) {
|
|
214
|
+
return {
|
|
215
|
+
line: originalLine,
|
|
216
|
+
modified: false,
|
|
217
|
+
eventType: 'tool_result',
|
|
218
|
+
};
|
|
219
|
+
}
|
|
220
|
+
// Ensure content is a string (MCP can return objects)
|
|
221
|
+
const contentStr = typeof content === 'string' ? content : JSON.stringify(content, null, 2);
|
|
222
|
+
// Get tool context
|
|
223
|
+
const toolContext = toolUseId ? this.activeTools.get(toolUseId) : this.lastToolUse;
|
|
224
|
+
const toolName = toolContext?.name ?? 'unknown';
|
|
225
|
+
const tokens = estimateTokens(contentStr);
|
|
226
|
+
// Analyze content for storage
|
|
227
|
+
const analysis = analyzeContent(contentStr);
|
|
228
|
+
const summary = generateHeuristicSummary(contentStr, analysis.contentType);
|
|
229
|
+
const sourceType = toolToSourceType(toolName);
|
|
230
|
+
const storeOptions = {
|
|
231
|
+
sourceType,
|
|
232
|
+
sourceTool: toolName,
|
|
233
|
+
contentType: analysis.contentType,
|
|
234
|
+
summary,
|
|
235
|
+
metadata: extractToolMetadata(toolName),
|
|
236
|
+
};
|
|
237
|
+
// Strip system reminders before storing - they shouldn't pollute RAG results
|
|
238
|
+
const contentForStorage = stripSystemReminders(contentStr);
|
|
239
|
+
// ALWAYS store to ContentStore for RAG indexing (regardless of size)
|
|
240
|
+
// Skip if content is empty/trivial after stripping system reminders
|
|
241
|
+
let stored;
|
|
242
|
+
try {
|
|
243
|
+
if (contentForStorage.trim().length > 10) {
|
|
244
|
+
stored = await this.contentStore.store(contentForStorage, storeOptions);
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
catch (err) {
|
|
248
|
+
// Log storage errors but don't fail the stream processing
|
|
249
|
+
console.error('[StreamProcessor] Failed to store content:', err);
|
|
250
|
+
// Continue without storing - return unmodified line
|
|
251
|
+
this.budget.consume(tokens);
|
|
252
|
+
if (toolUseId) {
|
|
253
|
+
this.activeTools.delete(toolUseId);
|
|
254
|
+
}
|
|
255
|
+
return {
|
|
256
|
+
line: originalLine,
|
|
257
|
+
modified: false,
|
|
258
|
+
eventType: 'tool_result',
|
|
259
|
+
};
|
|
260
|
+
}
|
|
261
|
+
// Decide whether to externalize: always externalize content above threshold,
|
|
262
|
+
// or any content that would exceed the budget (safety net for threshold-exempt content)
|
|
263
|
+
const shouldExternalize = (stored && tokens > this.minExternalizeTokens) || this.budget.wouldExceed(tokens);
|
|
264
|
+
if (!shouldExternalize || !stored) {
|
|
265
|
+
// Small content (error messages, status lines) that fits in budget — keep inline
|
|
266
|
+
this.budget.consume(tokens);
|
|
267
|
+
// Clean up tool context
|
|
268
|
+
if (toolUseId) {
|
|
269
|
+
this.activeTools.delete(toolUseId);
|
|
270
|
+
}
|
|
271
|
+
return {
|
|
272
|
+
line: originalLine,
|
|
273
|
+
modified: false,
|
|
274
|
+
storedContentId: stored?.id,
|
|
275
|
+
eventType: 'tool_result',
|
|
276
|
+
};
|
|
277
|
+
}
|
|
278
|
+
// Externalize: replace inline content with [STORED:xxx] reference
|
|
279
|
+
const replacement = this.createReplacementContent(stored.id, contentStr.length, tokens, summary, toolName);
|
|
280
|
+
// Consume tokens for the replacement (much smaller)
|
|
281
|
+
const replacementTokens = estimateTokens(replacement);
|
|
282
|
+
this.budget.consume(replacementTokens);
|
|
283
|
+
// Notify callback
|
|
284
|
+
if (this.onExternalize) {
|
|
285
|
+
this.onExternalize(stored.id, contentStr.length, summary);
|
|
286
|
+
}
|
|
287
|
+
// Create modified line
|
|
288
|
+
const modifiedParsed = this.createModifiedMessage(parsed, replacement, toolUseId);
|
|
289
|
+
const modifiedLine = JSON.stringify(modifiedParsed);
|
|
290
|
+
// Clean up tool context
|
|
291
|
+
if (toolUseId) {
|
|
292
|
+
this.activeTools.delete(toolUseId);
|
|
293
|
+
}
|
|
294
|
+
return {
|
|
295
|
+
line: modifiedLine,
|
|
296
|
+
modified: true,
|
|
297
|
+
storedContentId: stored.id,
|
|
298
|
+
eventType: 'tool_result',
|
|
299
|
+
};
|
|
300
|
+
}
|
|
301
|
+
/**
|
|
302
|
+
* Create the replacement content string for externalized content.
|
|
303
|
+
*/
|
|
304
|
+
createReplacementContent(contentId, originalSize, tokenEstimate, summary, toolName) {
|
|
305
|
+
const sizeKB = Math.round(originalSize / 1024);
|
|
306
|
+
return `[STORED:${contentId}] ${toolName} output stored (${sizeKB}KB, ~${tokenEstimate} tokens).
|
|
307
|
+
|
|
308
|
+
Summary: ${summary}
|
|
309
|
+
|
|
310
|
+
To access this content:
|
|
311
|
+
- retrieve_content("${contentId}") - get full content
|
|
312
|
+
- search_content("your query") - search within stored content
|
|
313
|
+
- read_content_chunk("${contentId}", 0) - read specific chunk`;
|
|
314
|
+
}
|
|
315
|
+
/**
|
|
316
|
+
* Create a modified message with replacement content.
|
|
317
|
+
*/
|
|
318
|
+
createModifiedMessage(original, replacement, _toolUseId) {
|
|
319
|
+
// Clone the original
|
|
320
|
+
const modified = JSON.parse(JSON.stringify(original));
|
|
321
|
+
// Replace content in various formats
|
|
322
|
+
if (modified.output !== undefined) {
|
|
323
|
+
modified.output = replacement;
|
|
324
|
+
}
|
|
325
|
+
if (modified.result !== undefined) {
|
|
326
|
+
modified.result = replacement;
|
|
327
|
+
}
|
|
328
|
+
if (modified.content !== undefined) {
|
|
329
|
+
modified.content = replacement;
|
|
330
|
+
}
|
|
331
|
+
if (modified.message?.content) {
|
|
332
|
+
for (const block of modified.message.content) {
|
|
333
|
+
if (block.type === 'tool_result') {
|
|
334
|
+
// tool_result blocks use 'content' field
|
|
335
|
+
block.content = replacement;
|
|
336
|
+
}
|
|
337
|
+
}
|
|
338
|
+
}
|
|
339
|
+
return modified;
|
|
340
|
+
}
|
|
341
|
+
/**
|
|
342
|
+
* Reset processor state for a new turn.
|
|
343
|
+
*/
|
|
344
|
+
reset() {
|
|
345
|
+
this.budget.reset();
|
|
346
|
+
this.activeTools.clear();
|
|
347
|
+
this.lastToolUse = null;
|
|
348
|
+
}
|
|
349
|
+
/**
|
|
350
|
+
* Get current budget stats.
|
|
351
|
+
*/
|
|
352
|
+
getBudgetStats() {
|
|
353
|
+
return this.budget.getStats();
|
|
354
|
+
}
|
|
355
|
+
}
|
|
356
|
+
/**
|
|
357
|
+
* Check if a user input should be externalized.
|
|
358
|
+
*/
|
|
359
|
+
export function shouldExternalizeUserInput(content, budget) {
|
|
360
|
+
const tokens = estimateTokens(content);
|
|
361
|
+
return budget.wouldExceed(tokens);
|
|
362
|
+
}
|
|
363
|
+
/**
|
|
364
|
+
* Externalize user input and return replacement message.
|
|
365
|
+
*/
|
|
366
|
+
export async function externalizeUserInput(content, contentStore) {
|
|
367
|
+
const analysis = analyzeContent(content);
|
|
368
|
+
const summary = generateHeuristicSummary(content, analysis.contentType);
|
|
369
|
+
const stored = await contentStore.store(content, {
|
|
370
|
+
sourceType: 'user_input',
|
|
371
|
+
contentType: analysis.contentType,
|
|
372
|
+
summary,
|
|
373
|
+
metadata: {},
|
|
374
|
+
});
|
|
375
|
+
const sizeKB = Math.round(content.length / 1024);
|
|
376
|
+
const tokens = estimateTokens(content);
|
|
377
|
+
const replacement = `[User provided large content - STORED:${stored.id}]
|
|
378
|
+
|
|
379
|
+
The user provided ${sizeKB}KB of content (~${tokens} tokens) which has been stored externally.
|
|
380
|
+
|
|
381
|
+
Summary: ${summary}
|
|
382
|
+
|
|
383
|
+
Use retrieve_content("${stored.id}") to access the full content.`;
|
|
384
|
+
return {
|
|
385
|
+
replacement,
|
|
386
|
+
contentId: stored.id,
|
|
387
|
+
};
|
|
388
|
+
}
|
|
389
|
+
//# sourceMappingURL=stream-processor.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"stream-processor.js","sourceRoot":"","sources":["../../src/lib/stream-processor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,wBAAwB,EAAE,MAAM,uBAAuB,CAAC;AAEjF,OAAO,EAAiB,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAEpE;;;GAGG;AACH,SAAS,oBAAoB,CAAC,OAAe;IAC3C,6EAA6E;IAC7E,uDAAuD;IACvD,OAAO,OAAO,CAAC,OAAO,CAAC,gDAAgD,EAAE,EAAE,CAAC,CAAC;AAC/E,CAAC;AAoED;;;;GAIG;AACH,MAAM,8BAA8B,GAAG,GAAG,CAAC;AAoB3C;;GAEG;AACH,SAAS,gBAAgB,CAAC,QAAgB;IACxC,MAAM,IAAI,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC;IACpC,IAAI,IAAI,KAAK,MAAM,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;QAAE,OAAO,WAAW,CAAC;IACjE,IAAI,IAAI,KAAK,MAAM,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC;QAAE,OAAO,aAAa,CAAC;IAC7F,IAAI,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;QAAE,OAAO,WAAW,CAAC;IAChG,OAAO,aAAa,CAAC;AACvB,CAAC;AAED;;GAEG;AACH,SAAS,mBAAmB,CAC1B,QAAgB,EAChB,KAA+B;IAE/B,MAAM,QAAQ,GAA4B,EAAE,QAAQ,EAAE,CAAC;IAEvD,IAAI,KAAK,EAAE,CAAC;QACV,6BAA6B;QAC7B,IAAI,KAAK,CAAC,SAAS;YAAE,QAAQ,CAAC,IAAI,GAAG,KAAK,CAAC,SAAS,CAAC;QACrD,IAAI,KAAK,CAAC,OAAO;YAAE,QAAQ,CAAC,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC;QACpD,IAAI,KAAK,CAAC,GAAG;YAAE,QAAQ,CAAC,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC;QACxC,IAAI,KAAK,CAAC,OAAO;YAAE,QAAQ,CAAC,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC;IACtD,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,MAAM,OAAO,eAAe;IAClB,MAAM,CAAgB;IACtB,YAAY,CAAe;IAC3B,aAAa,CAAsE;IACnF,oBAAoB,CAAS;IAErC,6CAA6C;IACrC,WAAW,GAA6B,IAAI,GAAG,EAAE,CAAC;IAE1D,iEAAiE;IACzD,WAAW,GAAuB,IAAI,CAAC;IAE/C,YAAY,MAA6B;QACvC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QAC5B,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC,YAAY,CAAC;QACxC,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC,aAAa,CAAC;QAC1C,IAAI,CAAC,oBAAoB,GAAG,MAAM,CAAC,oBAAoB,IAAI,8BAA8B,CAAC;IAC5F,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,WAAW,CAAC,QAAgB;QAChC,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC;QAChC,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,OAAO;gBACL,IAAI,EAAE,QAAQ;gBACd,QAAQ,EAAE,KAAK;gBACf,SAAS,EAAE,SAAS;aACrB,CAAC;QACJ,CAAC;QAED,IAAI,MAAqB,CAAC;QAC1B,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC/B,CAAC;QAAC,MAAM,CAAC;YACP,OAAO;gBACL,IAAI,EAAE,QAAQ;gBACd,QAAQ,EAAE,KAAK;gBACf,SAAS,EAAE,SAAS;gBACpB,KAAK,EAAE,sBAAsB;aAC9B,CAAC;QACJ,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;QAE/C,wBAAwB;QACxB,IAAI,SAAS,KAAK,UAAU,EAAE,CAAC;YAC7B,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;YAC3B,OAAO;gBACL,IAAI,EAAE,QAAQ;gBACd,QAAQ,EAAE,KAAK;gBACf,SAAS;aACV,CAAC;QACJ,CAAC;QAED,6BAA6B;QAC7B,IAAI,SAAS,KAAK,aAAa,EAAE,CAAC;YAChC,OAAO,IAAI,CAAC,gBAAgB,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QACjD,CAAC;QAED,4DAA4D;QAC5D,IAAI,SAAS,KAAK,WAAW,IAAI,MAAM,CAAC,OAAO,EAAE,OAAO,EAAE,CAAC;YACzD,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;gBAC3C,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;oBACxC,MAAM,MAAM,GAAG,cAAc,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;oBAC1C,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;gBAC9B,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO;YACL,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,KAAK;YACf,SAAS;SACV,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,eAAe,CAAC,MAAqB;QAC3C,kEAAkE;QAClE,IAAI,MAAM,CAAC,OAAO,EAAE,OAAO,EAAE,CAAC;YAC5B,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;gBAC3C,IAAI,KAAK,CAAC,IAAI,KAAK,UAAU;oBAAE,OAAO,UAAU,CAAC;gBACjD,IAAI,KAAK,CAAC,IAAI,KAAK,aAAa;oBAAE,OAAO,aAAa,CAAC;YACzD,CAAC;QACH,CAAC;QAED,kCAAkC;QAClC,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YAC/D,OAAO,aAAa,CAAC;QACvB,CAAC;QAED,8BAA8B;QAC9B,IAAI,MAAM,CAAC,IAAI,KAAK,WAAW;YAAE,OAAO,WAAW,CAAC;QACpD,IAAI,MAAM,CAAC,IAAI,KAAK,MAAM;YAAE,OAAO,MAAM,CAAC;QAC1C,IAAI,MAAM,CAAC,IAAI,KAAK,QAAQ;YAAE,OAAO,QAAQ,CAAC;QAC9C,IAAI,MAAM,CAAC,IAAI,KAAK,OAAO;YAAE,OAAO,OAAO,CAAC;QAE5C,OAAO,SAAS,CAAC;IACnB,CAAC;IAED;;OAEG;IACK,aAAa,CAAC,MAAqB;QACzC,IAAI,MAAM,CAAC,OAAO,EAAE,OAAO,EAAE,CAAC;YAC5B,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;gBAC3C,IAAI,KAAK,CAAC,IAAI,KAAK,UAAU,IAAI,KAAK,CAAC,WAAW,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;oBACjE,MAAM,OAAO,GAAgB;wBAC3B,IAAI,EAAE,KAAK,CAAC,IAAI;wBAChB,SAAS,EAAE,KAAK,CAAC,WAAW;wBAC5B,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;qBACtB,CAAC;oBACF,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;oBACjD,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC;gBAC7B,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,gBAAgB,CAC5B,MAAqB,EACrB,YAAoB;QAEpB,mCAAmC;QACnC,IAAI,OAA2B,CAAC;QAChC,IAAI,SAA6B,CAAC;QAElC,wBAAwB;QACxB,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;YAClB,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC;YACxB,SAAS,GAAG,MAAM,CAAC,WAAW,CAAC;QACjC,CAAC;aAAM,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;YACzB,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC;YACxB,SAAS,GAAG,MAAM,CAAC,WAAW,CAAC;QACjC,CAAC;aAAM,IAAI,MAAM,CAAC,OAAO,EAAE,OAAO,EAAE,CAAC;YACnC,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;gBAC3C,IAAI,KAAK,CAAC,IAAI,KAAK,aAAa,EAAE,CAAC;oBACjC,qDAAqD;oBACrD,OAAO,GAAG,KAAK,CAAC,OAAO,IAAI,KAAK,CAAC,IAAI,CAAC;oBACtC,SAAS,GAAG,KAAK,CAAC,WAAW,CAAC;oBAC9B,MAAM;gBACR,CAAC;YACH,CAAC;QACH,CAAC;QAED,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,OAAO;gBACL,IAAI,EAAE,YAAY;gBAClB,QAAQ,EAAE,KAAK;gBACf,SAAS,EAAE,aAAa;aACzB,CAAC;QACJ,CAAC;QAED,sDAAsD;QACtD,MAAM,UAAU,GAAG,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;QAE5F,mBAAmB;QACnB,MAAM,WAAW,GAAG,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC;QAEnF,MAAM,QAAQ,GAAG,WAAW,EAAE,IAAI,IAAI,SAAS,CAAC;QAChD,MAAM,MAAM,GAAG,cAAc,CAAC,UAAU,CAAC,CAAC;QAE1C,8BAA8B;QAC9B,MAAM,QAAQ,GAAG,cAAc,CAAC,UAAU,CAAC,CAAC;QAC5C,MAAM,OAAO,GAAG,wBAAwB,CAAC,UAAU,EAAE,QAAQ,CAAC,WAAW,CAAC,CAAC;QAC3E,MAAM,UAAU,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;QAE9C,MAAM,YAAY,GAAiB;YACjC,UAAU;YACV,UAAU,EAAE,QAAQ;YACpB,WAAW,EAAE,QAAQ,CAAC,WAAW;YACjC,OAAO;YACP,QAAQ,EAAE,mBAAmB,CAAC,QAAQ,CAAC;SACxC,CAAC;QAEF,6EAA6E;QAC7E,MAAM,iBAAiB,GAAG,oBAAoB,CAAC,UAAU,CAAC,CAAC;QAE3D,qEAAqE;QACrE,oEAAoE;QACpE,IAAI,MAAM,CAAC;QACX,IAAI,CAAC;YACH,IAAI,iBAAiB,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;gBACzC,MAAM,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,iBAAiB,EAAE,YAAY,CAAC,CAAC;YAC1E,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,0DAA0D;YAC1D,OAAO,CAAC,KAAK,CAAC,4CAA4C,EAAE,GAAG,CAAC,CAAC;YACjE,oDAAoD;YACpD,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;YAC5B,IAAI,SAAS,EAAE,CAAC;gBACd,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YACrC,CAAC;YACD,OAAO;gBACL,IAAI,EAAE,YAAY;gBAClB,QAAQ,EAAE,KAAK;gBACf,SAAS,EAAE,aAAa;aACzB,CAAC;QACJ,CAAC;QAED,6EAA6E;QAC7E,wFAAwF;QACxF,MAAM,iBAAiB,GACrB,CAAC,MAAM,IAAI,MAAM,GAAG,IAAI,CAAC,oBAAoB,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;QAEpF,IAAI,CAAC,iBAAiB,IAAI,CAAC,MAAM,EAAE,CAAC;YAClC,iFAAiF;YACjF,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;YAE5B,wBAAwB;YACxB,IAAI,SAAS,EAAE,CAAC;gBACd,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YACrC,CAAC;YAED,OAAO;gBACL,IAAI,EAAE,YAAY;gBAClB,QAAQ,EAAE,KAAK;gBACf,eAAe,EAAE,MAAM,EAAE,EAAE;gBAC3B,SAAS,EAAE,aAAa;aACzB,CAAC;QACJ,CAAC;QAED,kEAAkE;QAClE,MAAM,WAAW,GAAG,IAAI,CAAC,wBAAwB,CAC/C,MAAM,CAAC,EAAE,EACT,UAAU,CAAC,MAAM,EACjB,MAAM,EACN,OAAO,EACP,QAAQ,CACT,CAAC;QAEF,oDAAoD;QACpD,MAAM,iBAAiB,GAAG,cAAc,CAAC,WAAW,CAAC,CAAC;QACtD,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC;QAEvC,kBAAkB;QAClB,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACvB,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE,EAAE,UAAU,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAC5D,CAAC;QAED,uBAAuB;QACvB,MAAM,cAAc,GAAG,IAAI,CAAC,qBAAqB,CAAC,MAAM,EAAE,WAAW,EAAE,SAAS,CAAC,CAAC;QAClF,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;QAEpD,wBAAwB;QACxB,IAAI,SAAS,EAAE,CAAC;YACd,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QACrC,CAAC;QAED,OAAO;YACL,IAAI,EAAE,YAAY;YAClB,QAAQ,EAAE,IAAI;YACd,eAAe,EAAE,MAAM,CAAC,EAAE;YAC1B,SAAS,EAAE,aAAa;SACzB,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,wBAAwB,CAC9B,SAAiB,EACjB,YAAoB,EACpB,aAAqB,EACrB,OAAe,EACf,QAAgB;QAEhB,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,GAAG,IAAI,CAAC,CAAC;QAC/C,OAAO,WAAW,SAAS,KAAK,QAAQ,mBAAmB,MAAM,QAAQ,aAAa;;WAE/E,OAAO;;;sBAGI,SAAS;;wBAEP,SAAS,6BAA6B,CAAC;IAC7D,CAAC;IAED;;OAEG;IACK,qBAAqB,CAC3B,QAAuB,EACvB,WAAmB,EACnB,UAAmB;QAEnB,qBAAqB;QACrB,MAAM,QAAQ,GAAkB,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC;QAErE,qCAAqC;QACrC,IAAI,QAAQ,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YAClC,QAAQ,CAAC,MAAM,GAAG,WAAW,CAAC;QAChC,CAAC;QACD,IAAI,QAAQ,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YAClC,QAAQ,CAAC,MAAM,GAAG,WAAW,CAAC;QAChC,CAAC;QACD,IAAI,QAAQ,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;YACnC,QAAQ,CAAC,OAAO,GAAG,WAAW,CAAC;QACjC,CAAC;QACD,IAAI,QAAQ,CAAC,OAAO,EAAE,OAAO,EAAE,CAAC;YAC9B,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;gBAC7C,IAAI,KAAK,CAAC,IAAI,KAAK,aAAa,EAAE,CAAC;oBACjC,yCAAyC;oBACzC,KAAK,CAAC,OAAO,GAAG,WAAW,CAAC;gBAC9B,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED;;OAEG;IACH,KAAK;QACH,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QACpB,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;IAC1B,CAAC;IAED;;OAEG;IACH,cAAc;QAMZ,OAAO,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;IAChC,CAAC;CACF;AAED;;GAEG;AACH,MAAM,UAAU,0BAA0B,CAAC,OAAe,EAAE,MAAqB;IAC/E,MAAM,MAAM,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IACvC,OAAO,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;AACpC,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,OAAe,EACf,YAA0B;IAE1B,MAAM,QAAQ,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IACzC,MAAM,OAAO,GAAG,wBAAwB,CAAC,OAAO,EAAE,QAAQ,CAAC,WAAW,CAAC,CAAC;IAExE,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,KAAK,CAAC,OAAO,EAAE;QAC/C,UAAU,EAAE,YAAY;QACxB,WAAW,EAAE,QAAQ,CAAC,WAAW;QACjC,OAAO;QACP,QAAQ,EAAE,EAAE;KACb,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IACjD,MAAM,MAAM,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IAEvC,MAAM,WAAW,GAAG,yCAAyC,MAAM,CAAC,EAAE;;oBAEpD,MAAM,mBAAmB,MAAM;;WAExC,OAAO;;wBAEM,MAAM,CAAC,EAAE,gCAAgC,CAAC;IAEhE,OAAO;QACL,WAAW;QACX,SAAS,EAAE,MAAM,CAAC,EAAE;KACrB,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import { Message } from './history.js';
|
|
2
|
+
/**
|
|
3
|
+
* A summary of a chunk of conversation messages.
|
|
4
|
+
*/
|
|
5
|
+
export interface Summary {
|
|
6
|
+
id: string;
|
|
7
|
+
fromIndex: number;
|
|
8
|
+
toIndex: number;
|
|
9
|
+
generatedAt: number;
|
|
10
|
+
content: string;
|
|
11
|
+
topics: string[];
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* The full summary file structure for a thread.
|
|
15
|
+
*/
|
|
16
|
+
export interface SummaryFile {
|
|
17
|
+
threadId: string;
|
|
18
|
+
lastMessageIndex: number;
|
|
19
|
+
summaries: Summary[];
|
|
20
|
+
sessionSummary?: string;
|
|
21
|
+
updatedAt: number;
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Result from get_summary tool.
|
|
25
|
+
*/
|
|
26
|
+
export interface GetSummaryResult {
|
|
27
|
+
summary: string;
|
|
28
|
+
coversMessages: {
|
|
29
|
+
from: number;
|
|
30
|
+
to: number;
|
|
31
|
+
};
|
|
32
|
+
topics: string[];
|
|
33
|
+
generatedAt?: number;
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Get the summary file path for a thread.
|
|
37
|
+
*/
|
|
38
|
+
export declare function getSummaryPath(threadPath: string): string;
|
|
39
|
+
/**
|
|
40
|
+
* Load existing summaries for a thread.
|
|
41
|
+
*/
|
|
42
|
+
export declare function loadSummaries(threadPath: string): Promise<SummaryFile | null>;
|
|
43
|
+
/**
|
|
44
|
+
* Save summaries to file.
|
|
45
|
+
*/
|
|
46
|
+
export declare function saveSummaries(summaryFile: SummaryFile, threadPath: string): Promise<void>;
|
|
47
|
+
/**
|
|
48
|
+
* Check if summarization should be triggered.
|
|
49
|
+
*/
|
|
50
|
+
export declare function shouldSummarize(lastSummaryIndex: number, currentIndex: number, estimatedTokens?: number): boolean;
|
|
51
|
+
/**
|
|
52
|
+
* Generate a summary for a chunk of messages using sub_query.
|
|
53
|
+
*/
|
|
54
|
+
export declare function generateChunkSummary(messages: Message[], fromIndex: number, toIndex: number, subQueryFn: (prompt: string, context: string) => Promise<string>): Promise<Summary>;
|
|
55
|
+
/**
|
|
56
|
+
* Generate a high-level session summary from multiple chunk summaries.
|
|
57
|
+
*/
|
|
58
|
+
export declare function generateSessionSummary(chunks: Summary[], subQueryFn: (prompt: string, context: string) => Promise<string>): Promise<string>;
|
|
59
|
+
/**
|
|
60
|
+
* Run summarization for a thread if needed.
|
|
61
|
+
*/
|
|
62
|
+
export declare function runSummarization(threadPath: string, messages: Message[], subQueryFn: (prompt: string, context: string) => Promise<string>): Promise<void>;
|
|
63
|
+
/**
|
|
64
|
+
* Get summary based on scope.
|
|
65
|
+
*/
|
|
66
|
+
export declare function getSummary(threadPath: string, scope?: 'recent' | 'full' | 'range', startIndex?: number, endIndex?: number): Promise<GetSummaryResult | null>;
|
|
67
|
+
//# sourceMappingURL=summarizer.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"summarizer.d.ts","sourceRoot":"","sources":["../../src/lib/summarizer.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAOvC;;GAEG;AACH,MAAM,WAAW,OAAO;IACtB,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,gBAAgB,EAAE,MAAM,CAAC;IACzB,SAAS,EAAE,OAAO,EAAE,CAAC;IACrB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,MAAM,CAAA;KAAE,CAAC;IAC7C,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,CAEzD;AAED;;GAEG;AACH,wBAAsB,aAAa,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,CAanF;AAED;;GAEG;AACH,wBAAsB,aAAa,CAAC,WAAW,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAG/F;AAED;;GAEG;AACH,wBAAgB,eAAe,CAC7B,gBAAgB,EAAE,MAAM,EACxB,YAAY,EAAE,MAAM,EACpB,eAAe,GAAE,MAAU,GAC1B,OAAO,CAGT;AAuCD;;GAEG;AACH,wBAAsB,oBAAoB,CACxC,QAAQ,EAAE,OAAO,EAAE,EACnB,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,MAAM,EACf,UAAU,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,GAC/D,OAAO,CAAC,OAAO,CAAC,CAuBlB;AAED;;GAEG;AACH,wBAAsB,sBAAsB,CAC1C,MAAM,EAAE,OAAO,EAAE,EACjB,UAAU,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,GAC/D,OAAO,CAAC,MAAM,CAAC,CAcjB;AAED;;GAEG;AACH,wBAAsB,gBAAgB,CACpC,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,OAAO,EAAE,EACnB,UAAU,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,GAC/D,OAAO,CAAC,IAAI,CAAC,CA0Cf;AAED;;GAEG;AACH,wBAAsB,UAAU,CAC9B,UAAU,EAAE,MAAM,EAClB,KAAK,GAAE,QAAQ,GAAG,MAAM,GAAG,OAAkB,EAC7C,UAAU,CAAC,EAAE,MAAM,EACnB,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC,CAqElC"}
|