@osmosis-ai/core 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/distill/llm.d.ts +42 -0
- package/dist/distill/llm.js +145 -0
- package/dist/distill/llm.js.map +1 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
- package/src/distill/llm.ts +187 -0
- package/src/index.ts +2 -0
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* LLM-powered distillation — turns raw tool traces into actionable knowledge atoms.
|
|
3
|
+
*
|
|
4
|
+
* Groups related traces (same tool, similar errors) and asks an LLM to extract
|
|
5
|
+
* patterns, best practices, and failure workarounds.
|
|
6
|
+
*/
|
|
7
|
+
import type { CreateToolAtom, CreateNegativeAtom } from '../types/index.js';
|
|
8
|
+
import type { ToolTrace } from './index.js';
|
|
9
|
+
export interface LLMDistillConfig {
|
|
10
|
+
/** API endpoint (OpenAI-compatible) */
|
|
11
|
+
apiUrl: string;
|
|
12
|
+
/** API key */
|
|
13
|
+
apiKey: string;
|
|
14
|
+
/** Model to use (default: gpt-4o-mini for cost efficiency) */
|
|
15
|
+
model: string;
|
|
16
|
+
/** Minimum traces before distillation (default: 5) */
|
|
17
|
+
minTraces: number;
|
|
18
|
+
/** Max tokens for response */
|
|
19
|
+
maxTokens: number;
|
|
20
|
+
}
|
|
21
|
+
export declare const DEFAULT_LLM_CONFIG: LLMDistillConfig;
|
|
22
|
+
interface DistilledAtom {
|
|
23
|
+
kind: 'tool' | 'negative';
|
|
24
|
+
tool_name: string;
|
|
25
|
+
observation: string;
|
|
26
|
+
confidence: number;
|
|
27
|
+
error_signature: string | null;
|
|
28
|
+
anti_pattern: string | null;
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Group traces by tool name for batched distillation.
|
|
32
|
+
*/
|
|
33
|
+
export declare function groupTraces(traces: ToolTrace[]): Map<string, ToolTrace[]>;
|
|
34
|
+
/**
|
|
35
|
+
* Call an OpenAI-compatible API to distill traces.
|
|
36
|
+
*/
|
|
37
|
+
export declare function llmDistill(traces: ToolTrace[], config: LLMDistillConfig): Promise<DistilledAtom[]>;
|
|
38
|
+
/**
|
|
39
|
+
* Convert distilled atoms to store-ready format.
|
|
40
|
+
*/
|
|
41
|
+
export declare function toCreateAtoms(distilled: DistilledAtom[], agentHash?: string): Array<CreateToolAtom | CreateNegativeAtom>;
|
|
42
|
+
export {};
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* LLM-powered distillation — turns raw tool traces into actionable knowledge atoms.
|
|
3
|
+
*
|
|
4
|
+
* Groups related traces (same tool, similar errors) and asks an LLM to extract
|
|
5
|
+
* patterns, best practices, and failure workarounds.
|
|
6
|
+
*/
|
|
7
|
+
import { createHash } from 'node:crypto';
|
|
8
|
+
export const DEFAULT_LLM_CONFIG = {
|
|
9
|
+
apiUrl: 'https://api.openai.com/v1/chat/completions',
|
|
10
|
+
apiKey: '',
|
|
11
|
+
model: 'gpt-4o-mini',
|
|
12
|
+
minTraces: 5,
|
|
13
|
+
maxTokens: 1024,
|
|
14
|
+
};
|
|
15
|
+
const DISTILL_PROMPT = `You are a knowledge distillation engine for AI agents. You analyze tool call traces and extract reusable operational knowledge.
|
|
16
|
+
|
|
17
|
+
Given a batch of tool call traces (tool name, parameters, outcomes, errors), extract knowledge atoms:
|
|
18
|
+
|
|
19
|
+
Rules:
|
|
20
|
+
- Each atom must be a single, actionable insight an agent can use
|
|
21
|
+
- Focus on: failure patterns, parameter best practices, timing tips, error workarounds
|
|
22
|
+
- Skip trivial observations ("tool X works" is not useful)
|
|
23
|
+
- Be specific and concise (1-2 sentences max per observation)
|
|
24
|
+
- Include the error signature when relevant
|
|
25
|
+
|
|
26
|
+
Output JSON array of atoms:
|
|
27
|
+
[
|
|
28
|
+
{
|
|
29
|
+
"kind": "tool" | "negative",
|
|
30
|
+
"tool_name": "string",
|
|
31
|
+
"observation": "string (the insight)",
|
|
32
|
+
"confidence": 0.0-1.0,
|
|
33
|
+
"error_signature": "string | null",
|
|
34
|
+
"anti_pattern": "string | null (for negative atoms)"
|
|
35
|
+
}
|
|
36
|
+
]
|
|
37
|
+
|
|
38
|
+
If no meaningful patterns exist, return [].`;
|
|
39
|
+
/**
|
|
40
|
+
* Group traces by tool name for batched distillation.
|
|
41
|
+
*/
|
|
42
|
+
export function groupTraces(traces) {
|
|
43
|
+
const groups = new Map();
|
|
44
|
+
for (const trace of traces) {
|
|
45
|
+
const key = trace.toolName;
|
|
46
|
+
const existing = groups.get(key) ?? [];
|
|
47
|
+
existing.push(trace);
|
|
48
|
+
groups.set(key, existing);
|
|
49
|
+
}
|
|
50
|
+
return groups;
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Summarize traces for the LLM prompt (strip large payloads).
|
|
54
|
+
*/
|
|
55
|
+
function summarizeTraces(traces) {
|
|
56
|
+
return traces.map((t, i) => {
|
|
57
|
+
const params = Object.keys(t.params).join(', ');
|
|
58
|
+
const resultSnippet = t.result
|
|
59
|
+
? String(t.result).slice(0, 150).replace(/\n/g, ' ')
|
|
60
|
+
: 'null';
|
|
61
|
+
const errorSnippet = t.error ? t.error.slice(0, 150) : 'none';
|
|
62
|
+
return `[${i + 1}] ${t.toolName}(${params}) → ${t.outcome} | error: ${errorSnippet} | result: ${resultSnippet} | ${t.latencyMs}ms`;
|
|
63
|
+
}).join('\n');
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* Call an OpenAI-compatible API to distill traces.
|
|
67
|
+
*/
|
|
68
|
+
export async function llmDistill(traces, config) {
|
|
69
|
+
if (traces.length < config.minTraces)
|
|
70
|
+
return [];
|
|
71
|
+
if (!config.apiKey)
|
|
72
|
+
return [];
|
|
73
|
+
const summary = summarizeTraces(traces);
|
|
74
|
+
try {
|
|
75
|
+
const res = await fetch(config.apiUrl, {
|
|
76
|
+
method: 'POST',
|
|
77
|
+
headers: {
|
|
78
|
+
'Content-Type': 'application/json',
|
|
79
|
+
'Authorization': `Bearer ${config.apiKey}`,
|
|
80
|
+
},
|
|
81
|
+
body: JSON.stringify({
|
|
82
|
+
model: config.model,
|
|
83
|
+
messages: [
|
|
84
|
+
{ role: 'system', content: DISTILL_PROMPT },
|
|
85
|
+
{ role: 'user', content: `Analyze these ${traces.length} tool call traces:\n\n${summary}` },
|
|
86
|
+
],
|
|
87
|
+
max_tokens: config.maxTokens,
|
|
88
|
+
temperature: 0.3,
|
|
89
|
+
response_format: { type: 'json_object' },
|
|
90
|
+
}),
|
|
91
|
+
});
|
|
92
|
+
if (!res.ok) {
|
|
93
|
+
console.error(`LLM distill failed: ${res.status} ${await res.text()}`);
|
|
94
|
+
return [];
|
|
95
|
+
}
|
|
96
|
+
const body = await res.json();
|
|
97
|
+
const content = body.choices?.[0]?.message?.content;
|
|
98
|
+
if (!content)
|
|
99
|
+
return [];
|
|
100
|
+
const parsed = JSON.parse(content);
|
|
101
|
+
const atoms = Array.isArray(parsed) ? parsed : parsed.atoms ?? [];
|
|
102
|
+
return atoms.filter((a) => a.observation && typeof a.observation === 'string' && a.observation.length > 10);
|
|
103
|
+
}
|
|
104
|
+
catch (err) {
|
|
105
|
+
console.error(`LLM distill error: ${err}`);
|
|
106
|
+
return [];
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
/**
|
|
110
|
+
* Convert distilled atoms to store-ready format.
|
|
111
|
+
*/
|
|
112
|
+
export function toCreateAtoms(distilled, agentHash = 'distiller') {
|
|
113
|
+
return distilled.map(d => {
|
|
114
|
+
const base = {
|
|
115
|
+
observation: d.observation,
|
|
116
|
+
context: JSON.stringify({ source: 'llm-distillation', tool: d.tool_name }),
|
|
117
|
+
confidence: d.confidence,
|
|
118
|
+
fitness_score: d.confidence * 0.9, // Slightly lower than confidence
|
|
119
|
+
trust_tier: 'local',
|
|
120
|
+
source_agent_hash: agentHash,
|
|
121
|
+
decay_rate: 0.99,
|
|
122
|
+
};
|
|
123
|
+
if (d.kind === 'negative' && d.anti_pattern) {
|
|
124
|
+
return {
|
|
125
|
+
...base,
|
|
126
|
+
type: 'negative',
|
|
127
|
+
anti_pattern: d.anti_pattern,
|
|
128
|
+
failure_cluster_size: 1,
|
|
129
|
+
error_type: d.error_signature ?? 'unknown',
|
|
130
|
+
severity: d.confidence > 0.8 ? 'high' : 'medium',
|
|
131
|
+
};
|
|
132
|
+
}
|
|
133
|
+
return {
|
|
134
|
+
...base,
|
|
135
|
+
type: 'tool',
|
|
136
|
+
tool_name: d.tool_name,
|
|
137
|
+
params_hash: createHash('sha256').update(d.observation).digest('hex').slice(0, 16),
|
|
138
|
+
outcome: d.error_signature ? 'failure' : 'success',
|
|
139
|
+
error_signature: d.error_signature,
|
|
140
|
+
latency_ms: null,
|
|
141
|
+
reliability_score: d.error_signature ? 0.3 : 0.8,
|
|
142
|
+
};
|
|
143
|
+
});
|
|
144
|
+
}
|
|
145
|
+
//# sourceMappingURL=llm.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"llm.js","sourceRoot":"","sources":["../../src/distill/llm.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAIH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAezC,MAAM,CAAC,MAAM,kBAAkB,GAAqB;IAClD,MAAM,EAAE,4CAA4C;IACpD,MAAM,EAAE,EAAE;IACV,KAAK,EAAE,aAAa;IACpB,SAAS,EAAE,CAAC;IACZ,SAAS,EAAE,IAAI;CAChB,CAAC;AAEF,MAAM,cAAc,GAAG;;;;;;;;;;;;;;;;;;;;;;;4CAuBqB,CAAC;AAW7C;;GAEG;AACH,MAAM,UAAU,WAAW,CAAC,MAAmB;IAC7C,MAAM,MAAM,GAAG,IAAI,GAAG,EAAuB,CAAC;IAC9C,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,GAAG,GAAG,KAAK,CAAC,QAAQ,CAAC;QAC3B,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QACvC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACrB,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAC5B,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;GAEG;AACH,SAAS,eAAe,CAAC,MAAmB;IAC1C,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACzB,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAChD,MAAM,aAAa,GAAG,CAAC,CAAC,MAAM;YAC5B,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC;YACpD,CAAC,CAAC,MAAM,CAAC;QACX,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QAC9D,OAAO,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,QAAQ,IAAI,MAAM,OAAO,CAAC,CAAC,OAAO,aAAa,YAAY,cAAc,aAAa,MAAM,CAAC,CAAC,SAAS,IAAI,CAAC;IACrI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,MAAmB,EACnB,MAAwB;IAExB,IAAI,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,SAAS;QAAE,OAAO,EAAE,CAAC;IAChD,IAAI,CAAC,MAAM,CAAC,MAAM;QAAE,OAAO,EAAE,CAAC;IAE9B,MAAM,OAAO,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC;IAExC,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE;YACrC,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,cAAc,EAAE,kBAAkB;gBAClC,eAAe,EAAE,UAAU,MAAM,CAAC,MAAM,EAAE;aAC3C;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gBACnB,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,QAAQ,EAAE;oBACR,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,cAAc,EAAE;oBAC3C,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,iBAAiB,MAAM,CAAC,MAAM,yBAAyB,OAAO,EAAE,EAAE;iBAC5F;gBACD,UAAU,EAAE,MAAM,CAAC,SAAS;gBAC5B,WAAW,EAAE,GAAG;gBAChB,eAAe,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE;aACzC,CAAC;SACH,CAAC,CAAC;QAEH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,OAAO,CAAC,KAAK,CAAC,uBAAuB,GAAG,CAAC,MAAM,IAAI,MAAM,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YACvE,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAS,CAAC;QACrC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC;QACpD,IAAI,CAAC,OAAO;YAAE,OAAO,EAAE,CAAC;QAExB,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACnC,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC;QAElE,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,CAAM,EAAE,EAAE,CAC7B,CAAC,CAAC,WAAW,IAAI,OAAO,CAAC,CAAC,WAAW,KAAK,QAAQ,IAAI,CAAC,CAAC,WAAW,CAAC,MAAM,GAAG,EAAE,CAChF,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO,CAAC,KAAK,CAAC,sBAAsB,GAAG,EAAE,CAAC,CAAC;QAC3C,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,aAAa,CAC3B,SAA0B,EAC1B,YAAoB,WAAW;IAE/B,OAAO,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE;QACvB,MAAM,IAAI,GAAG;YACX,WAAW,EAAE,CAAC,CAAC,WAAW;YAC1B,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC,CAAC,SAAS,EAAE,CAAC;YAC1E,UAAU,EAAE,CAAC,CAAC,UAAU;YACxB,aAAa,EAAE,CAAC,CAAC,UAAU,GAAG,GAAG,EAAE,iCAAiC;YACpE,UAAU,EAAE,OAAgB;YAC5B,iBAAiB,EAAE,SAAS;YAC5B,UAAU,EAAE,IAAI;SACjB,CAAC;QAEF,IAAI,CAAC,CAAC,IAAI,KAAK,UAAU,IAAI,CAAC,CAAC,YAAY,EAAE,CAAC;YAC5C,OAAO;gBACL,GAAG,IAAI;gBACP,IAAI,EAAE,UAAmB;gBACzB,YAAY,EAAE,CAAC,CAAC,YAAY;gBAC5B,oBAAoB,EAAE,CAAC;gBACvB,UAAU,EAAE,CAAC,CAAC,eAAe,IAAI,SAAS;gBAC1C,QAAQ,EAAE,CAAC,CAAC,UAAU,GAAG,GAAG,CAAC,CAAC,CAAC,MAAe,CAAC,CAAC,CAAC,QAAiB;aACnE,CAAC;QACJ,CAAC;QAED,OAAO;YACL,GAAG,IAAI;YACP,IAAI,EAAE,MAAe;YACrB,SAAS,EAAE,CAAC,CAAC,SAAS;YACtB,WAAW,EAAE,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;YAClF,OAAO,EAAE,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,SAAkB,CAAC,CAAC,CAAC,SAAkB;YACpE,eAAe,EAAE,CAAC,CAAC,eAAe;YAClC,UAAU,EAAE,IAAI;YAChB,iBAAiB,EAAE,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG;SACjD,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC"}
|
package/dist/index.d.ts
CHANGED
|
@@ -6,5 +6,7 @@ export { computeFitness, recalculateFitness } from './fitness/index.js';
|
|
|
6
6
|
export { createServer } from './api/index.js';
|
|
7
7
|
export { searchAtoms, getTopAtoms } from './retrieval/index.js';
|
|
8
8
|
export { distillTrace, BatchDistiller } from './distill/index.js';
|
|
9
|
+
export { llmDistill, groupTraces, toCreateAtoms, DEFAULT_LLM_CONFIG } from './distill/llm.js';
|
|
10
|
+
export type { LLMDistillConfig } from './distill/llm.js';
|
|
9
11
|
export type { ToolTrace, DistillFn } from './distill/index.js';
|
|
10
12
|
export { seedAtoms } from './seeds/index.js';
|
package/dist/index.js
CHANGED
|
@@ -12,6 +12,7 @@ export { createServer } from './api/index.js';
|
|
|
12
12
|
export { searchAtoms, getTopAtoms } from './retrieval/index.js';
|
|
13
13
|
// Distill
|
|
14
14
|
export { distillTrace, BatchDistiller } from './distill/index.js';
|
|
15
|
+
export { llmDistill, groupTraces, toCreateAtoms, DEFAULT_LLM_CONFIG } from './distill/llm.js';
|
|
15
16
|
// Seeds
|
|
16
17
|
export { seedAtoms } from './seeds/index.js';
|
|
17
18
|
//# sourceMappingURL=index.js.map
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAmBA,QAAQ;AACR,OAAO,EAAE,SAAS,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAEhE,UAAU;AACV,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAErE,aAAa;AACb,OAAO,EACL,gBAAgB,EAChB,oBAAoB,EACpB,wBAAwB,EACxB,oBAAoB,EACpB,kBAAkB,EAClB,sBAAsB,EACtB,0BAA0B,GAC3B,MAAM,uBAAuB,CAAC;AAE/B,UAAU;AACV,OAAO,EAAE,cAAc,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAExE,MAAM;AACN,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAE9C,YAAY;AACZ,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AAEhE,UAAU;AACV,OAAO,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAmBA,QAAQ;AACR,OAAO,EAAE,SAAS,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAEhE,UAAU;AACV,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAErE,aAAa;AACb,OAAO,EACL,gBAAgB,EAChB,oBAAoB,EACpB,wBAAwB,EACxB,oBAAoB,EACpB,kBAAkB,EAClB,sBAAsB,EACtB,0BAA0B,GAC3B,MAAM,uBAAuB,CAAC;AAE/B,UAAU;AACV,OAAO,EAAE,cAAc,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAExE,MAAM;AACN,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAE9C,YAAY;AACZ,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AAEhE,UAAU;AACV,OAAO,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAClE,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAI9F,QAAQ;AACR,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC"}
|
package/package.json
CHANGED
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* LLM-powered distillation — turns raw tool traces into actionable knowledge atoms.
|
|
3
|
+
*
|
|
4
|
+
* Groups related traces (same tool, similar errors) and asks an LLM to extract
|
|
5
|
+
* patterns, best practices, and failure workarounds.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import type { KnowledgeAtom, CreateToolAtom, CreateNegativeAtom } from '../types/index.js';
|
|
9
|
+
import type { ToolTrace } from './index.js';
|
|
10
|
+
import { createHash } from 'node:crypto';
|
|
11
|
+
|
|
12
|
+
export interface LLMDistillConfig {
|
|
13
|
+
/** API endpoint (OpenAI-compatible) */
|
|
14
|
+
apiUrl: string;
|
|
15
|
+
/** API key */
|
|
16
|
+
apiKey: string;
|
|
17
|
+
/** Model to use (default: gpt-4o-mini for cost efficiency) */
|
|
18
|
+
model: string;
|
|
19
|
+
/** Minimum traces before distillation (default: 5) */
|
|
20
|
+
minTraces: number;
|
|
21
|
+
/** Max tokens for response */
|
|
22
|
+
maxTokens: number;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export const DEFAULT_LLM_CONFIG: LLMDistillConfig = {
|
|
26
|
+
apiUrl: 'https://api.openai.com/v1/chat/completions',
|
|
27
|
+
apiKey: '',
|
|
28
|
+
model: 'gpt-4o-mini',
|
|
29
|
+
minTraces: 5,
|
|
30
|
+
maxTokens: 1024,
|
|
31
|
+
};
|
|
32
|
+
|
|
33
|
+
const DISTILL_PROMPT = `You are a knowledge distillation engine for AI agents. You analyze tool call traces and extract reusable operational knowledge.
|
|
34
|
+
|
|
35
|
+
Given a batch of tool call traces (tool name, parameters, outcomes, errors), extract knowledge atoms:
|
|
36
|
+
|
|
37
|
+
Rules:
|
|
38
|
+
- Each atom must be a single, actionable insight an agent can use
|
|
39
|
+
- Focus on: failure patterns, parameter best practices, timing tips, error workarounds
|
|
40
|
+
- Skip trivial observations ("tool X works" is not useful)
|
|
41
|
+
- Be specific and concise (1-2 sentences max per observation)
|
|
42
|
+
- Include the error signature when relevant
|
|
43
|
+
|
|
44
|
+
Output JSON array of atoms:
|
|
45
|
+
[
|
|
46
|
+
{
|
|
47
|
+
"kind": "tool" | "negative",
|
|
48
|
+
"tool_name": "string",
|
|
49
|
+
"observation": "string (the insight)",
|
|
50
|
+
"confidence": 0.0-1.0,
|
|
51
|
+
"error_signature": "string | null",
|
|
52
|
+
"anti_pattern": "string | null (for negative atoms)"
|
|
53
|
+
}
|
|
54
|
+
]
|
|
55
|
+
|
|
56
|
+
If no meaningful patterns exist, return [].`;
|
|
57
|
+
|
|
58
|
+
interface DistilledAtom {
|
|
59
|
+
kind: 'tool' | 'negative';
|
|
60
|
+
tool_name: string;
|
|
61
|
+
observation: string;
|
|
62
|
+
confidence: number;
|
|
63
|
+
error_signature: string | null;
|
|
64
|
+
anti_pattern: string | null;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* Group traces by tool name for batched distillation.
|
|
69
|
+
*/
|
|
70
|
+
export function groupTraces(traces: ToolTrace[]): Map<string, ToolTrace[]> {
|
|
71
|
+
const groups = new Map<string, ToolTrace[]>();
|
|
72
|
+
for (const trace of traces) {
|
|
73
|
+
const key = trace.toolName;
|
|
74
|
+
const existing = groups.get(key) ?? [];
|
|
75
|
+
existing.push(trace);
|
|
76
|
+
groups.set(key, existing);
|
|
77
|
+
}
|
|
78
|
+
return groups;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* Summarize traces for the LLM prompt (strip large payloads).
|
|
83
|
+
*/
|
|
84
|
+
function summarizeTraces(traces: ToolTrace[]): string {
|
|
85
|
+
return traces.map((t, i) => {
|
|
86
|
+
const params = Object.keys(t.params).join(', ');
|
|
87
|
+
const resultSnippet = t.result
|
|
88
|
+
? String(t.result).slice(0, 150).replace(/\n/g, ' ')
|
|
89
|
+
: 'null';
|
|
90
|
+
const errorSnippet = t.error ? t.error.slice(0, 150) : 'none';
|
|
91
|
+
return `[${i + 1}] ${t.toolName}(${params}) → ${t.outcome} | error: ${errorSnippet} | result: ${resultSnippet} | ${t.latencyMs}ms`;
|
|
92
|
+
}).join('\n');
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* Call an OpenAI-compatible API to distill traces.
|
|
97
|
+
*/
|
|
98
|
+
export async function llmDistill(
|
|
99
|
+
traces: ToolTrace[],
|
|
100
|
+
config: LLMDistillConfig,
|
|
101
|
+
): Promise<DistilledAtom[]> {
|
|
102
|
+
if (traces.length < config.minTraces) return [];
|
|
103
|
+
if (!config.apiKey) return [];
|
|
104
|
+
|
|
105
|
+
const summary = summarizeTraces(traces);
|
|
106
|
+
|
|
107
|
+
try {
|
|
108
|
+
const res = await fetch(config.apiUrl, {
|
|
109
|
+
method: 'POST',
|
|
110
|
+
headers: {
|
|
111
|
+
'Content-Type': 'application/json',
|
|
112
|
+
'Authorization': `Bearer ${config.apiKey}`,
|
|
113
|
+
},
|
|
114
|
+
body: JSON.stringify({
|
|
115
|
+
model: config.model,
|
|
116
|
+
messages: [
|
|
117
|
+
{ role: 'system', content: DISTILL_PROMPT },
|
|
118
|
+
{ role: 'user', content: `Analyze these ${traces.length} tool call traces:\n\n${summary}` },
|
|
119
|
+
],
|
|
120
|
+
max_tokens: config.maxTokens,
|
|
121
|
+
temperature: 0.3,
|
|
122
|
+
response_format: { type: 'json_object' },
|
|
123
|
+
}),
|
|
124
|
+
});
|
|
125
|
+
|
|
126
|
+
if (!res.ok) {
|
|
127
|
+
console.error(`LLM distill failed: ${res.status} ${await res.text()}`);
|
|
128
|
+
return [];
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
const body = await res.json() as any;
|
|
132
|
+
const content = body.choices?.[0]?.message?.content;
|
|
133
|
+
if (!content) return [];
|
|
134
|
+
|
|
135
|
+
const parsed = JSON.parse(content);
|
|
136
|
+
const atoms = Array.isArray(parsed) ? parsed : parsed.atoms ?? [];
|
|
137
|
+
|
|
138
|
+
return atoms.filter((a: any) =>
|
|
139
|
+
a.observation && typeof a.observation === 'string' && a.observation.length > 10
|
|
140
|
+
);
|
|
141
|
+
} catch (err) {
|
|
142
|
+
console.error(`LLM distill error: ${err}`);
|
|
143
|
+
return [];
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/**
|
|
148
|
+
* Convert distilled atoms to store-ready format.
|
|
149
|
+
*/
|
|
150
|
+
export function toCreateAtoms(
|
|
151
|
+
distilled: DistilledAtom[],
|
|
152
|
+
agentHash: string = 'distiller',
|
|
153
|
+
): Array<CreateToolAtom | CreateNegativeAtom> {
|
|
154
|
+
return distilled.map(d => {
|
|
155
|
+
const base = {
|
|
156
|
+
observation: d.observation,
|
|
157
|
+
context: JSON.stringify({ source: 'llm-distillation', tool: d.tool_name }),
|
|
158
|
+
confidence: d.confidence,
|
|
159
|
+
fitness_score: d.confidence * 0.9, // Slightly lower than confidence
|
|
160
|
+
trust_tier: 'local' as const,
|
|
161
|
+
source_agent_hash: agentHash,
|
|
162
|
+
decay_rate: 0.99,
|
|
163
|
+
};
|
|
164
|
+
|
|
165
|
+
if (d.kind === 'negative' && d.anti_pattern) {
|
|
166
|
+
return {
|
|
167
|
+
...base,
|
|
168
|
+
type: 'negative' as const,
|
|
169
|
+
anti_pattern: d.anti_pattern,
|
|
170
|
+
failure_cluster_size: 1,
|
|
171
|
+
error_type: d.error_signature ?? 'unknown',
|
|
172
|
+
severity: d.confidence > 0.8 ? 'high' as const : 'medium' as const,
|
|
173
|
+
};
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
return {
|
|
177
|
+
...base,
|
|
178
|
+
type: 'tool' as const,
|
|
179
|
+
tool_name: d.tool_name,
|
|
180
|
+
params_hash: createHash('sha256').update(d.observation).digest('hex').slice(0, 16),
|
|
181
|
+
outcome: d.error_signature ? 'failure' as const : 'success' as const,
|
|
182
|
+
error_signature: d.error_signature,
|
|
183
|
+
latency_ms: null,
|
|
184
|
+
reliability_score: d.error_signature ? 0.3 : 0.8,
|
|
185
|
+
};
|
|
186
|
+
});
|
|
187
|
+
}
|
package/src/index.ts
CHANGED
|
@@ -45,6 +45,8 @@ export { searchAtoms, getTopAtoms } from './retrieval/index.js';
|
|
|
45
45
|
|
|
46
46
|
// Distill
|
|
47
47
|
export { distillTrace, BatchDistiller } from './distill/index.js';
|
|
48
|
+
export { llmDistill, groupTraces, toCreateAtoms, DEFAULT_LLM_CONFIG } from './distill/llm.js';
|
|
49
|
+
export type { LLMDistillConfig } from './distill/llm.js';
|
|
48
50
|
export type { ToolTrace, DistillFn } from './distill/index.js';
|
|
49
51
|
|
|
50
52
|
// Seeds
|