@eucoder/rag 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +384 -0
- package/dist/ab-testing.d.ts +52 -0
- package/dist/ab-testing.d.ts.map +1 -0
- package/dist/ab-testing.js +144 -0
- package/dist/ab-testing.js.map +1 -0
- package/dist/ab-testing.test.d.ts +2 -0
- package/dist/ab-testing.test.d.ts.map +1 -0
- package/dist/ab-testing.test.js +147 -0
- package/dist/ab-testing.test.js.map +1 -0
- package/dist/agentic-rag.d.ts +23 -0
- package/dist/agentic-rag.d.ts.map +1 -0
- package/dist/agentic-rag.js +170 -0
- package/dist/agentic-rag.js.map +1 -0
- package/dist/agentic-rag.test.d.ts +2 -0
- package/dist/agentic-rag.test.d.ts.map +1 -0
- package/dist/agentic-rag.test.js +174 -0
- package/dist/agentic-rag.test.js.map +1 -0
- package/dist/corrective-rag.d.ts +16 -0
- package/dist/corrective-rag.d.ts.map +1 -0
- package/dist/corrective-rag.js +85 -0
- package/dist/corrective-rag.js.map +1 -0
- package/dist/corrective-rag.test.d.ts +2 -0
- package/dist/corrective-rag.test.d.ts.map +1 -0
- package/dist/corrective-rag.test.js +140 -0
- package/dist/corrective-rag.test.js.map +1 -0
- package/dist/feedback.d.ts +77 -0
- package/dist/feedback.d.ts.map +1 -0
- package/dist/feedback.js +44 -0
- package/dist/feedback.js.map +1 -0
- package/dist/feedback.test.d.ts +2 -0
- package/dist/feedback.test.d.ts.map +1 -0
- package/dist/feedback.test.js +202 -0
- package/dist/feedback.test.js.map +1 -0
- package/dist/hybrid-search.d.ts +14 -0
- package/dist/hybrid-search.d.ts.map +1 -0
- package/dist/hybrid-search.js +70 -0
- package/dist/hybrid-search.js.map +1 -0
- package/dist/hybrid-search.test.d.ts +2 -0
- package/dist/hybrid-search.test.d.ts.map +1 -0
- package/dist/hybrid-search.test.js +93 -0
- package/dist/hybrid-search.test.js.map +1 -0
- package/dist/index.d.ts +17 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +12 -0
- package/dist/index.js.map +1 -0
- package/dist/knowledge-graph.d.ts +24 -0
- package/dist/knowledge-graph.d.ts.map +1 -0
- package/dist/knowledge-graph.js +131 -0
- package/dist/knowledge-graph.js.map +1 -0
- package/dist/knowledge-graph.test.d.ts +2 -0
- package/dist/knowledge-graph.test.d.ts.map +1 -0
- package/dist/knowledge-graph.test.js +140 -0
- package/dist/knowledge-graph.test.js.map +1 -0
- package/dist/llm-grader.d.ts +19 -0
- package/dist/llm-grader.d.ts.map +1 -0
- package/dist/llm-grader.js +63 -0
- package/dist/llm-grader.js.map +1 -0
- package/dist/metrics.d.ts +26 -0
- package/dist/metrics.d.ts.map +1 -0
- package/dist/metrics.js +100 -0
- package/dist/metrics.js.map +1 -0
- package/dist/optimizer.d.ts +52 -0
- package/dist/optimizer.d.ts.map +1 -0
- package/dist/optimizer.js +228 -0
- package/dist/optimizer.js.map +1 -0
- package/dist/optimizer.test.d.ts +2 -0
- package/dist/optimizer.test.d.ts.map +1 -0
- package/dist/optimizer.test.js +201 -0
- package/dist/optimizer.test.js.map +1 -0
- package/dist/self-improving.d.ts +85 -0
- package/dist/self-improving.d.ts.map +1 -0
- package/dist/self-improving.js +163 -0
- package/dist/self-improving.js.map +1 -0
- package/dist/self-improving.test.d.ts +2 -0
- package/dist/self-improving.test.d.ts.map +1 -0
- package/dist/self-improving.test.js +234 -0
- package/dist/self-improving.test.js.map +1 -0
- package/dist/types.d.ts +117 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +2 -0
- package/dist/types.js.map +1 -0
- package/package.json +42 -0
- package/src/ab-testing.test.ts +239 -0
- package/src/ab-testing.ts +214 -0
- package/src/agentic-rag.test.ts +201 -0
- package/src/agentic-rag.ts +220 -0
- package/src/corrective-rag.test.ts +166 -0
- package/src/corrective-rag.ts +115 -0
- package/src/feedback.test.ts +227 -0
- package/src/feedback.ts +118 -0
- package/src/hybrid-search.test.ts +107 -0
- package/src/hybrid-search.ts +86 -0
- package/src/index.ts +57 -0
- package/src/knowledge-graph.test.ts +170 -0
- package/src/knowledge-graph.ts +182 -0
- package/src/llm-grader.ts +69 -0
- package/src/metrics.ts +121 -0
- package/src/optimizer.test.ts +232 -0
- package/src/optimizer.ts +307 -0
- package/src/self-improving.test.ts +341 -0
- package/src/self-improving.ts +239 -0
- package/src/types.ts +139 -0
- package/tsconfig.json +9 -0
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import type { RagFeedback, RagMetrics } from "./feedback.js";
|
|
2
|
+
/**
|
|
3
|
+
* Calcola metriche aggregate da un array di feedback
|
|
4
|
+
*/
|
|
5
|
+
export declare class MetricsCalculator {
|
|
6
|
+
/**
|
|
7
|
+
* Calcola le metriche per una strategia specifica
|
|
8
|
+
*/
|
|
9
|
+
static calculateMetrics(feedbacks: RagFeedback[], strategy: string): RagMetrics;
|
|
10
|
+
/**
|
|
11
|
+
* Calcola la deviazione standard per una metrica specifica
|
|
12
|
+
*/
|
|
13
|
+
static calculateStandardDeviation(feedbacks: RagFeedback[], metric: "rating" | "relevance" | "completeness" | "citationsQuality"): number;
|
|
14
|
+
/**
|
|
15
|
+
* Calcola l'intervallo di confidenza al 95%
|
|
16
|
+
*/
|
|
17
|
+
static calculateConfidenceInterval(feedbacks: RagFeedback[], metric: "rating" | "relevance" | "completeness" | "citationsQuality"): {
|
|
18
|
+
lower: number;
|
|
19
|
+
upper: number;
|
|
20
|
+
};
|
|
21
|
+
/**
|
|
22
|
+
* Identifica trend nei feedback (miglioramento o peggioramento)
|
|
23
|
+
*/
|
|
24
|
+
static identifyTrend(feedbacks: RagFeedback[], windowSize?: number): "improving" | "declining" | "stable";
|
|
25
|
+
}
|
|
26
|
+
//# sourceMappingURL=metrics.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../src/metrics.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAE7D;;GAEG;AACH,qBAAa,iBAAiB;IAC5B;;OAEG;IACH,MAAM,CAAC,gBAAgB,CAAC,SAAS,EAAE,WAAW,EAAE,EAAE,QAAQ,EAAE,MAAM,GAAG,UAAU;IAqD/E;;OAEG;IACH,MAAM,CAAC,0BAA0B,CAC/B,SAAS,EAAE,WAAW,EAAE,EACxB,MAAM,EAAE,QAAQ,GAAG,WAAW,GAAG,cAAc,GAAG,kBAAkB,GACnE,MAAM;IAWT;;OAEG;IACH,MAAM,CAAC,2BAA2B,CAChC,SAAS,EAAE,WAAW,EAAE,EACxB,MAAM,EAAE,QAAQ,GAAG,WAAW,GAAG,cAAc,GAAG,kBAAkB,GACnE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE;IAiBnC;;OAEG;IACH,MAAM,CAAC,aAAa,CAAC,SAAS,EAAE,WAAW,EAAE,EAAE,UAAU,GAAE,MAAW,GAAG,WAAW,GAAG,WAAW,GAAG,QAAQ;CAe9G"}
|
package/dist/metrics.js
ADDED
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Calcola metriche aggregate da un array di feedback
|
|
3
|
+
*/
|
|
4
|
+
export class MetricsCalculator {
|
|
5
|
+
/**
|
|
6
|
+
* Calcola le metriche per una strategia specifica
|
|
7
|
+
*/
|
|
8
|
+
static calculateMetrics(feedbacks, strategy) {
|
|
9
|
+
if (feedbacks.length === 0) {
|
|
10
|
+
return {
|
|
11
|
+
strategy,
|
|
12
|
+
totalQueries: 0,
|
|
13
|
+
averageRating: 0,
|
|
14
|
+
averageRelevance: 0,
|
|
15
|
+
averageCompleteness: 0,
|
|
16
|
+
averageCitationsQuality: 0,
|
|
17
|
+
overallScore: 0,
|
|
18
|
+
confidence: 0,
|
|
19
|
+
};
|
|
20
|
+
}
|
|
21
|
+
const totalQueries = feedbacks.length;
|
|
22
|
+
// Calcola medie
|
|
23
|
+
const sumRating = feedbacks.reduce((sum, f) => sum + f.rating, 0);
|
|
24
|
+
const sumRelevance = feedbacks.reduce((sum, f) => sum + f.relevance, 0);
|
|
25
|
+
const sumCompleteness = feedbacks.reduce((sum, f) => sum + f.completeness, 0);
|
|
26
|
+
const sumCitationsQuality = feedbacks.reduce((sum, f) => sum + f.citationsQuality, 0);
|
|
27
|
+
const averageRating = sumRating / totalQueries;
|
|
28
|
+
const averageRelevance = sumRelevance / totalQueries;
|
|
29
|
+
const averageCompleteness = sumCompleteness / totalQueries;
|
|
30
|
+
const averageCitationsQuality = sumCitationsQuality / totalQueries;
|
|
31
|
+
// Overall score: media pesata
|
|
32
|
+
// Rating (30%), Relevance (30%), Completeness (25%), Citations (15%)
|
|
33
|
+
const normalizedRating = averageRating / 5; // Normalizza a 0-1
|
|
34
|
+
const overallScore = normalizedRating * 0.30 +
|
|
35
|
+
averageRelevance * 0.30 +
|
|
36
|
+
averageCompleteness * 0.25 +
|
|
37
|
+
averageCitationsQuality * 0.15;
|
|
38
|
+
// Confidence: basato sul numero di sample
|
|
39
|
+
// Minimo 10 sample per avere confidence > 0.5
|
|
40
|
+
// Massimo confidence a 50+ sample
|
|
41
|
+
const confidence = Math.min(1, totalQueries / 50);
|
|
42
|
+
return {
|
|
43
|
+
strategy,
|
|
44
|
+
totalQueries,
|
|
45
|
+
averageRating,
|
|
46
|
+
averageRelevance,
|
|
47
|
+
averageCompleteness,
|
|
48
|
+
averageCitationsQuality,
|
|
49
|
+
overallScore,
|
|
50
|
+
confidence,
|
|
51
|
+
};
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Calcola la deviazione standard per una metrica specifica
|
|
55
|
+
*/
|
|
56
|
+
static calculateStandardDeviation(feedbacks, metric) {
|
|
57
|
+
if (feedbacks.length < 2)
|
|
58
|
+
return 0;
|
|
59
|
+
const values = feedbacks.map((f) => f[metric]);
|
|
60
|
+
const mean = values.reduce((sum, v) => sum + v, 0) / values.length;
|
|
61
|
+
const squaredDiffs = values.map((v) => Math.pow(v - mean, 2));
|
|
62
|
+
const variance = squaredDiffs.reduce((sum, v) => sum + v, 0) / (values.length - 1);
|
|
63
|
+
return Math.sqrt(variance);
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* Calcola l'intervallo di confidenza al 95%
|
|
67
|
+
*/
|
|
68
|
+
static calculateConfidenceInterval(feedbacks, metric) {
|
|
69
|
+
if (feedbacks.length < 2) {
|
|
70
|
+
const value = feedbacks.length === 1 ? feedbacks[0][metric] : 0;
|
|
71
|
+
return { lower: value, upper: value };
|
|
72
|
+
}
|
|
73
|
+
const values = feedbacks.map((f) => f[metric]);
|
|
74
|
+
const mean = values.reduce((sum, v) => sum + v, 0) / values.length;
|
|
75
|
+
const stdDev = this.calculateStandardDeviation(feedbacks, metric);
|
|
76
|
+
const marginOfError = 1.96 * (stdDev / Math.sqrt(values.length)); // 95% confidence
|
|
77
|
+
return {
|
|
78
|
+
lower: Math.max(0, mean - marginOfError),
|
|
79
|
+
upper: Math.min(metric === "rating" ? 5 : 1, mean + marginOfError),
|
|
80
|
+
};
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* Identifica trend nei feedback (miglioramento o peggioramento)
|
|
84
|
+
*/
|
|
85
|
+
static identifyTrend(feedbacks, windowSize = 10) {
|
|
86
|
+
if (feedbacks.length < windowSize * 2)
|
|
87
|
+
return "stable";
|
|
88
|
+
const recent = feedbacks.slice(-windowSize);
|
|
89
|
+
const older = feedbacks.slice(-windowSize * 2, -windowSize);
|
|
90
|
+
const recentAvg = recent.reduce((sum, f) => sum + f.relevance, 0) / recent.length;
|
|
91
|
+
const olderAvg = older.reduce((sum, f) => sum + f.relevance, 0) / older.length;
|
|
92
|
+
const diff = recentAvg - olderAvg;
|
|
93
|
+
if (diff > 0.05)
|
|
94
|
+
return "improving";
|
|
95
|
+
if (diff < -0.05)
|
|
96
|
+
return "declining";
|
|
97
|
+
return "stable";
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
//# sourceMappingURL=metrics.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"metrics.js","sourceRoot":"","sources":["../src/metrics.ts"],"names":[],"mappings":"AAEA;;GAEG;AACH,MAAM,OAAO,iBAAiB;IAC5B;;OAEG;IACH,MAAM,CAAC,gBAAgB,CAAC,SAAwB,EAAE,QAAgB;QAChE,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,OAAO;gBACL,QAAQ;gBACR,YAAY,EAAE,CAAC;gBACf,aAAa,EAAE,CAAC;gBAChB,gBAAgB,EAAE,CAAC;gBACnB,mBAAmB,EAAE,CAAC;gBACtB,uBAAuB,EAAE,CAAC;gBAC1B,YAAY,EAAE,CAAC;gBACf,UAAU,EAAE,CAAC;aACd,CAAC;QACJ,CAAC;QAED,MAAM,YAAY,GAAG,SAAS,CAAC,MAAM,CAAC;QAEtC,gBAAgB;QAChB,MAAM,SAAS,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAClE,MAAM,YAAY,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;QACxE,MAAM,eAAe,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC;QAC9E,MAAM,mBAAmB,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC;QAEtF,MAAM,aAAa,GAAG,SAAS,GAAG,YAAY,CAAC;QAC/C,MAAM,gBAAgB,GAAG,YAAY,GAAG,YAAY,CAAC;QACrD,MAAM,mBAAmB,GAAG,eAAe,GAAG,YAAY,CAAC;QAC3D,MAAM,uBAAuB,GAAG,mBAAmB,GAAG,YAAY,CAAC;QAEnE,8BAA8B;QAC9B,qEAAqE;QACrE,MAAM,gBAAgB,GAAG,aAAa,GAAG,CAAC,CAAC,CAAC,mBAAmB;QAC/D,MAAM,YAAY,GAChB,gBAAgB,GAAG,IAAI;YACvB,gBAAgB,GAAG,IAAI;YACvB,mBAAmB,GAAG,IAAI;YAC1B,uBAAuB,GAAG,IAAI,CAAC;QAEjC,0CAA0C;QAC1C,8CAA8C;QAC9C,kCAAkC;QAClC,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,YAAY,GAAG,EAAE,CAAC,CAAC;QAElD,OAAO;YACL,QAAQ;YACR,YAAY;YACZ,aAAa;YACb,gBAAgB;YAChB,mBAAmB;YACnB,uBAAuB;YACvB,YAAY;YACZ,UAAU;SACX,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,0BAA0B,CAC/B,SAAwB,EACxB,MAAoE;QAEpE,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC;YAAE,OAAO,CAAC,CAAC;QAEnC,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;QAC/C,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;QACnE,MAAM,YAAY,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAC9D,MAAM,QAAQ,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAEnF,OAAO,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC7B,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,2BAA2B,CAChC,SAAwB,EACxB,MAAoE;QAEpE,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzB,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAChE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;QACxC,CAAC;QAED,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;QAC/C,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;QACnE,MAAM,MAAM,GAAG,IAAI,CAAC,0BAA0B,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QAClE,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,iBAAiB;QAEnF,OAAO;YACL,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,GAAG,aAAa,CAAC;YACxC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,GAAG,aAAa,CAAC;SACnE,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,aAAa,CAAC,SAAwB,EAAE,aAAqB,EAAE;QACpE,IAAI,SAAS,CAAC,MAAM,GAAG,UAAU,GAAG,CAAC;YAAE,OAAO,QAAQ,CAAC;QAEvD,MAAM,MAAM,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,UAAU,CAAC,CAAC;QAC5C,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,UAAU,GAAG,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC;QAE5D,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;QAClF,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC;QAE/E,MAAM,IAAI,GAAG,SAAS,GAAG,QAAQ,CAAC;QAElC,IAAI,IAAI,GAAG,IAAI;YAAE,OAAO,WAAW,CAAC;QACpC,IAAI,IAAI,GAAG,CAAC,IAAI;YAAE,OAAO,WAAW,CAAC;QACrC,OAAO,QAAQ,CAAC;IAClB,CAAC;CACF"}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import type { FeedbackStorage, OptimizationSuggestion } from "./feedback.js";
|
|
2
|
+
/**
|
|
3
|
+
* Parametri ottimizzabili per strategie RAG
|
|
4
|
+
*/
|
|
5
|
+
export interface OptimizableParams {
|
|
6
|
+
minGoodHits?: number;
|
|
7
|
+
minScore?: number;
|
|
8
|
+
maxRewrites?: number;
|
|
9
|
+
semanticWeight?: number;
|
|
10
|
+
keywordWeight?: number;
|
|
11
|
+
rrfK?: number;
|
|
12
|
+
maxIterations?: number;
|
|
13
|
+
enableSelfReflection?: boolean;
|
|
14
|
+
enableChainOfThought?: boolean;
|
|
15
|
+
topK?: number;
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* Ottimizzatore automatico per parametri RAG
|
|
19
|
+
*/
|
|
20
|
+
export declare class ParameterOptimizer {
|
|
21
|
+
private storage;
|
|
22
|
+
constructor(storage?: FeedbackStorage);
|
|
23
|
+
/**
|
|
24
|
+
* Analizza i feedback e suggerisce ottimizzazioni
|
|
25
|
+
*/
|
|
26
|
+
analyzeAndSuggest(strategy: string, currentParams: OptimizableParams): Promise<OptimizationSuggestion[]>;
|
|
27
|
+
/**
|
|
28
|
+
* Analizza problemi di relevance
|
|
29
|
+
*/
|
|
30
|
+
private analyzeRelevance;
|
|
31
|
+
/**
|
|
32
|
+
* Analizza problemi di completeness
|
|
33
|
+
*/
|
|
34
|
+
private analyzeCompleteness;
|
|
35
|
+
/**
|
|
36
|
+
* Analizza problemi di citations quality
|
|
37
|
+
*/
|
|
38
|
+
private analyzeCitationsQuality;
|
|
39
|
+
/**
|
|
40
|
+
* Analizza trend temporale
|
|
41
|
+
*/
|
|
42
|
+
private analyzeTrend;
|
|
43
|
+
/**
|
|
44
|
+
* Applica automaticamente le ottimizzazioni suggerite
|
|
45
|
+
*/
|
|
46
|
+
applySuggestions(currentParams: OptimizableParams, suggestions: OptimizationSuggestion[], minConfidence?: number): OptimizableParams;
|
|
47
|
+
/**
|
|
48
|
+
* Genera report di ottimizzazione
|
|
49
|
+
*/
|
|
50
|
+
generateReport(strategy: string, currentParams: OptimizableParams): Promise<string>;
|
|
51
|
+
}
|
|
52
|
+
//# sourceMappingURL=optimizer.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"optimizer.d.ts","sourceRoot":"","sources":["../src/optimizer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAe,eAAe,EAAE,sBAAsB,EAAE,MAAM,eAAe,CAAC;AAI1F;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAEhC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IAGrB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,IAAI,CAAC,EAAE,MAAM,CAAC;IAGd,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAC/B,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAG/B,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,qBAAa,kBAAkB;IAC7B,OAAO,CAAC,OAAO,CAAkB;gBAErB,OAAO,CAAC,EAAE,eAAe;IAIrC;;OAEG;IACG,iBAAiB,CACrB,QAAQ,EAAE,MAAM,EAChB,aAAa,EAAE,iBAAiB,GAC/B,OAAO,CAAC,sBAAsB,EAAE,CAAC;IAoCpC;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAkDxB;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAiD3B;;OAEG;IACH,OAAO,CAAC,uBAAuB;IAqC/B;;OAEG;IACH,OAAO,CAAC,YAAY;IAsBpB;;OAEG;IACH,gBAAgB,CACd,aAAa,EAAE,iBAAiB,EAChC,WAAW,EAAE,sBAAsB,EAAE,EACrC,aAAa,GAAE,MAAY,GAC1B,iBAAiB;IAYpB;;OAEG;IACG,cAAc,CAClB,QAAQ,EAAE,MAAM,EAChB,aAAa,EAAE,iBAAiB,GAC/B,OAAO,CAAC,MAAM,CAAC;CAgCnB"}
|
|
@@ -0,0 +1,228 @@
|
|
|
1
|
+
import { InMemoryFeedbackStorage } from "./feedback.js";
|
|
2
|
+
import { MetricsCalculator } from "./metrics.js";
|
|
3
|
+
/**
|
|
4
|
+
* Ottimizzatore automatico per parametri RAG
|
|
5
|
+
*/
|
|
6
|
+
export class ParameterOptimizer {
|
|
7
|
+
storage;
|
|
8
|
+
constructor(storage) {
|
|
9
|
+
this.storage = storage || new InMemoryFeedbackStorage();
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* Analizza i feedback e suggerisce ottimizzazioni
|
|
13
|
+
*/
|
|
14
|
+
async analyzeAndSuggest(strategy, currentParams) {
|
|
15
|
+
const feedbacks = await this.storage.getFeedback(strategy);
|
|
16
|
+
if (feedbacks.length < 10) {
|
|
17
|
+
return [{
|
|
18
|
+
type: "strategy",
|
|
19
|
+
target: "data_collection",
|
|
20
|
+
current: feedbacks.length,
|
|
21
|
+
suggested: 50,
|
|
22
|
+
expectedImprovement: 0,
|
|
23
|
+
confidence: 1,
|
|
24
|
+
reasoning: `Raccogli almeno 50 feedback per analisi affidabile (attualmente: ${feedbacks.length})`,
|
|
25
|
+
}];
|
|
26
|
+
}
|
|
27
|
+
const suggestions = [];
|
|
28
|
+
// Analizza problemi di relevance
|
|
29
|
+
const relevanceSuggestions = this.analyzeRelevance(feedbacks, currentParams);
|
|
30
|
+
suggestions.push(...relevanceSuggestions);
|
|
31
|
+
// Analizza problemi di completeness
|
|
32
|
+
const completenessSuggestions = this.analyzeCompleteness(feedbacks, currentParams);
|
|
33
|
+
suggestions.push(...completenessSuggestions);
|
|
34
|
+
// Analizza problemi di citations quality
|
|
35
|
+
const citationsSuggestions = this.analyzeCitationsQuality(feedbacks, currentParams);
|
|
36
|
+
suggestions.push(...citationsSuggestions);
|
|
37
|
+
// Analizza trend temporale
|
|
38
|
+
const trendSuggestions = this.analyzeTrend(feedbacks, currentParams);
|
|
39
|
+
suggestions.push(...trendSuggestions);
|
|
40
|
+
return suggestions;
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Analizza problemi di relevance
|
|
44
|
+
*/
|
|
45
|
+
analyzeRelevance(feedbacks, params) {
|
|
46
|
+
const suggestions = [];
|
|
47
|
+
const avgRelevance = feedbacks.reduce((sum, f) => sum + f.relevance, 0) / feedbacks.length;
|
|
48
|
+
// Se relevance è bassa, suggerisci di aumentare minScore o topK
|
|
49
|
+
if (avgRelevance < 0.6) {
|
|
50
|
+
if (params.minScore !== undefined && params.minScore < 0.5) {
|
|
51
|
+
suggestions.push({
|
|
52
|
+
type: "parameter",
|
|
53
|
+
target: "minScore",
|
|
54
|
+
current: params.minScore,
|
|
55
|
+
suggested: Math.min(0.7, params.minScore + 0.1),
|
|
56
|
+
expectedImprovement: 0.15,
|
|
57
|
+
confidence: 0.8,
|
|
58
|
+
reasoning: `Relevance media bassa (${avgRelevance.toFixed(2)}). Aumenta minScore per filtrare risultati meno rilevanti.`,
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
if (params.topK !== undefined && params.topK < 10) {
|
|
62
|
+
suggestions.push({
|
|
63
|
+
type: "parameter",
|
|
64
|
+
target: "topK",
|
|
65
|
+
current: params.topK,
|
|
66
|
+
suggested: Math.min(15, params.topK + 3),
|
|
67
|
+
expectedImprovement: 0.1,
|
|
68
|
+
confidence: 0.7,
|
|
69
|
+
reasoning: `Relevance media bassa (${avgRelevance.toFixed(2)}). Aumenta topK per avere più candidati da cui scegliere.`,
|
|
70
|
+
});
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
// Se relevance è molto alta, potresti essere troppo conservativo
|
|
74
|
+
if (avgRelevance > 0.9 && params.minScore !== undefined && params.minScore > 0.7) {
|
|
75
|
+
suggestions.push({
|
|
76
|
+
type: "parameter",
|
|
77
|
+
target: "minScore",
|
|
78
|
+
current: params.minScore,
|
|
79
|
+
suggested: Math.max(0.5, params.minScore - 0.1),
|
|
80
|
+
expectedImprovement: 0.05,
|
|
81
|
+
confidence: 0.6,
|
|
82
|
+
reasoning: `Relevance molto alta (${avgRelevance.toFixed(2)}) ma potresti perdere risultati utili. Riduci minScore per aumentare recall.`,
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
return suggestions;
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* Analizza problemi di completeness
|
|
89
|
+
*/
|
|
90
|
+
analyzeCompleteness(feedbacks, params) {
|
|
91
|
+
const suggestions = [];
|
|
92
|
+
const avgCompleteness = feedbacks.reduce((sum, f) => sum + f.completeness, 0) / feedbacks.length;
|
|
93
|
+
// Se completeness è bassa, suggerisci di abilitare self-reflection o aumentare iterazioni
|
|
94
|
+
if (avgCompleteness < 0.6) {
|
|
95
|
+
if (params.enableSelfReflection === false) {
|
|
96
|
+
suggestions.push({
|
|
97
|
+
type: "parameter",
|
|
98
|
+
target: "enableSelfReflection",
|
|
99
|
+
current: false,
|
|
100
|
+
suggested: true,
|
|
101
|
+
expectedImprovement: 0.2,
|
|
102
|
+
confidence: 0.85,
|
|
103
|
+
reasoning: `Completeness bassa (${avgCompleteness.toFixed(2)}). Abilita self-reflection per migliorare qualità risposte.`,
|
|
104
|
+
});
|
|
105
|
+
}
|
|
106
|
+
if (params.maxIterations !== undefined && params.maxIterations < 3) {
|
|
107
|
+
suggestions.push({
|
|
108
|
+
type: "parameter",
|
|
109
|
+
target: "maxIterations",
|
|
110
|
+
current: params.maxIterations,
|
|
111
|
+
suggested: Math.min(5, params.maxIterations + 1),
|
|
112
|
+
expectedImprovement: 0.15,
|
|
113
|
+
confidence: 0.75,
|
|
114
|
+
reasoning: `Completeness bassa (${avgCompleteness.toFixed(2)}). Aumenta maxIterations per permettere più tentativi di miglioramento.`,
|
|
115
|
+
});
|
|
116
|
+
}
|
|
117
|
+
if (params.enableChainOfThought === false) {
|
|
118
|
+
suggestions.push({
|
|
119
|
+
type: "parameter",
|
|
120
|
+
target: "enableChainOfThought",
|
|
121
|
+
current: false,
|
|
122
|
+
suggested: true,
|
|
123
|
+
expectedImprovement: 0.18,
|
|
124
|
+
confidence: 0.8,
|
|
125
|
+
reasoning: `Completeness bassa (${avgCompleteness.toFixed(2)}). Abilita chain-of-thought per risposte più strutturate.`,
|
|
126
|
+
});
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
return suggestions;
|
|
130
|
+
}
|
|
131
|
+
/**
|
|
132
|
+
* Analizza problemi di citations quality
|
|
133
|
+
*/
|
|
134
|
+
analyzeCitationsQuality(feedbacks, params) {
|
|
135
|
+
const suggestions = [];
|
|
136
|
+
const avgCitations = feedbacks.reduce((sum, f) => sum + f.citationsQuality, 0) / feedbacks.length;
|
|
137
|
+
// Se citations quality è bassa, suggerisci di migliorare hybrid search
|
|
138
|
+
if (avgCitations < 0.6) {
|
|
139
|
+
if (params.semanticWeight !== undefined && params.semanticWeight < 0.6) {
|
|
140
|
+
suggestions.push({
|
|
141
|
+
type: "parameter",
|
|
142
|
+
target: "semanticWeight",
|
|
143
|
+
current: params.semanticWeight,
|
|
144
|
+
suggested: Math.min(0.7, params.semanticWeight + 0.1),
|
|
145
|
+
expectedImprovement: 0.12,
|
|
146
|
+
confidence: 0.7,
|
|
147
|
+
reasoning: `Citations quality bassa (${avgCitations.toFixed(2)}). Aumenta semanticWeight per risultati più semanticamente rilevanti.`,
|
|
148
|
+
});
|
|
149
|
+
}
|
|
150
|
+
if (params.rrfK !== undefined && params.rrfK > 80) {
|
|
151
|
+
suggestions.push({
|
|
152
|
+
type: "parameter",
|
|
153
|
+
target: "rrfK",
|
|
154
|
+
current: params.rrfK,
|
|
155
|
+
suggested: Math.max(50, params.rrfK - 20),
|
|
156
|
+
expectedImprovement: 0.1,
|
|
157
|
+
confidence: 0.65,
|
|
158
|
+
reasoning: `Citations quality bassa (${avgCitations.toFixed(2)}). Riduci rrfK per dare più peso ai risultati top-ranked.`,
|
|
159
|
+
});
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
return suggestions;
|
|
163
|
+
}
|
|
164
|
+
/**
|
|
165
|
+
* Analizza trend temporale
|
|
166
|
+
*/
|
|
167
|
+
analyzeTrend(feedbacks, _params) {
|
|
168
|
+
const suggestions = [];
|
|
169
|
+
const trend = MetricsCalculator.identifyTrend(feedbacks);
|
|
170
|
+
if (trend === "declining") {
|
|
171
|
+
suggestions.push({
|
|
172
|
+
type: "strategy",
|
|
173
|
+
target: "overall_strategy",
|
|
174
|
+
current: "current",
|
|
175
|
+
suggested: "review",
|
|
176
|
+
expectedImprovement: 0.1,
|
|
177
|
+
confidence: 0.7,
|
|
178
|
+
reasoning: `Trend in declino negli ultimi feedback. Considera di rivedere la strategia o raccogliere feedback qualitativi per capire cosa non funziona.`,
|
|
179
|
+
});
|
|
180
|
+
}
|
|
181
|
+
return suggestions;
|
|
182
|
+
}
|
|
183
|
+
/**
|
|
184
|
+
* Applica automaticamente le ottimizzazioni suggerite
|
|
185
|
+
*/
|
|
186
|
+
applySuggestions(currentParams, suggestions, minConfidence = 0.7) {
|
|
187
|
+
const optimized = { ...currentParams };
|
|
188
|
+
for (const suggestion of suggestions) {
|
|
189
|
+
if (suggestion.confidence >= minConfidence && suggestion.type === "parameter") {
|
|
190
|
+
optimized[suggestion.target] = suggestion.suggested;
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
return optimized;
|
|
194
|
+
}
|
|
195
|
+
/**
|
|
196
|
+
* Genera report di ottimizzazione
|
|
197
|
+
*/
|
|
198
|
+
async generateReport(strategy, currentParams) {
|
|
199
|
+
const feedbacks = await this.storage.getFeedback(strategy);
|
|
200
|
+
const metrics = MetricsCalculator.calculateMetrics(feedbacks, strategy);
|
|
201
|
+
const suggestions = await this.analyzeAndSuggest(strategy, currentParams);
|
|
202
|
+
let report = `# Report Ottimizzazione: ${strategy}\n\n`;
|
|
203
|
+
report += `## Metriche Attuali\n`;
|
|
204
|
+
report += `- **Total Queries**: ${metrics.totalQueries}\n`;
|
|
205
|
+
report += `- **Overall Score**: ${metrics.overallScore.toFixed(3)}\n`;
|
|
206
|
+
report += `- **Average Relevance**: ${metrics.averageRelevance.toFixed(3)}\n`;
|
|
207
|
+
report += `- **Average Completeness**: ${metrics.averageCompleteness.toFixed(3)}\n`;
|
|
208
|
+
report += `- **Average Citations Quality**: ${metrics.averageCitationsQuality.toFixed(3)}\n`;
|
|
209
|
+
report += `- **Confidence**: ${(metrics.confidence * 100).toFixed(0)}%\n\n`;
|
|
210
|
+
report += `## Suggerimenti di Ottimizzazione\n\n`;
|
|
211
|
+
if (suggestions.length === 0) {
|
|
212
|
+
report += `Nessun suggerimento. I parametri attuali sembrano ottimali.\n`;
|
|
213
|
+
}
|
|
214
|
+
else {
|
|
215
|
+
for (const suggestion of suggestions) {
|
|
216
|
+
report += `### ${suggestion.target}\n`;
|
|
217
|
+
report += `- **Tipo**: ${suggestion.type}\n`;
|
|
218
|
+
report += `- **Valore attuale**: ${JSON.stringify(suggestion.current)}\n`;
|
|
219
|
+
report += `- **Valore suggerito**: ${JSON.stringify(suggestion.suggested)}\n`;
|
|
220
|
+
report += `- **Miglioramento atteso**: +${(suggestion.expectedImprovement * 100).toFixed(1)}%\n`;
|
|
221
|
+
report += `- **Confidence**: ${(suggestion.confidence * 100).toFixed(0)}%\n`;
|
|
222
|
+
report += `- **Ragionamento**: ${suggestion.reasoning}\n\n`;
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
return report;
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
//# sourceMappingURL=optimizer.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"optimizer.js","sourceRoot":"","sources":["../src/optimizer.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,uBAAuB,EAAE,MAAM,eAAe,CAAC;AACxD,OAAO,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAC;AAyBjD;;GAEG;AACH,MAAM,OAAO,kBAAkB;IACrB,OAAO,CAAkB;IAEjC,YAAY,OAAyB;QACnC,IAAI,CAAC,OAAO,GAAG,OAAO,IAAI,IAAI,uBAAuB,EAAE,CAAC;IAC1D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,iBAAiB,CACrB,QAAgB,EAChB,aAAgC;QAEhC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;QAE3D,IAAI,SAAS,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;YAC1B,OAAO,CAAC;oBACN,IAAI,EAAE,UAAU;oBAChB,MAAM,EAAE,iBAAiB;oBACzB,OAAO,EAAE,SAAS,CAAC,MAAM;oBACzB,SAAS,EAAE,EAAE;oBACb,mBAAmB,EAAE,CAAC;oBACtB,UAAU,EAAE,CAAC;oBACb,SAAS,EAAE,oEAAoE,SAAS,CAAC,MAAM,GAAG;iBACnG,CAAC,CAAC;QACL,CAAC;QAED,MAAM,WAAW,GAA6B,EAAE,CAAC;QAEjD,iCAAiC;QACjC,MAAM,oBAAoB,GAAG,IAAI,CAAC,gBAAgB,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC;QAC7E,WAAW,CAAC,IAAI,CAAC,GAAG,oBAAoB,CAAC,CAAC;QAE1C,oCAAoC;QACpC,MAAM,uBAAuB,GAAG,IAAI,CAAC,mBAAmB,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC;QACnF,WAAW,CAAC,IAAI,CAAC,GAAG,uBAAuB,CAAC,CAAC;QAE7C,yCAAyC;QACzC,MAAM,oBAAoB,GAAG,IAAI,CAAC,uBAAuB,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC;QACpF,WAAW,CAAC,IAAI,CAAC,GAAG,oBAAoB,CAAC,CAAC;QAE1C,2BAA2B;QAC3B,MAAM,gBAAgB,GAAG,IAAI,CAAC,YAAY,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC;QACrE,WAAW,CAAC,IAAI,CAAC,GAAG,gBAAgB,CAAC,CAAC;QAEtC,OAAO,WAAW,CAAC;IACrB,CAAC;IAED;;OAEG;IACK,gBAAgB,CACtB,SAAwB,EACxB,MAAyB;QAEzB,MAAM,WAAW,GAA6B,EAAE,CAAC;QACjD,MAAM,YAAY,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC;QAE3F,gEAAgE;QAChE,IAAI,YAAY,GAAG,GAAG,EAAE,CAAC;YACvB,IAAI,MAAM,CAAC,QAAQ,KAAK,SAAS,IAAI,MAAM,CAAC,QAAQ,GAAG,GAAG,EAAE,CAAC;gBAC3D,WAAW,CAAC,IAAI,CAAC;oBACf,IAAI,EAAE,WAAW;oBACjB,MAAM,EAAE,UAAU;oBAClB,OAAO,EAAE,MAAM,CAAC,QAAQ;oBACxB,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,QAAQ,GAAG,GAAG,CAAC;oBAC/C,mBAAmB,EAAE,IAAI;oBACzB,UAAU,EAAE,GAAG;oBACf,SAAS,EAAE,0BAA0B,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,4DAA4D;iBACzH,CAAC,CAAC;YACL,CAAC;YAED,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,IAAI,MAAM,CAAC,IAAI,GAAG,EAAE,EAAE,CAAC;gBAClD,WAAW,CAAC,IAAI,CAAC;oBACf,IAAI,EAAE,WAAW;oBACjB,MAAM,EAAE,MAAM;oBACd,OAAO,EAAE,MAAM,CAAC,IAAI;oBACpB,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,MAAM,CAAC,IAAI,GAAG,CAAC,CAAC;oBACxC,mBAAmB,EAAE,GAAG;oBACxB,UAAU,EAAE,GAAG;oBACf,SAAS,EAAE,0BAA0B,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,2DAA2D;iBACxH,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,iEAAiE;QACjE,IAAI,YAAY,GAAG,GAAG,IAAI,MAAM,CAAC,QAAQ,KAAK,SAAS,IAAI,MAAM,CAAC,QAAQ,GAAG,GAAG,EAAE,CAAC;YACjF,WAAW,CAAC,IAAI,CAAC;gBACf,IAAI,EAAE,WAAW;gBACjB,MAAM,EAAE,UAAU;gBAClB,OAAO,EAAE,MAAM,CAAC,QAAQ;gBACxB,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,QAAQ,GAAG,GAAG,CAAC;gBAC/C,mBAAmB,EAAE,IAAI;gBACzB,UAAU,EAAE,GAAG;gBACf,SAAS,EAAE,yBAAyB,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,8EAA8E;aAC1I,CAAC,CAAC;QACL,CAAC;QAED,OAAO,WAAW,CAAC;IACrB,CAAC;IAED;;OAEG;IACK,mBAAmB,CACzB,SAAwB,EACxB,MAAyB;QAEzB,MAAM,WAAW,GAA6B,EAAE,CAAC;QACjD,MAAM,eAAe,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,YAAY,EAAE,CAAC,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC;QAEjG,0FAA0F;QAC1F,IAAI,eAAe,GAAG,GAAG,EAAE,CAAC;YAC1B,IAAI,MAAM,CAAC,oBAAoB,KAAK,KAAK,EAAE,CAAC;gBAC1C,WAAW,CAAC,IAAI,CAAC;oBACf,IAAI,EAAE,WAAW;oBACjB,MAAM,EAAE,sBAAsB;oBAC9B,OAAO,EAAE,KAAK;oBACd,SAAS,EAAE,IAAI;oBACf,mBAAmB,EAAE,GAAG;oBACxB,UAAU,EAAE,IAAI;oBAChB,SAAS,EAAE,uBAAuB,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,6DAA6D;iBAC1H,CAAC,CAAC;YACL,CAAC;YAED,IAAI,MAAM,CAAC,aAAa,KAAK,SAAS,IAAI,MAAM,CAAC,aAAa,GAAG,CAAC,EAAE,CAAC;gBACnE,WAAW,CAAC,IAAI,CAAC;oBACf,IAAI,EAAE,WAAW;oBACjB,MAAM,EAAE,eAAe;oBACvB,OAAO,EAAE,MAAM,CAAC,aAAa;oBAC7B,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,aAAa,GAAG,CAAC,CAAC;oBAChD,mBAAmB,EAAE,IAAI;oBACzB,UAAU,EAAE,IAAI;oBAChB,SAAS,EAAE,uBAAuB,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,yEAAyE;iBACtI,CAAC,CAAC;YACL,CAAC;YAED,IAAI,MAAM,CAAC,oBAAoB,KAAK,KAAK,EAAE,CAAC;gBAC1C,WAAW,CAAC,IAAI,CAAC;oBACf,IAAI,EAAE,WAAW;oBACjB,MAAM,EAAE,sBAAsB;oBAC9B,OAAO,EAAE,KAAK;oBACd,SAAS,EAAE,IAAI;oBACf,mBAAmB,EAAE,IAAI;oBACzB,UAAU,EAAE,GAAG;oBACf,SAAS,EAAE,uBAAuB,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,2DAA2D;iBACxH,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,OAAO,WAAW,CAAC;IACrB,CAAC;IAED;;OAEG;IACK,uBAAuB,CAC7B,SAAwB,EACxB,MAAyB;QAEzB,MAAM,WAAW,GAA6B,EAAE,CAAC;QACjD,MAAM,YAAY,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,gBAAgB,EAAE,CAAC,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC;QAElG,uEAAuE;QACvE,IAAI,YAAY,GAAG,GAAG,EAAE,CAAC;YACvB,IAAI,MAAM,CAAC,cAAc,KAAK,SAAS,IAAI,MAAM,CAAC,cAAc,GAAG,GAAG,EAAE,CAAC;gBACvE,WAAW,CAAC,IAAI,CAAC;oBACf,IAAI,EAAE,WAAW;oBACjB,MAAM,EAAE,gBAAgB;oBACxB,OAAO,EAAE,MAAM,CAAC,cAAc;oBAC9B,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,cAAc,GAAG,GAAG,CAAC;oBACrD,mBAAmB,EAAE,IAAI;oBACzB,UAAU,EAAE,GAAG;oBACf,SAAS,EAAE,4BAA4B,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,uEAAuE;iBACtI,CAAC,CAAC;YACL,CAAC;YAED,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,IAAI,MAAM,CAAC,IAAI,GAAG,EAAE,EAAE,CAAC;gBAClD,WAAW,CAAC,IAAI,CAAC;oBACf,IAAI,EAAE,WAAW;oBACjB,MAAM,EAAE,MAAM;oBACd,OAAO,EAAE,MAAM,CAAC,IAAI;oBACpB,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,MAAM,CAAC,IAAI,GAAG,EAAE,CAAC;oBACzC,mBAAmB,EAAE,GAAG;oBACxB,UAAU,EAAE,IAAI;oBAChB,SAAS,EAAE,4BAA4B,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,2DAA2D;iBAC1H,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,OAAO,WAAW,CAAC;IACrB,CAAC;IAED;;OAEG;IACK,YAAY,CAClB,SAAwB,EACxB,OAA0B;QAE1B,MAAM,WAAW,GAA6B,EAAE,CAAC;QACjD,MAAM,KAAK,GAAG,iBAAiB,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;QAEzD,IAAI,KAAK,KAAK,WAAW,EAAE,CAAC;YAC1B,WAAW,CAAC,IAAI,CAAC;gBACf,IAAI,EAAE,UAAU;gBAChB,MAAM,EAAE,kBAAkB;gBAC1B,OAAO,EAAE,SAAS;gBAClB,SAAS,EAAE,QAAQ;gBACnB,mBAAmB,EAAE,GAAG;gBACxB,UAAU,EAAE,GAAG;gBACf,SAAS,EAAE,6IAA6I;aACzJ,CAAC,CAAC;QACL,CAAC;QAED,OAAO,WAAW,CAAC;IACrB,CAAC;IAED;;OAEG;IACH,gBAAgB,CACd,aAAgC,EAChC,WAAqC,EACrC,gBAAwB,GAAG;QAE3B,MAAM,SAAS,GAAG,EAAE,GAAG,aAAa,EAAE,CAAC;QAEvC,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;YACrC,IAAI,UAAU,CAAC,UAAU,IAAI,aAAa,IAAI,UAAU,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;gBAC7E,SAAiB,CAAC,UAAU,CAAC,MAAM,CAAC,GAAG,UAAU,CAAC,SAAS,CAAC;YAC/D,CAAC;QACH,CAAC;QAED,OAAO,SAAS,CAAC;IACnB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,cAAc,CAClB,QAAgB,EAChB,aAAgC;QAEhC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;QAC3D,MAAM,OAAO,GAAG,iBAAiB,CAAC,gBAAgB,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QACxE,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;QAE1E,IAAI,MAAM,GAAG,4BAA4B,QAAQ,MAAM,CAAC;QACxD,MAAM,IAAI,uBAAuB,CAAC;QAClC,MAAM,IAAI,wBAAwB,OAAO,CAAC,YAAY,IAAI,CAAC;QAC3D,MAAM,IAAI,wBAAwB,OAAO,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;QACtE,MAAM,IAAI,4BAA4B,OAAO,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;QAC9E,MAAM,IAAI,+BAA+B,OAAO,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;QACpF,MAAM,IAAI,oCAAoC,OAAO,CAAC,uBAAuB,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;QAC7F,MAAM,IAAI,qBAAqB,CAAC,OAAO,CAAC,UAAU,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC;QAE5E,MAAM,IAAI,uCAAuC,CAAC;QAElD,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC7B,MAAM,IAAI,+DAA+D,CAAC;QAC5E,CAAC;aAAM,CAAC;YACN,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;gBACrC,MAAM,IAAI,OAAO,UAAU,CAAC,MAAM,IAAI,CAAC;gBACvC,MAAM,IAAI,eAAe,UAAU,CAAC,IAAI,IAAI,CAAC;gBAC7C,MAAM,IAAI,yBAAyB,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC;gBAC1E,MAAM,IAAI,2BAA2B,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,CAAC;gBAC9E,MAAM,IAAI,gCAAgC,CAAC,UAAU,CAAC,mBAAmB,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC;gBACjG,MAAM,IAAI,qBAAqB,CAAC,UAAU,CAAC,UAAU,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC;gBAC7E,MAAM,IAAI,uBAAuB,UAAU,CAAC,SAAS,MAAM,CAAC;YAC9D,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;CACF"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"optimizer.test.d.ts","sourceRoot":"","sources":["../src/optimizer.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
import { describe, it, expect, beforeEach } from "vitest";
|
|
2
|
+
import { ParameterOptimizer } from "./optimizer.js";
|
|
3
|
+
import { InMemoryFeedbackStorage } from "./feedback.js";
|
|
4
|
+
describe("ParameterOptimizer", () => {
|
|
5
|
+
let optimizer;
|
|
6
|
+
let storage;
|
|
7
|
+
beforeEach(() => {
|
|
8
|
+
storage = new InMemoryFeedbackStorage();
|
|
9
|
+
optimizer = new ParameterOptimizer(storage);
|
|
10
|
+
});
|
|
11
|
+
it("should suggest collecting more data when feedback is insufficient", async () => {
|
|
12
|
+
const params = {
|
|
13
|
+
minScore: 0.5,
|
|
14
|
+
topK: 5,
|
|
15
|
+
};
|
|
16
|
+
// Add only 5 feedback (not enough)
|
|
17
|
+
for (let i = 0; i < 5; i++) {
|
|
18
|
+
await storage.saveFeedback({
|
|
19
|
+
id: `test-${i}`,
|
|
20
|
+
query: `query ${i}`,
|
|
21
|
+
answer: { text: `answer ${i}`, citations: [], steps: [], rewrites: 0 },
|
|
22
|
+
rating: 4,
|
|
23
|
+
relevance: 0.8,
|
|
24
|
+
completeness: 0.7,
|
|
25
|
+
citationsQuality: 0.9,
|
|
26
|
+
timestamp: new Date(),
|
|
27
|
+
strategy: "strategy-a",
|
|
28
|
+
});
|
|
29
|
+
}
|
|
30
|
+
const suggestions = await optimizer.analyzeAndSuggest("strategy-a", params);
|
|
31
|
+
expect(suggestions).toHaveLength(1);
|
|
32
|
+
expect(suggestions[0]?.type).toBe("strategy");
|
|
33
|
+
expect(suggestions[0]?.target).toBe("data_collection");
|
|
34
|
+
});
|
|
35
|
+
it("should suggest increasing minScore when relevance is low", async () => {
|
|
36
|
+
const params = {
|
|
37
|
+
minScore: 0.3,
|
|
38
|
+
topK: 10,
|
|
39
|
+
};
|
|
40
|
+
// Add 20 feedback with low relevance
|
|
41
|
+
for (let i = 0; i < 20; i++) {
|
|
42
|
+
await storage.saveFeedback({
|
|
43
|
+
id: `test-${i}`,
|
|
44
|
+
query: `query ${i}`,
|
|
45
|
+
answer: { text: `answer ${i}`, citations: [], steps: [], rewrites: 0 },
|
|
46
|
+
rating: 3,
|
|
47
|
+
relevance: 0.5,
|
|
48
|
+
completeness: 0.6,
|
|
49
|
+
citationsQuality: 0.7,
|
|
50
|
+
timestamp: new Date(),
|
|
51
|
+
strategy: "strategy-a",
|
|
52
|
+
});
|
|
53
|
+
}
|
|
54
|
+
const suggestions = await optimizer.analyzeAndSuggest("strategy-a", params);
|
|
55
|
+
const minScoreSuggestion = suggestions.find(s => s.target === "minScore");
|
|
56
|
+
expect(minScoreSuggestion).toBeDefined();
|
|
57
|
+
expect(minScoreSuggestion?.suggested).toBeGreaterThan(params.minScore);
|
|
58
|
+
});
|
|
59
|
+
it("should suggest enabling self-reflection when completeness is low", async () => {
|
|
60
|
+
const params = {
|
|
61
|
+
enableSelfReflection: false,
|
|
62
|
+
maxIterations: 2,
|
|
63
|
+
};
|
|
64
|
+
// Add 20 feedback with low completeness
|
|
65
|
+
for (let i = 0; i < 20; i++) {
|
|
66
|
+
await storage.saveFeedback({
|
|
67
|
+
id: `test-${i}`,
|
|
68
|
+
query: `query ${i}`,
|
|
69
|
+
answer: { text: `answer ${i}`, citations: [], steps: [], rewrites: 0 },
|
|
70
|
+
rating: 3,
|
|
71
|
+
relevance: 0.7,
|
|
72
|
+
completeness: 0.5,
|
|
73
|
+
citationsQuality: 0.8,
|
|
74
|
+
timestamp: new Date(),
|
|
75
|
+
strategy: "strategy-a",
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
const suggestions = await optimizer.analyzeAndSuggest("strategy-a", params);
|
|
79
|
+
const selfReflectionSuggestion = suggestions.find(s => s.target === "enableSelfReflection");
|
|
80
|
+
expect(selfReflectionSuggestion).toBeDefined();
|
|
81
|
+
expect(selfReflectionSuggestion?.suggested).toBe(true);
|
|
82
|
+
});
|
|
83
|
+
it("should suggest increasing semanticWeight when citations quality is low", async () => {
|
|
84
|
+
const params = {
|
|
85
|
+
semanticWeight: 0.5,
|
|
86
|
+
keywordWeight: 0.5,
|
|
87
|
+
};
|
|
88
|
+
// Add 20 feedback with low citations quality
|
|
89
|
+
for (let i = 0; i < 20; i++) {
|
|
90
|
+
await storage.saveFeedback({
|
|
91
|
+
id: `test-${i}`,
|
|
92
|
+
query: `query ${i}`,
|
|
93
|
+
answer: { text: `answer ${i}`, citations: [], steps: [], rewrites: 0 },
|
|
94
|
+
rating: 3,
|
|
95
|
+
relevance: 0.7,
|
|
96
|
+
completeness: 0.7,
|
|
97
|
+
citationsQuality: 0.5,
|
|
98
|
+
timestamp: new Date(),
|
|
99
|
+
strategy: "strategy-a",
|
|
100
|
+
});
|
|
101
|
+
}
|
|
102
|
+
const suggestions = await optimizer.analyzeAndSuggest("strategy-a", params);
|
|
103
|
+
const semanticWeightSuggestion = suggestions.find(s => s.target === "semanticWeight");
|
|
104
|
+
expect(semanticWeightSuggestion).toBeDefined();
|
|
105
|
+
expect(semanticWeightSuggestion?.suggested).toBeGreaterThan(params.semanticWeight);
|
|
106
|
+
});
|
|
107
|
+
it("should apply suggestions with sufficient confidence", () => {
|
|
108
|
+
const currentParams = {
|
|
109
|
+
minScore: 0.3,
|
|
110
|
+
topK: 5,
|
|
111
|
+
enableSelfReflection: false,
|
|
112
|
+
};
|
|
113
|
+
const suggestions = [
|
|
114
|
+
{
|
|
115
|
+
type: "parameter",
|
|
116
|
+
target: "minScore",
|
|
117
|
+
current: 0.3,
|
|
118
|
+
suggested: 0.5,
|
|
119
|
+
expectedImprovement: 0.15,
|
|
120
|
+
confidence: 0.8,
|
|
121
|
+
reasoning: "Test suggestion",
|
|
122
|
+
},
|
|
123
|
+
{
|
|
124
|
+
type: "parameter",
|
|
125
|
+
target: "topK",
|
|
126
|
+
current: 5,
|
|
127
|
+
suggested: 10,
|
|
128
|
+
expectedImprovement: 0.1,
|
|
129
|
+
confidence: 0.6, // Below threshold
|
|
130
|
+
reasoning: "Test suggestion",
|
|
131
|
+
},
|
|
132
|
+
];
|
|
133
|
+
const optimized = optimizer.applySuggestions(currentParams, suggestions, 0.7);
|
|
134
|
+
expect(optimized.minScore).toBe(0.5);
|
|
135
|
+
expect(optimized.topK).toBe(5); // Not applied due to low confidence
|
|
136
|
+
expect(optimized.enableSelfReflection).toBe(false);
|
|
137
|
+
});
|
|
138
|
+
it("should generate report", async () => {
|
|
139
|
+
const params = {
|
|
140
|
+
minScore: 0.5,
|
|
141
|
+
topK: 10,
|
|
142
|
+
};
|
|
143
|
+
// Add 15 feedback
|
|
144
|
+
for (let i = 0; i < 15; i++) {
|
|
145
|
+
await storage.saveFeedback({
|
|
146
|
+
id: `test-${i}`,
|
|
147
|
+
query: `query ${i}`,
|
|
148
|
+
answer: { text: `answer ${i}`, citations: [], steps: [], rewrites: 0 },
|
|
149
|
+
rating: 4,
|
|
150
|
+
relevance: 0.7,
|
|
151
|
+
completeness: 0.6,
|
|
152
|
+
citationsQuality: 0.8,
|
|
153
|
+
timestamp: new Date(),
|
|
154
|
+
strategy: "strategy-a",
|
|
155
|
+
});
|
|
156
|
+
}
|
|
157
|
+
const report = await optimizer.generateReport("strategy-a", params);
|
|
158
|
+
expect(report).toContain("Report Ottimizzazione: strategy-a");
|
|
159
|
+
expect(report).toContain("Metriche Attuali");
|
|
160
|
+
expect(report).toContain("Total Queries");
|
|
161
|
+
expect(report).toContain("15");
|
|
162
|
+
});
|
|
163
|
+
it("should detect declining trend", async () => {
|
|
164
|
+
const params = {
|
|
165
|
+
minScore: 0.5,
|
|
166
|
+
};
|
|
167
|
+
// Add older feedback with high relevance
|
|
168
|
+
for (let i = 0; i < 10; i++) {
|
|
169
|
+
await storage.saveFeedback({
|
|
170
|
+
id: `old-${i}`,
|
|
171
|
+
query: `query ${i}`,
|
|
172
|
+
answer: { text: `answer ${i}`, citations: [], steps: [], rewrites: 0 },
|
|
173
|
+
rating: 5,
|
|
174
|
+
relevance: 0.9,
|
|
175
|
+
completeness: 0.9,
|
|
176
|
+
citationsQuality: 0.9,
|
|
177
|
+
timestamp: new Date(Date.now() - 1000000),
|
|
178
|
+
strategy: "strategy-a",
|
|
179
|
+
});
|
|
180
|
+
}
|
|
181
|
+
// Add recent feedback with low relevance
|
|
182
|
+
for (let i = 0; i < 10; i++) {
|
|
183
|
+
await storage.saveFeedback({
|
|
184
|
+
id: `new-${i}`,
|
|
185
|
+
query: `query ${i}`,
|
|
186
|
+
answer: { text: `answer ${i}`, citations: [], steps: [], rewrites: 0 },
|
|
187
|
+
rating: 2,
|
|
188
|
+
relevance: 0.4,
|
|
189
|
+
completeness: 0.4,
|
|
190
|
+
citationsQuality: 0.4,
|
|
191
|
+
timestamp: new Date(),
|
|
192
|
+
strategy: "strategy-a",
|
|
193
|
+
});
|
|
194
|
+
}
|
|
195
|
+
const suggestions = await optimizer.analyzeAndSuggest("strategy-a", params);
|
|
196
|
+
const trendSuggestion = suggestions.find(s => s.target === "overall_strategy");
|
|
197
|
+
expect(trendSuggestion).toBeDefined();
|
|
198
|
+
expect(trendSuggestion?.reasoning).toContain("declino");
|
|
199
|
+
});
|
|
200
|
+
});
|
|
201
|
+
//# sourceMappingURL=optimizer.test.js.map
|