@goldensheepai/toknxr-cli 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +270 -9
- package/lib/audit-logger.js +500 -0
- package/lib/cli.js +1850 -129
- package/lib/cli.test.js +49 -0
- package/lib/code-analysis.js +349 -4
- package/lib/dashboard.js +4 -17
- package/lib/fixtures/canary-interaction.js +18 -0
- package/lib/plugin-system.js +266 -0
- package/lib/sync.js +27 -5
- package/lib/ui.js +129 -0
- package/lib/utils.js +117 -0
- package/package.json +51 -18
- package/.env +0 -21
- package/.env.example +0 -21
- package/interactions.log +0 -8
- package/src/ai-analytics.ts +0 -418
- package/src/auth.ts +0 -80
- package/src/cli.ts +0 -447
- package/src/code-analysis.ts +0 -365
- package/src/config.ts +0 -10
- package/src/dashboard.tsx +0 -391
- package/src/hallucination-detector.ts +0 -368
- package/src/policy.ts +0 -55
- package/src/pricing.ts +0 -21
- package/src/proxy.ts +0 -438
- package/src/sync.ts +0 -129
- package/start.sh +0 -56
- package/test-analysis.mjs +0 -77
- package/test-coding.mjs +0 -27
- package/test-generate-sample-data.js +0 -118
- package/test-proxy.mjs +0 -25
- package/toknxr.config.json +0 -63
- package/toknxr.policy.json +0 -18
- package/tsconfig.json +0 -19
package/.env.example
DELETED
@@ -1,21 +0,0 @@
|
|
1
|
-
# TokNXR Environment Variables
|
2
|
-
# Copy this file to .env and fill in your actual API keys
|
3
|
-
|
4
|
-
# Google AI API Key (used for both Gemini-Pro and Gemini-Free)
|
5
|
-
GEMINI_API_KEY=AIzaSyCtrQ8e5H66vbrAozWlWUmCcObJmrI2ovg
|
6
|
-
|
7
|
-
# OpenAI API Key
|
8
|
-
OPENAI_API_KEY=your-openai-api-key-here
|
9
|
-
|
10
|
-
# Anthropic Claude API Key
|
11
|
-
ANTHROPIC_API_KEY=your-anthropic-api-key-here
|
12
|
-
|
13
|
-
# Optional: Webhook URL for budget alerts
|
14
|
-
# WEBHOOK_URL=https://your-webhook-url.com/alerts
|
15
|
-
SUPABASE_URL=your-supabase-url
|
16
|
-
SUPABASE_ANON_KEY=your-anon-key
|
17
|
-
SUPABASE_SERVICE_ROLE_KEY=your-service-role-key
|
18
|
-
SUPABASE_JWT_SECRET=your-jwt-secret
|
19
|
-
|
20
|
-
# Optional: Custom port for the proxy server (default: 8787)
|
21
|
-
PORT=8787
|
package/interactions.log
DELETED
@@ -1,8 +0,0 @@
|
|
1
|
-
{"timestamp":"2025-10-05T20:14:35.420Z","provider":"Gemini-Pro","model":"unknown","promptTokens":9,"completionTokens":2056,"totalTokens":3506,"costUSD":0,"taskType":"chat"}
|
2
|
-
{"timestamp":"2025-10-05T20:18:24.093Z","provider":"Gemini-Pro","model":"unknown","promptTokens":9,"completionTokens":2236,"totalTokens":3564,"costUSD":0,"taskType":"chat"}
|
3
|
-
{"timestamp":"2025-10-05T20:21:58.826Z","provider":"Gemini-Pro","model":"unknown","promptTokens":9,"completionTokens":2054,"totalTokens":3322,"costUSD":0,"taskType":"chat"}
|
4
|
-
{"requestId":"84870610-70c2-4b31-834f-6c55f92c8495","timestamp":"2025-10-05T20:49:46.035Z","provider":"Gemini-Pro","model":"gemini-2.5-flash","promptTokens":2,"completionTokens":519,"totalTokens":1930,"costUSD":0.3117,"taskType":"chat"}
|
5
|
-
{"timestamp":"2025-10-05T21:44:57.972Z","provider":"Gemini-Pro","model":"gemini-2.5-flash","promptTokens":150,"completionTokens":200,"totalTokens":350,"costUSD":0.05,"taskType":"coding","userPrompt":"Create a React component for a todo list","aiResponse":"Here's a React todo component...","extractedCode":"function TodoList() {\n const [todos, setTodos] = useState([]);\n return (\n <div>\n <h1>My Todos</h1>\n {/* Todo implementation */}\n </div>\n );\n}","codeQualityScore":85,"codeQualityMetrics":{"syntaxValid":true,"estimatedReadability":0.8,"hasFunctions":true,"hasClasses":false,"linesOfCode":12,"potentialIssues":[]},"effectivenessScore":88}
|
6
|
-
{"timestamp":"2025-10-05T21:59:57.974Z","provider":"OpenAI-GPT4","model":"gpt-4","promptTokens":80,"completionTokens":120,"totalTokens":200,"costUSD":0.02,"taskType":"coding","userPrompt":"Write a Python function to calculate fibonacci","aiResponse":"def fibonacci(n):...","extractedCode":"def fibonacci(n):\n if n <= 1:\n return n\n return fibonacci(n-1) + fibonacci(n-2)","codeQualityScore":92,"codeQualityMetrics":{"syntaxValid":true,"estimatedReadability":0.9,"hasFunctions":true,"hasClasses":false,"linesOfCode":4,"potentialIssues":[]},"effectivenessScore":95}
|
7
|
-
{"timestamp":"2025-10-05T22:09:57.974Z","provider":"Gemini-Pro","model":"gemini-2.5-flash","promptTokens":200,"completionTokens":300,"totalTokens":500,"costUSD":0.08,"taskType":"coding","userPrompt":"Create a TypeScript API endpoint","aiResponse":"Here's a TypeScript API endpoint...","extractedCode":"import express from 'express';\n\nconst app = express();\n\napp.get('/api/users', async (req, res) => {\n // Implementation here\n});\n\nexport default app;","codeQualityScore":78,"codeQualityMetrics":{"syntaxValid":true,"estimatedReadability":0.7,"hasFunctions":true,"hasClasses":false,"linesOfCode":8,"potentialIssues":["Missing error handling"]},"effectivenessScore":82}
|
8
|
-
{"requestId":"b8d3c1cc-c3c5-427a-b9f3-0af12b7e8e8d","timestamp":"2025-10-05T22:19:07.944Z","provider":"Gemini-Pro","model":"gemini-2.5-flash","promptTokens":13,"completionTokens":1133,"totalTokens":2206,"costUSD":0.68175,"taskType":"coding","extractedCode":"/**\n * Calculates the factorial of a non-negative integer using recursion.\n *\n * @param {number} n The non-negative integer for which to calculate the factorial.\n * @returns {number} The factorial of n.\n * @throws {Error} If n is a negative number.\n * @throws {Error} If n is not an integer.\n */\nfunction factorial(n) {\n // Input validation: Factorial is traditionally defined for non-negative integers.\n if (n < 0) {\n throw new Error(\"Factorial is not defined for negative numbers.\");\n }\n if (!Number.isInteger(n)) {\n throw new Error(\"Factorial is only defined for integer numbers.\");\n }\n\n // Base Case: This is where the recursion stops.\n // The factorial of 0 is 1, and the factorial of 1 is 1.\n if (n === 0 || n === 1) {\n return 1;\n }\n\n // Recursive Step: The function calls itself with a smaller input.\n // n! = n * (n-1)!\n return n * factorial(n - 1);\n}\n\n// --- Examples ---\nconsole.log(`Factorial of 0: ${factorial(0)}`); // Expected: 1\nconsole.log(`Factorial of 1: ${factorial(1)}`); // Expected: 1\nconsole.log(`Factorial of 5: ${factorial(5)}`); // Expected: 120 (5 * 4 * 3 * 2 * 1)\nconsole.log(`Factorial of 7: ${factorial(7)}`); // Expected: 5040 (7 * 6 * 5 * 4 * 3 * 2 * 1)\nconsole.log(`Factorial of 10: ${factorial(10)}`); // Expected: 3628800\n\n// --- Error Handling Examples ---\ntry {\n console.log(`Factorial of -3: ${factorial(-3)}`);\n} catch (error) {\n console.error(`Error: ${error.message}`); // Expected: Error: Factorial is not defined for negative numbers.\n}\n\ntry {\n console.log(`Factorial of 3.5: ${factorial(3.5)}`);\n} catch (error) {\n console.error(`Error: ${error.message}`); // Expected: Error: Factorial is only defined for integer numbers.\n}\n\ntry {\n console.log(`Factorial of \"abc\": ${factorial(\"abc\")}`);\n} catch (error) {\n console.error(`Error: ${error.message}`); // Expected: Error: Factorial is only defined for integer numbers.\n}","codeQualityMetrics":{"syntaxValid":true,"linesOfCode":47,"complexity":10,"hasFunctions":true,"hasClasses":false,"hasTests":false,"estimatedReadability":6,"potentialIssues":[],"language":"javascript"},"codeQualityScore":100,"effectivenessScore":68,"userPrompt":"Write a JavaScript function to calculate the factorial of a number using recursion","aiResponse":"Here's a JavaScript function to calculate the factorial of a number using recursion, along with explanations and examples:\n\n```javascript\n/**\n * Calculates the factorial of a non-negative integer using recursion.\n *\n * @param {number} n The non-negative integer for which to calculate the factorial.\n * @returns {number} The factorial of n.\n * @throws {Error} If n is a negative number.\n * @throws {Error} If n is not an integer.\n */\nfunction factorial(n) {\n // Input validation: Factorial is traditionally defined for non-negative integers.\n if (n < 0) {\n throw new Error(\"Factorial is not defined for negative numbers.\");\n }\n if (!Number.isInteger(n)) {\n throw new Error(\"Factorial is only defined for integer numbers.\");\n }\n\n // Base Case: This is where the recursion stops.\n // The factorial of 0 is 1, and the factorial of 1 is 1.\n if (n === 0 || n === 1) {\n return 1;\n }\n\n // Recursive Step: The function calls itself with a smaller input.\n // n! = n * (n-1)!\n return n * factorial(n - 1);\n}\n\n// --- Examples ---\nconsole.log(`Factorial of 0: ${factorial(0)}`); // Expected: 1\nconsole.log(`Factorial of 1: ${factorial(1)}`); // Expected: 1\nconsole.log(`Factorial of 5: ${factorial(5)}`); // Expected: 120 (5 * 4 * 3 * 2 * 1)\nconsole.log(`Factorial of 7: ${factorial(7)}`); // Expected: 5040 (7 * 6 * 5 * 4 * 3 * 2 * 1)\nconsole.log(`Factorial of 10: ${factorial(10)}`); // Expected: 3628800\n\n// --- Error Handling Examples ---\ntry {\n console.log(`Factorial of -3: ${factorial(-3)}`);\n} catch (error) {\n console.error(`Error: ${error.message}`); // Expected: Error: Factorial is not defined for negative numbers.\n}\n\ntry {\n console.log(`Factorial of 3.5: ${factorial(3.5)}`);\n} catch (error) {\n console.error(`Error: ${error.message}`); // Expected: Error: Factorial is only defined for integer numbers.\n}\n\ntry {\n console.log(`Factorial of \"abc\": ${factorial(\"abc\")}`);\n} catch (error) {\n console.error(`Error: ${error.message}`); // Expected: Error: Factorial is only defined for integer numbers.\n}\n```\n\n### How it works (Recursive Breakdown):\n\nLet's trace `factorial(3)`:\n\n1. `factorial(3)` is called.\n * `n` is 3. It's not 0 or 1.\n * It returns `3 * factorial(2)`.\n\n2. `factorial(2)` is called (from inside the first call).\n * `n` is 2. It's not 0 or 1.\n * It returns `2 * factorial(1)`.\n\n3. `factorial(1)` is called (from inside the second call).\n * `n` is 1. **This hits the base case!**\n * It returns `1`.\n\n4. Now, the execution unwinds:\n * The `factorial(2)` call receives `1` from `factorial(1)`.\n * It calculates `2 * 1 = 2`.\n * It returns `2`.\n\n5. Finally, the `factorial(3)` call receives `2` from `factorial(2)`.\n * It calculates `3 * 2 = 6`.\n * It returns `6`.\n\nThe result `6` is then printed.\n\n### Key Concepts:\n\n1. **Base Case:** Every recursive function *must* have one or more base cases. This is a condition where the function stops calling itself and returns a direct value. Without a base case, the function would call itself indefinitely, leading to a \"Stack Overflow\" error. For factorial, `factorial(0) = 1` and `factorial(1) = 1` are the base cases.\n2. **Recursive Step:** This is where the function calls itself with a modified (usually smaller or simpler) input. The idea is to break down the problem into a smaller instance of the *same* problem. For factorial, `n! = n * (n-1)!` is the recursive step. The problem `n!` is solved by using the solution to `(n-1)!`.\n3. **Input Validation:** It's good practice to add checks for invalid inputs (like negative numbers or non-integers for factorial) to make the function more robust."}
|
package/src/ai-analytics.ts
DELETED
@@ -1,418 +0,0 @@
|
|
1
|
-
import fs from 'fs';
|
2
|
-
import path from 'path';
|
3
|
-
import { HallucinationDetector, HallucinationDetection, HallucinationMetrics, BusinessImpactMetrics } from './hallucination-detector.js';
|
4
|
-
|
5
|
-
export interface AIAnalyticsData {
|
6
|
-
timestamp: string;
|
7
|
-
provider: string;
|
8
|
-
model: string;
|
9
|
-
userPrompt: string;
|
10
|
-
aiResponse: string;
|
11
|
-
taskType: string;
|
12
|
-
|
13
|
-
// Existing metrics
|
14
|
-
codeQualityScore?: number;
|
15
|
-
effectivenessScore?: number;
|
16
|
-
|
17
|
-
// New hallucination metrics
|
18
|
-
hallucinationDetection?: HallucinationDetection;
|
19
|
-
businessImpact?: BusinessImpactMetrics;
|
20
|
-
|
21
|
-
// Cost and usage
|
22
|
-
costUSD: number;
|
23
|
-
totalTokens: number;
|
24
|
-
}
|
25
|
-
|
26
|
-
export interface ProviderAnalytics {
|
27
|
-
totalInteractions: number;
|
28
|
-
hallucinationRate: number;
|
29
|
-
avgQualityScore: number;
|
30
|
-
avgEffectivenessScore: number;
|
31
|
-
businessImpact: BusinessImpactMetrics;
|
32
|
-
}
|
33
|
-
|
34
|
-
export interface AggregatedAIAnalytics {
|
35
|
-
totalInteractions: number;
|
36
|
-
hallucinationMetrics: HallucinationMetrics;
|
37
|
-
providerComparison: Record<string, ProviderAnalytics>;
|
38
|
-
trends: {
|
39
|
-
hallucinationRateOverTime: Array<{ date: string; rate: number }>;
|
40
|
-
qualityScoreOverTime: Array<{ date: string; score: number }>;
|
41
|
-
costEfficiencyOverTime: Array<{ date: string; efficiency: number }>;
|
42
|
-
};
|
43
|
-
recommendations: string[];
|
44
|
-
}
|
45
|
-
|
46
|
-
/**
|
47
|
-
* Enhanced AI Analytics with hallucination tracking
|
48
|
-
*/
|
49
|
-
export class AIAnalytics {
|
50
|
-
private detector: HallucinationDetector;
|
51
|
-
private logFilePath: string;
|
52
|
-
|
53
|
-
constructor(logFilePath?: string) {
|
54
|
-
this.detector = new HallucinationDetector();
|
55
|
-
this.logFilePath = logFilePath || path.resolve(process.cwd(), 'interactions.log');
|
56
|
-
}
|
57
|
-
|
58
|
-
/**
|
59
|
-
* Analyze a single interaction for hallucinations and business impact
|
60
|
-
*/
|
61
|
-
analyzeInteraction(
|
62
|
-
userPrompt: string,
|
63
|
-
aiResponse: string,
|
64
|
-
context: {
|
65
|
-
provider: string;
|
66
|
-
model: string;
|
67
|
-
taskType: string;
|
68
|
-
costUSD: number;
|
69
|
-
totalTokens: number;
|
70
|
-
codeQualityScore?: number;
|
71
|
-
effectivenessScore?: number;
|
72
|
-
}
|
73
|
-
): AIAnalyticsData {
|
74
|
-
// Run hallucination detection
|
75
|
-
const hallucinationDetection = this.detector.detectHallucination(
|
76
|
-
userPrompt,
|
77
|
-
aiResponse
|
78
|
-
);
|
79
|
-
|
80
|
-
// Calculate business impact if hallucination detected
|
81
|
-
let businessImpact: BusinessImpactMetrics | undefined;
|
82
|
-
if (hallucinationDetection.isLikelyHallucination) {
|
83
|
-
businessImpact = this.detector.calculateBusinessImpact(
|
84
|
-
hallucinationDetection.confidence,
|
85
|
-
1, // This interaction
|
86
|
-
context.costUSD
|
87
|
-
);
|
88
|
-
}
|
89
|
-
|
90
|
-
return {
|
91
|
-
timestamp: new Date().toISOString(),
|
92
|
-
provider: context.provider,
|
93
|
-
model: context.model,
|
94
|
-
userPrompt,
|
95
|
-
aiResponse,
|
96
|
-
taskType: context.taskType,
|
97
|
-
costUSD: context.costUSD,
|
98
|
-
totalTokens: context.totalTokens,
|
99
|
-
codeQualityScore: context.codeQualityScore,
|
100
|
-
effectivenessScore: context.effectivenessScore,
|
101
|
-
hallucinationDetection,
|
102
|
-
businessImpact
|
103
|
-
};
|
104
|
-
}
|
105
|
-
|
106
|
-
/**
|
107
|
-
* Generate comprehensive analytics from interaction logs
|
108
|
-
*/
|
109
|
-
generateAnalytics(): AggregatedAIAnalytics {
|
110
|
-
if (!fs.existsSync(this.logFilePath)) {
|
111
|
-
return this.getEmptyAnalytics();
|
112
|
-
}
|
113
|
-
|
114
|
-
const fileContent = fs.readFileSync(this.logFilePath, 'utf8');
|
115
|
-
const lines = fileContent.trim().split('\n');
|
116
|
-
const interactions: AIAnalyticsData[] = [];
|
117
|
-
|
118
|
-
// Parse all interactions
|
119
|
-
for (const line of lines) {
|
120
|
-
try {
|
121
|
-
const interaction = JSON.parse(line);
|
122
|
-
if (interaction.userPrompt && interaction.aiResponse) {
|
123
|
-
interactions.push(interaction as AIAnalyticsData);
|
124
|
-
}
|
125
|
-
} catch (error) {
|
126
|
-
// Skip malformed lines
|
127
|
-
continue;
|
128
|
-
}
|
129
|
-
}
|
130
|
-
|
131
|
-
if (interactions.length === 0) {
|
132
|
-
return this.getEmptyAnalytics();
|
133
|
-
}
|
134
|
-
|
135
|
-
return this.aggregateAnalytics(interactions);
|
136
|
-
}
|
137
|
-
|
138
|
-
/**
|
139
|
-
* Aggregate analytics from interaction data
|
140
|
-
*/
|
141
|
-
private aggregateAnalytics(interactions: AIAnalyticsData[]): AggregatedAIAnalytics {
|
142
|
-
const totalInteractions = interactions.length;
|
143
|
-
|
144
|
-
// Calculate hallucination metrics
|
145
|
-
const hallucinations = interactions.filter(i => i.hallucinationDetection?.isLikelyHallucination);
|
146
|
-
const hallucinationCount = hallucinations.length;
|
147
|
-
const hallucinationRate = (hallucinationCount / totalInteractions) * 100;
|
148
|
-
|
149
|
-
const avgConfidence = hallucinations.length > 0
|
150
|
-
? hallucinations.reduce((sum, h) => sum + (h.hallucinationDetection?.confidence || 0), 0) / hallucinations.length
|
151
|
-
: 0;
|
152
|
-
|
153
|
-
// Category breakdown
|
154
|
-
const byCategory: Record<string, number> = {};
|
155
|
-
hallucinations.forEach(h => {
|
156
|
-
h.hallucinationDetection?.evidence.forEach(evidence => {
|
157
|
-
byCategory[evidence.type] = (byCategory[evidence.type] || 0) + 1;
|
158
|
-
});
|
159
|
-
});
|
160
|
-
|
161
|
-
// Provider comparison
|
162
|
-
const providerStats: Record<string, AIAnalyticsData[]> = {};
|
163
|
-
interactions.forEach(interaction => {
|
164
|
-
if (!providerStats[interaction.provider]) {
|
165
|
-
providerStats[interaction.provider] = [];
|
166
|
-
}
|
167
|
-
providerStats[interaction.provider].push(interaction);
|
168
|
-
});
|
169
|
-
|
170
|
-
const providerComparison: Record<string, ProviderAnalytics> = {};
|
171
|
-
Object.entries(providerStats).forEach(([provider, providerInteractions]) => {
|
172
|
-
const providerHallucinations = providerInteractions.filter(i => i.hallucinationDetection?.isLikelyHallucination);
|
173
|
-
const providerHallucinationRate = (providerHallucinations.length / providerInteractions.length) * 100;
|
174
|
-
|
175
|
-
const avgQualityScore = providerInteractions.reduce((sum, i) => sum + (i.codeQualityScore || 0), 0) / providerInteractions.length;
|
176
|
-
const avgEffectivenessScore = providerInteractions.reduce((sum, i) => sum + (i.effectivenessScore || 0), 0) / providerInteractions.length;
|
177
|
-
|
178
|
-
// Calculate business impact for this provider
|
179
|
-
const totalCost = providerInteractions.reduce((sum, i) => sum + i.costUSD, 0);
|
180
|
-
const businessImpact = this.detector.calculateBusinessImpact(
|
181
|
-
providerHallucinationRate,
|
182
|
-
providerInteractions.length,
|
183
|
-
totalCost / providerInteractions.length
|
184
|
-
);
|
185
|
-
|
186
|
-
providerComparison[provider] = {
|
187
|
-
totalInteractions: providerInteractions.length,
|
188
|
-
hallucinationRate: Math.round(providerHallucinationRate * 10) / 10,
|
189
|
-
avgQualityScore: Math.round(avgQualityScore),
|
190
|
-
avgEffectivenessScore: Math.round(avgEffectivenessScore),
|
191
|
-
businessImpact
|
192
|
-
};
|
193
|
-
});
|
194
|
-
|
195
|
-
// Calculate overall business impact
|
196
|
-
const totalCost = interactions.reduce((sum, i) => sum + i.costUSD, 0);
|
197
|
-
const avgCostPerInteraction = totalCost / totalInteractions;
|
198
|
-
const overallBusinessImpact = this.detector.calculateBusinessImpact(
|
199
|
-
hallucinationRate,
|
200
|
-
totalInteractions,
|
201
|
-
avgCostPerInteraction
|
202
|
-
);
|
203
|
-
|
204
|
-
const hallucinationMetrics: HallucinationMetrics = {
|
205
|
-
totalAnalyses: totalInteractions,
|
206
|
-
hallucinationCount,
|
207
|
-
hallucinationRate: Math.round(hallucinationRate * 10) / 10,
|
208
|
-
avgConfidence: Math.round(avgConfidence),
|
209
|
-
byCategory,
|
210
|
-
byProvider: Object.fromEntries(
|
211
|
-
Object.entries(providerComparison).map(([provider, stats]) => [provider, stats.hallucinationRate])
|
212
|
-
),
|
213
|
-
businessImpact: overallBusinessImpact
|
214
|
-
};
|
215
|
-
|
216
|
-
// Generate trends (last 30 days)
|
217
|
-
const trends = this.generateTrends(interactions);
|
218
|
-
|
219
|
-
// Generate recommendations
|
220
|
-
const recommendations = this.generateRecommendations(hallucinationMetrics, providerComparison);
|
221
|
-
|
222
|
-
return {
|
223
|
-
totalInteractions,
|
224
|
-
hallucinationMetrics,
|
225
|
-
providerComparison,
|
226
|
-
trends,
|
227
|
-
recommendations
|
228
|
-
};
|
229
|
-
}
|
230
|
-
|
231
|
-
/**
|
232
|
-
* Generate trend data from interactions
|
233
|
-
*/
|
234
|
-
private generateTrends(interactions: AIAnalyticsData[]) {
|
235
|
-
const last30Days = new Date();
|
236
|
-
last30Days.setDate(last30Days.getDate() - 30);
|
237
|
-
|
238
|
-
const recentInteractions = interactions.filter(i => new Date(i.timestamp) >= last30Days);
|
239
|
-
|
240
|
-
// Group by day
|
241
|
-
const dailyGroups: Record<string, AIAnalyticsData[]> = {};
|
242
|
-
recentInteractions.forEach(interaction => {
|
243
|
-
const date = new Date(interaction.timestamp).toISOString().split('T')[0];
|
244
|
-
if (!dailyGroups[date]) {
|
245
|
-
dailyGroups[date] = [];
|
246
|
-
}
|
247
|
-
dailyGroups[date].push(interaction);
|
248
|
-
});
|
249
|
-
|
250
|
-
// Calculate daily metrics
|
251
|
-
const hallucinationRateOverTime = Object.entries(dailyGroups)
|
252
|
-
.sort(([a], [b]) => a.localeCompare(b))
|
253
|
-
.map(([date, dayInteractions]) => {
|
254
|
-
const hallucinations = dayInteractions.filter(i => i.hallucinationDetection?.isLikelyHallucination);
|
255
|
-
const rate = dayInteractions.length > 0 ? (hallucinations.length / dayInteractions.length) * 100 : 0;
|
256
|
-
return { date, rate: Math.round(rate * 10) / 10 };
|
257
|
-
});
|
258
|
-
|
259
|
-
const qualityScoreOverTime = Object.entries(dailyGroups)
|
260
|
-
.sort(([a], [b]) => a.localeCompare(b))
|
261
|
-
.map(([date, dayInteractions]) => {
|
262
|
-
const avgQuality = dayInteractions.reduce((sum, i) => sum + (i.codeQualityScore || 0), 0) / dayInteractions.length;
|
263
|
-
return { date, score: Math.round(avgQuality) };
|
264
|
-
});
|
265
|
-
|
266
|
-
const costEfficiencyOverTime = Object.entries(dailyGroups)
|
267
|
-
.sort(([a], [b]) => a.localeCompare(b))
|
268
|
-
.map(([date, dayInteractions]) => {
|
269
|
-
const totalCost = dayInteractions.reduce((sum, i) => sum + i.costUSD, 0);
|
270
|
-
const avgQuality = dayInteractions.reduce((sum, i) => sum + (i.codeQualityScore || 0), 0) / dayInteractions.length;
|
271
|
-
const efficiency = totalCost > 0 ? (avgQuality / totalCost) * 1000 : 0; // Quality per dollar
|
272
|
-
return { date, efficiency: Math.round(efficiency * 10) / 10 };
|
273
|
-
});
|
274
|
-
|
275
|
-
return {
|
276
|
-
hallucinationRateOverTime,
|
277
|
-
qualityScoreOverTime,
|
278
|
-
costEfficiencyOverTime
|
279
|
-
};
|
280
|
-
}
|
281
|
-
|
282
|
-
/**
|
283
|
-
* Generate actionable recommendations
|
284
|
-
*/
|
285
|
-
private generateRecommendations(
|
286
|
-
metrics: HallucinationMetrics,
|
287
|
-
providerComparison: Record<string, ProviderAnalytics>
|
288
|
-
): string[] {
|
289
|
-
const recommendations: string[] = [];
|
290
|
-
|
291
|
-
// Hallucination rate recommendations
|
292
|
-
if (metrics.hallucinationRate > 20) {
|
293
|
-
recommendations.push(`🚨 CRITICAL: Hallucination rate is ${metrics.hallucinationRate}%. Consider reviewing AI-generated content more carefully.`);
|
294
|
-
} else if (metrics.hallucinationRate > 10) {
|
295
|
-
recommendations.push(`⚠️ WARNING: Hallucination rate is ${metrics.hallucinationRate}%. Monitor closely and verify important information.`);
|
296
|
-
} else if (metrics.hallucinationRate > 5) {
|
297
|
-
recommendations.push(`ℹ️ Hallucination rate is ${metrics.hallucinationRate}%. Generally acceptable but watch for patterns.`);
|
298
|
-
}
|
299
|
-
|
300
|
-
// Provider-specific recommendations
|
301
|
-
const worstProvider = Object.entries(providerComparison)
|
302
|
-
.sort(([,a], [,b]) => (b.hallucinationRate || 0) - (a.hallucinationRate || 0))[0];
|
303
|
-
|
304
|
-
if (worstProvider && (worstProvider[1].hallucinationRate || 0) > 15) {
|
305
|
-
recommendations.push(`🔄 Consider reducing usage of ${worstProvider[0]} (hallucination rate: ${worstProvider[1].hallucinationRate}%) or improve prompt quality.`);
|
306
|
-
}
|
307
|
-
|
308
|
-
// Business impact recommendations
|
309
|
-
if (metrics.businessImpact.estimatedDevTimeWasted > 5) {
|
310
|
-
recommendations.push(`⏱️ Hallucinations are wasting ~${metrics.businessImpact.estimatedDevTimeWasted} hours of development time. Consider AI response verification workflows.`);
|
311
|
-
}
|
312
|
-
|
313
|
-
if (metrics.businessImpact.roiImpact > 10) {
|
314
|
-
recommendations.push(`💰 Hallucinations are reducing ROI by ${metrics.businessImpact.roiImpact}%. Review AI usage strategy and prompt engineering.`);
|
315
|
-
}
|
316
|
-
|
317
|
-
// Quality improvement recommendations
|
318
|
-
const bestProvider = Object.entries(providerComparison)
|
319
|
-
.sort(([,a], [,b]) => (b.avgQualityScore || 0) - (a.avgQualityScore || 0))[0];
|
320
|
-
|
321
|
-
if (bestProvider && (bestProvider[1].avgQualityScore || 0) > 80) {
|
322
|
-
recommendations.push(`✅ ${bestProvider[0]} shows good performance (quality: ${bestProvider[1].avgQualityScore}/100). Consider using it more for critical tasks.`);
|
323
|
-
}
|
324
|
-
|
325
|
-
// Category-specific recommendations
|
326
|
-
if (metrics.byCategory.fabrication > metrics.byCategory.contradiction) {
|
327
|
-
recommendations.push(`🔧 High fabrication rate detected. Focus on improving technical prompt specificity and providing more context.`);
|
328
|
-
}
|
329
|
-
|
330
|
-
if (metrics.byCategory.contradiction > 3) {
|
331
|
-
recommendations.push(`⚖️ Multiple contradictions detected. Consider breaking complex requests into smaller, focused prompts.`);
|
332
|
-
}
|
333
|
-
|
334
|
-
if (recommendations.length === 0) {
|
335
|
-
recommendations.push('✨ AI hallucination metrics look good! Continue monitoring for any emerging patterns.');
|
336
|
-
}
|
337
|
-
|
338
|
-
return recommendations;
|
339
|
-
}
|
340
|
-
|
341
|
-
/**
|
342
|
-
* Get empty analytics for when no data is available
|
343
|
-
*/
|
344
|
-
private getEmptyAnalytics(): AggregatedAIAnalytics {
|
345
|
-
return {
|
346
|
-
totalInteractions: 0,
|
347
|
-
hallucinationMetrics: {
|
348
|
-
totalAnalyses: 0,
|
349
|
-
hallucinationCount: 0,
|
350
|
-
hallucinationRate: 0,
|
351
|
-
avgConfidence: 0,
|
352
|
-
byCategory: {},
|
353
|
-
byProvider: {},
|
354
|
-
businessImpact: {
|
355
|
-
estimatedDevTimeWasted: 0,
|
356
|
-
qualityDegradationScore: 0,
|
357
|
-
roiImpact: 0,
|
358
|
-
costOfHallucinations: 0
|
359
|
-
}
|
360
|
-
},
|
361
|
-
providerComparison: {},
|
362
|
-
trends: {
|
363
|
-
hallucinationRateOverTime: [],
|
364
|
-
qualityScoreOverTime: [],
|
365
|
-
costEfficiencyOverTime: []
|
366
|
-
},
|
367
|
-
recommendations: ['No data available yet. Start using AI through the proxy to see analytics.']
|
368
|
-
};
|
369
|
-
}
|
370
|
-
|
371
|
-
/**
|
372
|
-
* Export analytics to JSON file
|
373
|
-
*/
|
374
|
-
exportAnalytics(filePath?: string): void {
|
375
|
-
const analytics = this.generateAnalytics();
|
376
|
-
const exportPath = filePath || path.resolve(process.cwd(), 'ai-analytics.json');
|
377
|
-
|
378
|
-
fs.writeFileSync(exportPath, JSON.stringify(analytics, null, 2));
|
379
|
-
console.log(`AI analytics exported to ${exportPath}`);
|
380
|
-
}
|
381
|
-
|
382
|
-
/**
|
383
|
-
* Get real-time hallucination rate for a specific provider
|
384
|
-
*/
|
385
|
-
getProviderHallucinationRate(provider: string, hours: number = 24): number {
|
386
|
-
if (!fs.existsSync(this.logFilePath)) return 0;
|
387
|
-
|
388
|
-
const cutoffTime = new Date(Date.now() - hours * 60 * 60 * 1000);
|
389
|
-
const fileContent = fs.readFileSync(this.logFilePath, 'utf8');
|
390
|
-
const lines = fileContent.trim().split('\n');
|
391
|
-
|
392
|
-
let providerInteractions = 0;
|
393
|
-
let providerHallucinations = 0;
|
394
|
-
|
395
|
-
for (const line of lines) {
|
396
|
-
try {
|
397
|
-
const interaction = JSON.parse(line);
|
398
|
-
const interactionTime = new Date(interaction.timestamp);
|
399
|
-
|
400
|
-
if (interactionTime >= cutoffTime && interaction.provider === provider) {
|
401
|
-
providerInteractions++;
|
402
|
-
if (interaction.hallucinationDetection?.isLikelyHallucination) {
|
403
|
-
providerHallucinations++;
|
404
|
-
}
|
405
|
-
}
|
406
|
-
} catch {
|
407
|
-
continue;
|
408
|
-
}
|
409
|
-
}
|
410
|
-
|
411
|
-
return providerInteractions > 0 ? (providerHallucinations / providerInteractions) * 100 : 0;
|
412
|
-
}
|
413
|
-
}
|
414
|
-
|
415
|
-
/**
|
416
|
-
* Global AI analytics instance
|
417
|
-
*/
|
418
|
-
export const aiAnalytics = new AIAnalytics();
|
package/src/auth.ts
DELETED
@@ -1,80 +0,0 @@
|
|
1
|
-
import * as http from 'http';
|
2
|
-
import open from 'open';
|
3
|
-
import chalk from 'chalk';
|
4
|
-
import * as keytar from 'keytar'; // Import keytar
|
5
|
-
|
6
|
-
const CLI_LOGIN_PORT = 8789;
|
7
|
-
const WEB_APP_URL = 'http://localhost:3000';
|
8
|
-
const SERVICE_NAME = 'toknxr-cli'; // A unique name for our service in the keychain
|
9
|
-
const ACCOUNT_NAME = 'default-user'; // A generic account name for the stored token
|
10
|
-
|
11
|
-
// Function to securely store the token
|
12
|
-
const storeToken = async (token: string) => {
|
13
|
-
await keytar.setPassword(SERVICE_NAME, ACCOUNT_NAME, token);
|
14
|
-
console.log(chalk.green('Supabase JWT securely stored in system keychain.'));
|
15
|
-
};
|
16
|
-
|
17
|
-
// Function to retrieve the token
|
18
|
-
const getToken = async (): Promise<string | null> => {
|
19
|
-
return await keytar.getPassword(SERVICE_NAME, ACCOUNT_NAME);
|
20
|
-
};
|
21
|
-
|
22
|
-
export const login = async () => {
|
23
|
-
const server = new Promise<string>((resolve, reject) => {
|
24
|
-
const s = http.createServer(async (req, res) => {
|
25
|
-
// Handle CORS preflight requests
|
26
|
-
res.setHeader('Access-Control-Allow-Origin', WEB_APP_URL);
|
27
|
-
res.setHeader('Access-Control-Allow-Methods', 'POST, OPTIONS');
|
28
|
-
res.setHeader('Access-Control-Allow-Headers', 'Content-Type');
|
29
|
-
|
30
|
-
if (req.method === 'OPTIONS') {
|
31
|
-
res.writeHead(204);
|
32
|
-
res.end();
|
33
|
-
return;
|
34
|
-
}
|
35
|
-
|
36
|
-
if (req.method === 'POST' && req.url === '/token') {
|
37
|
-
const chunks: Buffer[] = [];
|
38
|
-
for await (const chunk of req) {
|
39
|
-
chunks.push(chunk);
|
40
|
-
}
|
41
|
-
const requestBody = Buffer.concat(chunks).toString();
|
42
|
-
const { token: supabaseJwt } = JSON.parse(requestBody); // Supabase JWT
|
43
|
-
|
44
|
-
if (supabaseJwt) {
|
45
|
-
console.log(chalk.green('CLI authentication successful!'));
|
46
|
-
res.writeHead(200, { 'Content-Type': 'application/json' });
|
47
|
-
res.end(JSON.stringify({ success: true }));
|
48
|
-
s.close();
|
49
|
-
resolve(supabaseJwt); // Resolve with the Supabase JWT
|
50
|
-
} else {
|
51
|
-
res.writeHead(400, { 'Content-Type': 'application/json' });
|
52
|
-
res.end(JSON.stringify({ error: 'No token provided' }));
|
53
|
-
s.close();
|
54
|
-
reject(new Error('No token provided'));
|
55
|
-
}
|
56
|
-
} else {
|
57
|
-
res.writeHead(404);
|
58
|
-
res.end();
|
59
|
-
}
|
60
|
-
});
|
61
|
-
|
62
|
-
s.listen(CLI_LOGIN_PORT, async () => {
|
63
|
-
const loginUrl = `${WEB_APP_URL}/cli-login?port=${CLI_LOGIN_PORT}`;
|
64
|
-
console.log(chalk.yellow('Your browser has been opened to complete the login process.'));
|
65
|
-
await open(loginUrl);
|
66
|
-
});
|
67
|
-
});
|
68
|
-
|
69
|
-
try {
|
70
|
-
const supabaseJwt = await server; // Get the Supabase JWT
|
71
|
-
await storeToken(supabaseJwt); // Store the Supabase JWT securely
|
72
|
-
console.log(chalk.cyan('Authentication complete. You can now use TokNxr CLI commands.'));
|
73
|
-
} catch (error: unknown) {
|
74
|
-
const message = error instanceof Error ? error.message : String(error);
|
75
|
-
console.error(chalk.red('Login failed:', message));
|
76
|
-
}
|
77
|
-
};
|
78
|
-
|
79
|
-
// Export getToken for other parts of the CLI to use
|
80
|
-
export { getToken };
|