@goldensheepai/toknxr-cli 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env +21 -0
- package/.env.example +21 -0
- package/README.md +238 -0
- package/interactions.log +8 -0
- package/lib/ai-analytics.js +296 -0
- package/lib/auth.js +73 -0
- package/lib/cli.js +382 -0
- package/lib/code-analysis.js +304 -0
- package/lib/code-review.js +319 -0
- package/lib/config.js +7 -0
- package/lib/dashboard.js +363 -0
- package/lib/hallucination-detector.js +272 -0
- package/lib/policy.js +49 -0
- package/lib/pricing.js +20 -0
- package/lib/proxy.js +359 -0
- package/lib/sync.js +95 -0
- package/package.json +38 -0
- package/src/ai-analytics.ts +418 -0
- package/src/auth.ts +80 -0
- package/src/cli.ts +447 -0
- package/src/code-analysis.ts +365 -0
- package/src/config.ts +10 -0
- package/src/dashboard.tsx +391 -0
- package/src/hallucination-detector.ts +368 -0
- package/src/policy.ts +55 -0
- package/src/pricing.ts +21 -0
- package/src/proxy.ts +438 -0
- package/src/sync.ts +129 -0
- package/start.sh +56 -0
- package/test-analysis.mjs +77 -0
- package/test-coding.mjs +27 -0
- package/test-generate-sample-data.js +118 -0
- package/test-proxy.mjs +25 -0
- package/toknxr.config.json +63 -0
- package/toknxr.policy.json +18 -0
- package/tsconfig.json +19 -0
package/lib/cli.js
ADDED
@@ -0,0 +1,382 @@
|
|
1
|
+
import 'dotenv/config';
|
2
|
+
import { Command } from 'commander';
|
3
|
+
import chalk from 'chalk';
|
4
|
+
import { startProxyServer } from './proxy.js';
|
5
|
+
import { login } from './auth.js';
|
6
|
+
import * as fs from 'node:fs';
|
7
|
+
import * as path from 'node:path';
|
8
|
+
import { createClient } from '@supabase/supabase-js';
|
9
|
+
import open from 'open';
|
10
|
+
import { syncInteractions } from './sync.js';
|
11
|
+
// Gracefully handle broken pipe (e.g., piping output to `head`)
|
12
|
+
process.stdout.on('error', (err) => {
|
13
|
+
if (err && err.code === 'EPIPE')
|
14
|
+
process.exit(0);
|
15
|
+
});
|
16
|
+
process.stderr.on('error', (err) => {
|
17
|
+
if (err && err.code === 'EPIPE')
|
18
|
+
process.exit(0);
|
19
|
+
});
|
20
|
+
const program = new Command();
|
21
|
+
// ASCII Art Welcome Screen with gradient colors
|
22
|
+
const asciiArt = `
|
23
|
+
${chalk.blue(' ████████╗')}${chalk.hex('#6B5BED')(' ██████╗ ')}${chalk.hex('#9B5BED')(' ██╗ ██╗')}${chalk.hex('#CB5BED')(' ███╗ ██╗')}${chalk.hex('#ED5B9B')(' ██╗ ██╗')}${chalk.hex('#ED5B6B')(' ██████╗ ')}
|
24
|
+
${chalk.blue(' ╚══██╔══╝')}${chalk.hex('#6B5BED')(' ██╔═══██╗')}${chalk.hex('#9B5BED')(' ██║ ██╔╝')}${chalk.hex('#CB5BED')(' ████╗ ██║')}${chalk.hex('#ED5B9B')(' ╚██╗██╔╝')}${chalk.hex('#ED5B6B')(' ██╔══██╗')}
|
25
|
+
${chalk.blue(' ██║ ')}${chalk.hex('#6B5BED')(' ██║ ██║')}${chalk.hex('#9B5BED')(' █████╔╝ ')}${chalk.hex('#CB5BED')(' ██╔██╗ ██║')}${chalk.hex('#ED5B9B')(' ╚███╔╝ ')}${chalk.hex('#ED5B6B')(' ██████╔╝')}
|
26
|
+
${chalk.blue(' ██║ ')}${chalk.hex('#6B5BED')(' ██║ ██║')}${chalk.hex('#9B5BED')(' ██╔═██╗ ')}${chalk.hex('#CB5BED')(' ██║╚██╗██║')}${chalk.hex('#ED5B9B')(' ██╔██╗ ')}${chalk.hex('#ED5B6B')(' ██╔══██╗')}
|
27
|
+
${chalk.blue(' ██║ ')}${chalk.hex('#6B5BED')(' ╚██████╔╝')}${chalk.hex('#9B5BED')(' ██║ ██╗')}${chalk.hex('#CB5BED')(' ██║ ╚████║')}${chalk.hex('#ED5B9B')(' ██╔╝ ██╗')}${chalk.hex('#ED5B6B')(' ██║ ██║')}
|
28
|
+
${chalk.blue(' ╚═╝ ')}${chalk.hex('#6B5BED')(' ╚═════╝ ')}${chalk.hex('#9B5BED')(' ╚═╝ ╚═╝')}${chalk.hex('#CB5BED')(' ╚═╝ ╚═══╝')}${chalk.hex('#ED5B9B')(' ╚═╝ ╚═╝')}${chalk.hex('#ED5B6B')(' ╚═╝ ╚═╝')}
|
29
|
+
|
30
|
+
${chalk.cyan('Tips for getting started:')}
|
31
|
+
${chalk.white('1. Start tracking:')} ${chalk.yellow('toknxr start')} ${chalk.gray('- Launch the proxy server')}
|
32
|
+
${chalk.white('2. View analytics:')} ${chalk.yellow('toknxr stats')} ${chalk.gray('- See token usage and code quality')}
|
33
|
+
${chalk.white('3. Deep dive:')} ${chalk.yellow('toknxr code-analysis')} ${chalk.gray('- Detailed quality insights')}
|
34
|
+
${chalk.white('4. Code review:')} ${chalk.yellow('toknxr review')} ${chalk.gray('- Review AI-generated code')}
|
35
|
+
${chalk.white('5. Login:')} ${chalk.yellow('toknxr login')} ${chalk.gray('- Authenticate with your account')}
|
36
|
+
${chalk.white('6. Set limits:')} ${chalk.yellow('toknxr policy:init')} ${chalk.gray('- Configure spending policies')}
|
37
|
+
${chalk.white('7. Need help?')} ${chalk.yellow('toknxr --help')} ${chalk.gray('- View all commands')}
|
38
|
+
|
39
|
+
${chalk.gray('━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━')}
|
40
|
+
|
41
|
+
${chalk.hex('#FFD700')('🐑 Powered by Golden Sheep AI')}
|
42
|
+
|
43
|
+
`;
|
44
|
+
console.log(asciiArt);
|
45
|
+
// --- Supabase Client ---
|
46
|
+
const supabaseUrl = process.env.SUPABASE_URL || '';
|
47
|
+
const supabaseKey = process.env.SUPABASE_KEY || '';
|
48
|
+
if (!supabaseUrl || !supabaseKey) {
|
49
|
+
console.error(chalk.red('Error: Supabase URL or Key not found in environment variables.'));
|
50
|
+
process.exit(1);
|
51
|
+
}
|
52
|
+
const supabase = createClient(supabaseUrl, supabaseKey);
|
53
|
+
program
|
54
|
+
.name('toknxr')
|
55
|
+
.description('AI Effectiveness & Code Quality Analysis CLI')
|
56
|
+
.version('0.1.0');
|
57
|
+
program
|
58
|
+
.command('start')
|
59
|
+
.description('Start the TokNxr proxy server to monitor AI interactions.')
|
60
|
+
.action(() => {
|
61
|
+
console.log(chalk.green('Starting TokNxr proxy server...'));
|
62
|
+
startProxyServer();
|
63
|
+
});
|
64
|
+
program
|
65
|
+
.command('sync')
|
66
|
+
.description('Sync local interaction logs to the Supabase dashboard.')
|
67
|
+
.option('--clear', 'Clear the log file after a successful sync.')
|
68
|
+
.action(async (options) => {
|
69
|
+
await syncInteractions(supabase, options);
|
70
|
+
});
|
71
|
+
program
|
72
|
+
.command('stats')
|
73
|
+
.description('Display token usage statistics from the local log.')
|
74
|
+
.action(() => {
|
75
|
+
const logFilePath = path.resolve(process.cwd(), 'interactions.log');
|
76
|
+
if (!fs.existsSync(logFilePath)) {
|
77
|
+
console.log(chalk.yellow('No interactions logged yet. Use the `start` command to begin tracking.'));
|
78
|
+
return;
|
79
|
+
}
|
80
|
+
const fileContent = fs.readFileSync(logFilePath, 'utf8');
|
81
|
+
const lines = fileContent.trim().split('\n');
|
82
|
+
const interactions = lines.map(line => {
|
83
|
+
try {
|
84
|
+
return JSON.parse(line);
|
85
|
+
}
|
86
|
+
catch (error) {
|
87
|
+
console.warn(`Skipping invalid log entry: ${line}`);
|
88
|
+
return null;
|
89
|
+
}
|
90
|
+
}).filter((interaction) => interaction !== null);
|
91
|
+
const stats = interactions.reduce((acc, interaction) => {
|
92
|
+
if (!acc[interaction.provider]) {
|
93
|
+
acc[interaction.provider] = {
|
94
|
+
totalTokens: 0, promptTokens: 0, completionTokens: 0, requestCount: 0, costUSD: 0,
|
95
|
+
codingCount: 0, avgQualityScore: 0, avgEffectivenessScore: 0, qualitySum: 0, effectivenessSum: 0
|
96
|
+
};
|
97
|
+
}
|
98
|
+
acc[interaction.provider].totalTokens += interaction.totalTokens;
|
99
|
+
acc[interaction.provider].promptTokens += interaction.promptTokens;
|
100
|
+
acc[interaction.provider].completionTokens += interaction.completionTokens;
|
101
|
+
acc[interaction.provider].requestCount += 1;
|
102
|
+
acc[interaction.provider].costUSD += interaction.costUSD || 0;
|
103
|
+
if (interaction.taskType === 'coding') {
|
104
|
+
acc[interaction.provider].codingCount += 1;
|
105
|
+
if (interaction.codeQualityScore !== undefined) {
|
106
|
+
acc[interaction.provider].qualitySum += interaction.codeQualityScore;
|
107
|
+
}
|
108
|
+
if (interaction.effectivenessScore !== undefined) {
|
109
|
+
acc[interaction.provider].effectivenessSum += interaction.effectivenessScore;
|
110
|
+
}
|
111
|
+
}
|
112
|
+
return acc;
|
113
|
+
}, {});
|
114
|
+
// Calculate averages
|
115
|
+
for (const provider in stats) {
|
116
|
+
const p = stats[provider];
|
117
|
+
if (p.codingCount > 0) {
|
118
|
+
p.avgQualityScore = Math.round(p.qualitySum / p.codingCount);
|
119
|
+
p.avgEffectivenessScore = Math.round(p.effectivenessSum / p.codingCount);
|
120
|
+
}
|
121
|
+
}
|
122
|
+
const grandTotals = Object.values(stats).reduce((acc, s) => {
|
123
|
+
acc.totalTokens += s.totalTokens;
|
124
|
+
acc.promptTokens += s.promptTokens;
|
125
|
+
acc.completionTokens += s.completionTokens;
|
126
|
+
acc.requestCount += s.requestCount;
|
127
|
+
acc.costUSD += s.costUSD;
|
128
|
+
acc.codingCount += s.codingCount;
|
129
|
+
acc.qualitySum += s.qualitySum;
|
130
|
+
acc.effectivenessSum += s.effectivenessSum;
|
131
|
+
return acc;
|
132
|
+
}, { totalTokens: 0, promptTokens: 0, completionTokens: 0, requestCount: 0, costUSD: 0, codingCount: 0, qualitySum: 0, effectivenessSum: 0 });
|
133
|
+
// Calculate grand averages
|
134
|
+
const codingTotal = grandTotals.codingCount;
|
135
|
+
const avgQuality = codingTotal > 0 ? Math.round(grandTotals.qualitySum / codingTotal) : 0;
|
136
|
+
const avgEffectiveness = codingTotal > 0 ? Math.round(grandTotals.effectivenessSum / codingTotal) : 0;
|
137
|
+
console.log(chalk.bold.underline('Token Usage Statistics'));
|
138
|
+
for (const provider in stats) {
|
139
|
+
console.log(chalk.bold(`\nProvider: ${provider}`));
|
140
|
+
console.log(` Total Requests: ${stats[provider].requestCount}`);
|
141
|
+
console.log(chalk.cyan(` Total Tokens: ${stats[provider].totalTokens}`));
|
142
|
+
console.log(` - Prompt Tokens: ${stats[provider].promptTokens}`);
|
143
|
+
console.log(` - Completion Tokens: ${stats[provider].completionTokens}`);
|
144
|
+
console.log(chalk.green(` Cost (USD): $${(stats[provider].costUSD).toFixed(4)}`));
|
145
|
+
if (stats[provider].codingCount > 0) {
|
146
|
+
console.log(chalk.blue(` Code Quality: ${stats[provider].avgQualityScore}/100 (avg)`));
|
147
|
+
console.log(chalk.magenta(` Effectiveness: ${stats[provider].avgEffectivenessScore}/100 (avg, ${stats[provider].codingCount} coding requests)`));
|
148
|
+
}
|
149
|
+
}
|
150
|
+
console.log(chalk.bold(`\nGrand Totals`));
|
151
|
+
console.log(` Requests: ${grandTotals.requestCount}`);
|
152
|
+
console.log(chalk.cyan(` Tokens: ${grandTotals.totalTokens}`));
|
153
|
+
console.log(` - Prompt: ${grandTotals.promptTokens}`);
|
154
|
+
console.log(` - Completion: ${grandTotals.completionTokens}`);
|
155
|
+
console.log(chalk.green(` Cost (USD): $${(grandTotals.costUSD).toFixed(4)}`));
|
156
|
+
if (codingTotal > 0) {
|
157
|
+
console.log(`\n${chalk.bold('Code Quality Insights:')}`);
|
158
|
+
console.log(chalk.blue(` Coding Requests: ${codingTotal}`));
|
159
|
+
console.log(chalk.blue(` Avg Code Quality: ${avgQuality}/100`));
|
160
|
+
console.log(chalk.magenta(` Avg Effectiveness: ${avgEffectiveness}/100`));
|
161
|
+
if (avgQuality < 70) {
|
162
|
+
console.log(chalk.red(' ⚠️ Low code quality - consider reviewing AI-generated code more carefully'));
|
163
|
+
}
|
164
|
+
if (avgEffectiveness < 70) {
|
165
|
+
console.log(chalk.red(' ⚠️ Low effectiveness - prompts may need improvement or different AI model'));
|
166
|
+
}
|
167
|
+
}
|
168
|
+
});
|
169
|
+
program
|
170
|
+
.command('init')
|
171
|
+
.description('Scaffold .env and toknxr.config.json in the current directory')
|
172
|
+
.action(() => {
|
173
|
+
const envPath = path.resolve(process.cwd(), '.env');
|
174
|
+
if (!fs.existsSync(envPath)) {
|
175
|
+
fs.writeFileSync(envPath, 'GEMINI_API_KEY=\n');
|
176
|
+
console.log(chalk.green(`Created ${envPath}`));
|
177
|
+
}
|
178
|
+
else {
|
179
|
+
console.log(chalk.yellow(`Skipped ${envPath} (exists)`));
|
180
|
+
}
|
181
|
+
const configPath = path.resolve(process.cwd(), 'toknxr.config.json');
|
182
|
+
if (!fs.existsSync(configPath)) {
|
183
|
+
const config = {
|
184
|
+
providers: [
|
185
|
+
{
|
186
|
+
name: 'Gemini-Pro',
|
187
|
+
routePrefix: '/gemini',
|
188
|
+
targetUrl: 'https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash:generateContent',
|
189
|
+
apiKeyEnvVar: 'GEMINI_API_KEY',
|
190
|
+
authHeader: 'x-goog-api-key',
|
191
|
+
tokenMapping: {
|
192
|
+
prompt: 'usageMetadata.promptTokenCount',
|
193
|
+
completion: 'usageMetadata.candidatesTokenCount',
|
194
|
+
total: 'usageMetadata.totalTokenCount'
|
195
|
+
}
|
196
|
+
}
|
197
|
+
]
|
198
|
+
};
|
199
|
+
fs.writeFileSync(configPath, JSON.stringify(config, null, 2));
|
200
|
+
console.log(chalk.green(`Created ${configPath}`));
|
201
|
+
}
|
202
|
+
else {
|
203
|
+
console.log(chalk.yellow(`Skipped ${configPath} (exists)`));
|
204
|
+
}
|
205
|
+
const policyPath = path.resolve(process.cwd(), 'toknxr.policy.json');
|
206
|
+
if (!fs.existsSync(policyPath)) {
|
207
|
+
const policy = {
|
208
|
+
version: '1',
|
209
|
+
monthlyUSD: 50,
|
210
|
+
perProviderMonthlyUSD: { 'Gemini-Pro': 30 },
|
211
|
+
webhookUrl: ''
|
212
|
+
};
|
213
|
+
fs.writeFileSync(policyPath, JSON.stringify(policy, null, 2));
|
214
|
+
console.log(chalk.green(`Created ${policyPath}`));
|
215
|
+
}
|
216
|
+
else {
|
217
|
+
console.log(chalk.yellow(`Skipped ${policyPath} (exists)`));
|
218
|
+
}
|
219
|
+
});
|
220
|
+
program
|
221
|
+
.command('tail')
|
222
|
+
.description('Follow interactions.log and pretty-print new lines')
|
223
|
+
.action(() => {
|
224
|
+
const logFilePath = path.resolve(process.cwd(), 'interactions.log');
|
225
|
+
if (!fs.existsSync(logFilePath)) {
|
226
|
+
console.log(chalk.yellow('No interactions.log found. Start the proxy first.'));
|
227
|
+
return;
|
228
|
+
}
|
229
|
+
console.log(chalk.gray(`Tailing ${logFilePath}... (Ctrl+C to stop)`));
|
230
|
+
fs.watchFile(logFilePath, { interval: 500 }, () => {
|
231
|
+
const content = fs.readFileSync(logFilePath, 'utf8').trim();
|
232
|
+
const lines = content.split('\n');
|
233
|
+
const last = lines[lines.length - 1];
|
234
|
+
try {
|
235
|
+
const j = JSON.parse(last);
|
236
|
+
console.log(`${chalk.bold(j.provider)} ${chalk.gray(j.timestamp)} id=${j.requestId} model=${j.model} tokens=${j.totalTokens} cost=$${(j.costUSD || 0).toFixed(4)}`);
|
237
|
+
}
|
238
|
+
catch {
|
239
|
+
console.log(last);
|
240
|
+
}
|
241
|
+
});
|
242
|
+
});
|
243
|
+
program
|
244
|
+
.command('dashboard')
|
245
|
+
.description('Open the minimal dashboard served by the proxy (/dashboard)')
|
246
|
+
.action(async () => {
|
247
|
+
const url = 'http://localhost:3000/dashboard'; // Assuming Next.js app serves the dashboard
|
248
|
+
console.log(chalk.gray(`Opening ${url}...`));
|
249
|
+
await open(url);
|
250
|
+
});
|
251
|
+
program
|
252
|
+
.command('policy:init')
|
253
|
+
.description('Scaffold toknxr.policy.json from the foundation starter pack if missing')
|
254
|
+
.action(() => {
|
255
|
+
const dest = path.resolve(process.cwd(), 'toknxr.policy.json');
|
256
|
+
if (fs.existsSync(dest)) {
|
257
|
+
console.log(chalk.yellow(`Skipped ${dest} (exists)`));
|
258
|
+
return;
|
259
|
+
}
|
260
|
+
// Fallback scaffold using sensible defaults if starter pack path is unavailable
|
261
|
+
const fallback = {
|
262
|
+
version: '1',
|
263
|
+
monthlyUSD: 50,
|
264
|
+
perProviderMonthlyUSD: { 'Gemini-Pro': 30 },
|
265
|
+
webhookUrl: ''
|
266
|
+
};
|
267
|
+
fs.writeFileSync(dest, JSON.stringify(fallback, null, 2));
|
268
|
+
console.log(chalk.green(`Created ${dest}`));
|
269
|
+
});
|
270
|
+
program
|
271
|
+
.command('code-analysis')
|
272
|
+
.description('Show detailed code quality analysis from coding interactions')
|
273
|
+
.action(() => {
|
274
|
+
const logFilePath = path.resolve(process.cwd(), 'interactions.log');
|
275
|
+
if (!fs.existsSync(logFilePath)) {
|
276
|
+
console.log(chalk.yellow('No interactions logged yet. Use the `start` command to begin tracking.'));
|
277
|
+
return;
|
278
|
+
}
|
279
|
+
const fileContent = fs.readFileSync(logFilePath, 'utf8');
|
280
|
+
const lines = fileContent.trim().split('\n');
|
281
|
+
const interactions = lines.map(line => {
|
282
|
+
try {
|
283
|
+
return JSON.parse(line);
|
284
|
+
}
|
285
|
+
catch (error) {
|
286
|
+
console.warn(`Skipping invalid log entry: ${line}`);
|
287
|
+
return null;
|
288
|
+
}
|
289
|
+
}).filter((interaction) => interaction !== null);
|
290
|
+
if (interactions.length === 0) {
|
291
|
+
console.log(chalk.yellow('No coding interactions found. Code analysis requires coding requests to the proxy.'));
|
292
|
+
return;
|
293
|
+
}
|
294
|
+
console.log(chalk.bold.underline('AI Code Quality Analysis'));
|
295
|
+
// Language distribution
|
296
|
+
const langStats = interactions.reduce((acc, i) => {
|
297
|
+
const lang = i.codeQualityMetrics?.language || 'unknown';
|
298
|
+
if (!acc[lang])
|
299
|
+
acc[lang] = 0;
|
300
|
+
acc[lang]++;
|
301
|
+
return acc;
|
302
|
+
}, {});
|
303
|
+
console.log(chalk.bold('\nLanguage Distribution:'));
|
304
|
+
for (const [lang, count] of Object.entries(langStats)) {
|
305
|
+
console.log(` ${lang}: ${count} requests`);
|
306
|
+
}
|
307
|
+
// Quality score distribution
|
308
|
+
const qualityRanges = { excellent: 0, good: 0, fair: 0, poor: 0 };
|
309
|
+
const effectivenessRanges = { excellent: 0, good: 0, fair: 0, poor: 0 };
|
310
|
+
interactions.forEach((i) => {
|
311
|
+
const q = i.codeQualityScore || 0;
|
312
|
+
const e = i.effectivenessScore || 0;
|
313
|
+
if (q >= 90)
|
314
|
+
qualityRanges.excellent++;
|
315
|
+
else if (q >= 75)
|
316
|
+
qualityRanges.good++;
|
317
|
+
else if (q >= 60)
|
318
|
+
qualityRanges.fair++;
|
319
|
+
else
|
320
|
+
qualityRanges.poor++;
|
321
|
+
if (e >= 90)
|
322
|
+
effectivenessRanges.excellent++;
|
323
|
+
else if (e >= 75)
|
324
|
+
effectivenessRanges.good++;
|
325
|
+
else if (e >= 60)
|
326
|
+
effectivenessRanges.fair++;
|
327
|
+
else
|
328
|
+
effectivenessRanges.poor++;
|
329
|
+
});
|
330
|
+
console.log(chalk.bold('\nCode Quality Scores:'));
|
331
|
+
console.log(chalk.green(` Excellent (90-100): ${qualityRanges.excellent}`));
|
332
|
+
console.log(chalk.blue(` Good (75-89): ${qualityRanges.good}`));
|
333
|
+
console.log(chalk.yellow(` Fair (60-74): ${qualityRanges.fair}`));
|
334
|
+
console.log(chalk.red(` Poor (0-59): ${qualityRanges.poor}`));
|
335
|
+
console.log(chalk.bold('\nEffectiveness Scores (Prompt ↔ Result):'));
|
336
|
+
console.log(chalk.green(` Excellent (90-100): ${effectivenessRanges.excellent}`));
|
337
|
+
console.log(chalk.blue(` Good (75-89): ${effectivenessRanges.good}`));
|
338
|
+
console.log(chalk.yellow(` Fair (60-74): ${effectivenessRanges.fair}`));
|
339
|
+
console.log(chalk.red(` Poor (0-59): ${effectivenessRanges.poor}`));
|
340
|
+
// Recent examples with low scores
|
341
|
+
const lowQuality = interactions.filter((i) => (i.codeQualityScore || 0) < 70).slice(-3);
|
342
|
+
if (lowQuality.length > 0) {
|
343
|
+
console.log(chalk.bold('\n🔍 Recent Low-Quality Code Examples:'));
|
344
|
+
lowQuality.forEach((i, idx) => {
|
345
|
+
console.log(`\n${idx + 1}. Quality: ${i.codeQualityScore}/100${i.effectivenessScore ? ` | Effectiveness: ${i.effectivenessScore}/100` : ''}`);
|
346
|
+
console.log(` Provider: ${i.provider} | Model: ${i.model}`);
|
347
|
+
if (i.userPrompt) {
|
348
|
+
const prompt = i.userPrompt.substring(0, 100);
|
349
|
+
console.log(` Prompt: ${prompt}${i.userPrompt.length > 100 ? '...' : ''}`);
|
350
|
+
}
|
351
|
+
if (i.codeQualityMetrics && i.codeQualityMetrics.potentialIssues && i.codeQualityMetrics.potentialIssues.length > 0) {
|
352
|
+
console.log(` Issues: ${i.codeQualityMetrics.potentialIssues.join(', ')}`);
|
353
|
+
}
|
354
|
+
});
|
355
|
+
}
|
356
|
+
// Improvement suggestions
|
357
|
+
const avgQuality = interactions.reduce((sum, i) => sum + (i.codeQualityScore || 0), 0) / interactions.length;
|
358
|
+
const avgEffectiveness = interactions.reduce((sum, i) => sum + (i.effectivenessScore || 0), 0) / interactions.length;
|
359
|
+
console.log(chalk.bold('\n💡 Improvement Suggestions:'));
|
360
|
+
if (avgQuality < 70) {
|
361
|
+
console.log(' • Consider reviewing AI-generated code more carefully before use');
|
362
|
+
console.log(' • Try more specific, detailed prompts for complex tasks');
|
363
|
+
}
|
364
|
+
if (avgEffectiveness < 70) {
|
365
|
+
console.log(' • Improve prompt clarity - be more specific about requirements');
|
366
|
+
console.log(' • Consider using different AI models for different types of tasks');
|
367
|
+
console.log(' • Break complex requests into smaller, focused prompts');
|
368
|
+
}
|
369
|
+
if (avgQuality >= 80 && avgEffectiveness >= 80) {
|
370
|
+
console.log(' • Great! Your AI coding setup is working well');
|
371
|
+
console.log(' • Consider establishing code review processes for edge cases');
|
372
|
+
}
|
373
|
+
console.log(`\n${chalk.gray('Total coding interactions analyzed: ' + interactions.length)}`);
|
374
|
+
});
|
375
|
+
program
|
376
|
+
.command('login')
|
377
|
+
.description('Authenticate with your TokNxr account')
|
378
|
+
.action(async () => {
|
379
|
+
console.log(chalk.blue('Starting CLI authentication process...'));
|
380
|
+
await login();
|
381
|
+
});
|
382
|
+
program.parse(process.argv);
|
@@ -0,0 +1,304 @@
|
|
1
|
+
/**
|
2
|
+
* Analyzes code quality and provides metrics
|
3
|
+
*/
|
4
|
+
export function analyzeCodeQuality(code, language) {
|
5
|
+
const metrics = {
|
6
|
+
syntaxValid: true,
|
7
|
+
linesOfCode: 0,
|
8
|
+
complexity: 0,
|
9
|
+
hasFunctions: false,
|
10
|
+
hasClasses: false,
|
11
|
+
hasTests: false,
|
12
|
+
estimatedReadability: 5,
|
13
|
+
potentialIssues: [],
|
14
|
+
language: detectLanguage(code) || language
|
15
|
+
};
|
16
|
+
if (!code || code.trim().length === 0) {
|
17
|
+
return metrics;
|
18
|
+
}
|
19
|
+
const lines = code.split('\n').filter(l => l.trim());
|
20
|
+
metrics.linesOfCode = lines.length;
|
21
|
+
// Detect language if not provided
|
22
|
+
const detectedLang = metrics.language || 'unknown';
|
23
|
+
// Basic complexity analysis
|
24
|
+
metrics.complexity = calculateComplexity(lines, detectedLang);
|
25
|
+
// Check for structural elements based on language
|
26
|
+
switch (detectedLang) {
|
27
|
+
case 'javascript':
|
28
|
+
case 'typescript':
|
29
|
+
metrics.hasFunctions = /function\s+\w+|const\s+\w+\s*=.*=>|class\s+\w+/.test(code);
|
30
|
+
metrics.hasClasses = /class\s+\w+/.test(code);
|
31
|
+
metrics.hasTests = /describe|test|it\(/.test(code);
|
32
|
+
// Syntax validation (basic)
|
33
|
+
metrics.syntaxValid = validateJSTypescript(code, detectedLang);
|
34
|
+
// Readability scoring
|
35
|
+
metrics.estimatedReadability = calculateJSReadability(code);
|
36
|
+
break;
|
37
|
+
case 'python':
|
38
|
+
metrics.hasFunctions = /def\s+\w+/.test(code);
|
39
|
+
metrics.hasClasses = /class\s+\w+/.test(code);
|
40
|
+
metrics.hasTests = /def test_|unittest|pytest/.test(code);
|
41
|
+
metrics.syntaxValid = validatePython(code);
|
42
|
+
metrics.estimatedReadability = calculatePythonReadability(code);
|
43
|
+
break;
|
44
|
+
default:
|
45
|
+
metrics.potentialIssues.push('Language not recognized for detailed analysis');
|
46
|
+
}
|
47
|
+
// Common issues
|
48
|
+
if (lines.length > 100) {
|
49
|
+
metrics.potentialIssues.push('Very long file - consider splitting');
|
50
|
+
}
|
51
|
+
if (!metrics.syntaxValid) {
|
52
|
+
metrics.potentialIssues.push('Potential syntax errors detected');
|
53
|
+
}
|
54
|
+
return metrics;
|
55
|
+
}
|
56
|
+
/**
|
57
|
+
* Calculate an effectiveness score comparing prompt to code output
|
58
|
+
*/
|
59
|
+
export function scoreEffectiveness(userPrompt, aiResponse, extractedCode) {
|
60
|
+
const code = extractedCode || aiResponse;
|
61
|
+
const score = {
|
62
|
+
promptClarityMatch: 0,
|
63
|
+
codeCompleteness: 0,
|
64
|
+
codeCorrectness: 0,
|
65
|
+
codeEfficiency: 0,
|
66
|
+
overallEffectiveness: 0
|
67
|
+
};
|
68
|
+
if (!userPrompt || !code)
|
69
|
+
return score;
|
70
|
+
const prompt = userPrompt.toLowerCase();
|
71
|
+
const response = aiResponse.toLowerCase();
|
72
|
+
// Analyze how well the AI understood the request
|
73
|
+
score.promptClarityMatch = analyzePromptMatch(prompt, response, code);
|
74
|
+
// Analyze code quality and completeness
|
75
|
+
const metrics = analyzeCodeQuality(code);
|
76
|
+
score.codeCorrectness = calculateCorrectnessScore(metrics);
|
77
|
+
score.codeCompleteness = analyzeCompleteness(prompt, code);
|
78
|
+
score.codeEfficiency = estimateEfficiencyScore(code, metrics);
|
79
|
+
// Calculate overall effectiveness (weighted average)
|
80
|
+
score.overallEffectiveness = Math.round((score.promptClarityMatch * 0.3) +
|
81
|
+
(score.codeCompleteness * 0.25) +
|
82
|
+
(score.codeCorrectness * 0.25) +
|
83
|
+
(score.codeEfficiency * 0.2));
|
84
|
+
return score;
|
85
|
+
}
|
86
|
+
/**
|
87
|
+
* Extract code blocks from AI responses
|
88
|
+
*/
|
89
|
+
export function extractCodeFromResponse(response) {
|
90
|
+
// Common code block patterns
|
91
|
+
const codeBlockRegex = /```(\w+)?\n?([\s\S]*?)```/g;
|
92
|
+
const match = codeBlockRegex.exec(response);
|
93
|
+
if (match) {
|
94
|
+
return {
|
95
|
+
code: match[2].trim(),
|
96
|
+
language: match[1]?.toLowerCase()
|
97
|
+
};
|
98
|
+
}
|
99
|
+
// Look for inline code that might be a complete solution
|
100
|
+
const inlineCodeRegex = /`([^`\n]+)`/g;
|
101
|
+
const inlineMatches = Array.from(response.matchAll(inlineCodeRegex));
|
102
|
+
if (inlineMatches.length > 0) {
|
103
|
+
// If multiple inline codes, combine them
|
104
|
+
const combined = inlineMatches.map(m => m[1]).join('\n\n');
|
105
|
+
if (combined.length > 50) { // Arbitrary threshold for "substantial" code
|
106
|
+
return { code: combined };
|
107
|
+
}
|
108
|
+
}
|
109
|
+
return null;
|
110
|
+
}
|
111
|
+
/**
|
112
|
+
* Detect programming language from code content
|
113
|
+
*/
|
114
|
+
function detectLanguage(code) {
|
115
|
+
if (/(?:import|export|function|const|let|var)\s+/.test(code)) {
|
116
|
+
return code.includes('interface') || code.includes(': string') ? 'typescript' : 'javascript';
|
117
|
+
}
|
118
|
+
if (/def\s+|import\s+|class\s+/.test(code) && /:/.test(code)) {
|
119
|
+
return 'python';
|
120
|
+
}
|
121
|
+
// Add more language detection as needed...
|
122
|
+
return undefined;
|
123
|
+
}
|
124
|
+
/**
|
125
|
+
* Calculate code complexity score
|
126
|
+
*/
|
127
|
+
function calculateComplexity(lines, language) {
|
128
|
+
let complexity = 1; // Base complexity
|
129
|
+
for (const line of lines) {
|
130
|
+
// Control flow increases complexity
|
131
|
+
if (/(if|for|while|switch|try|catch)/.test(line)) {
|
132
|
+
complexity += 0.5;
|
133
|
+
}
|
134
|
+
// Nested blocks
|
135
|
+
const indentLevel = line.length - line.trimStart().length;
|
136
|
+
complexity += indentLevel * 0.1;
|
137
|
+
// Function/class definitions
|
138
|
+
if (/(function|def|class)/.test(line)) {
|
139
|
+
complexity += 1;
|
140
|
+
}
|
141
|
+
}
|
142
|
+
return Math.min(complexity, 10); // Cap at 10
|
143
|
+
}
|
144
|
+
/**
|
145
|
+
* Basic JavaScript/TypeScript validation
|
146
|
+
*/
|
147
|
+
function validateJSTypescript(code, lang) {
|
148
|
+
try {
|
149
|
+
// Basic bracket matching
|
150
|
+
const brackets = { '(': 0, '[': 0, '{': 0 };
|
151
|
+
for (const char of code) {
|
152
|
+
if (char === '(')
|
153
|
+
brackets['(']++;
|
154
|
+
if (char === ')')
|
155
|
+
brackets['(']--;
|
156
|
+
if (char === '[')
|
157
|
+
brackets['[']++;
|
158
|
+
if (char === ']')
|
159
|
+
brackets['[']--;
|
160
|
+
if (char === '{')
|
161
|
+
brackets['{']++;
|
162
|
+
if (char === '}')
|
163
|
+
brackets['{']--;
|
164
|
+
if (brackets['('] < 0 || brackets['['] < 0 || brackets['{'] < 0) {
|
165
|
+
return false;
|
166
|
+
}
|
167
|
+
}
|
168
|
+
return brackets['('] === 0 && brackets['['] === 0 && brackets['{'] === 0;
|
169
|
+
}
|
170
|
+
catch {
|
171
|
+
return false;
|
172
|
+
}
|
173
|
+
}
|
174
|
+
/**
|
175
|
+
* Basic Python validation
|
176
|
+
*/
|
177
|
+
function validatePython(code) {
|
178
|
+
// Basic indentation check
|
179
|
+
const lines = code.split('\n');
|
180
|
+
let indentLevel = 0;
|
181
|
+
for (const line of lines) {
|
182
|
+
const trimmed = line.trim();
|
183
|
+
if (!trimmed || trimmed.startsWith('#'))
|
184
|
+
continue;
|
185
|
+
const currentIndent = line.length - line.trimStart().length;
|
186
|
+
if (currentIndent > indentLevel + 4)
|
187
|
+
return false; // Too much indentation
|
188
|
+
indentLevel = currentIndent;
|
189
|
+
}
|
190
|
+
return true;
|
191
|
+
}
|
192
|
+
/**
|
193
|
+
* Calculate readability for JS/TS
|
194
|
+
*/
|
195
|
+
function calculateJSReadability(code) {
|
196
|
+
let score = 5; // Base score
|
197
|
+
// Length factors
|
198
|
+
if (code.length > 2000)
|
199
|
+
score -= 2;
|
200
|
+
if (code.split('\n').length > 50)
|
201
|
+
score -= 1;
|
202
|
+
// Good practices
|
203
|
+
if (code.includes('//') || code.includes('/*'))
|
204
|
+
score += 1; // Has comments
|
205
|
+
if (/\w+_\w+/.test(code))
|
206
|
+
score -= 1; // Poor naming (underscores in JS)
|
207
|
+
if (/[a-z][A-Z]/.test(code.replace(/const|let|var/g, '')))
|
208
|
+
score += 1; // camelCase
|
209
|
+
return Math.max(1, Math.min(10, score));
|
210
|
+
}
|
211
|
+
/**
|
212
|
+
* Calculate readability for Python
|
213
|
+
*/
|
214
|
+
function calculatePythonReadability(code) {
|
215
|
+
let score = 7; // Python typically more readable
|
216
|
+
// Length factors
|
217
|
+
if (code.length > 1500)
|
218
|
+
score -= 1.5;
|
219
|
+
if (code.split('\n').length > 40)
|
220
|
+
score -= 1;
|
221
|
+
// Good practices
|
222
|
+
if (code.includes('#'))
|
223
|
+
score += 0.5; // Has comments
|
224
|
+
if (/"""[\s\S]*?"""/.test(code))
|
225
|
+
score += 1; // Has docstrings
|
226
|
+
if (/_/.test(code.replace(/__\w+__/g, '')))
|
227
|
+
score -= 0.5; // Uses underscores (good)
|
228
|
+
return Math.max(1, Math.min(10, score));
|
229
|
+
}
|
230
|
+
/**
|
231
|
+
* Analyze how well AI response matches the user's prompt
|
232
|
+
*/
|
233
|
+
function analyzePromptMatch(prompt, response, code) {
|
234
|
+
let match = 0;
|
235
|
+
// Keywords from prompt appearing in code
|
236
|
+
const promptWords = prompt.split(/\s+/).filter(w => w.length > 3);
|
237
|
+
const codeContent = code.toLowerCase();
|
238
|
+
const matchingWords = promptWords.filter(word => codeContent.includes(word.toLowerCase()));
|
239
|
+
match += (matchingWords.length / Math.max(promptWords.length, 1)) * 40;
|
240
|
+
// Check if response acknowledges the request
|
241
|
+
if (response.includes('here') || response.includes('below') || response.includes('code')) {
|
242
|
+
match += 20;
|
243
|
+
}
|
244
|
+
// Has explanation or context
|
245
|
+
if (response.length > code.length * 2) {
|
246
|
+
match += 15;
|
247
|
+
}
|
248
|
+
return Math.min(100, match);
|
249
|
+
}
|
250
|
+
/**
|
251
|
+
* Calculate correctness score from metrics
|
252
|
+
*/
|
253
|
+
function calculateCorrectnessScore(metrics) {
|
254
|
+
let score = 50; // Base score
|
255
|
+
if (metrics.syntaxValid)
|
256
|
+
score += 25;
|
257
|
+
else
|
258
|
+
score -= 20;
|
259
|
+
if (metrics.estimatedReadability > 6)
|
260
|
+
score += 15;
|
261
|
+
else if (metrics.estimatedReadability < 4)
|
262
|
+
score -= 10;
|
263
|
+
if (metrics.potentialIssues.length === 0)
|
264
|
+
score += 10;
|
265
|
+
return Math.max(0, Math.min(100, score));
|
266
|
+
}
|
267
|
+
/**
|
268
|
+
* Analyze code completeness
|
269
|
+
*/
|
270
|
+
function analyzeCompleteness(prompt, code) {
|
271
|
+
let completeness = 50; // Base
|
272
|
+
// Look for completion indicators
|
273
|
+
if (/(function|def|class)\s+\w+/.test(code))
|
274
|
+
completeness += 20;
|
275
|
+
if (/(return|yield|export)/.test(code))
|
276
|
+
completeness += 15;
|
277
|
+
// Check if code has implementation (not just skeleton)
|
278
|
+
const lines = code.split('\n').length;
|
279
|
+
if (lines > 10)
|
280
|
+
completeness += 10;
|
281
|
+
if (/(todo|fixme|implement)/i.test(code))
|
282
|
+
completeness -= 15;
|
283
|
+
return Math.max(0, Math.min(100, completeness));
|
284
|
+
}
|
285
|
+
/**
|
286
|
+
* Estimate efficiency score
|
287
|
+
*/
|
288
|
+
function estimateEfficiencyScore(code, metrics) {
|
289
|
+
let score = 60; // Base
|
290
|
+
// Complexity affects efficiency
|
291
|
+
if (metrics.complexity < 3)
|
292
|
+
score += 20;
|
293
|
+
else if (metrics.complexity > 7)
|
294
|
+
score -= 20;
|
295
|
+
// Length considerations
|
296
|
+
if (metrics.linesOfCode > 100)
|
297
|
+
score -= 10;
|
298
|
+
// Look for inefficient patterns
|
299
|
+
if (/while.*true|for.*;.*;.*\+\+/.test(code)) {
|
300
|
+
if (!/(break|return)/.test(code))
|
301
|
+
score -= 15; // Potential infinite loops
|
302
|
+
}
|
303
|
+
return Math.max(0, Math.min(100, score));
|
304
|
+
}
|