analyzthis_design 2.5.0 → 2.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/HOW-TO-USE.md +249 -326
- package/README.md +58 -13
- package/agents/calibration-benchmark.json +187 -0
- package/agents/cards/anuj.md +2 -2
- package/agents/cards/arjun.md +2 -2
- package/agents/cards/kavi.md +2 -1
- package/agents/cards/mood-board.md +3 -2
- package/agents/cards/noor.md +2 -2
- package/agents/cards/priya.md +1 -1
- package/agents/cards/zara.md +3 -3
- package/agents/chain.json +1 -1
- package/agents/design-spec-schema.json +18 -1
- package/agents/manifests/mood-board.json +1 -0
- package/agents/model-policy.json +12 -0
- package/agents/router.json +1 -1
- package/agents/session-schema.json +12 -2
- package/dist/HOW-TO-USE.md +249 -326
- package/dist/README.md +58 -13
- package/dist/agents/calibration-benchmark.json +187 -0
- package/dist/agents/cards/anuj.md +2 -2
- package/dist/agents/cards/arjun.md +2 -2
- package/dist/agents/cards/kavi.md +2 -1
- package/dist/agents/cards/mood-board.md +3 -2
- package/dist/agents/cards/noor.md +2 -2
- package/dist/agents/cards/priya.md +1 -1
- package/dist/agents/cards/zara.md +3 -3
- package/dist/agents/chain.json +1 -1
- package/dist/agents/design-spec-schema.json +18 -1
- package/dist/agents/manifests/mood-board.json +1 -0
- package/dist/agents/model-policy.json +12 -0
- package/dist/agents/router.json +1 -1
- package/dist/agents/session-schema.json +12 -2
- package/dist/bin/cli.js +154 -5
- package/dist/lib/calibrate.js +816 -0
- package/dist/lib/chunk-executor.js +41 -1
- package/dist/lib/chunk-models.js +4 -18
- package/dist/lib/chunk-router.js +22 -6
- package/dist/lib/chunk-run.js +55 -1
- package/dist/lib/chunk-synthesis.js +18 -2
- package/dist/lib/chunk-telemetry.js +77 -17
- package/dist/lib/collect.js +15 -1
- package/dist/lib/cost.js +15 -7
- package/dist/lib/design-spec.js +32 -2
- package/dist/lib/evolution-metrics.js +23 -1
- package/dist/lib/feedback-submit.js +50 -1
- package/dist/lib/feedback.js +15 -0
- package/dist/lib/lessons.js +8 -0
- package/dist/lib/mcp-server.js +119 -5
- package/dist/lib/model-policy.js +174 -0
- package/dist/lib/model-registry.js +71 -0
- package/dist/lib/moodboard.js +162 -28
- package/dist/lib/orchestrator/run.js +67 -8
- package/dist/lib/outcome.js +7 -2
- package/dist/lib/output-quality.js +95 -0
- package/dist/lib/reference-pack.js +9 -1
- package/dist/lib/session.js +48 -4
- package/dist/lib/system-prompt.js +3 -0
- package/dist/lib/ui-libraries.js +232 -0
- package/dist/skills/anuj/SKILL.md +4 -1
- package/dist/skills/arjun/SKILL.md +2 -2
- package/dist/skills/arjun/references/lens.md +15 -2
- package/dist/skills/design-director/SKILL.md +2 -0
- package/dist/skills/design-reference/SKILL.md +6 -0
- package/dist/skills/design-reference/motion.csv +11 -0
- package/dist/skills/design-reference/schema.json +12 -0
- package/dist/skills/design-reference/stacks/shadcn.csv +6 -0
- package/dist/skills/design-reference/ui-libraries.csv +46 -0
- package/dist/skills/design-reference/ux-guidelines.csv +12 -1
- package/dist/skills/design-spec/SKILL.md +1 -1
- package/dist/skills/getting-started/SKILL.md +2 -2
- package/dist/skills/kavi/SKILL.md +4 -3
- package/dist/skills/mood-board/SKILL.md +29 -3
- package/dist/skills/noor/SKILL.md +5 -1
- package/dist/skills/priya/SKILL.md +3 -0
- package/dist/skills/zara/SKILL.md +50 -3
- package/package.json +5 -2
- package/skills/anuj/SKILL.md +4 -1
- package/skills/arjun/SKILL.md +2 -2
- package/skills/arjun/references/lens.md +15 -2
- package/skills/design-director/SKILL.md +2 -0
- package/skills/design-reference/SKILL.md +6 -0
- package/skills/design-reference/motion.csv +11 -0
- package/skills/design-reference/schema.json +12 -0
- package/skills/design-reference/stacks/shadcn.csv +6 -0
- package/skills/design-reference/ui-libraries.csv +46 -0
- package/skills/design-reference/ux-guidelines.csv +12 -1
- package/skills/design-spec/SKILL.md +1 -1
- package/skills/getting-started/SKILL.md +2 -2
- package/skills/kavi/SKILL.md +4 -3
- package/skills/mood-board/SKILL.md +29 -3
- package/skills/noor/SKILL.md +5 -1
- package/skills/priya/SKILL.md +3 -0
- package/skills/zara/SKILL.md +50 -3
|
@@ -109,6 +109,15 @@ function patchIsApplied(patch) {
|
|
|
109
109
|
function collectSignals(projectIds, persona) {
|
|
110
110
|
var outcomes = []; // { value, source }
|
|
111
111
|
var ratings = []; // numbers
|
|
112
|
+
var byModel = {};
|
|
113
|
+
|
|
114
|
+
function modelBucket(attribution) {
|
|
115
|
+
var id = attribution && (attribution.model_id ||
|
|
116
|
+
(attribution.provider && attribution.model && attribution.provider + '/' + attribution.model));
|
|
117
|
+
id = id || 'unknown/unknown';
|
|
118
|
+
if (!byModel[id]) byModel[id] = { outcomes: 0, ratings: 0, rating_total: 0 };
|
|
119
|
+
return byModel[id];
|
|
120
|
+
}
|
|
112
121
|
|
|
113
122
|
for (var i = 0; i < projectIds.length; i++) {
|
|
114
123
|
var state = session.show({ project: projectIds[i] });
|
|
@@ -120,8 +129,10 @@ function collectSignals(projectIds, persona) {
|
|
|
120
129
|
// A confirmed outcome supersedes an inferred one for the same persona.
|
|
121
130
|
if (confirmed[persona] && confirmed[persona].value) {
|
|
122
131
|
outcomes.push({ value: confirmed[persona].value, source: 'confirmed' });
|
|
132
|
+
modelBucket(confirmed[persona].attribution).outcomes++;
|
|
123
133
|
} else if (inferred[persona] && inferred[persona].value) {
|
|
124
134
|
outcomes.push({ value: inferred[persona].value, source: 'inferred' });
|
|
135
|
+
modelBucket(inferred[persona].attribution).outcomes++;
|
|
125
136
|
}
|
|
126
137
|
|
|
127
138
|
var log = state.feedback_log || [];
|
|
@@ -129,10 +140,20 @@ function collectSignals(projectIds, persona) {
|
|
|
129
140
|
if (log[j].persona !== persona) continue;
|
|
130
141
|
if (log[j].rating == null) continue;
|
|
131
142
|
ratings.push(Number(log[j].rating));
|
|
143
|
+
var bucket = modelBucket(log[j].attribution);
|
|
144
|
+
bucket.ratings++;
|
|
145
|
+
bucket.rating_total += Number(log[j].rating);
|
|
132
146
|
}
|
|
133
147
|
}
|
|
134
148
|
|
|
135
|
-
|
|
149
|
+
Object.keys(byModel).forEach(function(id) {
|
|
150
|
+
var item = byModel[id];
|
|
151
|
+
item.evidence_count = item.outcomes + item.ratings;
|
|
152
|
+
item.avg_rating = item.ratings ? Math.round((item.rating_total / item.ratings) * 10) / 10 : null;
|
|
153
|
+
item.sufficient_evidence = item.evidence_count >= MIN_EVIDENCE;
|
|
154
|
+
delete item.rating_total;
|
|
155
|
+
});
|
|
156
|
+
return { outcomes: outcomes, ratings: ratings, by_model: byModel };
|
|
136
157
|
}
|
|
137
158
|
|
|
138
159
|
/**
|
|
@@ -243,6 +264,7 @@ function computeEvolutionMetrics(opts) {
|
|
|
243
264
|
outcome_breakdown: outcomeBreakdown,
|
|
244
265
|
ratings: signals.ratings.length,
|
|
245
266
|
avg_rating: avgRating,
|
|
267
|
+
model_breakdown: signals.by_model,
|
|
246
268
|
patches_proposed: personaPatches.length,
|
|
247
269
|
patches_applied: appliedPatches.length,
|
|
248
270
|
avg_signal: Math.round(avgSignal * 10) / 10,
|
|
@@ -240,7 +240,7 @@ function askConsentQuestion() {
|
|
|
240
240
|
rl.question(
|
|
241
241
|
'\nShare anonymized persona feedback with analyzthis_design maintainers?\n'
|
|
242
242
|
+ ' Sends: persona, rating, outcome (shipped/revised/missed), task type,\n'
|
|
243
|
-
+ ' tags,
|
|
243
|
+
+ ' tags, redacted corrections; or text-free fixed-benchmark aggregates\n'
|
|
244
244
|
+ ' Does NOT send: project paths, repo names, emails, or API keys\n'
|
|
245
245
|
+ 'Continue? [y/N] ',
|
|
246
246
|
(answer) => {
|
|
@@ -388,6 +388,53 @@ async function submitScores({ dryRun = false } = {}) {
|
|
|
388
388
|
return { sent, rows };
|
|
389
389
|
}
|
|
390
390
|
|
|
391
|
+
/**
|
|
392
|
+
* Build text-free, fixed-benchmark calibration cells. Random install IDs are
|
|
393
|
+
* deduplication hints only; the server and maintainer must not treat them as identity.
|
|
394
|
+
*/
|
|
395
|
+
function buildCalibrationPayloads(report, cfg) {
|
|
396
|
+
if (!report || report.official_benchmark !== true || report.private_tasks === true) {
|
|
397
|
+
throw new Error('Only the package fixed benchmark can be shared. Private-task results stay local.');
|
|
398
|
+
}
|
|
399
|
+
const cells = Array.isArray(report.community_rows) ? report.community_rows : [];
|
|
400
|
+
return cells.map((cell) => ({
|
|
401
|
+
install_id: cfg.installId,
|
|
402
|
+
package_version: cfg.packageVersion,
|
|
403
|
+
benchmark_version: String(report.benchmark_version || ''),
|
|
404
|
+
validator_version: String(report.validator_version || ''),
|
|
405
|
+
judge_policy_version: String(report.judge_policy_version || ''),
|
|
406
|
+
prompt_hash: String(cell.prompt_hash || ''),
|
|
407
|
+
persona: String(cell.persona || ''),
|
|
408
|
+
effort: String(cell.effort || ''),
|
|
409
|
+
output_format: String(cell.output_format || ''),
|
|
410
|
+
model_id: String(cell.model_id || ''),
|
|
411
|
+
sample_count: Number(cell.sample_count || 0),
|
|
412
|
+
score_mean: Number(cell.score_mean || 0),
|
|
413
|
+
score_variance: Number(cell.score_variance || 0),
|
|
414
|
+
pass_rate: Number(cell.pass_rate || 0),
|
|
415
|
+
latency_ms: Number(cell.latency_ms || 0),
|
|
416
|
+
availability_rate: Number(cell.availability_rate || 0),
|
|
417
|
+
cost_usd: Number(cell.cost_usd || 0),
|
|
418
|
+
}));
|
|
419
|
+
}
|
|
420
|
+
|
|
421
|
+
async function submitCalibration(report, { dryRun = false, yes = false } = {}) {
|
|
422
|
+
const cfg = resolveSubmitConfig();
|
|
423
|
+
if (!cfg.enabled) throw new Error('Feedback submit is disabled.');
|
|
424
|
+
const rows = buildCalibrationPayloads(report, cfg);
|
|
425
|
+
if (!rows.length) return { submitted: 0, skipped: 'no fixed-benchmark cells' };
|
|
426
|
+
if (dryRun) return { submitted: 0, dryRun: true, rows, endpoint: cfg.url };
|
|
427
|
+
const consented = await ensureConsent({ yes });
|
|
428
|
+
if (!consented) return { submitted: 0, cancelled: true };
|
|
429
|
+
let sent = 0;
|
|
430
|
+
for (let i = 0; i < rows.length; i += MAX_ROWS_PER_REQUEST) {
|
|
431
|
+
const batch = rows.slice(i, i + MAX_ROWS_PER_REQUEST);
|
|
432
|
+
await postJson(cfg.url, {}, { kind: 'calibration', rows: batch });
|
|
433
|
+
sent += batch.length;
|
|
434
|
+
}
|
|
435
|
+
return { submitted: sent, endpoint: cfg.url };
|
|
436
|
+
}
|
|
437
|
+
|
|
391
438
|
module.exports = {
|
|
392
439
|
anonymizeText,
|
|
393
440
|
entryToSubmitPayload,
|
|
@@ -395,6 +442,8 @@ module.exports = {
|
|
|
395
442
|
submitFeedback,
|
|
396
443
|
submitScores,
|
|
397
444
|
buildScorePayloads,
|
|
445
|
+
buildCalibrationPayloads,
|
|
446
|
+
submitCalibration,
|
|
398
447
|
submitStatus,
|
|
399
448
|
collectUnsentEntries,
|
|
400
449
|
markEntriesSubmitted,
|
package/dist/lib/feedback.js
CHANGED
|
@@ -75,6 +75,11 @@ function recordFeedback(opts = {}) {
|
|
|
75
75
|
if (!outputEntry) {
|
|
76
76
|
throw new Error(`No output recorded for persona "${persona}" in this session yet.`);
|
|
77
77
|
}
|
|
78
|
+
if (state.active_run && outputEntry.run_id !== state.active_run.id) {
|
|
79
|
+
const stale = new Error(`The "${persona}" output is from an older run. Run it again before recording feedback.`);
|
|
80
|
+
stale.code = 'STALE_OUTPUT';
|
|
81
|
+
throw stale;
|
|
82
|
+
}
|
|
78
83
|
|
|
79
84
|
const entry = {
|
|
80
85
|
id: crypto.randomBytes(8).toString('hex'),
|
|
@@ -86,6 +91,14 @@ function recordFeedback(opts = {}) {
|
|
|
86
91
|
comment: String(comment).trim(),
|
|
87
92
|
correction: String(correction).trim(),
|
|
88
93
|
tags: tags.length ? tags : (satisfied ? ['positive'] : ['other']),
|
|
94
|
+
run_id: outputEntry.run_id || null,
|
|
95
|
+
attribution: outputEntry.attribution || {
|
|
96
|
+
provider: 'host',
|
|
97
|
+
model: 'unknown',
|
|
98
|
+
model_id: 'host/unknown',
|
|
99
|
+
effort: 'unknown',
|
|
100
|
+
},
|
|
101
|
+
quality: outputEntry.quality || null,
|
|
89
102
|
original_output: typeof outputEntry.text === 'string'
|
|
90
103
|
? outputEntry.text.slice(0, 8000)
|
|
91
104
|
: JSON.stringify(outputEntry).slice(0, 8000),
|
|
@@ -155,6 +168,8 @@ function exportCorrections({ persona, project, all = false, output, includePosit
|
|
|
155
168
|
project_id: e.project_id,
|
|
156
169
|
rating: e.rating,
|
|
157
170
|
tags: e.tags,
|
|
171
|
+
run_id: e.run_id || null,
|
|
172
|
+
attribution: e.attribution || null,
|
|
158
173
|
system_card: loadCard(e.persona),
|
|
159
174
|
user: e.context?.task_map_summary || '',
|
|
160
175
|
assistant_rejected: e.original_output,
|
package/dist/lib/lessons.js
CHANGED
|
@@ -82,6 +82,8 @@ function extractLessonsFromSession(state, personaId) {
|
|
|
82
82
|
|
|
83
83
|
var lessons = [];
|
|
84
84
|
var text = entry.text || '';
|
|
85
|
+
var attribution = entry.attribution || null;
|
|
86
|
+
var runId = entry.run_id || null;
|
|
85
87
|
|
|
86
88
|
var fixesMatch = text.match(/Top\s*2\s*fixes:[\s\S]*?(?:\n\n|```|$)/i);
|
|
87
89
|
if (fixesMatch) {
|
|
@@ -103,6 +105,8 @@ function extractLessonsFromSession(state, personaId) {
|
|
|
103
105
|
session_id: state.project_id,
|
|
104
106
|
extracted_at: new Date().toISOString(),
|
|
105
107
|
citation: entry.citations ? entry.citations[0] : ''
|
|
108
|
+
,attribution: attribution
|
|
109
|
+
,run_id: runId
|
|
106
110
|
});
|
|
107
111
|
}
|
|
108
112
|
}
|
|
@@ -127,6 +131,8 @@ function extractLessonsFromSession(state, personaId) {
|
|
|
127
131
|
session_id: state.project_id,
|
|
128
132
|
extracted_at: new Date().toISOString(),
|
|
129
133
|
citation: ''
|
|
134
|
+
,attribution: attribution
|
|
135
|
+
,run_id: runId
|
|
130
136
|
});
|
|
131
137
|
break;
|
|
132
138
|
}
|
|
@@ -147,6 +153,8 @@ function extractLessonsFromSession(state, personaId) {
|
|
|
147
153
|
session_id: state.project_id,
|
|
148
154
|
extracted_at: new Date().toISOString(),
|
|
149
155
|
citation: ''
|
|
156
|
+
,attribution: attribution
|
|
157
|
+
,run_id: runId
|
|
150
158
|
});
|
|
151
159
|
}
|
|
152
160
|
}
|
package/dist/lib/mcp-server.js
CHANGED
|
@@ -72,7 +72,7 @@ var TOOLS = [
|
|
|
72
72
|
},
|
|
73
73
|
{
|
|
74
74
|
name: 'analyzthis_zara',
|
|
75
|
-
description: 'Zara — Delight moments. Identifies exactly ONE structural or surface delight moment
|
|
75
|
+
description: 'Zara — Delight moments, and scout for UI image boards. Identifies exactly ONE structural or surface delight moment. On a UI board she browses real shipped screens, picks 3-5 frames, and re-scouts if the jury skips. Use after the core critique, or when the user wants a UI image board.',
|
|
76
76
|
inputSchema: {
|
|
77
77
|
type: 'object',
|
|
78
78
|
properties: {
|
|
@@ -208,15 +208,24 @@ var TOOLS = [
|
|
|
208
208
|
},
|
|
209
209
|
{
|
|
210
210
|
name: 'analyzthis_mood_board',
|
|
211
|
-
description: 'Mood Board
|
|
211
|
+
description: 'Mood Board or UI image board. Mood: tagged URLs + team deliberation. UI: Zara scouts 3-5 real shipped screens, then Arjun/Noor/Priya/Meera jury. She re-scouts if they skip (max 3 rounds). Use for visual direction before designing.',
|
|
212
212
|
inputSchema: {
|
|
213
213
|
type: 'object',
|
|
214
214
|
properties: {
|
|
215
215
|
task: { type: 'string', description: 'Visual direction brief (e.g., "B2B fintech dashboard, trustworthy, high-contrast, data-dense")' },
|
|
216
|
-
|
|
217
|
-
|
|
216
|
+
kind: { type: 'string', enum: ['mood', 'ui'], description: 'mood = tagged URLs (default). ui = Zara image-board scout.' },
|
|
217
|
+
action: { type: 'string', enum: ['create', 'frames', 'jury', 'verdict'], description: 'create (default), frames (save Zara picks), jury (critique prompt), verdict (SHIP/REVISE/BLOCK after jury)' },
|
|
218
|
+
verdict: { type: 'string', enum: ['SHIP', 'REVISE', 'BLOCK'], description: 'Jury verdict (action=verdict)' },
|
|
219
|
+
summary: { type: 'string', description: 'Jury because-lines (action=verdict). Sent back to Zara if they skip.' },
|
|
220
|
+
board: { type: 'string', description: 'Board id (frames and jury)' },
|
|
221
|
+
urls: { type: 'array', items: { type: 'string' }, description: 'Web reference URLs to include (mood boards)' },
|
|
222
|
+
url: { type: 'string', description: 'One frame URL (ui boards, action=frames)' },
|
|
223
|
+
title: { type: 'string', description: 'Frame title' },
|
|
224
|
+
steal: { type: 'string', description: 'One thing to take from this frame' },
|
|
225
|
+
refuse: { type: 'string', description: 'One thing we will not copy' },
|
|
226
|
+
peak: { type: 'string', description: 'The first-impression this frame proves' },
|
|
227
|
+
auto: { type: 'boolean', description: 'Auto-collect references (mood boards only)' },
|
|
218
228
|
},
|
|
219
|
-
required: ['task'],
|
|
220
229
|
},
|
|
221
230
|
},
|
|
222
231
|
{
|
|
@@ -469,6 +478,111 @@ function handleCombinationPass(skillId, args) {
|
|
|
469
478
|
function handleMoodBoard(args) {
|
|
470
479
|
var skill = readSkill('mood-board');
|
|
471
480
|
var task = args.task || '';
|
|
481
|
+
var kind = args.kind === 'ui' ? 'ui' : 'mood';
|
|
482
|
+
var action = args.action || 'create';
|
|
483
|
+
var mb = require('./moodboard');
|
|
484
|
+
|
|
485
|
+
if (action === 'frames') {
|
|
486
|
+
try {
|
|
487
|
+
var added = mb.addFrames({
|
|
488
|
+
boardId: args.board,
|
|
489
|
+
frames: args.frames || [{
|
|
490
|
+
url: args.url,
|
|
491
|
+
title: args.title,
|
|
492
|
+
steal: args.steal,
|
|
493
|
+
refuse: args.refuse,
|
|
494
|
+
peak: args.peak,
|
|
495
|
+
}],
|
|
496
|
+
});
|
|
497
|
+
return {
|
|
498
|
+
result: 'Saved. ' + added.board.frames.length + ' frame(s). Next: action=jury on board ' + added.board.board_id,
|
|
499
|
+
board_id: added.board.board_id,
|
|
500
|
+
scout: added.board.scout,
|
|
501
|
+
};
|
|
502
|
+
} catch (e) {
|
|
503
|
+
return { error: e.message };
|
|
504
|
+
}
|
|
505
|
+
}
|
|
506
|
+
|
|
507
|
+
if (action === 'jury') {
|
|
508
|
+
try {
|
|
509
|
+
var juryBoard = mb.getBoard({ boardId: args.board });
|
|
510
|
+
if (!juryBoard.frames || !juryBoard.frames.length) {
|
|
511
|
+
return {
|
|
512
|
+
error: 'NEED_SCOUT',
|
|
513
|
+
board_id: juryBoard.board_id,
|
|
514
|
+
scout: juryBoard.scout,
|
|
515
|
+
prompt: 'Zara has not scouted yet.\n\n' + ((juryBoard.scout && juryBoard.scout.brief) || ''),
|
|
516
|
+
};
|
|
517
|
+
}
|
|
518
|
+
var frameLines = juryBoard.frames.map(function(f, idx) {
|
|
519
|
+
return (idx + 1) + '. ' + (f.title || 'Frame') + ' — ' + (f.url || '') +
|
|
520
|
+
'\n Steal: ' + (f.steal || '') +
|
|
521
|
+
'\n Refuse: ' + (f.refuse || '') +
|
|
522
|
+
'\n Peak: ' + (f.peak || '');
|
|
523
|
+
}).join('\n');
|
|
524
|
+
var juryPrompt = 'You are the UI image-board jury: Arjun, Noor, Priya, Meera. Zara does not vote.\n\n' +
|
|
525
|
+
'Job: ' + (juryBoard.task || '') + '\n\n' +
|
|
526
|
+
'Frames:\n' + frameLines + '\n\n' +
|
|
527
|
+
'KEEP or SKIP each frame with one because-sentence. Rank the best 2-3. Verdict SHIP only if the set fits the job. REVISE or BLOCK sends Zara to scout again.\n' +
|
|
528
|
+
'Do not reward cinematic marketing motion on a daily working surface.\n\n' +
|
|
529
|
+
'Then call analyzthis_mood_board with action=verdict, board=' + juryBoard.board_id +
|
|
530
|
+
', verdict=SHIP|REVISE|BLOCK, and summary=the because-lines.';
|
|
531
|
+
return { prompt: juryPrompt, board_id: juryBoard.board_id, scout: juryBoard.scout, frames: juryBoard.frames };
|
|
532
|
+
} catch (e) {
|
|
533
|
+
return { error: e.message };
|
|
534
|
+
}
|
|
535
|
+
}
|
|
536
|
+
|
|
537
|
+
if (action === 'verdict') {
|
|
538
|
+
try {
|
|
539
|
+
var judged = mb.recordJuryVerdict({
|
|
540
|
+
boardId: args.board,
|
|
541
|
+
verdict: args.verdict,
|
|
542
|
+
summary: args.summary,
|
|
543
|
+
});
|
|
544
|
+
var scout = judged.board.scout || {};
|
|
545
|
+
if (scout.status === 'awaiting_scout') {
|
|
546
|
+
return {
|
|
547
|
+
result: 'Jury skipped. Zara scouts again (round ' + scout.round + ').',
|
|
548
|
+
board_id: judged.board.board_id,
|
|
549
|
+
scout: scout,
|
|
550
|
+
prompt: 'You are Zara on a Scout Pass. Search and open real pages. Do not invent URLs.\n\n' + scout.brief +
|
|
551
|
+
'\n\nBoard id: ' + judged.board.board_id + '\nAfter 3-5 new frames, call action=frames, then action=jury.',
|
|
552
|
+
};
|
|
553
|
+
}
|
|
554
|
+
if (scout.status === 'raj') {
|
|
555
|
+
return {
|
|
556
|
+
result: 'Max scout rounds reached. Raj or the designer picks from the frames on this board.',
|
|
557
|
+
board_id: judged.board.board_id,
|
|
558
|
+
scout: scout,
|
|
559
|
+
};
|
|
560
|
+
}
|
|
561
|
+
return {
|
|
562
|
+
result: 'Jury SHIP. Board is done. User still /accept on the notes.',
|
|
563
|
+
board_id: judged.board.board_id,
|
|
564
|
+
scout: scout,
|
|
565
|
+
};
|
|
566
|
+
} catch (e) {
|
|
567
|
+
return { error: e.message };
|
|
568
|
+
}
|
|
569
|
+
}
|
|
570
|
+
|
|
571
|
+
if (kind === 'ui') {
|
|
572
|
+
if (!task) return { error: 'task is required to open a UI image board' };
|
|
573
|
+
try {
|
|
574
|
+
var created = mb.buildMoodBoard({ task: task, kind: 'ui', discover: false });
|
|
575
|
+
var prompt = 'You are Zara on a Scout Pass. Search and open real pages. Do not invent URLs.\n\n' +
|
|
576
|
+
created.board.scout.brief + '\n\n' +
|
|
577
|
+
'Board id: ' + created.board.board_id + '\n' +
|
|
578
|
+
'After 3-5 frames, call analyzthis_mood_board again with action=frames, board=' + created.board.board_id +
|
|
579
|
+
', and url/title/steal/refuse/peak. Then action=jury.\n';
|
|
580
|
+
return { prompt: prompt, board_id: created.board.board_id, scout: created.board.scout, task: task };
|
|
581
|
+
} catch (e) {
|
|
582
|
+
return { error: e.message };
|
|
583
|
+
}
|
|
584
|
+
}
|
|
585
|
+
|
|
472
586
|
var urls = (args.urls || []).join('\n');
|
|
473
587
|
var prompt = 'Build a mood board.\n\n' +
|
|
474
588
|
(skill ? skill.slice(0, 3000) : '') + '\n\n' +
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Shared model preference resolver for chunked and unchunked execution.
|
|
5
|
+
* Effort/gate constraints are established by the caller before preferences.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
var fs = require('fs');
|
|
9
|
+
var path = require('path');
|
|
10
|
+
var os = require('os');
|
|
11
|
+
var crypto = require('crypto');
|
|
12
|
+
var registry = require('./model-registry');
|
|
13
|
+
var provider = require('./provider');
|
|
14
|
+
var platforms = require('./platforms');
|
|
15
|
+
var outputQuality = require('./output-quality');
|
|
16
|
+
|
|
17
|
+
var PACKAGE_ROOT = platforms.resolvePackageRoot(__dirname);
|
|
18
|
+
var DEFAULT_LOCAL_FILE = path.join(os.homedir(), '.analyzthis_design', 'calibration.json');
|
|
19
|
+
var GLOBAL_FILE = path.join(PACKAGE_ROOT, 'agents', 'model-policy.json');
|
|
20
|
+
|
|
21
|
+
function readJson(file) {
|
|
22
|
+
if (!file || !fs.existsSync(file)) return {};
|
|
23
|
+
try { return JSON.parse(fs.readFileSync(file, 'utf8')); }
|
|
24
|
+
catch (e) { return {}; }
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
function localFile() {
|
|
28
|
+
return process.env.ANALYZTHIS_CALIBRATION_FILE || DEFAULT_LOCAL_FILE;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
function promptHash(persona) {
|
|
32
|
+
var manifestPath = path.join(PACKAGE_ROOT, 'agents', 'manifests', persona + '.json');
|
|
33
|
+
var manifest = readJson(manifestPath);
|
|
34
|
+
var parts = [];
|
|
35
|
+
[manifest.system_card, manifest.system_skill].forEach(function(relative) {
|
|
36
|
+
if (!relative) return;
|
|
37
|
+
var file = path.join(PACKAGE_ROOT, relative);
|
|
38
|
+
if (fs.existsSync(file)) parts.push(fs.readFileSync(file));
|
|
39
|
+
});
|
|
40
|
+
return crypto.createHash('sha256').update(Buffer.concat(parts)).digest('hex').slice(0, 16);
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
function calibrationContract() {
|
|
44
|
+
var file = path.join(PACKAGE_ROOT, 'agents', 'calibration-benchmark.json');
|
|
45
|
+
var benchmark = readJson(file);
|
|
46
|
+
return {
|
|
47
|
+
version: benchmark.version,
|
|
48
|
+
judge_policy_version: benchmark.judge_version,
|
|
49
|
+
hash: Object.keys(benchmark).length
|
|
50
|
+
? crypto.createHash('sha256').update(JSON.stringify(benchmark)).digest('hex').slice(0, 16)
|
|
51
|
+
: null,
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
function assignmentFrom(policy, persona, effort) {
|
|
56
|
+
var personas = policy.personas || policy.assignments || {};
|
|
57
|
+
return personas[persona] && personas[persona][effort];
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
function eligible(spec, effort) {
|
|
61
|
+
spec = registry.normalizeSpec(spec);
|
|
62
|
+
if (!spec.provider || !spec.model || registry.isExpired(spec)) return false;
|
|
63
|
+
if (spec.effort && spec.effort.indexOf(effort) === -1) return false;
|
|
64
|
+
if (spec.provider !== 'host' && !provider.hasKey(spec.provider)) return false;
|
|
65
|
+
return true;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
function validWinner(winner, persona, effort, requireContract) {
|
|
69
|
+
if (!winner || !eligible(winner, effort)) return false;
|
|
70
|
+
if (requireContract) {
|
|
71
|
+
var hasJudge = winner.judge_policy_version != null || winner.judge_version != null;
|
|
72
|
+
if (!winner.prompt_hash || winner.validator_version == null ||
|
|
73
|
+
winner.benchmark_version == null || !winner.benchmark_hash ||
|
|
74
|
+
!hasJudge || !winner.expires_at) return false;
|
|
75
|
+
}
|
|
76
|
+
if (winner.prompt_hash && winner.prompt_hash !== promptHash(persona)) return false;
|
|
77
|
+
if (winner.validator_version != null && Number(winner.validator_version) !== outputQuality.VERSION) return false;
|
|
78
|
+
var contract = calibrationContract();
|
|
79
|
+
if (winner.benchmark_version != null && winner.benchmark_version !== contract.version) return false;
|
|
80
|
+
if (winner.benchmark_hash && winner.benchmark_hash !== contract.hash) return false;
|
|
81
|
+
var judgeVersion = winner.judge_policy_version != null ? winner.judge_policy_version : winner.judge_version;
|
|
82
|
+
if (judgeVersion != null && judgeVersion !== contract.judge_policy_version) return false;
|
|
83
|
+
return true;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
function resolveAssignment(opts) {
|
|
87
|
+
opts = opts || {};
|
|
88
|
+
var persona = opts.persona || 'general';
|
|
89
|
+
var effort = opts.effort || 'standard';
|
|
90
|
+
var config = opts.config || {};
|
|
91
|
+
var candidates = [];
|
|
92
|
+
|
|
93
|
+
if (opts.explicitProvider || opts.explicitModel) {
|
|
94
|
+
var inferred = opts.explicitProvider;
|
|
95
|
+
if (!inferred && opts.explicitModel) {
|
|
96
|
+
var known = registry.MODELS.find(function(item) { return item.model === opts.explicitModel; });
|
|
97
|
+
inferred = known && known.provider;
|
|
98
|
+
}
|
|
99
|
+
candidates.push({
|
|
100
|
+
source: 'explicit',
|
|
101
|
+
value: { provider: inferred, model: opts.explicitModel },
|
|
102
|
+
strict: true,
|
|
103
|
+
});
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
var pinned = assignmentFrom(config.model_policy || {}, persona, effort);
|
|
107
|
+
if (!pinned && config.orchestrator && config.orchestrator.provider &&
|
|
108
|
+
config.orchestrator.provider !== 'auto' && config.orchestrator.model) {
|
|
109
|
+
pinned = { provider: config.orchestrator.provider, model: config.orchestrator.model };
|
|
110
|
+
}
|
|
111
|
+
if (pinned) candidates.push({ source: 'user_config', value: pinned, strict: false });
|
|
112
|
+
candidates.push({ source: 'local_calibration', value: assignmentFrom(readJson(localFile()), persona, effort) });
|
|
113
|
+
candidates.push({ source: 'package_policy', value: assignmentFrom(readJson(GLOBAL_FILE), persona, effort) });
|
|
114
|
+
|
|
115
|
+
for (var i = 0; i < candidates.length; i++) {
|
|
116
|
+
var candidate = candidates[i];
|
|
117
|
+
if (!candidate.value) continue;
|
|
118
|
+
var value = registry.normalizeSpec(candidate.value);
|
|
119
|
+
var requireContract = candidate.source === 'local_calibration' || candidate.source === 'package_policy';
|
|
120
|
+
if (validWinner(value, persona, effort, requireContract)) {
|
|
121
|
+
return { spec: value, source: candidate.source, prompt_hash: promptHash(persona) };
|
|
122
|
+
}
|
|
123
|
+
if (candidate.strict) {
|
|
124
|
+
var error = new Error('Requested model is unavailable or not eligible for effort=' + effort);
|
|
125
|
+
error.code = 'INELIGIBLE_MODEL';
|
|
126
|
+
throw error;
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
return null;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
function prioritizePool(pool, opts) {
|
|
133
|
+
opts = opts || {};
|
|
134
|
+
if (opts.explicitProvider && !opts.explicitModel) {
|
|
135
|
+
var providerOnly = pool.filter(function(item) {
|
|
136
|
+
return item.provider === opts.explicitProvider;
|
|
137
|
+
});
|
|
138
|
+
if (!providerOnly.length) {
|
|
139
|
+
var providerError = new Error('No eligible model for requested provider: ' + opts.explicitProvider);
|
|
140
|
+
providerError.code = 'INELIGIBLE_MODEL';
|
|
141
|
+
throw providerError;
|
|
142
|
+
}
|
|
143
|
+
return { pool: providerOnly, policy: { source: 'explicit_provider' } };
|
|
144
|
+
}
|
|
145
|
+
var resolved = resolveAssignment(opts);
|
|
146
|
+
if (!resolved) return { pool: pool, policy: null };
|
|
147
|
+
var wanted = registry.id(resolved.spec);
|
|
148
|
+
var index = pool.findIndex(function(item) {
|
|
149
|
+
return registry.id(item) === wanted;
|
|
150
|
+
});
|
|
151
|
+
if (index === -1) {
|
|
152
|
+
if (resolved.source === 'explicit' || resolved.source === 'user_config') {
|
|
153
|
+
var error = new Error('Requested model is not in the available model pool: ' + wanted);
|
|
154
|
+
error.code = 'MODEL_NOT_IN_POOL';
|
|
155
|
+
throw error;
|
|
156
|
+
}
|
|
157
|
+
return { pool: pool, policy: null };
|
|
158
|
+
}
|
|
159
|
+
var ordered = pool.slice();
|
|
160
|
+
var selected = ordered.splice(index, 1)[0];
|
|
161
|
+
ordered.unshift(selected);
|
|
162
|
+
return { pool: ordered, policy: resolved };
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
module.exports = {
|
|
166
|
+
VERSION: 1,
|
|
167
|
+
GLOBAL_FILE: GLOBAL_FILE,
|
|
168
|
+
localFile: localFile,
|
|
169
|
+
promptHash: promptHash,
|
|
170
|
+
calibrationContract: calibrationContract,
|
|
171
|
+
eligible: eligible,
|
|
172
|
+
resolveAssignment: resolveAssignment,
|
|
173
|
+
prioritizePool: prioritizePool,
|
|
174
|
+
};
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Canonical model metadata. Prices are USD per one million tokens.
|
|
5
|
+
* User-defined pool entries remain supported through normalizeSpec().
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
var MODELS = [
|
|
9
|
+
{ provider: 'groq', model: 'llama-3.1-8b-instant', input_per_m: 0.05, output_per_m: 0.08, effort: ['trivial', 'standard'], context: 131072, key_env: 'GROQ_API_KEY' },
|
|
10
|
+
{ provider: 'groq', model: 'mixtral-8x7b-32768', input_per_m: 0.24, output_per_m: 0.24, effort: ['trivial', 'standard', 'hard'], context: 32768, key_env: 'GROQ_API_KEY' },
|
|
11
|
+
{ provider: 'google', model: 'gemini-1.5-flash', input_per_m: 0.075, output_per_m: 0.30, effort: ['trivial', 'standard', 'hard'], context: 128000, key_env: 'GEMINI_API_KEY' },
|
|
12
|
+
{ provider: 'google', model: 'gemini-1.5-flash-8b', input_per_m: 0.0375, output_per_m: 0.15, effort: ['trivial', 'standard'], context: 128000, key_env: 'GEMINI_API_KEY' },
|
|
13
|
+
{ provider: 'google', model: 'gemini-2.5-flash', input_per_m: 0.30, output_per_m: 2.50, effort: ['trivial', 'standard', 'hard'], context: 1000000, key_env: 'GEMINI_API_KEY' },
|
|
14
|
+
{ provider: 'deepseek', model: 'deepseek-chat', input_per_m: 0.07, output_per_m: 0.27, effort: ['trivial', 'standard', 'hard'], context: 64000, key_env: 'DEEPSEEK_API_KEY' },
|
|
15
|
+
{ provider: 'openrouter', model: 'meta-llama/llama-3.1-8b-instruct:free', input_per_m: 0, output_per_m: 0, effort: ['trivial', 'standard'], context: 131072, key_env: 'OPENROUTER_API_KEY' },
|
|
16
|
+
{ provider: 'together', model: 'meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo', input_per_m: 0.18, output_per_m: 0.18, effort: ['trivial', 'standard'], context: 8192, key_env: 'TOGETHER_API_KEY' },
|
|
17
|
+
{ provider: 'openai', model: 'gpt-4o-mini', input_per_m: 0.15, output_per_m: 0.60, effort: ['trivial', 'standard'], context: 128000, key_env: 'OPENAI_API_KEY' },
|
|
18
|
+
{ provider: 'anthropic', model: 'claude-sonnet-4-20250514', input_per_m: 3, output_per_m: 15, effort: ['standard', 'hard'], context: 200000, key_env: 'ANTHROPIC_API_KEY' },
|
|
19
|
+
];
|
|
20
|
+
|
|
21
|
+
function id(spec) {
|
|
22
|
+
return String(spec.provider || '') + '/' + String(spec.model || '');
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
function normalizeSpec(spec) {
|
|
26
|
+
spec = Object.assign({}, spec || {});
|
|
27
|
+
var canonical = MODELS.find(function(item) {
|
|
28
|
+
return item.provider === spec.provider && item.model === spec.model;
|
|
29
|
+
}) || {};
|
|
30
|
+
var merged = Object.assign({}, canonical, spec);
|
|
31
|
+
if (merged.input_per_m == null && merged.cost != null) {
|
|
32
|
+
// v2.5.2 scalar costs were stored in per-1K units despite their comment.
|
|
33
|
+
merged.input_per_m = Number(merged.cost) * 1000;
|
|
34
|
+
}
|
|
35
|
+
if (merged.output_per_m == null && merged.cost != null) {
|
|
36
|
+
merged.output_per_m = Number(merged.cost) * 1000;
|
|
37
|
+
}
|
|
38
|
+
merged.input_per_m = Number(merged.input_per_m || 0);
|
|
39
|
+
merged.output_per_m = Number(merged.output_per_m || 0);
|
|
40
|
+
merged.cost = Math.max(merged.input_per_m, merged.output_per_m) / 1000;
|
|
41
|
+
merged.id = id(merged);
|
|
42
|
+
return merged;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
function estimateCost(spec, inputTokens, outputTokens) {
|
|
46
|
+
spec = normalizeSpec(spec);
|
|
47
|
+
return (Number(inputTokens || 0) / 1e6) * spec.input_per_m +
|
|
48
|
+
(Number(outputTokens || 0) / 1e6) * spec.output_per_m;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function find(provider, model) {
|
|
52
|
+
var found = MODELS.find(function(item) {
|
|
53
|
+
return item.provider === provider && item.model === model;
|
|
54
|
+
});
|
|
55
|
+
return found ? normalizeSpec(found) : null;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
function isExpired(spec, now) {
|
|
59
|
+
if (!spec || !spec.expires_at) return false;
|
|
60
|
+
return new Date(spec.expires_at).getTime() <= (now || Date.now());
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
module.exports = {
|
|
64
|
+
VERSION: 1,
|
|
65
|
+
MODELS: MODELS,
|
|
66
|
+
id: id,
|
|
67
|
+
find: find,
|
|
68
|
+
normalizeSpec: normalizeSpec,
|
|
69
|
+
estimateCost: estimateCost,
|
|
70
|
+
isExpired: isExpired,
|
|
71
|
+
};
|