analyzthis_design 2.5.1 ā 2.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/HOW-TO-USE.md +249 -326
- package/README.md +54 -20
- package/agents/calibration-benchmark.json +187 -0
- package/agents/cards/anuj.md +1 -1
- package/agents/cards/arjun.md +1 -1
- package/agents/cards/kavi.md +2 -1
- package/agents/cards/mood-board.md +3 -2
- package/agents/cards/noor.md +1 -1
- package/agents/cards/priya.md +1 -1
- package/agents/cards/zara.md +3 -3
- package/agents/chain.json +1 -1
- package/agents/design-spec-schema.json +18 -1
- package/agents/manifests/mood-board.json +1 -0
- package/agents/model-policy.json +12 -0
- package/agents/router.json +1 -1
- package/agents/session-schema.json +12 -2
- package/dist/HOW-TO-USE.md +249 -326
- package/dist/README.md +54 -20
- package/dist/agents/calibration-benchmark.json +187 -0
- package/dist/agents/cards/anuj.md +1 -1
- package/dist/agents/cards/arjun.md +1 -1
- package/dist/agents/cards/kavi.md +2 -1
- package/dist/agents/cards/mood-board.md +3 -2
- package/dist/agents/cards/noor.md +1 -1
- package/dist/agents/cards/priya.md +1 -1
- package/dist/agents/cards/zara.md +3 -3
- package/dist/agents/chain.json +1 -1
- package/dist/agents/design-spec-schema.json +18 -1
- package/dist/agents/manifests/mood-board.json +1 -0
- package/dist/agents/model-policy.json +12 -0
- package/dist/agents/router.json +1 -1
- package/dist/agents/session-schema.json +12 -2
- package/dist/bin/cli.js +154 -5
- package/dist/lib/calibrate.js +816 -0
- package/dist/lib/chunk-executor.js +41 -1
- package/dist/lib/chunk-models.js +4 -18
- package/dist/lib/chunk-router.js +22 -6
- package/dist/lib/chunk-run.js +55 -1
- package/dist/lib/chunk-synthesis.js +18 -2
- package/dist/lib/chunk-telemetry.js +77 -17
- package/dist/lib/collect.js +15 -1
- package/dist/lib/cost.js +15 -7
- package/dist/lib/design-spec.js +32 -2
- package/dist/lib/evolution-metrics.js +23 -1
- package/dist/lib/feedback-submit.js +50 -1
- package/dist/lib/feedback.js +15 -0
- package/dist/lib/lessons.js +8 -0
- package/dist/lib/mcp-server.js +119 -5
- package/dist/lib/model-policy.js +174 -0
- package/dist/lib/model-registry.js +71 -0
- package/dist/lib/moodboard.js +162 -28
- package/dist/lib/orchestrator/run.js +67 -8
- package/dist/lib/outcome.js +7 -2
- package/dist/lib/output-quality.js +95 -0
- package/dist/lib/reference-pack.js +9 -1
- package/dist/lib/session.js +48 -4
- package/dist/lib/system-prompt.js +3 -0
- package/dist/lib/ui-libraries.js +232 -0
- package/dist/skills/anuj/SKILL.md +2 -0
- package/dist/skills/arjun/references/lens.md +2 -0
- package/dist/skills/design-director/SKILL.md +2 -1
- package/dist/skills/design-reference/SKILL.md +6 -0
- package/dist/skills/design-reference/motion.csv +11 -0
- package/dist/skills/design-reference/schema.json +12 -0
- package/dist/skills/design-reference/stacks/shadcn.csv +6 -0
- package/dist/skills/design-reference/ui-libraries.csv +46 -0
- package/dist/skills/design-spec/SKILL.md +1 -1
- package/dist/skills/getting-started/SKILL.md +2 -2
- package/dist/skills/kavi/SKILL.md +3 -2
- package/dist/skills/mood-board/SKILL.md +29 -3
- package/dist/skills/noor/SKILL.md +2 -0
- package/dist/skills/priya/SKILL.md +3 -0
- package/dist/skills/zara/SKILL.md +50 -4
- package/package.json +5 -2
- package/skills/anuj/SKILL.md +2 -0
- package/skills/arjun/references/lens.md +2 -0
- package/skills/design-director/SKILL.md +2 -1
- package/skills/design-reference/SKILL.md +6 -0
- package/skills/design-reference/motion.csv +11 -0
- package/skills/design-reference/schema.json +12 -0
- package/skills/design-reference/stacks/shadcn.csv +6 -0
- package/skills/design-reference/ui-libraries.csv +46 -0
- package/skills/design-spec/SKILL.md +1 -1
- package/skills/getting-started/SKILL.md +2 -2
- package/skills/kavi/SKILL.md +3 -2
- package/skills/mood-board/SKILL.md +29 -3
- package/skills/noor/SKILL.md +2 -0
- package/skills/priya/SKILL.md +3 -0
- package/skills/zara/SKILL.md +50 -4
|
@@ -14,6 +14,7 @@ var chunkModels = require('./chunk-models');
|
|
|
14
14
|
var chunkRouter = require('./chunk-router');
|
|
15
15
|
var telemetry = require('./chunk-telemetry');
|
|
16
16
|
var referencePack = require('./reference-pack');
|
|
17
|
+
var outputQuality = require('./output-quality');
|
|
17
18
|
|
|
18
19
|
function buildChunkPrompt(chunk, priorResults, contextPack) {
|
|
19
20
|
var priorContext = '';
|
|
@@ -90,6 +91,9 @@ async function executeChunk(opts) {
|
|
|
90
91
|
var callLlmFn = opts.callLlmFn;
|
|
91
92
|
var attempt = 0;
|
|
92
93
|
var maxAttempts = opts.maxAttempts || 3;
|
|
94
|
+
var qualityFailures = 0;
|
|
95
|
+
var maxQualityFailures = opts.maxQualityFailures == null ? 1 : Number(opts.maxQualityFailures);
|
|
96
|
+
var outputFormat = chunk.output_schema || 'generic';
|
|
93
97
|
|
|
94
98
|
if (!callLlmFn) {
|
|
95
99
|
var orchestrator = require('./orchestrator/run');
|
|
@@ -112,8 +116,12 @@ async function executeChunk(opts) {
|
|
|
112
116
|
effort: chunk.effort || 'standard',
|
|
113
117
|
chunkType: chunk.id + '/' + chunk.persona,
|
|
114
118
|
persona: chunk.persona,
|
|
119
|
+
taskType: chunk.id,
|
|
120
|
+
outputFormat: outputFormat,
|
|
115
121
|
budget: budget,
|
|
116
122
|
config: config,
|
|
123
|
+
explicitProvider: opts.explicitProvider,
|
|
124
|
+
explicitModel: opts.explicitModel,
|
|
117
125
|
});
|
|
118
126
|
ranked = resolved.ranked;
|
|
119
127
|
} catch (e) {
|
|
@@ -154,27 +162,51 @@ async function executeChunk(opts) {
|
|
|
154
162
|
|
|
155
163
|
var inTok = Math.ceil((prompts.system.length + prompts.user.length) / 4);
|
|
156
164
|
var outTok = Math.ceil(text.length / 4);
|
|
165
|
+
var quality = outputQuality.validateOutput(text, {
|
|
166
|
+
persona: chunk.persona,
|
|
167
|
+
outputFormat: outputFormat,
|
|
168
|
+
});
|
|
157
169
|
|
|
158
170
|
telemetry.record({
|
|
171
|
+
scope_version: 1,
|
|
159
172
|
chunk_id: chunk.id,
|
|
160
173
|
persona: chunk.persona,
|
|
174
|
+
task_type: chunk.id,
|
|
175
|
+
output_format: outputFormat,
|
|
161
176
|
model: chunkModels.formatModelName(model),
|
|
162
177
|
provider: model.provider,
|
|
163
178
|
effort: chunk.effort || 'standard',
|
|
164
179
|
input_tokens: inTok,
|
|
165
180
|
output_tokens: outTok,
|
|
166
181
|
cost: chunkModels.estimateCost(model, inTok, outTok),
|
|
167
|
-
|
|
182
|
+
transport_success: true,
|
|
183
|
+
quality_pass: quality.pass,
|
|
184
|
+
quality_score: quality.score,
|
|
185
|
+
validator_version: quality.version,
|
|
186
|
+
success: quality.pass,
|
|
168
187
|
attempts: attempt,
|
|
169
188
|
at: new Date().toISOString(),
|
|
170
189
|
});
|
|
171
190
|
|
|
191
|
+
if (!quality.pass) {
|
|
192
|
+
qualityFailures++;
|
|
193
|
+
lastError = 'Output quality failed: ' + quality.issues.map(function(item) {
|
|
194
|
+
return item.code;
|
|
195
|
+
}).join(', ');
|
|
196
|
+
if (qualityFailures > maxQualityFailures) break;
|
|
197
|
+
modelIndex++;
|
|
198
|
+
continue;
|
|
199
|
+
}
|
|
200
|
+
|
|
172
201
|
return {
|
|
173
202
|
chunk_id: chunk.id,
|
|
174
203
|
persona: chunk.persona,
|
|
175
204
|
goal: chunk.goal,
|
|
205
|
+
effort: chunk.effort || 'standard',
|
|
206
|
+
output_format: outputFormat,
|
|
176
207
|
model_used: { provider: model.provider, model: model.model, cost: model.cost },
|
|
177
208
|
output: text,
|
|
209
|
+
full_prompt: prompts,
|
|
178
210
|
structured_output: parseStructured(text, chunk.output_schema),
|
|
179
211
|
tokens: { input: inTok, output: outTok },
|
|
180
212
|
cost: chunkModels.estimateCost(model, inTok, outTok),
|
|
@@ -182,16 +214,23 @@ async function executeChunk(opts) {
|
|
|
182
214
|
attempts: attempt,
|
|
183
215
|
cache_hit: cacheHit,
|
|
184
216
|
references: (refPack && refPack.citations) || [],
|
|
217
|
+
quality: quality,
|
|
185
218
|
success: true,
|
|
186
219
|
};
|
|
187
220
|
} catch (e) {
|
|
188
221
|
lastError = e.message;
|
|
189
222
|
telemetry.record({
|
|
223
|
+
scope_version: 1,
|
|
190
224
|
chunk_id: chunk.id,
|
|
191
225
|
persona: chunk.persona,
|
|
226
|
+
task_type: chunk.id,
|
|
227
|
+
output_format: outputFormat,
|
|
192
228
|
model: chunkModels.formatModelName(model),
|
|
193
229
|
provider: model.provider,
|
|
194
230
|
effort: chunk.effort || 'standard',
|
|
231
|
+
transport_success: false,
|
|
232
|
+
quality_pass: null,
|
|
233
|
+
failure_kind: outputQuality.classifyError(e),
|
|
195
234
|
success: false,
|
|
196
235
|
error: e.message,
|
|
197
236
|
attempts: attempt,
|
|
@@ -208,6 +247,7 @@ async function executeChunk(opts) {
|
|
|
208
247
|
success: false,
|
|
209
248
|
error: lastError || 'All chunk models failed',
|
|
210
249
|
attempts: attempt,
|
|
250
|
+
quality_failures: qualityFailures,
|
|
211
251
|
output: '',
|
|
212
252
|
};
|
|
213
253
|
}
|
package/dist/lib/chunk-models.js
CHANGED
|
@@ -14,6 +14,7 @@ var fs = require('fs');
|
|
|
14
14
|
var path = require('path');
|
|
15
15
|
var os = require('os');
|
|
16
16
|
var http = require('http');
|
|
17
|
+
var modelRegistry = require('./model-registry');
|
|
17
18
|
|
|
18
19
|
var CONFIG_FILE = path.join(os.homedir(), '.analyzthis_design', 'config.json');
|
|
19
20
|
|
|
@@ -29,20 +30,7 @@ function hasEnv(name) {
|
|
|
29
30
|
|
|
30
31
|
// Curated cloud models. Prices are estimates (USD per 1M tokens input/output).
|
|
31
32
|
// The router treats cost=0 as free; anything >0 as cheap/paid.
|
|
32
|
-
var DEFAULT_CLOUD_POOL =
|
|
33
|
-
// Free-ish / very cheap with user key
|
|
34
|
-
{ provider: 'groq', model: 'llama-3.1-8b-instant', cost: 0.00005, effort: ['trivial', 'standard'], context: 131072, key_env: 'GROQ_API_KEY' },
|
|
35
|
-
{ provider: 'groq', model: 'mixtral-8x7b-32768', cost: 0.00024, effort: ['trivial', 'standard', 'hard'], context: 32768, key_env: 'GROQ_API_KEY' },
|
|
36
|
-
{ provider: 'google', model: 'gemini-1.5-flash', cost: 0.000075, effort: ['trivial', 'standard', 'hard'], context: 128000, key_env: 'GEMINI_API_KEY' },
|
|
37
|
-
{ provider: 'google', model: 'gemini-1.5-flash-8b', cost: 0.0000375, effort: ['trivial', 'standard'], context: 128000, key_env: 'GEMINI_API_KEY' },
|
|
38
|
-
{ provider: 'deepseek', model: 'deepseek-chat', cost: 0.00007, effort: ['trivial', 'standard', 'hard'], context: 64000, key_env: 'DEEPSEEK_API_KEY' },
|
|
39
|
-
{ provider: 'openrouter', model: 'meta-llama/llama-3.1-8b-instruct:free', cost: 0, effort: ['trivial', 'standard'], context: 131072, key_env: 'OPENROUTER_API_KEY' },
|
|
40
|
-
{ provider: 'together', model: 'meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo', cost: 0.00018, effort: ['trivial', 'standard'], context: 8192, key_env: 'TOGETHER_API_KEY' },
|
|
41
|
-
// Paid but cheap fallback
|
|
42
|
-
{ provider: 'openai', model: 'gpt-4o-mini', cost: 0.00015, effort: ['trivial', 'standard'], context: 128000, key_env: 'OPENAI_API_KEY' },
|
|
43
|
-
// Frontier chunk fallback (only if budget allows)
|
|
44
|
-
{ provider: 'anthropic', model: 'claude-sonnet-4-20250514', cost: 0.003, effort: ['standard', 'hard'], context: 200000, key_env: 'ANTHROPIC_API_KEY' },
|
|
45
|
-
];
|
|
33
|
+
var DEFAULT_CLOUD_POOL = modelRegistry.MODELS.map(modelRegistry.normalizeSpec);
|
|
46
34
|
|
|
47
35
|
// Recommended Ollama models for chunk tasks, ordered by capability/cost trade-off.
|
|
48
36
|
var DEFAULT_OLLAMA_MODELS = [
|
|
@@ -124,7 +112,7 @@ async function discoverPool(opts) {
|
|
|
124
112
|
}
|
|
125
113
|
|
|
126
114
|
var userPool = getUserPool(config);
|
|
127
|
-
pool = pool.concat(userPool || DEFAULT_CLOUD_POOL);
|
|
115
|
+
pool = pool.concat((userPool || DEFAULT_CLOUD_POOL).map(modelRegistry.normalizeSpec));
|
|
128
116
|
|
|
129
117
|
return pool;
|
|
130
118
|
}
|
|
@@ -208,9 +196,7 @@ function formatModelName(spec) {
|
|
|
208
196
|
}
|
|
209
197
|
|
|
210
198
|
function estimateCost(spec, inputTokens, outputTokens) {
|
|
211
|
-
|
|
212
|
-
if (cost <= 0) return 0;
|
|
213
|
-
return (inputTokens / 1e6 + outputTokens / 1e6) * cost;
|
|
199
|
+
return modelRegistry.estimateCost(spec, inputTokens, outputTokens);
|
|
214
200
|
}
|
|
215
201
|
|
|
216
202
|
module.exports = {
|
package/dist/lib/chunk-router.js
CHANGED
|
@@ -12,9 +12,10 @@
|
|
|
12
12
|
|
|
13
13
|
var chunkModels = require('./chunk-models');
|
|
14
14
|
var telemetry = require('./chunk-telemetry');
|
|
15
|
+
var modelPolicy = require('./model-policy');
|
|
15
16
|
|
|
16
|
-
function applyTelemetryPenalty(pool, chunkType) {
|
|
17
|
-
var stats = telemetry.getStatsForChunkType(chunkType);
|
|
17
|
+
function applyTelemetryPenalty(pool, chunkType, scope) {
|
|
18
|
+
var stats = telemetry.getStatsForChunkType(chunkType, scope);
|
|
18
19
|
if (!stats || !stats.length) return pool;
|
|
19
20
|
|
|
20
21
|
var penalized = [];
|
|
@@ -23,11 +24,12 @@ function applyTelemetryPenalty(pool, chunkType) {
|
|
|
23
24
|
var name = chunkModels.formatModelName(spec);
|
|
24
25
|
var record = stats.find(function(s) { return s.model === name; });
|
|
25
26
|
var penalty = 0;
|
|
26
|
-
if (record && record.
|
|
27
|
-
//
|
|
28
|
-
var successRate = record.
|
|
27
|
+
if (record && record.quality_attempts >= 3) {
|
|
28
|
+
// Rate limits are availability failures and never lower a model's quality score.
|
|
29
|
+
var successRate = record.quality_successful / record.quality_attempts;
|
|
29
30
|
if (successRate < 0.5) penalty += 0.01;
|
|
30
31
|
if (successRate < 0.25) penalty += 0.02;
|
|
32
|
+
if (record.confidence === 'legacy') penalty *= 0.25;
|
|
31
33
|
}
|
|
32
34
|
penalized.push(Object.assign({}, spec, { effective_cost: (spec.cost || 0) + penalty }));
|
|
33
35
|
}
|
|
@@ -48,7 +50,20 @@ async function resolveChunkModel(opts) {
|
|
|
48
50
|
|
|
49
51
|
var pool = await chunkModels.discoverPool({ config: config });
|
|
50
52
|
var filtered = chunkModels.filterPool(pool, { effort: effort, budget: budget });
|
|
51
|
-
var ranked = applyTelemetryPenalty(filtered, chunkType
|
|
53
|
+
var ranked = applyTelemetryPenalty(filtered, chunkType, {
|
|
54
|
+
persona: opts.persona,
|
|
55
|
+
effort: effort,
|
|
56
|
+
taskType: opts.taskType || chunkType,
|
|
57
|
+
outputFormat: opts.outputFormat,
|
|
58
|
+
});
|
|
59
|
+
var prioritized = modelPolicy.prioritizePool(ranked, {
|
|
60
|
+
persona: opts.persona,
|
|
61
|
+
effort: effort,
|
|
62
|
+
config: config,
|
|
63
|
+
explicitProvider: opts.explicitProvider,
|
|
64
|
+
explicitModel: opts.explicitModel,
|
|
65
|
+
});
|
|
66
|
+
ranked = prioritized.pool;
|
|
52
67
|
|
|
53
68
|
if (!ranked.length) {
|
|
54
69
|
throw new Error('No chunk model available for effort=' + effort + ' budget=' + budget);
|
|
@@ -57,6 +72,7 @@ async function resolveChunkModel(opts) {
|
|
|
57
72
|
return {
|
|
58
73
|
ranked: ranked,
|
|
59
74
|
selected: ranked[0],
|
|
75
|
+
policy: prioritized.policy,
|
|
60
76
|
};
|
|
61
77
|
}
|
|
62
78
|
|
package/dist/lib/chunk-run.js
CHANGED
|
@@ -17,12 +17,15 @@
|
|
|
17
17
|
var fs = require('fs');
|
|
18
18
|
var os = require('os');
|
|
19
19
|
var path = require('path');
|
|
20
|
+
var crypto = require('crypto');
|
|
20
21
|
|
|
21
22
|
var session = require('./session');
|
|
22
23
|
var planner = require('./chunk-planner');
|
|
23
24
|
var executor = require('./chunk-executor');
|
|
24
25
|
var synthesis = require('./chunk-synthesis');
|
|
25
26
|
var chunkModels = require('./chunk-models');
|
|
27
|
+
var modelPolicy = require('./model-policy');
|
|
28
|
+
var outputQuality = require('./output-quality');
|
|
26
29
|
|
|
27
30
|
var CONFIG_FILE = path.join(os.homedir(), '.analyzthis_design', 'config.json');
|
|
28
31
|
|
|
@@ -47,6 +50,14 @@ async function runChunked(opts) {
|
|
|
47
50
|
if (!task) throw new Error('--task is required');
|
|
48
51
|
|
|
49
52
|
var state = session.show({ project: projectId }) || session.init({ project: projectId });
|
|
53
|
+
var runId = 'chunk-' + Date.now() + '-' + crypto.randomBytes(4).toString('hex');
|
|
54
|
+
state = session.update({
|
|
55
|
+
project: projectId,
|
|
56
|
+
patch: {
|
|
57
|
+
active_run: { id: runId, mode: 'chunked', at: new Date().toISOString() },
|
|
58
|
+
persona_outputs: {},
|
|
59
|
+
},
|
|
60
|
+
});
|
|
50
61
|
|
|
51
62
|
var planResult;
|
|
52
63
|
try {
|
|
@@ -106,6 +117,8 @@ async function runChunked(opts) {
|
|
|
106
117
|
config: config,
|
|
107
118
|
budget: budget,
|
|
108
119
|
callLlmFn: callLlmFn,
|
|
120
|
+
explicitProvider: opts.provider,
|
|
121
|
+
explicitModel: opts.model,
|
|
109
122
|
});
|
|
110
123
|
|
|
111
124
|
results.push(result);
|
|
@@ -140,6 +153,7 @@ async function runChunked(opts) {
|
|
|
140
153
|
}
|
|
141
154
|
|
|
142
155
|
var chunkRun = {
|
|
156
|
+
run_id: runId,
|
|
143
157
|
plan: planResult.plan,
|
|
144
158
|
planner_model: planResult.planner_model,
|
|
145
159
|
planner_warning: planResult.planner_warning,
|
|
@@ -154,10 +168,45 @@ async function runChunked(opts) {
|
|
|
154
168
|
mode: sequential ? 'sequential' : 'parallel',
|
|
155
169
|
};
|
|
156
170
|
|
|
171
|
+
// Default runs must feed the same feedback/evolution path as unchunked runs.
|
|
172
|
+
var personaOutputs = {};
|
|
173
|
+
var fullPrompts = {};
|
|
174
|
+
var structuredOutputs = {};
|
|
175
|
+
results.forEach(function(result) {
|
|
176
|
+
if (!result.persona || !result.success) return;
|
|
177
|
+
var model = result.model_used || {};
|
|
178
|
+
personaOutputs[result.persona] = {
|
|
179
|
+
text: result.output,
|
|
180
|
+
at: new Date().toISOString(),
|
|
181
|
+
accepted: null,
|
|
182
|
+
source: 'chunked',
|
|
183
|
+
run_id: runId,
|
|
184
|
+
structured_output: result.structured_output,
|
|
185
|
+
quality: result.quality,
|
|
186
|
+
attribution: {
|
|
187
|
+
provider: model.provider || 'unknown',
|
|
188
|
+
model: model.model || 'unknown',
|
|
189
|
+
model_id: model.provider && model.model ? chunkModels.formatModelName(model) : 'unknown/unknown',
|
|
190
|
+
effort: result.effort || 'standard',
|
|
191
|
+
policy_version: modelPolicy.VERSION,
|
|
192
|
+
prompt_hash: modelPolicy.promptHash(result.persona),
|
|
193
|
+
validator_version: outputQuality.VERSION,
|
|
194
|
+
},
|
|
195
|
+
};
|
|
196
|
+
if (result.full_prompt) fullPrompts[result.persona] = result.full_prompt;
|
|
197
|
+
if (result.structured_output) structuredOutputs[result.persona] = result.structured_output;
|
|
198
|
+
});
|
|
199
|
+
|
|
157
200
|
var final = session.update({
|
|
158
201
|
project: projectId,
|
|
159
202
|
patch: {
|
|
160
203
|
chunk_run: chunkRun,
|
|
204
|
+
active_run: { id: runId, mode: 'chunked', at: new Date().toISOString() },
|
|
205
|
+
persona_outputs: personaOutputs,
|
|
206
|
+
full_prompts: fullPrompts,
|
|
207
|
+
structured_outputs: structuredOutputs,
|
|
208
|
+
synthesis: syn.composite || syn,
|
|
209
|
+
synthesis_markdown: syn.markdown || '',
|
|
161
210
|
task_type: state.task_type || 'chunked_run',
|
|
162
211
|
mode: 'assess_only',
|
|
163
212
|
digest: Object.assign({}, state.digest || {}, {
|
|
@@ -178,8 +227,13 @@ async function runChunked(opts) {
|
|
|
178
227
|
effort_log: results.map(function(r) {
|
|
179
228
|
return {
|
|
180
229
|
persona: r.persona,
|
|
181
|
-
effort: r.
|
|
230
|
+
effort: r.effort || 'standard',
|
|
231
|
+
provider: r.model_used ? r.model_used.provider : null,
|
|
182
232
|
model: r.model_used ? chunkModels.formatModelName(r.model_used) : null,
|
|
233
|
+
model_id: r.model_used ? chunkModels.formatModelName(r.model_used) : null,
|
|
234
|
+
input_tokens: r.tokens ? r.tokens.input : 0,
|
|
235
|
+
output_tokens: r.tokens ? r.tokens.output : 0,
|
|
236
|
+
run_id: runId,
|
|
183
237
|
scoped_mode: null,
|
|
184
238
|
};
|
|
185
239
|
}),
|
|
@@ -13,6 +13,7 @@
|
|
|
13
13
|
var chunkModels = require('./chunk-models');
|
|
14
14
|
var chunkRouter = require('./chunk-router');
|
|
15
15
|
var synthesis = require('./synthesis');
|
|
16
|
+
var outputQuality = require('./output-quality');
|
|
16
17
|
|
|
17
18
|
function buildSynthesisPrompt(plan, results, contextPack) {
|
|
18
19
|
var lines = [
|
|
@@ -122,6 +123,9 @@ async function synthesize(opts) {
|
|
|
122
123
|
var resolved = await chunkRouter.resolveChunkModel({
|
|
123
124
|
effort: 'hard',
|
|
124
125
|
chunkType: 'synthesis',
|
|
126
|
+
persona: synChunk.persona || 'raj',
|
|
127
|
+
taskType: 'synthesis',
|
|
128
|
+
outputFormat: 'synthesis',
|
|
125
129
|
budget: budget,
|
|
126
130
|
config: config,
|
|
127
131
|
});
|
|
@@ -153,8 +157,19 @@ async function synthesize(opts) {
|
|
|
153
157
|
|
|
154
158
|
var inTok = Math.ceil((prompts.system.length + prompts.user.length) / 4);
|
|
155
159
|
var outTok = Math.ceil(text.length / 4);
|
|
156
|
-
|
|
157
|
-
|
|
160
|
+
var quality = outputQuality.validateOutput(text, {
|
|
161
|
+
persona: synChunk.persona || 'raj',
|
|
162
|
+
outputFormat: 'synthesis',
|
|
163
|
+
});
|
|
164
|
+
|
|
165
|
+
var personaOutputs = {};
|
|
166
|
+
successful.forEach(function(result) {
|
|
167
|
+
if (result.persona) personaOutputs[result.persona] = {
|
|
168
|
+
text: result.output,
|
|
169
|
+
structured_output: result.structured_output,
|
|
170
|
+
};
|
|
171
|
+
});
|
|
172
|
+
var syn = quality.pass && synthesis.buildSynthesis ? synthesis.buildSynthesis(personaOutputs) : null;
|
|
158
173
|
if (!syn) syn = fallbackSynthesis(successful);
|
|
159
174
|
|
|
160
175
|
return {
|
|
@@ -169,6 +184,7 @@ async function synthesize(opts) {
|
|
|
169
184
|
premise_challenge: extractPremiseCheck(text),
|
|
170
185
|
conflict_resolutions: extractConflictResolutions(text),
|
|
171
186
|
raw_synthesis: text,
|
|
187
|
+
quality: quality,
|
|
172
188
|
tokens: { input: inTok, output: outTok },
|
|
173
189
|
cost: chunkModels.estimateCost(model, inTok, outTok),
|
|
174
190
|
};
|
|
@@ -13,31 +13,37 @@ var fs = require('fs');
|
|
|
13
13
|
var path = require('path');
|
|
14
14
|
var os = require('os');
|
|
15
15
|
|
|
16
|
-
var
|
|
16
|
+
var DEFAULT_TELEMETRY_FILE = path.join(os.homedir(), '.analyzthis_design', 'chunk-telemetry.jsonl');
|
|
17
17
|
var MAX_LINES = 10000;
|
|
18
18
|
|
|
19
|
+
function telemetryFile() {
|
|
20
|
+
return process.env.ANALYZTHIS_TELEMETRY_FILE || DEFAULT_TELEMETRY_FILE;
|
|
21
|
+
}
|
|
22
|
+
|
|
19
23
|
function ensureDir() {
|
|
20
|
-
fs.mkdirSync(path.dirname(
|
|
24
|
+
fs.mkdirSync(path.dirname(telemetryFile()), { recursive: true });
|
|
21
25
|
}
|
|
22
26
|
|
|
23
27
|
function loadLines() {
|
|
24
|
-
|
|
28
|
+
var file = telemetryFile();
|
|
29
|
+
if (!fs.existsSync(file)) return [];
|
|
25
30
|
try {
|
|
26
|
-
var text = fs.readFileSync(
|
|
31
|
+
var text = fs.readFileSync(file, 'utf8');
|
|
27
32
|
return text.trim().split('\n').filter(function(l) { return l.trim(); });
|
|
28
33
|
} catch (e) { return []; }
|
|
29
34
|
}
|
|
30
35
|
|
|
31
36
|
function append(record) {
|
|
32
37
|
ensureDir();
|
|
38
|
+
var file = telemetryFile();
|
|
33
39
|
var line = JSON.stringify(record);
|
|
34
|
-
fs.appendFileSync(
|
|
40
|
+
fs.appendFileSync(file, line + '\n');
|
|
35
41
|
|
|
36
42
|
// Simple rotation: if file gets too large, keep last half.
|
|
37
43
|
var lines = loadLines();
|
|
38
44
|
if (lines.length > MAX_LINES) {
|
|
39
45
|
var keep = lines.slice(Math.floor(MAX_LINES / 2));
|
|
40
|
-
fs.writeFileSync(
|
|
46
|
+
fs.writeFileSync(file, keep.join('\n') + '\n');
|
|
41
47
|
}
|
|
42
48
|
}
|
|
43
49
|
|
|
@@ -48,23 +54,76 @@ function record(record) {
|
|
|
48
54
|
append(r);
|
|
49
55
|
}
|
|
50
56
|
|
|
51
|
-
function
|
|
52
|
-
|
|
57
|
+
function matchesScope(record, scope) {
|
|
58
|
+
if (!record.scope_version) return false;
|
|
59
|
+
if (scope.persona && record.persona !== scope.persona) return false;
|
|
60
|
+
if (scope.effort && record.effort !== scope.effort) return false;
|
|
61
|
+
if (scope.task_type && record.task_type !== scope.task_type) return false;
|
|
62
|
+
if (scope.output_format && record.output_format !== scope.output_format) return false;
|
|
63
|
+
return true;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
function aggregate(records, confidence) {
|
|
53
67
|
var byModel = {};
|
|
68
|
+
for (var i = 0; i < records.length; i++) {
|
|
69
|
+
var r = records[i];
|
|
70
|
+
if (!r.model) continue;
|
|
71
|
+
var key = r.model;
|
|
72
|
+
if (!byModel[key]) {
|
|
73
|
+
byModel[key] = {
|
|
74
|
+
model: key,
|
|
75
|
+
attempts: 0,
|
|
76
|
+
transport_successful: 0,
|
|
77
|
+
quality_attempts: 0,
|
|
78
|
+
quality_successful: 0,
|
|
79
|
+
rate_limits: 0,
|
|
80
|
+
total_cost: 0,
|
|
81
|
+
confidence: confidence,
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
byModel[key].attempts++;
|
|
85
|
+
if (r.transport_success === true || (r.transport_success == null && r.success)) {
|
|
86
|
+
byModel[key].transport_successful++;
|
|
87
|
+
}
|
|
88
|
+
if (r.failure_kind === 'rate_limit') byModel[key].rate_limits++;
|
|
89
|
+
if (r.quality_pass != null) {
|
|
90
|
+
byModel[key].quality_attempts++;
|
|
91
|
+
if (r.quality_pass) byModel[key].quality_successful++;
|
|
92
|
+
} else if (r.success != null && !r.failure_kind) {
|
|
93
|
+
// Legacy success means API availability only; retain it as low-confidence quality.
|
|
94
|
+
byModel[key].quality_attempts++;
|
|
95
|
+
if (r.success) byModel[key].quality_successful++;
|
|
96
|
+
}
|
|
97
|
+
byModel[key].total_cost += r.cost || 0;
|
|
98
|
+
}
|
|
99
|
+
return Object.keys(byModel).map(function(k) {
|
|
100
|
+
var item = byModel[k];
|
|
101
|
+
// Backward-compatible names for older callers.
|
|
102
|
+
item.successful = item.quality_successful;
|
|
103
|
+
return item;
|
|
104
|
+
});
|
|
105
|
+
}
|
|
54
106
|
|
|
107
|
+
function getStatsForChunkType(chunkType, opts) {
|
|
108
|
+
opts = opts || {};
|
|
109
|
+
var scope = {
|
|
110
|
+
persona: opts.persona || '',
|
|
111
|
+
effort: opts.effort || '',
|
|
112
|
+
task_type: opts.taskType || opts.task_type || chunkType || '',
|
|
113
|
+
output_format: opts.outputFormat || opts.output_format || '',
|
|
114
|
+
};
|
|
115
|
+
var lines = loadLines();
|
|
116
|
+
var parsed = [];
|
|
55
117
|
for (var i = 0; i < lines.length; i++) {
|
|
56
118
|
try {
|
|
57
119
|
var r = JSON.parse(lines[i]);
|
|
58
|
-
if (
|
|
59
|
-
var key = r.model;
|
|
60
|
-
if (!byModel[key]) byModel[key] = { model: key, attempts: 0, successful: 0, total_cost: 0 };
|
|
61
|
-
byModel[key].attempts++;
|
|
62
|
-
if (r.success) byModel[key].successful++;
|
|
63
|
-
byModel[key].total_cost += r.cost || 0;
|
|
120
|
+
if (r.model) parsed.push(r);
|
|
64
121
|
} catch (e) { /* skip corrupt line */ }
|
|
65
122
|
}
|
|
66
|
-
|
|
67
|
-
|
|
123
|
+
var scoped = parsed.filter(function(r) { return matchesScope(r, scope); });
|
|
124
|
+
if (scoped.length) return aggregate(scoped, 'scoped');
|
|
125
|
+
var legacy = parsed.filter(function(r) { return !r.scope_version; });
|
|
126
|
+
return aggregate(legacy, 'legacy');
|
|
68
127
|
}
|
|
69
128
|
|
|
70
129
|
function getRecentFailures(limit) {
|
|
@@ -84,5 +143,6 @@ module.exports = {
|
|
|
84
143
|
record: record,
|
|
85
144
|
getStatsForChunkType: getStatsForChunkType,
|
|
86
145
|
getRecentFailures: getRecentFailures,
|
|
87
|
-
|
|
146
|
+
telemetryFile: telemetryFile,
|
|
147
|
+
TELEMETRY_FILE: DEFAULT_TELEMETRY_FILE,
|
|
88
148
|
};
|
package/dist/lib/collect.js
CHANGED
|
@@ -25,6 +25,7 @@ const {
|
|
|
25
25
|
writeSourceManifestNotes,
|
|
26
26
|
connectDiscoveredSources,
|
|
27
27
|
} = require('./source-discovery');
|
|
28
|
+
const uiLibraries = require('./ui-libraries');
|
|
28
29
|
|
|
29
30
|
const { resolvePackageRoot } = require('./platforms');
|
|
30
31
|
const PACKAGE_ROOT = resolvePackageRoot(__dirname);
|
|
@@ -675,6 +676,7 @@ async function collect(opts = {}) {
|
|
|
675
676
|
console.log(` Mode: ${modeParts.join(' + ') || 'scan'}\n`);
|
|
676
677
|
|
|
677
678
|
const scan = scanAndDraft({ cwd, vaultPath, dryRun });
|
|
679
|
+
const uiLibDetection = uiLibraries.detectInstalledUiLibraries({ cwd });
|
|
678
680
|
|
|
679
681
|
const sourceDiscoveries = discoverSources
|
|
680
682
|
? discoverKnowledgeSources({ cwd, config })
|
|
@@ -721,10 +723,20 @@ async function collect(opts = {}) {
|
|
|
721
723
|
})),
|
|
722
724
|
webUrls: webPlan.urls.map((u) => ({ url: u.url, source: u.source })),
|
|
723
725
|
webQueries: webPlan.queries,
|
|
726
|
+
uiLibraries: uiLibraries.sessionPayload(uiLibDetection),
|
|
724
727
|
sample: scan.notes.slice(0, 8).map((n) => n.vaultRel),
|
|
725
728
|
}, null, 2));
|
|
726
729
|
console.log('');
|
|
727
|
-
return { vaultPath, projectId, dryRun: true, ...scan, sources: sourceDiscoveries, web: webPlan, enrich: null, sync: null };
|
|
730
|
+
return { vaultPath, projectId, dryRun: true, ...scan, sources: sourceDiscoveries, web: webPlan, uiLibraries: uiLibDetection, enrich: null, sync: null };
|
|
731
|
+
}
|
|
732
|
+
|
|
733
|
+
const uiLibNote = uiLibraries.writeInstalledUiLibrariesNote({
|
|
734
|
+
vaultPath,
|
|
735
|
+
detection: uiLibDetection,
|
|
736
|
+
dryRun: false,
|
|
737
|
+
});
|
|
738
|
+
if (uiLibNote.written) {
|
|
739
|
+
console.log(`\nš§© UI libraries: ${uiLibDetection.installed.length} detected ā Tech/Installed-UI-Libraries.md`);
|
|
728
740
|
}
|
|
729
741
|
|
|
730
742
|
let sourceManifest = { written: [] };
|
|
@@ -806,6 +818,7 @@ async function collect(opts = {}) {
|
|
|
806
818
|
content_sources: sources,
|
|
807
819
|
vault_path: vaultPath,
|
|
808
820
|
last_collect_at: new Date().toISOString(),
|
|
821
|
+
installed_ui_libraries: uiLibraries.sessionPayload(uiLibDetection),
|
|
809
822
|
digest: {
|
|
810
823
|
...(state.digest || {}),
|
|
811
824
|
task_map_summary: `Collected knowledge vault (${scan.notes.length} notes) from ${cwd}`,
|
|
@@ -847,6 +860,7 @@ async function collect(opts = {}) {
|
|
|
847
860
|
},
|
|
848
861
|
enrich: enrichResult,
|
|
849
862
|
sync: syncResult,
|
|
863
|
+
uiLibraries: uiLibDetection,
|
|
850
864
|
};
|
|
851
865
|
}
|
|
852
866
|
|
package/dist/lib/cost.js
CHANGED
|
@@ -20,6 +20,7 @@ const fs = require('fs');
|
|
|
20
20
|
const path = require('path');
|
|
21
21
|
const os = require('os');
|
|
22
22
|
const session = require('./session');
|
|
23
|
+
const modelRegistry = require('./model-registry');
|
|
23
24
|
|
|
24
25
|
const CONFIG_FILE = path.join(os.homedir(), '.analyzthis_design', 'config.json');
|
|
25
26
|
|
|
@@ -51,20 +52,27 @@ function computeRunCost(state, pricing = {}) {
|
|
|
51
52
|
|
|
52
53
|
let total = 0;
|
|
53
54
|
const byExpert = (log.length ? log : [{ persona: 'unknown', effort: m.mode || 'lite', model: null }]).map((entry) => {
|
|
54
|
-
const
|
|
55
|
+
const modelId = entry.model_id || entry.model;
|
|
56
|
+
let price = modelId ? (pricing[modelId] || pricing[entry.model]) : null;
|
|
57
|
+
if (!price && modelId && modelId.includes('/')) {
|
|
58
|
+
const split = modelId.indexOf('/');
|
|
59
|
+
price = modelRegistry.find(modelId.slice(0, split), modelId.slice(split + 1));
|
|
60
|
+
}
|
|
61
|
+
const inTok = entry.input_tokens != null ? entry.input_tokens : inPerCall;
|
|
62
|
+
const outTok = entry.output_tokens != null ? entry.output_tokens : outPerCall;
|
|
55
63
|
let cost = 0;
|
|
56
64
|
if (price) {
|
|
57
|
-
cost =
|
|
58
|
-
} else if (
|
|
59
|
-
unpriced.push(
|
|
65
|
+
cost = modelRegistry.estimateCost(price, inTok, outTok);
|
|
66
|
+
} else if (modelId) {
|
|
67
|
+
unpriced.push(modelId);
|
|
60
68
|
}
|
|
61
69
|
total += cost;
|
|
62
70
|
return {
|
|
63
71
|
persona: entry.persona,
|
|
64
72
|
effort: entry.effort,
|
|
65
|
-
model:
|
|
66
|
-
inTok
|
|
67
|
-
outTok
|
|
73
|
+
model: modelId,
|
|
74
|
+
inTok,
|
|
75
|
+
outTok,
|
|
68
76
|
cost: Number(cost.toFixed(6)),
|
|
69
77
|
};
|
|
70
78
|
});
|
package/dist/lib/design-spec.js
CHANGED
|
@@ -98,7 +98,28 @@ function validateDesignSpec(spec) {
|
|
|
98
98
|
}
|
|
99
99
|
}
|
|
100
100
|
|
|
101
|
-
|
|
101
|
+
if (spec.motion && spec.motion.enabled) {
|
|
102
|
+
for (const key of ['library', 'import_path', 'timing_ms', 'easing', 'reduced_motion']) {
|
|
103
|
+
if (!spec.motion[key] || !String(spec.motion[key]).trim()) {
|
|
104
|
+
errors.push(`motion.${key} is required when motion.enabled is true`);
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
const timing = String(spec.motion.timing_ms || '');
|
|
108
|
+
const ms = parseInt(timing, 10);
|
|
109
|
+
if (!Number.isNaN(ms) && ms > 400) {
|
|
110
|
+
errors.push('motion.timing_ms must stay at or under 400ms for product UI (cite motion.csv)');
|
|
111
|
+
}
|
|
112
|
+
const lib = String(spec.motion.library || '').toLowerCase();
|
|
113
|
+
const marketing = ['magic', 'aceternity', '21st', 'react bits', 'lottie', 'rive', 'lenis', 'spline'];
|
|
114
|
+
const notes = String(spec.motion.notes || '').toLowerCase();
|
|
115
|
+
const allowedSurface = /landing|onboard|first-run|marketing|empty/.test(notes);
|
|
116
|
+
if (marketing.some((k) => lib.indexOf(k) !== -1) && !allowedSurface) {
|
|
117
|
+
errors.push('motion.library looks like marketing FX ā set notes to landing/onboarding/first-run or pick a product library (Motion, Auto-Animate, CSS)');
|
|
118
|
+
}
|
|
119
|
+
if (Array.isArray(spec.motion.moments) && spec.motion.moments.length > 1) {
|
|
120
|
+
errors.push('motion.moments may contain at most one peak (Zara rule)');
|
|
121
|
+
}
|
|
122
|
+
}
|
|
102
123
|
if (Array.isArray(spec.components) && Array.isArray(spec.information_hierarchy) && spec.information_hierarchy.length) {
|
|
103
124
|
const hasRank1 = spec.components.some((c) => c.hierarchy_rank === 1);
|
|
104
125
|
if (!hasRank1) {
|
|
@@ -208,7 +229,16 @@ function templateMarkdown() {
|
|
|
208
229
|
"error": "Inline alert + retry",
|
|
209
230
|
"success": "Toast + updated primary data"
|
|
210
231
|
},
|
|
211
|
-
"motion": {
|
|
232
|
+
"motion": {
|
|
233
|
+
"enabled": false,
|
|
234
|
+
"library": "",
|
|
235
|
+
"import_path": "",
|
|
236
|
+
"timing_ms": "150-250",
|
|
237
|
+
"easing": "ease-out",
|
|
238
|
+
"reduced_motion": "opacity only or none",
|
|
239
|
+
"moments": [],
|
|
240
|
+
"notes": ""
|
|
241
|
+
},
|
|
212
242
|
"do": ["Single primary CTA above the fold", "Use existing shadcn components only"],
|
|
213
243
|
"dont": ["Invent hex colors", "Add second primary button"],
|
|
214
244
|
"wireframe_ref": "synthesized",
|