@onthink/prompt-observer 0.1.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/PROMPT_OBSERVER.md +36 -8
- package/README.md +46 -14
- package/bin/prompt-observer.mjs +734 -246
- package/package.json +2 -1
- package/pricing/models.json +23 -0
- package/schema/event.schema.json +48 -84
package/bin/prompt-observer.mjs
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
|
|
3
|
+
import { createReadStream } from "node:fs";
|
|
3
4
|
import {
|
|
4
5
|
appendFile,
|
|
5
6
|
copyFile,
|
|
@@ -9,10 +10,13 @@ import {
|
|
|
9
10
|
stat,
|
|
10
11
|
writeFile,
|
|
11
12
|
} from "node:fs/promises";
|
|
12
|
-
import {
|
|
13
|
+
import { createInterface } from "node:readline";
|
|
14
|
+
import { basename, dirname, join, resolve } from "node:path";
|
|
13
15
|
import { fileURLToPath } from "node:url";
|
|
14
16
|
|
|
15
|
-
const SCHEMA_VERSION = "1.
|
|
17
|
+
const SCHEMA_VERSION = "1.1";
|
|
18
|
+
const SUPPORTED_SCHEMA_VERSIONS = new Set(["1.0", "1.1"]);
|
|
19
|
+
const DEFAULT_REPORT_LIMIT = 50;
|
|
16
20
|
const SCORE_FIELDS = [
|
|
17
21
|
"intent_clarity",
|
|
18
22
|
"context_sufficiency",
|
|
@@ -21,16 +25,8 @@ const SCORE_FIELDS = [
|
|
|
21
25
|
"acceptance_criteria",
|
|
22
26
|
"verification_plan",
|
|
23
27
|
];
|
|
24
|
-
const
|
|
25
|
-
|
|
26
|
-
"debugging",
|
|
27
|
-
"planning",
|
|
28
|
-
"review",
|
|
29
|
-
"testing",
|
|
30
|
-
"documentation",
|
|
31
|
-
"refactoring",
|
|
32
|
-
"other",
|
|
33
|
-
]);
|
|
28
|
+
const INSIGHT_CATEGORIES = new Set([...SCORE_FIELDS, "output_format", "other"]);
|
|
29
|
+
const TASK_TYPES = new Set(["coding", "debugging", "planning", "review", "testing", "documentation", "refactoring", "other"]);
|
|
34
30
|
const RISK_LEVELS = new Set(["low", "medium", "high"]);
|
|
35
31
|
const WEAKNESS_CATEGORIES = new Set([
|
|
36
32
|
"missing_context",
|
|
@@ -48,72 +44,109 @@ const RESULT_STATUSES = new Set(["completed", "partial", "blocked"]);
|
|
|
48
44
|
const TEST_STATUSES = new Set(["passed", "failed", "not_run", "unknown"]);
|
|
49
45
|
const USAGE_SOURCES = new Set(["platform_reported", "estimated", "unavailable"]);
|
|
50
46
|
const ROOT_KEYS = new Set([
|
|
51
|
-
"schema_version",
|
|
52
|
-
"
|
|
53
|
-
"
|
|
54
|
-
"task_type",
|
|
55
|
-
"prompt_summary",
|
|
56
|
-
...SCORE_FIELDS,
|
|
57
|
-
"ambiguity_risk",
|
|
58
|
-
"strengths",
|
|
59
|
-
"weaknesses",
|
|
60
|
-
"improvement_suggestions",
|
|
61
|
-
"execution_signals",
|
|
62
|
-
"usage",
|
|
47
|
+
"schema_version", "event_id", "timestamp", "task_type", "prompt_summary",
|
|
48
|
+
...SCORE_FIELDS, "ambiguity_risk", "strengths", "weaknesses",
|
|
49
|
+
"improvement_suggestions", "execution_signals", "usage",
|
|
63
50
|
]);
|
|
64
51
|
const BANNED_KEYS = new Set([
|
|
65
|
-
"prompt",
|
|
66
|
-
"
|
|
67
|
-
"
|
|
68
|
-
"user_prompt",
|
|
69
|
-
"response",
|
|
70
|
-
"raw_response",
|
|
71
|
-
"full_response",
|
|
72
|
-
"assistant_response",
|
|
73
|
-
"reasoning",
|
|
74
|
-
"chain_of_thought",
|
|
75
|
-
"system_prompt",
|
|
52
|
+
"prompt", "raw_prompt", "full_prompt", "user_prompt", "response",
|
|
53
|
+
"raw_response", "full_response", "assistant_response", "reasoning",
|
|
54
|
+
"chain_of_thought", "system_prompt",
|
|
76
55
|
]);
|
|
77
56
|
|
|
78
57
|
function isObject(value) {
|
|
79
58
|
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
80
59
|
}
|
|
81
60
|
|
|
61
|
+
async function exists(path) {
|
|
62
|
+
try {
|
|
63
|
+
await stat(path);
|
|
64
|
+
return true;
|
|
65
|
+
} catch (error) {
|
|
66
|
+
if (error.code === "ENOENT") return false;
|
|
67
|
+
throw error;
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
async function loadPricingSnapshot() {
|
|
72
|
+
const selfDirectory = dirname(fileURLToPath(import.meta.url));
|
|
73
|
+
const candidates = [
|
|
74
|
+
join(selfDirectory, "pricing.json"),
|
|
75
|
+
join(selfDirectory, "..", "pricing", "models.json"),
|
|
76
|
+
];
|
|
77
|
+
for (const candidate of candidates) {
|
|
78
|
+
if (await exists(candidate)) return JSON.parse(await readFile(candidate, "utf8"));
|
|
79
|
+
}
|
|
80
|
+
return {
|
|
81
|
+
schema_version: "1.0",
|
|
82
|
+
updated_at: null,
|
|
83
|
+
currency: "USD",
|
|
84
|
+
unit_tokens: 1000000,
|
|
85
|
+
source_url: null,
|
|
86
|
+
models: {},
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
const PRICING = await loadPricingSnapshot();
|
|
91
|
+
|
|
82
92
|
function addUnknownKeyErrors(value, allowed, path, errors) {
|
|
83
93
|
for (const key of Object.keys(value)) {
|
|
84
|
-
if (!allowed.has(key)) errors.push(
|
|
94
|
+
if (!allowed.has(key)) errors.push(path + "." + key + " is not allowed");
|
|
85
95
|
}
|
|
86
96
|
}
|
|
87
97
|
|
|
88
|
-
function validateShortText(value, path, errors,
|
|
98
|
+
function validateShortText(value, path, errors, options = {}) {
|
|
99
|
+
const max = options.max ?? 500;
|
|
89
100
|
if (typeof value !== "string" || value.trim().length === 0) {
|
|
90
|
-
errors.push(
|
|
101
|
+
errors.push(path + " must be a non-empty string");
|
|
91
102
|
} else if (value.length > max) {
|
|
92
|
-
errors.push(
|
|
103
|
+
errors.push(path + " must be at most " + max + " characters");
|
|
93
104
|
}
|
|
94
105
|
}
|
|
95
106
|
|
|
96
107
|
function validateTextArray(value, path, errors, maxItems = 10) {
|
|
97
108
|
if (!Array.isArray(value)) {
|
|
98
|
-
errors.push(
|
|
109
|
+
errors.push(path + " must be an array");
|
|
99
110
|
return;
|
|
100
111
|
}
|
|
101
|
-
if (value.length > maxItems) errors.push(
|
|
102
|
-
value.forEach((item, index) => validateShortText(item,
|
|
112
|
+
if (value.length > maxItems) errors.push(path + " must contain at most " + maxItems + " items");
|
|
113
|
+
value.forEach((item, index) => validateShortText(item, path + "[" + index + "]", errors));
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
function validateInsightArray(value, path, errors, schemaVersion) {
|
|
117
|
+
if (schemaVersion === "1.0") {
|
|
118
|
+
validateTextArray(value, path, errors);
|
|
119
|
+
return;
|
|
120
|
+
}
|
|
121
|
+
if (!Array.isArray(value)) {
|
|
122
|
+
errors.push(path + " must be an array");
|
|
123
|
+
return;
|
|
124
|
+
}
|
|
125
|
+
if (value.length > 10) errors.push(path + " must contain at most 10 items");
|
|
126
|
+
value.forEach((item, index) => {
|
|
127
|
+
const itemPath = path + "[" + index + "]";
|
|
128
|
+
if (!isObject(item)) {
|
|
129
|
+
errors.push(itemPath + " must be an object in schema 1.1");
|
|
130
|
+
return;
|
|
131
|
+
}
|
|
132
|
+
addUnknownKeyErrors(item, new Set(["category", "message"]), itemPath, errors);
|
|
133
|
+
if (!INSIGHT_CATEGORIES.has(item.category)) errors.push(itemPath + ".category is invalid");
|
|
134
|
+
validateShortText(item.message, itemPath + ".message", errors);
|
|
135
|
+
});
|
|
103
136
|
}
|
|
104
137
|
|
|
105
138
|
function findBannedKeys(value, path, errors) {
|
|
106
139
|
if (Array.isArray(value)) {
|
|
107
|
-
value.forEach((item, index) => findBannedKeys(item,
|
|
140
|
+
value.forEach((item, index) => findBannedKeys(item, path + "[" + index + "]", errors));
|
|
108
141
|
return;
|
|
109
142
|
}
|
|
110
143
|
if (!isObject(value)) return;
|
|
111
144
|
for (const [key, child] of Object.entries(value)) {
|
|
112
145
|
const normalized = key.toLowerCase().replaceAll("-", "_");
|
|
113
146
|
if (BANNED_KEYS.has(normalized)) {
|
|
114
|
-
errors.push(
|
|
147
|
+
errors.push(path + "." + key + " is prohibited because raw prompt, response, or private reasoning must not be stored");
|
|
115
148
|
}
|
|
116
|
-
findBannedKeys(child,
|
|
149
|
+
findBannedKeys(child, path + "." + key, errors);
|
|
117
150
|
}
|
|
118
151
|
}
|
|
119
152
|
|
|
@@ -126,65 +159,77 @@ function findLikelySecrets(value, errors) {
|
|
|
126
159
|
[/\bBearer\s+[A-Za-z0-9._~+/=-]{12,}/i, "a bearer token"],
|
|
127
160
|
];
|
|
128
161
|
for (const [pattern, label] of patterns) {
|
|
129
|
-
if (pattern.test(serialized)) errors.push(
|
|
162
|
+
if (pattern.test(serialized)) errors.push("event appears to contain " + label + "; redact it before logging");
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
export function calculateEstimatedCost(usage, pricing = PRICING) {
|
|
167
|
+
if (!usage || !usage.model || usage.input_tokens === null || usage.output_tokens === null) return null;
|
|
168
|
+
const rate = pricing.models?.[usage.model];
|
|
169
|
+
if (!rate) return null;
|
|
170
|
+
const unit = pricing.unit_tokens || 1000000;
|
|
171
|
+
return (usage.input_tokens * rate.input_usd + usage.output_tokens * rate.output_usd) / unit;
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
export function estimateTextTokens(text) {
|
|
175
|
+
if (typeof text !== "string" || text.length === 0) return 0;
|
|
176
|
+
let latin = 0;
|
|
177
|
+
let nonLatin = 0;
|
|
178
|
+
let symbols = 0;
|
|
179
|
+
for (const character of text) {
|
|
180
|
+
if (/\s/u.test(character)) continue;
|
|
181
|
+
if (/[A-Za-z0-9]/u.test(character)) latin += 1;
|
|
182
|
+
else if (/[\p{L}\p{N}\p{M}]/u.test(character)) nonLatin += 1;
|
|
183
|
+
else symbols += 1;
|
|
130
184
|
}
|
|
185
|
+
return Math.ceil(latin / 4 + nonLatin / 2 + symbols / 2);
|
|
131
186
|
}
|
|
132
187
|
|
|
133
188
|
export function validateEvent(event) {
|
|
134
189
|
const errors = [];
|
|
135
190
|
if (!isObject(event)) return ["event must be a JSON object"];
|
|
136
|
-
|
|
137
191
|
addUnknownKeyErrors(event, ROOT_KEYS, "$", errors);
|
|
138
192
|
findBannedKeys(event, "$", errors);
|
|
139
193
|
findLikelySecrets(event, errors);
|
|
140
|
-
|
|
141
194
|
for (const key of ROOT_KEYS) {
|
|
142
|
-
if (!(key in event)) errors.push(
|
|
195
|
+
if (!(key in event)) errors.push("$." + key + " is required");
|
|
143
196
|
}
|
|
144
197
|
if (errors.some((error) => error.endsWith(" is required"))) return errors;
|
|
145
198
|
|
|
146
|
-
if (event.schema_version
|
|
147
|
-
errors.push(
|
|
199
|
+
if (!SUPPORTED_SCHEMA_VERSIONS.has(event.schema_version)) {
|
|
200
|
+
errors.push("$.schema_version must equal 1.0 or 1.1");
|
|
148
201
|
}
|
|
149
|
-
if (
|
|
150
|
-
typeof event.event_id !== "string" ||
|
|
151
|
-
!/^[A-Za-z0-9][A-Za-z0-9._:-]{7,127}$/.test(event.event_id)
|
|
152
|
-
) {
|
|
202
|
+
if (typeof event.event_id !== "string" || !/^[A-Za-z0-9][A-Za-z0-9._:-]{7,127}$/.test(event.event_id)) {
|
|
153
203
|
errors.push("$.event_id must be 8-128 safe identifier characters");
|
|
154
204
|
}
|
|
155
|
-
if (
|
|
156
|
-
typeof event.timestamp !== "string" ||
|
|
157
|
-
!event.timestamp.includes("T") ||
|
|
158
|
-
Number.isNaN(Date.parse(event.timestamp))
|
|
159
|
-
) {
|
|
205
|
+
if (typeof event.timestamp !== "string" || !event.timestamp.includes("T") || Number.isNaN(Date.parse(event.timestamp))) {
|
|
160
206
|
errors.push("$.timestamp must be a valid ISO-8601 date-time");
|
|
161
207
|
}
|
|
162
208
|
if (!TASK_TYPES.has(event.task_type)) errors.push("$.task_type is invalid");
|
|
163
209
|
validateShortText(event.prompt_summary, "$.prompt_summary", errors);
|
|
164
|
-
|
|
165
210
|
for (const field of SCORE_FIELDS) {
|
|
166
211
|
if (!Number.isInteger(event[field]) || event[field] < 0 || event[field] > 10) {
|
|
167
|
-
errors.push(
|
|
212
|
+
errors.push("$." + field + " must be an integer from 0 to 10");
|
|
168
213
|
}
|
|
169
214
|
}
|
|
170
215
|
if (!RISK_LEVELS.has(event.ambiguity_risk)) errors.push("$.ambiguity_risk is invalid");
|
|
171
|
-
|
|
172
|
-
|
|
216
|
+
validateInsightArray(event.strengths, "$.strengths", errors, event.schema_version);
|
|
217
|
+
validateInsightArray(event.improvement_suggestions, "$.improvement_suggestions", errors, event.schema_version);
|
|
173
218
|
|
|
174
219
|
if (!Array.isArray(event.weaknesses)) {
|
|
175
220
|
errors.push("$.weaknesses must be an array");
|
|
176
221
|
} else {
|
|
177
222
|
if (event.weaknesses.length > 10) errors.push("$.weaknesses must contain at most 10 items");
|
|
178
223
|
event.weaknesses.forEach((weakness, index) => {
|
|
179
|
-
const path =
|
|
224
|
+
const path = "$.weaknesses[" + index + "]";
|
|
180
225
|
if (!isObject(weakness)) {
|
|
181
|
-
errors.push(
|
|
226
|
+
errors.push(path + " must be an object");
|
|
182
227
|
return;
|
|
183
228
|
}
|
|
184
229
|
addUnknownKeyErrors(weakness, new Set(["category", "severity", "message"]), path, errors);
|
|
185
|
-
if (!WEAKNESS_CATEGORIES.has(weakness.category)) errors.push(
|
|
186
|
-
if (!SEVERITIES.has(weakness.severity)) errors.push(
|
|
187
|
-
validateShortText(weakness.message,
|
|
230
|
+
if (!WEAKNESS_CATEGORIES.has(weakness.category)) errors.push(path + ".category is invalid");
|
|
231
|
+
if (!SEVERITIES.has(weakness.severity)) errors.push(path + ".severity is invalid");
|
|
232
|
+
validateShortText(weakness.message, path + ".message", errors);
|
|
188
233
|
});
|
|
189
234
|
}
|
|
190
235
|
|
|
@@ -193,25 +238,21 @@ export function validateEvent(event) {
|
|
|
193
238
|
errors.push("$.execution_signals must be an object");
|
|
194
239
|
} else {
|
|
195
240
|
addUnknownKeyErrors(signals, new Set(["result_status", "files_changed", "tests_run"]), "$.execution_signals", errors);
|
|
196
|
-
if (!RESULT_STATUSES.has(signals.result_status))
|
|
197
|
-
errors.push("$.execution_signals.result_status is invalid");
|
|
198
|
-
}
|
|
241
|
+
if (!RESULT_STATUSES.has(signals.result_status)) errors.push("$.execution_signals.result_status is invalid");
|
|
199
242
|
validateTextArray(signals.files_changed, "$.execution_signals.files_changed", errors, 500);
|
|
200
243
|
if (!Array.isArray(signals.tests_run)) {
|
|
201
244
|
errors.push("$.execution_signals.tests_run must be an array");
|
|
202
245
|
} else {
|
|
203
|
-
if (signals.tests_run.length > 100)
|
|
204
|
-
errors.push("$.execution_signals.tests_run must contain at most 100 items");
|
|
205
|
-
}
|
|
246
|
+
if (signals.tests_run.length > 100) errors.push("$.execution_signals.tests_run must contain at most 100 items");
|
|
206
247
|
signals.tests_run.forEach((test, index) => {
|
|
207
|
-
const path =
|
|
248
|
+
const path = "$.execution_signals.tests_run[" + index + "]";
|
|
208
249
|
if (!isObject(test)) {
|
|
209
|
-
errors.push(
|
|
250
|
+
errors.push(path + " must be an object");
|
|
210
251
|
return;
|
|
211
252
|
}
|
|
212
253
|
addUnknownKeyErrors(test, new Set(["name", "status"]), path, errors);
|
|
213
|
-
validateShortText(test.name,
|
|
214
|
-
if (!TEST_STATUSES.has(test.status)) errors.push(
|
|
254
|
+
validateShortText(test.name, path + ".name", errors);
|
|
255
|
+
if (!TEST_STATUSES.has(test.status)) errors.push(path + ".status is invalid");
|
|
215
256
|
});
|
|
216
257
|
}
|
|
217
258
|
}
|
|
@@ -220,34 +261,24 @@ export function validateEvent(event) {
|
|
|
220
261
|
if (!isObject(usage)) {
|
|
221
262
|
errors.push("$.usage must be an object");
|
|
222
263
|
} else {
|
|
223
|
-
const usageKeys = new Set([
|
|
224
|
-
"model",
|
|
225
|
-
"input_tokens",
|
|
226
|
-
"output_tokens",
|
|
227
|
-
"cost_usd",
|
|
228
|
-
"source",
|
|
229
|
-
"estimation_method",
|
|
230
|
-
]);
|
|
264
|
+
const usageKeys = new Set(["model", "input_tokens", "output_tokens", "cost_usd", "source", "estimation_method"]);
|
|
231
265
|
addUnknownKeyErrors(usage, usageKeys, "$.usage", errors);
|
|
232
266
|
for (const key of usageKeys) {
|
|
233
|
-
if (!(key in usage)) errors.push(
|
|
267
|
+
if (!(key in usage)) errors.push("$.usage." + key + " is required");
|
|
234
268
|
}
|
|
235
269
|
if (usage.model !== null && (typeof usage.model !== "string" || usage.model.length > 200)) {
|
|
236
270
|
errors.push("$.usage.model must be null or a string up to 200 characters");
|
|
237
271
|
}
|
|
238
272
|
for (const field of ["input_tokens", "output_tokens"]) {
|
|
239
273
|
if (usage[field] !== null && (!Number.isInteger(usage[field]) || usage[field] < 0)) {
|
|
240
|
-
errors.push(
|
|
274
|
+
errors.push("$.usage." + field + " must be null or a non-negative integer");
|
|
241
275
|
}
|
|
242
276
|
}
|
|
243
277
|
if (usage.cost_usd !== null && (typeof usage.cost_usd !== "number" || usage.cost_usd < 0)) {
|
|
244
278
|
errors.push("$.usage.cost_usd must be null or a non-negative number");
|
|
245
279
|
}
|
|
246
280
|
if (!USAGE_SOURCES.has(usage.source)) errors.push("$.usage.source is invalid");
|
|
247
|
-
if (
|
|
248
|
-
usage.estimation_method !== null &&
|
|
249
|
-
(typeof usage.estimation_method !== "string" || usage.estimation_method.trim() === "" || usage.estimation_method.length > 300)
|
|
250
|
-
) {
|
|
281
|
+
if (usage.estimation_method !== null && (typeof usage.estimation_method !== "string" || usage.estimation_method.trim() === "" || usage.estimation_method.length > 300)) {
|
|
251
282
|
errors.push("$.usage.estimation_method must be null or a non-empty string up to 300 characters");
|
|
252
283
|
}
|
|
253
284
|
const metrics = [usage.model, usage.input_tokens, usage.output_tokens, usage.cost_usd];
|
|
@@ -264,6 +295,14 @@ export function validateEvent(event) {
|
|
|
264
295
|
if (usage.input_tokens === null && usage.output_tokens === null && usage.cost_usd === null) {
|
|
265
296
|
errors.push("$.usage estimated source requires at least one estimated numeric metric");
|
|
266
297
|
}
|
|
298
|
+
if (event.schema_version === "1.1" && usage.cost_usd !== null) {
|
|
299
|
+
const expected = calculateEstimatedCost(usage);
|
|
300
|
+
if (expected === null) {
|
|
301
|
+
errors.push("$.usage.cost_usd requires both token counts and an exact model match in pricing.json");
|
|
302
|
+
} else if (Math.abs(expected - usage.cost_usd) > Math.max(1e-9, expected * 0.000001)) {
|
|
303
|
+
errors.push("$.usage.cost_usd does not match the versioned pricing snapshot");
|
|
304
|
+
}
|
|
305
|
+
}
|
|
267
306
|
}
|
|
268
307
|
if (usage.source === "platform_reported" && usage.estimation_method !== null) {
|
|
269
308
|
errors.push("$.usage.estimation_method must be null when source is platform_reported");
|
|
@@ -272,20 +311,9 @@ export function validateEvent(event) {
|
|
|
272
311
|
errors.push("$.usage platform_reported source requires at least one reported metric");
|
|
273
312
|
}
|
|
274
313
|
}
|
|
275
|
-
|
|
276
314
|
return errors;
|
|
277
315
|
}
|
|
278
316
|
|
|
279
|
-
async function exists(path) {
|
|
280
|
-
try {
|
|
281
|
-
await stat(path);
|
|
282
|
-
return true;
|
|
283
|
-
} catch (error) {
|
|
284
|
-
if (error.code === "ENOENT") return false;
|
|
285
|
-
throw error;
|
|
286
|
-
}
|
|
287
|
-
}
|
|
288
|
-
|
|
289
317
|
async function findResourceRoot() {
|
|
290
318
|
const selfDirectory = dirname(fileURLToPath(import.meta.url));
|
|
291
319
|
const candidates = [resolve(selfDirectory, ".."), selfDirectory];
|
|
@@ -294,29 +322,32 @@ async function findResourceRoot() {
|
|
|
294
322
|
const schema = (await exists(join(candidate, "schema", "event.schema.json")))
|
|
295
323
|
? join(candidate, "schema", "event.schema.json")
|
|
296
324
|
: join(candidate, "event.schema.json");
|
|
297
|
-
|
|
298
|
-
|
|
325
|
+
const pricing = (await exists(join(candidate, "pricing", "models.json")))
|
|
326
|
+
? join(candidate, "pricing", "models.json")
|
|
327
|
+
: join(candidate, "pricing.json");
|
|
328
|
+
if ((await exists(contract)) && (await exists(schema)) && (await exists(pricing))) {
|
|
329
|
+
return { root: candidate, contract, schema, pricing };
|
|
299
330
|
}
|
|
300
331
|
}
|
|
301
|
-
throw new Error("Could not locate
|
|
332
|
+
throw new Error("Could not locate the Prompt Observer contract, schema, and pricing snapshot next to the CLI.");
|
|
302
333
|
}
|
|
303
334
|
|
|
304
335
|
async function copyUnlessPresent(source, destination, actions) {
|
|
305
336
|
if (await exists(destination)) {
|
|
306
|
-
actions.push(
|
|
337
|
+
actions.push("kept " + destination);
|
|
307
338
|
return;
|
|
308
339
|
}
|
|
309
340
|
await copyFile(source, destination);
|
|
310
|
-
actions.push(
|
|
341
|
+
actions.push("created " + destination);
|
|
311
342
|
}
|
|
312
343
|
|
|
313
344
|
async function writeUnlessPresent(destination, content, actions) {
|
|
314
345
|
if (await exists(destination)) {
|
|
315
|
-
actions.push(
|
|
346
|
+
actions.push("kept " + destination);
|
|
316
347
|
return;
|
|
317
348
|
}
|
|
318
349
|
await writeFile(destination, content, "utf8");
|
|
319
|
-
actions.push(
|
|
350
|
+
actions.push("created " + destination);
|
|
320
351
|
}
|
|
321
352
|
|
|
322
353
|
export async function initProject(targetPath) {
|
|
@@ -325,22 +356,21 @@ export async function initProject(targetPath) {
|
|
|
325
356
|
const resources = await findResourceRoot();
|
|
326
357
|
const selfPath = fileURLToPath(import.meta.url);
|
|
327
358
|
const actions = [];
|
|
328
|
-
|
|
329
359
|
await mkdir(observerDirectory, { recursive: true });
|
|
330
360
|
await mkdir(join(observerDirectory, "pending"), { recursive: true });
|
|
331
361
|
await copyUnlessPresent(resources.contract, join(observerDirectory, "PROMPT_OBSERVER.md"), actions);
|
|
332
362
|
await copyUnlessPresent(resources.schema, join(observerDirectory, "event.schema.json"), actions);
|
|
363
|
+
await copyUnlessPresent(resources.pricing, join(observerDirectory, "pricing.json"), actions);
|
|
333
364
|
await copyUnlessPresent(selfPath, join(observerDirectory, "prompt-observer.mjs"), actions);
|
|
334
365
|
await writeUnlessPresent(
|
|
335
366
|
join(observerDirectory, ".gitignore"),
|
|
336
|
-
["events.jsonl", "report.md", "pending/", "*.tmp", ""].join("\n"),
|
|
367
|
+
["events.jsonl", "report.md", "report.html", "pending/", "*.tmp", ""].join("\n"),
|
|
337
368
|
actions,
|
|
338
369
|
);
|
|
339
370
|
const eventsFile = join(observerDirectory, "events.jsonl");
|
|
340
371
|
const file = await open(eventsFile, "a");
|
|
341
372
|
await file.close();
|
|
342
|
-
actions.push((await stat(eventsFile)).size === 0 ?
|
|
343
|
-
|
|
373
|
+
actions.push((await stat(eventsFile)).size === 0 ? "ready " + eventsFile : "kept " + eventsFile);
|
|
344
374
|
return { target, observerDirectory, actions };
|
|
345
375
|
}
|
|
346
376
|
|
|
@@ -349,9 +379,8 @@ async function resolveObserverDirectory(targetPath) {
|
|
|
349
379
|
const resolved = resolve(targetPath);
|
|
350
380
|
const candidate = basename(resolved) === ".prompt-observer" ? resolved : join(resolved, ".prompt-observer");
|
|
351
381
|
if (await exists(candidate)) return candidate;
|
|
352
|
-
throw new Error(
|
|
382
|
+
throw new Error("Prompt Observer is not initialized at " + resolved + ". Run init first.");
|
|
353
383
|
}
|
|
354
|
-
|
|
355
384
|
let current = resolve(".");
|
|
356
385
|
while (true) {
|
|
357
386
|
const candidate = join(current, ".prompt-observer");
|
|
@@ -363,25 +392,64 @@ async function resolveObserverDirectory(targetPath) {
|
|
|
363
392
|
throw new Error("No .prompt-observer directory found from the current directory upward. Run init first.");
|
|
364
393
|
}
|
|
365
394
|
|
|
366
|
-
async function
|
|
367
|
-
if (!(await exists(eventsPath))) return
|
|
368
|
-
const
|
|
369
|
-
const
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
395
|
+
async function* streamEvents(eventsPath) {
|
|
396
|
+
if (!(await exists(eventsPath))) return;
|
|
397
|
+
const input = createReadStream(eventsPath, { encoding: "utf8" });
|
|
398
|
+
const lines = createInterface({ input, crlfDelay: Infinity });
|
|
399
|
+
let lineNumber = 0;
|
|
400
|
+
try {
|
|
401
|
+
for await (const line of lines) {
|
|
402
|
+
lineNumber += 1;
|
|
403
|
+
if (line.trim() === "") continue;
|
|
404
|
+
let event;
|
|
405
|
+
try {
|
|
406
|
+
event = JSON.parse(line);
|
|
407
|
+
} catch (error) {
|
|
408
|
+
throw new Error("Invalid JSON in " + eventsPath + " at line " + lineNumber + ": " + error.message);
|
|
409
|
+
}
|
|
410
|
+
const validationErrors = validateEvent(event);
|
|
411
|
+
if (validationErrors.length > 0) {
|
|
412
|
+
throw new Error("Invalid event in " + eventsPath + " at line " + lineNumber + ":\n- " + validationErrors.join("\n- "));
|
|
413
|
+
}
|
|
414
|
+
yield event;
|
|
381
415
|
}
|
|
382
|
-
|
|
416
|
+
} finally {
|
|
417
|
+
lines.close();
|
|
418
|
+
input.destroy();
|
|
383
419
|
}
|
|
384
|
-
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
async function selectReportWindow(eventsPath, options = {}) {
|
|
423
|
+
const all = options.all === true;
|
|
424
|
+
const limit = options.limit ?? DEFAULT_REPORT_LIMIT;
|
|
425
|
+
const retained = [];
|
|
426
|
+
let totalEventCount = 0;
|
|
427
|
+
for await (const event of streamEvents(eventsPath)) {
|
|
428
|
+
totalEventCount += 1;
|
|
429
|
+
retained.push(event);
|
|
430
|
+
if (!all && retained.length > limit * 2) retained.shift();
|
|
431
|
+
}
|
|
432
|
+
if (all) return { current: retained, previous: [], totalEventCount };
|
|
433
|
+
const currentStart = Math.max(0, retained.length - limit);
|
|
434
|
+
const previousStart = Math.max(0, currentStart - limit);
|
|
435
|
+
return {
|
|
436
|
+
current: retained.slice(currentStart),
|
|
437
|
+
previous: retained.slice(previousStart, currentStart),
|
|
438
|
+
totalEventCount,
|
|
439
|
+
};
|
|
440
|
+
}
|
|
441
|
+
|
|
442
|
+
function enrichEstimatedUsage(event) {
|
|
443
|
+
if (event.schema_version !== "1.1" || event.usage.source !== "estimated" || event.usage.cost_usd !== null) return event;
|
|
444
|
+
const calculated = calculateEstimatedCost(event.usage);
|
|
445
|
+
if (calculated === null) return event;
|
|
446
|
+
const enriched = structuredClone(event);
|
|
447
|
+
enriched.usage.cost_usd = calculated;
|
|
448
|
+
const marker = "pricing_snapshot:" + PRICING.updated_at;
|
|
449
|
+
if (!enriched.usage.estimation_method.includes(marker)) {
|
|
450
|
+
enriched.usage.estimation_method += "; " + marker;
|
|
451
|
+
}
|
|
452
|
+
return enriched;
|
|
385
453
|
}
|
|
386
454
|
|
|
387
455
|
export async function logEvent(eventFile, targetPath) {
|
|
@@ -391,176 +459,587 @@ export async function logEvent(eventFile, targetPath) {
|
|
|
391
459
|
try {
|
|
392
460
|
event = JSON.parse(await readFile(eventPath, "utf8"));
|
|
393
461
|
} catch (error) {
|
|
394
|
-
throw new Error(
|
|
395
|
-
}
|
|
396
|
-
|
|
397
|
-
const validationErrors = validateEvent(event);
|
|
398
|
-
if (validationErrors.length > 0) {
|
|
399
|
-
throw new Error(`Event validation failed:\n- ${validationErrors.join("\n- ")}`);
|
|
462
|
+
throw new Error("Could not read event JSON from " + eventPath + ": " + error.message);
|
|
400
463
|
}
|
|
464
|
+
let validationErrors = validateEvent(event);
|
|
465
|
+
if (validationErrors.length > 0) throw new Error("Event validation failed:\n- " + validationErrors.join("\n- "));
|
|
466
|
+
event = enrichEstimatedUsage(event);
|
|
467
|
+
validationErrors = validateEvent(event);
|
|
468
|
+
if (validationErrors.length > 0) throw new Error("Enriched event validation failed:\n- " + validationErrors.join("\n- "));
|
|
401
469
|
|
|
402
470
|
const observerDirectory = await resolveObserverDirectory(targetPath);
|
|
403
471
|
const eventsPath = join(observerDirectory, "events.jsonl");
|
|
404
|
-
const
|
|
405
|
-
|
|
406
|
-
|
|
472
|
+
for await (const existing of streamEvents(eventsPath)) {
|
|
473
|
+
if (existing.event_id === event.event_id) {
|
|
474
|
+
throw new Error("Event ID " + event.event_id + " already exists; refusing to create a duplicate.");
|
|
475
|
+
}
|
|
407
476
|
}
|
|
408
|
-
await appendFile(eventsPath,
|
|
477
|
+
await appendFile(eventsPath, JSON.stringify(event) + "\n", "utf8");
|
|
409
478
|
return { eventId: event.event_id, eventsPath };
|
|
410
479
|
}
|
|
411
480
|
|
|
412
481
|
function average(values) {
|
|
413
|
-
|
|
414
|
-
return values.reduce((sum, value) => sum + value, 0) / values.length;
|
|
482
|
+
return values.length === 0 ? null : values.reduce((sum, value) => sum + value, 0) / values.length;
|
|
415
483
|
}
|
|
416
484
|
|
|
417
|
-
function
|
|
418
|
-
return
|
|
485
|
+
function eventHealth(event) {
|
|
486
|
+
return average(SCORE_FIELDS.map((field) => event[field]));
|
|
419
487
|
}
|
|
420
488
|
|
|
421
|
-
function
|
|
422
|
-
return
|
|
489
|
+
function normalizeInsight(item) {
|
|
490
|
+
return typeof item === "string" ? { category: "other", message: item } : item;
|
|
423
491
|
}
|
|
424
492
|
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
493
|
+
function normalizeEvent(event) {
|
|
494
|
+
return {
|
|
495
|
+
...event,
|
|
496
|
+
strengths: event.strengths.map(normalizeInsight),
|
|
497
|
+
improvement_suggestions: event.improvement_suggestions.map(normalizeInsight),
|
|
498
|
+
};
|
|
499
|
+
}
|
|
500
|
+
|
|
501
|
+
function increment(map, key) {
|
|
502
|
+
map.set(key, (map.get(key) ?? 0) + 1);
|
|
503
|
+
}
|
|
504
|
+
|
|
505
|
+
function summarizeInsights(events, field) {
|
|
506
|
+
const counts = new Map();
|
|
507
|
+
const examples = [];
|
|
508
|
+
const seen = new Set();
|
|
509
|
+
for (const event of [...events].reverse()) {
|
|
510
|
+
for (const insight of event[field]) {
|
|
511
|
+
increment(counts, insight.category);
|
|
512
|
+
const signature = insight.category + "\n" + insight.message.toLowerCase();
|
|
513
|
+
if (examples.length < 5 && !seen.has(signature)) {
|
|
514
|
+
seen.add(signature);
|
|
515
|
+
examples.push({ category: insight.category, message: insight.message });
|
|
516
|
+
}
|
|
517
|
+
}
|
|
429
518
|
}
|
|
430
|
-
|
|
431
|
-
|
|
519
|
+
return { counts, examples };
|
|
520
|
+
}
|
|
432
521
|
|
|
433
|
-
|
|
434
|
-
|
|
522
|
+
function sortedCounts(map) {
|
|
523
|
+
return [...map.entries()].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]));
|
|
524
|
+
}
|
|
525
|
+
|
|
526
|
+
export function analyzeEvents(inputEvents) {
|
|
527
|
+
const events = inputEvents.map(normalizeEvent);
|
|
528
|
+
const dimensions = new Map();
|
|
529
|
+
for (const field of SCORE_FIELDS) dimensions.set(field, average(events.map((event) => event[field])));
|
|
435
530
|
const riskCounts = new Map();
|
|
436
531
|
const resultCounts = new Map();
|
|
437
532
|
const testCounts = new Map();
|
|
533
|
+
const severityCounts = new Map();
|
|
534
|
+
const weaknessCounts = new Map();
|
|
535
|
+
const weaknessExamples = [];
|
|
536
|
+
const weaknessSeen = new Set();
|
|
537
|
+
const usage = new Map();
|
|
538
|
+
for (const source of USAGE_SOURCES) {
|
|
539
|
+
usage.set(source, { events: 0, inputTokens: 0, outputTokens: 0, cost: 0, hasInput: false, hasOutput: false, hasCost: false });
|
|
540
|
+
}
|
|
438
541
|
let eventsWithTests = 0;
|
|
439
|
-
let unavailableUsage = 0;
|
|
440
|
-
let totalInputTokens = 0;
|
|
441
|
-
let totalOutputTokens = 0;
|
|
442
|
-
let totalCost = 0;
|
|
443
|
-
let hasInputTokens = false;
|
|
444
|
-
let hasOutputTokens = false;
|
|
445
|
-
let hasCost = false;
|
|
446
|
-
|
|
447
542
|
for (const event of events) {
|
|
448
|
-
riskCounts
|
|
449
|
-
|
|
450
|
-
resultCounts.set(status, (resultCounts.get(status) ?? 0) + 1);
|
|
543
|
+
increment(riskCounts, event.ambiguity_risk);
|
|
544
|
+
increment(resultCounts, event.execution_signals.result_status);
|
|
451
545
|
if (event.execution_signals.tests_run.length > 0) eventsWithTests += 1;
|
|
452
|
-
for (const test of event.execution_signals.tests_run)
|
|
453
|
-
testCounts.set(test.status, (testCounts.get(test.status) ?? 0) + 1);
|
|
454
|
-
}
|
|
546
|
+
for (const test of event.execution_signals.tests_run) increment(testCounts, test.status);
|
|
455
547
|
for (const weakness of event.weaknesses) {
|
|
456
|
-
weaknessCounts
|
|
457
|
-
severityCounts
|
|
548
|
+
increment(weaknessCounts, weakness.category);
|
|
549
|
+
increment(severityCounts, weakness.severity);
|
|
458
550
|
}
|
|
459
|
-
|
|
551
|
+
const bucket = usage.get(event.usage.source);
|
|
552
|
+
bucket.events += 1;
|
|
460
553
|
if (event.usage.input_tokens !== null) {
|
|
461
|
-
|
|
462
|
-
|
|
554
|
+
bucket.inputTokens += event.usage.input_tokens;
|
|
555
|
+
bucket.hasInput = true;
|
|
463
556
|
}
|
|
464
557
|
if (event.usage.output_tokens !== null) {
|
|
465
|
-
|
|
466
|
-
|
|
558
|
+
bucket.outputTokens += event.usage.output_tokens;
|
|
559
|
+
bucket.hasOutput = true;
|
|
467
560
|
}
|
|
468
561
|
if (event.usage.cost_usd !== null) {
|
|
469
|
-
|
|
470
|
-
hasCost = true;
|
|
562
|
+
bucket.cost += event.usage.cost_usd;
|
|
563
|
+
bucket.hasCost = true;
|
|
471
564
|
}
|
|
472
565
|
}
|
|
566
|
+
for (const event of [...events].reverse()) {
|
|
567
|
+
for (const weakness of event.weaknesses) {
|
|
568
|
+
const signature = weakness.category + "\n" + weakness.message.toLowerCase();
|
|
569
|
+
if (weaknessExamples.length < 5 && !weaknessSeen.has(signature)) {
|
|
570
|
+
weaknessSeen.add(signature);
|
|
571
|
+
weaknessExamples.push(weakness);
|
|
572
|
+
}
|
|
573
|
+
}
|
|
574
|
+
}
|
|
575
|
+
const totalTests = [...testCounts.values()].reduce((sum, count) => sum + count, 0);
|
|
576
|
+
const reportedOrEstimated = (usage.get("platform_reported")?.events ?? 0) + (usage.get("estimated")?.events ?? 0);
|
|
577
|
+
return {
|
|
578
|
+
events,
|
|
579
|
+
count: events.length,
|
|
580
|
+
overallAverage: average(events.map(eventHealth)),
|
|
581
|
+
dimensions,
|
|
582
|
+
riskCounts,
|
|
583
|
+
resultCounts,
|
|
584
|
+
testCounts,
|
|
585
|
+
severityCounts,
|
|
586
|
+
weaknessCounts,
|
|
587
|
+
weaknessExamples,
|
|
588
|
+
strengths: summarizeInsights(events, "strengths"),
|
|
589
|
+
suggestions: summarizeInsights(events, "improvement_suggestions"),
|
|
590
|
+
eventsWithTests,
|
|
591
|
+
totalTests,
|
|
592
|
+
usage,
|
|
593
|
+
completionRate: events.length ? (resultCounts.get("completed") ?? 0) / events.length : 0,
|
|
594
|
+
lowRiskRate: events.length ? (riskCounts.get("low") ?? 0) / events.length : 0,
|
|
595
|
+
verificationCoverage: events.length ? eventsWithTests / events.length : 0,
|
|
596
|
+
usageCoverage: events.length ? reportedOrEstimated / events.length : 0,
|
|
597
|
+
};
|
|
598
|
+
}
|
|
473
599
|
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
.join("\n");
|
|
478
|
-
const generatedAt = new Date().toISOString();
|
|
600
|
+
function formatAverage(value) {
|
|
601
|
+
return value === null ? "N/A" : value.toFixed(2);
|
|
602
|
+
}
|
|
479
603
|
|
|
480
|
-
|
|
604
|
+
function formatPercent(value) {
|
|
605
|
+
return (value * 100).toFixed(0) + "%";
|
|
606
|
+
}
|
|
481
607
|
|
|
482
|
-
|
|
608
|
+
function formatNumber(value) {
|
|
609
|
+
return new Intl.NumberFormat("en-US").format(value);
|
|
610
|
+
}
|
|
483
611
|
|
|
484
|
-
|
|
612
|
+
function progressBar(value, max = 10) {
|
|
613
|
+
if (value === null) return "N/A";
|
|
614
|
+
const filled = Math.max(0, Math.min(10, Math.round((value / max) * 10)));
|
|
615
|
+
return "█".repeat(filled) + "░".repeat(10 - filled);
|
|
616
|
+
}
|
|
617
|
+
|
|
618
|
+
function markdownText(value) {
|
|
619
|
+
return String(value).replaceAll("|", "\\|").replace(/\r?\n/g, " ").trim();
|
|
620
|
+
}
|
|
621
|
+
|
|
622
|
+
function deltaLabel(current, previous) {
|
|
623
|
+
if (current === null || previous === null) return "N/A";
|
|
624
|
+
const delta = current - previous;
|
|
625
|
+
if (Math.abs(delta) < 0.005) return "→ 0.00";
|
|
626
|
+
return (delta > 0 ? "↑ +" : "↓ ") + delta.toFixed(2);
|
|
627
|
+
}
|
|
628
|
+
|
|
629
|
+
function countRows(map, order) {
|
|
630
|
+
return order.map((key) => "| " + key + " | " + (map.get(key) ?? 0) + " |").join("\n");
|
|
631
|
+
}
|
|
485
632
|
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
633
|
+
function insightMarkdown(title, summary, emptyMessage) {
|
|
634
|
+
const categories = sortedCounts(summary.counts);
|
|
635
|
+
const categoryRows = categories.length
|
|
636
|
+
? categories.map(([category, count]) => "| " + category + " | " + count + " |").join("\n")
|
|
637
|
+
: "| None | 0 |";
|
|
638
|
+
const examples = summary.examples.length
|
|
639
|
+
? summary.examples.map((item) => "- **" + item.category + ":** " + markdownText(item.message)).join("\n")
|
|
640
|
+
: "- " + emptyMessage;
|
|
641
|
+
return "## " + title + "\n\n| Category | Count |\n| --- | ---: |\n" + categoryRows + "\n\n### Recent examples\n\n" + examples;
|
|
642
|
+
}
|
|
490
643
|
|
|
491
|
-
|
|
644
|
+
export function buildReport(inputEvents, options = {}) {
|
|
645
|
+
const current = analyzeEvents(inputEvents);
|
|
646
|
+
const previous = analyzeEvents(options.previousEvents ?? []);
|
|
647
|
+
const totalEventCount = options.totalEventCount ?? current.count;
|
|
648
|
+
const selectedLabel = options.all ? "all events" : "latest " + current.count + " events";
|
|
649
|
+
const usageRows = ["platform_reported", "estimated", "unavailable"].map((source) => {
|
|
650
|
+
const bucket = current.usage.get(source);
|
|
651
|
+
return "| " + source + " | " + bucket.events + " | " +
|
|
652
|
+
(bucket.hasInput ? formatNumber(bucket.inputTokens) : "N/A") + " | " +
|
|
653
|
+
(bucket.hasOutput ? formatNumber(bucket.outputTokens) : "N/A") + " | " +
|
|
654
|
+
(bucket.hasCost ? "$" + bucket.cost.toFixed(6) : "N/A") + " |";
|
|
655
|
+
}).join("\n");
|
|
656
|
+
const dimensionRows = SCORE_FIELDS.map((field) => {
|
|
657
|
+
const value = current.dimensions.get(field);
|
|
658
|
+
const prior = previous.dimensions.get(field);
|
|
659
|
+
return "| " + field + " | " + progressBar(value) + " | " + formatAverage(value) + " | " + deltaLabel(value, prior) + " |";
|
|
660
|
+
}).join("\n");
|
|
661
|
+
const weaknessSummary = { counts: current.weaknessCounts, examples: current.weaknessExamples };
|
|
662
|
+
const generatedAt = new Date().toISOString();
|
|
663
|
+
return [
|
|
664
|
+
"# Prompt Observer Report",
|
|
665
|
+
"",
|
|
666
|
+
"> Window: **" + selectedLabel + "** of " + totalEventCount + " stored events · Generated " + generatedAt,
|
|
667
|
+
"",
|
|
668
|
+
"## Health snapshot",
|
|
669
|
+
"",
|
|
670
|
+
"| Events | Prompt health | vs previous window | Completed | Low ambiguity | Verified tasks | Usage coverage |",
|
|
671
|
+
"| ---: | ---: | ---: | ---: | ---: | ---: | ---: |",
|
|
672
|
+
"| " + current.count + " | **" + formatAverage(current.overallAverage) + "/10** | " + deltaLabel(current.overallAverage, previous.overallAverage) + " | " + formatPercent(current.completionRate) + " | " + formatPercent(current.lowRiskRate) + " | " + formatPercent(current.verificationCoverage) + " | " + formatPercent(current.usageCoverage) + " |",
|
|
673
|
+
"",
|
|
674
|
+
"## Quality dimensions",
|
|
675
|
+
"",
|
|
676
|
+
"| Dimension | Health | Average | Trend |",
|
|
677
|
+
"| --- | --- | ---: | ---: |",
|
|
678
|
+
dimensionRows,
|
|
679
|
+
"",
|
|
680
|
+
insightMarkdown("Strengths", current.strengths, "No strengths recorded in this window."),
|
|
681
|
+
"",
|
|
682
|
+
insightMarkdown("Weaknesses", weaknessSummary, "No material weaknesses detected."),
|
|
683
|
+
"",
|
|
684
|
+
insightMarkdown("Improvement suggestions", current.suggestions, "No change needed."),
|
|
685
|
+
"",
|
|
686
|
+
"## Execution and verification",
|
|
687
|
+
"",
|
|
688
|
+
"| Result | Count |",
|
|
689
|
+
"| --- | ---: |",
|
|
690
|
+
countRows(current.resultCounts, ["completed", "partial", "blocked"]),
|
|
691
|
+
"",
|
|
692
|
+
"| Test status | Count |",
|
|
693
|
+
"| --- | ---: |",
|
|
694
|
+
countRows(current.testCounts, ["passed", "failed", "not_run", "unknown"]),
|
|
695
|
+
"",
|
|
696
|
+
"| Weakness severity | Count |",
|
|
697
|
+
"| --- | ---: |",
|
|
698
|
+
countRows(current.severityCounts, ["high", "medium", "low"]),
|
|
699
|
+
"",
|
|
700
|
+
"| Ambiguity risk | Count |",
|
|
701
|
+
"| --- | ---: |",
|
|
702
|
+
countRows(current.riskCounts, ["high", "medium", "low"]),
|
|
703
|
+
"",
|
|
704
|
+
"## Usage",
|
|
705
|
+
"",
|
|
706
|
+
"| Source | Events | Input tokens | Output tokens | Cost (USD) |",
|
|
707
|
+
"| --- | ---: | ---: | ---: | ---: |",
|
|
708
|
+
usageRows,
|
|
709
|
+
"",
|
|
710
|
+
"Usage coverage is " + formatPercent(current.usageCoverage) + ". Exact and estimated values are intentionally kept separate; unavailable values are never invented.",
|
|
711
|
+
"",
|
|
712
|
+
PRICING.updated_at
|
|
713
|
+
? "Estimated costs use pricing snapshot **" + PRICING.updated_at + "** ([source](" + PRICING.source_url + ")). Cached tokens, tools, subscriptions, discounts, and other provider charges are excluded."
|
|
714
|
+
: "No pricing snapshot was available, so estimated costs were not calculated.",
|
|
715
|
+
"",
|
|
716
|
+
].join("\n");
|
|
717
|
+
}
|
|
492
718
|
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
719
|
+
function dashboardApp(payload) {
|
|
720
|
+
const scoreFields = payload.scoreFields;
|
|
721
|
+
let range = payload.defaultRange;
|
|
722
|
+
const filters = { task: "all", status: "all", risk: "all" };
|
|
723
|
+
const byId = (id) => document.getElementById(id);
|
|
724
|
+
const pct = (value) => Math.round(value * 100) + "%";
|
|
725
|
+
const avg = (values) => values.length ? values.reduce((a, b) => a + b, 0) / values.length : null;
|
|
726
|
+
const health = (event) => avg(scoreFields.map((field) => event[field]));
|
|
727
|
+
const number = (value) => new Intl.NumberFormat("en-US").format(value);
|
|
728
|
+
const addOption = (select, value) => {
|
|
729
|
+
const option = document.createElement("option");
|
|
730
|
+
option.value = value;
|
|
731
|
+
option.textContent = value;
|
|
732
|
+
select.append(option);
|
|
733
|
+
};
|
|
734
|
+
const unique = (field) => [...new Set(payload.events.map(field))].sort();
|
|
735
|
+
unique((event) => event.task_type).forEach((value) => addOption(byId("task-filter"), value));
|
|
736
|
+
unique((event) => event.execution_signals.result_status).forEach((value) => addOption(byId("status-filter"), value));
|
|
737
|
+
unique((event) => event.ambiguity_risk).forEach((value) => addOption(byId("risk-filter"), value));
|
|
496
738
|
|
|
497
|
-
|
|
739
|
+
function selectedEvents() {
|
|
740
|
+
return payload.events.slice(-range).filter((event) =>
|
|
741
|
+
(filters.task === "all" || event.task_type === filters.task) &&
|
|
742
|
+
(filters.status === "all" || event.execution_signals.result_status === filters.status) &&
|
|
743
|
+
(filters.risk === "all" || event.ambiguity_risk === filters.risk)
|
|
744
|
+
);
|
|
745
|
+
}
|
|
498
746
|
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
747
|
+
function setCard(id, value, note) {
|
|
748
|
+
byId(id).querySelector(".value").textContent = value;
|
|
749
|
+
byId(id).querySelector(".note").textContent = note;
|
|
750
|
+
}
|
|
502
751
|
|
|
503
|
-
|
|
752
|
+
function renderDimensions(events) {
|
|
753
|
+
const container = byId("dimensions");
|
|
754
|
+
container.replaceChildren();
|
|
755
|
+
scoreFields.forEach((field) => {
|
|
756
|
+
const value = avg(events.map((event) => event[field]));
|
|
757
|
+
const row = document.createElement("div");
|
|
758
|
+
row.className = "bar-row";
|
|
759
|
+
const label = document.createElement("span");
|
|
760
|
+
label.textContent = field;
|
|
761
|
+
const track = document.createElement("div");
|
|
762
|
+
track.className = "bar-track";
|
|
763
|
+
const fill = document.createElement("div");
|
|
764
|
+
fill.className = "bar-fill";
|
|
765
|
+
fill.style.width = (value === null ? 0 : value * 10) + "%";
|
|
766
|
+
const score = document.createElement("strong");
|
|
767
|
+
score.textContent = value === null ? "N/A" : value.toFixed(2);
|
|
768
|
+
track.append(fill);
|
|
769
|
+
row.append(label, track, score);
|
|
770
|
+
container.append(row);
|
|
771
|
+
});
|
|
772
|
+
}
|
|
504
773
|
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
774
|
+
function renderTrend(events) {
|
|
775
|
+
const svg = byId("trend");
|
|
776
|
+
svg.replaceChildren();
|
|
777
|
+
const values = events.map(health);
|
|
778
|
+
if (values.length < 2) {
|
|
779
|
+
const message = document.createElementNS("http://www.w3.org/2000/svg", "text");
|
|
780
|
+
message.setAttribute("x", "20");
|
|
781
|
+
message.setAttribute("y", "55");
|
|
782
|
+
message.textContent = values.length ? "Add another event to see a trend." : "No matching events.";
|
|
783
|
+
svg.append(message);
|
|
784
|
+
return;
|
|
785
|
+
}
|
|
786
|
+
const width = 600;
|
|
787
|
+
const height = 120;
|
|
788
|
+
const points = values.map((value, index) => {
|
|
789
|
+
const x = 10 + index * (width - 20) / (values.length - 1);
|
|
790
|
+
const y = height - 10 - value / 10 * (height - 20);
|
|
791
|
+
return x + "," + y;
|
|
792
|
+
}).join(" ");
|
|
793
|
+
const line = document.createElementNS("http://www.w3.org/2000/svg", "polyline");
|
|
794
|
+
line.setAttribute("points", points);
|
|
795
|
+
line.setAttribute("fill", "none");
|
|
796
|
+
line.setAttribute("stroke", "#7c5cff");
|
|
797
|
+
line.setAttribute("stroke-width", "4");
|
|
798
|
+
line.setAttribute("stroke-linecap", "round");
|
|
799
|
+
line.setAttribute("stroke-linejoin", "round");
|
|
800
|
+
svg.append(line);
|
|
801
|
+
}
|
|
508
802
|
|
|
509
|
-
|
|
803
|
+
function renderDistribution(id, values, order) {
|
|
804
|
+
const container = byId(id);
|
|
805
|
+
container.replaceChildren();
|
|
806
|
+
const total = values.length;
|
|
807
|
+
order.forEach((name) => {
|
|
808
|
+
const count = values.filter((value) => value === name).length;
|
|
809
|
+
const row = document.createElement("div");
|
|
810
|
+
row.className = "bar-row";
|
|
811
|
+
const label = document.createElement("span");
|
|
812
|
+
label.textContent = name;
|
|
813
|
+
const track = document.createElement("div");
|
|
814
|
+
track.className = "bar-track";
|
|
815
|
+
const fill = document.createElement("div");
|
|
816
|
+
fill.className = "bar-fill";
|
|
817
|
+
fill.style.width = (total ? count / total * 100 : 0) + "%";
|
|
818
|
+
const score = document.createElement("strong");
|
|
819
|
+
score.textContent = String(count);
|
|
820
|
+
track.append(fill);
|
|
821
|
+
row.append(label, track, score);
|
|
822
|
+
container.append(row);
|
|
823
|
+
});
|
|
824
|
+
}
|
|
510
825
|
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
826
|
+
function insightItems(events, field, weakness) {
|
|
827
|
+
const counts = new Map();
|
|
828
|
+
const examples = [];
|
|
829
|
+
const seen = new Set();
|
|
830
|
+
[...events].reverse().forEach((event) => {
|
|
831
|
+
event[field].forEach((raw) => {
|
|
832
|
+
const item = typeof raw === "string" ? { category: "other", message: raw } : raw;
|
|
833
|
+
counts.set(item.category, (counts.get(item.category) || 0) + 1);
|
|
834
|
+
const key = item.category + "\n" + item.message.toLowerCase();
|
|
835
|
+
if (examples.length < 5 && !seen.has(key)) {
|
|
836
|
+
seen.add(key);
|
|
837
|
+
examples.push({ ...item, severity: weakness ? item.severity : null });
|
|
838
|
+
}
|
|
839
|
+
});
|
|
840
|
+
});
|
|
841
|
+
return { counts: [...counts.entries()].sort((a, b) => b[1] - a[1]), examples };
|
|
842
|
+
}
|
|
514
843
|
|
|
515
|
-
|
|
844
|
+
function renderInsights(id, title, data, empty) {
|
|
845
|
+
const root = byId(id);
|
|
846
|
+
root.replaceChildren();
|
|
847
|
+
const heading = document.createElement("div");
|
|
848
|
+
heading.className = "insight-heading";
|
|
849
|
+
const name = document.createElement("h3");
|
|
850
|
+
name.textContent = title;
|
|
851
|
+
heading.append(name);
|
|
852
|
+
data.counts.slice(0, 4).forEach(([category, count]) => {
|
|
853
|
+
const chip = document.createElement("span");
|
|
854
|
+
chip.className = "chip";
|
|
855
|
+
chip.textContent = category + " · " + count;
|
|
856
|
+
heading.append(chip);
|
|
857
|
+
});
|
|
858
|
+
root.append(heading);
|
|
859
|
+
if (!data.examples.length) {
|
|
860
|
+
const blank = document.createElement("p");
|
|
861
|
+
blank.className = "empty";
|
|
862
|
+
blank.textContent = empty;
|
|
863
|
+
root.append(blank);
|
|
864
|
+
return;
|
|
865
|
+
}
|
|
866
|
+
const list = document.createElement("ul");
|
|
867
|
+
data.examples.forEach((item) => {
|
|
868
|
+
const entry = document.createElement("li");
|
|
869
|
+
const category = document.createElement("strong");
|
|
870
|
+
category.textContent = item.category + ": ";
|
|
871
|
+
entry.append(category, document.createTextNode(item.message));
|
|
872
|
+
list.append(entry);
|
|
873
|
+
});
|
|
874
|
+
root.append(list);
|
|
875
|
+
}
|
|
516
876
|
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
877
|
+
function renderUsage(events) {
|
|
878
|
+
const sources = ["platform_reported", "estimated", "unavailable"];
|
|
879
|
+
const body = byId("usage-body");
|
|
880
|
+
body.replaceChildren();
|
|
881
|
+
sources.forEach((source) => {
|
|
882
|
+
const matching = events.filter((event) => event.usage.source === source);
|
|
883
|
+
const sum = (field) => matching.reduce((total, event) => total + (event.usage[field] ?? 0), 0);
|
|
884
|
+
const has = (field) => matching.some((event) => event.usage[field] !== null);
|
|
885
|
+
const row = document.createElement("tr");
|
|
886
|
+
[source, matching.length, has("input_tokens") ? number(sum("input_tokens")) : "N/A", has("output_tokens") ? number(sum("output_tokens")) : "N/A", has("cost_usd") ? "$" + sum("cost_usd").toFixed(6) : "N/A"].forEach((value) => {
|
|
887
|
+
const cell = document.createElement("td");
|
|
888
|
+
cell.textContent = value;
|
|
889
|
+
row.append(cell);
|
|
890
|
+
});
|
|
891
|
+
body.append(row);
|
|
892
|
+
});
|
|
893
|
+
}
|
|
520
894
|
|
|
521
|
-
|
|
895
|
+
function render() {
|
|
896
|
+
const events = selectedEvents();
|
|
897
|
+
const healthValue = avg(events.map(health));
|
|
898
|
+
const completed = events.filter((event) => event.execution_signals.result_status === "completed").length;
|
|
899
|
+
const lowRisk = events.filter((event) => event.ambiguity_risk === "low").length;
|
|
900
|
+
const verified = events.filter((event) => event.execution_signals.tests_run.length > 0).length;
|
|
901
|
+
const covered = events.filter((event) => event.usage.source !== "unavailable").length;
|
|
902
|
+
setCard("health-card", healthValue === null ? "N/A" : healthValue.toFixed(2), "out of 10");
|
|
903
|
+
setCard("completion-card", events.length ? pct(completed / events.length) : "N/A", completed + " completed");
|
|
904
|
+
setCard("ambiguity-card", events.length ? pct(lowRisk / events.length) : "N/A", lowRisk + " low-risk");
|
|
905
|
+
setCard("verification-card", events.length ? pct(verified / events.length) : "N/A", verified + " with tests");
|
|
906
|
+
setCard("usage-card", events.length ? pct(covered / events.length) : "N/A", covered + " exact or estimated");
|
|
907
|
+
byId("event-count").textContent = events.length + " matching event" + (events.length === 1 ? "" : "s");
|
|
908
|
+
renderDimensions(events);
|
|
909
|
+
renderTrend(events);
|
|
910
|
+
renderDistribution("outcomes", events.map((event) => event.execution_signals.result_status), ["completed", "partial", "blocked"]);
|
|
911
|
+
renderDistribution("risks", events.map((event) => event.ambiguity_risk), ["low", "medium", "high"]);
|
|
912
|
+
renderDistribution("tests", events.flatMap((event) => event.execution_signals.tests_run.map((test) => test.status)), ["passed", "failed", "not_run", "unknown"]);
|
|
913
|
+
renderInsights("strengths", "Strengths", insightItems(events, "strengths", false), "No strengths recorded in this window.");
|
|
914
|
+
renderInsights("weaknesses", "Weaknesses", insightItems(events, "weaknesses", true), "No material weaknesses detected.");
|
|
915
|
+
renderInsights("suggestions", "Improvement suggestions", insightItems(events, "improvement_suggestions", false), "No change needed.");
|
|
916
|
+
renderUsage(events);
|
|
917
|
+
document.querySelectorAll("[data-range]").forEach((button) => button.classList.toggle("active", Number(button.dataset.range) === range));
|
|
918
|
+
}
|
|
522
919
|
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
920
|
+
document.querySelectorAll("[data-range]").forEach((button) => button.addEventListener("click", () => {
|
|
921
|
+
range = Number(button.dataset.range);
|
|
922
|
+
render();
|
|
923
|
+
}));
|
|
924
|
+
[["task-filter", "task"], ["status-filter", "status"], ["risk-filter", "risk"]].forEach(([id, key]) => {
|
|
925
|
+
byId(id).addEventListener("change", (event) => {
|
|
926
|
+
filters[key] = event.target.value;
|
|
927
|
+
render();
|
|
928
|
+
});
|
|
929
|
+
});
|
|
930
|
+
render();
|
|
931
|
+
}
|
|
526
932
|
|
|
527
|
-
|
|
933
|
+
function escapeHtml(value) {
|
|
934
|
+
return String(value).replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">").replaceAll('"', """);
|
|
935
|
+
}
|
|
528
936
|
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
937
|
+
export function buildHtmlReport(inputEvents, options = {}) {
|
|
938
|
+
const events = inputEvents.map(normalizeEvent).map((event) => ({
|
|
939
|
+
timestamp: event.timestamp,
|
|
940
|
+
task_type: event.task_type,
|
|
941
|
+
ambiguity_risk: event.ambiguity_risk,
|
|
942
|
+
...Object.fromEntries(SCORE_FIELDS.map((field) => [field, event[field]])),
|
|
943
|
+
strengths: event.strengths,
|
|
944
|
+
weaknesses: event.weaknesses,
|
|
945
|
+
improvement_suggestions: event.improvement_suggestions,
|
|
946
|
+
execution_signals: {
|
|
947
|
+
result_status: event.execution_signals.result_status,
|
|
948
|
+
tests_run: event.execution_signals.tests_run.map((test) => ({ status: test.status })),
|
|
949
|
+
},
|
|
950
|
+
usage: event.usage,
|
|
951
|
+
}));
|
|
952
|
+
const defaultRange = options.all ? events.length : 50;
|
|
953
|
+
const payload = JSON.stringify({ events, scoreFields: SCORE_FIELDS, defaultRange }).replaceAll("<", "\\u003c");
|
|
954
|
+
const generatedAt = new Date().toISOString();
|
|
955
|
+
const totalEventCount = options.totalEventCount ?? events.length;
|
|
956
|
+
const windowLabel = options.all ? "All events" : "Latest " + events.length + " events";
|
|
957
|
+
const styles = [
|
|
958
|
+
":root{color-scheme:dark;--bg:#0b1020;--panel:#141a2e;--panel2:#1b2340;--text:#eef1ff;--muted:#9ca8c7;--accent:#7c5cff;--accent2:#36d1a0;--danger:#ff6b82;--border:#293251}",
|
|
959
|
+
"*{box-sizing:border-box}body{margin:0;background:radial-gradient(circle at 15% 0,#24205a 0,transparent 35%),var(--bg);color:var(--text);font:14px/1.55 Inter,ui-sans-serif,system-ui,-apple-system,Segoe UI,sans-serif}",
|
|
960
|
+
"main{max-width:1180px;margin:auto;padding:44px 24px 80px}header{display:flex;justify-content:space-between;gap:20px;align-items:end;margin-bottom:28px}h1{font-size:34px;margin:0 0 6px}h2,h3{margin:0}.sub,.note,.empty{color:var(--muted)}",
|
|
961
|
+
".toolbar{display:flex;flex-wrap:wrap;gap:10px;padding:14px;background:rgba(20,26,46,.85);border:1px solid var(--border);border-radius:16px;margin-bottom:18px;position:sticky;top:10px;z-index:3;backdrop-filter:blur(12px)}",
|
|
962
|
+
"button,select{background:var(--panel2);color:var(--text);border:1px solid var(--border);border-radius:9px;padding:8px 11px}button{cursor:pointer}button.active{background:var(--accent);border-color:var(--accent)}",
|
|
963
|
+
".cards{display:grid;grid-template-columns:repeat(5,1fr);gap:14px;margin:18px 0}.card,.panel,.insight{background:linear-gradient(145deg,rgba(27,35,64,.95),rgba(20,26,46,.95));border:1px solid var(--border);border-radius:18px;padding:20px;box-shadow:0 14px 45px rgba(0,0,0,.18)}",
|
|
964
|
+
".card .label{color:var(--muted);font-size:12px;text-transform:uppercase;letter-spacing:.08em}.card .value{font-size:28px;font-weight:750;margin:8px 0 2px}.grid{display:grid;grid-template-columns:1fr 1fr;gap:16px}.panel h2{font-size:18px;margin-bottom:18px}",
|
|
965
|
+
".bar-row{display:grid;grid-template-columns:165px 1fr 42px;gap:12px;align-items:center;margin:12px 0}.bar-track{height:9px;background:#252d4c;border-radius:9px;overflow:hidden}.bar-fill{height:100%;background:linear-gradient(90deg,var(--accent),var(--accent2));border-radius:9px}",
|
|
966
|
+
"#trend{width:100%;height:140px;background:rgba(8,12,27,.35);border-radius:12px}#trend text{fill:var(--muted);font-size:13px}.insights{display:grid;grid-template-columns:repeat(3,1fr);gap:16px;margin-top:16px}.insight-heading{display:flex;gap:7px;align-items:center;flex-wrap:wrap;margin-bottom:14px}.insight-heading h3{width:100%;font-size:18px}.chip{font-size:11px;background:#282f53;color:#cbd2ef;padding:4px 7px;border-radius:20px}",
|
|
967
|
+
"ul{padding-left:20px;margin:8px 0}li{margin:9px 0;color:#dbe0f5}table{width:100%;border-collapse:collapse;margin-top:10px}th,td{text-align:left;padding:10px;border-bottom:1px solid var(--border)}th{color:var(--muted);font-size:12px;text-transform:uppercase}.usage{margin-top:16px}.footnote{margin-top:12px;color:var(--muted);font-size:12px}",
|
|
968
|
+
"@media(max-width:900px){.cards{grid-template-columns:repeat(2,1fr)}.grid,.insights{grid-template-columns:1fr}header{display:block}.bar-row{grid-template-columns:130px 1fr 38px}}@media(max-width:520px){main{padding:24px 14px}.cards{grid-template-columns:1fr}.toolbar{position:static}}",
|
|
969
|
+
].join("");
|
|
970
|
+
return [
|
|
971
|
+
"<!doctype html><html lang=\"en\"><head><meta charset=\"utf-8\"><meta name=\"viewport\" content=\"width=device-width,initial-scale=1\">",
|
|
972
|
+
"<title>Prompt Observer Report</title><style>" + styles + "</style></head><body><main>",
|
|
973
|
+
"<header><div><h1>Prompt Observer</h1><div class=\"sub\">" + escapeHtml(windowLabel) + " of " + totalEventCount + " stored · " + escapeHtml(generatedAt) + "</div></div><strong id=\"event-count\"></strong></header>",
|
|
974
|
+
"<section class=\"toolbar\"><button data-range=\"10\">Last 10</button><button data-range=\"25\">Last 25</button><button data-range=\"50\">Last 50</button>" + (options.all ? "<button data-range=\"" + events.length + "\" class=\"active\">All loaded</button>" : "") + "<select id=\"task-filter\"><option value=\"all\">All task types</option></select><select id=\"status-filter\"><option value=\"all\">All outcomes</option></select><select id=\"risk-filter\"><option value=\"all\">All ambiguity levels</option></select></section>",
|
|
975
|
+
"<section class=\"cards\"><article class=\"card\" id=\"health-card\"><div class=\"label\">Prompt health</div><div class=\"value\"></div><div class=\"note\"></div></article><article class=\"card\" id=\"completion-card\"><div class=\"label\">Completion</div><div class=\"value\"></div><div class=\"note\"></div></article><article class=\"card\" id=\"ambiguity-card\"><div class=\"label\">Low ambiguity</div><div class=\"value\"></div><div class=\"note\"></div></article><article class=\"card\" id=\"verification-card\"><div class=\"label\">Verification</div><div class=\"value\"></div><div class=\"note\"></div></article><article class=\"card\" id=\"usage-card\"><div class=\"label\">Usage coverage</div><div class=\"value\"></div><div class=\"note\"></div></article></section>",
|
|
976
|
+
"<section class=\"grid\"><article class=\"panel\"><h2>Quality dimensions</h2><div id=\"dimensions\"></div></article><article class=\"panel\"><h2>Prompt-health trend</h2><svg id=\"trend\" viewBox=\"0 0 600 120\" preserveAspectRatio=\"none\"></svg></article></section>",
|
|
977
|
+
"<section class=\"grid usage\"><article class=\"panel\"><h2>Execution outcomes</h2><div id=\"outcomes\"></div><h2>Ambiguity risk</h2><div id=\"risks\"></div></article><article class=\"panel\"><h2>Verification results</h2><div id=\"tests\"></div></article></section>",
|
|
978
|
+
"<section class=\"insights\"><article class=\"insight\" id=\"strengths\"></article><article class=\"insight\" id=\"weaknesses\"></article><article class=\"insight\" id=\"suggestions\"></article></section>",
|
|
979
|
+
"<section class=\"panel usage\"><h2>Usage by source</h2><table><thead><tr><th>Source</th><th>Events</th><th>Input tokens</th><th>Output tokens</th><th>Cost</th></tr></thead><tbody id=\"usage-body\"></tbody></table><p class=\"footnote\">Exact and estimated values are separated. Estimated costs use pricing snapshot " + escapeHtml(PRICING.updated_at ?? "unavailable") + " and exclude cached tokens, tools, subscriptions, discounts, and provider-specific charges.</p></section>",
|
|
980
|
+
"</main><script>const reportData=" + payload + ";(" + dashboardApp.toString() + ")(reportData);</script></body></html>",
|
|
981
|
+
].join("");
|
|
533
982
|
}
|
|
534
983
|
|
|
535
|
-
export async function generateReport(targetPath) {
|
|
984
|
+
export async function generateReport(targetPath, options = {}) {
|
|
536
985
|
const observerDirectory = await resolveObserverDirectory(targetPath ?? ".");
|
|
537
986
|
const eventsPath = join(observerDirectory, "events.jsonl");
|
|
538
|
-
const
|
|
539
|
-
const report = buildReport(
|
|
987
|
+
const selected = await selectReportWindow(eventsPath, options);
|
|
988
|
+
const report = buildReport(selected.current, {
|
|
989
|
+
previousEvents: selected.previous,
|
|
990
|
+
totalEventCount: selected.totalEventCount,
|
|
991
|
+
all: options.all,
|
|
992
|
+
});
|
|
993
|
+
const html = buildHtmlReport(selected.current, {
|
|
994
|
+
totalEventCount: selected.totalEventCount,
|
|
995
|
+
all: options.all,
|
|
996
|
+
});
|
|
540
997
|
const reportPath = join(observerDirectory, "report.md");
|
|
541
|
-
|
|
542
|
-
|
|
998
|
+
const htmlPath = join(observerDirectory, "report.html");
|
|
999
|
+
await Promise.all([writeFile(reportPath, report, "utf8"), writeFile(htmlPath, html, "utf8")]);
|
|
1000
|
+
return {
|
|
1001
|
+
eventCount: selected.current.length,
|
|
1002
|
+
totalEventCount: selected.totalEventCount,
|
|
1003
|
+
reportPath,
|
|
1004
|
+
htmlPath,
|
|
1005
|
+
report,
|
|
1006
|
+
html,
|
|
1007
|
+
};
|
|
543
1008
|
}
|
|
544
1009
|
|
|
545
1010
|
function printHelp() {
|
|
546
|
-
process.stdout.write(
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
prompt-observer
|
|
551
|
-
prompt-observer
|
|
552
|
-
prompt-observer
|
|
553
|
-
prompt-observer
|
|
554
|
-
|
|
1011
|
+
process.stdout.write([
|
|
1012
|
+
"Prompt Observer " + SCHEMA_VERSION,
|
|
1013
|
+
"",
|
|
1014
|
+
"Usage:",
|
|
1015
|
+
" prompt-observer init <target-path>",
|
|
1016
|
+
" prompt-observer log <event-file> [--target <target-path>]",
|
|
1017
|
+
" prompt-observer report [target-path] [--limit <count> | --all]",
|
|
1018
|
+
" prompt-observer validate <event-file>",
|
|
1019
|
+
" prompt-observer help",
|
|
1020
|
+
"",
|
|
1021
|
+
].join("\n"));
|
|
555
1022
|
}
|
|
556
1023
|
|
|
557
1024
|
function optionValue(args, name) {
|
|
558
1025
|
const index = args.indexOf(name);
|
|
559
1026
|
if (index === -1) return undefined;
|
|
560
|
-
if (!args[index + 1]) throw new Error(
|
|
1027
|
+
if (!args[index + 1] || args[index + 1].startsWith("--")) throw new Error(name + " requires a value.");
|
|
561
1028
|
return args[index + 1];
|
|
562
1029
|
}
|
|
563
1030
|
|
|
1031
|
+
function positionalArgs(args, optionsWithValues = []) {
|
|
1032
|
+
const positions = [];
|
|
1033
|
+
for (let index = 0; index < args.length; index += 1) {
|
|
1034
|
+
if (optionsWithValues.includes(args[index])) {
|
|
1035
|
+
index += 1;
|
|
1036
|
+
} else if (!args[index].startsWith("--")) {
|
|
1037
|
+
positions.push(args[index]);
|
|
1038
|
+
}
|
|
1039
|
+
}
|
|
1040
|
+
return positions;
|
|
1041
|
+
}
|
|
1042
|
+
|
|
564
1043
|
export async function runCli(args) {
|
|
565
1044
|
const [command, ...rest] = args;
|
|
566
1045
|
if (!command || command === "help" || command === "--help" || command === "-h") {
|
|
@@ -569,35 +1048,44 @@ export async function runCli(args) {
|
|
|
569
1048
|
}
|
|
570
1049
|
if (command === "init") {
|
|
571
1050
|
const result = await initProject(rest[0] ?? ".");
|
|
572
|
-
process.stdout.write(
|
|
1051
|
+
process.stdout.write("Initialized Prompt Observer at " + result.observerDirectory + "\n" + result.actions.join("\n") + "\n");
|
|
573
1052
|
return;
|
|
574
1053
|
}
|
|
575
1054
|
if (command === "validate") {
|
|
576
1055
|
if (!rest[0]) throw new Error("validate requires an event JSON file.");
|
|
577
1056
|
const event = JSON.parse(await readFile(resolve(rest[0]), "utf8"));
|
|
578
1057
|
const errors = validateEvent(event);
|
|
579
|
-
if (errors.length > 0) throw new Error(
|
|
580
|
-
process.stdout.write(
|
|
1058
|
+
if (errors.length > 0) throw new Error("Event validation failed:\n- " + errors.join("\n- "));
|
|
1059
|
+
process.stdout.write("Valid event: " + event.event_id + "\n");
|
|
581
1060
|
return;
|
|
582
1061
|
}
|
|
583
1062
|
if (command === "log") {
|
|
584
1063
|
const target = optionValue(rest, "--target");
|
|
585
|
-
const
|
|
586
|
-
|
|
1064
|
+
const positions = positionalArgs(rest, ["--target"]);
|
|
1065
|
+
const result = await logEvent(positions[0], target);
|
|
1066
|
+
process.stdout.write("Logged " + result.eventId + " to " + result.eventsPath + "\n");
|
|
587
1067
|
return;
|
|
588
1068
|
}
|
|
589
1069
|
if (command === "report") {
|
|
590
|
-
const
|
|
591
|
-
|
|
1070
|
+
const all = rest.includes("--all");
|
|
1071
|
+
const rawLimit = optionValue(rest, "--limit");
|
|
1072
|
+
if (all && rawLimit !== undefined) throw new Error("--all and --limit cannot be used together.");
|
|
1073
|
+
const limit = rawLimit === undefined ? DEFAULT_REPORT_LIMIT : Number(rawLimit);
|
|
1074
|
+
if (!all && (!Number.isInteger(limit) || limit < 1 || limit > 100000)) {
|
|
1075
|
+
throw new Error("--limit must be an integer from 1 to 100000.");
|
|
1076
|
+
}
|
|
1077
|
+
const positions = positionalArgs(rest, ["--limit"]);
|
|
1078
|
+
const result = await generateReport(positions[0] ?? ".", { all, limit });
|
|
1079
|
+
process.stdout.write("Generated " + result.reportPath + " and " + result.htmlPath + " from " + result.eventCount + " of " + result.totalEventCount + " event(s).\n");
|
|
592
1080
|
return;
|
|
593
1081
|
}
|
|
594
|
-
throw new Error(
|
|
1082
|
+
throw new Error("Unknown command: " + command);
|
|
595
1083
|
}
|
|
596
1084
|
|
|
597
1085
|
const isDirectExecution = process.argv[1] && resolve(process.argv[1]) === resolve(fileURLToPath(import.meta.url));
|
|
598
1086
|
if (isDirectExecution) {
|
|
599
1087
|
runCli(process.argv.slice(2)).catch((error) => {
|
|
600
|
-
process.stderr.write(
|
|
1088
|
+
process.stderr.write("Error: " + error.message + "\n");
|
|
601
1089
|
process.exitCode = 1;
|
|
602
1090
|
});
|
|
603
1091
|
}
|