@onthink/prompt-observer 0.1.1 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,5 +1,6 @@
1
1
  #!/usr/bin/env node
2
2
 
3
+ import { createReadStream } from "node:fs";
3
4
  import {
4
5
  appendFile,
5
6
  copyFile,
@@ -9,10 +10,13 @@ import {
9
10
  stat,
10
11
  writeFile,
11
12
  } from "node:fs/promises";
12
- import { dirname, basename, join, resolve } from "node:path";
13
+ import { createInterface } from "node:readline";
14
+ import { basename, dirname, join, resolve } from "node:path";
13
15
  import { fileURLToPath } from "node:url";
14
16
 
15
- const SCHEMA_VERSION = "1.0";
17
+ const SCHEMA_VERSION = "1.1";
18
+ const SUPPORTED_SCHEMA_VERSIONS = new Set(["1.0", "1.1"]);
19
+ const DEFAULT_REPORT_LIMIT = 50;
16
20
  const SCORE_FIELDS = [
17
21
  "intent_clarity",
18
22
  "context_sufficiency",
@@ -21,16 +25,8 @@ const SCORE_FIELDS = [
21
25
  "acceptance_criteria",
22
26
  "verification_plan",
23
27
  ];
24
- const TASK_TYPES = new Set([
25
- "coding",
26
- "debugging",
27
- "planning",
28
- "review",
29
- "testing",
30
- "documentation",
31
- "refactoring",
32
- "other",
33
- ]);
28
+ const INSIGHT_CATEGORIES = new Set([...SCORE_FIELDS, "output_format", "other"]);
29
+ const TASK_TYPES = new Set(["coding", "debugging", "planning", "review", "testing", "documentation", "refactoring", "other"]);
34
30
  const RISK_LEVELS = new Set(["low", "medium", "high"]);
35
31
  const WEAKNESS_CATEGORIES = new Set([
36
32
  "missing_context",
@@ -48,72 +44,109 @@ const RESULT_STATUSES = new Set(["completed", "partial", "blocked"]);
48
44
  const TEST_STATUSES = new Set(["passed", "failed", "not_run", "unknown"]);
49
45
  const USAGE_SOURCES = new Set(["platform_reported", "estimated", "unavailable"]);
50
46
  const ROOT_KEYS = new Set([
51
- "schema_version",
52
- "event_id",
53
- "timestamp",
54
- "task_type",
55
- "prompt_summary",
56
- ...SCORE_FIELDS,
57
- "ambiguity_risk",
58
- "strengths",
59
- "weaknesses",
60
- "improvement_suggestions",
61
- "execution_signals",
62
- "usage",
47
+ "schema_version", "event_id", "timestamp", "task_type", "prompt_summary",
48
+ ...SCORE_FIELDS, "ambiguity_risk", "strengths", "weaknesses",
49
+ "improvement_suggestions", "execution_signals", "usage",
63
50
  ]);
64
51
  const BANNED_KEYS = new Set([
65
- "prompt",
66
- "raw_prompt",
67
- "full_prompt",
68
- "user_prompt",
69
- "response",
70
- "raw_response",
71
- "full_response",
72
- "assistant_response",
73
- "reasoning",
74
- "chain_of_thought",
75
- "system_prompt",
52
+ "prompt", "raw_prompt", "full_prompt", "user_prompt", "response",
53
+ "raw_response", "full_response", "assistant_response", "reasoning",
54
+ "chain_of_thought", "system_prompt",
76
55
  ]);
77
56
 
78
57
  function isObject(value) {
79
58
  return value !== null && typeof value === "object" && !Array.isArray(value);
80
59
  }
81
60
 
61
+ async function exists(path) {
62
+ try {
63
+ await stat(path);
64
+ return true;
65
+ } catch (error) {
66
+ if (error.code === "ENOENT") return false;
67
+ throw error;
68
+ }
69
+ }
70
+
71
+ async function loadPricingSnapshot() {
72
+ const selfDirectory = dirname(fileURLToPath(import.meta.url));
73
+ const candidates = [
74
+ join(selfDirectory, "pricing.json"),
75
+ join(selfDirectory, "..", "pricing", "models.json"),
76
+ ];
77
+ for (const candidate of candidates) {
78
+ if (await exists(candidate)) return JSON.parse(await readFile(candidate, "utf8"));
79
+ }
80
+ return {
81
+ schema_version: "1.0",
82
+ updated_at: null,
83
+ currency: "USD",
84
+ unit_tokens: 1000000,
85
+ source_url: null,
86
+ models: {},
87
+ };
88
+ }
89
+
90
+ const PRICING = await loadPricingSnapshot();
91
+
82
92
  function addUnknownKeyErrors(value, allowed, path, errors) {
83
93
  for (const key of Object.keys(value)) {
84
- if (!allowed.has(key)) errors.push(`${path}.${key} is not allowed`);
94
+ if (!allowed.has(key)) errors.push(path + "." + key + " is not allowed");
85
95
  }
86
96
  }
87
97
 
88
- function validateShortText(value, path, errors, { max = 500 } = {}) {
98
+ function validateShortText(value, path, errors, options = {}) {
99
+ const max = options.max ?? 500;
89
100
  if (typeof value !== "string" || value.trim().length === 0) {
90
- errors.push(`${path} must be a non-empty string`);
101
+ errors.push(path + " must be a non-empty string");
91
102
  } else if (value.length > max) {
92
- errors.push(`${path} must be at most ${max} characters`);
103
+ errors.push(path + " must be at most " + max + " characters");
93
104
  }
94
105
  }
95
106
 
96
107
  function validateTextArray(value, path, errors, maxItems = 10) {
97
108
  if (!Array.isArray(value)) {
98
- errors.push(`${path} must be an array`);
109
+ errors.push(path + " must be an array");
99
110
  return;
100
111
  }
101
- if (value.length > maxItems) errors.push(`${path} must contain at most ${maxItems} items`);
102
- value.forEach((item, index) => validateShortText(item, `${path}[${index}]`, errors));
112
+ if (value.length > maxItems) errors.push(path + " must contain at most " + maxItems + " items");
113
+ value.forEach((item, index) => validateShortText(item, path + "[" + index + "]", errors));
114
+ }
115
+
116
+ function validateInsightArray(value, path, errors, schemaVersion) {
117
+ if (schemaVersion === "1.0") {
118
+ validateTextArray(value, path, errors);
119
+ return;
120
+ }
121
+ if (!Array.isArray(value)) {
122
+ errors.push(path + " must be an array");
123
+ return;
124
+ }
125
+ if (value.length > 10) errors.push(path + " must contain at most 10 items");
126
+ value.forEach((item, index) => {
127
+ const itemPath = path + "[" + index + "]";
128
+ if (!isObject(item)) {
129
+ errors.push(itemPath + " must be an object in schema 1.1");
130
+ return;
131
+ }
132
+ addUnknownKeyErrors(item, new Set(["category", "message"]), itemPath, errors);
133
+ if (!INSIGHT_CATEGORIES.has(item.category)) errors.push(itemPath + ".category is invalid");
134
+ validateShortText(item.message, itemPath + ".message", errors);
135
+ });
103
136
  }
104
137
 
105
138
  function findBannedKeys(value, path, errors) {
106
139
  if (Array.isArray(value)) {
107
- value.forEach((item, index) => findBannedKeys(item, `${path}[${index}]`, errors));
140
+ value.forEach((item, index) => findBannedKeys(item, path + "[" + index + "]", errors));
108
141
  return;
109
142
  }
110
143
  if (!isObject(value)) return;
111
144
  for (const [key, child] of Object.entries(value)) {
112
145
  const normalized = key.toLowerCase().replaceAll("-", "_");
113
146
  if (BANNED_KEYS.has(normalized)) {
114
- errors.push(`${path}.${key} is prohibited because raw prompt, response, or private reasoning must not be stored`);
147
+ errors.push(path + "." + key + " is prohibited because raw prompt, response, or private reasoning must not be stored");
115
148
  }
116
- findBannedKeys(child, `${path}.${key}`, errors);
149
+ findBannedKeys(child, path + "." + key, errors);
117
150
  }
118
151
  }
119
152
 
@@ -126,65 +159,77 @@ function findLikelySecrets(value, errors) {
126
159
  [/\bBearer\s+[A-Za-z0-9._~+/=-]{12,}/i, "a bearer token"],
127
160
  ];
128
161
  for (const [pattern, label] of patterns) {
129
- if (pattern.test(serialized)) errors.push(`event appears to contain ${label}; redact it before logging`);
162
+ if (pattern.test(serialized)) errors.push("event appears to contain " + label + "; redact it before logging");
163
+ }
164
+ }
165
+
166
+ export function calculateEstimatedCost(usage, pricing = PRICING) {
167
+ if (!usage || !usage.model || usage.input_tokens === null || usage.output_tokens === null) return null;
168
+ const rate = pricing.models?.[usage.model];
169
+ if (!rate) return null;
170
+ const unit = pricing.unit_tokens || 1000000;
171
+ return (usage.input_tokens * rate.input_usd + usage.output_tokens * rate.output_usd) / unit;
172
+ }
173
+
174
+ export function estimateTextTokens(text) {
175
+ if (typeof text !== "string" || text.length === 0) return 0;
176
+ let latin = 0;
177
+ let nonLatin = 0;
178
+ let symbols = 0;
179
+ for (const character of text) {
180
+ if (/\s/u.test(character)) continue;
181
+ if (/[A-Za-z0-9]/u.test(character)) latin += 1;
182
+ else if (/[\p{L}\p{N}\p{M}]/u.test(character)) nonLatin += 1;
183
+ else symbols += 1;
130
184
  }
185
+ return Math.ceil(latin / 4 + nonLatin / 2 + symbols / 2);
131
186
  }
132
187
 
133
188
  export function validateEvent(event) {
134
189
  const errors = [];
135
190
  if (!isObject(event)) return ["event must be a JSON object"];
136
-
137
191
  addUnknownKeyErrors(event, ROOT_KEYS, "$", errors);
138
192
  findBannedKeys(event, "$", errors);
139
193
  findLikelySecrets(event, errors);
140
-
141
194
  for (const key of ROOT_KEYS) {
142
- if (!(key in event)) errors.push(`$.${key} is required`);
195
+ if (!(key in event)) errors.push("$." + key + " is required");
143
196
  }
144
197
  if (errors.some((error) => error.endsWith(" is required"))) return errors;
145
198
 
146
- if (event.schema_version !== SCHEMA_VERSION) {
147
- errors.push(`$.schema_version must equal ${SCHEMA_VERSION}`);
199
+ if (!SUPPORTED_SCHEMA_VERSIONS.has(event.schema_version)) {
200
+ errors.push("$.schema_version must equal 1.0 or 1.1");
148
201
  }
149
- if (
150
- typeof event.event_id !== "string" ||
151
- !/^[A-Za-z0-9][A-Za-z0-9._:-]{7,127}$/.test(event.event_id)
152
- ) {
202
+ if (typeof event.event_id !== "string" || !/^[A-Za-z0-9][A-Za-z0-9._:-]{7,127}$/.test(event.event_id)) {
153
203
  errors.push("$.event_id must be 8-128 safe identifier characters");
154
204
  }
155
- if (
156
- typeof event.timestamp !== "string" ||
157
- !event.timestamp.includes("T") ||
158
- Number.isNaN(Date.parse(event.timestamp))
159
- ) {
205
+ if (typeof event.timestamp !== "string" || !event.timestamp.includes("T") || Number.isNaN(Date.parse(event.timestamp))) {
160
206
  errors.push("$.timestamp must be a valid ISO-8601 date-time");
161
207
  }
162
208
  if (!TASK_TYPES.has(event.task_type)) errors.push("$.task_type is invalid");
163
209
  validateShortText(event.prompt_summary, "$.prompt_summary", errors);
164
-
165
210
  for (const field of SCORE_FIELDS) {
166
211
  if (!Number.isInteger(event[field]) || event[field] < 0 || event[field] > 10) {
167
- errors.push(`$.${field} must be an integer from 0 to 10`);
212
+ errors.push("$." + field + " must be an integer from 0 to 10");
168
213
  }
169
214
  }
170
215
  if (!RISK_LEVELS.has(event.ambiguity_risk)) errors.push("$.ambiguity_risk is invalid");
171
- validateTextArray(event.strengths, "$.strengths", errors);
172
- validateTextArray(event.improvement_suggestions, "$.improvement_suggestions", errors);
216
+ validateInsightArray(event.strengths, "$.strengths", errors, event.schema_version);
217
+ validateInsightArray(event.improvement_suggestions, "$.improvement_suggestions", errors, event.schema_version);
173
218
 
174
219
  if (!Array.isArray(event.weaknesses)) {
175
220
  errors.push("$.weaknesses must be an array");
176
221
  } else {
177
222
  if (event.weaknesses.length > 10) errors.push("$.weaknesses must contain at most 10 items");
178
223
  event.weaknesses.forEach((weakness, index) => {
179
- const path = `$.weaknesses[${index}]`;
224
+ const path = "$.weaknesses[" + index + "]";
180
225
  if (!isObject(weakness)) {
181
- errors.push(`${path} must be an object`);
226
+ errors.push(path + " must be an object");
182
227
  return;
183
228
  }
184
229
  addUnknownKeyErrors(weakness, new Set(["category", "severity", "message"]), path, errors);
185
- if (!WEAKNESS_CATEGORIES.has(weakness.category)) errors.push(`${path}.category is invalid`);
186
- if (!SEVERITIES.has(weakness.severity)) errors.push(`${path}.severity is invalid`);
187
- validateShortText(weakness.message, `${path}.message`, errors);
230
+ if (!WEAKNESS_CATEGORIES.has(weakness.category)) errors.push(path + ".category is invalid");
231
+ if (!SEVERITIES.has(weakness.severity)) errors.push(path + ".severity is invalid");
232
+ validateShortText(weakness.message, path + ".message", errors);
188
233
  });
189
234
  }
190
235
 
@@ -193,25 +238,21 @@ export function validateEvent(event) {
193
238
  errors.push("$.execution_signals must be an object");
194
239
  } else {
195
240
  addUnknownKeyErrors(signals, new Set(["result_status", "files_changed", "tests_run"]), "$.execution_signals", errors);
196
- if (!RESULT_STATUSES.has(signals.result_status)) {
197
- errors.push("$.execution_signals.result_status is invalid");
198
- }
241
+ if (!RESULT_STATUSES.has(signals.result_status)) errors.push("$.execution_signals.result_status is invalid");
199
242
  validateTextArray(signals.files_changed, "$.execution_signals.files_changed", errors, 500);
200
243
  if (!Array.isArray(signals.tests_run)) {
201
244
  errors.push("$.execution_signals.tests_run must be an array");
202
245
  } else {
203
- if (signals.tests_run.length > 100) {
204
- errors.push("$.execution_signals.tests_run must contain at most 100 items");
205
- }
246
+ if (signals.tests_run.length > 100) errors.push("$.execution_signals.tests_run must contain at most 100 items");
206
247
  signals.tests_run.forEach((test, index) => {
207
- const path = `$.execution_signals.tests_run[${index}]`;
248
+ const path = "$.execution_signals.tests_run[" + index + "]";
208
249
  if (!isObject(test)) {
209
- errors.push(`${path} must be an object`);
250
+ errors.push(path + " must be an object");
210
251
  return;
211
252
  }
212
253
  addUnknownKeyErrors(test, new Set(["name", "status"]), path, errors);
213
- validateShortText(test.name, `${path}.name`, errors);
214
- if (!TEST_STATUSES.has(test.status)) errors.push(`${path}.status is invalid`);
254
+ validateShortText(test.name, path + ".name", errors);
255
+ if (!TEST_STATUSES.has(test.status)) errors.push(path + ".status is invalid");
215
256
  });
216
257
  }
217
258
  }
@@ -220,34 +261,24 @@ export function validateEvent(event) {
220
261
  if (!isObject(usage)) {
221
262
  errors.push("$.usage must be an object");
222
263
  } else {
223
- const usageKeys = new Set([
224
- "model",
225
- "input_tokens",
226
- "output_tokens",
227
- "cost_usd",
228
- "source",
229
- "estimation_method",
230
- ]);
264
+ const usageKeys = new Set(["model", "input_tokens", "output_tokens", "cost_usd", "source", "estimation_method"]);
231
265
  addUnknownKeyErrors(usage, usageKeys, "$.usage", errors);
232
266
  for (const key of usageKeys) {
233
- if (!(key in usage)) errors.push(`$.usage.${key} is required`);
267
+ if (!(key in usage)) errors.push("$.usage." + key + " is required");
234
268
  }
235
269
  if (usage.model !== null && (typeof usage.model !== "string" || usage.model.length > 200)) {
236
270
  errors.push("$.usage.model must be null or a string up to 200 characters");
237
271
  }
238
272
  for (const field of ["input_tokens", "output_tokens"]) {
239
273
  if (usage[field] !== null && (!Number.isInteger(usage[field]) || usage[field] < 0)) {
240
- errors.push(`$.usage.${field} must be null or a non-negative integer`);
274
+ errors.push("$.usage." + field + " must be null or a non-negative integer");
241
275
  }
242
276
  }
243
277
  if (usage.cost_usd !== null && (typeof usage.cost_usd !== "number" || usage.cost_usd < 0)) {
244
278
  errors.push("$.usage.cost_usd must be null or a non-negative number");
245
279
  }
246
280
  if (!USAGE_SOURCES.has(usage.source)) errors.push("$.usage.source is invalid");
247
- if (
248
- usage.estimation_method !== null &&
249
- (typeof usage.estimation_method !== "string" || usage.estimation_method.trim() === "" || usage.estimation_method.length > 300)
250
- ) {
281
+ if (usage.estimation_method !== null && (typeof usage.estimation_method !== "string" || usage.estimation_method.trim() === "" || usage.estimation_method.length > 300)) {
251
282
  errors.push("$.usage.estimation_method must be null or a non-empty string up to 300 characters");
252
283
  }
253
284
  const metrics = [usage.model, usage.input_tokens, usage.output_tokens, usage.cost_usd];
@@ -264,6 +295,14 @@ export function validateEvent(event) {
264
295
  if (usage.input_tokens === null && usage.output_tokens === null && usage.cost_usd === null) {
265
296
  errors.push("$.usage estimated source requires at least one estimated numeric metric");
266
297
  }
298
+ if (event.schema_version === "1.1" && usage.cost_usd !== null) {
299
+ const expected = calculateEstimatedCost(usage);
300
+ if (expected === null) {
301
+ errors.push("$.usage.cost_usd requires both token counts and an exact model match in pricing.json");
302
+ } else if (Math.abs(expected - usage.cost_usd) > Math.max(1e-9, expected * 0.000001)) {
303
+ errors.push("$.usage.cost_usd does not match the versioned pricing snapshot");
304
+ }
305
+ }
267
306
  }
268
307
  if (usage.source === "platform_reported" && usage.estimation_method !== null) {
269
308
  errors.push("$.usage.estimation_method must be null when source is platform_reported");
@@ -272,20 +311,9 @@ export function validateEvent(event) {
272
311
  errors.push("$.usage platform_reported source requires at least one reported metric");
273
312
  }
274
313
  }
275
-
276
314
  return errors;
277
315
  }
278
316
 
279
- async function exists(path) {
280
- try {
281
- await stat(path);
282
- return true;
283
- } catch (error) {
284
- if (error.code === "ENOENT") return false;
285
- throw error;
286
- }
287
- }
288
-
289
317
  async function findResourceRoot() {
290
318
  const selfDirectory = dirname(fileURLToPath(import.meta.url));
291
319
  const candidates = [resolve(selfDirectory, ".."), selfDirectory];
@@ -294,29 +322,32 @@ async function findResourceRoot() {
294
322
  const schema = (await exists(join(candidate, "schema", "event.schema.json")))
295
323
  ? join(candidate, "schema", "event.schema.json")
296
324
  : join(candidate, "event.schema.json");
297
- if ((await exists(contract)) && (await exists(schema))) {
298
- return { root: candidate, contract, schema };
325
+ const pricing = (await exists(join(candidate, "pricing", "models.json")))
326
+ ? join(candidate, "pricing", "models.json")
327
+ : join(candidate, "pricing.json");
328
+ if ((await exists(contract)) && (await exists(schema)) && (await exists(pricing))) {
329
+ return { root: candidate, contract, schema, pricing };
299
330
  }
300
331
  }
301
- throw new Error("Could not locate PROMPT_OBSERVER.md and event.schema.json next to the CLI.");
332
+ throw new Error("Could not locate the Prompt Observer contract, schema, and pricing snapshot next to the CLI.");
302
333
  }
303
334
 
304
335
  async function copyUnlessPresent(source, destination, actions) {
305
336
  if (await exists(destination)) {
306
- actions.push(`kept ${destination}`);
337
+ actions.push("kept " + destination);
307
338
  return;
308
339
  }
309
340
  await copyFile(source, destination);
310
- actions.push(`created ${destination}`);
341
+ actions.push("created " + destination);
311
342
  }
312
343
 
313
344
  async function writeUnlessPresent(destination, content, actions) {
314
345
  if (await exists(destination)) {
315
- actions.push(`kept ${destination}`);
346
+ actions.push("kept " + destination);
316
347
  return;
317
348
  }
318
349
  await writeFile(destination, content, "utf8");
319
- actions.push(`created ${destination}`);
350
+ actions.push("created " + destination);
320
351
  }
321
352
 
322
353
  export async function initProject(targetPath) {
@@ -325,22 +356,21 @@ export async function initProject(targetPath) {
325
356
  const resources = await findResourceRoot();
326
357
  const selfPath = fileURLToPath(import.meta.url);
327
358
  const actions = [];
328
-
329
359
  await mkdir(observerDirectory, { recursive: true });
330
360
  await mkdir(join(observerDirectory, "pending"), { recursive: true });
331
361
  await copyUnlessPresent(resources.contract, join(observerDirectory, "PROMPT_OBSERVER.md"), actions);
332
362
  await copyUnlessPresent(resources.schema, join(observerDirectory, "event.schema.json"), actions);
363
+ await copyUnlessPresent(resources.pricing, join(observerDirectory, "pricing.json"), actions);
333
364
  await copyUnlessPresent(selfPath, join(observerDirectory, "prompt-observer.mjs"), actions);
334
365
  await writeUnlessPresent(
335
366
  join(observerDirectory, ".gitignore"),
336
- ["events.jsonl", "report.md", "pending/", "*.tmp", ""].join("\n"),
367
+ ["events.jsonl", "report.md", "report.html", "pending/", "*.tmp", ""].join("\n"),
337
368
  actions,
338
369
  );
339
370
  const eventsFile = join(observerDirectory, "events.jsonl");
340
371
  const file = await open(eventsFile, "a");
341
372
  await file.close();
342
- actions.push((await stat(eventsFile)).size === 0 ? `ready ${eventsFile}` : `kept ${eventsFile}`);
343
-
373
+ actions.push((await stat(eventsFile)).size === 0 ? "ready " + eventsFile : "kept " + eventsFile);
344
374
  return { target, observerDirectory, actions };
345
375
  }
346
376
 
@@ -349,9 +379,8 @@ async function resolveObserverDirectory(targetPath) {
349
379
  const resolved = resolve(targetPath);
350
380
  const candidate = basename(resolved) === ".prompt-observer" ? resolved : join(resolved, ".prompt-observer");
351
381
  if (await exists(candidate)) return candidate;
352
- throw new Error(`Prompt Observer is not initialized at ${resolved}. Run init first.`);
382
+ throw new Error("Prompt Observer is not initialized at " + resolved + ". Run init first.");
353
383
  }
354
-
355
384
  let current = resolve(".");
356
385
  while (true) {
357
386
  const candidate = join(current, ".prompt-observer");
@@ -363,25 +392,64 @@ async function resolveObserverDirectory(targetPath) {
363
392
  throw new Error("No .prompt-observer directory found from the current directory upward. Run init first.");
364
393
  }
365
394
 
366
- async function readEvents(eventsPath) {
367
- if (!(await exists(eventsPath))) return [];
368
- const contents = await readFile(eventsPath, "utf8");
369
- const events = [];
370
- for (const [index, line] of contents.split(/\r?\n/).entries()) {
371
- if (line.trim() === "") continue;
372
- let event;
373
- try {
374
- event = JSON.parse(line);
375
- } catch (error) {
376
- throw new Error(`Invalid JSON in ${eventsPath} at line ${index + 1}: ${error.message}`);
377
- }
378
- const validationErrors = validateEvent(event);
379
- if (validationErrors.length > 0) {
380
- throw new Error(`Invalid event in ${eventsPath} at line ${index + 1}:\n- ${validationErrors.join("\n- ")}`);
395
+ async function* streamEvents(eventsPath) {
396
+ if (!(await exists(eventsPath))) return;
397
+ const input = createReadStream(eventsPath, { encoding: "utf8" });
398
+ const lines = createInterface({ input, crlfDelay: Infinity });
399
+ let lineNumber = 0;
400
+ try {
401
+ for await (const line of lines) {
402
+ lineNumber += 1;
403
+ if (line.trim() === "") continue;
404
+ let event;
405
+ try {
406
+ event = JSON.parse(line);
407
+ } catch (error) {
408
+ throw new Error("Invalid JSON in " + eventsPath + " at line " + lineNumber + ": " + error.message);
409
+ }
410
+ const validationErrors = validateEvent(event);
411
+ if (validationErrors.length > 0) {
412
+ throw new Error("Invalid event in " + eventsPath + " at line " + lineNumber + ":\n- " + validationErrors.join("\n- "));
413
+ }
414
+ yield event;
381
415
  }
382
- events.push(event);
416
+ } finally {
417
+ lines.close();
418
+ input.destroy();
383
419
  }
384
- return events;
420
+ }
421
+
422
+ async function selectReportWindow(eventsPath, options = {}) {
423
+ const all = options.all === true;
424
+ const limit = options.limit ?? DEFAULT_REPORT_LIMIT;
425
+ const retained = [];
426
+ let totalEventCount = 0;
427
+ for await (const event of streamEvents(eventsPath)) {
428
+ totalEventCount += 1;
429
+ retained.push(event);
430
+ if (!all && retained.length > limit * 2) retained.shift();
431
+ }
432
+ if (all) return { current: retained, previous: [], totalEventCount };
433
+ const currentStart = Math.max(0, retained.length - limit);
434
+ const previousStart = Math.max(0, currentStart - limit);
435
+ return {
436
+ current: retained.slice(currentStart),
437
+ previous: retained.slice(previousStart, currentStart),
438
+ totalEventCount,
439
+ };
440
+ }
441
+
442
+ function enrichEstimatedUsage(event) {
443
+ if (event.schema_version !== "1.1" || event.usage.source !== "estimated" || event.usage.cost_usd !== null) return event;
444
+ const calculated = calculateEstimatedCost(event.usage);
445
+ if (calculated === null) return event;
446
+ const enriched = structuredClone(event);
447
+ enriched.usage.cost_usd = calculated;
448
+ const marker = "pricing_snapshot:" + PRICING.updated_at;
449
+ if (!enriched.usage.estimation_method.includes(marker)) {
450
+ enriched.usage.estimation_method += "; " + marker;
451
+ }
452
+ return enriched;
385
453
  }
386
454
 
387
455
  export async function logEvent(eventFile, targetPath) {
@@ -391,176 +459,587 @@ export async function logEvent(eventFile, targetPath) {
391
459
  try {
392
460
  event = JSON.parse(await readFile(eventPath, "utf8"));
393
461
  } catch (error) {
394
- throw new Error(`Could not read event JSON from ${eventPath}: ${error.message}`);
395
- }
396
-
397
- const validationErrors = validateEvent(event);
398
- if (validationErrors.length > 0) {
399
- throw new Error(`Event validation failed:\n- ${validationErrors.join("\n- ")}`);
462
+ throw new Error("Could not read event JSON from " + eventPath + ": " + error.message);
400
463
  }
464
+ let validationErrors = validateEvent(event);
465
+ if (validationErrors.length > 0) throw new Error("Event validation failed:\n- " + validationErrors.join("\n- "));
466
+ event = enrichEstimatedUsage(event);
467
+ validationErrors = validateEvent(event);
468
+ if (validationErrors.length > 0) throw new Error("Enriched event validation failed:\n- " + validationErrors.join("\n- "));
401
469
 
402
470
  const observerDirectory = await resolveObserverDirectory(targetPath);
403
471
  const eventsPath = join(observerDirectory, "events.jsonl");
404
- const existingEvents = await readEvents(eventsPath);
405
- if (existingEvents.some((item) => item.event_id === event.event_id)) {
406
- throw new Error(`Event ID ${event.event_id} already exists; refusing to create a duplicate.`);
472
+ for await (const existing of streamEvents(eventsPath)) {
473
+ if (existing.event_id === event.event_id) {
474
+ throw new Error("Event ID " + event.event_id + " already exists; refusing to create a duplicate.");
475
+ }
407
476
  }
408
- await appendFile(eventsPath, `${JSON.stringify(event)}\n`, "utf8");
477
+ await appendFile(eventsPath, JSON.stringify(event) + "\n", "utf8");
409
478
  return { eventId: event.event_id, eventsPath };
410
479
  }
411
480
 
412
481
  function average(values) {
413
- if (values.length === 0) return null;
414
- return values.reduce((sum, value) => sum + value, 0) / values.length;
482
+ return values.length === 0 ? null : values.reduce((sum, value) => sum + value, 0) / values.length;
415
483
  }
416
484
 
417
- function formatAverage(value) {
418
- return value === null ? "N/A" : value.toFixed(2);
485
+ function eventHealth(event) {
486
+ return average(SCORE_FIELDS.map((field) => event[field]));
419
487
  }
420
488
 
421
- function tableRows(counts, preferredOrder) {
422
- return preferredOrder.map((key) => `| ${key} | ${counts.get(key) ?? 0} |`).join("\n");
489
+ function normalizeInsight(item) {
490
+ return typeof item === "string" ? { category: "other", message: item } : item;
423
491
  }
424
492
 
425
- export function buildReport(events) {
426
- const dimensionAverages = new Map();
427
- for (const field of SCORE_FIELDS) {
428
- dimensionAverages.set(field, average(events.map((event) => event[field])));
493
+ function normalizeEvent(event) {
494
+ return {
495
+ ...event,
496
+ strengths: event.strengths.map(normalizeInsight),
497
+ improvement_suggestions: event.improvement_suggestions.map(normalizeInsight),
498
+ };
499
+ }
500
+
501
+ function increment(map, key) {
502
+ map.set(key, (map.get(key) ?? 0) + 1);
503
+ }
504
+
505
+ function summarizeInsights(events, field) {
506
+ const counts = new Map();
507
+ const examples = [];
508
+ const seen = new Set();
509
+ for (const event of [...events].reverse()) {
510
+ for (const insight of event[field]) {
511
+ increment(counts, insight.category);
512
+ const signature = insight.category + "\n" + insight.message.toLowerCase();
513
+ if (examples.length < 5 && !seen.has(signature)) {
514
+ seen.add(signature);
515
+ examples.push({ category: insight.category, message: insight.message });
516
+ }
517
+ }
429
518
  }
430
- const overallScores = events.map((event) => average(SCORE_FIELDS.map((field) => event[field])));
431
- const overallAverage = average(overallScores.filter((value) => value !== null));
519
+ return { counts, examples };
520
+ }
432
521
 
433
- const weaknessCounts = new Map();
434
- const severityCounts = new Map();
522
+ function sortedCounts(map) {
523
+ return [...map.entries()].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]));
524
+ }
525
+
526
+ export function analyzeEvents(inputEvents) {
527
+ const events = inputEvents.map(normalizeEvent);
528
+ const dimensions = new Map();
529
+ for (const field of SCORE_FIELDS) dimensions.set(field, average(events.map((event) => event[field])));
435
530
  const riskCounts = new Map();
436
531
  const resultCounts = new Map();
437
532
  const testCounts = new Map();
533
+ const severityCounts = new Map();
534
+ const weaknessCounts = new Map();
535
+ const weaknessExamples = [];
536
+ const weaknessSeen = new Set();
537
+ const usage = new Map();
538
+ for (const source of USAGE_SOURCES) {
539
+ usage.set(source, { events: 0, inputTokens: 0, outputTokens: 0, cost: 0, hasInput: false, hasOutput: false, hasCost: false });
540
+ }
438
541
  let eventsWithTests = 0;
439
- let unavailableUsage = 0;
440
- let totalInputTokens = 0;
441
- let totalOutputTokens = 0;
442
- let totalCost = 0;
443
- let hasInputTokens = false;
444
- let hasOutputTokens = false;
445
- let hasCost = false;
446
-
447
542
  for (const event of events) {
448
- riskCounts.set(event.ambiguity_risk, (riskCounts.get(event.ambiguity_risk) ?? 0) + 1);
449
- const status = event.execution_signals.result_status;
450
- resultCounts.set(status, (resultCounts.get(status) ?? 0) + 1);
543
+ increment(riskCounts, event.ambiguity_risk);
544
+ increment(resultCounts, event.execution_signals.result_status);
451
545
  if (event.execution_signals.tests_run.length > 0) eventsWithTests += 1;
452
- for (const test of event.execution_signals.tests_run) {
453
- testCounts.set(test.status, (testCounts.get(test.status) ?? 0) + 1);
454
- }
546
+ for (const test of event.execution_signals.tests_run) increment(testCounts, test.status);
455
547
  for (const weakness of event.weaknesses) {
456
- weaknessCounts.set(weakness.category, (weaknessCounts.get(weakness.category) ?? 0) + 1);
457
- severityCounts.set(weakness.severity, (severityCounts.get(weakness.severity) ?? 0) + 1);
548
+ increment(weaknessCounts, weakness.category);
549
+ increment(severityCounts, weakness.severity);
458
550
  }
459
- if (event.usage.source === "unavailable") unavailableUsage += 1;
551
+ const bucket = usage.get(event.usage.source);
552
+ bucket.events += 1;
460
553
  if (event.usage.input_tokens !== null) {
461
- totalInputTokens += event.usage.input_tokens;
462
- hasInputTokens = true;
554
+ bucket.inputTokens += event.usage.input_tokens;
555
+ bucket.hasInput = true;
463
556
  }
464
557
  if (event.usage.output_tokens !== null) {
465
- totalOutputTokens += event.usage.output_tokens;
466
- hasOutputTokens = true;
558
+ bucket.outputTokens += event.usage.output_tokens;
559
+ bucket.hasOutput = true;
467
560
  }
468
561
  if (event.usage.cost_usd !== null) {
469
- totalCost += event.usage.cost_usd;
470
- hasCost = true;
562
+ bucket.cost += event.usage.cost_usd;
563
+ bucket.hasCost = true;
471
564
  }
472
565
  }
566
+ for (const event of [...events].reverse()) {
567
+ for (const weakness of event.weaknesses) {
568
+ const signature = weakness.category + "\n" + weakness.message.toLowerCase();
569
+ if (weaknessExamples.length < 5 && !weaknessSeen.has(signature)) {
570
+ weaknessSeen.add(signature);
571
+ weaknessExamples.push(weakness);
572
+ }
573
+ }
574
+ }
575
+ const totalTests = [...testCounts.values()].reduce((sum, count) => sum + count, 0);
576
+ const reportedOrEstimated = (usage.get("platform_reported")?.events ?? 0) + (usage.get("estimated")?.events ?? 0);
577
+ return {
578
+ events,
579
+ count: events.length,
580
+ overallAverage: average(events.map(eventHealth)),
581
+ dimensions,
582
+ riskCounts,
583
+ resultCounts,
584
+ testCounts,
585
+ severityCounts,
586
+ weaknessCounts,
587
+ weaknessExamples,
588
+ strengths: summarizeInsights(events, "strengths"),
589
+ suggestions: summarizeInsights(events, "improvement_suggestions"),
590
+ eventsWithTests,
591
+ totalTests,
592
+ usage,
593
+ completionRate: events.length ? (resultCounts.get("completed") ?? 0) / events.length : 0,
594
+ lowRiskRate: events.length ? (riskCounts.get("low") ?? 0) / events.length : 0,
595
+ verificationCoverage: events.length ? eventsWithTests / events.length : 0,
596
+ usageCoverage: events.length ? reportedOrEstimated / events.length : 0,
597
+ };
598
+ }
473
599
 
474
- const topWeaknesses = [...weaknessCounts.entries()]
475
- .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
476
- .map(([category, count]) => `| ${category} | ${count} |`)
477
- .join("\n");
478
- const generatedAt = new Date().toISOString();
600
+ function formatAverage(value) {
601
+ return value === null ? "N/A" : value.toFixed(2);
602
+ }
479
603
 
480
- return `# Prompt Observer Report
604
+ function formatPercent(value) {
605
+ return (value * 100).toFixed(0) + "%";
606
+ }
481
607
 
482
- Generated: ${generatedAt}
608
+ function formatNumber(value) {
609
+ return new Intl.NumberFormat("en-US").format(value);
610
+ }
483
611
 
484
- ## Overview
612
+ function progressBar(value, max = 10) {
613
+ if (value === null) return "N/A";
614
+ const filled = Math.max(0, Math.min(10, Math.round((value / max) * 10)));
615
+ return "█".repeat(filled) + "░".repeat(10 - filled);
616
+ }
617
+
618
+ function markdownText(value) {
619
+ return String(value).replaceAll("|", "\\|").replace(/\r?\n/g, " ").trim();
620
+ }
621
+
622
+ function deltaLabel(current, previous) {
623
+ if (current === null || previous === null) return "N/A";
624
+ const delta = current - previous;
625
+ if (Math.abs(delta) < 0.005) return "→ 0.00";
626
+ return (delta > 0 ? "↑ +" : "↓ ") + delta.toFixed(2);
627
+ }
628
+
629
+ function countRows(map, order) {
630
+ return order.map((key) => "| " + key + " | " + (map.get(key) ?? 0) + " |").join("\n");
631
+ }
485
632
 
486
- - Events: ${events.length}
487
- - Average prompt health: ${formatAverage(overallAverage)}/10
488
- - Events with tests: ${eventsWithTests}
489
- - Events with unavailable usage: ${unavailableUsage}
633
+ function insightMarkdown(title, summary, emptyMessage) {
634
+ const categories = sortedCounts(summary.counts);
635
+ const categoryRows = categories.length
636
+ ? categories.map(([category, count]) => "| " + category + " | " + count + " |").join("\n")
637
+ : "| None | 0 |";
638
+ const examples = summary.examples.length
639
+ ? summary.examples.map((item) => "- **" + item.category + ":** " + markdownText(item.message)).join("\n")
640
+ : "- " + emptyMessage;
641
+ return "## " + title + "\n\n| Category | Count |\n| --- | ---: |\n" + categoryRows + "\n\n### Recent examples\n\n" + examples;
642
+ }
490
643
 
491
- ## Quality dimensions
644
+ export function buildReport(inputEvents, options = {}) {
645
+ const current = analyzeEvents(inputEvents);
646
+ const previous = analyzeEvents(options.previousEvents ?? []);
647
+ const totalEventCount = options.totalEventCount ?? current.count;
648
+ const selectedLabel = options.all ? "all events" : "latest " + current.count + " events";
649
+ const usageRows = ["platform_reported", "estimated", "unavailable"].map((source) => {
650
+ const bucket = current.usage.get(source);
651
+ return "| " + source + " | " + bucket.events + " | " +
652
+ (bucket.hasInput ? formatNumber(bucket.inputTokens) : "N/A") + " | " +
653
+ (bucket.hasOutput ? formatNumber(bucket.outputTokens) : "N/A") + " | " +
654
+ (bucket.hasCost ? "$" + bucket.cost.toFixed(6) : "N/A") + " |";
655
+ }).join("\n");
656
+ const dimensionRows = SCORE_FIELDS.map((field) => {
657
+ const value = current.dimensions.get(field);
658
+ const prior = previous.dimensions.get(field);
659
+ return "| " + field + " | " + progressBar(value) + " | " + formatAverage(value) + " | " + deltaLabel(value, prior) + " |";
660
+ }).join("\n");
661
+ const weaknessSummary = { counts: current.weaknessCounts, examples: current.weaknessExamples };
662
+ const generatedAt = new Date().toISOString();
663
+ return [
664
+ "# Prompt Observer Report",
665
+ "",
666
+ "> Window: **" + selectedLabel + "** of " + totalEventCount + " stored events · Generated " + generatedAt,
667
+ "",
668
+ "## Health snapshot",
669
+ "",
670
+ "| Events | Prompt health | vs previous window | Completed | Low ambiguity | Verified tasks | Usage coverage |",
671
+ "| ---: | ---: | ---: | ---: | ---: | ---: | ---: |",
672
+ "| " + current.count + " | **" + formatAverage(current.overallAverage) + "/10** | " + deltaLabel(current.overallAverage, previous.overallAverage) + " | " + formatPercent(current.completionRate) + " | " + formatPercent(current.lowRiskRate) + " | " + formatPercent(current.verificationCoverage) + " | " + formatPercent(current.usageCoverage) + " |",
673
+ "",
674
+ "## Quality dimensions",
675
+ "",
676
+ "| Dimension | Health | Average | Trend |",
677
+ "| --- | --- | ---: | ---: |",
678
+ dimensionRows,
679
+ "",
680
+ insightMarkdown("Strengths", current.strengths, "No strengths recorded in this window."),
681
+ "",
682
+ insightMarkdown("Weaknesses", weaknessSummary, "No material weaknesses detected."),
683
+ "",
684
+ insightMarkdown("Improvement suggestions", current.suggestions, "No change needed."),
685
+ "",
686
+ "## Execution and verification",
687
+ "",
688
+ "| Result | Count |",
689
+ "| --- | ---: |",
690
+ countRows(current.resultCounts, ["completed", "partial", "blocked"]),
691
+ "",
692
+ "| Test status | Count |",
693
+ "| --- | ---: |",
694
+ countRows(current.testCounts, ["passed", "failed", "not_run", "unknown"]),
695
+ "",
696
+ "| Weakness severity | Count |",
697
+ "| --- | ---: |",
698
+ countRows(current.severityCounts, ["high", "medium", "low"]),
699
+ "",
700
+ "| Ambiguity risk | Count |",
701
+ "| --- | ---: |",
702
+ countRows(current.riskCounts, ["high", "medium", "low"]),
703
+ "",
704
+ "## Usage",
705
+ "",
706
+ "| Source | Events | Input tokens | Output tokens | Cost (USD) |",
707
+ "| --- | ---: | ---: | ---: | ---: |",
708
+ usageRows,
709
+ "",
710
+ "Usage coverage is " + formatPercent(current.usageCoverage) + ". Exact and estimated values are intentionally kept separate; unavailable values are never invented.",
711
+ "",
712
+ PRICING.updated_at
713
+ ? "Estimated costs use pricing snapshot **" + PRICING.updated_at + "** ([source](" + PRICING.source_url + ")). Cached tokens, tools, subscriptions, discounts, and other provider charges are excluded."
714
+ : "No pricing snapshot was available, so estimated costs were not calculated.",
715
+ "",
716
+ ].join("\n");
717
+ }
492
718
 
493
- | Dimension | Average |
494
- | --- | ---: |
495
- ${SCORE_FIELDS.map((field) => `| ${field} | ${formatAverage(dimensionAverages.get(field))} |`).join("\n")}
719
+ function dashboardApp(payload) {
720
+ const scoreFields = payload.scoreFields;
721
+ let range = payload.defaultRange;
722
+ const filters = { task: "all", status: "all", risk: "all" };
723
+ const byId = (id) => document.getElementById(id);
724
+ const pct = (value) => Math.round(value * 100) + "%";
725
+ const avg = (values) => values.length ? values.reduce((a, b) => a + b, 0) / values.length : null;
726
+ const health = (event) => avg(scoreFields.map((field) => event[field]));
727
+ const number = (value) => new Intl.NumberFormat("en-US").format(value);
728
+ const addOption = (select, value) => {
729
+ const option = document.createElement("option");
730
+ option.value = value;
731
+ option.textContent = value;
732
+ select.append(option);
733
+ };
734
+ const unique = (field) => [...new Set(payload.events.map(field))].sort();
735
+ unique((event) => event.task_type).forEach((value) => addOption(byId("task-filter"), value));
736
+ unique((event) => event.execution_signals.result_status).forEach((value) => addOption(byId("status-filter"), value));
737
+ unique((event) => event.ambiguity_risk).forEach((value) => addOption(byId("risk-filter"), value));
496
738
 
497
- ## Weaknesses
739
+ function selectedEvents() {
740
+ return payload.events.slice(-range).filter((event) =>
741
+ (filters.task === "all" || event.task_type === filters.task) &&
742
+ (filters.status === "all" || event.execution_signals.result_status === filters.status) &&
743
+ (filters.risk === "all" || event.ambiguity_risk === filters.risk)
744
+ );
745
+ }
498
746
 
499
- | Category | Count |
500
- | --- | ---: |
501
- ${topWeaknesses || "| None recorded | 0 |"}
747
+ function setCard(id, value, note) {
748
+ byId(id).querySelector(".value").textContent = value;
749
+ byId(id).querySelector(".note").textContent = note;
750
+ }
502
751
 
503
- ### Severity
752
+ function renderDimensions(events) {
753
+ const container = byId("dimensions");
754
+ container.replaceChildren();
755
+ scoreFields.forEach((field) => {
756
+ const value = avg(events.map((event) => event[field]));
757
+ const row = document.createElement("div");
758
+ row.className = "bar-row";
759
+ const label = document.createElement("span");
760
+ label.textContent = field;
761
+ const track = document.createElement("div");
762
+ track.className = "bar-track";
763
+ const fill = document.createElement("div");
764
+ fill.className = "bar-fill";
765
+ fill.style.width = (value === null ? 0 : value * 10) + "%";
766
+ const score = document.createElement("strong");
767
+ score.textContent = value === null ? "N/A" : value.toFixed(2);
768
+ track.append(fill);
769
+ row.append(label, track, score);
770
+ container.append(row);
771
+ });
772
+ }
504
773
 
505
- | Severity | Count |
506
- | --- | ---: |
507
- ${tableRows(severityCounts, ["high", "medium", "low"])}
774
+ function renderTrend(events) {
775
+ const svg = byId("trend");
776
+ svg.replaceChildren();
777
+ const values = events.map(health);
778
+ if (values.length < 2) {
779
+ const message = document.createElementNS("http://www.w3.org/2000/svg", "text");
780
+ message.setAttribute("x", "20");
781
+ message.setAttribute("y", "55");
782
+ message.textContent = values.length ? "Add another event to see a trend." : "No matching events.";
783
+ svg.append(message);
784
+ return;
785
+ }
786
+ const width = 600;
787
+ const height = 120;
788
+ const points = values.map((value, index) => {
789
+ const x = 10 + index * (width - 20) / (values.length - 1);
790
+ const y = height - 10 - value / 10 * (height - 20);
791
+ return x + "," + y;
792
+ }).join(" ");
793
+ const line = document.createElementNS("http://www.w3.org/2000/svg", "polyline");
794
+ line.setAttribute("points", points);
795
+ line.setAttribute("fill", "none");
796
+ line.setAttribute("stroke", "#7c5cff");
797
+ line.setAttribute("stroke-width", "4");
798
+ line.setAttribute("stroke-linecap", "round");
799
+ line.setAttribute("stroke-linejoin", "round");
800
+ svg.append(line);
801
+ }
508
802
 
509
- ## Ambiguity risk
803
+ function renderDistribution(id, values, order) {
804
+ const container = byId(id);
805
+ container.replaceChildren();
806
+ const total = values.length;
807
+ order.forEach((name) => {
808
+ const count = values.filter((value) => value === name).length;
809
+ const row = document.createElement("div");
810
+ row.className = "bar-row";
811
+ const label = document.createElement("span");
812
+ label.textContent = name;
813
+ const track = document.createElement("div");
814
+ track.className = "bar-track";
815
+ const fill = document.createElement("div");
816
+ fill.className = "bar-fill";
817
+ fill.style.width = (total ? count / total * 100 : 0) + "%";
818
+ const score = document.createElement("strong");
819
+ score.textContent = String(count);
820
+ track.append(fill);
821
+ row.append(label, track, score);
822
+ container.append(row);
823
+ });
824
+ }
510
825
 
511
- | Risk | Count |
512
- | --- | ---: |
513
- ${tableRows(riskCounts, ["high", "medium", "low"])}
826
+ function insightItems(events, field, weakness) {
827
+ const counts = new Map();
828
+ const examples = [];
829
+ const seen = new Set();
830
+ [...events].reverse().forEach((event) => {
831
+ event[field].forEach((raw) => {
832
+ const item = typeof raw === "string" ? { category: "other", message: raw } : raw;
833
+ counts.set(item.category, (counts.get(item.category) || 0) + 1);
834
+ const key = item.category + "\n" + item.message.toLowerCase();
835
+ if (examples.length < 5 && !seen.has(key)) {
836
+ seen.add(key);
837
+ examples.push({ ...item, severity: weakness ? item.severity : null });
838
+ }
839
+ });
840
+ });
841
+ return { counts: [...counts.entries()].sort((a, b) => b[1] - a[1]), examples };
842
+ }
514
843
 
515
- ## Execution outcomes
844
+ function renderInsights(id, title, data, empty) {
845
+ const root = byId(id);
846
+ root.replaceChildren();
847
+ const heading = document.createElement("div");
848
+ heading.className = "insight-heading";
849
+ const name = document.createElement("h3");
850
+ name.textContent = title;
851
+ heading.append(name);
852
+ data.counts.slice(0, 4).forEach(([category, count]) => {
853
+ const chip = document.createElement("span");
854
+ chip.className = "chip";
855
+ chip.textContent = category + " · " + count;
856
+ heading.append(chip);
857
+ });
858
+ root.append(heading);
859
+ if (!data.examples.length) {
860
+ const blank = document.createElement("p");
861
+ blank.className = "empty";
862
+ blank.textContent = empty;
863
+ root.append(blank);
864
+ return;
865
+ }
866
+ const list = document.createElement("ul");
867
+ data.examples.forEach((item) => {
868
+ const entry = document.createElement("li");
869
+ const category = document.createElement("strong");
870
+ category.textContent = item.category + ": ";
871
+ entry.append(category, document.createTextNode(item.message));
872
+ list.append(entry);
873
+ });
874
+ root.append(list);
875
+ }
516
876
 
517
- | Status | Count |
518
- | --- | ---: |
519
- ${tableRows(resultCounts, ["completed", "partial", "blocked"])}
877
+ function renderUsage(events) {
878
+ const sources = ["platform_reported", "estimated", "unavailable"];
879
+ const body = byId("usage-body");
880
+ body.replaceChildren();
881
+ sources.forEach((source) => {
882
+ const matching = events.filter((event) => event.usage.source === source);
883
+ const sum = (field) => matching.reduce((total, event) => total + (event.usage[field] ?? 0), 0);
884
+ const has = (field) => matching.some((event) => event.usage[field] !== null);
885
+ const row = document.createElement("tr");
886
+ [source, matching.length, has("input_tokens") ? number(sum("input_tokens")) : "N/A", has("output_tokens") ? number(sum("output_tokens")) : "N/A", has("cost_usd") ? "$" + sum("cost_usd").toFixed(6) : "N/A"].forEach((value) => {
887
+ const cell = document.createElement("td");
888
+ cell.textContent = value;
889
+ row.append(cell);
890
+ });
891
+ body.append(row);
892
+ });
893
+ }
520
894
 
521
- ## Verification
895
+ function render() {
896
+ const events = selectedEvents();
897
+ const healthValue = avg(events.map(health));
898
+ const completed = events.filter((event) => event.execution_signals.result_status === "completed").length;
899
+ const lowRisk = events.filter((event) => event.ambiguity_risk === "low").length;
900
+ const verified = events.filter((event) => event.execution_signals.tests_run.length > 0).length;
901
+ const covered = events.filter((event) => event.usage.source !== "unavailable").length;
902
+ setCard("health-card", healthValue === null ? "N/A" : healthValue.toFixed(2), "out of 10");
903
+ setCard("completion-card", events.length ? pct(completed / events.length) : "N/A", completed + " completed");
904
+ setCard("ambiguity-card", events.length ? pct(lowRisk / events.length) : "N/A", lowRisk + " low-risk");
905
+ setCard("verification-card", events.length ? pct(verified / events.length) : "N/A", verified + " with tests");
906
+ setCard("usage-card", events.length ? pct(covered / events.length) : "N/A", covered + " exact or estimated");
907
+ byId("event-count").textContent = events.length + " matching event" + (events.length === 1 ? "" : "s");
908
+ renderDimensions(events);
909
+ renderTrend(events);
910
+ renderDistribution("outcomes", events.map((event) => event.execution_signals.result_status), ["completed", "partial", "blocked"]);
911
+ renderDistribution("risks", events.map((event) => event.ambiguity_risk), ["low", "medium", "high"]);
912
+ renderDistribution("tests", events.flatMap((event) => event.execution_signals.tests_run.map((test) => test.status)), ["passed", "failed", "not_run", "unknown"]);
913
+ renderInsights("strengths", "Strengths", insightItems(events, "strengths", false), "No strengths recorded in this window.");
914
+ renderInsights("weaknesses", "Weaknesses", insightItems(events, "weaknesses", true), "No material weaknesses detected.");
915
+ renderInsights("suggestions", "Improvement suggestions", insightItems(events, "improvement_suggestions", false), "No change needed.");
916
+ renderUsage(events);
917
+ document.querySelectorAll("[data-range]").forEach((button) => button.classList.toggle("active", Number(button.dataset.range) === range));
918
+ }
522
919
 
523
- | Test status | Count |
524
- | --- | ---: |
525
- ${tableRows(testCounts, ["passed", "failed", "not_run", "unknown"])}
920
+ document.querySelectorAll("[data-range]").forEach((button) => button.addEventListener("click", () => {
921
+ range = Number(button.dataset.range);
922
+ render();
923
+ }));
924
+ [["task-filter", "task"], ["status-filter", "status"], ["risk-filter", "risk"]].forEach(([id, key]) => {
925
+ byId(id).addEventListener("change", (event) => {
926
+ filters[key] = event.target.value;
927
+ render();
928
+ });
929
+ });
930
+ render();
931
+ }
526
932
 
527
- ## Reported usage
933
+ function escapeHtml(value) {
934
+ return String(value).replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;").replaceAll('"', "&quot;");
935
+ }
528
936
 
529
- - Input tokens: ${hasInputTokens ? totalInputTokens : "N/A"}
530
- - Output tokens: ${hasOutputTokens ? totalOutputTokens : "N/A"}
531
- - Cost (USD): ${hasCost ? totalCost.toFixed(6) : "N/A"}
532
- `;
937
+ export function buildHtmlReport(inputEvents, options = {}) {
938
+ const events = inputEvents.map(normalizeEvent).map((event) => ({
939
+ timestamp: event.timestamp,
940
+ task_type: event.task_type,
941
+ ambiguity_risk: event.ambiguity_risk,
942
+ ...Object.fromEntries(SCORE_FIELDS.map((field) => [field, event[field]])),
943
+ strengths: event.strengths,
944
+ weaknesses: event.weaknesses,
945
+ improvement_suggestions: event.improvement_suggestions,
946
+ execution_signals: {
947
+ result_status: event.execution_signals.result_status,
948
+ tests_run: event.execution_signals.tests_run.map((test) => ({ status: test.status })),
949
+ },
950
+ usage: event.usage,
951
+ }));
952
+ const defaultRange = options.all ? events.length : 50;
953
+ const payload = JSON.stringify({ events, scoreFields: SCORE_FIELDS, defaultRange }).replaceAll("<", "\\u003c");
954
+ const generatedAt = new Date().toISOString();
955
+ const totalEventCount = options.totalEventCount ?? events.length;
956
+ const windowLabel = options.all ? "All events" : "Latest " + events.length + " events";
957
+ const styles = [
958
+ ":root{color-scheme:dark;--bg:#0b1020;--panel:#141a2e;--panel2:#1b2340;--text:#eef1ff;--muted:#9ca8c7;--accent:#7c5cff;--accent2:#36d1a0;--danger:#ff6b82;--border:#293251}",
959
+ "*{box-sizing:border-box}body{margin:0;background:radial-gradient(circle at 15% 0,#24205a 0,transparent 35%),var(--bg);color:var(--text);font:14px/1.55 Inter,ui-sans-serif,system-ui,-apple-system,Segoe UI,sans-serif}",
960
+ "main{max-width:1180px;margin:auto;padding:44px 24px 80px}header{display:flex;justify-content:space-between;gap:20px;align-items:end;margin-bottom:28px}h1{font-size:34px;margin:0 0 6px}h2,h3{margin:0}.sub,.note,.empty{color:var(--muted)}",
961
+ ".toolbar{display:flex;flex-wrap:wrap;gap:10px;padding:14px;background:rgba(20,26,46,.85);border:1px solid var(--border);border-radius:16px;margin-bottom:18px;position:sticky;top:10px;z-index:3;backdrop-filter:blur(12px)}",
962
+ "button,select{background:var(--panel2);color:var(--text);border:1px solid var(--border);border-radius:9px;padding:8px 11px}button{cursor:pointer}button.active{background:var(--accent);border-color:var(--accent)}",
963
+ ".cards{display:grid;grid-template-columns:repeat(5,1fr);gap:14px;margin:18px 0}.card,.panel,.insight{background:linear-gradient(145deg,rgba(27,35,64,.95),rgba(20,26,46,.95));border:1px solid var(--border);border-radius:18px;padding:20px;box-shadow:0 14px 45px rgba(0,0,0,.18)}",
964
+ ".card .label{color:var(--muted);font-size:12px;text-transform:uppercase;letter-spacing:.08em}.card .value{font-size:28px;font-weight:750;margin:8px 0 2px}.grid{display:grid;grid-template-columns:1fr 1fr;gap:16px}.panel h2{font-size:18px;margin-bottom:18px}",
965
+ ".bar-row{display:grid;grid-template-columns:165px 1fr 42px;gap:12px;align-items:center;margin:12px 0}.bar-track{height:9px;background:#252d4c;border-radius:9px;overflow:hidden}.bar-fill{height:100%;background:linear-gradient(90deg,var(--accent),var(--accent2));border-radius:9px}",
966
+ "#trend{width:100%;height:140px;background:rgba(8,12,27,.35);border-radius:12px}#trend text{fill:var(--muted);font-size:13px}.insights{display:grid;grid-template-columns:repeat(3,1fr);gap:16px;margin-top:16px}.insight-heading{display:flex;gap:7px;align-items:center;flex-wrap:wrap;margin-bottom:14px}.insight-heading h3{width:100%;font-size:18px}.chip{font-size:11px;background:#282f53;color:#cbd2ef;padding:4px 7px;border-radius:20px}",
967
+ "ul{padding-left:20px;margin:8px 0}li{margin:9px 0;color:#dbe0f5}table{width:100%;border-collapse:collapse;margin-top:10px}th,td{text-align:left;padding:10px;border-bottom:1px solid var(--border)}th{color:var(--muted);font-size:12px;text-transform:uppercase}.usage{margin-top:16px}.footnote{margin-top:12px;color:var(--muted);font-size:12px}",
968
+ "@media(max-width:900px){.cards{grid-template-columns:repeat(2,1fr)}.grid,.insights{grid-template-columns:1fr}header{display:block}.bar-row{grid-template-columns:130px 1fr 38px}}@media(max-width:520px){main{padding:24px 14px}.cards{grid-template-columns:1fr}.toolbar{position:static}}",
969
+ ].join("");
970
+ return [
971
+ "<!doctype html><html lang=\"en\"><head><meta charset=\"utf-8\"><meta name=\"viewport\" content=\"width=device-width,initial-scale=1\">",
972
+ "<title>Prompt Observer Report</title><style>" + styles + "</style></head><body><main>",
973
+ "<header><div><h1>Prompt Observer</h1><div class=\"sub\">" + escapeHtml(windowLabel) + " of " + totalEventCount + " stored · " + escapeHtml(generatedAt) + "</div></div><strong id=\"event-count\"></strong></header>",
974
+ "<section class=\"toolbar\"><button data-range=\"10\">Last 10</button><button data-range=\"25\">Last 25</button><button data-range=\"50\">Last 50</button>" + (options.all ? "<button data-range=\"" + events.length + "\" class=\"active\">All loaded</button>" : "") + "<select id=\"task-filter\"><option value=\"all\">All task types</option></select><select id=\"status-filter\"><option value=\"all\">All outcomes</option></select><select id=\"risk-filter\"><option value=\"all\">All ambiguity levels</option></select></section>",
975
+ "<section class=\"cards\"><article class=\"card\" id=\"health-card\"><div class=\"label\">Prompt health</div><div class=\"value\"></div><div class=\"note\"></div></article><article class=\"card\" id=\"completion-card\"><div class=\"label\">Completion</div><div class=\"value\"></div><div class=\"note\"></div></article><article class=\"card\" id=\"ambiguity-card\"><div class=\"label\">Low ambiguity</div><div class=\"value\"></div><div class=\"note\"></div></article><article class=\"card\" id=\"verification-card\"><div class=\"label\">Verification</div><div class=\"value\"></div><div class=\"note\"></div></article><article class=\"card\" id=\"usage-card\"><div class=\"label\">Usage coverage</div><div class=\"value\"></div><div class=\"note\"></div></article></section>",
976
+ "<section class=\"grid\"><article class=\"panel\"><h2>Quality dimensions</h2><div id=\"dimensions\"></div></article><article class=\"panel\"><h2>Prompt-health trend</h2><svg id=\"trend\" viewBox=\"0 0 600 120\" preserveAspectRatio=\"none\"></svg></article></section>",
977
+ "<section class=\"grid usage\"><article class=\"panel\"><h2>Execution outcomes</h2><div id=\"outcomes\"></div><h2>Ambiguity risk</h2><div id=\"risks\"></div></article><article class=\"panel\"><h2>Verification results</h2><div id=\"tests\"></div></article></section>",
978
+ "<section class=\"insights\"><article class=\"insight\" id=\"strengths\"></article><article class=\"insight\" id=\"weaknesses\"></article><article class=\"insight\" id=\"suggestions\"></article></section>",
979
+ "<section class=\"panel usage\"><h2>Usage by source</h2><table><thead><tr><th>Source</th><th>Events</th><th>Input tokens</th><th>Output tokens</th><th>Cost</th></tr></thead><tbody id=\"usage-body\"></tbody></table><p class=\"footnote\">Exact and estimated values are separated. Estimated costs use pricing snapshot " + escapeHtml(PRICING.updated_at ?? "unavailable") + " and exclude cached tokens, tools, subscriptions, discounts, and provider-specific charges.</p></section>",
980
+ "</main><script>const reportData=" + payload + ";(" + dashboardApp.toString() + ")(reportData);</script></body></html>",
981
+ ].join("");
533
982
  }
534
983
 
535
- export async function generateReport(targetPath) {
984
+ export async function generateReport(targetPath, options = {}) {
536
985
  const observerDirectory = await resolveObserverDirectory(targetPath ?? ".");
537
986
  const eventsPath = join(observerDirectory, "events.jsonl");
538
- const events = await readEvents(eventsPath);
539
- const report = buildReport(events);
987
+ const selected = await selectReportWindow(eventsPath, options);
988
+ const report = buildReport(selected.current, {
989
+ previousEvents: selected.previous,
990
+ totalEventCount: selected.totalEventCount,
991
+ all: options.all,
992
+ });
993
+ const html = buildHtmlReport(selected.current, {
994
+ totalEventCount: selected.totalEventCount,
995
+ all: options.all,
996
+ });
540
997
  const reportPath = join(observerDirectory, "report.md");
541
- await writeFile(reportPath, report, "utf8");
542
- return { eventCount: events.length, reportPath, report };
998
+ const htmlPath = join(observerDirectory, "report.html");
999
+ await Promise.all([writeFile(reportPath, report, "utf8"), writeFile(htmlPath, html, "utf8")]);
1000
+ return {
1001
+ eventCount: selected.current.length,
1002
+ totalEventCount: selected.totalEventCount,
1003
+ reportPath,
1004
+ htmlPath,
1005
+ report,
1006
+ html,
1007
+ };
543
1008
  }
544
1009
 
545
1010
  function printHelp() {
546
- process.stdout.write(`Prompt Observer ${SCHEMA_VERSION}
547
-
548
- Usage:
549
- prompt-observer init <target-path>
550
- prompt-observer log <event-file> [--target <target-path>]
551
- prompt-observer report [target-path]
552
- prompt-observer validate <event-file>
553
- prompt-observer help
554
- `);
1011
+ process.stdout.write([
1012
+ "Prompt Observer " + SCHEMA_VERSION,
1013
+ "",
1014
+ "Usage:",
1015
+ " prompt-observer init <target-path>",
1016
+ " prompt-observer log <event-file> [--target <target-path>]",
1017
+ " prompt-observer report [target-path] [--limit <count> | --all]",
1018
+ " prompt-observer validate <event-file>",
1019
+ " prompt-observer help",
1020
+ "",
1021
+ ].join("\n"));
555
1022
  }
556
1023
 
557
1024
  function optionValue(args, name) {
558
1025
  const index = args.indexOf(name);
559
1026
  if (index === -1) return undefined;
560
- if (!args[index + 1]) throw new Error(`${name} requires a value.`);
1027
+ if (!args[index + 1] || args[index + 1].startsWith("--")) throw new Error(name + " requires a value.");
561
1028
  return args[index + 1];
562
1029
  }
563
1030
 
1031
+ function positionalArgs(args, optionsWithValues = []) {
1032
+ const positions = [];
1033
+ for (let index = 0; index < args.length; index += 1) {
1034
+ if (optionsWithValues.includes(args[index])) {
1035
+ index += 1;
1036
+ } else if (!args[index].startsWith("--")) {
1037
+ positions.push(args[index]);
1038
+ }
1039
+ }
1040
+ return positions;
1041
+ }
1042
+
564
1043
  export async function runCli(args) {
565
1044
  const [command, ...rest] = args;
566
1045
  if (!command || command === "help" || command === "--help" || command === "-h") {
@@ -569,35 +1048,44 @@ export async function runCli(args) {
569
1048
  }
570
1049
  if (command === "init") {
571
1050
  const result = await initProject(rest[0] ?? ".");
572
- process.stdout.write(`Initialized Prompt Observer at ${result.observerDirectory}\n${result.actions.join("\n")}\n`);
1051
+ process.stdout.write("Initialized Prompt Observer at " + result.observerDirectory + "\n" + result.actions.join("\n") + "\n");
573
1052
  return;
574
1053
  }
575
1054
  if (command === "validate") {
576
1055
  if (!rest[0]) throw new Error("validate requires an event JSON file.");
577
1056
  const event = JSON.parse(await readFile(resolve(rest[0]), "utf8"));
578
1057
  const errors = validateEvent(event);
579
- if (errors.length > 0) throw new Error(`Event validation failed:\n- ${errors.join("\n- ")}`);
580
- process.stdout.write(`Valid event: ${event.event_id}\n`);
1058
+ if (errors.length > 0) throw new Error("Event validation failed:\n- " + errors.join("\n- "));
1059
+ process.stdout.write("Valid event: " + event.event_id + "\n");
581
1060
  return;
582
1061
  }
583
1062
  if (command === "log") {
584
1063
  const target = optionValue(rest, "--target");
585
- const result = await logEvent(rest[0], target);
586
- process.stdout.write(`Logged ${result.eventId} to ${result.eventsPath}\n`);
1064
+ const positions = positionalArgs(rest, ["--target"]);
1065
+ const result = await logEvent(positions[0], target);
1066
+ process.stdout.write("Logged " + result.eventId + " to " + result.eventsPath + "\n");
587
1067
  return;
588
1068
  }
589
1069
  if (command === "report") {
590
- const result = await generateReport(rest[0] ?? ".");
591
- process.stdout.write(`Generated ${result.reportPath} from ${result.eventCount} event(s).\n`);
1070
+ const all = rest.includes("--all");
1071
+ const rawLimit = optionValue(rest, "--limit");
1072
+ if (all && rawLimit !== undefined) throw new Error("--all and --limit cannot be used together.");
1073
+ const limit = rawLimit === undefined ? DEFAULT_REPORT_LIMIT : Number(rawLimit);
1074
+ if (!all && (!Number.isInteger(limit) || limit < 1 || limit > 100000)) {
1075
+ throw new Error("--limit must be an integer from 1 to 100000.");
1076
+ }
1077
+ const positions = positionalArgs(rest, ["--limit"]);
1078
+ const result = await generateReport(positions[0] ?? ".", { all, limit });
1079
+ process.stdout.write("Generated " + result.reportPath + " and " + result.htmlPath + " from " + result.eventCount + " of " + result.totalEventCount + " event(s).\n");
592
1080
  return;
593
1081
  }
594
- throw new Error(`Unknown command: ${command}`);
1082
+ throw new Error("Unknown command: " + command);
595
1083
  }
596
1084
 
597
1085
  const isDirectExecution = process.argv[1] && resolve(process.argv[1]) === resolve(fileURLToPath(import.meta.url));
598
1086
  if (isDirectExecution) {
599
1087
  runCli(process.argv.slice(2)).catch((error) => {
600
- process.stderr.write(`Error: ${error.message}\n`);
1088
+ process.stderr.write("Error: " + error.message + "\n");
601
1089
  process.exitCode = 1;
602
1090
  });
603
1091
  }