behavior-wrapped 0.8.1 → 0.8.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/assets/index-CAStSiD9.js +11 -0
- package/dist/assets/index-CudcHj4C.css +1 -0
- package/dist/index.html +2 -2
- package/fixtures/cowork-sessions/synthetic-account/synthetic-workspace/local_44444444-4444-4444-8444-444444444444/audit.jsonl +1 -1
- package/package.json +1 -1
- package/server/analysis.mjs +139 -18
- package/server/cli.mjs +1 -1
- package/server/discovery.mjs +19 -2
- package/server/interaction-evidence.mjs +14 -12
- package/server/launcher.mjs +2 -2
- package/server/model-pricing.mjs +61 -0
- package/server/public-report-schema.mjs +57 -0
- package/dist/assets/index-CdzEeY5M.js +0 -11
- package/dist/assets/index-hFREdA4j.css +0 -1
package/server/analysis.mjs
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import crypto from "node:crypto";
|
|
2
|
-
import { safeEvidenceText, redactText } from "./privacy.mjs";
|
|
2
|
+
import { safeEvidenceText, redactAggregateText, redactText } from "./privacy.mjs";
|
|
3
3
|
import { isFrustratedMessage, isGratefulMessage } from "./frustration-card.mjs";
|
|
4
|
+
import { estimateModelUsageCost } from "./model-pricing.mjs";
|
|
4
5
|
import { displayModelName } from "./model-names.mjs";
|
|
5
6
|
|
|
6
7
|
export { displayModelName } from "./model-names.mjs";
|
|
@@ -77,6 +78,90 @@ function stockPhraseCounts(texts) {
|
|
|
77
78
|
}));
|
|
78
79
|
}
|
|
79
80
|
|
|
81
|
+
const instructionOpeningPattern = /^(?:please\s+)?(?:always|never|do not|don['’]t|avoid|be|check|commit|continue|focus|give|include|keep|make|only|prefer|push|remember|respond|run|show|stop|tell|use|write)\b/i;
|
|
82
|
+
const userAccountabilityPattern = /\b(?:my (?:mistake|fault|bad)|i (?:was|am|got (?:that|this|it)) wrong|you (?:were|are) right|(?:i['’]m )?sorry[,—–-]? (?:i (?:was|got|missed|misunderstood|overlooked|forgot|made)|that was|you were))\b/i;
|
|
83
|
+
const agentAccountabilityPattern = /\b(?:my (?:mistake|fault|error)|i (?:was|am|got (?:that|this|it)) wrong|you(?:['’]re| are| were) right|(?:i['’]m )?sorry[,—–-]? (?:i (?:was|got|missed|misunderstood|overlooked|forgot|made)|that was|you were))\b/i;
|
|
84
|
+
|
|
85
|
+
function repeatedInstructions(sessionRecords) {
|
|
86
|
+
const instructions = new Map();
|
|
87
|
+
for (let sessionIndex = 0; sessionIndex < sessionRecords.length; sessionIndex++) {
|
|
88
|
+
for (const record of sessionRecords[sessionIndex].records) {
|
|
89
|
+
if (record.type !== "user" || record.isMeta) continue;
|
|
90
|
+
const cleaned = redactAggregateText(proseText(visibleText(record)));
|
|
91
|
+
if (!cleaned || /\[(?:REDACTED|REMOVED)[^\]]*\]/i.test(cleaned)) continue;
|
|
92
|
+
for (const rawClause of cleaned.split(/(?<=[.!?])\s+|[;\n]+/)) {
|
|
93
|
+
const instruction = rawClause.replace(/^[-*\d.)\s]+/, "").replace(/\s+/g, " ").trim();
|
|
94
|
+
const words = instruction.match(/\p{L}+(?:['’]\p{L}+)?/gu) || [];
|
|
95
|
+
if (words.length < 3 || words.length > 18 || instruction.length > 160 || !instructionOpeningPattern.test(instruction)) continue;
|
|
96
|
+
if (/https?:\/\/|(?:\/Users\/|\/home\/)|```|<[^>]+>|[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/i.test(instruction)) continue;
|
|
97
|
+
const key = instruction.normalize("NFKC").replace(/[’‘]/g, "'").toLocaleLowerCase().replace(/[^\p{L}'\s]/gu, "").replace(/\s+/g, " ").trim();
|
|
98
|
+
if (!key) continue;
|
|
99
|
+
const item = instructions.get(key) || { instruction, occurrences: 0, sessions: new Set() };
|
|
100
|
+
item.occurrences++;
|
|
101
|
+
item.sessions.add(sessionIndex);
|
|
102
|
+
instructions.set(key, item);
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
return [...instructions.values()]
|
|
107
|
+
.filter((item) => item.occurrences >= 2)
|
|
108
|
+
.sort((left, right) => right.sessions.size - left.sessions.size || right.occurrences - left.occurrences || left.instruction.localeCompare(right.instruction))
|
|
109
|
+
.slice(0, 4)
|
|
110
|
+
.map((item) => ({ instruction: item.instruction, occurrences: item.occurrences, distinctSessions: item.sessions.size }));
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
function permissionScore(record, agent) {
|
|
114
|
+
const mode = String(record.permissionMode || "").toLowerCase();
|
|
115
|
+
if (mode) {
|
|
116
|
+
const scores = { plan: 0, default: agent === "cowork" ? 40 : 25, acceptedits: 65, dontask: 80, bypasspermissions: 100 };
|
|
117
|
+
return scores[mode] ?? null;
|
|
118
|
+
}
|
|
119
|
+
if (agent !== "codex" || (!record.approvalPolicy && !record.sandboxPolicy)) return null;
|
|
120
|
+
const approvalScores = { untrusted: 0, "on-request": 20, "on-failure": 40, never: 60 };
|
|
121
|
+
const sandboxScores = { "read-only": 0, "workspace-write": 25, "danger-full-access": 40 };
|
|
122
|
+
const approval = approvalScores[String(record.approvalPolicy || "").toLowerCase()] ?? 0;
|
|
123
|
+
const sandbox = sandboxScores[String(record.sandboxPolicy || "").toLowerCase()] ?? 0;
|
|
124
|
+
return Math.min(100, approval + sandbox);
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
function buildTrustCurve(sessionRecords) {
|
|
128
|
+
const days = new Map();
|
|
129
|
+
let observations = 0;
|
|
130
|
+
let autonomousObservations = 0;
|
|
131
|
+
for (const { records, agent = "claude" } of sessionRecords) {
|
|
132
|
+
for (const record of records) {
|
|
133
|
+
const score = permissionScore(record, agent);
|
|
134
|
+
const timestamp = record.timestamp ? new Date(record.timestamp) : null;
|
|
135
|
+
if (score === null || !timestamp || !Number.isFinite(timestamp.getTime())) continue;
|
|
136
|
+
const date = timestamp.toISOString().slice(0, 10);
|
|
137
|
+
const item = days.get(date) || { total: 0, observations: 0 };
|
|
138
|
+
item.total += score;
|
|
139
|
+
item.observations++;
|
|
140
|
+
days.set(date, item);
|
|
141
|
+
observations++;
|
|
142
|
+
if (score >= 65) autonomousObservations++;
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
const ordered = [...days].sort((left, right) => left[0].localeCompare(right[0]));
|
|
146
|
+
if (ordered.length < 2) return null;
|
|
147
|
+
const firstDay = new Date(`${ordered[0][0]}T00:00:00.000Z`).getTime();
|
|
148
|
+
const points = ordered.map(([date, item]) => ({
|
|
149
|
+
dayOffset: Math.round((new Date(`${date}T00:00:00.000Z`).getTime() - firstDay) / 86_400_000),
|
|
150
|
+
score: Number((item.total / item.observations).toFixed(1)),
|
|
151
|
+
observations: item.observations,
|
|
152
|
+
}));
|
|
153
|
+
return {
|
|
154
|
+
points,
|
|
155
|
+
startScore: points[0].score,
|
|
156
|
+
endScore: points.at(-1).score,
|
|
157
|
+
change: Number((points.at(-1).score - points[0].score).toFixed(1)),
|
|
158
|
+
observations,
|
|
159
|
+
autonomousObservations,
|
|
160
|
+
autonomousPercentage: Number((autonomousObservations / observations * 100).toFixed(1)),
|
|
161
|
+
method: "Scores source-specific permission modes from 0 (planning/read-only with approvals) to 100 (no approvals with full access), then averages observations by day.",
|
|
162
|
+
};
|
|
163
|
+
}
|
|
164
|
+
|
|
80
165
|
const anomalyScripts = [
|
|
81
166
|
{ language: "Japanese", locale: "ja", expression: /[\p{Script=Hiragana}\p{Script=Katakana}]/gu },
|
|
82
167
|
{ language: "Korean", locale: "ko", expression: /\p{Script=Hangul}/gu },
|
|
@@ -251,15 +336,6 @@ function finding(kind, title, summary, method, score, evidence) {
|
|
|
251
336
|
return { id: crypto.randomUUID(), kind, title, summary, method, confidence: confidence(score), evidence };
|
|
252
337
|
}
|
|
253
338
|
|
|
254
|
-
function ratesFor(model, agent) {
|
|
255
|
-
const value = String(model || "").toLowerCase();
|
|
256
|
-
if (value.includes("opus")) return { input: 5, output: 25, cacheWrite: 6.25, cacheRead: 0.5 };
|
|
257
|
-
if (value.includes("sonnet")) return { input: 3, output: 15, cacheWrite: 3.75, cacheRead: 0.3 };
|
|
258
|
-
if (value.includes("haiku")) return { input: 1, output: 5, cacheWrite: 1.25, cacheRead: 0.1 };
|
|
259
|
-
if (agent === "codex" || value.startsWith("gpt")) return { input: 1.25, output: 10, cacheWrite: 1.25, cacheRead: 0.125 };
|
|
260
|
-
return { input: 3, output: 15, cacheWrite: 3.75, cacheRead: 0.3 };
|
|
261
|
-
}
|
|
262
|
-
|
|
263
339
|
function analyzeBehavior(sessionRecords) {
|
|
264
340
|
const findings = [];
|
|
265
341
|
for (const { sessionId, records } of sessionRecords) {
|
|
@@ -339,6 +415,7 @@ export function analyzeSessions(sessionRecords) {
|
|
|
339
415
|
const toolCounts = new Map();
|
|
340
416
|
const agentCounts = new Map(agentDefinitions.map(({ agent }) => [agent, 0]));
|
|
341
417
|
const modelTokens = new Map();
|
|
418
|
+
const interruptionModels = new Map();
|
|
342
419
|
const activeDays = new Set();
|
|
343
420
|
let prompts = 0;
|
|
344
421
|
let toolCalls = 0;
|
|
@@ -353,15 +430,20 @@ export function analyzeSessions(sessionRecords) {
|
|
|
353
430
|
let agentResponseCount = 0;
|
|
354
431
|
let frustratedMessages = 0;
|
|
355
432
|
let gratefulMessages = 0;
|
|
433
|
+
let userApologies = 0;
|
|
434
|
+
let agentApologies = 0;
|
|
435
|
+
const apologyReview = { user: [], agent: [] };
|
|
436
|
+
let longestUninterruptedRun = null;
|
|
356
437
|
const assistantProse = [];
|
|
357
438
|
const sessionTurnCounts = [];
|
|
358
|
-
for (const { records, agent = "claude" } of sessionRecords) {
|
|
439
|
+
for (const { sessionId, records, agent = "claude" } of sessionRecords) {
|
|
359
440
|
agentCounts.set(agent, (agentCounts.get(agent) || 0) + 1);
|
|
360
441
|
const timestamps = records.map((r) => r.timestamp).filter(Boolean).map((value) => new Date(value).getTime()).filter(Number.isFinite);
|
|
361
442
|
if (timestamps.length > 1) totalDurationMs += Math.max(...timestamps) - Math.min(...timestamps);
|
|
362
443
|
let currentResponseWords = 0;
|
|
363
444
|
let hasCurrentPrompt = false;
|
|
364
445
|
let sessionTurns = 0;
|
|
446
|
+
let currentModel = `${agent === "codex" ? "Codex" : agent === "cowork" ? "Cowork" : "Claude"} model`;
|
|
365
447
|
const finishResponse = () => {
|
|
366
448
|
if (hasCurrentPrompt && currentResponseWords > 0) {
|
|
367
449
|
agentResponseWords += currentResponseWords;
|
|
@@ -369,8 +451,17 @@ export function analyzeSessions(sessionRecords) {
|
|
|
369
451
|
}
|
|
370
452
|
currentResponseWords = 0;
|
|
371
453
|
};
|
|
372
|
-
for (const record of records) {
|
|
454
|
+
for (const [recordIndex, record] of records.entries()) {
|
|
373
455
|
const text = visibleText(record);
|
|
456
|
+
const declaredModel = record?.message?.model || record?.model;
|
|
457
|
+
if (typeof declaredModel === "string" && declaredModel) currentModel = declaredModel;
|
|
458
|
+
if (record.type === "system" && record.subtype === "turn_duration") {
|
|
459
|
+
const durationMs = Number(record.durationMs);
|
|
460
|
+
if (Number.isFinite(durationMs) && durationMs > 0 && (!longestUninterruptedRun || durationMs > longestUninterruptedRun.durationMs)) {
|
|
461
|
+
const definition = agentDefinitions.find((item) => item.agent === agent);
|
|
462
|
+
longestUninterruptedRun = { durationMs: Math.round(durationMs), agent, agentName: definition?.name || "Agent" };
|
|
463
|
+
}
|
|
464
|
+
}
|
|
374
465
|
if (record.type === "user" && !record.isMeta && text) {
|
|
375
466
|
finishResponse();
|
|
376
467
|
hasCurrentPrompt = true;
|
|
@@ -379,14 +470,33 @@ export function analyzeSessions(sessionRecords) {
|
|
|
379
470
|
sessionTurns++;
|
|
380
471
|
if (isFrustratedMessage(text)) frustratedMessages++;
|
|
381
472
|
if (isGratefulMessage(text)) gratefulMessages++;
|
|
473
|
+
if (userAccountabilityPattern.test(proseText(text))) {
|
|
474
|
+
userApologies++;
|
|
475
|
+
apologyReview.user.push({
|
|
476
|
+
candidateId: `apology-user-${userApologies}`,
|
|
477
|
+
location: { sessionId, recordIndex, timestamp: record.timestamp || null },
|
|
478
|
+
});
|
|
479
|
+
}
|
|
382
480
|
} else if (record.type === "assistant" && hasCurrentPrompt && text) {
|
|
383
481
|
currentResponseWords += wordCount(text);
|
|
384
482
|
}
|
|
385
|
-
if (record.type === "assistant" && text)
|
|
483
|
+
if (record.type === "assistant" && text) {
|
|
484
|
+
assistantProse.push(text);
|
|
485
|
+
if (agentAccountabilityPattern.test(proseText(text))) {
|
|
486
|
+
agentApologies++;
|
|
487
|
+
apologyReview.agent.push({
|
|
488
|
+
candidateId: `apology-agent-${agentApologies}`,
|
|
489
|
+
location: { sessionId, recordIndex, timestamp: record.timestamp || null },
|
|
490
|
+
});
|
|
491
|
+
}
|
|
492
|
+
}
|
|
386
493
|
const d = day(record.timestamp);
|
|
387
494
|
if (d) activeDays.add(d);
|
|
388
495
|
if (record.type === "user" && !record.isMeta && text) prompts++;
|
|
389
|
-
if (record.type === "system" && /interrupt/i.test(`${record.subtype || ""} ${record.content || ""}`))
|
|
496
|
+
if ((record.type === "system" && /interrupt/i.test(`${record.subtype || ""} ${record.content || ""}`)) || record.interruptedMessageId) {
|
|
497
|
+
interruptions++;
|
|
498
|
+
interruptionModels.set(currentModel, (interruptionModels.get(currentModel) || 0) + 1);
|
|
499
|
+
}
|
|
390
500
|
const usage = record?.message?.usage;
|
|
391
501
|
if (usage) {
|
|
392
502
|
const input = Number(usage.input_tokens) || 0;
|
|
@@ -403,8 +513,7 @@ export function analyzeSessions(sessionRecords) {
|
|
|
403
513
|
tokenBreakdown.reasoning += reasoning;
|
|
404
514
|
const model = record?.message?.model || `${agent === "codex" ? "Codex" : "Claude"} model`;
|
|
405
515
|
modelTokens.set(model, (modelTokens.get(model) || 0) + recordTokens);
|
|
406
|
-
|
|
407
|
-
estimatedCostUsd += (input * rates.input + (output + reasoning) * rates.output + cacheWrite * rates.cacheWrite + cacheRead * rates.cacheRead) / 1_000_000;
|
|
516
|
+
estimatedCostUsd += estimateModelUsageCost(usage, model, agent);
|
|
408
517
|
}
|
|
409
518
|
for (const tool of toolUses(record)) {
|
|
410
519
|
toolCalls++;
|
|
@@ -427,6 +536,9 @@ export function analyzeSessions(sessionRecords) {
|
|
|
427
536
|
tokens: modelTokenCount,
|
|
428
537
|
percentage: tokens ? Number((modelTokenCount / tokens * 100).toFixed(1)) : 0,
|
|
429
538
|
}));
|
|
539
|
+
const interruptionsByModel = [...interruptionModels]
|
|
540
|
+
.sort((left, right) => right[1] - left[1] || left[0].localeCompare(right[0]))
|
|
541
|
+
.map(([model, count]) => ({ model, name: displayModelName(model), count }));
|
|
430
542
|
const stats = {
|
|
431
543
|
sessions: totalSessions,
|
|
432
544
|
activeDays: activeDays.size,
|
|
@@ -434,6 +546,7 @@ export function analyzeSessions(sessionRecords) {
|
|
|
434
546
|
prompts,
|
|
435
547
|
toolCalls,
|
|
436
548
|
interruptions,
|
|
549
|
+
interruptionsByModel,
|
|
437
550
|
tokens,
|
|
438
551
|
tokenBreakdown,
|
|
439
552
|
agentWords: agentResponseWords,
|
|
@@ -450,7 +563,15 @@ export function analyzeSessions(sessionRecords) {
|
|
|
450
563
|
analyzedMessages: userInputCount,
|
|
451
564
|
method: "Counts user messages matching conservative frustration or gratitude phrase patterns; this is an approximate tone signal, not a judgment of emotion.",
|
|
452
565
|
},
|
|
566
|
+
apologyCounts: {
|
|
567
|
+
user: userApologies,
|
|
568
|
+
agent: agentApologies,
|
|
569
|
+
method: "Counts visible messages containing explicit admissions of error or fault; generic capability apologies are excluded.",
|
|
570
|
+
},
|
|
571
|
+
longestUninterruptedRun,
|
|
572
|
+
trustCurve: buildTrustCurve(sessionRecords),
|
|
453
573
|
stockPhrases: stockPhraseCounts(assistantProse),
|
|
574
|
+
repeatedInstructions: repeatedInstructions(sessionRecords),
|
|
454
575
|
outputLanguages: languageBreakdown(assistantProse),
|
|
455
576
|
languageAnomaly: languageAnomalyBreakdown(sessionRecords),
|
|
456
577
|
languageMethod: "Estimates natural-language word share in assistant text after removing fenced code, inline code, URLs, paths, and markup. Script detection and small Latin-language lexicons are approximate.",
|
|
@@ -460,9 +581,9 @@ export function analyzeSessions(sessionRecords) {
|
|
|
460
581
|
agents,
|
|
461
582
|
models,
|
|
462
583
|
estimatedCostUsd: Number(estimatedCostUsd.toFixed(2)),
|
|
463
|
-
costEstimateMethod: "API-equivalent estimate using
|
|
584
|
+
costEstimateMethod: "API-equivalent estimate using current standard list prices by exact model, including distinct cache-read, 5-minute cache-write, and 1-hour cache-write rates.",
|
|
464
585
|
};
|
|
465
|
-
return { stats, findings: analyzeBehavior(sessionRecords) };
|
|
586
|
+
return { stats, findings: analyzeBehavior(sessionRecords), apologyReview };
|
|
466
587
|
}
|
|
467
588
|
|
|
468
589
|
function donationRedactionInventory(detections) {
|
package/server/cli.mjs
CHANGED
|
@@ -259,7 +259,7 @@ async function createWrapped() {
|
|
|
259
259
|
const id = createReportId();
|
|
260
260
|
const safeFindings = analyzed.findings.map(({ evidence, method, ...finding }) => finding);
|
|
261
261
|
const hasPrivateWorkaroundEvidence = Boolean(analyzed.workaroundReview?.occurrences?.length || analyzed.workaroundReview?.borderline?.length);
|
|
262
|
-
const report = { id, createdAt: new Date().toISOString(), rangeLabel: formatRange(chosenSessions), source: formatAgentSource(chosenSessions), stats: analyzed.stats, findings: safeFindings, phraseCard: analyzed.phraseCard, interactionCard: analyzed.interactionCard, interactionReview: analyzed.interactionReview, workaroundCard: analyzed.workaroundCard, workaroundReview: analyzed.workaroundReview, sessionSummaries: analyzed.sessionSummaries || [], sessionIds: chosenSessions.map((session) => session.id), donationHelperUrl: `${baseUrl}/donate/${id}`, privacy: { shareSafe: !hasPrivateWorkaroundEvidence, containsTranscriptText: hasPrivateWorkaroundEvidence, externalTransmission: !localOnly, analysisMode: localOnly ? "local-only" : "remote", leaderboardParticipation: localOnly ? "excluded" : "included-by-default", ...(localOnly ? { transmittedData: `None; ${testMode ? "test" : "local-only"} mode stays on this device.`, externalRecipient: "None" } : { transmittedData: "redacted phrase, interaction-tone, and session-topic candidates; locally redacted context windows around explicit blockers for workaround discovery; aggregate report statistics; and a random client ID only", externalRecipient: "Behavior Wrapped relay, OpenRouter, a zero-data-retention GPT-5.6 Luna provider, and public report hosting" }) } };
|
|
262
|
+
const report = { id, createdAt: new Date().toISOString(), rangeLabel: formatRange(chosenSessions), source: formatAgentSource(chosenSessions), stats: analyzed.stats, findings: safeFindings, phraseCard: analyzed.phraseCard, interactionCard: analyzed.interactionCard, interactionReview: analyzed.interactionReview, apologyReview: analyzed.apologyReview, workaroundCard: analyzed.workaroundCard, workaroundReview: analyzed.workaroundReview, sessionSummaries: analyzed.sessionSummaries || [], sessionIds: chosenSessions.map((session) => session.id), donationHelperUrl: `${baseUrl}/donate/${id}`, privacy: { shareSafe: !hasPrivateWorkaroundEvidence, containsTranscriptText: hasPrivateWorkaroundEvidence, externalTransmission: !localOnly, analysisMode: localOnly ? "local-only" : "remote", leaderboardParticipation: localOnly ? "excluded" : "included-by-default", ...(localOnly ? { transmittedData: `None; ${testMode ? "test" : "local-only"} mode stays on this device.`, externalRecipient: "None" } : { transmittedData: "redacted phrase, interaction-tone, and session-topic candidates; locally redacted context windows around explicit blockers for workaround discovery; aggregate report statistics; and a random client ID only", externalRecipient: "Behavior Wrapped relay, OpenRouter, a zero-data-retention GPT-5.6 Luna provider, and public report hosting" }) } };
|
|
263
263
|
let publicUrl = null;
|
|
264
264
|
if (!localOnly) {
|
|
265
265
|
progress.start("Publishing share-safe Wrapped", "strict aggregate-only schema");
|
package/server/discovery.mjs
CHANGED
|
@@ -126,7 +126,7 @@ function shouldKeepCoworkRecord(record, seenUuids) {
|
|
|
126
126
|
if (seenUuids.has(record.uuid)) return false;
|
|
127
127
|
seenUuids.add(record.uuid);
|
|
128
128
|
}
|
|
129
|
-
return ["user", "assistant", "system"].includes(record.type);
|
|
129
|
+
return ["user", "assistant", "system", "result"].includes(record.type);
|
|
130
130
|
}
|
|
131
131
|
|
|
132
132
|
function coworkMetadataFromRecords(file, stat, records, metadata = readCoworkMetadata(file)) {
|
|
@@ -427,6 +427,14 @@ function normalizeCodexRecords(records, { includePrivateToolDetails = false } =
|
|
|
427
427
|
if (record.type === "turn_context" && typeof payload.model === "string") {
|
|
428
428
|
currentModel = payload.model;
|
|
429
429
|
hasSeenModelContext = true;
|
|
430
|
+
const sandboxPolicy = payload.sandbox_policy?.type || payload.permission_profile?.type || (typeof payload.sandbox_policy === "string" ? payload.sandbox_policy : null);
|
|
431
|
+
if (payload.approval_policy || sandboxPolicy) normalized.push({
|
|
432
|
+
type: "system",
|
|
433
|
+
subtype: "permission_mode",
|
|
434
|
+
timestamp: record.timestamp,
|
|
435
|
+
approvalPolicy: typeof payload.approval_policy === "string" ? payload.approval_policy : null,
|
|
436
|
+
sandboxPolicy,
|
|
437
|
+
});
|
|
430
438
|
} else if (record.type === "response_item" && payload.type === "message" && (payload.role === "user" || payload.role === "assistant")) {
|
|
431
439
|
const content = textBlocks(payload.content);
|
|
432
440
|
if (content.length) normalized.push({ type: payload.role, timestamp: record.timestamp, message: { content, ...(payload.role === "assistant" ? { model: currentModel } : {}) } });
|
|
@@ -447,8 +455,10 @@ function normalizeCodexRecords(records, { includePrivateToolDetails = false } =
|
|
|
447
455
|
const canSummarizeRestriction = status.failed || (!status.wrapped && restrictionEligibleActions.has(semantics?.action));
|
|
448
456
|
const errorSummary = canSummarizeRestriction ? restrictionErrorSummary(output) : null;
|
|
449
457
|
normalized.push({ type: "user", isMeta: true, timestamp: record.timestamp, message: { content: [{ type: "tool_result", is_error: Boolean(errorSummary) || status.failed || unwrappedFailure, error_summary: errorSummary, ...(includePrivateToolDetails && output ? { content: output } : {}) }] } });
|
|
458
|
+
} else if (record.type === "event_msg" && payload.type === "task_complete" && Number(payload.duration_ms) > 0) {
|
|
459
|
+
normalized.push({ type: "system", subtype: "turn_duration", timestamp: record.timestamp, durationMs: Number(payload.duration_ms), model: currentModel });
|
|
450
460
|
} else if (record.type === "event_msg" && payload.type === "turn_aborted") {
|
|
451
|
-
normalized.push({ type: "system", subtype: "interrupt", timestamp: record.timestamp, content: "interrupt" });
|
|
461
|
+
normalized.push({ type: "system", subtype: "interrupt", timestamp: record.timestamp, content: "interrupt", model: currentModel });
|
|
452
462
|
} else if (record.type === "event_msg" && payload.type === "token_count" && payload.info?.total_token_usage) {
|
|
453
463
|
const total = payload.info.total_token_usage;
|
|
454
464
|
const usage = Object.fromEntries(Object.keys(previousUsage).map((key) => [key, Math.max(0, (Number(total[key]) || 0) - previousUsage[key])]));
|
|
@@ -477,6 +487,11 @@ function normalizeCoworkRecords(records) {
|
|
|
477
487
|
for (const record of records) {
|
|
478
488
|
if (!shouldKeepCoworkRecord(record, seenUuids)) continue;
|
|
479
489
|
const timestamp = coworkRecordTimestamp(record);
|
|
490
|
+
if (record.type === "result") {
|
|
491
|
+
const durationMs = Number(record.duration_ms);
|
|
492
|
+
if (!record.is_error && record.subtype === "success" && durationMs > 0) normalized.push({ type: "system", subtype: "turn_duration", ...(timestamp ? { timestamp } : {}), durationMs, model: typeof record.model === "string" ? record.model : "Cowork model" });
|
|
493
|
+
continue;
|
|
494
|
+
}
|
|
480
495
|
const message = record.message && typeof record.message === "object" ? {
|
|
481
496
|
...(record.message.content !== undefined ? { content: record.message.content } : {}),
|
|
482
497
|
...(typeof record.message.model === "string" ? { model: record.message.model } : {}),
|
|
@@ -488,6 +503,8 @@ function normalizeCoworkRecords(records) {
|
|
|
488
503
|
...(record.isMeta ? { isMeta: true } : {}),
|
|
489
504
|
...(record.subtype ? { subtype: record.subtype } : {}),
|
|
490
505
|
...(record.content !== undefined ? { content: record.content } : {}),
|
|
506
|
+
...(typeof record.model === "string" ? { model: record.model } : {}),
|
|
507
|
+
...(typeof record.permissionMode === "string" ? { permissionMode: record.permissionMode } : {}),
|
|
491
508
|
...(message ? { message } : {}),
|
|
492
509
|
};
|
|
493
510
|
const messageId = record.type === "assistant" && typeof record?.message?.id === "string" ? record.message.id : null;
|
|
@@ -15,12 +15,12 @@ function transcriptRole(record) {
|
|
|
15
15
|
return record?.type === "assistant" ? "assistant" : record?.type === "user" && !record?.isMeta ? "user" : null;
|
|
16
16
|
}
|
|
17
17
|
|
|
18
|
-
function matchingRecordIndex(reference, records) {
|
|
18
|
+
function matchingRecordIndex(reference, records, expectedRole) {
|
|
19
19
|
const index = reference?.location?.recordIndex;
|
|
20
20
|
if (Number.isInteger(index) && index >= 0 && index < records.length) return index;
|
|
21
21
|
const timestamp = reference?.location?.timestamp;
|
|
22
22
|
if (!timestamp) return null;
|
|
23
|
-
const fallback = records.findIndex((record) => record?.timestamp === timestamp && record
|
|
23
|
+
const fallback = records.findIndex((record) => record?.timestamp === timestamp && transcriptRole(record) === expectedRole);
|
|
24
24
|
return fallback >= 0 ? fallback : null;
|
|
25
25
|
}
|
|
26
26
|
|
|
@@ -33,12 +33,12 @@ function adjacentMessage(records, fromIndex, direction) {
|
|
|
33
33
|
return null;
|
|
34
34
|
}
|
|
35
35
|
|
|
36
|
-
function exactOccurrence(reference, index, records, metadata) {
|
|
37
|
-
const recordIndex = matchingRecordIndex(reference, records);
|
|
36
|
+
function exactOccurrence(reference, index, records, metadata, expectedRole) {
|
|
37
|
+
const recordIndex = matchingRecordIndex(reference, records, expectedRole);
|
|
38
38
|
if (recordIndex === null) return null;
|
|
39
39
|
const record = records[recordIndex];
|
|
40
40
|
const text = visibleText(record);
|
|
41
|
-
if (record
|
|
41
|
+
if (transcriptRole(record) !== expectedRole || !text) return null;
|
|
42
42
|
const before = adjacentMessage(records, recordIndex, -1);
|
|
43
43
|
const after = adjacentMessage(records, recordIndex, 1);
|
|
44
44
|
return {
|
|
@@ -50,16 +50,16 @@ function exactOccurrence(reference, index, records, metadata) {
|
|
|
50
50
|
startedAt: metadata?.startedAt || records.find((item) => item?.timestamp)?.timestamp || null,
|
|
51
51
|
},
|
|
52
52
|
timestamp: record.timestamp || null,
|
|
53
|
-
messages: [before, { role:
|
|
53
|
+
messages: [before, { role: expectedRole, text, timestamp: record.timestamp || null, highlighted: true }, after].filter(Boolean),
|
|
54
54
|
};
|
|
55
55
|
}
|
|
56
56
|
|
|
57
|
-
function buildKind(
|
|
58
|
-
return (
|
|
57
|
+
function buildKind(review, kind, expectedRole, recordsById, metadataById) {
|
|
58
|
+
return (review?.[kind] || []).slice(0, MAX_OCCURRENCES_PER_KIND).flatMap((reference, index) => {
|
|
59
59
|
const sessionId = reference?.location?.sessionId;
|
|
60
60
|
const records = recordsById.get(sessionId);
|
|
61
61
|
if (!records) return [];
|
|
62
|
-
const occurrence = exactOccurrence(reference, index, records, metadataById.get(sessionId));
|
|
62
|
+
const occurrence = exactOccurrence(reference, index, records, metadataById.get(sessionId), expectedRole);
|
|
63
63
|
return occurrence ? [occurrence] : [];
|
|
64
64
|
});
|
|
65
65
|
}
|
|
@@ -67,11 +67,13 @@ function buildKind(report, kind, recordsById, metadataById) {
|
|
|
67
67
|
export function makeInteractionEvidencePreview(report, sessionRecords, metadataById) {
|
|
68
68
|
const recordsById = new Map(sessionRecords.map((session) => [session.sessionId, session.records]));
|
|
69
69
|
return {
|
|
70
|
-
format: "behavior-wrapped-interaction-evidence-
|
|
70
|
+
format: "behavior-wrapped-interaction-evidence-v2",
|
|
71
71
|
localPrivate: true,
|
|
72
72
|
standardRedactionsApplied: false,
|
|
73
73
|
reportId: report?.id,
|
|
74
|
-
frustrated: buildKind(report, "frustrated", recordsById, metadataById),
|
|
75
|
-
grateful: buildKind(report, "grateful", recordsById, metadataById),
|
|
74
|
+
frustrated: buildKind(report?.interactionReview, "frustrated", "user", recordsById, metadataById),
|
|
75
|
+
grateful: buildKind(report?.interactionReview, "grateful", "user", recordsById, metadataById),
|
|
76
|
+
userApologies: buildKind(report?.apologyReview, "user", "user", recordsById, metadataById),
|
|
77
|
+
agentApologies: buildKind(report?.apologyReview, "agent", "assistant", recordsById, metadataById),
|
|
76
78
|
};
|
|
77
79
|
}
|
package/server/launcher.mjs
CHANGED
|
@@ -107,7 +107,7 @@ const server = http.createServer(async (request, response) => {
|
|
|
107
107
|
if (request.method === "GET" && reportMatch) {
|
|
108
108
|
const report = loadReport(reportMatch[1]);
|
|
109
109
|
if (!report) return json(response, 404, { error: "Saved report not found" });
|
|
110
|
-
const { sessionIds, workaroundReview, interactionReview, ...shareSafeReport } = report;
|
|
110
|
+
const { sessionIds, workaroundReview, interactionReview, apologyReview, ...shareSafeReport } = report;
|
|
111
111
|
shareSafeReport.privacy = { ...shareSafeReport.privacy, shareSafe: true, containsTranscriptText: false };
|
|
112
112
|
return json(response, 200, shareSafeReport);
|
|
113
113
|
}
|
|
@@ -133,7 +133,7 @@ const server = http.createServer(async (request, response) => {
|
|
|
133
133
|
const report = loadReport(interactionEvidenceMatch[1]);
|
|
134
134
|
if (!report) return json(response, 404, { error: "Saved report not found" });
|
|
135
135
|
const allowed = new Set(report.sessionIds || []);
|
|
136
|
-
const references = [...(report.interactionReview?.frustrated || []), ...(report.interactionReview?.grateful || [])];
|
|
136
|
+
const references = [...(report.interactionReview?.frustrated || []), ...(report.interactionReview?.grateful || []), ...(report.apologyReview?.user || []), ...(report.apologyReview?.agent || [])];
|
|
137
137
|
const ids = [...new Set(references.map((reference) => reference?.location?.sessionId).filter((id) => allowed.has(id) && catalog.index.has(id)))].slice(0, 200);
|
|
138
138
|
const records = await chosenRecords(ids);
|
|
139
139
|
const labels = new Map(publicCatalog().sessions.map((session) => [session.id, session]));
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
// Standard API list prices in USD per million tokens, verified 2026-08-26.
|
|
2
|
+
// OpenAI: https://developers.openai.com/api/docs/models/compare
|
|
3
|
+
// Anthropic: https://platform.claude.com/docs/en/about-claude/pricing
|
|
4
|
+
const scaled = (value, multiplier) => Number((value * multiplier).toFixed(12));
|
|
5
|
+
|
|
6
|
+
const price = (input, output, cacheRead, cacheWrite5m, cacheWrite1h) => ({
|
|
7
|
+
input,
|
|
8
|
+
output,
|
|
9
|
+
cacheRead: cacheRead ?? scaled(input, 0.1),
|
|
10
|
+
cacheWrite5m: cacheWrite5m ?? scaled(input, 1.25),
|
|
11
|
+
cacheWrite1h: cacheWrite1h ?? scaled(input, 2),
|
|
12
|
+
});
|
|
13
|
+
|
|
14
|
+
const modelRates = [
|
|
15
|
+
[/gpt-5\.6-sol/, price(4, 20)],
|
|
16
|
+
[/gpt-5\.6-terra/, price(2, 12)],
|
|
17
|
+
[/gpt-5\.6-luna/, price(0.2, 1.2)],
|
|
18
|
+
[/gpt-5\.5-pro/, price(30, 180, 0, 30, 30)],
|
|
19
|
+
[/gpt-5\.5/, price(5, 30)],
|
|
20
|
+
[/gpt-5\.4-pro/, price(30, 180, 0, 30, 30)],
|
|
21
|
+
[/gpt-5\.4-mini/, price(0.75, 4.5)],
|
|
22
|
+
[/gpt-5\.4-nano/, price(0.2, 1.25)],
|
|
23
|
+
[/gpt-5\.4/, price(2.5, 15)],
|
|
24
|
+
[/gpt-5\.3-codex/, price(1.75, 14)],
|
|
25
|
+
[/gpt-5\.2/, price(1.75, 14)],
|
|
26
|
+
[/(?:claude-)?(?:fable|mythos)(?:-|\s)?5/, price(10, 50)],
|
|
27
|
+
[/(?:claude-)?opus(?:-|\s)?5/, price(5, 25)],
|
|
28
|
+
[/(?:claude-)?opus(?:-|\s)?4(?:-|\s)?(?:8|7|6|5)/, price(5, 25)],
|
|
29
|
+
[/(?:claude-)?opus(?:-|\s)?4(?:-|\s)?1/, price(15, 75)],
|
|
30
|
+
[/(?:claude-)?opus(?:-|\s)?4(?:\b|-20)/, price(15, 75)],
|
|
31
|
+
[/(?:claude-)?sonnet(?:-|\s)?5/, price(2, 10)],
|
|
32
|
+
[/(?:claude-)?sonnet(?:-|\s)?4/, price(3, 15)],
|
|
33
|
+
[/(?:claude-)?haiku(?:-|\s)?4(?:-|\s)?5/, price(1, 5)],
|
|
34
|
+
[/(?:claude-)?haiku(?:-|\s)?3(?:-|\s)?5/, price(0.8, 4)],
|
|
35
|
+
];
|
|
36
|
+
|
|
37
|
+
export function ratesFor(model, agent) {
|
|
38
|
+
const value = String(model || "").toLowerCase();
|
|
39
|
+
const match = modelRates.find(([pattern]) => pattern.test(value));
|
|
40
|
+
if (match) return match[1];
|
|
41
|
+
return agent === "codex" || value.startsWith("gpt") ? price(2.5, 15) : price(3, 15);
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export function estimateModelUsageCost(usage, model, agent) {
|
|
45
|
+
const input = Number(usage?.input_tokens) || 0;
|
|
46
|
+
const output = Number(usage?.output_tokens) || 0;
|
|
47
|
+
const reasoning = Number(usage?.reasoning_output_tokens) || 0;
|
|
48
|
+
const cacheRead = Number(usage?.cache_read_input_tokens) || 0;
|
|
49
|
+
const cacheWrite = Number(usage?.cache_creation_input_tokens) || 0;
|
|
50
|
+
const declared5m = Number(usage?.cache_creation?.ephemeral_5m_input_tokens) || 0;
|
|
51
|
+
const declared1h = Number(usage?.cache_creation?.ephemeral_1h_input_tokens) || 0;
|
|
52
|
+
const cacheWrite5m = declared5m + Math.max(0, cacheWrite - declared5m - declared1h);
|
|
53
|
+
const rates = ratesFor(model, agent);
|
|
54
|
+
return (
|
|
55
|
+
input * rates.input
|
|
56
|
+
+ (output + reasoning) * rates.output
|
|
57
|
+
+ cacheRead * rates.cacheRead
|
|
58
|
+
+ cacheWrite5m * rates.cacheWrite5m
|
|
59
|
+
+ declared1h * rates.cacheWrite1h
|
|
60
|
+
) / 1_000_000;
|
|
61
|
+
}
|
|
@@ -36,6 +36,44 @@ function safeStockPhrases(value) {
|
|
|
36
36
|
return stockPhraseLabels.map((phrase) => ({ phrase, count: counts.get(phrase) || 0 }));
|
|
37
37
|
}
|
|
38
38
|
|
|
39
|
+
function safeRepeatedInstructions(value) {
|
|
40
|
+
if (!Array.isArray(value)) return [];
|
|
41
|
+
return value.slice(0, 4).flatMap((item) => {
|
|
42
|
+
const instruction = safeText(item?.instruction, 160).trim();
|
|
43
|
+
const occurrences = Math.round(safeNumber(item?.occurrences, 1_000_000));
|
|
44
|
+
const distinctSessions = Math.round(safeNumber(item?.distinctSessions, 1_000_000));
|
|
45
|
+
if (!instruction || occurrences < 2 || distinctSessions < 1 || distinctSessions > occurrences) return [];
|
|
46
|
+
if (/\[(?:REDACTED|REMOVED)[^\]]*\]|https?:\/\/|(?:\/Users\/|\/home\/)|```|<[^>]+>|[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/i.test(instruction)) return [];
|
|
47
|
+
return [{ instruction, occurrences, distinctSessions }];
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function safeTrustCurve(value) {
|
|
52
|
+
if (!value || typeof value !== "object" || !Array.isArray(value.points)) return null;
|
|
53
|
+
let previousOffset = -1;
|
|
54
|
+
const points = value.points.slice(0, 4_000).flatMap((item) => {
|
|
55
|
+
const dayOffset = Math.round(safeNumber(item?.dayOffset, 3_650));
|
|
56
|
+
const score = safeNumber(item?.score, 100);
|
|
57
|
+
const observations = Math.round(safeNumber(item?.observations, 1_000_000));
|
|
58
|
+
if (dayOffset <= previousOffset || observations < 1) return [];
|
|
59
|
+
previousOffset = dayOffset;
|
|
60
|
+
return [{ dayOffset, score: Number(score.toFixed(1)), observations }];
|
|
61
|
+
});
|
|
62
|
+
if (points.length < 2) return null;
|
|
63
|
+
const observations = Math.round(safeNumber(value.observations, 10_000_000));
|
|
64
|
+
const autonomousObservations = Math.round(safeNumber(value.autonomousObservations, observations));
|
|
65
|
+
if (!observations || autonomousObservations > observations) return null;
|
|
66
|
+
return {
|
|
67
|
+
points,
|
|
68
|
+
startScore: points[0].score,
|
|
69
|
+
endScore: points.at(-1).score,
|
|
70
|
+
change: Number((points.at(-1).score - points[0].score).toFixed(1)),
|
|
71
|
+
observations,
|
|
72
|
+
autonomousObservations,
|
|
73
|
+
autonomousPercentage: Number((autonomousObservations / observations * 100).toFixed(1)),
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
|
|
39
77
|
export function sanitizePublicReport(value) {
|
|
40
78
|
if (!value || typeof value !== "object" || Array.isArray(value) || !/^[A-Za-z0-9_-]{8,32}$/.test(value.id || "")) return null;
|
|
41
79
|
const stats = value.stats;
|
|
@@ -52,6 +90,12 @@ export function sanitizePublicReport(value) {
|
|
|
52
90
|
const safeTokenBreakdown = tokenBreakdown && Object.values(tokenBreakdown).reduce((sum, count) => sum + count, 0) === safeTokens ? tokenBreakdown : null;
|
|
53
91
|
const safeStockPhraseCounts = safeStockPhrases(stats.stockPhrases);
|
|
54
92
|
const safeSessionTurnCounts = safeTurnCounts(stats.sessionTurnCounts);
|
|
93
|
+
const runAgent = allowedAgents.has(stats.longestUninterruptedRun?.agent) ? stats.longestUninterruptedRun.agent : null;
|
|
94
|
+
const safeLongestUninterruptedRun = runAgent && safeNumber(stats.longestUninterruptedRun?.durationMs, 7 * 24 * 60 * 60 * 1000) > 0 ? {
|
|
95
|
+
durationMs: Math.round(safeNumber(stats.longestUninterruptedRun.durationMs, 7 * 24 * 60 * 60 * 1000)),
|
|
96
|
+
agent: runAgent,
|
|
97
|
+
agentName: safeText(stats.longestUninterruptedRun?.agentName, 30),
|
|
98
|
+
} : null;
|
|
55
99
|
const phrase = value.phraseCard?.phrase;
|
|
56
100
|
const safePhrase = typeof phrase === "string" && /^[a-z]+(?:'[a-z]+)?(?: [a-z]+(?:'[a-z]+)?){3,9}$/.test(phrase) ? {
|
|
57
101
|
phrase,
|
|
@@ -90,16 +134,29 @@ export function sanitizePublicReport(value) {
|
|
|
90
134
|
sessions: Math.round(safeNumber(stats.sessions, 1_000_000)), activeDays: Math.round(safeNumber(stats.activeDays, 1_000_000)),
|
|
91
135
|
durationMinutes: Math.round(safeNumber(stats.durationMinutes)), prompts: Math.round(safeNumber(stats.prompts)), toolCalls: Math.round(safeNumber(stats.toolCalls)),
|
|
92
136
|
interruptions: Math.round(safeNumber(stats.interruptions)), tokens: safeTokens, ...(safeTokenBreakdown ? { tokenBreakdown: safeTokenBreakdown } : {}), agentWords: Math.round(safeNumber(stats.agentWords)),
|
|
137
|
+
interruptionsByModel: Array.isArray(stats.interruptionsByModel) ? stats.interruptionsByModel.slice(0, 10).flatMap((item) => {
|
|
138
|
+
const model = safeText(item?.model, 80);
|
|
139
|
+
const name = safeText(item?.name, 80);
|
|
140
|
+
const count = Math.round(safeNumber(item?.count, 1_000_000));
|
|
141
|
+
return model && name && count > 0 ? [{ model, name, count }] : [];
|
|
142
|
+
}) : [],
|
|
93
143
|
userWords: Math.round(safeNumber(stats.userWords)), agentUserWordRatio: safeNumber(stats.agentUserWordRatio, 10_000),
|
|
94
144
|
averageAgentResponseWords: Math.round(safeNumber(stats.averageAgentResponseWords)), averageUserInputWords: Math.round(safeNumber(stats.averageUserInputWords)),
|
|
95
145
|
longestSessionTurns: Math.max(0, ...safeSessionTurnCounts),
|
|
96
146
|
sessionTurnCounts: safeSessionTurnCounts,
|
|
147
|
+
longestUninterruptedRun: safeLongestUninterruptedRun,
|
|
148
|
+
trustCurve: safeTrustCurve(stats.trustCurve),
|
|
97
149
|
interactionTone: {
|
|
98
150
|
frustratedMessages: Math.round(safeNumber(stats.interactionTone?.frustratedMessages, 1_000_000)),
|
|
99
151
|
gratefulMessages: Math.round(safeNumber(stats.interactionTone?.gratefulMessages, 1_000_000)),
|
|
100
152
|
analyzedMessages: Math.round(safeNumber(stats.interactionTone?.analyzedMessages, 1_000_000)),
|
|
101
153
|
},
|
|
154
|
+
apologyCounts: {
|
|
155
|
+
user: Math.round(safeNumber(stats.apologyCounts?.user, 1_000_000)),
|
|
156
|
+
agent: Math.round(safeNumber(stats.apologyCounts?.agent, 1_000_000)),
|
|
157
|
+
},
|
|
102
158
|
...(safeStockPhraseCounts ? { stockPhrases: safeStockPhraseCounts } : {}),
|
|
159
|
+
repeatedInstructions: safeRepeatedInstructions(stats.repeatedInstructions),
|
|
103
160
|
outputLanguages: safeBreakdown(stats.outputLanguages, "language", "words", allowedLanguages),
|
|
104
161
|
languageAnomaly: safeLanguageAnomaly,
|
|
105
162
|
topics: safeBreakdown(stats.topics, "topic", "tokens", new Set(["Coding", "Writing", "Personal advice", "Research & search", "Planning", "Data & analysis", "Other"])),
|