cortad 0.3.9 → 0.3.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/contact-text.mjs +13 -33
- package/lib/keeping.mjs +1 -1
- package/lib/mcp.mjs +3 -3
- package/lib/read-text.mjs +15 -15
- package/lib/stores.mjs +2 -2
- package/lib/text.mjs +17 -16
- package/lib/verbs.mjs +1 -1
- package/local.mjs +7 -5
- package/package.json +1 -1
- package/skill/SKILL.md +23 -11
- package/skill/references/results.md +17 -17
package/lib/contact-text.mjs
CHANGED
|
@@ -93,40 +93,21 @@ function doorLines(v) {
|
|
|
93
93
|
return [...lines, ...inner.filter(Boolean).map((l) => ` ${l}`)];
|
|
94
94
|
}
|
|
95
95
|
|
|
96
|
-
// What a run would play, take and cost,
|
|
97
|
-
//
|
|
98
|
-
//
|
|
99
|
-
//
|
|
96
|
+
// What a run would play, take and cost, in one short line the agent hands the person: how many
|
|
97
|
+
// simulated users, about how long, and one price on their own key. A first run's price is what it is
|
|
98
|
+
// expected to cost (`expected`, from factors measured on recorded first runs); the most it could cost,
|
|
99
|
+
// the follow-ups and the pace at one request at a time were three more numbers nobody acted on.
|
|
100
100
|
function readyLine(r) {
|
|
101
101
|
if (!r || !r.trials) return null;
|
|
102
|
-
const
|
|
103
|
-
const
|
|
104
|
-
|
|
105
|
-
const
|
|
106
|
-
|
|
107
|
-
// a range when that is slower, and where fewer conversations fit at that pace, how many is said.
|
|
108
|
-
const upTo = has(r.atOne?.minutes) && r.atOne.minutes > r.minutes ? r.atOne.minutes : null;
|
|
109
|
-
const atOne = has(r.atOne?.trials) && r.atOne.trials < r.trials ? ` If your app answers one request at a time, about ${num(r.atOne.trials)} of them fit in that time.` : "";
|
|
110
|
-
const minutes = has(r.minutes) && `in about ${upTo ? `${num(r.minutes)} to ${plural(upTo, "minute")}` : plural(r.minutes, "minute")}`;
|
|
111
|
-
const size = [has(r.replies) && `about ${plural(r.replies, "reply", "replies")}`, minutes].filter(Boolean).join(" ");
|
|
112
|
-
const usd = (x) => `$${x.toFixed(2)}`;
|
|
113
|
-
// Some endpoints had no published rate or token counts: the price covers the others and says so.
|
|
114
|
-
// A first run is priced at what it is expected to cost (`expected`, from factors measured on recorded
|
|
115
|
-
// first runs) and the most it costs (`upTo`); a server from before `expected` sends the most alone.
|
|
116
|
-
// After a run of the app's own, as what that run measured. An older server sends a range (`usdHigh`).
|
|
117
|
-
// Where some endpoints had no price, on how many.
|
|
118
|
-
const amount = () => `${r.upTo ? "up to" : "about"} ${usd(r.usd)}${has(r.usdHigh) ? ` to ${usd(r.usdHigh)}` : ""}`;
|
|
119
|
-
const some = r.pricedOn ? `, on the ${num(r.pricedOn.endpoints)} of ${plural(r.pricedOn.of, "endpoint")} Cortad could price` : "";
|
|
120
|
-
const onKey = `on your ${r.provider ? `${r.provider} ` : ""}key${r.model ? ` for ${r.model}` : ""}`;
|
|
121
|
-
const both = r.upTo && has(r.expected);
|
|
122
|
-
// No price is said, as the card says it (guide.ts quoteLine), never left out.
|
|
123
|
-
const cost = has(r.usd)
|
|
124
|
-
? ` ${both ? `It is expected to cost ${usd(r.expected)}${soFar} ${onKey}, and the most it costs is ${usd(r.usd)}` : `It costs ${amount()}${soFar} ${onKey}`}${some}${r.estimated ? ", estimated from the length of the text since your provider sent no token counts" : ""}.`
|
|
125
|
-
: " It runs on your app's own model keys; Cortad could not price it from what your app's model calls reported.";
|
|
126
|
-
// No rules is a fact of a finished read: said while the code is still being read, it was false a minute later.
|
|
127
|
-
const rules = !has(r.rules) || (r.reading && r.rules === 0) ? "" : r.rules === 0 ? " No rules of your own were read from the code, so replies are checked against general standards only." : ` Every reply is checked against the ${plural(r.rules, "rule")} read from your code.`;
|
|
128
|
-
return `For the person: a run plays ${plays}${followUps}${size ? `: ${size}` : ""}.${atOne}${cost}${rules}${r.reading ? " Your code is still being read; the run waits for it and plays every conversation it adds." : ""}`;
|
|
102
|
+
const plays = `${plural(r.trials, "simulated user")}${r.reading ? " so far" : ""}`;
|
|
103
|
+
const minutes = has(r.minutes) ? ` in about ${plural(r.minutes, "minute")}` : "";
|
|
104
|
+
const key = `your ${r.provider ? `${r.provider} ` : ""}key`;
|
|
105
|
+
const price = has(r.usd) ? `About $${(r.upTo && has(r.expected) ? r.expected : r.usd).toFixed(2)} on ${key}.` : `It runs on ${key}.`;
|
|
106
|
+
return `A run plays ${plays}${minutes}. ${price}`;
|
|
129
107
|
}
|
|
108
|
+
// No rules is a fact of a finished read, said before a run is spent on general standards alone: said
|
|
109
|
+
// while the code is still being read, it was false a minute later.
|
|
110
|
+
const noRulesLine = (r) => (r?.trials && r.rules === 0 && !r.reading ? "No rules of your own were read from the code, so replies are checked against general standards only." : null);
|
|
130
111
|
|
|
131
112
|
// The endpoints no request has reached, as status lists them and reach sends them. `receipts`: doors
|
|
132
113
|
// the App section says answered at once with their answer out of Cortad's sight, so not said here.
|
|
@@ -165,8 +146,7 @@ export function contactLines(c, data = [], receipts = []) {
|
|
|
165
146
|
const lines = proven.length ? [head, ...proven.flatMap((v) => doorLines(alike ? { ...v, standing: null } : v))] : [];
|
|
166
147
|
const missing = unreached(c, receipts);
|
|
167
148
|
if (missing.length) lines.push(`Endpoints found in your code that have not answered yet (${num(missing.length)}):`, ...missing.map(unreachedLine), ...[reachLine(missing)].filter(Boolean));
|
|
168
|
-
const
|
|
169
|
-
if (ready) lines.push(ready);
|
|
149
|
+
for (const line of [readyLine(c.ready), noRulesLine(c.ready)]) if (line) lines.push(line);
|
|
170
150
|
// What holds Run is the server's next line (src/local/contact.ts heldLine), said once: printed here
|
|
171
151
|
// as well, ulaim's hold on its Qdrant was read twice in one status.
|
|
172
152
|
return [...lines, ...said];
|
package/lib/keeping.mjs
CHANGED
|
@@ -45,7 +45,7 @@ export function makeKeeping({ root, work, ledgerFile, onPath, say, settings, lau
|
|
|
45
45
|
let backedUp = false;
|
|
46
46
|
let seen = 0;
|
|
47
47
|
const tell = (lines) => { for (const line of lines) say(lower(line)); };
|
|
48
|
-
const held = (blocked) => tell(blocked.map((b) => `
|
|
48
|
+
const held = (blocked) => tell(blocked.map((b) => `One question for the person on the card in the browser: ${lower(b.why)}`));
|
|
49
49
|
|
|
50
50
|
async function keepAll(started) {
|
|
51
51
|
if (typed) {
|
package/lib/mcp.mjs
CHANGED
|
@@ -10,8 +10,8 @@ export const PROTOCOL = "2025-06-18";
|
|
|
10
10
|
// Codex keeps only the first 512 characters self-contained and Claude Code shares about 4 KB across
|
|
11
11
|
// every server, so the first paragraph stands alone and the whole stays under 1,500 characters.
|
|
12
12
|
export const instructions = (spec = "cortad") => [
|
|
13
|
-
"Cortad tests the AI in this repository: simulated users written from the app's code talk to the app running on this machine, and every reply is checked. Cortad sends each AI endpoint a test request itself. status
|
|
14
|
-
`Tell the person what status found in
|
|
13
|
+
"Cortad tests the AI in this repository: simulated users written from the app's code talk to the app running on this machine, and every reply is checked. Cortad sends each AI endpoint a test request itself. status says whether the app is up, what Cortad's audit of the AI code found, and what a run would play and cost. A run starts when the person presses Run in the browser or asks you.",
|
|
14
|
+
`Tell the person what status found in three or four short, plain lines, leaving out file paths, endpoints, status codes and setting names unless they ask. Before you stop, start \`npx -y ${spec} wait\` as a background command: it returns when the next run or verify ends, with what it found. After a run, call findings and tell the person what their users ran into and what you would change, worst first, three problems at most; the person then says which fix to make, and nothing is edited before their go. On their go, one fix at a time: verify replays the failing conversations and says whether the failure is gone, and a further round plays only when verify is called again. findings has each failure whole and findings numbers every rate. field_connect gives the steps to connect production. dispute records a check that reads this app wrong.`,
|
|
15
15
|
].join("\n\n");
|
|
16
16
|
export const INSTRUCTIONS = instructions();
|
|
17
17
|
|
|
@@ -23,7 +23,7 @@ const ABOUT = ["status", "reach", "run", "run_status", "findings", "verify", "di
|
|
|
23
23
|
|
|
24
24
|
export const TOOLS = [
|
|
25
25
|
{ name: "status", title: "Cortad status", annotations: readOnly,
|
|
26
|
-
description: "What Cortad found in this repository: whether the app is up and on which port,
|
|
26
|
+
description: "What Cortad found in this repository: whether the app is up and on which port, its audit of the prompts and of the code around the model, each endpoint's answer to Cortad's test request or the file and line it failed at, the plan with runs left, and the latest run. Applies at the start of a session, right after a connect, and before a commit that changes prompts, tools, models or retrieval. With show, one section in full, in pages.",
|
|
27
27
|
inputSchema: { type: "object", properties: { show: { type: "string", enum: SHOW, description: "A section to list in full: rules, standards, journeys, endpoints, trials, records or reviews; or machine, what Cortad does on this machine." }, page: { type: "integer", minimum: 1, description: "The page of that list. Default 1." } }, additionalProperties: false } },
|
|
28
28
|
{ name: "reach", title: "Send Cortad's test requests again",
|
|
29
29
|
description: "Sends Cortad's test request again to the endpoints status lists as failed or not yet answered, or to the paths named, from this machine to the app on its port. Each endpoint comes back with its HTTP status, its time and, when it is not 2xx, the start of what the app answered.",
|
package/lib/read-text.mjs
CHANGED
|
@@ -87,30 +87,30 @@ const listed = (names) => (names.length > 1 ? `${names.slice(0, -1).join(", ")}
|
|
|
87
87
|
|
|
88
88
|
// What the connect command does on this machine, from the facts it keeps in the runner's record:
|
|
89
89
|
// what it started and stops, the database copies, the settings its app runs with, what it loads,
|
|
90
|
-
// what a run holds and puts back, and what it keeps
|
|
91
|
-
//
|
|
90
|
+
// what a run holds and puts back, and what it keeps outside this repository. The first line is the
|
|
91
|
+
// one the terminal says at connect (connectedText); the rest is read when someone asks.
|
|
92
92
|
export function machineText(f) {
|
|
93
93
|
if (!f) return ["No connect command is running for this repository now, so nothing Cortad started is running here."];
|
|
94
94
|
const lines = [
|
|
95
95
|
f.started
|
|
96
|
-
? `
|
|
97
|
-
: `
|
|
98
|
-
f.copies === "made"
|
|
99
|
-
? "For the person: Where Cortad could copy a database, your app runs on the copy, made from your own at every connect and deleted when this command ends, so records created in a copy are gone at the next connect; keep records every run needs in your own database. Each Data line in status says what happens to one store."
|
|
100
|
-
: "For the person: No copy of a database was made for this session, so what test conversations create stays where your app keeps its data; each Data line in status names one store.",
|
|
96
|
+
? `Cortad started your app in ${f.dir === "." ? "this folder" : f.dir} and stops it when this command stops. Your code and settings files are never edited.`
|
|
97
|
+
: `Cortad uses your app already running on port ${f.port} and leaves it running. Your code and settings files are never edited.`,
|
|
101
98
|
];
|
|
99
|
+
if (f.copies === "made") lines.push("Test conversations save into a copy of your database, made fresh at each connect and deleted when this command stops. Your own data is not changed.");
|
|
102
100
|
// Env values reach only a process the command started itself.
|
|
103
|
-
if (f.started) lines.push(
|
|
104
|
-
lines.push(f.hooked ? "
|
|
105
|
-
: f.proxy ? `A local proxy on port ${f.proxy} sees each request sent to it and the model calls your app makes through it
|
|
106
|
-
: "
|
|
101
|
+
if (f.started) lines.push(`Your env values stay on this machine${f.changed?.length ? `, with ${listed(f.changed)} set for this session only` : ""}. None is sent to Cortad.`);
|
|
102
|
+
lines.push(f.hooked ? "Cortad sees each request that reaches your model and the model calls it makes. Request headers stay on this machine and are deleted when the command stops."
|
|
103
|
+
: f.proxy ? `A local proxy on port ${f.proxy} sees each request sent to it and the model calls your app makes through it. Request headers stay on this machine and are deleted when the command stops.`
|
|
104
|
+
: "Cortad did not start your app, so it does not see the calls your app makes to its model.");
|
|
107
105
|
// Only the hook holds a write or records a file: through the proxy, or with nothing loaded, both go as the app does them.
|
|
108
|
-
lines.push(f.hooked ? "During a run,
|
|
109
|
-
: "During a run,
|
|
110
|
-
lines.push(f.hooked ? "Files your app writes in its own folder during a run are put back when
|
|
111
|
-
lines.push("Kept
|
|
106
|
+
lines.push(f.hooked ? "During a run, anything your app would send to an outside service, such as a payment or an email, is answered inside your app and never sent, unless it carries a test key, goes to a provider's sandbox, or goes to a host you allowed on the card. A call that only reads goes out."
|
|
107
|
+
: "During a run, what your app sends to an outside service, such as a payment or an email, is sent for real.");
|
|
108
|
+
lines.push(f.hooked ? "Files your app writes in its own folder during a run are put back when the run ends. Its log files are left as they are." : "Files your app writes in its own folder stay as it writes them.");
|
|
109
|
+
lines.push("Kept in ~/.cortad on this machine between connects: this repository's key, the key the test accounts' passwords are made from, what to put back after a run, and your app's output from the last session with your env values masked.");
|
|
112
110
|
return lines;
|
|
113
111
|
}
|
|
112
|
+
// What the terminal says once the app answers: the one line the person needs.
|
|
113
|
+
export const connectedText = (f) => machineText(f).slice(0, 1);
|
|
114
114
|
|
|
115
115
|
export function showText(d, show, page = 1) {
|
|
116
116
|
if (!d.read || (d.read.complete === false && show !== "reviews")) return "Cortad is still reading this repository.";
|
package/lib/stores.mjs
CHANGED
|
@@ -232,7 +232,7 @@ const HELD = {
|
|
|
232
232
|
"neon-key": (s) => ["there is no NEON_API_KEY in its settings to make a branch with", `Set NEON_API_KEY, or point ${s.names[0]} at a database on this machine`],
|
|
233
233
|
neon: (s) => ["Neon did not make a branch of it", `Check NEON_API_KEY, or point ${s.names[0]} at a database on this machine`],
|
|
234
234
|
supabase: (s) => ["a local Supabase needs supabase/config.toml, Docker and the supabase command", `Add them, or point ${s.names[0]} at a local Supabase`],
|
|
235
|
-
engine: (s) => [`
|
|
235
|
+
engine: (s) => [`no copy of a hosted ${ENGINE[s.engine]} is made`, `Point ${s.names[0]} at a ${ENGINE[s.engine]} on this machine`],
|
|
236
236
|
parts: (s) => ["its settings name it in parts, which cannot be pointed at a copy", `Put its whole address in one setting, or point ${s.names[0]} at a database on this machine`],
|
|
237
237
|
"no-migrations": (s) => ["no migrations were found to build a copy on this machine from", `Point ${s.names[0]} at a database on this machine`],
|
|
238
238
|
"no-local": (s) => ["no Postgres on this machine let us make a database to build a copy in", "Start Postgres on this machine"],
|
|
@@ -247,7 +247,7 @@ const HELD = {
|
|
|
247
247
|
// card told the owner, on the card, that "the owner can say yes to it on the card in the browser".
|
|
248
248
|
const heldWhy = (s, why, tool) => {
|
|
249
249
|
const [reason, fix] = (HELD[why] ?? HELD.other)(s, tool);
|
|
250
|
-
return `${listed(s.names)} ${s.names.length === 1 ? "points" : "point"} at ${s.engine === "redis" || s.db ? "the" : "a"} ${whatOf(s)}
|
|
250
|
+
return `${listed(s.names)} ${s.names.length === 1 ? "points" : "point"} at ${s.engine === "redis" || s.db ? "the" : "a"} ${whatOf(s)} that is not on this machine, and ${reason}, so test conversations would save into it. ${fix}, then run the connect command again.`;
|
|
251
251
|
};
|
|
252
252
|
// A Qdrant server the app writes into is kept by the hook in the app instead of by a copy made here:
|
|
253
253
|
// the first write a run makes into a collection copies that collection on the same server and the
|
package/lib/text.mjs
CHANGED
|
@@ -4,8 +4,8 @@ import { failedLines, heldLines } from "./failed-text.mjs";
|
|
|
4
4
|
import { readLines } from "./read-text.mjs";
|
|
5
5
|
import { at, clip, has, num, PAGE_CHARS, pageOf, plural, SIDE, upper } from "./words.mjs";
|
|
6
6
|
|
|
7
|
-
// What each verb prints. A result is data for the coding agent
|
|
8
|
-
//
|
|
7
|
+
// What each verb prints. A result is data for the coding agent, in plain sentences it can hand the
|
|
8
|
+
// person as they are, and a run_status result ends with the next call. A field the API leaves
|
|
9
9
|
// out prints nothing, so the text follows the API as it grows.
|
|
10
10
|
|
|
11
11
|
const DONE = new Set(["succeeded", "failed", "canceled"]);
|
|
@@ -39,8 +39,7 @@ export function statusText(d) {
|
|
|
39
39
|
lines.push(...contactLines(d.contact, d.app?.data, receipts));
|
|
40
40
|
// While the read runs its lists are zeros ("Journeys (0)", "Questions: 0; 0 asked..."): it is said
|
|
41
41
|
// once, and the lists come when it lands (the server's news says so).
|
|
42
|
-
|
|
43
|
-
if (d.read?.complete === false) { if (!d.contact?.ready?.reading || !d.contact.ready.trials) lines.push("Your code is still being read."); }
|
|
42
|
+
if (d.read?.complete === false) lines.push("Your code is still being read.");
|
|
44
43
|
else if (d.read) lines.push(...readLines(d.read));
|
|
45
44
|
if (d.run) lines.push(...runLines(d.run, "latest "));
|
|
46
45
|
// Trials the read wrote after this run chose its own: the one count that is not the run's.
|
|
@@ -49,7 +48,7 @@ export function statusText(d) {
|
|
|
49
48
|
// Production comes up after the first run (skill: The second run and production).
|
|
50
49
|
if (d.field && d.run) lines.push(`Production: ${d.field.connected ? "connected" : "not connected"}.`);
|
|
51
50
|
if (d.next) lines.push(d.next);
|
|
52
|
-
if (d.links?.lab && (d.read || d.run)) lines.push(`
|
|
51
|
+
if (d.links?.lab && (d.read || d.run)) lines.push(`The person sees this at ${d.links.lab}`);
|
|
53
52
|
if (d.run && !finished(d.run)) lines.push(`next: run_status ${d.run.jobId}`);
|
|
54
53
|
return lines.join("\n");
|
|
55
54
|
}
|
|
@@ -61,8 +60,8 @@ export const unseenText = (door) => `A request to ${door} was answered in words
|
|
|
61
60
|
// to it, or there is no AI behind it. `followed`: a model call came after it that carried none of its
|
|
62
61
|
// words.
|
|
63
62
|
export const receiptText = (door, followed) => (followed
|
|
64
|
-
? `${door} answered your request at once, with no model call inside it; a model call came after it, but its prompt carried none of the words your request sent, so Cortad cannot tell it was this request's answer. That gap is on Cortad's side, and a run leaves this endpoint out
|
|
65
|
-
: `${door} answered your request at once with no model call inside it, and no model call followed in any process Cortad started. If a worker in another process writes its answer, Cortad does not see that worker, so a run leaves this endpoint out for now; that gap is on Cortad's side. If this endpoint has no AI behind it, nothing is missing
|
|
63
|
+
? `${door} answered your request at once, with no model call inside it; a model call came after it, but its prompt carried none of the words your request sent, so Cortad cannot tell it was this request's answer. That gap is on Cortad's side, and a run leaves this endpoint out.`
|
|
64
|
+
: `${door} answered your request at once with no model call inside it, and no model call followed in any process Cortad started. If a worker in another process writes its answer, Cortad does not see that worker, so a run leaves this endpoint out for now; that gap is on Cortad's side. If this endpoint has no AI behind it, nothing is missing.`);
|
|
66
65
|
// Only a door the read lists as one no request has reached, and no request has proven since: a sign-up
|
|
67
66
|
// or a guard's reply to a request a later one proved is no worker's answer.
|
|
68
67
|
const receiptsSaid = (receipts = [], c) => {
|
|
@@ -185,7 +184,9 @@ function runLines(d, prefix = "") {
|
|
|
185
184
|
for (const f of d.faults ?? []) lines.push(`Fault${SIDE[f.side] ? ` ${SIDE[f.side]}` : ""}: ${f.what}${f.fix ? ` ${f.fix}` : ""}`);
|
|
186
185
|
for (const s of d.data ?? []) lines.push(`Data: ${s}`);
|
|
187
186
|
// A run the stop line already explains is not said twice as an error.
|
|
188
|
-
|
|
187
|
+
// A failed run with no stop line of its own was stopped by Cortad, never by their app: said as that, with
|
|
188
|
+
// our error after it for a note to the team. "Error: timeout exceeded when trying to connect" read as the app's.
|
|
189
|
+
if (d.error && !(d.stopped && d.error.startsWith(d.stopped.why))) lines.push(d.status === "failed" && !d.stopped ? `The run stopped on Cortad's side, not your app's. Start another run. (Cortad's error: ${d.error})` : `Error: ${d.error}`);
|
|
189
190
|
if (d.verify) lines.push(...verifyLines(d.verify));
|
|
190
191
|
return lines;
|
|
191
192
|
}
|
|
@@ -247,10 +248,10 @@ export function nextCall(d) {
|
|
|
247
248
|
return d.kind === "verify" || d.findings > 0 ? "findings" : "status";
|
|
248
249
|
}
|
|
249
250
|
|
|
250
|
-
// One line
|
|
251
|
+
// One line to hand the person while a run plays: conversations done of the total, whose side any failed
|
|
251
252
|
// reply is on, and the minutes left at the pace measured so far. Cost is measured only at the end.
|
|
252
253
|
export function progressLine(d, now = Date.now()) {
|
|
253
|
-
if (!d.of) return "
|
|
254
|
+
if (!d.of) return "The run has started. No conversation has played yet.";
|
|
254
255
|
const parts = [`${num(d.played ?? 0)} of ${plural(d.of, "conversation")} done.`];
|
|
255
256
|
const total = d.replies?.total ?? d.waited?.of;
|
|
256
257
|
const crashed = d.replies?.crashed ?? 0;
|
|
@@ -262,7 +263,7 @@ export function progressLine(d, now = Date.now()) {
|
|
|
262
263
|
if (total > 0 && ours) parts.push(`${num(ours)} of ${plural(total, "reply", "replies")} so far failed on Cortad's side, not your app's.`);
|
|
263
264
|
const left = Date.parse(d.pace?.endsAt ?? "") - now;
|
|
264
265
|
if (left > 0) parts.push(`About ${plural(Math.ceil(left / 60_000), "minute")} left at the pace your app has answered so far.`);
|
|
265
|
-
return
|
|
266
|
+
return parts.join(" ");
|
|
266
267
|
}
|
|
267
268
|
|
|
268
269
|
export function runText(d) {
|
|
@@ -273,7 +274,7 @@ export function runText(d) {
|
|
|
273
274
|
if (plan) lines.push(plan);
|
|
274
275
|
const offer = productionOffer(d);
|
|
275
276
|
if (offer) lines.push(offer);
|
|
276
|
-
if (finished(d) && d.url) lines.push(`
|
|
277
|
+
if (finished(d) && d.url) lines.push(`The report is at ${d.url}`);
|
|
277
278
|
lines.push(`next: ${nextCall(d)}`);
|
|
278
279
|
return verifyOnceAPage(lines.join("\n"));
|
|
279
280
|
}
|
|
@@ -283,7 +284,7 @@ export function runText(d) {
|
|
|
283
284
|
function productionOffer(d) {
|
|
284
285
|
const v = d.verify;
|
|
285
286
|
if (d.kind !== "verify" || !finished(d) || v?.verdict !== "gone" || !["gone", "less often"].includes(v.heldOut?.moved) || d.field?.connected !== false) return null;
|
|
286
|
-
return `
|
|
287
|
+
return `The failure in ${v.findingId} is gone on its replays. Connecting production shows what real users are doing as it happens; field_connect has the steps.`;
|
|
287
288
|
}
|
|
288
289
|
|
|
289
290
|
export function startedText(d, kind, findingId, said = []) {
|
|
@@ -311,7 +312,7 @@ export function waitingText(p, now, limitMs, runner, app) {
|
|
|
311
312
|
const kind = p.kind === "verify" ? "verify" : "run";
|
|
312
313
|
const waited = `${num((now - Date.parse(p.startedAt)) / 1000)} of up to ${num(limitMs / 1000)} seconds`;
|
|
313
314
|
const head = runner?.state === "up" ? `${answeringText(runner, app)}\nThe ${kind} has waited ${waited}.` : `Your app is still starting for the ${kind}: ${waited}.`;
|
|
314
|
-
return `${head}\
|
|
315
|
+
return `${head}\nnext: run_status pending`;
|
|
315
316
|
}
|
|
316
317
|
|
|
317
318
|
// A spent plan: what the last run found, what was fixed since, what the next run would play.
|
|
@@ -322,7 +323,7 @@ export function refusedText(d) {
|
|
|
322
323
|
if (d.ledger) lines.push(...ledgerLines(d.ledger));
|
|
323
324
|
if (!d.ledger?.plan && d.plans?.length) lines.push(`Plans: ${d.plans.map((p) => `${p.name} $${p.monthlyUsd} a month`).join(", ")}.`);
|
|
324
325
|
lines.push("Nothing ran.");
|
|
325
|
-
if (d.checkout) lines.push(`
|
|
326
|
+
if (d.checkout) lines.push(`Plans and checkout are at ${d.checkout}`);
|
|
326
327
|
return lines.join("\n");
|
|
327
328
|
}
|
|
328
329
|
|
|
@@ -536,7 +537,7 @@ function alsoLines(f) {
|
|
|
536
537
|
export function fieldText(d) {
|
|
537
538
|
const t = d.totals;
|
|
538
539
|
const pct = (k, n) => (n ? `${Math.round((100 * k) / n)}%` : "n/a");
|
|
539
|
-
const link = d.url ? [`
|
|
540
|
+
const link = d.url ? [`The person sees this at ${d.url}`] : [];
|
|
540
541
|
if (!t || !t.n) return [`No production conversations read in the last ${plural(d.days, "day")}.`, ...link].join("\n");
|
|
541
542
|
const lines = [
|
|
542
543
|
`Production, last ${plural(d.days, "day")}: ${plural(t.n, "conversation")}, ${num(t.read)} read.`,
|
package/lib/verbs.mjs
CHANGED
|
@@ -16,7 +16,7 @@ import { fieldConnectText, fieldText, findingsText, finished, linesOf, notStarte
|
|
|
16
16
|
// the run before them has its numbers in and its job is still ending: the server holds them until it
|
|
17
17
|
// has, up to the same 45 seconds.
|
|
18
18
|
|
|
19
|
-
const NOT_CONNECTED = "This folder is not connected to Cortad. Run npx cortad in this folder as a background command: it prints a link for the person to approve
|
|
19
|
+
const NOT_CONNECTED = "This folder is not connected to Cortad. Run npx cortad in this folder as a background command: it prints a link for the person to approve in a browser where they are signed in to Cortad.";
|
|
20
20
|
const HOLD_S = 45;
|
|
21
21
|
// `wait`: how often it asks whether a run has started, how long the connect command may be gone
|
|
22
22
|
// before no run can start, and how long it waits for one at all.
|
package/local.mjs
CHANGED
|
@@ -28,7 +28,7 @@ import { changesOf, commandText, RELOADER, sourceOf } from "./lib/fresh.mjs";
|
|
|
28
28
|
import { chainOf, elapsedMs, holderOf, listening, listenerOn, portInError, spawnTied, stopTree, supervisorOf } from "./lib/proc.mjs";
|
|
29
29
|
import { claimRunner, readRunner, releaseRunner, replaceRunner, startedAtOf, writeRunner } from "./lib/runner.mjs";
|
|
30
30
|
import { registerAll } from "./lib/register.mjs";
|
|
31
|
-
import {
|
|
31
|
+
import { connectedText } from "./lib/read-text.mjs";
|
|
32
32
|
import { VERSION } from "./lib/spec.mjs";
|
|
33
33
|
import { maskedLines, maskWith, secretValues } from "./lib/env-secrets.mjs";
|
|
34
34
|
import { finished, unseenText } from "./lib/text.mjs";
|
|
@@ -598,7 +598,9 @@ async function verb(job) {
|
|
|
598
598
|
} catch { return { text: "", root, size: 0 }; }
|
|
599
599
|
}
|
|
600
600
|
// The person changed what they said yes to while this command runs.
|
|
601
|
-
|
|
601
|
+
// Said in the terminal: an agent that read "one question for the person" here told them the run
|
|
602
|
+
// was still waiting two minutes after they had answered.
|
|
603
|
+
case "consent": await allowOutbound(b); say("the person's answer on the card in the browser arrived"); return { ok: true };
|
|
602
604
|
// A line for the agent reading this terminal: the read of the code landing, said the moment it does.
|
|
603
605
|
// One marked news (a run ending) is also kept for the hook, so an agent that was not listening
|
|
604
606
|
// hears it on its next prompt or edit.
|
|
@@ -1198,7 +1200,7 @@ if (explain) {
|
|
|
1198
1200
|
].join("\n"));
|
|
1199
1201
|
process.exit(0);
|
|
1200
1202
|
}
|
|
1201
|
-
if (kept.length) say(`
|
|
1203
|
+
if (kept.length) say(`left on this machine because it holds a key: ${kept.join(", ")}`);
|
|
1202
1204
|
const keepSecret = (v) => { if (v && v.length >= 12 && !secrets.includes(v)) secrets.push(v); };
|
|
1203
1205
|
identities = makeIdentities({ root, work, envFiles, sourceFiles: () => files, say, keepSecret, appDir: () => appDir });
|
|
1204
1206
|
// A real request that reached their model proves its door (lib/proof.mjs). What it proved goes up,
|
|
@@ -1297,7 +1299,7 @@ stepDone(moved ? "connected: your change is in, and what Cortad read before stil
|
|
|
1297
1299
|
if (!viaToken && process.env.CORTAD_NO_REGISTER !== "1") {
|
|
1298
1300
|
const got = await registerAll().catch((err) => { if (verbose) say(`could not register with your coding agents: ${err?.message ?? err}`); return { clients: [], changed: [] }; });
|
|
1299
1301
|
const and = (list) => new Intl.ListFormat("en", { type: "conjunction" }).format(list);
|
|
1300
|
-
if (got.changed.length) say(
|
|
1302
|
+
if (got.changed.length) say(`${and(got.clients)} can now use Cortad: its tools and skill were added to ${got.changed.map((f) => `~/${f}`).join(", ")}. This repository was not changed.`);
|
|
1301
1303
|
}
|
|
1302
1304
|
|
|
1303
1305
|
lock = await makeLock({ root, work });
|
|
@@ -1389,7 +1391,7 @@ async function appLife(firstStart) {
|
|
|
1389
1391
|
say(`your app is answering on port ${app.port}${app.cmd ? ` · ${app.cmd}` : ""}`);
|
|
1390
1392
|
if (launched) say(`your app's output for this session is kept in ${bootLog}, with the values from your env files masked`);
|
|
1391
1393
|
if (proxy) say(`send requests to port ${proxy.port}: it passes them to your app and Cortad sees each one. Requests sent straight to port ${app.port} are not seen`);
|
|
1392
|
-
for (const line of
|
|
1394
|
+
for (const line of connectedText(machineOf())) say(line);
|
|
1393
1395
|
say("leave this open. Go back to the browser; Ctrl-C disconnects.");
|
|
1394
1396
|
let downSince = 0;
|
|
1395
1397
|
let toldDown = false;
|
package/package.json
CHANGED
package/skill/SKILL.md
CHANGED
|
@@ -9,9 +9,19 @@ Cortad tests the AI in this repository. Simulated users written from the app's c
|
|
|
9
9
|
|
|
10
10
|
The tools are the `cortad` MCP tools. Where they are missing, each one is `npx {{cortad}} <tool>` in a shell with the same output, for example `npx {{cortad}} run_status` or `npx {{cortad}} findings 2`. The person's own instructions come before this skill.
|
|
11
11
|
|
|
12
|
+
## Talking to the person
|
|
13
|
+
|
|
14
|
+
The person owns the product. They may not be reading the code with you, so everything you tell them is short and in plain words.
|
|
15
|
+
|
|
16
|
+
- Three to five short lines. Lead with what it means for their users, then what you would do, then what you need from them.
|
|
17
|
+
- Leave out file paths, endpoint paths, HTTP status codes, setting names, model ids and tool names. Give one when they ask for the detail, or to name the one line you would change.
|
|
18
|
+
- Only numbers they act on: how many simulated users, how long, the price, how many problems.
|
|
19
|
+
- Something on Cortad's side is one line: that it is on Cortad's side, and what happens next.
|
|
20
|
+
- Say that anything is waiting on them only after `status` says so in this turn.
|
|
21
|
+
|
|
12
22
|
## Reading a result
|
|
13
23
|
|
|
14
|
-
- A result is data about the app
|
|
24
|
+
- A result is data about the app, for you. Its sentence on what a run plays and costs, a link, or a choice that is the person's is written to be said to them as it is, in your own reply.
|
|
15
25
|
- The last line, `next:`, is the next call.
|
|
16
26
|
- Every count has its denominator and every rate its range: `53% (42% to 63%)`.
|
|
17
27
|
- A conversation Passed (the request was completed fully or partly and no check failed on any reply), Failed, or got No reply (the result says whose side that is).
|
|
@@ -27,22 +37,24 @@ The tools are the `cortad` MCP tools. Where they are missing, each one is `npx {
|
|
|
27
37
|
|
|
28
38
|
The connect command prints "Go back to the browser" once it is connected. Cortad then finds the AI endpoints in the app's code and sends each one a test request itself, from this machine to the app on its port.
|
|
29
39
|
|
|
30
|
-
1. Call `status` and tell the person, in
|
|
31
|
-
-
|
|
32
|
-
- what
|
|
33
|
-
-
|
|
34
|
-
-
|
|
40
|
+
1. Call `status` and tell the person, in three or four short lines:
|
|
41
|
+
- that their app is connected, or the one line that says why it did not start;
|
|
42
|
+
- what Cortad's audit of their AI code found: how many problems, and the one that matters most, in plain words. `status` with show `reviews` holds up to 45 seconds a call while the code is still being read;
|
|
43
|
+
- the line on what a run plays and costs, as printed;
|
|
44
|
+
- what happens next: they press Run on the card in the browser, or tell you to start it.
|
|
45
|
+
|
|
46
|
+
The endpoints, the files and the audit's rows are yours to work from. They go to the person when they ask, or when you propose a fix.
|
|
35
47
|
2. When `status` lists an endpoint as `Needs a signed-in user`, Cortad could not make a sign-in for it on this machine. Tell the person; when they ask for it, sign in a test account through the app's own sign-in on this machine, send that endpoint one request in that session, then call `status` again. That is the only sign-in this needs, and only for the endpoints listed that way.
|
|
36
48
|
3. A failure on the app's side that every conversation would repeat, such as a model provider refusing, a retrieval with no answer or a tool returning an error: say what you would change, change it when the person agrees, then call `reach` to send Cortad's test request to that endpoint again.
|
|
37
|
-
4. The first run starts when the person presses Run on the card, or asks you, and then you call `run`. Before you stop, start `npx -y {{cortad}} wait` as a background command (in Claude Code: Bash with run_in_background). It returns when the next run or verify ends, with what it found, and its ending wakes you. Then follow "After a run".
|
|
49
|
+
4. A question for the person on the card in the browser is said to them once, in one line. The first run starts when the person presses Run on the card, or asks you, and then you call `run`. Before you stop, start `npx -y {{cortad}} wait` as a background command (in Claude Code: Bash with run_in_background). It returns when the next run or verify ends, with what it found, and its ending wakes you. Then follow "After a run".
|
|
38
50
|
|
|
39
|
-
`status` with `show` lists one section in full: `rules`, `standards`, `journeys`, `endpoints`, `trials`, `records` or `reviews` (in a shell, `npx {{cortad}} status rules`). `status` with show `machine` says what Cortad does on this machine; hand the person its
|
|
51
|
+
`status` with `show` lists one section in full: `rules`, `standards`, `journeys`, `endpoints`, `trials`, `records` or `reviews` (in a shell, `npx {{cortad}} status rules`). `status` with show `machine` says what Cortad does on this machine; hand the person its lines when they ask.
|
|
40
52
|
|
|
41
53
|
## After a run
|
|
42
54
|
|
|
43
55
|
The run is Cortad's; the code is the person's. You report first, and you change nothing until they say so.
|
|
44
56
|
|
|
45
|
-
1. Call `findings`. Tell the person, in plain lines: how many
|
|
57
|
+
1. Call `findings`. Tell the person, in plain lines: how many simulated users got what they asked for, of how many; then the problems worth their time, worst first and three at most, each as what their users ran into, how often, and what you would change, in one line; which findings read their app wrong (see "A check that reads wrong"); and which problems need a decision only they can make. Name a file only for the change you propose.
|
|
46
58
|
2. Then stop, and let them answer. Until their go, nothing is edited, no verify starts and no run starts. They choose which problems to fix, in what order, and whether the change you propose fits their product.
|
|
47
59
|
3. On their go, one problem at a time: one change, in the file and near the line the finding names, then `verify <findingId>`. Its first line says how many conversations the round replays and what that counts against this month; start `npx -y {{cortad}} wait` in the background, and it wakes you when the verify ends, so `run_status` is called once at most.
|
|
48
60
|
4. Tell them what the verify decided, in its own words; the next change comes after their next go. A verify that cannot tell offers another round in its last line; it plays only when you call `verify` again, and only when the person wants it.
|
|
@@ -50,7 +62,7 @@ The run is Cortad's; the code is the person's. You report first, and you change
|
|
|
50
62
|
|
|
51
63
|
## Following a run
|
|
52
64
|
|
|
53
|
-
`run` answers within a second; when the app is still starting it says so, and `run_status` holds until the run has an id. `run_status` follows the run this machine started last, or the latest run. Each call holds up to 45 seconds, returns as soon as the count moves, and ends with the next call; `npx -y {{cortad}} wait` in the background does the following for you and wakes you at the end of a run or a verify.
|
|
65
|
+
`run` answers within a second; when the app is still starting it says so, and `run_status` holds until the run has an id. `run_status` follows the run this machine started last, or the latest run. Each call holds up to 45 seconds, returns as soon as the count moves, and ends with the next call; `npx -y {{cortad}} wait` in the background does the following for you and wakes you at the end of a run or a verify. Tell the person how far the run is at most once every two minutes. When the plan is spent, `run` says so and nothing runs.
|
|
54
66
|
|
|
55
67
|
## The first run: the baseline
|
|
56
68
|
|
|
@@ -82,7 +94,7 @@ The conversations, checks and saved conversations belong to Cortad, and the app
|
|
|
82
94
|
|
|
83
95
|
After the fixes, `run` again. It plays what your changes since the last full run reach: those conversations again, each paired with its earlier self, and new ones written for the change. Its text leads with what moved, worse first.
|
|
84
96
|
|
|
85
|
-
Once a fix is verified gone and production is not connected,
|
|
97
|
+
Once a fix is verified gone and production is not connected, the result offers production: connect it to see what real users are doing as it happens and where the AI lets them down. `field_connect` gives the steps; the owner creates the key in the browser. `field` gives the numbers; message text stays out.
|
|
86
98
|
|
|
87
99
|
## Making it stick
|
|
88
100
|
|
|
@@ -35,15 +35,15 @@ Endpoints found in your code that have not answered yet (2):
|
|
|
35
35
|
POST /api/homework/explain apps/api/src/routes/homework.ts:18 Cortad's test request was answered with HTTP 500.
|
|
36
36
|
POST /api/progress/summary apps/api/src/routes/progress.ts:9 Needs a signed-in user.
|
|
37
37
|
Cortad sends each of them a test request.
|
|
38
|
-
|
|
38
|
+
A run plays 36 simulated users in about 6 minutes. About $0.38 on your OpenAI key.
|
|
39
39
|
A run starts when the person presses Run on the card in the browser or asks you for one; if they say they pressed it, call run rather than ask them again.
|
|
40
40
|
```
|
|
41
41
|
|
|
42
|
-
When the app keeps its data in a store
|
|
42
|
+
When the app keeps its data in a hosted store no copy was made of, the decision is the person's, said once as the last line before the link:
|
|
43
43
|
|
|
44
44
|
```
|
|
45
|
-
|
|
46
|
-
|
|
45
|
+
DATABASE_URL points at the Postgres database shop that is not on this machine, and no migrations were found to build a copy on this machine from, so test conversations would save into it. Point DATABASE_URL at a database on this machine, then run the connect command again. Or the person allows it on the card in the browser: one press lets test conversations save into it.
|
|
46
|
+
The person sees this at https://cortad.com/lab
|
|
47
47
|
```
|
|
48
48
|
|
|
49
49
|
Once a run is playing, what reading your code found and the run:
|
|
@@ -67,7 +67,7 @@ Conversations set aside:
|
|
|
67
67
|
POST /api/homework/upload: 7 conversations, on the app's side: the route needs a signed file URL, and no test account can make one
|
|
68
68
|
Latest run 8f2a1c4e-5b6d-4e7f-9a0b-1c2d3e4f5a6b: running, 14 of 51 conversations played.
|
|
69
69
|
Production: not connected.
|
|
70
|
-
|
|
70
|
+
The person sees this at https://cortad.com/lab
|
|
71
71
|
next: run_status 8f2a1c4e-5b6d-4e7f-9a0b-1c2d3e4f5a6b
|
|
72
72
|
```
|
|
73
73
|
|
|
@@ -131,7 +131,7 @@ The next run would play 58 conversations, 7 saved to test fixes, 5 new from the
|
|
|
131
131
|
The Pro plan, $99 a month, includes 10 runs.
|
|
132
132
|
Production: not connected.
|
|
133
133
|
Nothing ran.
|
|
134
|
-
|
|
134
|
+
Plans and checkout are at https://cortad.com/pricing?checkout=ship
|
|
135
135
|
```
|
|
136
136
|
|
|
137
137
|
## run_status
|
|
@@ -140,7 +140,7 @@ Playing. The call waited up to 45 seconds:
|
|
|
140
140
|
|
|
141
141
|
```
|
|
142
142
|
Run 8f2a1c4e-5b6d-4e7f-9a0b-1c2d3e4f5a6b: running, 14 of 51 conversations played.
|
|
143
|
-
|
|
143
|
+
14 of 51 conversations done.
|
|
144
144
|
next: run_status 8f2a1c4e-5b6d-4e7f-9a0b-1c2d3e4f5a6b
|
|
145
145
|
```
|
|
146
146
|
|
|
@@ -208,7 +208,7 @@ Connect production to see what your real users are doing as it happens and where
|
|
|
208
208
|
"Answer in the language the student writes in." (apps/api/src/agent/system.ts:12): failed in 2 of 26 conversations.
|
|
209
209
|
112 of 140 checks measured.
|
|
210
210
|
Free: this month's 1 run is used; 60 verify conversations left. Pro $99 a month: runs and reruns included, 100,000 production replies read.
|
|
211
|
-
|
|
211
|
+
The report is at https://cortad.com/lab
|
|
212
212
|
next: findings
|
|
213
213
|
```
|
|
214
214
|
|
|
@@ -219,7 +219,7 @@ A run after the first one plays what the files changed since the last run that p
|
|
|
219
219
|
```
|
|
220
220
|
Run 9a10b3d2-7c8d-4e9f-8a1b-2c3d4e5f6a7b: running, 9 of 34 conversations played.
|
|
221
221
|
Since run 8f2a1c4e, 1 file changed (apps/api/src/agent/prompt.ts), touching the prompt in apps/api/src/agent/prompt.ts. They reach POST /api/chat. This run replays the 22 conversations there from that run, so each pairs with its earlier self, and adds 12 new conversations written for the change on billing question, order tracking (5 of them open the way your production customers did). Left out: 7 conversations on POST /api/homework/upload, which nothing that changed reaches.
|
|
222
|
-
|
|
222
|
+
9 of 34 conversations done.
|
|
223
223
|
next: run_status 9a10b3d2-7c8d-4e9f-8a1b-2c3d4e5f6a7b
|
|
224
224
|
```
|
|
225
225
|
|
|
@@ -242,7 +242,7 @@ Replies took 2.9 s at the median and 9.8 s at the slowest, over 81 replies.
|
|
|
242
242
|
Nothing broke that only running your app could show.
|
|
243
243
|
The next run is compared with run 8f2a1c4e, the last run that played every conversation, with everything that changed since then in its scope.
|
|
244
244
|
Pro: 8 of 10 runs left this month; 560 verify conversations left.
|
|
245
|
-
|
|
245
|
+
The report is at https://cortad.com/lab
|
|
246
246
|
next: status
|
|
247
247
|
```
|
|
248
248
|
|
|
@@ -253,7 +253,7 @@ Run 8f2a1c4e-5b6d-4e7f-9a0b-1c2d3e4f5a6b: finished, 11 of 51 conversations playe
|
|
|
253
253
|
1 finding.
|
|
254
254
|
Stopped at 11 of 51 conversations, on the app's side: your app stopped answering after 11 replies. Bring your app back up, then run again.
|
|
255
255
|
Fault on the app's side: Your code names llama-3.1-8b-instant, which api.groq.com does not serve. Rename the model in your code, then run again.
|
|
256
|
-
|
|
256
|
+
The report is at https://cortad.com/lab
|
|
257
257
|
next: findings
|
|
258
258
|
```
|
|
259
259
|
|
|
@@ -281,8 +281,8 @@ Verify 7c31e0aa-1b2c-4d3e-8f4a-5b6c7d8e9f0a: finished, 5 of 5 conversations play
|
|
|
281
281
|
Verify of finding:1 at apps/api/src/agent/prompt.ts:41.
|
|
282
282
|
The failure is gone on its own conversations: none of 3 replays failed, against 3 of 3 conversations in the run it was found in. On the conversations saved to test the fix the failure is gone: 3 of 8 failed before the fix, 0 of 8 after, a fall beyond chance.
|
|
283
283
|
Free: this month's 1 run is used; 55 verify conversations left. Pro $99 a month: runs and reruns included, 100,000 production replies read.
|
|
284
|
-
|
|
285
|
-
|
|
284
|
+
The failure in finding:1 is gone on its replays. Connecting production shows what real users are doing as it happens; field_connect has the steps.
|
|
285
|
+
The report is at https://cortad.com/lab
|
|
286
286
|
next: findings
|
|
287
287
|
```
|
|
288
288
|
|
|
@@ -303,7 +303,7 @@ The failure stayed. The conversations still fail more than the same question doe
|
|
|
303
303
|
Verify 7c31e0aa-1b2c-4d3e-8f4a-5b6c7d8e9f0a: finished, 4 of 4 conversations played.
|
|
304
304
|
Verify of finding:4 at apps/api/src/agent/system.ts:12.
|
|
305
305
|
The failure stayed: 4 of 4 replays failed, against 4 of 4 conversations in the run it was found in; the same question fails on 3 of 18 conversations elsewhere in your app, so these still stand out beyond chance. After the fix 4 conversations still failed: conversation 0b19a3f2 said "Sure! Let's solve this together." at reply 1, as it did before the fix; conversation 3e8d0c47 said "Sure! Let's solve this together." at reply 1, as it did before the fix; conversation a61f9b05 said "Sure! Let's solve this together." at reply 1, as it did before the fix; and 1 more.
|
|
306
|
-
|
|
306
|
+
The report is at https://cortad.com/lab
|
|
307
307
|
next: findings
|
|
308
308
|
```
|
|
309
309
|
|
|
@@ -313,7 +313,7 @@ Where the conversations saved to test the fix disagree with the finding's own, t
|
|
|
313
313
|
Verify 7c31e0aa-1b2c-4d3e-8f4a-5b6c7d8e9f0a: finished, 43 of 43 conversations played.
|
|
314
314
|
Verify of finding:5 at apps/api/src/agent/lessons.ts:58.
|
|
315
315
|
The failure stayed on its own conversations (3 of 35 failed, against 4 of 18 before) and is gone on the conversations saved to test the fix (3 of 8 failed before the fix, 0 of 8 after, beyond chance). On its own conversations that is not a fall beyond chance. After the fix 3 conversations still failed: conversation e4a1c9d2 said "I don't have this information about your homework plan." at reply 2, where before the fix it said "Let me look up your lesson plan."; conversation f0000000 said "I don't have this information." at reply 1; conversation f0000001 said "I don't have this information." at reply 1. The reason changed: in conversation e4a1c9d2 the reply declined to help, where before the fix the conversation ended without what they asked for.
|
|
316
|
-
|
|
316
|
+
The report is at https://cortad.com/lab
|
|
317
317
|
next: findings
|
|
318
318
|
```
|
|
319
319
|
|
|
@@ -323,7 +323,7 @@ Replies refused or failed on the replay come first. A refusal is not an answer,
|
|
|
323
323
|
Verify 7c31e0aa-1b2c-4d3e-8f4a-5b6c7d8e9f0a: finished, 5 of 5 conversations played.
|
|
324
324
|
Verify of finding:2 at backend/tools/refunds.py.
|
|
325
325
|
Your app refused 5 of 5 replays (HTTP 422), so they got no answer: the fix changed what the endpoint accepts. The failure stayed: 5 of 5 replays got no answer, against 5 of 5 conversations in the run it was found in; the same question fails on 2 of 16 conversations elsewhere in your app, so these still stand out beyond chance.
|
|
326
|
-
|
|
326
|
+
The report is at https://cortad.com/lab
|
|
327
327
|
next: findings
|
|
328
328
|
```
|
|
329
329
|
|
|
@@ -434,5 +434,5 @@ Production, last 30 days: 4,812 conversations, 4,790 read.
|
|
|
434
434
|
Rule checks passed: 93% of 61,204 (1,120 too close to call). Resolved 71%, frustrated 6%, asked for a human 2%, unanswered 4%.
|
|
435
435
|
Rules broken most: rule:answer-first (412), rule:language (188), rule:cite-source (97).
|
|
436
436
|
By journey: homework help 3,102 conversations, 94% passed; billing question 410 conversations, 88% passed.
|
|
437
|
-
|
|
437
|
+
The person sees this at https://cortad.com/lab#field
|
|
438
438
|
```
|