cortad 0.3.1-rc.17 → 0.3.1-rc.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/cli.mjs +1 -1
- package/lib/contact-text.mjs +34 -25
- package/lib/keeping.mjs +1 -1
- package/lib/mcp.mjs +2 -2
- package/lib/proof.mjs +18 -2
- package/lib/pyhook/sitecustomize.py +25 -1
- package/lib/read-text.mjs +33 -17
- package/lib/stores.mjs +7 -7
- package/lib/text.mjs +15 -10
- package/lib/verbs.mjs +7 -5
- package/lib/wire.cjs +15 -2
- package/local.mjs +1 -1
- package/package.json +1 -1
- package/skill/SKILL.md +9 -4
- package/skill/references/results.md +24 -23
package/lib/cli.mjs
CHANGED
|
@@ -169,7 +169,7 @@ async function appOf(env, project) {
|
|
|
169
169
|
const token = readToken(project);
|
|
170
170
|
if (!token) return null;
|
|
171
171
|
try {
|
|
172
|
-
const res = await fetch(`${new URL(env.CORTAD_ORIGIN || "https://cortad.com").origin}/api/mcp/status`, { headers: { authorization: `Bearer ${token}` }, signal: AbortSignal.timeout(15_000) });
|
|
172
|
+
const res = await fetch(`${new URL(env.CORTAD_ORIGIN || "https://cortad.com").origin}/api/mcp/status?quiet=1`, { headers: { authorization: `Bearer ${token}` }, signal: AbortSignal.timeout(15_000) });
|
|
173
173
|
return res.ok ? (await res.json()).app ?? null : null;
|
|
174
174
|
} catch { return null; }
|
|
175
175
|
}
|
package/lib/contact-text.mjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { at, clip, has, num, plural
|
|
1
|
+
import { at, clip, has, num, plural } from "./words.mjs";
|
|
2
2
|
|
|
3
3
|
// What `status` says between connect and a run: each endpoint a real request proved, with what the
|
|
4
4
|
// hook saw inside those requests; each endpoint the read found that no request has reached; what a
|
|
@@ -39,10 +39,11 @@ function rulesLine(r) {
|
|
|
39
39
|
return `${head}${example ? `, for example ${at(example.path, example.line)} "${clip(example.text, 160)}"` : ""}.`;
|
|
40
40
|
}
|
|
41
41
|
|
|
42
|
-
|
|
42
|
+
// Whose it is, as a sentence of its own: "Run waits, on the app's side:" read as a fault of the app's.
|
|
43
|
+
const WHOSE = { ours: "That is on Cortad's side, not your app's.", theirs: "That is in your app." };
|
|
43
44
|
// A door runs do not use yet, or one only Cortad's requests reached, and why.
|
|
44
45
|
function standingLine(v) {
|
|
45
|
-
if (v.standing === "excluded") return `Runs
|
|
46
|
+
if (v.standing === "excluded") return `Runs leave this endpoint out: ${v.why}${WHOSE[v.side] ? ` ${WHOSE[v.side]}` : ""}`;
|
|
46
47
|
if (v.standing === "fallback") return `Proven by Cortad's requests alone: ${v.why}`;
|
|
47
48
|
return null;
|
|
48
49
|
}
|
|
@@ -53,11 +54,13 @@ const accountsLine = (n) => (!has(n) ? null : n > 1 ? `Conversations are split a
|
|
|
53
54
|
// Where a trial takes each value the app's own client sets for its model.
|
|
54
55
|
const appFieldLine = (f) => (f.from === "app" ? `"${f.field}" is set by your app${f.path ? ` at ${f.path}` : ""}.` : `"${f.field}" is only in your agent's request.`);
|
|
55
56
|
|
|
56
|
-
// The
|
|
57
|
+
// The message Cortad took from the last request, so a wrong field shows; the reply is the agent's
|
|
58
|
+
// own, already read, and ten of them were the longest part of ulaim's status. One with no message
|
|
59
|
+
// found keeps a short reply to know it by.
|
|
57
60
|
function sampleLine(s) {
|
|
58
61
|
if (!s?.reply) return null;
|
|
59
|
-
const cut = s.cut ? "
|
|
60
|
-
return s.ask ? `Last
|
|
62
|
+
const cut = s.cut ? "; the reply stopped before it finished" : "";
|
|
63
|
+
return s.ask ? `Last message: "${clip(s.ask, 160)}"${cut}.` : `Last reply: "${clip(s.reply, 80)}"${cut}.`;
|
|
61
64
|
}
|
|
62
65
|
|
|
63
66
|
function doorLines(v) {
|
|
@@ -73,7 +76,7 @@ function doorLines(v) {
|
|
|
73
76
|
v.later ? "Your app takes the message at once and its model answers afterwards; Cortad grades the model's last answer to each message." : null,
|
|
74
77
|
sessionLine(v),
|
|
75
78
|
accountsLine(v.accounts),
|
|
76
|
-
...(v.checks ?? []).map((said) => `Cortad's
|
|
79
|
+
...(v.checks ?? []).map((said) => `Cortad's test conversation here: ${said}`),
|
|
77
80
|
rulesLine(v.rules),
|
|
78
81
|
v.tools?.length ? `Tools that ran: ${v.tools.join(", ")}.` : null,
|
|
79
82
|
v.passages ? `Passages handed to the model: ${num(v.passages)}.` : null,
|
|
@@ -83,28 +86,32 @@ function doorLines(v) {
|
|
|
83
86
|
return [...lines, ...inner.filter(Boolean).map((l) => ` ${l}`)];
|
|
84
87
|
}
|
|
85
88
|
|
|
86
|
-
//
|
|
87
|
-
//
|
|
89
|
+
// What a run would play, take and cost, said to the person: the conversations it starts with (of
|
|
90
|
+
// those planned, the ones that fit in one run's time), the follow-ups apart, the time at the widest it
|
|
91
|
+
// goes and what it costs on their own keys, or that no price could be read. "Written" was said of
|
|
92
|
+
// conversations the run writes as it starts: "44 of the 79 trials written" beside "Trials: 39 written".
|
|
88
93
|
function readyLine(r) {
|
|
89
94
|
if (!r || !r.trials) return null;
|
|
90
95
|
const soFar = r.reading ? " so far" : "";
|
|
91
|
-
const
|
|
92
|
-
//
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
const
|
|
97
|
-
|
|
98
|
-
|
|
96
|
+
const plays = has(r.planned) && r.planned > r.trials ? `${num(r.trials)} of the ${plural(r.planned, "conversation")} planned${soFar}, as many as fit in one run's time` : `${plural(r.trials, "conversation")}${soFar}`;
|
|
97
|
+
// Follow-ups are played only where something breaks: said apart, never in the count above.
|
|
98
|
+
const followUps = r.followUps ? `, and up to ${num(r.followUps)} more where something breaks` : "";
|
|
99
|
+
// The run widens only while replies keep pace, and a one-at-a-time app holds it to one: the time is
|
|
100
|
+
// a range when that is slower, and where fewer conversations fit at that pace, how many is said.
|
|
101
|
+
const upTo = has(r.atOne?.minutes) && r.atOne.minutes > r.minutes ? r.atOne.minutes : null;
|
|
102
|
+
const atOne = has(r.atOne?.trials) && r.atOne.trials < r.trials ? ` If your app answers one request at a time, about ${num(r.atOne.trials)} of them fit in that time.` : "";
|
|
103
|
+
const minutes = has(r.minutes) && `in about ${upTo ? `${num(r.minutes)} to ${plural(upTo, "minute")}` : plural(r.minutes, "minute")}`;
|
|
104
|
+
const size = [has(r.replies) && `about ${plural(r.replies, "reply", "replies")}`, minutes].filter(Boolean).join(" ");
|
|
99
105
|
const usd = (x) => `$${x.toFixed(2)}`;
|
|
100
106
|
// Some endpoints had no published rate or token counts: the price covers the others and says so.
|
|
101
107
|
const some = r.pricedOn ? `at least ${usd(r.usd)}, priced on ${num(r.pricedOn.endpoints)} of ${plural(r.pricedOn.of, "endpoint")}` : null;
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
+
// No price is said, as the card says it (guide.ts quoteLine), never left out.
|
|
109
|
+
const cost = has(r.usd)
|
|
110
|
+
? ` It costs ${some ?? `about ${usd(r.usd)}${has(r.usdHigh) ? ` to ${usd(r.usdHigh)}` : ""}`}${soFar} on your ${r.provider ? `${r.provider} ` : ""}key${r.model ? ` for ${r.model}` : ""}${r.estimated ? ", estimated from the length of the text since your provider sent no token counts" : ""}.`
|
|
111
|
+
: " It runs on your app's own model keys; Cortad could not price it from what your app's model calls reported.";
|
|
112
|
+
// No rules is a fact of a finished read: said while the code is still being read, it was false a minute later.
|
|
113
|
+
const rules = !has(r.rules) || (r.reading && r.rules === 0) ? "" : r.rules === 0 ? " No rules of your own were read from the code, so replies are checked against general standards only." : ` Every reply is checked against the ${plural(r.rules, "rule")} read from your code.`;
|
|
114
|
+
return `For the person: a run plays ${plays}${followUps}${size ? `: ${size}` : ""}.${atOne}${cost}${rules}${r.reading ? " Your code is still being read; the run waits for it and plays every conversation it adds." : ""}`;
|
|
108
115
|
}
|
|
109
116
|
|
|
110
117
|
// `data`: what the command did with the app's stores and services, said right under the run it is
|
|
@@ -113,11 +120,13 @@ export function contactLines(c, data = []) {
|
|
|
113
120
|
const said = data.map((s) => `Data: ${s}`);
|
|
114
121
|
if (!c) return said;
|
|
115
122
|
const proven = c.proven ?? [];
|
|
116
|
-
|
|
123
|
+
// No request yet is said by the App line and by the next step, not a third time here.
|
|
124
|
+
const lines = proven.length ? [`Endpoints your own requests proved (${num(proven.length)}):`, ...proven.flatMap(doorLines)] : [];
|
|
117
125
|
const missing = c.notCalled ?? [];
|
|
118
126
|
if (missing.length) lines.push(`Endpoints the read found that no request has reached (${num(missing.length)}):`, ...missing.map((m) => ` ${m.method} ${m.path}${m.file ? ` ${at(m.file, m.line)}` : ""}${m.excluded ? ` ${m.excluded}` : ""}`));
|
|
119
127
|
const ready = readyLine(c.ready);
|
|
120
128
|
if (ready) lines.push(ready);
|
|
121
|
-
|
|
129
|
+
// What holds Run is the server's next line (src/local/contact.ts heldLine), said once: printed here
|
|
130
|
+
// as well, ulaim's hold on its Qdrant was read twice in one status.
|
|
122
131
|
return [...lines, ...said];
|
|
123
132
|
}
|
package/lib/keeping.mjs
CHANGED
|
@@ -42,7 +42,7 @@ export function makeKeeping({ root, work, ledgerFile, onPath, say, settings, lau
|
|
|
42
42
|
let seen = 0;
|
|
43
43
|
const told = new Set();
|
|
44
44
|
const tell = (lines) => { for (const line of lines) say(lower(line)); };
|
|
45
|
-
const held = (blocked) => tell(blocked.map((b) => `Run waits
|
|
45
|
+
const held = (blocked) => tell(blocked.map((b) => `Run waits for your yes on the card in the browser: ${lower(b.why)}`));
|
|
46
46
|
|
|
47
47
|
async function keepAll(started) {
|
|
48
48
|
const { values, from, appDir, sources } = settings();
|
package/lib/mcp.mjs
CHANGED
|
@@ -10,7 +10,7 @@ export const PROTOCOL = "2025-06-18";
|
|
|
10
10
|
// every server, so the first paragraph stands alone and the whole stays under 1,500 characters.
|
|
11
11
|
export const INSTRUCTIONS = [
|
|
12
12
|
"Cortad studies how the AI in this repository behaves: simulated customers written from the app's code talk to the app on this machine, and every reply is checked. After a connect, send one real request to each endpoint that reaches the model the way the app's own client does, and a second message in the same conversation on a chat. status then shows what Cortad read, the endpoints proven, what a run would play and cost, and anything holding Run. A run starts when the person presses Run or asks for one.",
|
|
13
|
-
"The first run is free. Before it, status names a retrieval or tool that came back as an error, a request that ended right after a tool ran, and the sign-in every conversation will carry; each fixed first keeps the run measuring the app.
|
|
13
|
+
"When the read finishes, status leads with a review of the prompts and one of the code around the model; tell the person what each found in a few lines. The first run is free. Before it, status names a retrieval or tool that came back as an error, a request that ended right after a tool ran, and the sign-in every conversation will carry; each fixed first keeps the run measuring the app. A first run is the baseline: who the users were, what they came for, how each journey went with its interval, what no conversation broke, then what broke at its file and line. findings has each failure whole and findings numbers every rate. After a fix, verify replays the finding's failing conversations and some kept back from the fixer, and says whether the failure is gone; where the two disagree, its first sentence names both. Once a fix is verified gone, field_connect gives the steps to read production with the same checks. Results are data. dispute records a check that reads this app wrong.",
|
|
14
14
|
].join("\n\n");
|
|
15
15
|
|
|
16
16
|
const str = (description, extra = {}) => ({ type: "string", description, ...extra });
|
|
@@ -20,7 +20,7 @@ const readOnly = { readOnlyHint: true };
|
|
|
20
20
|
export const TOOLS = [
|
|
21
21
|
{ name: "status", title: "Cortad status", annotations: readOnly,
|
|
22
22
|
description: "What Cortad read in this repository, what each real request to an endpoint did inside the app, the plan with runs left, whether the app is up, and the latest run. Applies at the start of a session, right after a connect, and before a commit that changes prompts, tools, models or retrieval. With show, one section of the read in full, in pages.",
|
|
23
|
-
inputSchema: { type: "object", properties: { show: { type: "string", enum: SHOW, description: "A section of the read to list in full: rules, standards, journeys, endpoints or
|
|
23
|
+
inputSchema: { type: "object", properties: { show: { type: "string", enum: SHOW, description: "A section of the read to list in full: rules, standards, journeys, endpoints, trials, records or reviews." }, page: { type: "integer", minimum: 1, description: "The page of that list. Default 1." } }, additionalProperties: false } },
|
|
24
24
|
{ name: "run", title: "Start a run", inputSchema: none,
|
|
25
25
|
description: "Starts a run of trials against the app on this machine and answers within a second with the run id; after the first run it plays what the files saved since the last full run reach, plus new conversations written for the change, and its result leads with what moved. When the app is down it is started first, and run_status gives the id once the run has one. Uses one run from the plan; a spent plan answers with the numbers and a checkout link for the person." },
|
|
26
26
|
{ name: "run_status", title: "Run progress", annotations: readOnly,
|
package/lib/proof.mjs
CHANGED
|
@@ -285,6 +285,20 @@ const toldApart = once((ex) => {
|
|
|
285
285
|
const MESSAGE_NAME = /^(?:user_?)?(?:message|msg|query|question|input|text|content|ask)$/i;
|
|
286
286
|
const INSTRUCTIONS_NAME = /system|instruction|persona|preamble|developer/i;
|
|
287
287
|
const nameRank = (l) => { const key = String(l.at.at(-1)); return INSTRUCTIONS_NAME.test(key) ? 2 : MESSAGE_NAME.test(key) ? 0 : 1; };
|
|
288
|
+
// What the person can be asking in this request: never a turn's side ("user", "assistant"), and in a
|
|
289
|
+
// list of turns only its newest, unless its side names the app, since a history ends on the app's reply.
|
|
290
|
+
// The person's side goes by any word (a "parent", an id), so only the app's own are told apart.
|
|
291
|
+
// One word counts only where the person's turn carried it: a word of the app's instructions matches by
|
|
292
|
+
// chance. ulaim's "the Arabic root" made "language" its message, and its history's sides made
|
|
293
|
+
// "conversationHistory" its message after that.
|
|
294
|
+
const MODEL_SIDE = /^(?:assistant|ai|bot|chatbot|model|system|developer|tool|function|agent)$/i;
|
|
295
|
+
const askable = (body, at) => {
|
|
296
|
+
if (TURN_ROLE.test(String(at.at(-1)))) return false;
|
|
297
|
+
const i = at.findIndex((k) => typeof k === "number");
|
|
298
|
+
const list = i < 0 ? null : at.slice(0, i).reduce((v, k) => v?.[k], body);
|
|
299
|
+
return !isTurns(list) || (at[i] === list.length - 1 && !Object.entries(list[at[i]]).some(([k, v]) => TURN_ROLE.test(k) && MODEL_SIDE.test(String(v))));
|
|
300
|
+
};
|
|
301
|
+
const words = (v) => /\s/.test(v) || (v.match(/[\p{L}\p{N}]/gu) ?? []).length >= TOLD_MIN;
|
|
288
302
|
|
|
289
303
|
// The body leaves the app's own client set for its model: words the model was given as a whole
|
|
290
304
|
// instruction message or only inside one, while another field reached it as the person's turn, and a
|
|
@@ -315,7 +329,9 @@ function askOf(exchanges) {
|
|
|
315
329
|
const prompt = promptOf(ex);
|
|
316
330
|
const body = bodyOf(ex);
|
|
317
331
|
if (!prompt || !body) continue;
|
|
318
|
-
const
|
|
332
|
+
const { person } = toldApart(ex);
|
|
333
|
+
const found = leaves(body).map((l) => ({ ...l, v: norm(l.value) })).filter((l) => l.v.length >= 3 && /[a-z\u00c0-\uffff]/.test(l.v) && askable(body, l.at) && prompt.includes(l.v)
|
|
334
|
+
&& (words(l.v) || person.some((t) => t.includes(l.v))));
|
|
319
335
|
for (const l of appLeaves(ex, found)) app.add(fieldOf(l.at));
|
|
320
336
|
for (const l of found) {
|
|
321
337
|
const field = fieldOf(l.at);
|
|
@@ -339,7 +355,7 @@ function askIn(ex, ask) {
|
|
|
339
355
|
const body = bodyOf(ex);
|
|
340
356
|
if (!ask || !body) return "";
|
|
341
357
|
const prompt = promptOf(ex);
|
|
342
|
-
return leaves(body).filter((l) => fieldOf(l.at) === ask.field && (!ask.list || prompt.includes(norm(l.value)))).at(-1)?.value ?? "";
|
|
358
|
+
return leaves(body).filter((l) => fieldOf(l.at) === ask.field && askable(body, l.at) && (!ask.list || prompt.includes(norm(l.value)))).at(-1)?.value ?? "";
|
|
343
359
|
}
|
|
344
360
|
|
|
345
361
|
// ---- the session
|
|
@@ -1153,6 +1153,30 @@ def _install():
|
|
|
1153
1153
|
except (TypeError, ValueError):
|
|
1154
1154
|
return ""
|
|
1155
1155
|
|
|
1156
|
+
# A model request kept whole when it fits; a longer one keeps its shape and every value short
|
|
1157
|
+
# enough to be a turn, its longest texts cut in their middle (lib/wire.cjs fitted).
|
|
1158
|
+
def fitted(body):
|
|
1159
|
+
whole = text(body, reply_max)
|
|
1160
|
+
if len(whole) <= limit:
|
|
1161
|
+
return whole
|
|
1162
|
+
try:
|
|
1163
|
+
parsed = json.loads(whole)
|
|
1164
|
+
except ValueError:
|
|
1165
|
+
parsed = None
|
|
1166
|
+
cap = limit // 2
|
|
1167
|
+
while isinstance(parsed, (dict, list)) and cap >= 256:
|
|
1168
|
+
def cut(v):
|
|
1169
|
+
if isinstance(v, str):
|
|
1170
|
+
return v if len(v) <= cap else v[:cap // 2] + " [...] " + v[-(cap // 2):]
|
|
1171
|
+
if isinstance(v, list):
|
|
1172
|
+
return [cut(x) for x in v]
|
|
1173
|
+
return {k: cut(x) for k, x in v.items()} if isinstance(v, dict) else v
|
|
1174
|
+
out = json.dumps(cut(parsed), ensure_ascii=False)
|
|
1175
|
+
if len(out) <= limit:
|
|
1176
|
+
return out
|
|
1177
|
+
cap //= 2
|
|
1178
|
+
return whole[:limit]
|
|
1179
|
+
|
|
1156
1180
|
# An exchange is kept when the request makes its first model call, for the first few requests per
|
|
1157
1181
|
# route, and written once the app has answered. Model calls and tool rows carry the request's id
|
|
1158
1182
|
# whether kept or not; the command joins them to the exchange it has.
|
|
@@ -1202,7 +1226,7 @@ def _install():
|
|
|
1202
1226
|
per_route[key] = n + 1
|
|
1203
1227
|
req["kept"] = True
|
|
1204
1228
|
if req["kept"] and len(req["sent"]) < sent_max:
|
|
1205
|
-
req["sent"].append(
|
|
1229
|
+
req["sent"].append(fitted(body))
|
|
1206
1230
|
return req
|
|
1207
1231
|
|
|
1208
1232
|
def noted(url, sent):
|
package/lib/read-text.mjs
CHANGED
|
@@ -6,7 +6,10 @@ import { at, clip, has, num, pack, pageOf, plural, SIDE } from "./words.mjs";
|
|
|
6
6
|
// The full list comes back from GET /mcp/status?show=<name>: read.rules.all, read.machine.misses
|
|
7
7
|
// complete with read.machine.mets, read.journeys.all, read.doors.all, read.trials.all. Where the API
|
|
8
8
|
// has not sent it, what it did send is listed with "n of N listed".
|
|
9
|
-
|
|
9
|
+
// `records` lists the plain status's own `read.records`: the server sends them with every status.
|
|
10
|
+
// `reviews` are the two reviews the read leaves, of the prompts and of the code around the model:
|
|
11
|
+
// status hands them over once as news, and this lists them again.
|
|
12
|
+
export const SHOW = ["rules", "standards", "journeys", "endpoints", "trials", "records", "reviews"];
|
|
10
13
|
|
|
11
14
|
const rulesHead = (rules) => {
|
|
12
15
|
const files = Array.isArray(rules.files) ? rules.files.length : rules.files;
|
|
@@ -16,13 +19,17 @@ const rulesHead = (rules) => {
|
|
|
16
19
|
const measuredOf = (m) => (m === null ? "; asked of no reply yet" : m ? `; asked of ${plural(m.asked, "reply", "replies")}, passed in ${num(m.held)}${m.asked > m.held ? `, broke in ${num(m.asked - m.held)}` : ""}` : "");
|
|
17
20
|
const ruleLine = (e) => ` ${e.path ? `${at(e.path, e.line)} ` : ""}"${clip(e.text, 300)}"${"measured" in e ? measuredOf(e.measured) : ""}`;
|
|
18
21
|
const headOf = (label, parts) => (parts.some(Boolean) ? `${label}: ${parts.filter(Boolean).join(", ")}.` : `${label}:`);
|
|
19
|
-
|
|
22
|
+
// "28 decided, 28 met, 0 missed, from 0 problems" was four counts for one fact.
|
|
23
|
+
// A review that decided nothing (it failed, or no standard applied) makes no claim: "meets all 0".
|
|
24
|
+
const standardsHead = (m) => (!has(m.met) || !(m.met + (m.missed ?? 0)) ? "Engineering standards:"
|
|
25
|
+
: m.missed ? `Engineering standards: your code misses ${num(m.missed)} of the ${num(m.decided ?? m.met + m.missed)} Cortad checks${has(m.problems) ? `, through ${plural(m.problems, "problem")}` : ""}.`
|
|
26
|
+
: `Engineering standards: your code meets all ${num(m.met)} that Cortad checks.`);
|
|
20
27
|
const decided = (x) => (x.decidedBy ? ` (decided by ${x.decidedBy === "code" ? "code" : "a model"})` : "");
|
|
21
28
|
// A miss is one problem with the standards it fails beneath it; an older API sent one standard a row.
|
|
22
29
|
// A problem their own documents waive says so, with the line; the server lists it last.
|
|
23
30
|
const designed = (x) => (x.byDesign ? `\n By design here, per ${x.byDesign.at}: "${clip(x.byDesign.said, 200)}"` : "");
|
|
24
31
|
const standardLine = (x) => Array.isArray(x.standards)
|
|
25
|
-
? ` ${x.path ? `${at(x.path, x.line)} ` : ""}${clip(x.problem, 240)}${decided(x)}\n ${x.standards.length === 1 ? "Fails" : `Fails ${num(x.standards.length)}`}: ${x.standards.join("; ")}${designed(x)}`
|
|
32
|
+
? ` ${x.path ? `${at(x.path, x.line)} ` : ""}${clip(x.problem, 240)}${decided(x)}${x.fix ? `\n ${clip(x.fix, 300)}` : ""}\n ${x.standards.length === 1 ? "Fails" : `Fails ${num(x.standards.length)}`}: ${x.standards.join("; ")}${designed(x)}`
|
|
26
33
|
: ` ${x.path ? `${at(x.path, x.line)} ` : ""}${x.standard}${x.detail ? `: ${clip(x.detail, 240)}` : ""}${decided(x)}`;
|
|
27
34
|
// Which trials can be played is decided when a run knocks: a read that said "21 playable, none held
|
|
28
35
|
// back" was followed by a run that held 28 back, and three agents called the counts contradictory.
|
|
@@ -32,40 +39,43 @@ const named = (label, group, max = 12) => {
|
|
|
32
39
|
if (!group) return null;
|
|
33
40
|
const names = group.names ?? [];
|
|
34
41
|
const more = (group.count ?? names.length) - Math.min(names.length, max);
|
|
35
|
-
|
|
36
|
-
|
|
42
|
+
// A name may hold commas ("A student, Grade 10 Sem 2 (Jordan), English"): one name ends at a semicolon.
|
|
43
|
+
const list = names.slice(0, max).join("; ");
|
|
44
|
+
return `${label} (${num(group.count ?? names.length)})${list ? `: ${list}${more > 0 ? `; and ${more} more` : ""}` : ""}.`;
|
|
37
45
|
};
|
|
38
46
|
// "Journeys (6):" when every one is here, "Journeys (6), 4 listed:" when not.
|
|
39
47
|
const counted = (label, count, n) => `${label} (${num(count ?? n)})${has(count) && n < count ? `, ${num(n)} listed` : ""}:`;
|
|
40
48
|
|
|
41
49
|
// The records the app already holds, read from its own data files, that a trial may name: an agent
|
|
42
50
|
// that sees a trial ask about SO20260810001 can tell it is a real order, not an invented one.
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
return `Records the trials can name: ${collections.map((c) => `${c.name} (${c.examples.slice(0, 3).map((e) => clip(e, 40)).join(", ")})`).join(", ")}.`;
|
|
47
|
-
};
|
|
51
|
+
// Listed with show records, never in the walkthrough: ulaim's was 45 ids in one line.
|
|
52
|
+
const recordLines = (records) => (records?.collections ?? []).filter((c) => c?.name && c.examples?.length)
|
|
53
|
+
.map((c) => ` ${c.name}: ${c.examples.slice(0, 3).map((e) => clip(e, 40)).join(", ")}`);
|
|
48
54
|
|
|
49
55
|
// The walkthrough, in the order the person is walked through it.
|
|
56
|
+
// The endpoints are listed above it, proven or not reached, and the questions are the rules and
|
|
57
|
+
// standards counted again: neither is said here, and status with show lists every section in full.
|
|
50
58
|
export function readLines(r) {
|
|
51
59
|
const lines = ["What Cortad read:"];
|
|
52
60
|
if (r.rules) {
|
|
53
61
|
const examples = (r.rules.examples ?? []).slice(0, 3);
|
|
54
|
-
|
|
62
|
+
// "Asked of no reply yet" on each rule before any run is a fact about no run.
|
|
63
|
+
const ran = has(r.rules.measured);
|
|
64
|
+
lines.push(`${rulesHead(r.rules)}${examples.length ? ` ${examples.length} of them:` : ""}`, ...examples.map((e) => ruleLine(ran ? e : { text: e.text, path: e.path, line: e.line })));
|
|
55
65
|
}
|
|
56
|
-
for (const line of [named("Journeys", r.journeys), named("Simulated users", r.profiles)
|
|
66
|
+
for (const line of [named("Journeys", r.journeys), named("Simulated users", r.profiles)]) if (line) lines.push(line);
|
|
57
67
|
const m = r.machine;
|
|
58
|
-
if (m) {
|
|
68
|
+
if (m && (m.met || m.missed || (m.misses ?? []).length)) {
|
|
59
69
|
const misses = m.misses ?? [];
|
|
60
70
|
// "3 missed" was read off a list of three under a head that said thirteen: the list says it is a part.
|
|
61
71
|
lines.push(standardsHead(m), ...(misses.length > 3 && has(m.missed) && m.missed > 3 ? [` 3 of the ${num(m.missed)} missed:`] : []), ...misses.slice(0, 3).map(standardLine));
|
|
62
72
|
const more = (m.problems ?? misses.length) - Math.min(misses.length, 3);
|
|
63
73
|
if (more > 0) lines.push(` and ${plural(more, "more problem")}; status with show standards lists them all.`);
|
|
64
74
|
}
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
const
|
|
68
|
-
if (
|
|
75
|
+
if (r.reviews?.lines.length) lines.push("The reviews of your prompts and of the code around your model are listed whole by status with show reviews.");
|
|
76
|
+
// What a run plays is the quote's to say; here only conversations set aside, and why.
|
|
77
|
+
const held = r.trials?.held ?? [];
|
|
78
|
+
if (held.length) lines.push("Conversations set aside:", ...held.map((h) => ` ${h.surface}: ${plural(h.n, "conversation")}${heldWhy(h)}`));
|
|
69
79
|
return lines;
|
|
70
80
|
}
|
|
71
81
|
|
|
@@ -100,6 +110,12 @@ function sectionOf(r, show) {
|
|
|
100
110
|
const all = r.doors?.all ?? (r.doors?.names ?? []).map((name) => ({ name }));
|
|
101
111
|
return { head: counted("Endpoints", r.doors?.count, all.length), lines: all.map((e) => ` ${e.name}${e.path ? `: ${[e.method, e.path].filter(Boolean).join(" ")}` : ""}`), continued: "Endpoints, continued." };
|
|
102
112
|
}
|
|
113
|
+
case "records":
|
|
114
|
+
return recordLines(r.records).length
|
|
115
|
+
? { head: "Records the trials can name, three of each:", lines: recordLines(r.records), continued: "Records, continued." }
|
|
116
|
+
: { head: "The read found no records a trial can name.", lines: [], continued: "" };
|
|
117
|
+
case "reviews":
|
|
118
|
+
return { head: "The reviews of your prompts and of the code around your model:", lines: r.reviews?.lines.length ? r.reviews.lines : [r.complete === false ? "Your code is still being read; the reviews come when that is done." : r.reviews ? "The read left no review of this app's prompts or code." : "This status carries no reviews."], continued: "The reviews, continued." };
|
|
103
119
|
case "trials":
|
|
104
120
|
default: {
|
|
105
121
|
const t = r.trials ?? {};
|
package/lib/stores.mjs
CHANGED
|
@@ -228,22 +228,22 @@ const HELD = {
|
|
|
228
228
|
"neon-key": (s) => ["there is no NEON_API_KEY in its settings to make a branch with", `Set NEON_API_KEY, or point ${s.names[0]} at a database on this machine`],
|
|
229
229
|
neon: (s) => ["Neon did not make a branch of it", `Check NEON_API_KEY, or point ${s.names[0]} at a database on this machine`],
|
|
230
230
|
supabase: (s) => ["a local Supabase needs supabase/config.toml, Docker and the supabase command", `Add them, or point ${s.names[0]} at a local Supabase`],
|
|
231
|
-
engine: (s) => [`a hosted ${ENGINE[s.engine]}
|
|
231
|
+
engine: (s) => [`Cortad cannot copy a hosted ${ENGINE[s.engine]}`, `Point ${s.names[0]} at a ${ENGINE[s.engine]} on this machine`],
|
|
232
232
|
parts: (s) => ["its settings name it in parts, which cannot be pointed at a copy", `Put its whole address in one setting, or point ${s.names[0]} at a database on this machine`],
|
|
233
233
|
"no-migrations": (s) => ["no migrations were found to build a copy on this machine from", `Point ${s.names[0]} at a database on this machine`],
|
|
234
234
|
"no-local": (s) => ["no Postgres on this machine let us make a database to build a copy in", "Start Postgres on this machine"],
|
|
235
235
|
migrations: (s, tool) => [`its ${tool} migrations did not run against a copy on this machine`, `Run them against a local database once by hand, or point ${s.names[0]} at one`],
|
|
236
|
-
started: () => ["your app was already running when this command started
|
|
236
|
+
started: () => ["your app was already running when this command started", "Stop your app"],
|
|
237
237
|
bypassed: () => ["your app reached it anyway, since it loads its settings file over the settings it was started with", "Load that file without overriding what is already set"],
|
|
238
238
|
other: (s) => ["the copy did not go through", `Point ${s.names[0]} at a database on this machine`],
|
|
239
239
|
};
|
|
240
|
-
// The
|
|
241
|
-
//
|
|
242
|
-
//
|
|
240
|
+
// The fact and the way to keep the store out of the run, read by the person on the card and by their
|
|
241
|
+
// agent in status. The other way, the owner's yes, is each surface's own to offer: the card as its
|
|
242
|
+
// button, status in the sentence after this one (src/local/contact.ts heldLine). Written in here, the
|
|
243
|
+
// card told the owner, on the card, that "the owner can say yes to it on the card in the browser".
|
|
243
244
|
const heldWhy = (s, why, tool) => {
|
|
244
245
|
const [reason, fix] = (HELD[why] ?? HELD.other)(s, tool);
|
|
245
|
-
|
|
246
|
-
return `Trials would write into the ${whatOf(s)} that ${listed(s.names)} ${s.names.length === 1 ? "names" : "name"}, on a server off this machine, and ${reason}. ${fix}, then run the command again.${yes}`;
|
|
246
|
+
return `${listed(s.names)} ${s.names.length === 1 ? "points" : "point"} at ${s.engine === "redis" || s.db ? "the" : "a"} ${whatOf(s)} off this machine, and ${reason}, so the run would write into it as it is. ${fix}, then run the connect command again.`;
|
|
247
247
|
};
|
|
248
248
|
const hostedKept = (s, got) => (got.how === "neon"
|
|
249
249
|
? `Your app runs against a Neon branch of its ${whatOf(s)}, made for this session through ${listed(s.names)}; the original is not touched, and the branch is deleted when this command ends.`
|
package/lib/text.mjs
CHANGED
|
@@ -36,14 +36,17 @@ export function statusText(d) {
|
|
|
36
36
|
if (plan) lines.push(plan);
|
|
37
37
|
lines.push(...appLines(d));
|
|
38
38
|
lines.push(...contactLines(d.contact, d.app?.data));
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
39
|
+
// While the read runs its lists are zeros ("Journeys (0)", "Questions: 0; 0 asked..."): it is said
|
|
40
|
+
// once, and the lists come when it lands (the server's news says so).
|
|
41
|
+
// The quote says it when it is said (it names the read with what the run does about it).
|
|
42
|
+
if (d.read?.complete === false) { if (!d.contact?.ready?.reading || !d.contact.ready.trials) lines.push("Your code is still being read."); }
|
|
43
|
+
else if (d.read) lines.push(...readLines(d.read));
|
|
44
|
+
if (d.run) lines.push(...runLines(d.run, "latest "));
|
|
43
45
|
// Trials the read wrote after this run chose its own: the one count that is not the run's.
|
|
44
46
|
const since = (d.read?.trials?.written ?? 0) - (d.run?.trials?.written ?? Infinity);
|
|
45
47
|
if (since > 0 && d.run?.kind !== "verify") lines.push(`${plural(since, "more trial")} written since this run; the next run chooses from them too.`);
|
|
46
|
-
|
|
48
|
+
// Production comes up after the first run (skill: The second run and production).
|
|
49
|
+
if (d.field && d.run) lines.push(`Production: ${d.field.connected ? "connected" : "not connected"}.`);
|
|
47
50
|
if (d.next) lines.push(d.next);
|
|
48
51
|
if (d.links?.lab && (d.read || d.run)) lines.push(`For the person: ${d.links.lab} shows this in the browser.`);
|
|
49
52
|
if (d.run && !finished(d.run)) lines.push(`next: run_status ${d.run.jobId}`);
|
|
@@ -103,7 +106,7 @@ function promiseLines(p) {
|
|
|
103
106
|
|
|
104
107
|
const promiseLine = (d) => `${upper(`${num(d.promises.held)} of the ${plural(d.promises.decided, "trial")}`)} that asked your app's own promises kept every one.`;
|
|
105
108
|
|
|
106
|
-
const haltLine = (h) => `${h.side === "ours" ? `Stopped ${SIDE.ours}` : "Your app stopped answering"} after ${plural(h.after, "
|
|
109
|
+
const haltLine = (h) => `${h.side === "ours" ? `Stopped ${SIDE.ours}` : "Your app stopped answering"} after ${plural(h.after, "reply", "replies")}${h.why ? `: ${clip(h.why, 240)}` : ""}. The numbers cover the replies before that.`;
|
|
107
110
|
// The pace a slow app was played at, and the replies our own wait cut off: both said as ours. A
|
|
108
111
|
// verify says the second among the replays it could not read.
|
|
109
112
|
const waitLines = (d) => [d.pace?.said, d.kind === "verify" ? null : d.waited?.said].filter(Boolean);
|
|
@@ -182,8 +185,9 @@ function trialsLine(d) {
|
|
|
182
185
|
// before the first turn has no turn to name, and a reason written as a sentence keeps one full stop.
|
|
183
186
|
function stoppedLine(d) {
|
|
184
187
|
const s = d.stopped;
|
|
185
|
-
// A stop before the trials, on the endpoints' first conversations, names each one instead of a
|
|
186
|
-
|
|
188
|
+
// A stop before the trials, on the endpoints' first conversations, names each one instead of a count.
|
|
189
|
+
// `after` counts the replies of the whole run, never one conversation's turn.
|
|
190
|
+
const turn = s.after > 0 && !s.first ? ` after ${plural(s.after, "reply", "replies")}` : "";
|
|
187
191
|
const why = String(s.why).replace(/\.$/, "");
|
|
188
192
|
const fix = s.fix ? ` ${s.fix}` : "";
|
|
189
193
|
if (d.of && d.played < d.of) return `Stopped at ${num(d.played)} of ${plural(d.of, "trial")}${SIDE[s.side] ? `, ${SIDE[s.side]}` : ""}: ${why}${turn}.${fix}`;
|
|
@@ -325,7 +329,7 @@ export function linesOf(d) {
|
|
|
325
329
|
export const findingsText = (d, page = 1) => pageOf(findingsPages(d), page, (n) => `findings with page ${n}`);
|
|
326
330
|
|
|
327
331
|
function findingsHead(d) {
|
|
328
|
-
if (!d.runId) return "No run has finished on this repository yet.";
|
|
332
|
+
if (!d.runId) return d.why ?? "No run has finished on this repository yet.";
|
|
329
333
|
const settled = (d.findings ?? []).filter((f) => f.trials && !f.unsettled).length;
|
|
330
334
|
const lines = [`Run ${d.runId}: ${plural(d.findings?.length ?? 0, "finding")}${settled ? `, ${num(settled)} settled` : ""}.`];
|
|
331
335
|
const cost = costSaid(d);
|
|
@@ -341,7 +345,8 @@ function findingsTail(d) {
|
|
|
341
345
|
if (d.halted) lines.push(haltLine(d.halted));
|
|
342
346
|
lines.push(...waitLines(d));
|
|
343
347
|
if (d.baseRunId) lines.push(`Compared with run ${d.baseRunId}.`);
|
|
344
|
-
|
|
348
|
+
// With no run to read, the head already said why.
|
|
349
|
+
for (const s of [d.read, d.heldBack, d.findings?.length || !d.runId ? null : d.why]) if (s) lines.push(s);
|
|
345
350
|
// A finding in a situation the run keeps back: what failed and where, so the rule can be fixed;
|
|
346
351
|
// its inputs and trials stay unseen, and verify replays it all the same.
|
|
347
352
|
for (const k of d.keptBack ?? []) lines.push(`Kept back ${k.id}: "${clip(k.asks, 160)}"${k.file ? ` at ${at(k.file, k.line)}` : ""}, ${k.trials ? `failed in ${num(k.trials.failed)} of ${plural(k.trials.of, "trial")}` : `passed in ${kn(k.rate)}${has(k.reply) ? ` at reply ${num(k.reply)}` : ""}`}${k.quote ? `; the reply said "${clip(k.quote, 120)}"` : ""}. Its inputs are not shown; verify ${k.id} replays them.`);
|
package/lib/verbs.mjs
CHANGED
|
@@ -58,7 +58,9 @@ export function makeVerbs({ api, token, runner = () => null, fetchImpl = fetch,
|
|
|
58
58
|
// With show, one section of the read in full, paged; a name that is not a section is the plain status.
|
|
59
59
|
const status = async ({ show, page } = {}) => {
|
|
60
60
|
const section = SHOW.includes(show) ? show : null;
|
|
61
|
-
|
|
61
|
+
// The plain status is the one read that prints the server's news, and says so; a section prints
|
|
62
|
+
// none, so it takes none.
|
|
63
|
+
const res = await call("GET", `/mcp/status${section ? `?show=${section}&quiet=1` : "?news=1"}`);
|
|
62
64
|
if (!res.ok) return failed(res);
|
|
63
65
|
// Status says a finished run itself, so the line kept to tell the agent of it is not said again.
|
|
64
66
|
if (finished(res.data?.run) && res.data.run.jobId) heard(res.data.run.jobId);
|
|
@@ -70,7 +72,7 @@ export function makeVerbs({ api, token, runner = () => null, fetchImpl = fetch,
|
|
|
70
72
|
// The files saved since the run the next one is compared with (status `baseline`), which scope a
|
|
71
73
|
// run after a change on Cortad's side. None when this machine cannot say: the run replays everything.
|
|
72
74
|
const changeOf = async (seen) => {
|
|
73
|
-
const base = (seen ?? (await call("GET", "/mcp/status")).data)?.baseline;
|
|
75
|
+
const base = (seen ?? (await call("GET", "/mcp/status?quiet=1")).data)?.baseline;
|
|
74
76
|
const files = base?.jobId ? code?.saved?.(base.startedAt) : null;
|
|
75
77
|
return files && files.length <= MOST_CHANGED ? { since: base.jobId, files } : null;
|
|
76
78
|
};
|
|
@@ -99,7 +101,7 @@ export function makeVerbs({ api, token, runner = () => null, fetchImpl = fetch,
|
|
|
99
101
|
const start = async (args, kind) => {
|
|
100
102
|
const here = runner();
|
|
101
103
|
if (here?.state === "failed") return { text: notStartedText(here), data: { app: here }, isError: true };
|
|
102
|
-
const before = await call("GET", "/mcp/status");
|
|
104
|
+
const before = await call("GET", "/mcp/status?quiet=1");
|
|
103
105
|
if (!before.ok) return failed(before);
|
|
104
106
|
if (here?.state === "up" && before.data.app?.state === "ready-to-test") {
|
|
105
107
|
const idle = !before.data.run || finished(before.data.run);
|
|
@@ -121,7 +123,7 @@ export function makeVerbs({ api, token, runner = () => null, fetchImpl = fetch,
|
|
|
121
123
|
// With no id: the run this machine is still starting, held here until it has an id, or else the
|
|
122
124
|
// latest run on the server.
|
|
123
125
|
const resolve = async (until) => {
|
|
124
|
-
const latest = await call("GET", "/mcp/status");
|
|
126
|
+
const latest = await call("GET", "/mcp/status?quiet=1");
|
|
125
127
|
if (!latest.ok) return failed(latest);
|
|
126
128
|
const newest = latest.data.run?.jobId ?? null;
|
|
127
129
|
let p = pending.read();
|
|
@@ -217,7 +219,7 @@ export function makeVerbs({ api, token, runner = () => null, fetchImpl = fetch,
|
|
|
217
219
|
// `status --changed`: the files Cortad cites that changed since the latest run started, with the
|
|
218
220
|
// findings at them. Empty when there is nothing to say, or nothing could be asked.
|
|
219
221
|
const changed = async () => {
|
|
220
|
-
const [s, f] = await Promise.all([call("GET", "/mcp/status"), gather()]);
|
|
222
|
+
const [s, f] = await Promise.all([call("GET", "/mcp/status?quiet=1"), gather()]);
|
|
221
223
|
const r = s.ok ? s.data?.run : null;
|
|
222
224
|
if (!r) return { text: "" };
|
|
223
225
|
const kept = pending.read();
|
package/lib/wire.cjs
CHANGED
|
@@ -577,6 +577,19 @@ const decoded = (buf, encoding) => {
|
|
|
577
577
|
} catch { return ""; }
|
|
578
578
|
};
|
|
579
579
|
const parse = (text) => { try { return JSON.parse(text); } catch { return null; } };
|
|
580
|
+
// A model request is kept whole when it fits. A longer one keeps its shape and every value short
|
|
581
|
+
// enough to be a turn, its longest texts cut in their middle: every provider puts the newest turn
|
|
582
|
+
// last, and cutting the body's end kept ulaim's 25,000-token prompt without the person's words.
|
|
583
|
+
// ponytail: a turn longer than the cap that fits is cut too; keep turns whole if one is ever lost.
|
|
584
|
+
const fitted = (text) => {
|
|
585
|
+
if (text.length <= MAX) return text;
|
|
586
|
+
const body = parse(text);
|
|
587
|
+
for (let cap = MAX / 2; body && typeof body === "object" && cap >= 256; cap /= 2) {
|
|
588
|
+
const out = JSON.stringify(body, (_, v) => (typeof v === "string" && v.length > cap ? `${v.slice(0, cap / 2)} [...] ${v.slice(-cap / 2)}` : v));
|
|
589
|
+
if (out.length <= MAX) return out;
|
|
590
|
+
}
|
|
591
|
+
return text.slice(0, MAX);
|
|
592
|
+
};
|
|
580
593
|
const n = (v) => (Number.isInteger(v) && v > 0 ? v : 0);
|
|
581
594
|
// The shapes the providers answer in. promptTokens is the whole input, cache included, so totals add up.
|
|
582
595
|
const tokensOf = (u) => {
|
|
@@ -735,7 +748,7 @@ function makeRecorder({ row, rulesFile, onTurn = () => {}, onEnded = () => {} })
|
|
|
735
748
|
const n = perRoute.get(key) || 0;
|
|
736
749
|
if (n < max && (n || perRoute.size < ROUTES_KEPT)) { perRoute.set(key, n + 1); ctx.kept = true; }
|
|
737
750
|
}
|
|
738
|
-
if (ctx.kept && ctx.sent.length < SENT_MAX) ctx.sent.push(String(sent || "")
|
|
751
|
+
if (ctx.kept && ctx.sent.length < SENT_MAX) ctx.sent.push(fitted(String(sent || "")));
|
|
739
752
|
return ctx;
|
|
740
753
|
};
|
|
741
754
|
const rowOf = (ctx) => {
|
|
@@ -818,5 +831,5 @@ function makeRecorder({ row, rulesFile, onTurn = () => {}, onEnded = () => {} })
|
|
|
818
831
|
module.exports = {
|
|
819
832
|
MAX, REPLY_MAX, EMBEDDING, TOOL_TEXT, MODEL_HOST,
|
|
820
833
|
scrubbed, list, clipped, argsText, turnOf, toolText, isModelCall, isRetrieval, passagesFrom, decoded, cookieNames,
|
|
821
|
-
makeRecorder,
|
|
834
|
+
fitted, makeRecorder,
|
|
822
835
|
};
|
package/local.mjs
CHANGED
|
@@ -1371,7 +1371,7 @@ const IDLE_MS = 10 * 60_000;
|
|
|
1371
1371
|
if (argv.includes("--until-idle") && stored) {
|
|
1372
1372
|
let idleSince = Date.now();
|
|
1373
1373
|
setInterval(async () => {
|
|
1374
|
-
const res = await call("GET", "/mcp/status", undefined, { headers: { authorization: `Bearer ${stored}` } }).catch(() => null);
|
|
1374
|
+
const res = await call("GET", "/mcp/status?quiet=1", undefined, { headers: { authorization: `Bearer ${stored}` } }).catch(() => null);
|
|
1375
1375
|
if (res?.ok && res.data?.run && !finished(res.data.run)) idleSince = Date.now();
|
|
1376
1376
|
else if (Date.now() - idleSince > IDLE_MS) { say("no run for ten minutes, leaving"); await close(0); }
|
|
1377
1377
|
}, 60_000).unref();
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "cortad",
|
|
3
|
-
"version": "0.3.1-rc.
|
|
3
|
+
"version": "0.3.1-rc.19",
|
|
4
4
|
"description": "Connects the AI app on your machine to Cortad for test conversations, and gives your coding agent the MCP and skill to run them. No dependencies.",
|
|
5
5
|
"bin": {
|
|
6
6
|
"cortad": "local.mjs"
|
package/skill/SKILL.md
CHANGED
|
@@ -26,15 +26,20 @@ The connect command ends with "Go back to the browser". The person is waiting on
|
|
|
26
26
|
1. The moment the command says "Go back to the browser", tell the person: connected, the app is up on which port (or did not start, and what the command names), and that you are now sending it one real request per endpoint. If the app did not start, say what you would change and change it only if they say so; the command starts the app again by itself after a save.
|
|
27
27
|
2. Send one real request to each endpoint that reaches the model, the way the app's own client does: its route, its body, its sign-in. On a chat, send a second message in the same conversation. When `status` asks for a request signed in as another customer, send it: each customer you sign in as becomes an account of its own in the run.
|
|
28
28
|
3. A reply that is an error, or a `Problem` line in `status` (a provider refusing, a retrieval with no answer, a tool that failed), is the app's own state, not yours to repair in silence. Say it in two lines: what failed, at which file and line, whose side it is on. Then ask whether they want it fixed first or the run as the app stands. Two minutes of reading for the cause at most, then say what you have.
|
|
29
|
-
4. `status` shows what Cortad read (rules, journeys, simulated users, endpoints, standards) and each endpoint your requests reached with what it did inside the app: the model and how many calls a request made, the seconds a reply took, which of the rules read from the code the prompts carried, the tools that ran, the passages handed to the model, and problems a line of code decides, at their file and line. Endpoints the read found that no request reached are listed apart. The
|
|
30
|
-
5.
|
|
29
|
+
4. `status` shows what Cortad read (rules, journeys, simulated users, endpoints, standards) and each endpoint your requests reached with what it did inside the app: the model and how many calls a request made, the seconds a reply took, which of the rules read from the code the prompts carried, the tools that ran, the passages handed to the model, and problems a line of code decides, at their file and line. Endpoints the read found that no request reached are listed apart. Those lines are for you. The lines that start with `For the person:` are for them: what a run would play, take and cost, and a decision only they can make, such as a store off this machine the run would write into. They allow that with one press on the card, which also starts the run, or keep the store out by pointing its setting at one on this machine; you cannot give that yes, `run` is refused until they do, and you move the store only if they ask.
|
|
30
|
+
5. When the read of the code finishes, `status` leads with two reviews under `New since your last call`: one of the instructions in the app's prompts, one of the code around its model, each problem at its file and line with what to change. `status` with show `reviews` lists them again. The person hears them from you without asking; change nothing they name until the person says so.
|
|
31
|
+
6. The first run is free. Before it, read `status` for what would make the run measure the setup instead of the app:
|
|
31
32
|
- a service down: a `Problem` line where a retrieval got no answer, or a tool came back as an error;
|
|
32
33
|
- a request that ended on a tool call: `The request ended right after ... ran: no model call came after it`, which every conversation on that endpoint would repeat;
|
|
33
34
|
- the sign-in: `Every trial sends the sign-in your agent's own request carried, so all trials act as one account`, or a request your app answered 401 or 403.
|
|
34
35
|
Say what applies and what you would do about it; fix it when they agree, send the request again, and read `status` again. A run that measured nothing is not counted against the plan.
|
|
35
|
-
|
|
36
|
+
7. Then tell the person three things, a sentence or two each, and nothing else:
|
|
37
|
+
- what Cortad found: how many of their endpoints answered, the rules, journeys and kinds of users under `What Cortad read`, and what the two reviews found: how many problems each, and the one or two that matter most at their file and line;
|
|
38
|
+
- the decision that is theirs, from the `For the person:` line that asks one, if there is one;
|
|
39
|
+
- what the run will play, take and cost, from the `For the person:` line that starts `a run plays`, whether it is their free run, any endpoint a `Runs leave this endpoint out` line names with whose side that is, and the link to the card.
|
|
40
|
+
Models, reply times, fields, passages, messages and the endpoints no request reached stay with you unless they ask. Say it once, then stop: the run is theirs to start, from the card or by asking you. If they say they pressed and `status` shows no run, call `run`; their press was the ask, so call it without asking them again. When it ends you are told, on your next prompt or edit, what it found and the call to make, if `npx {{cortad}} stick` has been run in this repository (below); say that in one line, and offer to run it.
|
|
36
41
|
|
|
37
|
-
`status` with `show` lists one section of the read in full: `rules`, `standards`, `journeys`, `endpoints` or `
|
|
42
|
+
`status` with `show` lists one section of the read in full: `rules`, `standards`, `journeys`, `endpoints`, `trials`, `records` or `reviews` (in a shell, `npx {{cortad}} status rules`).
|
|
38
43
|
|
|
39
44
|
## Following a run
|
|
40
45
|
|
|
@@ -10,17 +10,15 @@ Right after a connect, before the read is done:
|
|
|
10
10
|
|
|
11
11
|
```
|
|
12
12
|
Cortad · tutor-app
|
|
13
|
-
Free: 1 of 1 run left this month; 60 verify trials left.
|
|
13
|
+
Free: 1 of 1 run left this month; 60 verify trials left. Pro $99 a month: runs and reruns included, 100,000 production replies read.
|
|
14
14
|
App: Your app is starting on this machine.
|
|
15
|
-
No run yet.
|
|
16
|
-
Production: not connected.
|
|
17
15
|
```
|
|
18
16
|
|
|
19
17
|
After the agent sent one real request to each endpoint that reaches the model, two on the chat, and before any run:
|
|
20
18
|
|
|
21
19
|
```
|
|
22
20
|
Cortad · tutor-app
|
|
23
|
-
Free: 1 of 1 run left this month; 60 verify trials left.
|
|
21
|
+
Free: 1 of 1 run left this month; 60 verify trials left. Pro $99 a month: runs and reruns included, 100,000 production replies read.
|
|
24
22
|
App: Your app answered on port 3100.
|
|
25
23
|
Endpoints your own requests proved (1):
|
|
26
24
|
POST /api/chat: 2 requests reached gpt-4o-mini, 2 model calls each, 3.1 seconds a reply.
|
|
@@ -28,43 +26,46 @@ Endpoints your own requests proved (1):
|
|
|
28
26
|
Tools that ran: search_lessons.
|
|
29
27
|
Passages handed to the model: 3.
|
|
30
28
|
Problem at apps/api/src/agent/system.ts:30: The reply carried "</student_profile>", markup from the prompt your app sent the model.
|
|
31
|
-
Last
|
|
29
|
+
Last message: "and the second question?".
|
|
32
30
|
Endpoints the read found that no request has reached (1):
|
|
33
31
|
POST /api/homework/explain apps/api/src/routes/homework.ts:18
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
32
|
+
For the person: a run plays 36 conversations: about 72 replies in about 6 minutes. It costs about $0.38 on your OpenAI key for gpt-4o-mini. Every reply is checked against the 46 rules read from your code.
|
|
33
|
+
A run starts when the person presses Run on the card in the browser or asks you for one; if they say they pressed it, call run rather than ask them again.
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
When the app keeps its data in a store off this machine that no copy can be made of, the decision is the person's, said once as the last line before the link:
|
|
37
|
+
|
|
38
|
+
```
|
|
39
|
+
For the person: DATABASE_URL points at the Postgres database shop off this machine, and no migrations were found to build a copy on this machine from, so the run would write into it as it is. Point DATABASE_URL at a database on this machine, then run the connect command again. Or one press on the card in the browser lets the run write into it and starts the run.
|
|
40
|
+
For the person: https://cortad.com/lab shows this in the browser.
|
|
38
41
|
```
|
|
39
42
|
|
|
40
43
|
Once a run is playing, the read and the run:
|
|
41
44
|
|
|
42
45
|
```
|
|
43
46
|
Cortad · tutor-app
|
|
44
|
-
Free: this month's 1 run is used; 60 verify trials left.
|
|
47
|
+
Free: this month's 1 run is used; 60 verify trials left. Pro $99 a month: runs and reruns included, 100,000 production replies read.
|
|
45
48
|
App: Your app answered on port 3100.
|
|
46
49
|
What Cortad read:
|
|
47
50
|
Rules in the code: 46, in 5 files. 3 of them:
|
|
48
51
|
apps/api/src/agent/prompt.ts:41 "Never state a refund policy the product does not publish."
|
|
49
52
|
apps/api/src/agent/system.ts:12 "Answer in the language the student writes in."
|
|
50
53
|
apps/api/src/tools/search.ts:8 "Cite the lesson a fact comes from."
|
|
51
|
-
Journeys (4): homework help
|
|
52
|
-
Simulated users (3): student in grade 9
|
|
53
|
-
|
|
54
|
-
Engineering standards: 38 decided, 35 met, 3 missed.
|
|
54
|
+
Journeys (4): homework help; billing question; account recovery; first lesson.
|
|
55
|
+
Simulated users (3): student in grade 9; parent paying for the plan; teacher checking progress.
|
|
56
|
+
Engineering standards: your code misses 3 of the 38 Cortad checks.
|
|
55
57
|
apps/api/src/agent/client.ts:9 The model call has a timeout: the OpenAI client is created with no timeout, so a slow reply holds the request open (decided by code)
|
|
56
58
|
apps/api/src/agent/system.ts:30 User text stays out of the system prompt: the student's name is written into the system prompt (decided by a model)
|
|
57
59
|
apps/api/src/tools/search.ts:51 Tool errors reach the model as errors: search returns an empty list when the index is down (decided by a model)
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
POST /api/homework/upload: 7 trials set aside, on the app's side: the route needs a signed file URL, and no test account can make one
|
|
60
|
+
Conversations set aside:
|
|
61
|
+
POST /api/homework/upload: 7 conversations, on the app's side: the route needs a signed file URL, and no test account can make one
|
|
61
62
|
Latest run 8f2a1c4e-5b6d-4e7f-9a0b-1c2d3e4f5a6b: running, 14 of 51 trials played.
|
|
62
63
|
Production: not connected.
|
|
63
64
|
For the person: https://cortad.com/lab shows this in the browser.
|
|
64
65
|
next: run_status 8f2a1c4e-5b6d-4e7f-9a0b-1c2d3e4f5a6b
|
|
65
66
|
```
|
|
66
67
|
|
|
67
|
-
Each section in full, with `show` (`rules`, `standards`, `journeys`, `endpoints` or `
|
|
68
|
+
Each section in full, with `show` (`rules`, `standards`, `journeys`, `endpoints`, `trials`, `records` or `reviews`). A list longer than one page ends with `page 1 of 3, call status with show rules and page 2`.
|
|
68
69
|
|
|
69
70
|
```
|
|
70
71
|
Rules in the code: 16, in 5 files.
|
|
@@ -110,7 +111,7 @@ Last run 8f2a1c4e-5b6d-4e7f-9a0b-1c2d3e4f5a6b: 3 findings, 51 of 51 trials playe
|
|
|
110
111
|
apps/api/src/agent/prompt.ts:41 (finding:1): gone; 0 of 3 replays failed, against 3 of 3 before; on the conversations kept back from you, 1 of 2 failed before, 0 of 2 after, not beyond chance.
|
|
111
112
|
apps/api/src/agent/system.ts:12 (finding:2): gone; 0 of 7 replays failed, against 4 of 4 before; the fix made the failure worse on the conversations kept back from you (0 of 8 failed before, 6 of 8 after).
|
|
112
113
|
The next run would play 58 trials, 7 kept back, 5 new from the changes.
|
|
113
|
-
The
|
|
114
|
+
The Pro plan, $99 a month, includes 10 runs.
|
|
114
115
|
Production: not connected.
|
|
115
116
|
Nothing ran.
|
|
116
117
|
For the person: plans and checkout at https://cortad.com/pricing?checkout=ship
|
|
@@ -188,7 +189,7 @@ Once production is connected, every production reply is read with these same che
|
|
|
188
189
|
"Never state a refund policy the product does not publish." (apps/api/src/agent/prompt.ts:41): broke in 9 of 12 trials.
|
|
189
190
|
"Answer in the language the student writes in." (apps/api/src/agent/system.ts:12): broke in 2 of 26 trials.
|
|
190
191
|
112 of 140 checks measured.
|
|
191
|
-
Free: this month's 1 run is used; 60 verify trials left.
|
|
192
|
+
Free: this month's 1 run is used; 60 verify trials left. Pro $99 a month: runs and reruns included, 100,000 production replies read.
|
|
192
193
|
For the person: the report is at https://cortad.com/lab
|
|
193
194
|
next: findings
|
|
194
195
|
```
|
|
@@ -221,7 +222,7 @@ Your app answered on gpt-4o-mini. The run took 3 min 8 s and spent $0.19 on your
|
|
|
221
222
|
Replies took 2.9 s at the median and 9.8 s at the slowest, over 81 replies.
|
|
222
223
|
Nothing broke that only running your app could show.
|
|
223
224
|
The next run is compared with run 8f2a1c4e, the last run that played every conversation, with everything that changed since then in its scope.
|
|
224
|
-
|
|
225
|
+
Pro: 8 of 10 runs left this month; 560 verify trials left.
|
|
225
226
|
For the person: the report is at https://cortad.com/lab
|
|
226
227
|
next: status
|
|
227
228
|
```
|
|
@@ -231,7 +232,7 @@ Stopped early. Each stop and fault names the side it is on and what comes next:
|
|
|
231
232
|
```
|
|
232
233
|
Run 8f2a1c4e-5b6d-4e7f-9a0b-1c2d3e4f5a6b: finished, 11 of 51 trials played.
|
|
233
234
|
1 finding.
|
|
234
|
-
Stopped at 11 of 51 trials, on the app's side: your app stopped answering
|
|
235
|
+
Stopped at 11 of 51 trials, on the app's side: your app stopped answering after 11 replies. Bring your app back up, then run again.
|
|
235
236
|
Fault on the app's side: Your code names llama-3.1-8b-instant, which api.groq.com does not serve. Rename the model in your code, then run again.
|
|
236
237
|
For the person: the report is at https://cortad.com/lab
|
|
237
238
|
next: findings
|
|
@@ -260,7 +261,7 @@ A verify replays the conversations the finding failed on, word for word, and dec
|
|
|
260
261
|
Verify 7c31e0aa-1b2c-4d3e-8f4a-5b6c7d8e9f0a: finished, 5 of 5 trials played.
|
|
261
262
|
Verify of finding:1 at apps/api/src/agent/prompt.ts:41.
|
|
262
263
|
The failure is gone on its own conversations: none of 3 replays failed, against 3 of 3 conversations in the run it was found in. On the conversations kept back from you the failure is gone: 3 of 8 failed before the fix, 0 of 8 after, a fall beyond chance.
|
|
263
|
-
Free: this month's 1 run is used; 55 verify trials left.
|
|
264
|
+
Free: this month's 1 run is used; 55 verify trials left. Pro $99 a month: runs and reruns included, 100,000 production replies read.
|
|
264
265
|
For the person: the failure in finding:1 is gone on its replays. Connecting production reads every real reply with these same checks; field_connect has the steps.
|
|
265
266
|
For the person: the report is at https://cortad.com/lab
|
|
266
267
|
next: findings
|