@intentic/sandbox-contract 1.223.0 → 1.225.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -14
- package/dist/agent-catalog.js +3 -3
- package/dist/agent-catalog.js.map +1 -1
- package/dist/chores/chores.js +43 -43
- package/dist/chores/chores.js.map +1 -1
- package/dist/chores/extension-update.js +2 -2
- package/dist/chores/extension-update.js.map +1 -1
- package/dist/chores/fix-deps.js +1 -1
- package/dist/chores/fix-deps.js.map +1 -1
- package/dist/chores/probes.js +1 -1
- package/dist/chores/probes.js.map +1 -1
- package/dist/chores/prompt.d.ts.map +1 -1
- package/dist/chores/prompt.js +3 -3
- package/dist/chores/prompt.js.map +1 -1
- package/dist/chores/verdict.js +2 -2
- package/dist/chores/verdict.js.map +1 -1
- package/dist/contracts/capabilities.contract.d.ts +41 -0
- package/dist/contracts/capabilities.contract.d.ts.map +1 -1
- package/dist/contracts/exit.contract.d.ts +80 -0
- package/dist/contracts/exit.contract.d.ts.map +1 -0
- package/dist/contracts/exit.contract.js +13 -0
- package/dist/contracts/exit.contract.js.map +1 -0
- package/dist/contracts/ports.contract.d.ts +11 -0
- package/dist/contracts/ports.contract.d.ts.map +1 -1
- package/dist/contracts/settings.contract.d.ts +12 -0
- package/dist/contracts/settings.contract.d.ts.map +1 -1
- package/dist/events.d.ts +4 -4
- package/dist/events.d.ts.map +1 -1
- package/dist/events.js +5 -5
- package/dist/events.js.map +1 -1
- package/dist/fast-tier.d.ts +9 -0
- package/dist/fast-tier.d.ts.map +1 -0
- package/dist/fast-tier.js +19 -0
- package/dist/fast-tier.js.map +1 -0
- package/dist/history-state.js +3 -3
- package/dist/history-state.js.map +1 -1
- package/dist/index.d.ts +214 -68
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -0
- package/dist/index.js.map +1 -1
- package/dist/model-order.d.ts +1 -0
- package/dist/model-order.d.ts.map +1 -1
- package/dist/model-order.js +5 -0
- package/dist/model-order.js.map +1 -1
- package/dist/output-fields.d.ts.map +1 -1
- package/dist/output-fields.js +2 -2
- package/dist/output-fields.js.map +1 -1
- package/dist/prompt-complexity.d.ts +19 -0
- package/dist/prompt-complexity.d.ts.map +1 -0
- package/dist/prompt-complexity.js +83 -0
- package/dist/prompt-complexity.js.map +1 -0
- package/dist/publish-drafts.js +2 -2
- package/dist/publish-drafts.js.map +1 -1
- package/dist/schemas.d.ts +236 -0
- package/dist/schemas.d.ts.map +1 -1
- package/dist/schemas.js +116 -2
- package/dist/schemas.js.map +1 -1
- package/dist/workflow-faults.js +3 -3
- package/dist/workflow-faults.js.map +1 -1
- package/dist/workspace-state.d.ts +20 -20
- package/dist/workspace-state.d.ts.map +1 -1
- package/dist/workspace-state.js +20 -20
- package/dist/workspace-state.js.map +1 -1
- package/package.json +5 -5
- package/src/agent-catalog.test.ts +25 -25
- package/src/agent-catalog.ts +81 -81
- package/src/agent-run-model.test.ts +3 -3
- package/src/agent-run-model.ts +8 -8
- package/src/capability-env.ts +3 -3
- package/src/capability-ledger.test.ts +13 -13
- package/src/capability-secrets.ts +5 -5
- package/src/chores/chores.test.ts +1 -1
- package/src/chores/chores.ts +109 -109
- package/src/chores/digest.test.ts +1 -1
- package/src/chores/digest.ts +3 -3
- package/src/chores/extension-update.ts +5 -5
- package/src/chores/fix-deps.ts +4 -4
- package/src/chores/probes.test.ts +6 -6
- package/src/chores/probes.ts +17 -17
- package/src/chores/prompt.ts +9 -9
- package/src/chores/stack.test.ts +3 -3
- package/src/chores/stack.ts +17 -17
- package/src/chores/verdict.test.ts +20 -20
- package/src/chores/verdict.ts +25 -25
- package/src/contract-lock.test.ts +1 -1
- package/src/contract-lock.ts +7 -7
- package/src/contracts/activity.contract.ts +1 -1
- package/src/contracts/agent.contract.ts +2 -2
- package/src/contracts/agents.contract.ts +11 -11
- package/src/contracts/automations.contract.ts +5 -5
- package/src/contracts/capabilities.contract.ts +7 -7
- package/src/contracts/chores.contract.ts +3 -3
- package/src/contracts/ci.contract.ts +2 -2
- package/src/contracts/claude.contract.ts +3 -3
- package/src/contracts/drafts.contract.ts +1 -1
- package/src/contracts/endpoints.contract.ts +2 -2
- package/src/contracts/exit.contract.ts +42 -0
- package/src/contracts/extensions.contract.ts +8 -8
- package/src/contracts/git.contract.ts +6 -6
- package/src/contracts/grok.contract.ts +4 -4
- package/src/contracts/host.contract.ts +5 -5
- package/src/contracts/intentic.contract.ts +2 -2
- package/src/contracts/logs.contract.ts +1 -1
- package/src/contracts/loops.contract.ts +8 -8
- package/src/contracts/personas.contract.ts +8 -8
- package/src/contracts/ports.contract.ts +1 -1
- package/src/contracts/prepush.contract.ts +2 -2
- package/src/contracts/providers.contract.ts +4 -4
- package/src/contracts/public.contract.ts +1 -1
- package/src/contracts/push.contract.ts +2 -2
- package/src/contracts/secrets.contract.ts +2 -2
- package/src/contracts/sessions.contract.ts +2 -2
- package/src/contracts/settings.contract.ts +3 -3
- package/src/contracts/share.contract.ts +1 -1
- package/src/contracts/skills.contract.ts +2 -2
- package/src/contracts/system.contract.ts +9 -9
- package/src/contracts/usage.contract.ts +1 -1
- package/src/contracts/vpn.contract.ts +5 -5
- package/src/contracts/workflows.contract.ts +12 -12
- package/src/contracts/workspace.contract.ts +13 -13
- package/src/conversation-ids.ts +8 -8
- package/src/events.test.ts +3 -3
- package/src/events.ts +142 -142
- package/src/fast-tier.test.ts +88 -0
- package/src/fast-tier.ts +72 -0
- package/src/history-state.ts +15 -15
- package/src/host-protocol.ts +7 -7
- package/src/hostnames.test.ts +1 -1
- package/src/hostnames.ts +15 -15
- package/src/index.ts +13 -8
- package/src/listener-protocol.ts +12 -12
- package/src/model-order.test.ts +11 -11
- package/src/model-order.ts +52 -30
- package/src/output-fields.ts +11 -11
- package/src/path-refs.test.ts +4 -4
- package/src/path-refs.ts +6 -6
- package/src/prompt-complexity.test.ts +160 -0
- package/src/prompt-complexity.ts +271 -0
- package/src/publish-drafts.ts +8 -8
- package/src/quick-model.test.ts +11 -11
- package/src/quick-model.ts +15 -15
- package/src/routes.test.ts +11 -5
- package/src/routes.ts +19 -19
- package/src/runtime-state.test.ts +1 -1
- package/src/runtime-state.ts +22 -22
- package/src/schemas.test.ts +8 -8
- package/src/schemas.ts +1238 -921
- package/src/search-globs.test.ts +2 -2
- package/src/search-globs.ts +6 -6
- package/src/session-names.ts +5 -5
- package/src/share-paths.test.ts +1 -1
- package/src/share-paths.ts +7 -7
- package/src/sse.ts +1 -1
- package/src/state-portability.ts +8 -8
- package/src/terminal-protocol.ts +3 -3
- package/src/title.test.ts +9 -9
- package/src/title.ts +21 -21
- package/src/tunnel-ids.test.ts +3 -3
- package/src/tunnel-ids.ts +7 -7
- package/src/versions.test.ts +3 -3
- package/src/versions.ts +6 -6
- package/src/workflow-faults.test.ts +6 -6
- package/src/workflow-faults.ts +14 -14
- package/src/workspace-state.test.ts +32 -32
- package/src/workspace-state.ts +139 -139
package/src/model-order.test.ts
CHANGED
|
@@ -11,7 +11,7 @@ const CODEX = ["gpt-5.1-codex", "gpt-5.4-mini", "gpt-5.5", "gpt-5.6-luna", "gpt-
|
|
|
11
11
|
|
|
12
12
|
test("ranks the frontier line above the cheap one and the newest release above its predecessors", () => {
|
|
13
13
|
// The base line (no tier word) leads, newest first; the mini rung sinks under all of it regardless of how
|
|
14
|
-
// recently it shipped
|
|
14
|
+
// recently it shipped, which is the whole decision a user makes in this list.
|
|
15
15
|
expect(CODEX.toSorted(compareModelIds)).toEqual(["gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna", "gpt-5.5", "gpt-5.1-codex", "gpt-5.4-mini"]);
|
|
16
16
|
});
|
|
17
17
|
|
|
@@ -47,7 +47,7 @@ test("the Codex release-tier order is stable across catalog refreshes", () => {
|
|
|
47
47
|
expect(["gpt-5.4-mini", ...arrivals[0]!].toSorted(compareUnrankedModelIds).at(-1)).toBe("gpt-5.4-mini");
|
|
48
48
|
});
|
|
49
49
|
|
|
50
|
-
test("leaves a RANKED catalog's ties alone
|
|
50
|
+
test("leaves a RANKED catalog's ties alone: the id tiebreak is for sets, and Anthropic publishes an opinion", () => {
|
|
51
51
|
// compareUnrankedModelIds would seat claude-fable-5 ahead of claude-opus-5 on the id alone. Anthropic's
|
|
52
52
|
// catalog arrives newest-first, so that order is a fact about the provider, not a leftover to be broken.
|
|
53
53
|
expect(["claude-opus-5", "claude-fable-5"].toSorted(compareModelIds)).toEqual(["claude-opus-5", "claude-fable-5"]);
|
|
@@ -82,7 +82,7 @@ test("reads Kimi's k-prefixed generation so K3 leads the K2.x catalog", () => {
|
|
|
82
82
|
});
|
|
83
83
|
|
|
84
84
|
test("the rightmost tier word wins, because tier words compose", () => {
|
|
85
|
-
// flash-lite is the cheap end of Flash, codex-max the frontier end of Codex
|
|
85
|
+
// flash-lite is the cheap end of Flash, codex-max the frontier end of Codex: reading the leftmost word
|
|
86
86
|
// instead would file both under the tier they modify.
|
|
87
87
|
expect(tierRankOf(familyOf("gemini-3-flash-lite"))).toBe(tierRankOf("lite"));
|
|
88
88
|
expect(tierRankOf(familyOf("gpt-5.1-codex-max"))).toBe(tierRankOf("max"));
|
|
@@ -95,7 +95,7 @@ test("leads with a family carrying no tier word at all, so a brand-new flagship
|
|
|
95
95
|
|
|
96
96
|
test("files a re-served open-weights model on the cheap rung, not at the head of the catalog it visits", () => {
|
|
97
97
|
// Google's channel vends gpt-oss beside Gemini and Claude. It carries no tier word of its own, so the
|
|
98
|
-
// lead-the-unknown rule would open that whole section on it
|
|
98
|
+
// lead-the-unknown rule would open that whole section on it: above Opus.
|
|
99
99
|
expect(["gpt-oss-120b-medium", "claude-opus-4-6-thinking", "gemini-pro-agent"].toSorted(compareModelIds)).toEqual([
|
|
100
100
|
"claude-opus-4-6-thinking",
|
|
101
101
|
"gemini-pro-agent",
|
|
@@ -103,7 +103,7 @@ test("files a re-served open-weights model on the cheap rung, not at the head of
|
|
|
103
103
|
]);
|
|
104
104
|
});
|
|
105
105
|
|
|
106
|
-
test("keeps the arrival order between ids the rule cannot separate
|
|
106
|
+
test("keeps the arrival order between ids the rule cannot separate: Anthropic's catalog IS ranked", () => {
|
|
107
107
|
// Same tier, same version: nothing here outranks the order the provider itself reported.
|
|
108
108
|
expect(["claude-opus-5", "claude-fable-5"].toSorted(compareModelIds)).toEqual(["claude-opus-5", "claude-fable-5"]);
|
|
109
109
|
expect(["claude-fable-5", "claude-opus-5"].toSorted(compareModelIds)).toEqual(["claude-fable-5", "claude-opus-5"]);
|
|
@@ -124,7 +124,7 @@ test("stands an id with nothing but numbers (and an ACP row's empty one) as its
|
|
|
124
124
|
|
|
125
125
|
test("holds date stamps apart from version components, or a dated build outranks the point release after it", () => {
|
|
126
126
|
// The failure this prevents: claude-opus-4-1-20250805 (Opus 4.1) read as (4,1,20250805) loses to
|
|
127
|
-
// claude-opus-4-20250514 (Opus 4.0) read as (4,20250514)
|
|
127
|
+
// claude-opus-4-20250514 (Opus 4.0) read as (4,20250514): the older model, by six digits.
|
|
128
128
|
expect(releaseOf("claude-opus-4-1-20250805")).toEqual({ version: [4, 1], date: 20250805 });
|
|
129
129
|
expect(["claude-opus-4-20250514", "claude-opus-4-1-20250805"].toSorted(compareModelIds)).toEqual([
|
|
130
130
|
"claude-opus-4-1-20250805",
|
|
@@ -153,7 +153,7 @@ test("sorts an unversioned rolling alias under the releases that name their vers
|
|
|
153
153
|
// What the quick model behind a one-click helper resolves against: the same tier scale, read for the weakest
|
|
154
154
|
// row instead of the strongest.
|
|
155
155
|
|
|
156
|
-
test("opens on the efficient rung and buries the frontier one
|
|
156
|
+
test("opens on the efficient rung and buries the frontier one: the exact inverse of the picker's order", () => {
|
|
157
157
|
const claude = ["claude-opus-5", "claude-sonnet-5", "claude-haiku-4-5-20251001"];
|
|
158
158
|
|
|
159
159
|
expect(claude.toSorted(compareCheapestFirst)).toEqual(["claude-haiku-4-5-20251001", "claude-sonnet-5", "claude-opus-5"]);
|
|
@@ -162,7 +162,7 @@ test("opens on the efficient rung and buries the frontier one — the exact inve
|
|
|
162
162
|
|
|
163
163
|
test("keeps an UNRANKED family off the cheap end, where a plain reversal would have seated it first", () => {
|
|
164
164
|
// The whole reason this is not `-compareModelIds`. An id with no tier word is the provider's base line, and
|
|
165
|
-
// an unheard-of family is likelier the next flagship than the next budget tier
|
|
165
|
+
// an unheard-of family is likelier the next flagship than the next budget tier, so both orders agree it is
|
|
166
166
|
// not the efficient rung, and a helper never spends frontier money on a commit message.
|
|
167
167
|
expect(["gpt-5.6", "gpt-5.4-mini"].toSorted(compareCheapestFirst)).toEqual(["gpt-5.4-mini", "gpt-5.6"]);
|
|
168
168
|
expect(["claude-mythos-1", "claude-haiku-4-5", "claude-sonnet-5"].toSorted(compareCheapestFirst).at(-1)).toBe("claude-mythos-1");
|
|
@@ -177,7 +177,7 @@ test("takes the NEWEST build of the cheap rung, not merely any of them", () => {
|
|
|
177
177
|
test("finds each vendor's own cheap rung, including a re-served open-weights row", () => {
|
|
178
178
|
expect(["gemini-3-pro", "gemini-3-flash", "gemini-3-flash-lite"].toSorted(compareCheapestFirst)[0]).toBe("gemini-3-flash-lite");
|
|
179
179
|
// Google's channel vends gpt-oss beside Gemini's own line; it is there to be the cheap option, and `oss`
|
|
180
|
-
// is what says so
|
|
180
|
+
// is what says so: without that word the id carries no tier at all and would sink to the bottom.
|
|
181
181
|
expect(["claude-opus-4-6-thinking", "gpt-oss-120b-medium"].toSorted(compareCheapestFirst)[0]).toBe("gpt-oss-120b-medium");
|
|
182
182
|
expect(["grok-4", "grok-4-fast"].toSorted(compareCheapestFirst)[0]).toBe("grok-4-fast");
|
|
183
183
|
});
|
|
@@ -188,7 +188,7 @@ test("reads a release-local tier ladder from the cheap end too", () => {
|
|
|
188
188
|
|
|
189
189
|
test("refuses the thinking variant of a model, however new it is", () => {
|
|
190
190
|
/* The bug this rule exists for, in the shape the live catalog actually publishes it: a routed channel vends
|
|
191
|
-
* one row per thinking LEVEL, and the newest row of the cheapest model was the high one
|
|
191
|
+
* one row per thinking LEVEL, and the newest row of the cheapest model was the high one, so the ladder
|
|
192
192
|
* whose whole job is to be the cheap rung reached for the most expensive reading of it, and a commit
|
|
193
193
|
* message that takes 2s took closer to 30. */
|
|
194
194
|
expect(["gemini-3.6-flash-high", "gemini-3.5-flash-extra-low"].toSorted(compareCheapestFirst)[0]).toBe("gemini-3.5-flash-extra-low");
|
|
@@ -219,6 +219,6 @@ test("names the thinking rows, and only those", () => {
|
|
|
219
219
|
|
|
220
220
|
test("falls back on the newest release for a catalog that publishes no cheap tier at all", () => {
|
|
221
221
|
// Kimi names no tier word anywhere, so every row is UNRANKED and the tier term cancels. Serving the newest
|
|
222
|
-
// of what it does publish is the honest answer
|
|
222
|
+
// of what it does publish is the honest answer: there is no cheaper rung to find.
|
|
223
223
|
expect(["kimi-k2-0711-preview", "kimi-k2-0905-preview"].toSorted(compareCheapestFirst)[0]).toBe("kimi-k2-0905-preview");
|
|
224
224
|
});
|
package/src/model-order.ts
CHANGED
|
@@ -1,35 +1,35 @@
|
|
|
1
|
-
/* HOW A MODEL CATALOG IS ORDERED
|
|
1
|
+
/* HOW A MODEL CATALOG IS ORDERED, one rule for every provider, because only one provider publishes an order
|
|
2
2
|
* worth keeping.
|
|
3
3
|
*
|
|
4
4
|
* Anthropic's REST /v1/models answers newest-first: that IS a provider opinion, and Claude's catalog rides it
|
|
5
|
-
* (claude-models.ts). Every other provider here is read through an OpenAI-compatible /v1/models
|
|
6
|
-
* Gemini and Kimi via the bundled translator
|
|
5
|
+
* (claude-models.ts). Every other provider here is read through an OpenAI-compatible /v1/models. Codex and
|
|
6
|
+
* Gemini and Kimi via the bundled translator, or out of xAI's "Did you mean" rejection, and those
|
|
7
7
|
* endpoints publish a SET, not a ranking: they hand the ids back in whatever order their registry iterates,
|
|
8
8
|
* which in practice is alphabetical. Reading that as a preference is what put "GPT 5.4 Mini" at the head of the
|
|
9
9
|
* Codex group with GPT 5.6 below it, and what made a fresh Codex conversation start on whichever id happened to
|
|
10
|
-
* sort first
|
|
10
|
+
* sort first, models[0] is the provider default.
|
|
11
11
|
*
|
|
12
12
|
* So for those providers the order is DERIVED from the id, out of the only two facts an id reliably carries:
|
|
13
|
-
* which TIER the model is (the adjective) and which RELEASE it is (the numbers). Both are provider-agnostic
|
|
14
|
-
* every vendor names its models the same way
|
|
13
|
+
* which TIER the model is (the adjective) and which RELEASE it is (the numbers). Both are provider-agnostic,
|
|
14
|
+
* every vendor names its models the same way, which is what lets the daemon's four catalog services and the
|
|
15
15
|
* web's picker share one rule instead of each inventing a local one. */
|
|
16
16
|
|
|
17
17
|
// A version-ish segment: digits and dots, optionally prefixed by the vendor's version marker (`4`, `5.1`, `v2`,
|
|
18
18
|
// `k3`, `k2.7`, `20251001`). Kimi is the one provider that fuses the marker with the generation; treating `k3`
|
|
19
19
|
// as a name made the current flagship look unversioned, so K2.x sorted above it. Everything else is a NAME
|
|
20
|
-
// segment and belongs to the family
|
|
20
|
+
// segment and belongs to the family, which is what makes the split below exhaustive.
|
|
21
21
|
const VERSION_SEGMENT = /^(?:v|k)?[\d.]+$/i;
|
|
22
22
|
|
|
23
23
|
// A date stamp rather than a version component: six digits or more (20251001, 250514). The distinction is not
|
|
24
|
-
// cosmetic
|
|
25
|
-
// with the stamps held apart, and as (4,1,20250805) vs (4,20250514)
|
|
24
|
+
// cosmetic, claude-opus-4-1-20250805 (Opus 4.1) and claude-opus-4-20250514 (Opus 4.0) compare as (4,1) vs (4)
|
|
25
|
+
// with the stamps held apart, and as (4,1,20250805) vs (4,20250514), the OLDER model winning, without.
|
|
26
26
|
const DATE_SEGMENT = /^\d{6,}$/;
|
|
27
27
|
|
|
28
28
|
const segmentsOf = (id: string): string[] => id.split(/[-_]/);
|
|
29
29
|
|
|
30
|
-
// A model's FAMILY
|
|
30
|
+
// A model's FAMILY, its id with every version-ish segment dropped, so claude-opus-5 and claude-opus-4-8 land
|
|
31
31
|
// together (as do gpt-5.1/gpt-5, and claude-haiku-4-5-20251001 with its date suffix). Derived, never listed: a
|
|
32
|
-
// family that ships tomorrow groups itself. The id is the stable key here
|
|
32
|
+
// family that ships tomorrow groups itself. The id is the stable key here, labels get renamed, ids don't.
|
|
33
33
|
export const familyOf = (id: string): string => {
|
|
34
34
|
const stem = segmentsOf(id)
|
|
35
35
|
.filter((segment) => !VERSION_SEGMENT.test(segment))
|
|
@@ -43,7 +43,7 @@ export interface ModelRelease {
|
|
|
43
43
|
// id (kimi-latest, gemini-pro-agent), which therefore reads as the oldest of its tier: a rolling alias names
|
|
44
44
|
// no release, and inventing one for it would outrank the models that do name theirs.
|
|
45
45
|
readonly version: readonly number[];
|
|
46
|
-
// The id's date stamp, 0 for none
|
|
46
|
+
// The id's date stamp, 0 for none, the tiebreak between two builds of the SAME version.
|
|
47
47
|
readonly date: number;
|
|
48
48
|
}
|
|
49
49
|
|
|
@@ -75,14 +75,14 @@ const compareRelease = (left: ModelRelease, right: ModelRelease): number => {
|
|
|
75
75
|
|
|
76
76
|
/* THE ONE CURATED FACT in this file, and the only one the providers publish nowhere the app can read: which tier
|
|
77
77
|
* is the frontier and which is the cheap one. It ranks FAMILIES, never models, and it is a vocabulary of tier
|
|
78
|
-
* ADJECTIVES rather than a table of ids
|
|
78
|
+
* ADJECTIVES rather than a table of ids, that scoping is the whole point, because a per-model ranking table
|
|
79
79
|
* failed here once already. The words are the ones every vendor reaches for, so a release that ships tomorrow
|
|
80
80
|
* ranks itself as long as it is named like its predecessors, and a release named some other way ranks as unknown.
|
|
81
81
|
*
|
|
82
82
|
* An UNKNOWN family LEADS rather than sinks, and that direction is the point: the ranking this replaced sank
|
|
83
83
|
* unrecognized ids to a floor below the everyday tier, so a brand-new flagship sorted beneath the model it
|
|
84
|
-
* replaced. An id carrying no tier word at all is the provider's BASE line (gpt-5.6, grok-4, kimi-k2)
|
|
85
|
-
* exactly the line a user reaches for
|
|
84
|
+
* replaced. An id carrying no tier word at all is the provider's BASE line (gpt-5.6, grok-4, kimi-k2), which is
|
|
85
|
+
* exactly the line a user reaches for, and a family nobody here has heard of is far likelier to be the next
|
|
86
86
|
* flagship than the next budget tier. Being wrong costs one row's position; being wrong the other way hides a
|
|
87
87
|
* launch. */
|
|
88
88
|
const TIER_RANK: Readonly<Record<string, number>> = {
|
|
@@ -113,7 +113,7 @@ const UNRANKED = -1;
|
|
|
113
113
|
|
|
114
114
|
/* Some providers name a capability ladder INSIDE one release instead of using the cross-release adjectives
|
|
115
115
|
* above. Codex 5.6's Sol/Terra/Luna rows are that shape: they must remain together ahead of the older 5.5 line,
|
|
116
|
-
* but their order is not an arbitrary id tiebreak
|
|
116
|
+
* but their order is not an arbitrary id tiebreak. Sol is the strongest, followed by Terra, then Luna. Keeping
|
|
117
117
|
* this as a separate rank lets release recency still win across generations (a future GPT 5.7 base model must
|
|
118
118
|
* not be buried under a recognized 5.6 suffix), while the three siblings sort by their real tier. */
|
|
119
119
|
const RELEASE_TIER_RANK: Readonly<Record<string, number>> = {
|
|
@@ -135,23 +135,23 @@ const lastRankOf = (family: string, ranks: Readonly<Record<string, number>>): nu
|
|
|
135
135
|
|
|
136
136
|
const releaseTierRankOf = (family: string): number => lastRankOf(family, RELEASE_TIER_RANK);
|
|
137
137
|
|
|
138
|
-
/* HOW HARD AN ID SAYS IT WILL THINK. A routed catalog does not publish one row per model
|
|
138
|
+
/* HOW HARD AN ID SAYS IT WILL THINK. A routed catalog does not publish one row per model, it publishes one row
|
|
139
139
|
* per model PER THINKING LEVEL, spelling the level into the id: `gemini-3.6-flash-high` and
|
|
140
140
|
* `gemini-3.5-flash-extra-low` are the same Flash at opposite ends of its reasoning budget.
|
|
141
141
|
*
|
|
142
142
|
* Which the ranking above cannot see at all, and that blind spot has a direction: `high` and `low` are both
|
|
143
|
-
* unrecognized words, so two variants of one model tie on tier and the RELEASE tiebreak settles them
|
|
143
|
+
* unrecognized words, so two variants of one model tie on tier and the RELEASE tiebreak settles them, and the
|
|
144
144
|
* newest variant a channel publishes is routinely the high one. The quick model, whose entire job is to be the
|
|
145
145
|
* cheap rung, would therefore reach for the most expensive reading of the cheapest model it can find.
|
|
146
146
|
*
|
|
147
147
|
* That is not a small mis-sort. Thinking is the difference between a commit message that is in the box before
|
|
148
|
-
* the user has finished reading the file list and one that arrives half a minute later
|
|
148
|
+
* the user has finished reading the file list and one that arrives half a minute later, measured at ~2s versus
|
|
149
149
|
* ~27s on the same model and the same diff (agent/one-shot.ts, which disables thinking for the rungs where a
|
|
150
150
|
* request parameter can). For a routed rung there is no such parameter: the id IS the setting, so this ranking
|
|
151
151
|
* is the only place the choice can be made.
|
|
152
152
|
*
|
|
153
153
|
* Read ONLY by the cheap-end order. A picker orders a catalog by what a person reaches for, and a person
|
|
154
|
-
* pinning `-high` on purpose means it
|
|
154
|
+
* pinning `-high` on purpose means it, see compareCheapestFirst for the seam this belongs to. */
|
|
155
155
|
const THINKING_RANK: Readonly<Record<string, number>> = {
|
|
156
156
|
minimal: 0,
|
|
157
157
|
none: 0,
|
|
@@ -164,7 +164,7 @@ const THINKING_RANK: Readonly<Record<string, number>> = {
|
|
|
164
164
|
thinking: 4,
|
|
165
165
|
};
|
|
166
166
|
|
|
167
|
-
/* An id naming NO level sits between the two ends rather than at either
|
|
167
|
+
/* An id naming NO level sits between the two ends rather than at either, the provider's own default, which for
|
|
168
168
|
* a model that can think is usually some thinking. Ranking it cheapest would seat a silent default ahead of an
|
|
169
169
|
* id that explicitly says `minimal`, and ranking it dearest would bury every model whose channel publishes no
|
|
170
170
|
* variants at all (Anthropic's, Kimi's) beneath one that does. Neither is what the id claims. */
|
|
@@ -180,7 +180,7 @@ const thinkingRankOf = (family: string): number => {
|
|
|
180
180
|
export const tierRankOf = (family: string): number => lastRankOf(family, TIER_RANK);
|
|
181
181
|
|
|
182
182
|
// The canonical order of two model ids: broad tier first, then release, then a tier declared within that release.
|
|
183
|
-
// Hand it straight to Array#toSorted
|
|
183
|
+
// Hand it straight to Array#toSorted, that sort is stable, so two ids this rule cannot separate keep the order
|
|
184
184
|
// they arrived in (for Claude, the provider's own).
|
|
185
185
|
export const compareModelIds = (left: string, right: string): number => {
|
|
186
186
|
const leftFamily = familyOf(left);
|
|
@@ -192,28 +192,28 @@ export const compareModelIds = (left: string, right: string): number => {
|
|
|
192
192
|
);
|
|
193
193
|
};
|
|
194
194
|
|
|
195
|
-
/* The order for a catalog its endpoint published as a SET
|
|
195
|
+
/* The order for a catalog its endpoint published as a SET. Codex, Gemini, Kimi and Grok, i.e. everything but
|
|
196
196
|
* Anthropic's ranked list. Falling back on arrival order is what the rule above does with a tie, and for a RANKED
|
|
197
197
|
* catalog that is exactly right: the tie is the provider's own opinion, so claude-opus-5 stays ahead of
|
|
198
198
|
* claude-fable-5. For a set there is no opinion to keep, and the header of this file assumed the leftover order
|
|
199
|
-
* was at least alphabetical
|
|
199
|
+
* was at least alphabetical, it is not. A subscription can hand tied rows back in whatever order its registry
|
|
200
200
|
* iterated THIS request, so the tie decides which model a fresh conversation opens on and can flip between
|
|
201
201
|
* catalog refreshes.
|
|
202
202
|
*
|
|
203
|
-
* So a set breaks its own ties on the id. Which sibling that seats first is arbitrary
|
|
203
|
+
* So a set breaks its own ties on the id. Which sibling that seats first is arbitrary, but it is the same
|
|
204
204
|
* arbitrary answer every refresh, which is the property `default` actually needs. */
|
|
205
205
|
export const compareUnrankedModelIds = (left: string, right: string): number => compareModelIds(left, right) || left.localeCompare(right);
|
|
206
206
|
|
|
207
207
|
/* THE SAME TIER SCALE READ FROM THE OTHER END, for the one caller that wants the WEAKEST model rather than the
|
|
208
208
|
* strongest: the quick model behind an automatic helper (the commit message written at land time). A picker
|
|
209
209
|
* orders a catalog by what a user reaches for; this orders it by what a helper should spend, and the two are
|
|
210
|
-
* exact opposites
|
|
210
|
+
* exact opposites, so they share TIER_RANK rather than each naming its own list of cheap ids.
|
|
211
211
|
*
|
|
212
212
|
* The direction of UNRANKED is the reason this can't just be compareModelIds reversed. There, an unrecognized
|
|
213
213
|
* family LEADS, because an id carrying no tier word is the provider's base line and a family nobody here has
|
|
214
214
|
* heard of is likelier the next flagship than the next budget tier. Reversing would therefore seat exactly that
|
|
215
|
-
* unknown-probably-flagship id as the cheap pick. So unknown sinks to LAST here too
|
|
216
|
-
* not the efficient rung
|
|
215
|
+
* unknown-probably-flagship id as the cheap pick. So unknown sinks to LAST here too, both orders agree it is
|
|
216
|
+
* not the efficient rung, and the cheap end is only ever a family whose tier word is actually recognized.
|
|
217
217
|
* Falling off the end of a catalog with no efficient tier at all (Kimi publishes none) is then honest: the
|
|
218
218
|
* newest of what it does publish, chosen by the release tiebreak below. */
|
|
219
219
|
export const compareCheapestFirst = (left: string, right: string): number => {
|
|
@@ -223,14 +223,36 @@ export const compareCheapestFirst = (left: string, right: string): number => {
|
|
|
223
223
|
tierRankOf(rightFamily) - tierRankOf(leftFamily) ||
|
|
224
224
|
// BEFORE release, and that placement is the point: two rows of one model differing only in thinking
|
|
225
225
|
// level are the same model, so recency has nothing to say between them, and letting it speak is what
|
|
226
|
-
// seated the high variant. Tier still wins over both
|
|
226
|
+
// seated the high variant. Tier still wins over both, a thinking Haiku is cheaper than a silent Opus.
|
|
227
227
|
thinkingRankOf(leftFamily) - thinkingRankOf(rightFamily) ||
|
|
228
228
|
compareRelease(releaseOf(left), releaseOf(right)) ||
|
|
229
229
|
releaseTierRankOf(rightFamily) - releaseTierRankOf(leftFamily)
|
|
230
230
|
);
|
|
231
231
|
};
|
|
232
232
|
|
|
233
|
-
/*
|
|
233
|
+
/* IS `candidate` A STRICTLY CHEAPER RUNG THAN `pick`, on the tier ladder alone. The predicate automatic tier
|
|
234
|
+
* selection runs on (prompt-complexity.ts judges the turn, fast-tier.ts spends this to find something cheaper
|
|
235
|
+
* to run it on), and it lives here because it is a question about the tier vocabulary, which is this file's
|
|
236
|
+
* one curated fact.
|
|
237
|
+
*
|
|
238
|
+
* TIER ONLY, deliberately: not release, not thinking level. A downgrade has to be legible as a downgrade, and
|
|
239
|
+
* "the same model one release older" or "the same model with less thinking" are not the saving this mechanism
|
|
240
|
+
* promises, they are a different turn wearing the user's model name. Haiku under Sonnet is; Sonnet 4 under
|
|
241
|
+
* Sonnet 5 is not.
|
|
242
|
+
*
|
|
243
|
+
* AN UNRANKED FAMILY IS FALSE ON EITHER SIDE, and both directions matter. A candidate carrying no tier word is
|
|
244
|
+
* a provider's base line or a family nobody here has heard of, so calling it the budget option is a guess, and
|
|
245
|
+
* the scale already says the likelier reading is "next flagship". The PICK being unranked is the subtler half:
|
|
246
|
+
* the safety argument for automatic tier selection is that it can only ever route DOWN, and against an id whose
|
|
247
|
+
* tier is unknown nothing can be shown to be down. So an unrecognised pick is left alone rather than swapped
|
|
248
|
+
* for something merely known to be cheap, which is how a saving turns into a bill. */
|
|
249
|
+
export const isCheaperRung = (candidate: string, pick: string): boolean => {
|
|
250
|
+
const candidateRank = tierRankOf(familyOf(candidate));
|
|
251
|
+
const pickRank = tierRankOf(familyOf(pick));
|
|
252
|
+
return candidateRank !== UNRANKED && pickRank !== UNRANKED && candidateRank > pickRank;
|
|
253
|
+
};
|
|
254
|
+
|
|
255
|
+
/* WOULD RUNNING THIS ID MAKE THE MODEL THINK, as far as its name admits, which for a routed catalog is as far
|
|
234
256
|
* as anyone can tell without running it. True only for an id that spells out a level ABOVE the quiet end, so an
|
|
235
257
|
* ordinary id nobody has annotated (claude-haiku-4-5, kimi-k2) is never accused of it.
|
|
236
258
|
*
|
package/src/output-fields.ts
CHANGED
|
@@ -1,23 +1,23 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
2
|
|
|
3
|
-
/* A DECLARED OUTPUT SHAPE
|
|
3
|
+
/* A DECLARED OUTPUT SHAPE, the answer to "what does this session produce?", written once and used three ways.
|
|
4
4
|
*
|
|
5
5
|
* An agentic session's natural output is prose, and prose is unusable as an input to the next session: a step
|
|
6
6
|
* that must hand "the three files worth changing" to the step after it cannot hand over a paragraph that
|
|
7
7
|
* mentions them. So a session that feeds another one declares its shape here, and that one declaration
|
|
8
8
|
* becomes: the sentence in the prompt that tells the model what to write, the validator that decides whether
|
|
9
9
|
* it complied, and the table the run view renders. Written once because the three drift apart the moment they
|
|
10
|
-
* are written twice
|
|
10
|
+
* are written twice, a prompt asking for `files` and a validator wanting `paths` fails on every iteration and
|
|
11
11
|
* says nothing useful about why.
|
|
12
12
|
*
|
|
13
13
|
* WHY A FIELD LIST AND NOT JSON SCHEMA. JSON Schema is strictly more expressive and completely unauthorable in
|
|
14
14
|
* a form: nobody designing a workflow is going to hand-write `{"type":"object","properties":{...}}`, and a UI
|
|
15
15
|
* that generates it becomes a schema editor, which is a product of its own. Four scalar types plus a string
|
|
16
|
-
* list covers what one session actually hands another
|
|
16
|
+
* list covers what one session actually hands another, a verdict, a count, a list of paths, a summary, and
|
|
17
17
|
* anything past that is better carried as a file the next step reads.
|
|
18
18
|
*
|
|
19
19
|
* `description` IS REQUIRED, and that is the field that decides whether this works at all. `{name: "risk"}`
|
|
20
|
-
* gets you the model's guess at what risk means; `{name: "risk", description: "high | medium | low
|
|
20
|
+
* gets you the model's guess at what risk means; `{name: "risk", description: "high | medium | low, how
|
|
21
21
|
* likely this change is to break something at runtime"}` gets you the answer to the question you asked.
|
|
22
22
|
*/
|
|
23
23
|
|
|
@@ -28,14 +28,14 @@ const FIELD_NAME = /^[a-zA-Z_][a-zA-Z0-9_]{0,39}$/;
|
|
|
28
28
|
export const OutputFieldSchema = z.object({
|
|
29
29
|
name: z.string().regex(FIELD_NAME),
|
|
30
30
|
type: z.enum(["string", "number", "boolean", "string[]"]),
|
|
31
|
-
// What the field means, in the words the model is given. Not optional
|
|
31
|
+
// What the field means, in the words the model is given. Not optional, see the note above.
|
|
32
32
|
description: z.string().min(1),
|
|
33
33
|
// An absent optional field validates; an absent required one does not, and the iteration is told which.
|
|
34
34
|
required: z.boolean(),
|
|
35
35
|
});
|
|
36
36
|
export type OutputField = z.infer<typeof OutputFieldSchema>;
|
|
37
37
|
|
|
38
|
-
// How many fields one output may declare. A shape past this is not a handoff, it is a report
|
|
38
|
+
// How many fields one output may declare. A shape past this is not a handoff, it is a report, and a report is
|
|
39
39
|
// what the prose half of the document is for.
|
|
40
40
|
export const OUTPUT_FIELDS_MAX = 16;
|
|
41
41
|
|
|
@@ -81,18 +81,18 @@ const validatorFor = (field: OutputField): z.ZodType => {
|
|
|
81
81
|
/* The declared shape as a validator. Unknown keys are ALLOWED THROUGH: a model that answered everything asked
|
|
82
82
|
* of it and then added a `notes` key has complied, and failing it there would burn an iteration teaching it to
|
|
83
83
|
* write less. What is enforced is that every required field is present and every present field has the
|
|
84
|
-
* declared type
|
|
84
|
+
* declared type, the two things the reader downstream is entitled to assume.
|
|
85
85
|
*/
|
|
86
86
|
export const fieldsValidator = (fields: readonly OutputField[]): z.ZodType =>
|
|
87
87
|
z.looseObject(Object.fromEntries(fields.map((field) => [field.name, field.required ? validatorFor(field) : validatorFor(field).optional()])));
|
|
88
88
|
|
|
89
89
|
// A worked example of the declared shape, so the prompt can show rather than describe. Values are the field's
|
|
90
|
-
// own description
|
|
90
|
+
// own description, a model copying the example's structure has the description in front of it as it fills each
|
|
91
91
|
// slot, which is where it is needed rather than in a legend three lines up.
|
|
92
92
|
export const fieldsExample = (fields: readonly OutputField[]): Record<string, unknown> =>
|
|
93
93
|
Object.fromEntries(
|
|
94
94
|
fields.map((field) => {
|
|
95
|
-
const hint = `${field.description}${field.required ? "" : " (optional
|
|
95
|
+
const hint = `${field.description}${field.required ? "" : " (optional, omit if it does not apply)"}`;
|
|
96
96
|
if (field.type === "number") {
|
|
97
97
|
return [field.name, 0];
|
|
98
98
|
}
|
|
@@ -106,6 +106,6 @@ export const fieldsExample = (fields: readonly OutputField[]): Record<string, un
|
|
|
106
106
|
}),
|
|
107
107
|
);
|
|
108
108
|
|
|
109
|
-
// One line per field, for surfaces with no room to render an example: "risk (string, required)
|
|
109
|
+
// One line per field, for surfaces with no room to render an example: "risk (string, required), how likely …".
|
|
110
110
|
export const describeFields = (fields: readonly OutputField[]): string =>
|
|
111
|
-
fields.map((field) => `- \`${field.name}\` (${field.type}${field.required ? ", required" : ", optional"})
|
|
111
|
+
fields.map((field) => `- \`${field.name}\` (${field.type}${field.required ? ", required" : ", optional"}), ${field.description}`).join(`\n`);
|
package/src/path-refs.test.ts
CHANGED
|
@@ -15,7 +15,7 @@ describe("referenceTails", () => {
|
|
|
15
15
|
expect(referenceTails("/history/worktrees/agent-7/_apps/foo.ts", "/work")).toContain("_apps/foo.ts");
|
|
16
16
|
});
|
|
17
17
|
|
|
18
|
-
test("never cuts down to a bare filename
|
|
18
|
+
test("never cuts down to a bare filename: one `index.ts` is as good as another", () => {
|
|
19
19
|
expect(referenceTails("a/b/index.ts", "/work").at(-1)).toBe("b/index.ts");
|
|
20
20
|
expect(referenceTails("index.ts", "/work")).toEqual([]);
|
|
21
21
|
});
|
|
@@ -28,11 +28,11 @@ describe("referenceTails", () => {
|
|
|
28
28
|
|
|
29
29
|
describe("rankRefCandidates", () => {
|
|
30
30
|
test("keeps only matches that end in the tail on a segment boundary", () => {
|
|
31
|
-
// `mypages/` merely ends with the same characters
|
|
31
|
+
// `mypages/` merely ends with the same characters: the daemon's glob can't tell, so this must.
|
|
32
32
|
expect(rankRefCandidates("pages/Foo.vue", ["app/mypages/Foo.vue", "app/pages/Foo.vue"])).toEqual(["app/pages/Foo.vue"]);
|
|
33
33
|
});
|
|
34
34
|
|
|
35
|
-
test("ranks the shallowest match first
|
|
35
|
+
test("ranks the shallowest match first: the app's file, not a copy in a fixture tree", () => {
|
|
36
36
|
expect(rankRefCandidates("pages/Foo.vue", ["a/b/c/pages/Foo.vue", "a/pages/Foo.vue"])).toEqual(["a/pages/Foo.vue", "a/b/c/pages/Foo.vue"]);
|
|
37
37
|
});
|
|
38
38
|
|
|
@@ -41,7 +41,7 @@ describe("rankRefCandidates", () => {
|
|
|
41
41
|
});
|
|
42
42
|
});
|
|
43
43
|
|
|
44
|
-
test("isTestPath: test files, fixture dirs and runner configs
|
|
44
|
+
test("isTestPath: test files, fixture dirs and runner configs: never product code that merely says 'test'", () => {
|
|
45
45
|
for (const path of [
|
|
46
46
|
`src/agents/land.test.ts`,
|
|
47
47
|
`src/pages/Foo.spec.tsx`,
|
package/src/path-refs.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/* Which file a NAMED reference means. A path written in prose is only loosely anchored to the workspace: an
|
|
2
2
|
* agent that has been working in `_editor/web/src` writes `pages/workspace/Foo.vue`, and a turn running in an
|
|
3
|
-
* isolated worktree prints `/history/worktrees/<id>/_editor/web/src/foo.ts
|
|
3
|
+
* isolated worktree prints `/history/worktrees/<id>/_editor/web/src/foo.ts`, neither is the workspace-relative
|
|
4
4
|
* path the file routes speak, but both END in it.
|
|
5
5
|
*
|
|
6
6
|
* So a reference is resolved by matching progressively shorter TAILS of it against the real tree. The rules
|
|
@@ -32,17 +32,17 @@ export const referenceTails = (raw: string, root: string): readonly string[] =>
|
|
|
32
32
|
return tails;
|
|
33
33
|
};
|
|
34
34
|
|
|
35
|
-
// The paths that genuinely END in `tail` on a segment boundary, best first
|
|
35
|
+
// The paths that genuinely END in `tail` on a segment boundary, best first, the shared ranking both matchers
|
|
36
36
|
// return their candidates in. Shallowest wins: `pages/Foo.vue` means the app's page, not the copy six
|
|
37
|
-
// directories down in a fixture tree. (The daemon's glob is anchored only at the string level
|
|
38
|
-
// also matches `mypages/x.vue
|
|
37
|
+
// directories down in a fixture tree. (The daemon's glob is anchored only at the string level, `**/pages/x.vue`
|
|
38
|
+
// also matches `mypages/x.vue`, so the boundary is enforced here rather than by the pattern.)
|
|
39
39
|
export const rankRefCandidates = (tail: string, paths: readonly string[]): readonly string[] =>
|
|
40
40
|
paths
|
|
41
41
|
.filter((path) => path === tail || path.endsWith(`/${tail}`))
|
|
42
42
|
.toSorted((a, b) => a.split(`/`).length - b.split(`/`).length || a.length - b.length || (a < b ? -1 : 1))
|
|
43
43
|
.slice(0, MAX_REF_CANDIDATES);
|
|
44
44
|
|
|
45
|
-
/* IS THIS FILE TEST CODE
|
|
45
|
+
/* IS THIS FILE TEST CODE, the one classification rule for every surface that splits a diff into "the
|
|
46
46
|
* change" and "the proof". The agent review header answers "how much of this is tests?" with it; anything
|
|
47
47
|
* else that wants the split (fleet cards, commit summaries) must use this same predicate, because two
|
|
48
48
|
* classifiers that disagree turn the readout into a lie the user can't detect.
|
|
@@ -50,7 +50,7 @@ export const rankRefCandidates = (tail: string, paths: readonly string[]): reado
|
|
|
50
50
|
* Convention-based, matching what this monorepo (and the ecosystems it scaffolds) actually writes: a
|
|
51
51
|
* `.test.` / `.spec.` filename in any extension, a `__tests__` / `__fixtures__` directory anywhere on the
|
|
52
52
|
* path, an `e2e-harness`, or a test-runner config. Deliberately NOT "anything containing 'test'": a
|
|
53
|
-
* `testimonials/` page or a `latest.ts` is product code, and a false "tests" tag is worse than a missed one
|
|
53
|
+
* `testimonials/` page or a `latest.ts` is product code, and a false "tests" tag is worse than a missed one,
|
|
54
54
|
* it tells a reviewer not to look. */
|
|
55
55
|
const TEST_DIRS = new Set([`__tests__`, `__fixtures__`, `__mocks__`, `__snapshots__`]);
|
|
56
56
|
const TEST_FILE =
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
import { expect, test } from "vitest";
|
|
2
|
+
import { type ComplexityInput, FAST_CEILING, judgeComplexity } from "./prompt-complexity.js";
|
|
3
|
+
|
|
4
|
+
/* WHETHER A TURN COULD HAVE RUN ON THE CHEAP RUNG, judged before anything is spent.
|
|
5
|
+
*
|
|
6
|
+
* What these tests pin is the ASYMMETRY, not the accuracy: the judge can only ever route down, so every one of
|
|
7
|
+
* its mistakes in the "standard" direction costs a fraction of a cent and every mistake in the "fast" direction
|
|
8
|
+
* costs a user their turn. So the properties worth nailing down are that silence resolves to standard, that a
|
|
9
|
+
* gate or an escalating rule ends the question whatever else the sentence says, and that no rule ORDER can
|
|
10
|
+
* change an answer. The weights themselves are a hypothesis with a ledger under it (settings.autoTier
|
|
11
|
+
* "shadow"), and a test that froze them would be a test that made them impossible to fit. */
|
|
12
|
+
|
|
13
|
+
const turn = (prompt: string, over: Partial<ComplexityInput> = {}): ComplexityInput => ({
|
|
14
|
+
prompt,
|
|
15
|
+
attachments: 0,
|
|
16
|
+
hasImages: false,
|
|
17
|
+
editorContext: false,
|
|
18
|
+
unattended: false,
|
|
19
|
+
planMode: false,
|
|
20
|
+
afterHardTurn: false,
|
|
21
|
+
...over,
|
|
22
|
+
});
|
|
23
|
+
|
|
24
|
+
const tierOf = (prompt: string, over: Partial<ComplexityInput> = {}) => judgeComplexity(turn(prompt, over)).tier;
|
|
25
|
+
|
|
26
|
+
// --- silence, and what it is read as -----------------------------------------------------------------
|
|
27
|
+
|
|
28
|
+
test("a prompt matching nothing at all stays on the model the user picked", () => {
|
|
29
|
+
// The single most important default in the file. An unrecognised request is MEDIUM, never simple: the
|
|
30
|
+
// conservative reading of silence is the only safe one when a wrong downgrade costs a retry and a
|
|
31
|
+
// user who stops trusting the feature.
|
|
32
|
+
const verdict = judgeComplexity(turn(`Have another go at the thing we were discussing yesterday afternoon`));
|
|
33
|
+
|
|
34
|
+
expect(verdict.tier).toBe(`standard`);
|
|
35
|
+
expect(verdict.score).toBeGreaterThan(FAST_CEILING);
|
|
36
|
+
// Nothing in the sentence made a positive claim in either direction, so only the two absence features
|
|
37
|
+
// fired, and the pair of them cannot reach the ceiling by design.
|
|
38
|
+
expect(verdict.rules).toEqual([`short-prompt`, `no-workspace-reference`]);
|
|
39
|
+
});
|
|
40
|
+
|
|
41
|
+
test("a short vague request is not read as an easy one", () => {
|
|
42
|
+
// Absence of complexity is not evidence of simplicity. Weighted the obvious way, "short and naming no
|
|
43
|
+
// file" reached the ceiling by itself and downgraded every terse request in the product.
|
|
44
|
+
expect(tierOf(`fix the bug`)).toBe(`standard`);
|
|
45
|
+
expect(tierOf(`have a look at the thing we discussed`)).toBe(`standard`);
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
test("an empty-ish prompt is not mistaken for an easy one", () => {
|
|
49
|
+
expect(tierOf(` `)).toBe(`standard`);
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
// --- the fast end ------------------------------------------------------------------------------------
|
|
53
|
+
|
|
54
|
+
test("a short question about nothing in the workspace is the case this feature exists for", () => {
|
|
55
|
+
expect(tierOf(`what is a closure?`)).toBe(`fast`);
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
test("naming a file keeps an otherwise easy question on the user's own model", () => {
|
|
59
|
+
// "explain" is the easiest word in the lexicon, but the turn is now about real code in this repo rather
|
|
60
|
+
// than about a concept, and the cheap rung's failures on real code are the silent kind.
|
|
61
|
+
expect(tierOf(`explain what is a closure`)).toBe(`fast`);
|
|
62
|
+
expect(tierOf(`explain what src/agent/turn-plan.ts does`)).toBe(`standard`);
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
test("a trivial aside inside a hard conversation still gets through", () => {
|
|
66
|
+
// The reason afterHardTurn is a weight and not a lock: a conversation that has been doing hard work is
|
|
67
|
+
// still allowed to be asked an easy question, and locking it out is a mechanism that saves nothing.
|
|
68
|
+
expect(tierOf(`what is a closure?`, { afterHardTurn: true })).toBe(`fast`);
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
test("a workspace-adjacent errand stops being cheap once the conversation has done hard work", () => {
|
|
72
|
+
// The contrast with the test above is the whole rule. A pure knowledge question survives the penalty
|
|
73
|
+
// because it earns every easing feature there is; an errand about this repo does not, and after hard work
|
|
74
|
+
// it is far likelier to be the deceptive follow-up than a genuine aside.
|
|
75
|
+
expect(tierOf(`list the exports`, { afterHardTurn: false })).toBe(`fast`);
|
|
76
|
+
expect(tierOf(`list the exports`, { afterHardTurn: true })).toBe(`standard`);
|
|
77
|
+
});
|
|
78
|
+
|
|
79
|
+
// --- gates: the turn's situation, whatever its words say ---------------------------------------------
|
|
80
|
+
|
|
81
|
+
test("a screenshot is never sent to the cheap rung, however simple the question about it", () => {
|
|
82
|
+
// The tier most likely to misread an image, on the turn least likely to notice that it did.
|
|
83
|
+
expect(tierOf(`what is this?`, { hasImages: true, attachments: 1 })).toBe(`standard`);
|
|
84
|
+
});
|
|
85
|
+
|
|
86
|
+
test("plan mode is a request to think, so it is never answered by the model that thinks least", () => {
|
|
87
|
+
expect(tierOf(`what is a closure?`, { planMode: true })).toBe(`standard`);
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
test("a surface-started run is never downgraded, because nobody is watching it fail", () => {
|
|
91
|
+
// Same call agentRunModels already makes in the other direction: a run billed whole, with a worktree in
|
|
92
|
+
// it, is not the place to spend a guess.
|
|
93
|
+
expect(tierOf(`what is a closure?`, { unattended: true })).toBe(`standard`);
|
|
94
|
+
});
|
|
95
|
+
|
|
96
|
+
test("a gate reports itself and scores 1, so the ledger can tell a gate from a hard sentence", () => {
|
|
97
|
+
const verdict = judgeComplexity(turn(`hi`, { unattended: true }));
|
|
98
|
+
|
|
99
|
+
expect(verdict.score).toBe(1);
|
|
100
|
+
expect(verdict.rules).toContain(`unattended`);
|
|
101
|
+
});
|
|
102
|
+
|
|
103
|
+
// --- escalating rules: any one ends the question ------------------------------------------------------
|
|
104
|
+
|
|
105
|
+
test.each([
|
|
106
|
+
[`pasted code`, "explain this\n```ts\nconst x = 1;\n```"],
|
|
107
|
+
[`a stack trace`, "it broke\n at Object.run (/work/x.ts:12:3)"],
|
|
108
|
+
[`a thrown error`, "help\nTypeError: cannot read properties of undefined"],
|
|
109
|
+
[`a hard word`, `why does the picker reset`],
|
|
110
|
+
[`another hard word`, `refactor this`],
|
|
111
|
+
[`a second job`, `rename it and then update the tests`],
|
|
112
|
+
[`a checklist`, `- rename it\n- update the tests`],
|
|
113
|
+
[`a cross-cutting scope`, `rename it across the codebase`],
|
|
114
|
+
])("%s forces the user's own model even in an otherwise tiny prompt", (_name, prompt) => {
|
|
115
|
+
expect(tierOf(prompt)).toBe(`standard`);
|
|
116
|
+
});
|
|
117
|
+
|
|
118
|
+
test("an escalating rule beats every easing feature at once, so rule order cannot change an answer", () => {
|
|
119
|
+
// Short, easy-worded, no workspace reference, a bare question: every negative weight in the file, plus one
|
|
120
|
+
// escalating rule. Monotone escalation means the rule wins, which is what makes adding a rule tomorrow a
|
|
121
|
+
// safe change: it can only ever move turns UP a tier.
|
|
122
|
+
const verdict = judgeComplexity(turn(`what is a race condition?`));
|
|
123
|
+
|
|
124
|
+
expect(verdict.tier).toBe(`standard`);
|
|
125
|
+
expect(verdict.score).toBe(1);
|
|
126
|
+
expect(verdict.rules).toEqual([`hard-words`]);
|
|
127
|
+
});
|
|
128
|
+
|
|
129
|
+
test("a long brief is standard on its length alone, whatever words it happens to use", () => {
|
|
130
|
+
expect(tierOf(`explain `.repeat(400))).toBe(`standard`);
|
|
131
|
+
});
|
|
132
|
+
|
|
133
|
+
test("three files in, the job is about a shape rather than about a file", () => {
|
|
134
|
+
expect(tierOf(`have a look`, { attachments: 3 })).toBe(`standard`);
|
|
135
|
+
expect(tierOf(`have a look`, { attachments: 1 })).toBe(`standard`);
|
|
136
|
+
});
|
|
137
|
+
|
|
138
|
+
// --- the verdict as a record --------------------------------------------------------------------------
|
|
139
|
+
|
|
140
|
+
test("names every rule that fired, because a score alone cannot say which feature did the work", () => {
|
|
141
|
+
// The ledger stores these. Re-fitting the weights against real traffic needs to know WHICH feature moved a
|
|
142
|
+
// turn, not merely that a threshold was crossed.
|
|
143
|
+
const verdict = judgeComplexity(turn(`what is this?`));
|
|
144
|
+
|
|
145
|
+
expect(verdict.rules).toEqual([`short-prompt`, `easy-words`, `bare-question`, `no-workspace-reference`]);
|
|
146
|
+
});
|
|
147
|
+
|
|
148
|
+
test("scores are rounded, so two turns the same rules judged compare equal on the ledger", () => {
|
|
149
|
+
const score = judgeComplexity(turn(`what is this?`)).score;
|
|
150
|
+
|
|
151
|
+
expect(score).toBe(Number(score.toFixed(3)));
|
|
152
|
+
});
|
|
153
|
+
|
|
154
|
+
test("the score never leaves 0..1, so a stored row is always comparable against the ceiling", () => {
|
|
155
|
+
const floor = judgeComplexity(turn(`what is this?`));
|
|
156
|
+
const ceiling = judgeComplexity(turn(`refactor everything`));
|
|
157
|
+
|
|
158
|
+
expect(floor.score).toBeGreaterThanOrEqual(0);
|
|
159
|
+
expect(ceiling.score).toBeLessThanOrEqual(1);
|
|
160
|
+
});
|