@crossworks/client-types 0.400.34 → 0.400.35

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@crossworks/client-types",
3
- "version": "0.400.34",
3
+ "version": "0.400.35",
4
4
  "type": "module",
5
5
  "main": "./src/index.ts",
6
6
  "types": "./src/index.ts",
@@ -0,0 +1,155 @@
1
+ /**
2
+ * Decision model profiles: what we measured about each OpenRouter decision
3
+ * model, beside what the live catalog already says (context size, input
4
+ * types, price). The catalog cannot say whether a model takes Mantle's
5
+ * request at all, at what privacy level, or how it bills, so this table does.
6
+ *
7
+ * Source: the 2026-10-30 probe of all 28 models (one model at a time; a small
8
+ * request, then one full-size request of 25 passages, about 19,000 tokens,
9
+ * under the 1.5 s worker limit). One full-size timing per model, so `slow`
10
+ * and `fails_full_size` are a first read, not a verdict. A model that is not
11
+ * here is untested: it may run, and nothing claims it works.
12
+ *
13
+ * Today this feeds the pool's refusal (`decisionModelRefusal`, read by
14
+ * `poolModelIssue`) and the decider pool's re-curation (`poolPosition`). The request planner
15
+ * and the worker screen read it later.
16
+ */
17
+
18
+ export type DecisionModelStatus =
19
+ /** Full-size request answered under 1.5 s with `zdr`. */
20
+ | 'works'
21
+ /** Answers only with privacy `no_training`: `zdr` returns 404. */
22
+ | 'no_training_only'
23
+ /** Answers, but a full-size request took 1.7 to 5.7 s. */
24
+ | 'slow'
25
+ /** A full-size request failed: a timeout, a 16-question limit, a
26
+ * 16,384-token limit or a byte limit. Smaller requests may pass. */
27
+ | 'fails_full_size'
28
+ /** Cannot take Mantle's request: another state shape or no live endpoint.
29
+ * The decider pool refuses it. */
30
+ | 'refused';
31
+
32
+ export type DecisionModelProfile = {
33
+ /** The OpenRouter id, lower case. */
34
+ id: string;
35
+ status: DecisionModelStatus;
36
+ /** The strictest privacy level its OpenRouter route accepts. */
37
+ privacy: 'zdr' | 'no_training';
38
+ /** Bills the state once per question: one search billed about 450,000
39
+ * tokens. */
40
+ perQuestionBilling?: true;
41
+ /** Reported exactly this many input tokens for a larger request: a
42
+ * suspected silent cut. Unsafe until checked. */
43
+ suspectedCutAt?: number;
44
+ /** A start value for passage scoring's threshold where Jev's 1.5 drops
45
+ * right passages. Tune it with eval:route. */
46
+ thresholdStart?: number;
47
+ /** Its place in the shipped decider pool. A product decision (2026-10-30),
48
+ * not a ranking: re-curation offers only these, in this order. */
49
+ poolPosition?: number;
50
+ note?: string;
51
+ };
52
+
53
+ const REFUSED =
54
+ "cannot take Mantle's decision request (another state shape, or no live endpoint; probe 2026-10-30)";
55
+
56
+ export const DECISION_MODEL_PROFILES: readonly DecisionModelProfile[] = [
57
+ // Group A: work today, under 1.5 s.
58
+ {
59
+ id: '~typesafe/jev-latest',
60
+ status: 'works',
61
+ privacy: 'zdr',
62
+ poolPosition: 0,
63
+ note: 'the shipped default',
64
+ },
65
+ { id: 'typesafe/jev-1.14', status: 'works', privacy: 'zdr' },
66
+ { id: 'typesafe/jev-1.13', status: 'works', privacy: 'zdr' },
67
+ {
68
+ id: 'perplexity/pplx-decider-v1.2',
69
+ status: 'works',
70
+ privacy: 'zdr',
71
+ poolPosition: 1,
72
+ note: 'the second choice: tied Jev on quality at about half the cost; slower (median 643 ms)',
73
+ },
74
+ {
75
+ id: 'microsoft/microsoft-decision-1',
76
+ status: 'works',
77
+ privacy: 'zdr',
78
+ note: 'tied Jev on quality; built on Qwen3.5-9B',
79
+ },
80
+ { id: 'nace-ai/drex-v1.5', status: 'works', privacy: 'zdr' },
81
+ {
82
+ id: 'cloudflare/clef-omni',
83
+ status: 'works',
84
+ privacy: 'zdr',
85
+ note: 'ranked passages worse than search alone; do not use',
86
+ },
87
+ // Group B: blocked only by zdr.
88
+ {
89
+ id: 'openai/gpt-6-luna-decisions',
90
+ status: 'no_training_only',
91
+ privacy: 'no_training',
92
+ thresholdStart: 1.0,
93
+ note: 'no zero-retention route on OpenRouter; scored some right passages near 0',
94
+ },
95
+ {
96
+ id: 'respan/span-02',
97
+ status: 'no_training_only',
98
+ privacy: 'no_training',
99
+ perQuestionBilling: true,
100
+ note: 'also 1.6 s on a full-size request',
101
+ },
102
+ // Group C: work, but too slow for 1.5 s.
103
+ { id: 'liquid/d1', status: 'slow', privacy: 'zdr' },
104
+ { id: 'liquid/d1.1', status: 'slow', privacy: 'zdr' },
105
+ { id: 'liquid/d1-ultra', status: 'slow', privacy: 'zdr' },
106
+ { id: 'inception/mercury-decide', status: 'slow', privacy: 'zdr' },
107
+ { id: 'cloudflare/clef', status: 'slow', privacy: 'zdr' },
108
+ { id: 'cloudflare/clef-flash', status: 'slow', privacy: 'zdr', suspectedCutAt: 16_384 },
109
+ {
110
+ id: 'perplexity/pplx-decider-flash-v1.2',
111
+ status: 'slow',
112
+ privacy: 'zdr',
113
+ perQuestionBilling: true,
114
+ },
115
+ { id: 'upstage/solar-decide-flash', status: 'slow', privacy: 'zdr', perQuestionBilling: true },
116
+ { id: 'jaredpalmer/kev-4b', status: 'slow', privacy: 'zdr' },
117
+ // Group D: fail on a full-size request.
118
+ { id: 'upstage/solar-decide', status: 'fails_full_size', privacy: 'zdr' },
119
+ { id: 'nace-ai/drex-v2.0', status: 'fails_full_size', privacy: 'zdr' },
120
+ { id: 'nace-ai/drex-max', status: 'fails_full_size', privacy: 'zdr' },
121
+ { id: 'baseten/joey-1', status: 'fails_full_size', privacy: 'zdr' },
122
+ { id: 'baseten/elephant-1', status: 'fails_full_size', privacy: 'zdr' },
123
+ { id: 'baseten/mammoth-1', status: 'fails_full_size', privacy: 'zdr' },
124
+ { id: 'togethercomputer/tev1-4b-experimental', status: 'fails_full_size', privacy: 'zdr' },
125
+ // Group E: cannot take our request at all.
126
+ { id: 'scaledown/classify-1', status: 'refused', privacy: 'zdr', note: REFUSED },
127
+ { id: 'respan/span-01', status: 'refused', privacy: 'zdr', note: REFUSED },
128
+ { id: 'respan/span-01-lite', status: 'refused', privacy: 'zdr', note: REFUSED },
129
+ { id: 'cloudflare/clairic', status: 'refused', privacy: 'zdr', note: REFUSED },
130
+ ];
131
+
132
+ const BY_ID = new Map(DECISION_MODEL_PROFILES.map((p) => [p.id, p]));
133
+
134
+ /** The profile of a decision model id, or null when it is untested. A
135
+ * variant suffix (`:free`) is the same model, so it shares the profile. */
136
+ export function decisionModelProfile(
137
+ model: string | null | undefined,
138
+ ): DecisionModelProfile | null {
139
+ if (!model) return null;
140
+ const id = model
141
+ .trim()
142
+ .toLowerCase()
143
+ .replace(/:[a-z0-9_-]+$/, '');
144
+ return BY_ID.get(id) ?? null;
145
+ }
146
+
147
+ /** Why the decider pool refuses this model, or null when it does not. */
148
+ export function decisionModelRefusal(model: string | null | undefined): string | null {
149
+ const p = decisionModelProfile(model);
150
+ if (p?.status !== 'refused') return null;
151
+ return (
152
+ `this model ${p.note ?? REFUSED}. ` +
153
+ 'Pick a tested decision model such as ~typesafe/jev-latest or perplexity/pplx-decider-v1.2.'
154
+ );
155
+ }
@@ -2038,5 +2038,26 @@
2038
2038
  },
2039
2039
  "rating": 5,
2040
2040
  "note": "the shipped default for this pool; auto-updating alias — tracks this vendor’s current model"
2041
+ },
2042
+ {
2043
+ "pool": "decider",
2044
+ "position": 1,
2045
+ "name": "Perplexity: Decider V1.2",
2046
+ "vendor": "Perplexity",
2047
+ "routes": [
2048
+ {
2049
+ "provider": "openrouter",
2050
+ "model": "perplexity/pplx-decider-v1.2"
2051
+ }
2052
+ ],
2053
+ "pricing": {
2054
+ "inputPerM": 0.02,
2055
+ "outputPerM": 0,
2056
+ "currency": "USD",
2057
+ "capturedAt": "2026-10-30T00:00:00.000Z",
2058
+ "source": "openrouter"
2059
+ },
2060
+ "rating": 4,
2061
+ "note": "second choice, not the default: tied Jev on quality at about half the cost in the 2026-10-30 test; slower (median 643 ms)"
2041
2062
  }
2042
2063
  ]
@@ -8,6 +8,11 @@
8
8
  import { describe, expect, it } from 'vitest';
9
9
  import { MODEL_POOLS, poolModelIssue, kindFromModalities } from './model-pools';
10
10
  import { CURATED_MODEL_POOLS } from './model-pools-data';
11
+ import {
12
+ DECISION_MODEL_PROFILES,
13
+ decisionModelProfile,
14
+ decisionModelRefusal,
15
+ } from './decision-models';
11
16
 
12
17
  const READER = { input: ['text', 'image'], output: ['text'] };
13
18
  const GENERATOR = { input: ['image', 'text'], output: ['image', 'text'] };
@@ -154,4 +159,82 @@ describe('poolModelIssue: the decider pool', () => {
154
159
  expect(poolModelIssue('decider', null)).toBeNull();
155
160
  expect(poolModelIssue('decider', { input: [], output: [] })).toBeNull();
156
161
  });
162
+
163
+ it('refuses the 4 models that cannot take the request, catalog or not', () => {
164
+ const decisions = { input: ['text'], output: ['decisions'] };
165
+ for (const id of [
166
+ 'scaledown/classify-1',
167
+ 'respan/span-01',
168
+ 'respan/span-01-lite',
169
+ 'respan/span-01-lite:free',
170
+ 'Cloudflare/Clairic',
171
+ ]) {
172
+ expect(poolModelIssue('decider', decisions, id), id).toMatch(/cannot take/);
173
+ expect(poolModelIssue('decider', null, id), id).toMatch(/cannot take/);
174
+ }
175
+ });
176
+
177
+ it('accepts the tested models, the slow ones and an untested id', () => {
178
+ const decisions = { input: ['text'], output: ['decisions'] };
179
+ for (const id of [
180
+ '~typesafe/jev-latest',
181
+ 'perplexity/pplx-decider-v1.2',
182
+ 'openai/gpt-6-luna-decisions',
183
+ 'liquid/d1',
184
+ 'some-vendor/new-decider',
185
+ ]) {
186
+ expect(poolModelIssue('decider', decisions, id), id).toBeNull();
187
+ }
188
+ });
189
+
190
+ it('the refusal is the decider pool only', () => {
191
+ expect(poolModelIssue('summarizer', null, 'respan/span-01')).toBeNull();
192
+ });
193
+ });
194
+
195
+ describe('decision model profiles', () => {
196
+ it('holds the 28 probed models once each, plus the shipped alias', () => {
197
+ const ids = DECISION_MODEL_PROFILES.map((p) => p.id);
198
+ expect(new Set(ids).size).toBe(ids.length);
199
+ expect(ids.filter((id) => !id.startsWith('~'))).toHaveLength(28);
200
+ expect(ids).toContain('~typesafe/jev-latest');
201
+ for (const id of ids) expect(id, id).toBe(id.toLowerCase());
202
+ });
203
+
204
+ it('counts each group as the probe found it', () => {
205
+ const count = (s: string) =>
206
+ DECISION_MODEL_PROFILES.filter((p) => p.status === s && !p.id.startsWith('~')).length;
207
+ expect(count('works')).toBe(6);
208
+ expect(count('no_training_only')).toBe(2);
209
+ expect(count('slow')).toBe(9);
210
+ expect(count('fails_full_size')).toBe(7);
211
+ expect(count('refused')).toBe(4);
212
+ });
213
+
214
+ it('a model that needs no_training says so, and the rest take zdr', () => {
215
+ for (const p of DECISION_MODEL_PROFILES) {
216
+ expect(p.privacy, p.id).toBe(p.status === 'no_training_only' ? 'no_training' : 'zdr');
217
+ }
218
+ });
219
+
220
+ it('looks up by id, case and variant suffix aside, and says nothing of an untested id', () => {
221
+ expect(decisionModelProfile('OpenAI/GPT-6-Luna-Decisions')?.thresholdStart).toBe(1.0);
222
+ expect(decisionModelProfile('respan/span-01-lite:free')?.status).toBe('refused');
223
+ expect(decisionModelProfile('some-vendor/new-decider')).toBeNull();
224
+ expect(decisionModelProfile(null)).toBeNull();
225
+ expect(decisionModelRefusal('typesafe/jev-1.14')).toBeNull();
226
+ });
227
+
228
+ it('every shipped decider template model is profiled as working', () => {
229
+ const shipped = CURATED_MODEL_POOLS.filter((e) => e.pool === 'decider').flatMap((e) =>
230
+ e.routes.map((r) => r.model),
231
+ );
232
+ expect(shipped).toEqual(['~typesafe/jev-latest', 'perplexity/pplx-decider-v1.2']);
233
+ for (const id of shipped) expect(decisionModelProfile(id)?.status, id).toBe('works');
234
+ // The profile's shortlist and the template agree, in order.
235
+ const shortlist = DECISION_MODEL_PROFILES.filter((p) => p.poolPosition !== undefined)
236
+ .sort((a, b) => a.poolPosition! - b.poolPosition!)
237
+ .map((p) => p.id);
238
+ expect(shortlist).toEqual(shipped);
239
+ });
157
240
  });
@@ -10,6 +10,8 @@
10
10
  * not switchable (docs/embeddings.md), so curating it would be a trap.
11
11
  */
12
12
 
13
+ import { decisionModelRefusal } from './decision-models';
14
+
13
15
  export type ModelPoolDef = {
14
16
  id: string;
15
17
  label: string;
@@ -209,9 +211,18 @@ export function kindFromModalities(output: readonly string[]): CatalogKind {
209
211
  export function poolModelIssue(
210
212
  poolId: string,
211
213
  modalities: ModelModalities | null | undefined,
214
+ /** The model id, when the caller has it: the decider pool also refuses a
215
+ * decision model our own probe found cannot take the request. That is our
216
+ * evidence, not the catalog's, so it applies with no catalog too. */
217
+ modelId?: string | null,
212
218
  ): string | null {
213
219
  const pool = POOL_BY_ID.get(poolId);
214
- if (!pool || !modalities) return null;
220
+ if (!pool) return null;
221
+ if (pool.modality.output === 'decisions') {
222
+ const refusal = decisionModelRefusal(modelId);
223
+ if (refusal) return refusal;
224
+ }
225
+ if (!modalities) return null;
215
226
  const want = pool.modality;
216
227
  const outputs = modalities.output ?? [];
217
228
  const inputs = modalities.input ?? [];