@aidriftline/driftline 0.1.14 → 0.1.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/registry/models.json +83 -13
- package/src/cli.js +5 -1
- package/src/index.js +11 -2
- package/src/mcp.js +21 -21
- package/src/migration.js +101 -0
- package/src/providers.js +2 -2
- package/src/registry.js +34 -0
- package/src/report.js +24 -4
- package/src/run.js +26 -11
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@aidriftline/driftline",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.16",
|
|
4
4
|
"description": "Finds OpenAI, Anthropic and Google model IDs in your code and warns before they are retired. CLI, GitHub Action and MCP server.",
|
|
5
5
|
"mcpName": "io.github.AI-Driftline/driftline",
|
|
6
6
|
"type": "module",
|
package/registry/models.json
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
{
|
|
2
|
-
"checked": "2026-11-
|
|
2
|
+
"checked": "2026-11-13",
|
|
3
3
|
"note": "Hand-maintained from the official deprecation pages. Re-check before each release. Entries with a 'kind' other than text are scanned and reported but never drift-tested. gpt-4-1106-preview is listed on OpenAI's page under both 2026-03-26 and 2026-10-23; the later date is used so builds are not failed on a conflicting record. Entries with routes: true keep answering after the date, but the provider sends the requests to the replacement model (Google calls this automatic routing); the 'retires' date is the day routing starts. gemini-3.5-flash and gemini-3.7-flash already route to newer models with no date given, so they are left out.",
|
|
4
4
|
"sources": {
|
|
5
5
|
"openai": "https://developers.openai.com/api/docs/deprecations",
|
|
@@ -151,7 +151,9 @@
|
|
|
151
151
|
"provider": "openai",
|
|
152
152
|
"retires": "2026-12-11",
|
|
153
153
|
"qualifier": "exact",
|
|
154
|
-
"replacement": "gpt-
|
|
154
|
+
"replacement": "gpt-6.1-sol",
|
|
155
|
+
"previous_replacement": "gpt-5.6-sol",
|
|
156
|
+
"replacement_changed": "2026-11-13",
|
|
155
157
|
"announced": "2026-06-11"
|
|
156
158
|
},
|
|
157
159
|
{
|
|
@@ -159,7 +161,9 @@
|
|
|
159
161
|
"provider": "openai",
|
|
160
162
|
"retires": "2026-12-11",
|
|
161
163
|
"qualifier": "exact",
|
|
162
|
-
"replacement": "gpt-
|
|
164
|
+
"replacement": "gpt-6.1-sol",
|
|
165
|
+
"previous_replacement": "gpt-5.6-sol",
|
|
166
|
+
"replacement_changed": "2026-11-13",
|
|
163
167
|
"announced": "2026-06-11"
|
|
164
168
|
},
|
|
165
169
|
{
|
|
@@ -167,7 +171,10 @@
|
|
|
167
171
|
"provider": "openai",
|
|
168
172
|
"retires": "2026-12-11",
|
|
169
173
|
"qualifier": "exact",
|
|
170
|
-
"replacement": "gpt-
|
|
174
|
+
"replacement": "gpt-6.1-sol",
|
|
175
|
+
"replacement_note": "reasoning.mode: pro, in the Responses API",
|
|
176
|
+
"previous_replacement": "gpt-5.6-sol",
|
|
177
|
+
"replacement_changed": "2026-11-13",
|
|
171
178
|
"announced": "2026-06-11"
|
|
172
179
|
},
|
|
173
180
|
{
|
|
@@ -175,7 +182,10 @@
|
|
|
175
182
|
"provider": "openai",
|
|
176
183
|
"retires": "2026-12-11",
|
|
177
184
|
"qualifier": "exact",
|
|
178
|
-
"replacement": "gpt-
|
|
185
|
+
"replacement": "gpt-6.1-sol",
|
|
186
|
+
"replacement_note": "reasoning.mode: pro, in the Responses API",
|
|
187
|
+
"previous_replacement": "gpt-5.6-sol",
|
|
188
|
+
"replacement_changed": "2026-11-13",
|
|
179
189
|
"announced": "2026-06-11"
|
|
180
190
|
},
|
|
181
191
|
{
|
|
@@ -183,7 +193,9 @@
|
|
|
183
193
|
"provider": "openai",
|
|
184
194
|
"retires": "2026-12-11",
|
|
185
195
|
"qualifier": "exact",
|
|
186
|
-
"replacement": "gpt-
|
|
196
|
+
"replacement": "gpt-6.1-sol",
|
|
197
|
+
"previous_replacement": "gpt-5.6-sol",
|
|
198
|
+
"replacement_changed": "2026-11-13",
|
|
187
199
|
"announced": "2026-06-11"
|
|
188
200
|
},
|
|
189
201
|
{
|
|
@@ -191,7 +203,9 @@
|
|
|
191
203
|
"provider": "openai",
|
|
192
204
|
"retires": "2026-12-11",
|
|
193
205
|
"qualifier": "exact",
|
|
194
|
-
"replacement": "gpt-
|
|
206
|
+
"replacement": "gpt-6.1-sol",
|
|
207
|
+
"previous_replacement": "gpt-5.6-sol",
|
|
208
|
+
"replacement_changed": "2026-11-13",
|
|
195
209
|
"announced": "2026-06-11"
|
|
196
210
|
},
|
|
197
211
|
{
|
|
@@ -199,7 +213,10 @@
|
|
|
199
213
|
"provider": "openai",
|
|
200
214
|
"retires": "2026-12-11",
|
|
201
215
|
"qualifier": "exact",
|
|
202
|
-
"replacement": "gpt-
|
|
216
|
+
"replacement": "gpt-6.1-sol",
|
|
217
|
+
"replacement_note": "reasoning.mode: pro, in the Responses API",
|
|
218
|
+
"previous_replacement": "gpt-5.6-sol",
|
|
219
|
+
"replacement_changed": "2026-11-13",
|
|
203
220
|
"announced": "2026-06-11"
|
|
204
221
|
},
|
|
205
222
|
{
|
|
@@ -207,7 +224,10 @@
|
|
|
207
224
|
"provider": "openai",
|
|
208
225
|
"retires": "2026-12-11",
|
|
209
226
|
"qualifier": "exact",
|
|
210
|
-
"replacement": "gpt-
|
|
227
|
+
"replacement": "gpt-6.1-sol",
|
|
228
|
+
"replacement_note": "reasoning.mode: pro, in the Responses API",
|
|
229
|
+
"previous_replacement": "gpt-5.6-sol",
|
|
230
|
+
"replacement_changed": "2026-11-13",
|
|
211
231
|
"announced": "2026-06-11"
|
|
212
232
|
},
|
|
213
233
|
{
|
|
@@ -215,7 +235,9 @@
|
|
|
215
235
|
"provider": "openai",
|
|
216
236
|
"retires": "2026-12-11",
|
|
217
237
|
"qualifier": "exact",
|
|
218
|
-
"replacement": "gpt-
|
|
238
|
+
"replacement": "gpt-6.1-terra",
|
|
239
|
+
"previous_replacement": "gpt-5.6-terra",
|
|
240
|
+
"replacement_changed": "2026-11-13",
|
|
219
241
|
"announced": "2026-06-11"
|
|
220
242
|
},
|
|
221
243
|
{
|
|
@@ -223,7 +245,9 @@
|
|
|
223
245
|
"provider": "openai",
|
|
224
246
|
"retires": "2026-12-11",
|
|
225
247
|
"qualifier": "exact",
|
|
226
|
-
"replacement": "gpt-
|
|
248
|
+
"replacement": "gpt-6.1-terra",
|
|
249
|
+
"previous_replacement": "gpt-5.6-terra",
|
|
250
|
+
"replacement_changed": "2026-11-13",
|
|
227
251
|
"announced": "2026-06-11"
|
|
228
252
|
},
|
|
229
253
|
{
|
|
@@ -231,7 +255,9 @@
|
|
|
231
255
|
"provider": "openai",
|
|
232
256
|
"retires": "2026-12-11",
|
|
233
257
|
"qualifier": "exact",
|
|
234
|
-
"replacement": "gpt-
|
|
258
|
+
"replacement": "gpt-6-luna",
|
|
259
|
+
"previous_replacement": "gpt-5.6-luna",
|
|
260
|
+
"replacement_changed": "2026-11-13",
|
|
235
261
|
"announced": "2026-06-11"
|
|
236
262
|
},
|
|
237
263
|
{
|
|
@@ -239,7 +265,9 @@
|
|
|
239
265
|
"provider": "openai",
|
|
240
266
|
"retires": "2026-12-11",
|
|
241
267
|
"qualifier": "exact",
|
|
242
|
-
"replacement": "gpt-
|
|
268
|
+
"replacement": "gpt-6-luna",
|
|
269
|
+
"previous_replacement": "gpt-5.6-luna",
|
|
270
|
+
"replacement_changed": "2026-11-13",
|
|
243
271
|
"announced": "2026-06-11"
|
|
244
272
|
},
|
|
245
273
|
{
|
|
@@ -344,6 +372,7 @@
|
|
|
344
372
|
"retires": "2026-10-23",
|
|
345
373
|
"qualifier": "exact",
|
|
346
374
|
"replacement": "gpt-5.6-sol",
|
|
375
|
+
"replacement_note": "reasoning.mode: pro",
|
|
347
376
|
"announced": "2026-04-22"
|
|
348
377
|
},
|
|
349
378
|
{
|
|
@@ -352,6 +381,7 @@
|
|
|
352
381
|
"retires": "2026-10-23",
|
|
353
382
|
"qualifier": "exact",
|
|
354
383
|
"replacement": "gpt-5.6-sol",
|
|
384
|
+
"replacement_note": "reasoning.mode: pro",
|
|
355
385
|
"announced": "2026-04-22"
|
|
356
386
|
},
|
|
357
387
|
{
|
|
@@ -1944,6 +1974,46 @@
|
|
|
1944
1974
|
"qualifier": "earliest",
|
|
1945
1975
|
"replacement": "antigravity-preview-09-2026",
|
|
1946
1976
|
"kind": "agent"
|
|
1977
|
+
},
|
|
1978
|
+
{
|
|
1979
|
+
"id": "gpt-3.5-turbo-completions",
|
|
1980
|
+
"provider": "openai",
|
|
1981
|
+
"retires": "2026-10-23",
|
|
1982
|
+
"qualifier": "exact",
|
|
1983
|
+
"replacement": "gpt-5.6-terra",
|
|
1984
|
+
"announced": "2026-04-22"
|
|
1985
|
+
},
|
|
1986
|
+
{
|
|
1987
|
+
"id": "gpt-4-completions",
|
|
1988
|
+
"provider": "openai",
|
|
1989
|
+
"retires": "2026-10-23",
|
|
1990
|
+
"qualifier": "exact",
|
|
1991
|
+
"replacement": "gpt-5.6-sol",
|
|
1992
|
+
"announced": "2026-04-22"
|
|
1993
|
+
},
|
|
1994
|
+
{
|
|
1995
|
+
"id": "gpt-4-0613-completions",
|
|
1996
|
+
"provider": "openai",
|
|
1997
|
+
"retires": "2026-10-23",
|
|
1998
|
+
"qualifier": "exact",
|
|
1999
|
+
"replacement": "gpt-5.6-sol",
|
|
2000
|
+
"announced": "2026-04-22"
|
|
2001
|
+
},
|
|
2002
|
+
{
|
|
2003
|
+
"id": "gpt-4-turbo-completions",
|
|
2004
|
+
"provider": "openai",
|
|
2005
|
+
"retires": "2026-10-23",
|
|
2006
|
+
"qualifier": "exact",
|
|
2007
|
+
"replacement": "gpt-5.6-sol",
|
|
2008
|
+
"announced": "2026-04-22"
|
|
2009
|
+
},
|
|
2010
|
+
{
|
|
2011
|
+
"id": "gpt-4-turbo-preview-completions",
|
|
2012
|
+
"provider": "openai",
|
|
2013
|
+
"retires": "2026-03-26",
|
|
2014
|
+
"qualifier": "exact",
|
|
2015
|
+
"replacement": "gpt-5",
|
|
2016
|
+
"announced": "2025-09-26"
|
|
1947
2017
|
}
|
|
1948
2018
|
]
|
|
1949
2019
|
}
|
package/src/cli.js
CHANGED
|
@@ -7,6 +7,8 @@
|
|
|
7
7
|
// driftline [directory] --json same scan, machine-readable (for scripts and coding agents)
|
|
8
8
|
// driftline mcp MCP server on stdio for coding agents (Claude Code, Cursor, ...)
|
|
9
9
|
// driftline --help | --version
|
|
10
|
+
import { loadBestRegistry } from "./registry.js";
|
|
11
|
+
import { migrationFor } from "./migration.js";
|
|
10
12
|
import { resolve } from "node:path";
|
|
11
13
|
import { readFileSync } from "node:fs";
|
|
12
14
|
import { execFileSync } from "node:child_process";
|
|
@@ -67,8 +69,10 @@ if (cmd === "-h" || cmd === "--help" || cmd === "help") {
|
|
|
67
69
|
if (!r.ok) { console.error(r.reason); process.exitCode = 1; }
|
|
68
70
|
else console.log("Deactivated. That slot is free for another GitHub account.");
|
|
69
71
|
} else {
|
|
72
|
+
const { reg } = await loadBestRegistry();
|
|
70
73
|
const result = await run({
|
|
71
74
|
root: resolve(cmd || "."),
|
|
75
|
+
registry: reg,
|
|
72
76
|
licenseKey: process.env.DRIFTLINE_LICENSE_KEY || "",
|
|
73
77
|
account: process.env.GITHUB_REPOSITORY_OWNER || accountFromGit(),
|
|
74
78
|
log: (m) => console.error(m),
|
|
@@ -77,7 +81,7 @@ if (cmd === "-h" || cmd === "--help" || cmd === "help") {
|
|
|
77
81
|
const pick = (f) => ({
|
|
78
82
|
id: f.id, file: f.file, line: f.line, status: f.status, docOnly: !!f.doc,
|
|
79
83
|
provider: f.entry?.provider, retires: f.entry?.retires, qualifier: f.entry?.qualifier,
|
|
80
|
-
replacement: f.entry?.replacement, routes: !!f.entry?.routes, aliasOf: f.entry?.aliasOf,
|
|
84
|
+
replacement: f.entry?.replacement, migration: migrationFor(f.entry) || undefined, routes: !!f.entry?.routes, aliasOf: f.entry?.aliasOf,
|
|
81
85
|
});
|
|
82
86
|
console.log(JSON.stringify({ findings: result.findings.map(pick), drift: result.drift || [] }, null, 2));
|
|
83
87
|
} else console.log(result.markdown);
|
package/src/index.js
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
// GitHub Action entry point.
|
|
3
|
+
import { replacementText, replacementChange, loadBestRegistry } from "./registry.js";
|
|
4
|
+
import { migrationFor } from "./migration.js";
|
|
3
5
|
import { resolve } from "node:path";
|
|
4
6
|
import { statSync } from "node:fs";
|
|
5
7
|
import { randomUUID } from "node:crypto";
|
|
@@ -26,9 +28,12 @@ async function main() {
|
|
|
26
28
|
failOn = "retired";
|
|
27
29
|
}
|
|
28
30
|
|
|
31
|
+
const { reg, source } = await loadBestRegistry({ offline: input("registry", "live").toLowerCase() === "bundled" });
|
|
32
|
+
console.log(`Registry: ${source === "live" ? "live copy from aidriftline.com" : "copy bundled with this version"}, checked ${reg.checked}.`);
|
|
29
33
|
const result = await run({
|
|
30
34
|
root,
|
|
31
35
|
repoRoot: workspace,
|
|
36
|
+
registry: reg,
|
|
32
37
|
warnDays,
|
|
33
38
|
promptsFile: input("prompts-file", ".driftline/prompts.yml"),
|
|
34
39
|
licenseKey: input("license-key"),
|
|
@@ -48,7 +53,10 @@ async function main() {
|
|
|
48
53
|
}
|
|
49
54
|
for (const f of toAnnotate.slice(0, MAX_ANNOTATIONS)) {
|
|
50
55
|
const level = f.status === "deprecated" || f.doc ? "notice" : "warning";
|
|
51
|
-
const
|
|
56
|
+
const chg = replacementChange(f.entry);
|
|
57
|
+
const mig = f.entry.routes ? null : migrationFor(f.entry);
|
|
58
|
+
const more = mig && !mig.drop_in && mig.change !== "settings" ? ` Not just a rename: ${mig.note}` : mig?.change === "choose" ? ` ${mig.note}` : "";
|
|
59
|
+
const repl = f.entry?.replacement && !f.entry.routes ? ` Replacement: ${replacementText(f.entry)}.${chg ? ` (Changed by the provider on ${chg.on}; before it was ${chg.was}.)` : ""}` : "";
|
|
52
60
|
const text = f.entry.routes && f.entry.replacement
|
|
53
61
|
? (f.days <= 0
|
|
54
62
|
? `Since ${f.entry.retires}, requests to ${f.id} are answered by ${f.entry.replacement}. The call still works, but the model behind it changed.`
|
|
@@ -56,7 +64,8 @@ async function main() {
|
|
|
56
64
|
: f.entry.qualifier === "earliest" && f.days <= 0
|
|
57
65
|
? `${f.id} is past its earliest shutdown date (${f.entry.retires}) and may stop working at any time.`
|
|
58
66
|
: `${f.id} ${STATUS_TEXT[f.status]} (${f.entry.retires}).`;
|
|
59
|
-
|
|
67
|
+
const routedMore = f.entry.routes && migrationFor(f.entry)?.change === "params" ? ` Before then, apply the request changes the provider lists: ${migrationFor(f.entry).note}` : "";
|
|
68
|
+
annotate(level, text + repl + more + routedMore, f.file, f.line);
|
|
60
69
|
}
|
|
61
70
|
|
|
62
71
|
// The report contains model answers; stop GitHub from reading any "::command::" lines inside it.
|
package/src/mcp.js
CHANGED
|
@@ -6,39 +6,33 @@
|
|
|
6
6
|
import { readFileSync, statSync } from "node:fs";
|
|
7
7
|
import { resolve } from "node:path";
|
|
8
8
|
import { createInterface } from "node:readline";
|
|
9
|
-
import {
|
|
9
|
+
import { loadBestRegistry, classify, normalizeId, replacementChange } from "./registry.js";
|
|
10
|
+
import { migrationFor } from "./migration.js";
|
|
10
11
|
import { scanDirectory } from "./scan.js";
|
|
11
12
|
|
|
12
13
|
const VERSION = JSON.parse(readFileSync(new URL("../package.json", import.meta.url), "utf8")).version;
|
|
13
|
-
const LIVE_URL = "https://aidriftline.com/models.json";
|
|
14
14
|
const page = (id) => `https://aidriftline.com/models/${id}/`;
|
|
15
15
|
const WHO = { openai: "OpenAI", anthropic: "Anthropic", google: "Google" };
|
|
16
16
|
|
|
17
|
-
export
|
|
18
|
-
const bundled = JSON.parse(readFileSync(REGISTRY_PATH, "utf8"));
|
|
19
|
-
if (offline) return { reg: buildRegistry(bundled), source: "bundled" };
|
|
20
|
-
try {
|
|
21
|
-
const res = await fetchImpl(LIVE_URL, { signal: AbortSignal.timeout(4000) });
|
|
22
|
-
const live = res.ok ? await res.json() : null;
|
|
23
|
-
if (live?.models?.length && live.checked >= bundled.checked) return { reg: buildRegistry(live), source: "live" };
|
|
24
|
-
} catch { /* offline or slow: use the bundled copy */ }
|
|
25
|
-
return { reg: buildRegistry(bundled), source: "bundled" };
|
|
26
|
-
}
|
|
17
|
+
export { loadBestRegistry }; // shared with the Action and CLI (src/registry.js)
|
|
27
18
|
|
|
28
19
|
// One plain sentence an agent can pass on to the user.
|
|
29
|
-
function advice(c) {
|
|
20
|
+
function advice(c, today = new Date()) {
|
|
30
21
|
const m = c.entry;
|
|
31
22
|
if (!m) return "Not in Driftline's registry: no retirement is announced for it on the OpenAI, Anthropic or Google deprecation pages (or it is not a model ID).";
|
|
32
23
|
const who = WHO[m.provider] || m.provider;
|
|
33
|
-
const
|
|
24
|
+
const chg = replacementChange(m, today);
|
|
25
|
+
const mig = migrationFor(m);
|
|
26
|
+
const migText = mig && !mig.drop_in && mig.change !== "settings" ? ` More than a rename: ${mig.note}` : mig?.change === "choose" ? ` ${mig.note}` : "";
|
|
27
|
+
const repl = m.replacement ? ` Replacement: ${m.replacement}${m.replacement_note ? ` (${m.replacement_note})` : ""}.${chg ? ` ${who} changed this recommendation on ${chg.on}; before it was ${chg.was}.` : ""}` : "";
|
|
34
28
|
if (m.qualifier === "not-before") return `Active. ${who} says it will not be retired before ${m.retires}.`;
|
|
35
29
|
if (m.routes) return c.days <= 0
|
|
36
|
-
? `Since ${m.retires}, ${who} answers requests for it with ${m.replacement}. No error, but answers come from a different model
|
|
37
|
-
: `From ${m.retires} (in ${c.days} days), ${who} will answer requests for it with ${m.replacement}
|
|
30
|
+
? `Since ${m.retires}, ${who} answers requests for it with ${m.replacement}. No error, but answers come from a different model.${migText}`
|
|
31
|
+
: `From ${m.retires} (in ${c.days} days), ${who} will answer requests for it with ${m.replacement}.${migText}`;
|
|
38
32
|
if (m.qualifier === "earliest") return c.days <= 0
|
|
39
|
-
? `${who} listed ${m.retires} as the earliest shutdown date; it can stop working at any time now.${repl}`
|
|
40
|
-
: `${who} may shut it down from ${m.retires} (in ${c.days} days).${repl}`;
|
|
41
|
-
return c.days <= 0 ? `Retired on ${m.retires}. Calls to it fail.${repl}` : `Retires on ${m.retires} (in ${c.days} days). Calls fail after that.${repl}`;
|
|
33
|
+
? `${who} listed ${m.retires} as the earliest shutdown date; it can stop working at any time now.${repl}${migText}`
|
|
34
|
+
: `${who} may shut it down from ${m.retires} (in ${c.days} days).${repl}${migText}`;
|
|
35
|
+
return c.days <= 0 ? `Retired on ${m.retires}. Calls to it fail.${repl}${migText}` : `Retires on ${m.retires} (in ${c.days} days). Calls fail after that.${repl}${migText}`;
|
|
42
36
|
}
|
|
43
37
|
|
|
44
38
|
function describe(rawId, reg, today) {
|
|
@@ -49,9 +43,11 @@ function describe(rawId, reg, today) {
|
|
|
49
43
|
id: rawId, ...(id !== rawId ? { normalized: id } : {}),
|
|
50
44
|
status: c.status, ...(m ? {
|
|
51
45
|
provider: m.provider, retires: m.retires, qualifier: m.qualifier, days_left: c.days,
|
|
52
|
-
replacement: m.replacement || null,
|
|
46
|
+
replacement: m.replacement || null, ...(m.replacement_note ? { replacement_note: m.replacement_note } : {}),
|
|
47
|
+
...(m.previous_replacement ? { previous_replacement: m.previous_replacement, replacement_changed: m.replacement_changed } : {}), routes: !!m.routes, ...(m.aliasOf ? { alias_of: m.aliasOf } : {}), page: page(m.aliasOf || m.id),
|
|
53
48
|
} : {}),
|
|
54
|
-
|
|
49
|
+
...(m && migrationFor(m) ? { migration: migrationFor(m) } : {}),
|
|
50
|
+
advice: advice(c, today),
|
|
55
51
|
};
|
|
56
52
|
}
|
|
57
53
|
|
|
@@ -66,6 +62,10 @@ const MODEL_RESULT = {
|
|
|
66
62
|
qualifier: { type: "string", enum: ["exact", "earliest", "not-before"], description: "exact date, earliest possible shutdown (Google), or not before this date (Anthropic)" },
|
|
67
63
|
days_left: { type: "number", description: "Days until the date; 0 or negative when it has passed" },
|
|
68
64
|
replacement: { type: ["string", "null"] },
|
|
65
|
+
previous_replacement: { type: "string", description: "The replacement the provider recommended before it changed its advice" },
|
|
66
|
+
replacement_changed: { type: "string", description: "YYYY-MM-DD: when the change was seen on the provider's page" },
|
|
67
|
+
migration: { type: "object", description: "Whether moving to the replacement is just a new model name (drop_in true) or needs more: change = rename | settings | params | api | re-embed | choose, with a note and the provider's guide", properties: { change: { type: "string" }, drop_in: { type: "boolean" }, note: { type: "string" }, guide: { type: ["string", "null"] } } },
|
|
68
|
+
replacement_note: { type: "string", description: "Extra instruction the provider gives with the replacement (e.g. a setting or API to use). Without it, switching the model name alone is not enough" },
|
|
69
69
|
routes: { type: "boolean", description: "true: after the date the provider answers with the replacement model instead of failing" },
|
|
70
70
|
alias_of: { type: "string" }, page: { type: "string", description: "aidriftline.com page with the source for this date" },
|
|
71
71
|
advice: { type: "string", description: "One plain sentence to pass on to the user" },
|
package/src/migration.js
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
// Is the replacement a drop-in, or does moving to it need more than a new model name?
|
|
2
|
+
// Rules per model family, written from the providers' own migration docs (checked 13 Nov 2026).
|
|
3
|
+
// Anything no rule matches is a plain rename, which is how Driftline has always presented it.
|
|
4
|
+
//
|
|
5
|
+
// migrationFor(entry) → { change, drop_in, note, guide }
|
|
6
|
+
// change: "rename" the model name is the only change
|
|
7
|
+
// "settings" also set a parameter the provider names (e.g. reasoning.mode: pro)
|
|
8
|
+
// "params" some request parameters are removed or renamed
|
|
9
|
+
// "api" a different API, method or endpoint
|
|
10
|
+
// "re-embed" stored embeddings have to be recomputed
|
|
11
|
+
// "choose" the provider names more than one replacement; which one depends on how you use it
|
|
12
|
+
|
|
13
|
+
const OPENAI = "https://developers.openai.com/api/docs/guides/";
|
|
14
|
+
const GOOGLE = "https://ai.google.dev/gemini-api/docs/";
|
|
15
|
+
|
|
16
|
+
// Claude family version from an ID: claude-sonnet-4-5-20250929 → 4.5, claude-3-haiku-20240307 → 3, claude-opus-4-8 → 4.8
|
|
17
|
+
function claudeVersion(id) {
|
|
18
|
+
const m = String(id || "").match(/^claude-(?:(?:opus|sonnet|haiku|fable|mythos)-)?(\d+)(?:[-.](\d))?(?:-|$)/);
|
|
19
|
+
return m ? Number(m[1]) + (m[2] ? Number(m[2]) / 10 : 0) : null;
|
|
20
|
+
}
|
|
21
|
+
const geminiMajor = (id) => { const m = String(id || "").match(/^gemini-(\d+)/); return m ? Number(m[1]) : null; };
|
|
22
|
+
|
|
23
|
+
const RULES = [
|
|
24
|
+
{
|
|
25
|
+
when: (e) => e.provider === "openai" && (/^(davinci-002|babbage-002|gpt-3\.5-turbo-instruct)/.test(e.id) || /-completions$/.test(e.id)),
|
|
26
|
+
change: "api",
|
|
27
|
+
note: "The old Completions endpoint (a plain prompt) doesn't serve the replacement: move the call to the Responses API or Chat Completions (messages).",
|
|
28
|
+
guide: OPENAI + "migrate-to-responses",
|
|
29
|
+
},
|
|
30
|
+
{
|
|
31
|
+
when: (e) => e.provider === "openai" && /^(tts-1|gpt-4o-mini-tts)/.test(e.id),
|
|
32
|
+
change: "api",
|
|
33
|
+
note: "OpenAI calls this a change of API workflow: the replacement isn't a model for /v1/audio/speech and runs through the Realtime API.",
|
|
34
|
+
guide: OPENAI + "text-to-speech",
|
|
35
|
+
},
|
|
36
|
+
{
|
|
37
|
+
when: (e) => e.provider === "openai" && /realtime-preview/.test(e.id),
|
|
38
|
+
change: "api",
|
|
39
|
+
note: "The preview models ran on the Realtime beta (header OpenAI-Beta: realtime=v1), which OpenAI shut down; the replacement needs the current Realtime API.",
|
|
40
|
+
guide: OPENAI + "realtime",
|
|
41
|
+
},
|
|
42
|
+
{
|
|
43
|
+
when: (e) => e.provider === "openai" && /^(whisper-1|gpt-4o(-mini)?-transcribe)/.test(e.id),
|
|
44
|
+
change: "choose",
|
|
45
|
+
note: "OpenAI names two replacements: gpt-transcribe for recorded files, gpt-live-transcribe for live audio. Word timestamps, subtitles and translation work differently, so check those if you use them.",
|
|
46
|
+
guide: OPENAI + "transcription",
|
|
47
|
+
},
|
|
48
|
+
{
|
|
49
|
+
when: (e) => e.provider === "openai" && e.kind === "image" && /-sunburst$/.test(e.replacement || ""),
|
|
50
|
+
change: "rename",
|
|
51
|
+
note: "Same Images API. OpenAI offers two: …-sunburst for editing precision, …-flare for fast everyday generation.",
|
|
52
|
+
guide: OPENAI + "image-generation",
|
|
53
|
+
},
|
|
54
|
+
{
|
|
55
|
+
when: (e) => e.kind === "embedding",
|
|
56
|
+
change: "re-embed",
|
|
57
|
+
note: "Embeddings from different models can't be compared: re-embed your stored data with the new model, and query with that same model. The vector size may differ too.",
|
|
58
|
+
guide: (e) => e.provider === "google" ? GOOGLE + "embeddings" : OPENAI + "embeddings",
|
|
59
|
+
},
|
|
60
|
+
{
|
|
61
|
+
when: (e) => e.provider === "google" && /^imagen-/.test(e.id),
|
|
62
|
+
change: "api",
|
|
63
|
+
note: "Use generate_content instead of generate_images; images come back as content parts, not an image response object.",
|
|
64
|
+
guide: GOOGLE + "imagen",
|
|
65
|
+
},
|
|
66
|
+
{
|
|
67
|
+
when: (e) => e.provider === "google" && /^veo-/.test(e.id),
|
|
68
|
+
change: "api",
|
|
69
|
+
note: "Gemini Omni Flash uses the Interactions API (interactions.create), not Veo's video-generation call. Google also offers Veo itself on Gemini Enterprise Agent Platform.",
|
|
70
|
+
guide: GOOGLE + "video",
|
|
71
|
+
},
|
|
72
|
+
{
|
|
73
|
+
when: (e) => e.provider === "google" && (!e.kind || e.kind === "text") && geminiMajor(e.id) === 2 && geminiMajor(e.replacement) >= 3,
|
|
74
|
+
change: "params",
|
|
75
|
+
note: "Gemini 3 changes the request: remove temperature, top_p, top_k, candidate_count and prefilled model turns, replace thinking_budget with thinking_level, and check function calling.",
|
|
76
|
+
guide: (e) => GOOGLE + (/^gemini-3\.8/.test(e.replacement) ? "whats-new-gemini-3.8" : "whats-new-gemini-3.6"),
|
|
77
|
+
},
|
|
78
|
+
{
|
|
79
|
+
when: (e) => e.provider === "anthropic" && claudeVersion(e.id) !== null && claudeVersion(e.id) < 4.7 && claudeVersion(e.replacement) >= 4.7,
|
|
80
|
+
change: "params",
|
|
81
|
+
note: "Claude 4.7 and later accept only the default temperature, top_p and top_k: if your code sets them, remove them.",
|
|
82
|
+
guide: "https://platform.claude.com/docs/en/api/migration-2026-10-21",
|
|
83
|
+
},
|
|
84
|
+
];
|
|
85
|
+
|
|
86
|
+
export function migrationFor(entry) {
|
|
87
|
+
if (!entry?.replacement) return null;
|
|
88
|
+
// A provider's own instruction next to the replacement (registry replacement_note) comes first.
|
|
89
|
+
const setting = entry.replacement_note
|
|
90
|
+
? { change: "settings", drop_in: false, note: `Also set ${entry.replacement_note}.`, guide: /reasoning\.mode/.test(entry.replacement_note) ? OPENAI + "reasoning" : null }
|
|
91
|
+
: null;
|
|
92
|
+
const rule = RULES.find((r) => r.when(entry));
|
|
93
|
+
if (!rule) return setting || { change: "rename", drop_in: true, note: "", guide: null };
|
|
94
|
+
const found = { change: rule.change, drop_in: rule.change === "rename", note: rule.note, guide: typeof rule.guide === "function" ? rule.guide(entry) : rule.guide };
|
|
95
|
+
return setting ? { ...found, change: found.change === "rename" ? "settings" : found.change, drop_in: false, note: `${setting.note} ${found.note}` } : found;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
// Short label for tables: "Rename" or what else is needed.
|
|
99
|
+
export const MIGRATION_LABEL = {
|
|
100
|
+
rename: "Rename", settings: "Rename + setting", params: "Parameter changes", api: "API change", "re-embed": "Re-embed data", choose: "Pick one of two",
|
|
101
|
+
};
|
package/src/providers.js
CHANGED
|
@@ -63,7 +63,7 @@ async function postOnce(fetchImpl, url, headers, body) {
|
|
|
63
63
|
return json;
|
|
64
64
|
}
|
|
65
65
|
|
|
66
|
-
export async function complete({ model, prompt, system, env = process.env, fetchImpl = fetch }) {
|
|
66
|
+
export async function complete({ model, prompt, system, reasoningMode = null, env = process.env, fetchImpl = fetch }) {
|
|
67
67
|
const provider = providerOf(model);
|
|
68
68
|
const key = keyFor(provider, env);
|
|
69
69
|
if (!key) throw new Error(missingKeyMessage(provider));
|
|
@@ -71,7 +71,7 @@ export async function complete({ model, prompt, system, env = process.env, fetch
|
|
|
71
71
|
if (provider === "openai") {
|
|
72
72
|
const data = await postJson(fetchImpl, "https://api.openai.com/v1/responses",
|
|
73
73
|
{ authorization: `Bearer ${key}` },
|
|
74
|
-
{ model, input: prompt, ...(system ? { instructions: system } : {}) });
|
|
74
|
+
{ model, input: prompt, ...(system ? { instructions: system } : {}), ...(reasoningMode ? { reasoning: { mode: reasoningMode } } : {}) });
|
|
75
75
|
const text = typeof data.output_text === "string" ? data.output_text : (data.output || [])
|
|
76
76
|
.flatMap((o) => o.content || [])
|
|
77
77
|
.filter((c) => c.type === "output_text" || typeof c.text === "string")
|
package/src/registry.js
CHANGED
|
@@ -11,6 +11,26 @@ export function loadRegistry(path = REGISTRY_PATH) {
|
|
|
11
11
|
return buildRegistry(JSON.parse(readFileSync(path, "utf8")));
|
|
12
12
|
}
|
|
13
13
|
|
|
14
|
+
// The live registry is updated as soon as a provider changes a date or replacement; the copy bundled with a
|
|
15
|
+
// release can be days or weeks old. Everything (Action, CLI, MCP) uses the live copy when it can be read, looks
|
|
16
|
+
// like ours and is at least as new as the bundled one; otherwise the bundled copy. reg.source says which.
|
|
17
|
+
export const LIVE_URL = "https://aidriftline.com/models.json";
|
|
18
|
+
export function validRegistry(data) {
|
|
19
|
+
return Array.isArray(data?.models) && data.models.length >= 50 && /^\d{4}-\d{2}-\d{2}$/.test(data.checked || "") &&
|
|
20
|
+
data.models.every((m) => typeof m?.id === "string" && ["openai", "anthropic", "google"].includes(m.provider) && /^\d{4}-\d{2}-\d{2}$/.test(m.retires || ""));
|
|
21
|
+
}
|
|
22
|
+
export async function loadBestRegistry({ fetchImpl = fetch, offline = process.env.DRIFTLINE_OFFLINE === "1", timeoutMs = 4000 } = {}) {
|
|
23
|
+
const bundled = JSON.parse(readFileSync(REGISTRY_PATH, "utf8"));
|
|
24
|
+
const use = (data, source, why = "") => { const reg = buildRegistry(data); reg.source = source; reg.sourceWhy = why; return { reg, source, why }; };
|
|
25
|
+
if (offline) return use(bundled, "bundled", "offline");
|
|
26
|
+
try {
|
|
27
|
+
const res = await fetchImpl(LIVE_URL, { signal: AbortSignal.timeout(timeoutMs) });
|
|
28
|
+
const live = res.ok ? await res.json() : null;
|
|
29
|
+
if (validRegistry(live)) return live.checked >= bundled.checked ? use(live, "live") : use(bundled, "bundled", "newer");
|
|
30
|
+
} catch { /* offline, slow or malformed: use the bundled copy */ }
|
|
31
|
+
return use(bundled, "bundled", "unreachable");
|
|
32
|
+
}
|
|
33
|
+
|
|
14
34
|
// Same as loadRegistry, from already-parsed JSON (e.g. the live copy at aidriftline.com/models.json).
|
|
15
35
|
export function buildRegistry(data) {
|
|
16
36
|
const byId = new Map(data.models.map((m) => [m.id, { ...m }]));
|
|
@@ -72,6 +92,20 @@ export function looksLikeModelId(id, known) {
|
|
|
72
92
|
// Only text models can be drift-tested; audio, image, video, embedding and the rest are scan-only.
|
|
73
93
|
export const isTextModel = (entry) => !entry?.kind || entry.kind === "text";
|
|
74
94
|
|
|
95
|
+
// The replacement as people should read it: the model, plus the provider's extra instruction when there is one,
|
|
96
|
+
// e.g. "gpt-6.1-sol (reasoning.mode: pro, in the Responses API)". Without the note, a rename alone is not the move.
|
|
97
|
+
export const replacementText = (entry) => entry?.replacement
|
|
98
|
+
? entry.replacement + (entry.replacement_note ? ` (${entry.replacement_note})` : "") : "";
|
|
99
|
+
|
|
100
|
+
// Providers sometimes change the recommended replacement after announcing a retirement. The registry keeps
|
|
101
|
+
// the old one and the date we saw the change, so reports can say so instead of silently switching targets.
|
|
102
|
+
export const RECENT_CHANGE_DAYS = 120;
|
|
103
|
+
export function replacementChange(entry, today = new Date()) {
|
|
104
|
+
if (!entry?.replacement_changed || !entry.previous_replacement) return null;
|
|
105
|
+
const days = Math.floor((today - new Date(entry.replacement_changed + "T00:00:00Z")) / DAY);
|
|
106
|
+
return days >= 0 && days <= RECENT_CHANGE_DAYS ? { on: entry.replacement_changed, was: entry.previous_replacement, days } : null;
|
|
107
|
+
}
|
|
108
|
+
|
|
75
109
|
/**
|
|
76
110
|
* Classify one model ID against the registry on a given date.
|
|
77
111
|
* status: retired | at-risk | deprecated | active | unknown
|
package/src/report.js
CHANGED
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
import { replacementChange } from "./registry.js";
|
|
2
|
+
import { migrationFor, MIGRATION_LABEL } from "./migration.js";
|
|
3
|
+
|
|
1
4
|
const STATUS_LABEL = {
|
|
2
5
|
retired: "🔴 Retired",
|
|
3
6
|
"at-risk": "🟠 Retires soon",
|
|
@@ -7,6 +10,9 @@ const STATUS_LABEL = {
|
|
|
7
10
|
};
|
|
8
11
|
const RESULT_LABEL = { same: "✅ Same", changed: "🟡 Changed", broken: "🔴 Broken", error: "⚠️ Error", skipped: "⏭️ Skipped" };
|
|
9
12
|
|
|
13
|
+
// "<br>changed 2026-11-13, was `gpt-5.6-sol`" when the provider changed its advice recently.
|
|
14
|
+
const changed = (entry, today) => { const c = replacementChange(entry, today); return c ? `<br>_changed ${c.on}, was ${code(c.was)}_` : ""; };
|
|
15
|
+
|
|
10
16
|
// Google gives only earliest dates; once one has passed the model may be gone already, so "retires soon" would be wrong.
|
|
11
17
|
// Routed models keep answering after the date, but from the replacement model ("retired" would be wrong too).
|
|
12
18
|
const label = (c) => c.status === "at-risk" && c.entry?.qualifier === "earliest" && c.days <= 0
|
|
@@ -43,7 +49,7 @@ const MAX_ANSWER = 1500;
|
|
|
43
49
|
const MAX_ROWS = 150;
|
|
44
50
|
const clip = (s) => { s = String(s || ""); return s.length > MAX_ANSWER ? s.slice(0, MAX_ANSWER) + "\n[…cut]" : s; };
|
|
45
51
|
|
|
46
|
-
export function renderReport({ findings, drift, registry, notices = [], answers = true }) {
|
|
52
|
+
export function renderReport({ findings, drift, registry, notices = [], answers = true, today = new Date() }) {
|
|
47
53
|
const lines = ["## Driftline report", ""];
|
|
48
54
|
const flagged = findings.filter((f) => ["retired", "at-risk", "deprecated"].includes(f.status));
|
|
49
55
|
const byModel = new Map();
|
|
@@ -60,16 +66,30 @@ export function renderReport({ findings, drift, registry, notices = [], answers
|
|
|
60
66
|
lines.push(n
|
|
61
67
|
? `**${n} reference${n === 1 ? "" : "s"} to retired or retiring models** across ${byModel.size} model ID${byModel.size === 1 ? "" : "s"} found.`
|
|
62
68
|
: `Found ${byModel.size} model ID${byModel.size === 1 ? "" : "s"}. None are scheduled for retirement.`);
|
|
63
|
-
lines.push("", "| Model | Status | Retirement | Replacement | Where |", "
|
|
69
|
+
lines.push("", "| Model | Status | Retirement | Replacement | Change needed | Where |", "|---|---|---|---|---|---|");
|
|
70
|
+
const migNotes = new Map(); // note → { n, ids, guide } for the list under the table
|
|
64
71
|
const order = ["retired", "at-risk", "deprecated", "active", "unknown"];
|
|
65
72
|
const all = [...byModel.values()].sort((a, b) => order.indexOf(a.status) - order.indexOf(b.status));
|
|
66
73
|
const rows = all.slice(0, MAX_ROWS); // flagged models sort first, so only unknown/active rows get cut
|
|
67
74
|
for (const r of rows) {
|
|
75
|
+
const mig = migrationFor(r.entry);
|
|
76
|
+
let changeCell = "";
|
|
77
|
+
if (mig) {
|
|
78
|
+
if (mig.note && !(mig.change === "settings" && r.entry.replacement_note && mig.note === `Also set ${r.entry.replacement_note}.`)) {
|
|
79
|
+
if (!migNotes.has(mig.note)) migNotes.set(mig.note, { n: migNotes.size + 1, ids: [], guide: mig.guide });
|
|
80
|
+
migNotes.get(mig.note).ids.push(r.id);
|
|
81
|
+
changeCell = `${mig.drop_in ? "✅" : "⚠️"} ${MIGRATION_LABEL[mig.change]} [${migNotes.get(mig.note).n}]`;
|
|
82
|
+
} else changeCell = `${mig.drop_in ? "✅" : "⚠️"} ${MIGRATION_LABEL[mig.change]}`;
|
|
83
|
+
}
|
|
68
84
|
const places = r.places.slice(0, 3).join(", ") + (r.places.length > 3 ? ` +${r.places.length - 3} more` : "");
|
|
69
85
|
const docsOnly = r.places.length && byModel.get(r.id).docs === r.places.length ? " · docs only" : "";
|
|
70
|
-
lines.push(`| ${code(r.id)}${r.entry?.aliasOf ? " · alias of " + code(r.entry.aliasOf) : ""}${r.entry?.kind && r.entry.kind !== "text" ? " · " + r.entry.kind : ""} | ${label(r)}${docsOnly} | ${esc(when(r))} | ${r.entry?.replacement ? code(r.entry.replacement) : ""} | ${places} |`);
|
|
86
|
+
lines.push(`| ${code(r.id)}${r.entry?.aliasOf ? " · alias of " + code(r.entry.aliasOf) : ""}${r.entry?.kind && r.entry.kind !== "text" ? " · " + r.entry.kind : ""} | ${label(r)}${docsOnly} | ${esc(when(r))} | ${r.entry?.replacement ? code(r.entry.replacement) + (r.entry.replacement_note ? " (" + esc(r.entry.replacement_note) + ")" : "") + changed(r.entry, today) : ""} | ${changeCell} | ${places} |`);
|
|
71
87
|
}
|
|
72
88
|
if (all.length > rows.length) lines.push("", `_${all.length - rows.length} more model ID(s) with no retirement listed are not shown._`);
|
|
89
|
+
if (migNotes.size) {
|
|
90
|
+
lines.push("", "**More than a rename:**", "");
|
|
91
|
+
for (const [note, m] of migNotes) lines.push(`${m.n}. ${m.ids.map(code).join(", ")}: ${esc(note)}${m.guide ? ` ([provider guide](${m.guide}))` : ""}`);
|
|
92
|
+
}
|
|
73
93
|
}
|
|
74
94
|
|
|
75
95
|
if (drift?.length) {
|
|
@@ -96,7 +116,7 @@ export function renderReport({ findings, drift, registry, notices = [], answers
|
|
|
96
116
|
}
|
|
97
117
|
|
|
98
118
|
if (notices.length) lines.push("", ...notices.map((n) => `> ${n}`));
|
|
99
|
-
lines.push("", `<sub>Retirement dates checked against official provider pages on ${registry.checked}. ` +
|
|
119
|
+
lines.push("", `<sub>Retirement dates checked against official provider pages on ${registry.checked}${registry.source === "live" ? " (live list at aidriftline.com)" : ({ newer: " (list bundled with this version, newer than the live list)", unreachable: " (list bundled with this version; the live list could not be read)", offline: " (list bundled with this version)" })[registry.sourceWhy] || ""}. ` +
|
|
100
120
|
`Sources: [OpenAI](${registry.sources.openai}) · [Anthropic](${registry.sources.anthropic}) · [Google](${registry.sources.google}).</sub>`);
|
|
101
121
|
return lines.join("\n");
|
|
102
122
|
}
|
package/src/run.js
CHANGED
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
import { existsSync } from "node:fs";
|
|
2
2
|
import { join, relative, sep } from "node:path";
|
|
3
|
-
import { loadRegistry, classify, isTextModel } from "./registry.js";
|
|
3
|
+
import { loadRegistry, classify, isTextModel, replacementChange } from "./registry.js";
|
|
4
4
|
import { scanDirectory } from "./scan.js";
|
|
5
5
|
import { loadPrompts } from "./prompts.js";
|
|
6
6
|
import { complete, providerOf, keyFor, missingKeyMessage } from "./providers.js";
|
|
7
7
|
import { compare, extractJson } from "./compare.js";
|
|
8
8
|
import { checkLicense } from "./license.js";
|
|
9
9
|
import { renderReport } from "./report.js";
|
|
10
|
+
import { migrationFor } from "./migration.js";
|
|
10
11
|
|
|
11
12
|
const MAX_PROMPTS = 20;
|
|
12
13
|
const oneLine = (s) => String(s).replace(/[\r\n]+/g, " ");
|
|
@@ -61,9 +62,9 @@ export async function run({
|
|
|
61
62
|
}
|
|
62
63
|
|
|
63
64
|
const atRisk = findings.filter((f) => !f.doc && (f.status === "retired" || f.status === "at-risk")).length;
|
|
64
|
-
const markdown = renderReport({ findings, drift, registry, notices, answers });
|
|
65
|
+
const markdown = renderReport({ findings, drift, registry, notices, answers, today });
|
|
65
66
|
// The issue never carries model answers: in a public repository anyone can read it.
|
|
66
|
-
const issueMarkdown = renderReport({ findings, drift, registry, notices, answers: false });
|
|
67
|
+
const issueMarkdown = renderReport({ findings, drift, registry, notices, answers: false, today });
|
|
67
68
|
return { findings, drift, notices, atRisk, markdown, issueMarkdown };
|
|
68
69
|
}
|
|
69
70
|
|
|
@@ -92,25 +93,39 @@ async function runDrift({ prompts, findings, registry, env, fetchImpl, today, wa
|
|
|
92
93
|
for (const from of froms) {
|
|
93
94
|
const status = classify(from, registry, { today, warnDays });
|
|
94
95
|
const to = p.compare_to || status.entry?.replacement;
|
|
96
|
+
// The provider's replacement comes with an extra instruction (e.g. "reasoning.mode: pro") that this test
|
|
97
|
+
// doesn't apply: say so on every row, so nobody reads a plain-mode answer as the replacement's best.
|
|
98
|
+
const chg = !p.compare_to && replacementChange(status.entry, today);
|
|
99
|
+
// The provider says the replacement runs in a mode (e.g. reasoning.mode: pro): test it that way, as the customer would run it.
|
|
100
|
+
const proMode = !p.compare_to && /reasoning\.mode: pro/.test(status.entry?.replacement_note || "") && providerOf(to) === "openai" ? "pro" : null;
|
|
101
|
+
const mig = !p.compare_to ? migrationFor(status.entry) : null;
|
|
102
|
+
const caveat = [
|
|
103
|
+
proMode ? "Replacement tested with reasoning.mode: pro, as the provider recommends." : "",
|
|
104
|
+
!p.compare_to && !proMode && status.entry?.replacement_note
|
|
105
|
+
? `Tested without "${status.entry.replacement_note}", which the provider says the replacement needs; answers after a correct switch may differ.` : "",
|
|
106
|
+
mig && !mig.drop_in && mig.change !== "settings" ? `Beyond the model name: ${mig.note}` : "",
|
|
107
|
+
chg ? `The provider changed this replacement on ${chg.on} (before: ${chg.was}); drift results from earlier runs compared against ${chg.was}.` : "",
|
|
108
|
+
].filter(Boolean).join(" ");
|
|
109
|
+
const push = (row) => out.push(caveat ? { ...row, note: [row.note, caveat].filter(Boolean).join(" ") } : row);
|
|
95
110
|
if (!to) {
|
|
96
|
-
|
|
111
|
+
push({ id: p.id, from, to: "—", result: "skipped", note: "No known replacement; set compare_to" });
|
|
97
112
|
continue;
|
|
98
113
|
}
|
|
99
114
|
const missing = [from, to].map(providerOf).find((pr) => !keyFor(pr, env));
|
|
100
115
|
if (missing) {
|
|
101
|
-
|
|
116
|
+
push({ id: p.id, from, to, result: "error", note: missingKeyMessage(missing) });
|
|
102
117
|
continue;
|
|
103
118
|
}
|
|
104
119
|
log(oneLine(`Drift test ${p.id}: ${from} → ${to}`));
|
|
105
120
|
try {
|
|
106
|
-
const after = await complete({ model: to, prompt: p.prompt, system: p.system, env, fetchImpl });
|
|
121
|
+
const after = await complete({ model: to, prompt: p.prompt, system: p.system, reasoningMode: proMode, env, fetchImpl });
|
|
107
122
|
if (status.status === "retired") {
|
|
108
123
|
if (!after.trim()) {
|
|
109
|
-
|
|
124
|
+
push({ id: p.id, from, to, after, before: "(current model already retired)", result: "broken", note: "Replacement returned an empty answer" });
|
|
110
125
|
continue;
|
|
111
126
|
}
|
|
112
127
|
const jsonOk = p.expect !== "json" || extractJson(after).ok;
|
|
113
|
-
|
|
128
|
+
push({ id: p.id, from, to, after, before: "(current model already retired)", result: jsonOk ? "changed" : "broken",
|
|
114
129
|
note: jsonOk ? "Current model is retired; review the replacement's answer" : "Replacement did not return valid JSON" });
|
|
115
130
|
continue;
|
|
116
131
|
}
|
|
@@ -120,12 +135,12 @@ async function runDrift({ prompts, findings, registry, env, fetchImpl, today, wa
|
|
|
120
135
|
} catch (e) {
|
|
121
136
|
if (!/^(Empty answer|Answer cut off)/.test(e.message)) throw e;
|
|
122
137
|
// The current model gave nothing usable; the replacement's answer is still worth showing.
|
|
123
|
-
|
|
138
|
+
push({ id: p.id, from, to, after, before: `(${e.message})`, result: "changed", note: `Current model: ${e.message}. Review the replacement's answer` });
|
|
124
139
|
continue;
|
|
125
140
|
}
|
|
126
|
-
|
|
141
|
+
push({ id: p.id, from, to, before, after, ...compare(before, after, p.expect) });
|
|
127
142
|
} catch (e) {
|
|
128
|
-
|
|
143
|
+
push({ id: p.id, from, to, result: "error", note: e.message.slice(0, 200) });
|
|
129
144
|
}
|
|
130
145
|
}
|
|
131
146
|
}
|