@modelstatus/cli 0.1.0 → 0.1.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/api.js +3 -0
- package/src/ci.js +143 -0
- package/src/detect/core.js +26 -1
- package/src/index.js +200 -10
- package/src/openUrl.js +43 -1
- package/src/registry/local.js +23 -4
- package/src/sources/filesystem.js +0 -0
- package/src/telemetry.js +66 -0
- package/src/tui/app.js +173 -91
- package/src/tui/scan-stream.js +234 -0
- package/src/tui/signin.js +142 -0
- package/src/tui/snippet.js +127 -0
- package/src/tui/ui.js +661 -16
- package/src/tui/views/account.js +43 -13
- package/src/tui/views/add.js +33 -11
- package/src/tui/views/alerts.js +91 -39
- package/src/tui/views/inventory.js +149 -47
- package/src/tui/views/local.js +229 -0
- package/src/tui/views/scan.js +231 -72
- package/src/tui/views/whatsnew.js +92 -50
- package/src/updater.js +170 -0
- package/src/version.js +32 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@modelstatus/cli",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.25",
|
|
4
4
|
"description": "Track which AI models you use, where, and never get surprised by a retirement. Free offline model-health for any repo (mm status), browser sign-in for cloud inventory + alerts.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"llm",
|
package/src/api.js
CHANGED
|
@@ -64,6 +64,9 @@ export function createClient({ apiBase, apiKey }) {
|
|
|
64
64
|
linkUsage: (id, modelId) => req("POST", `/usages/${id}/link`, { model_id: modelId }),
|
|
65
65
|
bulkUpload: (projectId, usages) => req("POST", "/usages/bulk", { project_id: projectId, usages }),
|
|
66
66
|
|
|
67
|
+
// ci runs (Pro) — record a CI evaluation for the dashboard + alerts
|
|
68
|
+
ciRun: (body) => req("POST", "/ci/runs", body),
|
|
69
|
+
|
|
67
70
|
// notification rules
|
|
68
71
|
listRules: () => req("GET", "/notification-rules"),
|
|
69
72
|
createRule: (body) => req("POST", "/notification-rules", body),
|
package/src/ci.js
ADDED
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
/* CI evaluation engine: scan a repo, resolve against the signed registry, and
|
|
2
|
+
* report any deprecated/retiring/retired AI-model usage with file:line. Pure
|
|
3
|
+
* (no process.exit / no direct GitHub I/O) so it's testable; the `mm ci` command
|
|
4
|
+
* in index.js turns the result into GitHub annotations + a step summary + an
|
|
5
|
+
* exit code. Offline-capable — uses the public signed registry, no account. */
|
|
6
|
+
import { execFileSync } from "node:child_process";
|
|
7
|
+
import path from "node:path";
|
|
8
|
+
import { getRegistry } from "./registry/fetch.js";
|
|
9
|
+
import { resolveLocal, computeHealth } from "./registry/local.js";
|
|
10
|
+
import { collectFrom } from "./sources/index.js";
|
|
11
|
+
|
|
12
|
+
export const HEALTH_RANK = { ok: 0, deprecating: 1, retiring: 2, retired: 3 };
|
|
13
|
+
// `--fail-on` threshold → the minimum health rank that fails the build.
|
|
14
|
+
export const FAIL_THRESHOLD = { none: 99, deprecating: 1, retiring: 2, retired: 3 };
|
|
15
|
+
const BADGE = { ok: "🟢", deprecating: "🟡", retiring: "🟠", retired: "🔴" };
|
|
16
|
+
|
|
17
|
+
/** Evaluate `dir` and return { findings, failing, threshold, failOn, counts, snapshot }.
|
|
18
|
+
* `findings` are per-(model, location) entries with health worse than ok. */
|
|
19
|
+
export async function evaluateCi({ dir, sources = ["filesystem"], scanOpts = {}, failOn = "retired", offline = false, log = () => {} }) {
|
|
20
|
+
// Online by default so CI checks against the LATEST registry; --offline (or the
|
|
21
|
+
// env) falls back to the cached snapshot. cacheFile env keeps tests hermetic.
|
|
22
|
+
const snapshot = await getRegistry({
|
|
23
|
+
offline: offline || process.env.LLMSTATUS_REGISTRY_OFFLINE === "1",
|
|
24
|
+
cacheFile: process.env.LLMSTATUS_REGISTRY_CACHE || undefined,
|
|
25
|
+
log: (m) => log(`${m}\n`),
|
|
26
|
+
});
|
|
27
|
+
const candidates = await collectFrom(sources, { root: dir, ...scanOpts }, snapshot.detection);
|
|
28
|
+
const resolved = resolveLocal(snapshot, [...new Set(candidates.map((c) => c.model_string))]);
|
|
29
|
+
const byStr = new Map(resolved.map((r) => [r.input.toLowerCase(), r]));
|
|
30
|
+
const today = new Date();
|
|
31
|
+
const threshold = FAIL_THRESHOLD[failOn] ?? FAIL_THRESHOLD.retired;
|
|
32
|
+
|
|
33
|
+
const seen = new Set();
|
|
34
|
+
const findings = [];
|
|
35
|
+
for (const c of candidates) {
|
|
36
|
+
const r = byStr.get(c.model_string.toLowerCase());
|
|
37
|
+
if (!r?.model || !r.model_slug) continue; // only registry-known models can be scored
|
|
38
|
+
const health = computeHealth(r.model, 90, today);
|
|
39
|
+
if (health === "ok") continue;
|
|
40
|
+
const loc = `${c.source_path || c.location_label || ""}:${c.source_line || ""}`;
|
|
41
|
+
const key = r.model_slug + "|" + loc;
|
|
42
|
+
if (seen.has(key)) continue;
|
|
43
|
+
seen.add(key);
|
|
44
|
+
findings.push({
|
|
45
|
+
slug: r.model_slug,
|
|
46
|
+
health,
|
|
47
|
+
retires: r.model.retires_date || null,
|
|
48
|
+
replacement: r.model.replacement_slug || null,
|
|
49
|
+
path: c.source_path || null,
|
|
50
|
+
line: c.source_line || null,
|
|
51
|
+
location: c.location_label || c.source_path || c.model_string,
|
|
52
|
+
env: c.environment || null,
|
|
53
|
+
});
|
|
54
|
+
}
|
|
55
|
+
findings.sort((a, b) => HEALTH_RANK[b.health] - HEALTH_RANK[a.health] || String(a.retires || "9999").localeCompare(String(b.retires || "9999")));
|
|
56
|
+
|
|
57
|
+
const counts = { ok: 0, deprecating: 0, retiring: 0, retired: 0 };
|
|
58
|
+
for (const f of findings) counts[f.health]++;
|
|
59
|
+
const failing = findings.filter((f) => HEALTH_RANK[f.health] >= threshold);
|
|
60
|
+
return { snapshot, candidates, findings, failing, threshold, failOn, counts };
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* Pure filter: keep only findings whose `path` is in `changedPaths`. `changedPaths`
|
|
65
|
+
* is a Set of dir-relative posix paths (the same shape as `finding.path`). A `null`
|
|
66
|
+
* (or undefined) set means "couldn't determine the diff" → return findings unchanged
|
|
67
|
+
* (no filtering). Findings without a `path` are always dropped in diff mode, since we
|
|
68
|
+
* can't prove they belong to a changed file. No git / no I/O — trivially testable.
|
|
69
|
+
*/
|
|
70
|
+
export function filterToChangedFiles(findings, changedPaths) {
|
|
71
|
+
if (!changedPaths) return findings; // null/undefined → no diff info → don't filter
|
|
72
|
+
const norm = (p) => String(p || "").replace(/\\/g, "/").replace(/^\.\//, "");
|
|
73
|
+
const want = new Set([...changedPaths].map(norm));
|
|
74
|
+
return findings.filter((f) => f.path && want.has(norm(f.path)));
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Run `git diff --name-only <base>...HEAD` in `dir` and return a Set of paths made
|
|
79
|
+
* RELATIVE TO `dir` (so they match `finding.path`, which is dir-relative). git reports
|
|
80
|
+
* repo-root-relative paths, so each is resolved against the repo root then re-based on
|
|
81
|
+
* `dir`. Returns `null` (caller falls back to the full finding set) when git is missing,
|
|
82
|
+
* `dir` is not a repo, or `base` is unknown — i.e. on ANY git failure. Optional `log`
|
|
83
|
+
* receives a human note on fallback. Synchronous + offline (pure local git).
|
|
84
|
+
*/
|
|
85
|
+
export function getChangedFiles(dir, base, { log = () => {}, git = "git" } = {}) {
|
|
86
|
+
if (!base) return null;
|
|
87
|
+
const run = (args) => execFileSync(git, args, { cwd: dir, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] });
|
|
88
|
+
let repoRoot;
|
|
89
|
+
try {
|
|
90
|
+
repoRoot = run(["rev-parse", "--show-toplevel"]).trim();
|
|
91
|
+
} catch {
|
|
92
|
+
log(`--diff: not a git repo (or git not installed) — scanning all files.\n`);
|
|
93
|
+
return null;
|
|
94
|
+
}
|
|
95
|
+
let out;
|
|
96
|
+
try {
|
|
97
|
+
out = run(["diff", "--name-only", `${base}...HEAD`]);
|
|
98
|
+
} catch {
|
|
99
|
+
log(`--diff: couldn't diff against "${base}" (unknown ref?) — scanning all files.\n`);
|
|
100
|
+
return null;
|
|
101
|
+
}
|
|
102
|
+
const set = new Set();
|
|
103
|
+
for (const line of out.split(/\r?\n/)) {
|
|
104
|
+
const rel = line.trim();
|
|
105
|
+
if (!rel) continue;
|
|
106
|
+
// git path is repo-root-relative → make it dir-relative to match finding.path.
|
|
107
|
+
const relToDir = path.relative(dir, path.resolve(repoRoot, rel)).replace(/\\/g, "/");
|
|
108
|
+
set.add(relToDir);
|
|
109
|
+
}
|
|
110
|
+
return set;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
const ann = (s) => String(s).replace(/%/g, "%25").replace(/\r/g, "%0D").replace(/\n/g, "%0A");
|
|
114
|
+
|
|
115
|
+
/** GitHub Actions workflow-command annotations (one per finding). Inline on the PR. */
|
|
116
|
+
export function annotationLines(findings, threshold) {
|
|
117
|
+
return findings.map((f) => {
|
|
118
|
+
const sev = HEALTH_RANK[f.health] >= threshold ? "error" : "warning";
|
|
119
|
+
const where = f.path ? `file=${ann(f.path)},line=${f.line || 1},` : "";
|
|
120
|
+
const msg = `${f.slug} is ${f.health}${f.retires ? ` (retires ${f.retires})` : ""}${f.replacement ? ` — switch to ${f.replacement}` : ""}`;
|
|
121
|
+
return `::${sev} ${where}title=AI model ${f.health}::${ann(msg)}`;
|
|
122
|
+
});
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/** Markdown for $GITHUB_STEP_SUMMARY (and a decent human report). */
|
|
126
|
+
export function summaryMarkdown(findings, { failing, failOn } = {}) {
|
|
127
|
+
const lines = ["## LLM Status — model lifecycle check", ""];
|
|
128
|
+
if (!findings.length) {
|
|
129
|
+
lines.push("✅ No deprecated, retiring, or retired AI models found.");
|
|
130
|
+
return lines.join("\n") + "\n";
|
|
131
|
+
}
|
|
132
|
+
const fc = failing ? failing.length : 0;
|
|
133
|
+
lines.push(fc ? `🔴 **${fc}** usage(s) at or past the \`${failOn}\` threshold — this check fails.` : `Found ${findings.length} aging model usage(s) (below the \`${failOn}\` fail threshold).`, "");
|
|
134
|
+
lines.push("| | Model | Where | Retires | Replacement |", "|---|---|---|---|---|");
|
|
135
|
+
for (const f of findings.slice(0, 100)) {
|
|
136
|
+
lines.push(`| ${BADGE[f.health]} ${f.health} | \`${f.slug}\` | \`${f.location || ""}\` | ${f.retires || "—"} | ${f.replacement ? `\`${f.replacement}\`` : "—"} |`);
|
|
137
|
+
}
|
|
138
|
+
if (findings.length > 100) lines.push(`\n…and ${findings.length - 100} more.`);
|
|
139
|
+
lines.push("", "_Powered by [LLM Status](https://llmstatus.ai)._");
|
|
140
|
+
return lines.join("\n") + "\n";
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
export { BADGE };
|
package/src/detect/core.js
CHANGED
|
@@ -12,6 +12,28 @@ function cleanGeneric(s) {
|
|
|
12
12
|
return s.replace(/^[.\-_]+/, "").replace(/[.\-_]+$/, "");
|
|
13
13
|
}
|
|
14
14
|
|
|
15
|
+
/** A char is part of a model token if it's alnum or one of - _ . / : (slug-ish).
|
|
16
|
+
* Mirrors the server PR scanner (apps/web/lib/github/scan-pr.ts) so the CLI and
|
|
17
|
+
* the GitHub check agree on what counts as a boundary. */
|
|
18
|
+
function isTokenChar(ch) {
|
|
19
|
+
return /[A-Za-z0-9._/:-]/.test(ch);
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/** True when `term` occurs in `haystack` at a word-ish boundary (not embedded
|
|
23
|
+
* inside a longer identifier). Both are already lower-cased. This is what stops
|
|
24
|
+
* the alias "gpt-4" from matching inside "gpt-4o-mini" (→ wrong, older model). */
|
|
25
|
+
function matchesAtBoundary(haystack, term) {
|
|
26
|
+
let from = 0;
|
|
27
|
+
for (;;) {
|
|
28
|
+
const at = haystack.indexOf(term, from);
|
|
29
|
+
if (at < 0) return false;
|
|
30
|
+
const before = at > 0 ? haystack[at - 1] : "";
|
|
31
|
+
const after = at + term.length < haystack.length ? haystack[at + term.length] : "";
|
|
32
|
+
if ((before === "" || !isTokenChar(before)) && (after === "" || !isTokenChar(after))) return true;
|
|
33
|
+
from = at + 1; // a later occurrence may be bounded
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
|
|
15
37
|
/** Family globs catch brand-NEW versioned models before they're in the
|
|
16
38
|
* registry, so a real hit virtually always carries a version digit. Requiring
|
|
17
39
|
* one (and rejecting filename/domain tails) kills the bulk of false positives. */
|
|
@@ -33,7 +55,10 @@ export function compilePatterns(patterns) {
|
|
|
33
55
|
export function detectInLine(line, compiled) {
|
|
34
56
|
const lower = line.toLowerCase();
|
|
35
57
|
const found = new Set();
|
|
36
|
-
|
|
58
|
+
// Exact registry strings must match at a token boundary, not as a raw substring
|
|
59
|
+
// — otherwise a short alias ("gpt-4") resolves inside a longer id ("gpt-4o-mini")
|
|
60
|
+
// to the wrong model. Matches the server PR scanner's boundary semantics.
|
|
61
|
+
for (const s of compiled.exact) if (matchesAtBoundary(lower, s)) found.add(s);
|
|
37
62
|
for (const re of compiled.generic) {
|
|
38
63
|
re.lastIndex = 0;
|
|
39
64
|
let m;
|
package/src/index.js
CHANGED
|
@@ -1,15 +1,19 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
+
import fs from "node:fs";
|
|
2
3
|
import path from "node:path";
|
|
3
4
|
import { resolveAuth, loadConfig, saveConfig, clearAuth, configFilePath } from "./config.js";
|
|
4
5
|
import { createClient } from "./api.js";
|
|
5
6
|
import { collectFrom, availability, ALL_SOURCE_IDS } from "./sources/index.js";
|
|
6
7
|
import { loginViaBrowser } from "./auth.js";
|
|
8
|
+
import { maybeCheckForUpdate } from "./updater.js";
|
|
9
|
+
import { track, maybeAnalyticsNotice } from "./telemetry.js";
|
|
10
|
+
import { BUILD_VERSION } from "./version.js";
|
|
7
11
|
|
|
8
12
|
function parseArgs(argv) {
|
|
9
13
|
const flags = {};
|
|
10
14
|
const positional = [];
|
|
11
15
|
const valueFlags = new Set([
|
|
12
|
-
"api", "key", "project", "dir",
|
|
16
|
+
"api", "key", "project", "dir", "fail-on", "diff", "json-out",
|
|
13
17
|
"sources", "region", "namespace", "kube-context", "db", "sql-table", "env",
|
|
14
18
|
]);
|
|
15
19
|
for (let i = 0; i < argv.length; i++) {
|
|
@@ -29,6 +33,12 @@ function parseArgs(argv) {
|
|
|
29
33
|
|
|
30
34
|
const uuidish = (s) => /^[0-9a-f]{8}-[0-9a-f]{4}-/i.test(s || "");
|
|
31
35
|
|
|
36
|
+
/** The "owner/name" repo slug for source deep-links. In GitHub Actions
|
|
37
|
+
* GITHUB_REPOSITORY is exactly that and the checkout is repo-root-relative (so
|
|
38
|
+
* our source_path lines up). Outside CI we return "" (omit it) rather than guess
|
|
39
|
+
* from a git remote, since a local scan root may not be the repo root. */
|
|
40
|
+
const ghRepoSlug = () => (process.env.GITHUB_REPOSITORY || "").trim();
|
|
41
|
+
|
|
32
42
|
/** Resolve the requested sources: default filesystem, "all", or a comma list. */
|
|
33
43
|
function parseSources(flags) {
|
|
34
44
|
const raw = (flags.sources || "").trim();
|
|
@@ -95,14 +105,16 @@ async function cmdUpgrade(_positional, flags) {
|
|
|
95
105
|
}
|
|
96
106
|
|
|
97
107
|
async function launchTui(initialView, flags) {
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
}
|
|
108
|
+
// Pass apiKey straight through (may be null). The TUI's Bootstrap wrapper
|
|
109
|
+
// renders an interactive SignIn screen when there's no key, then swaps to
|
|
110
|
+
// the main App on success — no separate browser-login polling phase that the
|
|
111
|
+
// user has to wait through without seeing anything interactive.
|
|
112
|
+
const { apiBase, apiKey } = resolveAuth(flags);
|
|
103
113
|
const dir = path.resolve(flags.dir || ".");
|
|
104
114
|
const { runApp } = await import("./tui/app.js");
|
|
105
|
-
|
|
115
|
+
// --scan / --rescan / --fresh force a fresh filesystem walk instead of loading
|
|
116
|
+
// the last persisted scan.
|
|
117
|
+
await runApp({ apiBase, apiKey, dir, initialView, fresh: !!(flags.scan || flags.rescan || flags.fresh) });
|
|
106
118
|
}
|
|
107
119
|
|
|
108
120
|
async function cmdScan(positional, flags) {
|
|
@@ -161,6 +173,7 @@ async function cmdScan(positional, flags) {
|
|
|
161
173
|
custom_model_name: r.model_id ? undefined : r.model_string,
|
|
162
174
|
environment: r.environment,
|
|
163
175
|
location_label: r.location_label,
|
|
176
|
+
source_repo: ghRepoSlug() || undefined,
|
|
164
177
|
source_path: r.source_path,
|
|
165
178
|
source_line: r.source_line ?? undefined,
|
|
166
179
|
}));
|
|
@@ -190,6 +203,7 @@ async function cmdScan(positional, flags) {
|
|
|
190
203
|
}
|
|
191
204
|
|
|
192
205
|
const res = await client.bulkUpload(projectId, usages);
|
|
206
|
+
track("usages_uploaded", { count: usages.length, created: res.created, updated: res.updated, source: "cli" });
|
|
193
207
|
if (flags.json) {
|
|
194
208
|
console.log(
|
|
195
209
|
JSON.stringify(
|
|
@@ -205,6 +219,144 @@ async function cmdScan(positional, flags) {
|
|
|
205
219
|
}
|
|
206
220
|
}
|
|
207
221
|
|
|
222
|
+
/** Pro cloud sync for `mm ci --report`: push this run's usages to the account so
|
|
223
|
+
* the dashboard tracks per-commit drift + alerts can fire. Plan-gated server-side
|
|
224
|
+
* (402 → skipped, the local check still ran). Best-effort: never fails the build. */
|
|
225
|
+
async function ciReport(dir, flags, res) {
|
|
226
|
+
const { apiBase, apiKey } = resolveAuth(flags);
|
|
227
|
+
if (!apiKey) {
|
|
228
|
+
process.stderr.write("! --report needs an API key (set LLMSTATUS_API_KEY) — skipping cloud sync.\n");
|
|
229
|
+
return null;
|
|
230
|
+
}
|
|
231
|
+
const client = createClient({ apiBase, apiKey });
|
|
232
|
+
const uniq = [...new Set(res.candidates.map((c) => c.model_string))];
|
|
233
|
+
const resolved = uniq.length ? (await client.resolve(uniq)).data || [] : [];
|
|
234
|
+
const byStr = new Map(resolved.map((r) => [r.input.toLowerCase(), r]));
|
|
235
|
+
const seen = new Set();
|
|
236
|
+
const usages = [];
|
|
237
|
+
for (const c of res.candidates) {
|
|
238
|
+
const r = byStr.get(c.model_string.toLowerCase());
|
|
239
|
+
const k = `${r?.model_id ?? "custom:" + c.model_string}|${c.location_label}`;
|
|
240
|
+
if (seen.has(k)) continue;
|
|
241
|
+
seen.add(k);
|
|
242
|
+
usages.push({
|
|
243
|
+
model_id: r?.model_id ?? undefined,
|
|
244
|
+
custom_model_name: r?.model_id ? undefined : c.model_string,
|
|
245
|
+
environment: c.environment,
|
|
246
|
+
location_label: c.location_label,
|
|
247
|
+
source_path: c.source_path,
|
|
248
|
+
source_line: c.source_line ?? undefined,
|
|
249
|
+
});
|
|
250
|
+
}
|
|
251
|
+
try {
|
|
252
|
+
const projectName = flags.project || (process.env.GITHUB_REPOSITORY || "").split("/").pop() || path.basename(dir);
|
|
253
|
+
const projects = (await client.listProjects()).data || [];
|
|
254
|
+
const projectId = projects.find((p) => p.name === projectName)?.id || (await client.createProject(projectName)).id;
|
|
255
|
+
const r = await client.bulkUpload(projectId, usages);
|
|
256
|
+
process.stderr.write(`✓ Reported ${usages.length} usage(s) to LLM Status (project "${projectName}").\n`);
|
|
257
|
+
return { project: projectName, ...r };
|
|
258
|
+
} catch (e) {
|
|
259
|
+
if (e.status === 402) process.stderr.write("! Cloud CI reporting is a Pro feature — the local check still ran. Upgrade with: mm upgrade\n");
|
|
260
|
+
else process.stderr.write(`! Cloud report skipped: ${e.message}\n`);
|
|
261
|
+
return null;
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
/** Record this run's summary to /ci/runs (the dashboard's source). Pulls commit/
|
|
266
|
+
* branch/repo from the GitHub Actions env (sensible local fallbacks). Best-effort,
|
|
267
|
+
* Pro-gated server-side (402 → skipped). Never throws into the CI gate. */
|
|
268
|
+
async function postCiRun(dir, flags, res, failOn) {
|
|
269
|
+
const { apiBase, apiKey } = resolveAuth(flags);
|
|
270
|
+
if (!apiKey) return null;
|
|
271
|
+
const env = process.env;
|
|
272
|
+
const client = createClient({ apiBase, apiKey });
|
|
273
|
+
try {
|
|
274
|
+
await client.ciRun({
|
|
275
|
+
commit_sha: env.GITHUB_SHA || "local",
|
|
276
|
+
branch: env.GITHUB_REF_NAME || "",
|
|
277
|
+
ref: env.GITHUB_REF || "",
|
|
278
|
+
repo: env.GITHUB_REPOSITORY || path.basename(path.resolve(dir)),
|
|
279
|
+
counts: res.counts,
|
|
280
|
+
fail_on: failOn,
|
|
281
|
+
failing_count: res.failing.length,
|
|
282
|
+
findings: res.findings.slice(0, 200),
|
|
283
|
+
});
|
|
284
|
+
return true;
|
|
285
|
+
} catch (e) {
|
|
286
|
+
if (e.status === 402) process.stderr.write("! CI run recording is a Pro feature — the local check still ran. Upgrade: mm upgrade\n");
|
|
287
|
+
return null;
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
/** `mm ci [dir]` — evaluate the repo for deprecated/retiring models. Emits GitHub
|
|
292
|
+
* annotations + a step summary under GITHUB_ACTIONS; exits non-zero per --fail-on
|
|
293
|
+
* so it gates merges. Offline (public registry) by default; --report syncs (Pro). */
|
|
294
|
+
async function cmdCi(positional, flags) {
|
|
295
|
+
const dir = path.resolve(positional[1] || flags.dir || ".");
|
|
296
|
+
const failOn = String(flags["fail-on"] || "retired").toLowerCase();
|
|
297
|
+
const { evaluateCi, annotationLines, summaryMarkdown, filterToChangedFiles, getChangedFiles, HEALTH_RANK } = await import("./ci.js");
|
|
298
|
+
const res = await evaluateCi({
|
|
299
|
+
dir,
|
|
300
|
+
sources: parseSources(flags),
|
|
301
|
+
scanOpts: scanOpts(flags, dir),
|
|
302
|
+
failOn,
|
|
303
|
+
offline: !!flags.offline,
|
|
304
|
+
log: (m) => process.stderr.write(`! ${m}`),
|
|
305
|
+
});
|
|
306
|
+
let { findings, failing, threshold, counts } = res;
|
|
307
|
+
|
|
308
|
+
// --diff <base> (or a GitHub PR: GITHUB_BASE_REF as base, HEAD as head) restricts
|
|
309
|
+
// findings to files CHANGED vs base — we still scan the whole repo, then drop findings
|
|
310
|
+
// whose source_path isn't in the changed set. getChangedFiles returns null on any git
|
|
311
|
+
// failure (missing git / not a repo / bad base), and filterToChangedFiles treats null
|
|
312
|
+
// as "don't filter", so we transparently fall back to the full set with a stderr note.
|
|
313
|
+
const diffBase = flags.diff && flags.diff !== true ? flags.diff : process.env.GITHUB_BASE_REF || null;
|
|
314
|
+
if (diffBase) {
|
|
315
|
+
const changed = getChangedFiles(dir, diffBase, { log: (m) => process.stderr.write(`! ${m}`) });
|
|
316
|
+
if (changed) {
|
|
317
|
+
findings = filterToChangedFiles(findings, changed);
|
|
318
|
+
failing = findings.filter((f) => HEALTH_RANK[f.health] >= threshold);
|
|
319
|
+
counts = { ok: 0, deprecating: 0, retiring: 0, retired: 0 };
|
|
320
|
+
for (const f of findings) counts[f.health]++;
|
|
321
|
+
process.stderr.write(`! --diff ${diffBase}: ${findings.length} finding(s) in ${changed.size} changed file(s).\n`);
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
if (process.env.GITHUB_ACTIONS) {
|
|
326
|
+
for (const a of annotationLines(findings, threshold)) console.log(a);
|
|
327
|
+
if (process.env.GITHUB_STEP_SUMMARY) {
|
|
328
|
+
try { fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, summaryMarkdown(findings, { failing, failOn })); } catch { /* best effort */ }
|
|
329
|
+
}
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
// Record the run to the account (Pro): inventory drift (bulkUpload) + a CI-run
|
|
333
|
+
// row for the dashboard. Best-effort, plan-gated server-side, never fails CI.
|
|
334
|
+
const reported = flags.report ? await ciReport(dir, flags, res).catch(() => null) : null;
|
|
335
|
+
if (flags.report) await postCiRun(dir, flags, res, failOn).catch(() => null);
|
|
336
|
+
|
|
337
|
+
const report = { scanned: dir, fail_on: failOn, counts, findings, failing: failing.length, reported };
|
|
338
|
+
// --json-out writes CLEAN JSON to a file (annotations still go to stdout for
|
|
339
|
+
// GitHub) so a CI step can parse it without stdout pollution.
|
|
340
|
+
if (flags["json-out"]) { try { fs.writeFileSync(flags["json-out"], JSON.stringify(report, null, 2)); } catch { /* best effort */ } }
|
|
341
|
+
|
|
342
|
+
if (flags.json) {
|
|
343
|
+
console.log(JSON.stringify(report, null, 2));
|
|
344
|
+
} else {
|
|
345
|
+
const ICON = { ok: "🟢", deprecating: "🟡", retiring: "🟠", retired: "🔴" };
|
|
346
|
+
console.log(`LLM Status CI — scanned ${dir} (fail-on: ${failOn})`);
|
|
347
|
+
if (!findings.length) {
|
|
348
|
+
console.log("✓ No deprecated, retiring, or retired AI models found.");
|
|
349
|
+
} else {
|
|
350
|
+
for (const f of findings) {
|
|
351
|
+
console.log(` ${ICON[f.health]} ${f.health.padEnd(11)} ${f.slug.padEnd(30)} ${String(f.location || "").padEnd(28)}${f.retires ? ` retires ${f.retires}` : ""}${f.replacement ? ` → ${f.replacement}` : ""}`);
|
|
352
|
+
}
|
|
353
|
+
console.log("");
|
|
354
|
+
console.log(failing.length ? `✗ ${failing.length} usage(s) at or past "${failOn}" — failing the build.` : `✓ ${findings.length} aging usage(s), none past "${failOn}".`);
|
|
355
|
+
}
|
|
356
|
+
}
|
|
357
|
+
if (failing.length) process.exit(1);
|
|
358
|
+
}
|
|
359
|
+
|
|
208
360
|
/** List detection sources and whether each can run right now. */
|
|
209
361
|
async function cmdSources(_positional, flags) {
|
|
210
362
|
const dir = path.resolve(flags.dir || ".");
|
|
@@ -292,15 +444,17 @@ async function cmdStatus(positional, flags) {
|
|
|
292
444
|
const HELP = `LLM Status CLI
|
|
293
445
|
|
|
294
446
|
Usage:
|
|
295
|
-
mm Launch the TUI (
|
|
447
|
+
mm Launch the TUI (signs you in via in-TUI device flow if needed)
|
|
296
448
|
mm status [dir] Offline model-health check for a dir — no account needed
|
|
297
449
|
mm login [api_key] Browser sign-in with polling (or paste a key)
|
|
298
450
|
mm signup Create an account in the browser, then poll
|
|
299
451
|
mm logout Forget the saved API key
|
|
300
452
|
mm scan [dir] Scan for model usage; interactive TUI, or --ci/--json for pipelines
|
|
453
|
+
mm ci [dir] CI gate: fail the build on deprecated/retiring models (GitHub annotations)
|
|
454
|
+
(--diff <base> limits findings to files changed vs base; auto on PRs via GITHUB_BASE_REF)
|
|
301
455
|
mm sources List detection sources and whether each can run here
|
|
302
456
|
mm upgrade Open Stripe checkout and poll until Pro is active
|
|
303
|
-
mm tui
|
|
457
|
+
mm tui Force-launch the TUI (logs you in first if needed)
|
|
304
458
|
|
|
305
459
|
Scan sources (--sources, default filesystem; "all" for everything):
|
|
306
460
|
filesystem repo files aws-secrets AWS Secrets Manager + SSM
|
|
@@ -314,24 +468,60 @@ Flags: --api <url> · --key <key> · --project <id|name> · --yes · --json · -
|
|
|
314
468
|
|
|
315
469
|
Get started: \`mm login\` (opens your browser).`;
|
|
316
470
|
|
|
471
|
+
/** Awaits the updater promise; prints a one-liner if an update completed. Never throws. */
|
|
472
|
+
async function maybePrintUpdate(promise) {
|
|
473
|
+
try {
|
|
474
|
+
const r = await promise;
|
|
475
|
+
if (!r) return;
|
|
476
|
+
if (r.manual) {
|
|
477
|
+
// Can't auto-update in place (root-owned dir like /usr/local/bin) — would
|
|
478
|
+
// be unsafe. Point the user at a one-line reinstall to a user-owned dir.
|
|
479
|
+
process.stderr.write(`\n✦ mm ${r.to} is available (you're on ${r.from}). Auto-update can't write ${process.execPath} — reinstall:\n curl -fsSL https://llmstatus.ai/install.sh | bash\n`);
|
|
480
|
+
} else {
|
|
481
|
+
process.stderr.write(`\n✓ Updated mm ${r.from} → ${r.to} — your next run uses the new version.\n`);
|
|
482
|
+
}
|
|
483
|
+
} catch {
|
|
484
|
+
/* swallow */
|
|
485
|
+
}
|
|
486
|
+
}
|
|
487
|
+
|
|
317
488
|
async function main() {
|
|
318
489
|
const { positional, flags } = parseArgs(process.argv.slice(2));
|
|
319
490
|
const cmd = positional[0];
|
|
491
|
+
|
|
492
|
+
// --version / -v: print + exit (skip dispatch + updater).
|
|
493
|
+
if (cmd === "version" || flags.version || flags.v) {
|
|
494
|
+
console.log(BUILD_VERSION);
|
|
495
|
+
return;
|
|
496
|
+
}
|
|
497
|
+
|
|
498
|
+
// Anonymous, opt-out usage analytics (one-time disclosure, then a single
|
|
499
|
+
// event per invocation). No-op without a baked key / when opted out.
|
|
500
|
+
maybeAnalyticsNotice();
|
|
501
|
+
track("cli_command", { command: cmd || "tui" });
|
|
502
|
+
|
|
503
|
+
// Kick off the background self-update check — runs in parallel with the
|
|
504
|
+
// user's command, capped at 1/24h, only for shell-installed binaries.
|
|
505
|
+
const updatePromise = maybeCheckForUpdate(flags);
|
|
506
|
+
|
|
320
507
|
try {
|
|
321
508
|
if (cmd === "login") await cmdLogin(positional, flags);
|
|
322
509
|
else if (cmd === "signup") await cmdSignup(positional, flags);
|
|
323
510
|
else if (cmd === "logout") cmdLogout();
|
|
324
511
|
else if (cmd === "scan") await cmdScan(positional, flags);
|
|
512
|
+
else if (cmd === "ci") await cmdCi(positional, flags);
|
|
325
513
|
else if (cmd === "status") await cmdStatus(positional, flags);
|
|
326
514
|
else if (cmd === "sources") await cmdSources(positional, flags);
|
|
327
515
|
else if (cmd === "upgrade") await cmdUpgrade(positional, flags);
|
|
328
|
-
else if (cmd === "tui" || !cmd) await launchTui(positional[1]
|
|
516
|
+
else if (cmd === "tui" || !cmd) await launchTui(positional[1], flags);
|
|
329
517
|
else if (cmd === "help" || flags.help) console.log(HELP);
|
|
330
518
|
else console.log(HELP);
|
|
331
519
|
} catch (e) {
|
|
332
520
|
console.error(`Error: ${e?.message ?? e}`);
|
|
521
|
+
await maybePrintUpdate(updatePromise);
|
|
333
522
|
process.exit(1);
|
|
334
523
|
}
|
|
524
|
+
await maybePrintUpdate(updatePromise);
|
|
335
525
|
}
|
|
336
526
|
|
|
337
527
|
main();
|
package/src/openUrl.js
CHANGED
|
@@ -1,4 +1,46 @@
|
|
|
1
|
-
import { spawn } from "node:child_process";
|
|
1
|
+
import { spawn, spawnSync } from "node:child_process";
|
|
2
|
+
|
|
3
|
+
function hasCmd(c) {
|
|
4
|
+
try {
|
|
5
|
+
return spawnSync(process.platform === "win32" ? "where" : "which", [c], { stdio: "ignore" }).status === 0;
|
|
6
|
+
} catch {
|
|
7
|
+
return false;
|
|
8
|
+
}
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
/** Open a source location in the user's editor. Prefers a line-aware editor
|
|
12
|
+
* ($MM_EDITOR template, then `code -g`/`cursor -g`); falls back to the OS opener
|
|
13
|
+
* (no line jump). Non-blocking, best-effort, never throws. Test-gated by
|
|
14
|
+
* LLMSTATUS_NO_OPEN. Returns true if something was launched. */
|
|
15
|
+
export function openLocation(absPath, line) {
|
|
16
|
+
if (process.env.LLMSTATUS_NO_OPEN) return false;
|
|
17
|
+
const target = line ? `${absPath}:${line}` : absPath;
|
|
18
|
+
const editorEnv = (process.env.MM_EDITOR || "").trim();
|
|
19
|
+
let cmd, args;
|
|
20
|
+
if (editorEnv) {
|
|
21
|
+
const parts = editorEnv.split(/\s+/);
|
|
22
|
+
cmd = parts[0];
|
|
23
|
+
args = [...parts.slice(1), target];
|
|
24
|
+
} else if (hasCmd("code")) {
|
|
25
|
+
cmd = "code"; args = ["-g", target];
|
|
26
|
+
} else if (hasCmd("cursor")) {
|
|
27
|
+
cmd = "cursor"; args = ["-g", target];
|
|
28
|
+
} else if (process.platform === "darwin") {
|
|
29
|
+
cmd = "open"; args = [absPath];
|
|
30
|
+
} else if (process.platform === "win32") {
|
|
31
|
+
cmd = "cmd"; args = ["/c", "start", "", absPath];
|
|
32
|
+
} else {
|
|
33
|
+
cmd = "xdg-open"; args = [absPath];
|
|
34
|
+
}
|
|
35
|
+
try {
|
|
36
|
+
const child = spawn(cmd, args, { stdio: "ignore", detached: true });
|
|
37
|
+
child.on("error", () => {});
|
|
38
|
+
child.unref();
|
|
39
|
+
return true;
|
|
40
|
+
} catch {
|
|
41
|
+
return false;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
2
44
|
|
|
3
45
|
/** Open a URL in the user's default browser, cross-platform. Best-effort: if it
|
|
4
46
|
* fails we still printed the URL, so the user can open it manually. */
|
package/src/registry/local.js
CHANGED
|
@@ -1,14 +1,33 @@
|
|
|
1
1
|
/* Offline resolution + health from a verified snapshot — mirrors the server's
|
|
2
2
|
* computeHealth (apps/web/lib/serialize.ts) so an offline run matches online. */
|
|
3
3
|
|
|
4
|
-
/** Resolve model strings against the snapshot's detection table
|
|
4
|
+
/** Resolve model strings against the snapshot's detection table.
|
|
5
|
+
*
|
|
6
|
+
* Exact match stays AUTHORITATIVE: only an exact hit sets `model` (and thus
|
|
7
|
+
* lifecycle/health), so a custom id never inherits a real model's retirement
|
|
8
|
+
* date by accident. For misses we attach a non-authoritative `suggestion` — the
|
|
9
|
+
* nearest known id by two-way substring containment (mirrors the server's 0.6
|
|
10
|
+
* fuzzy in apps/web/lib/services/registry.ts) — so callers can hint "≈ gpt-4?"
|
|
11
|
+
* on a typo without treating it as a confirmed match. `confidence`: 1 exact,
|
|
12
|
+
* 0.6 suggestion, 0 unknown. */
|
|
5
13
|
export function resolveLocal(snapshot, strings) {
|
|
6
14
|
const exact = new Map((snapshot.detection?.model_strings || []).map((s) => [s.match.toLowerCase(), s.model_slug]));
|
|
7
15
|
const bySlug = new Map((snapshot.models || []).map((m) => [m.slug, m]));
|
|
16
|
+
const keys = [...exact.keys()];
|
|
8
17
|
return strings.map((s) => {
|
|
9
|
-
const
|
|
10
|
-
const
|
|
11
|
-
|
|
18
|
+
const n = String(s).toLowerCase();
|
|
19
|
+
const slug = exact.get(n) || null;
|
|
20
|
+
if (slug) {
|
|
21
|
+
const model = bySlug.get(slug) || null;
|
|
22
|
+
return { input: s, model_slug: slug, display: model?.display ?? s, model, confidence: 1, suggestion: null };
|
|
23
|
+
}
|
|
24
|
+
// No exact hit — find the nearest known id (longest two-way substring overlap).
|
|
25
|
+
let near = null;
|
|
26
|
+
for (const k of keys) {
|
|
27
|
+
if (k.length >= 4 && (k.includes(n) || n.includes(k)) && (!near || k.length > near.length)) near = k;
|
|
28
|
+
}
|
|
29
|
+
const suggestion = near ? exact.get(near) || null : null;
|
|
30
|
+
return { input: s, model_slug: null, display: s, model: null, confidence: suggestion ? 0.6 : 0, suggestion };
|
|
12
31
|
});
|
|
13
32
|
}
|
|
14
33
|
|
|
Binary file
|
package/src/telemetry.js
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
/* Anonymous, opt-out usage analytics for the CLI/TUI (PostHog).
|
|
2
|
+
*
|
|
3
|
+
* Privacy: we send event NAMES + counts + version/OS only — never code, model
|
|
4
|
+
* strings, file paths, repo names, or your API key. The distinct id is a random
|
|
5
|
+
* per-machine UUID stored in the config (not tied to your account). Disable with
|
|
6
|
+
* any of MM_NO_ANALYTICS=1, DO_NOT_TRACK=1, or CI=1. The PostHog project token is
|
|
7
|
+
* a public ingest key (the same kind shipped in web bundles), baked at build via
|
|
8
|
+
* --define __POSTHOG_KEY__; with no key, every function here is a silent no-op. */
|
|
9
|
+
import crypto from "node:crypto";
|
|
10
|
+
import { loadConfig, setConfigValue } from "./config.js";
|
|
11
|
+
import { BUILD_VERSION, UPDATE_CHANNEL } from "./version.js";
|
|
12
|
+
|
|
13
|
+
// eslint-disable-next-line no-undef
|
|
14
|
+
const POSTHOG_KEY = typeof __POSTHOG_KEY__ !== "undefined" ? __POSTHOG_KEY__ : (process.env.MM_POSTHOG_KEY || "");
|
|
15
|
+
const POSTHOG_HOST = (process.env.MM_POSTHOG_HOST || "https://us.i.posthog.com").replace(/\/$/, "");
|
|
16
|
+
|
|
17
|
+
const optedOut = () => !!(process.env.MM_NO_ANALYTICS || process.env.DO_NOT_TRACK || process.env.CI);
|
|
18
|
+
const enabled = () => !!POSTHOG_KEY && !optedOut();
|
|
19
|
+
|
|
20
|
+
let cachedId = null;
|
|
21
|
+
function distinctId() {
|
|
22
|
+
if (cachedId) return cachedId;
|
|
23
|
+
const cfg = loadConfig();
|
|
24
|
+
cachedId = cfg.anonId || "cli_" + crypto.randomUUID();
|
|
25
|
+
if (!cfg.anonId) { try { setConfigValue("anonId", cachedId); } catch { /* best effort */ } }
|
|
26
|
+
return cachedId;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/** One-time, pre-TUI stderr disclosure. Honors opt-out + only shows once. */
|
|
30
|
+
export function maybeAnalyticsNotice() {
|
|
31
|
+
if (!enabled()) return;
|
|
32
|
+
if (loadConfig().analyticsNoticeShown) return;
|
|
33
|
+
try { setConfigValue("analyticsNoticeShown", true); } catch { /* best effort */ }
|
|
34
|
+
process.stderr.write(
|
|
35
|
+
"ℹ llmstatus sends anonymous usage analytics (event names + counts only — never code, model names, or paths). Opt out: MM_NO_ANALYTICS=1\n",
|
|
36
|
+
);
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/** Fire-and-forget capture. Never throws, never blocks the CLI meaningfully. */
|
|
40
|
+
export function track(event, properties = {}) {
|
|
41
|
+
if (!enabled()) return;
|
|
42
|
+
try {
|
|
43
|
+
const ctrl = new AbortController();
|
|
44
|
+
const timer = setTimeout(() => ctrl.abort(), 1500);
|
|
45
|
+
fetch(`${POSTHOG_HOST}/i/v0/e/`, {
|
|
46
|
+
method: "POST",
|
|
47
|
+
headers: { "content-type": "application/json" },
|
|
48
|
+
body: JSON.stringify({
|
|
49
|
+
api_key: POSTHOG_KEY,
|
|
50
|
+
event,
|
|
51
|
+
distinct_id: distinctId(),
|
|
52
|
+
properties: {
|
|
53
|
+
$process_person_profile: false, // anonymous events, no person profiles
|
|
54
|
+
cli_version: BUILD_VERSION,
|
|
55
|
+
channel: UPDATE_CHANNEL,
|
|
56
|
+
os: process.platform,
|
|
57
|
+
arch: process.arch,
|
|
58
|
+
...properties,
|
|
59
|
+
},
|
|
60
|
+
timestamp: new Date().toISOString(),
|
|
61
|
+
}),
|
|
62
|
+
signal: ctrl.signal,
|
|
63
|
+
keepalive: true,
|
|
64
|
+
}).catch(() => {}).finally(() => clearTimeout(timer));
|
|
65
|
+
} catch { /* analytics must never break the CLI */ }
|
|
66
|
+
}
|