@modelstatus/cli 0.1.1 → 0.1.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/api.js +3 -0
- package/src/ci.js +143 -0
- package/src/detect/core.js +48 -4
- package/src/index.js +204 -11
- package/src/openUrl.js +43 -1
- package/src/registry/local.js +23 -4
- package/src/sources/filesystem.js +0 -0
- package/src/telemetry.js +66 -0
- package/src/tui/app.js +173 -91
- package/src/tui/scan-stream.js +234 -0
- package/src/tui/signin.js +142 -0
- package/src/tui/snippet.js +127 -0
- package/src/tui/ui.js +661 -16
- package/src/tui/views/account.js +43 -13
- package/src/tui/views/add.js +33 -11
- package/src/tui/views/alerts.js +91 -39
- package/src/tui/views/inventory.js +149 -47
- package/src/tui/views/local.js +229 -0
- package/src/tui/views/scan.js +231 -72
- package/src/tui/views/whatsnew.js +92 -50
- package/src/updater.js +170 -0
- package/src/version.js +32 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@modelstatus/cli",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.26",
|
|
4
4
|
"description": "Track which AI models you use, where, and never get surprised by a retirement. Free offline model-health for any repo (mm status), browser sign-in for cloud inventory + alerts.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"llm",
|
package/src/api.js
CHANGED
|
@@ -64,6 +64,9 @@ export function createClient({ apiBase, apiKey }) {
|
|
|
64
64
|
linkUsage: (id, modelId) => req("POST", `/usages/${id}/link`, { model_id: modelId }),
|
|
65
65
|
bulkUpload: (projectId, usages) => req("POST", "/usages/bulk", { project_id: projectId, usages }),
|
|
66
66
|
|
|
67
|
+
// ci runs (Pro) — record a CI evaluation for the dashboard + alerts
|
|
68
|
+
ciRun: (body) => req("POST", "/ci/runs", body),
|
|
69
|
+
|
|
67
70
|
// notification rules
|
|
68
71
|
listRules: () => req("GET", "/notification-rules"),
|
|
69
72
|
createRule: (body) => req("POST", "/notification-rules", body),
|
package/src/ci.js
ADDED
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
/* CI evaluation engine: scan a repo, resolve against the signed registry, and
|
|
2
|
+
* report any deprecated/retiring/retired AI-model usage with file:line. Pure
|
|
3
|
+
* (no process.exit / no direct GitHub I/O) so it's testable; the `mm ci` command
|
|
4
|
+
* in index.js turns the result into GitHub annotations + a step summary + an
|
|
5
|
+
* exit code. Offline-capable — uses the public signed registry, no account. */
|
|
6
|
+
import { execFileSync } from "node:child_process";
|
|
7
|
+
import path from "node:path";
|
|
8
|
+
import { getRegistry } from "./registry/fetch.js";
|
|
9
|
+
import { resolveLocal, computeHealth } from "./registry/local.js";
|
|
10
|
+
import { collectFrom } from "./sources/index.js";
|
|
11
|
+
|
|
12
|
+
export const HEALTH_RANK = { ok: 0, deprecating: 1, retiring: 2, retired: 3 };
|
|
13
|
+
// `--fail-on` threshold → the minimum health rank that fails the build.
|
|
14
|
+
export const FAIL_THRESHOLD = { none: 99, deprecating: 1, retiring: 2, retired: 3 };
|
|
15
|
+
const BADGE = { ok: "🟢", deprecating: "🟡", retiring: "🟠", retired: "🔴" };
|
|
16
|
+
|
|
17
|
+
/** Evaluate `dir` and return { findings, failing, threshold, failOn, counts, snapshot }.
|
|
18
|
+
* `findings` are per-(model, location) entries with health worse than ok. */
|
|
19
|
+
export async function evaluateCi({ dir, sources = ["filesystem"], scanOpts = {}, failOn = "retired", offline = false, log = () => {} }) {
|
|
20
|
+
// Online by default so CI checks against the LATEST registry; --offline (or the
|
|
21
|
+
// env) falls back to the cached snapshot. cacheFile env keeps tests hermetic.
|
|
22
|
+
const snapshot = await getRegistry({
|
|
23
|
+
offline: offline || process.env.LLMSTATUS_REGISTRY_OFFLINE === "1",
|
|
24
|
+
cacheFile: process.env.LLMSTATUS_REGISTRY_CACHE || undefined,
|
|
25
|
+
log: (m) => log(`${m}\n`),
|
|
26
|
+
});
|
|
27
|
+
const candidates = await collectFrom(sources, { root: dir, ...scanOpts }, snapshot.detection);
|
|
28
|
+
const resolved = resolveLocal(snapshot, [...new Set(candidates.map((c) => c.model_string))]);
|
|
29
|
+
const byStr = new Map(resolved.map((r) => [r.input.toLowerCase(), r]));
|
|
30
|
+
const today = new Date();
|
|
31
|
+
const threshold = FAIL_THRESHOLD[failOn] ?? FAIL_THRESHOLD.retired;
|
|
32
|
+
|
|
33
|
+
const seen = new Set();
|
|
34
|
+
const findings = [];
|
|
35
|
+
for (const c of candidates) {
|
|
36
|
+
const r = byStr.get(c.model_string.toLowerCase());
|
|
37
|
+
if (!r?.model || !r.model_slug) continue; // only registry-known models can be scored
|
|
38
|
+
const health = computeHealth(r.model, 90, today);
|
|
39
|
+
if (health === "ok") continue;
|
|
40
|
+
const loc = `${c.source_path || c.location_label || ""}:${c.source_line || ""}`;
|
|
41
|
+
const key = r.model_slug + "|" + loc;
|
|
42
|
+
if (seen.has(key)) continue;
|
|
43
|
+
seen.add(key);
|
|
44
|
+
findings.push({
|
|
45
|
+
slug: r.model_slug,
|
|
46
|
+
health,
|
|
47
|
+
retires: r.model.retires_date || null,
|
|
48
|
+
replacement: r.model.replacement_slug || null,
|
|
49
|
+
path: c.source_path || null,
|
|
50
|
+
line: c.source_line || null,
|
|
51
|
+
location: c.location_label || c.source_path || c.model_string,
|
|
52
|
+
env: c.environment || null,
|
|
53
|
+
});
|
|
54
|
+
}
|
|
55
|
+
findings.sort((a, b) => HEALTH_RANK[b.health] - HEALTH_RANK[a.health] || String(a.retires || "9999").localeCompare(String(b.retires || "9999")));
|
|
56
|
+
|
|
57
|
+
const counts = { ok: 0, deprecating: 0, retiring: 0, retired: 0 };
|
|
58
|
+
for (const f of findings) counts[f.health]++;
|
|
59
|
+
const failing = findings.filter((f) => HEALTH_RANK[f.health] >= threshold);
|
|
60
|
+
return { snapshot, candidates, findings, failing, threshold, failOn, counts };
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* Pure filter: keep only findings whose `path` is in `changedPaths`. `changedPaths`
|
|
65
|
+
* is a Set of dir-relative posix paths (the same shape as `finding.path`). A `null`
|
|
66
|
+
* (or undefined) set means "couldn't determine the diff" → return findings unchanged
|
|
67
|
+
* (no filtering). Findings without a `path` are always dropped in diff mode, since we
|
|
68
|
+
* can't prove they belong to a changed file. No git / no I/O — trivially testable.
|
|
69
|
+
*/
|
|
70
|
+
export function filterToChangedFiles(findings, changedPaths) {
|
|
71
|
+
if (!changedPaths) return findings; // null/undefined → no diff info → don't filter
|
|
72
|
+
const norm = (p) => String(p || "").replace(/\\/g, "/").replace(/^\.\//, "");
|
|
73
|
+
const want = new Set([...changedPaths].map(norm));
|
|
74
|
+
return findings.filter((f) => f.path && want.has(norm(f.path)));
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Run `git diff --name-only <base>...HEAD` in `dir` and return a Set of paths made
|
|
79
|
+
* RELATIVE TO `dir` (so they match `finding.path`, which is dir-relative). git reports
|
|
80
|
+
* repo-root-relative paths, so each is resolved against the repo root then re-based on
|
|
81
|
+
* `dir`. Returns `null` (caller falls back to the full finding set) when git is missing,
|
|
82
|
+
* `dir` is not a repo, or `base` is unknown — i.e. on ANY git failure. Optional `log`
|
|
83
|
+
* receives a human note on fallback. Synchronous + offline (pure local git).
|
|
84
|
+
*/
|
|
85
|
+
export function getChangedFiles(dir, base, { log = () => {}, git = "git" } = {}) {
|
|
86
|
+
if (!base) return null;
|
|
87
|
+
const run = (args) => execFileSync(git, args, { cwd: dir, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] });
|
|
88
|
+
let repoRoot;
|
|
89
|
+
try {
|
|
90
|
+
repoRoot = run(["rev-parse", "--show-toplevel"]).trim();
|
|
91
|
+
} catch {
|
|
92
|
+
log(`--diff: not a git repo (or git not installed) — scanning all files.\n`);
|
|
93
|
+
return null;
|
|
94
|
+
}
|
|
95
|
+
let out;
|
|
96
|
+
try {
|
|
97
|
+
out = run(["diff", "--name-only", `${base}...HEAD`]);
|
|
98
|
+
} catch {
|
|
99
|
+
log(`--diff: couldn't diff against "${base}" (unknown ref?) — scanning all files.\n`);
|
|
100
|
+
return null;
|
|
101
|
+
}
|
|
102
|
+
const set = new Set();
|
|
103
|
+
for (const line of out.split(/\r?\n/)) {
|
|
104
|
+
const rel = line.trim();
|
|
105
|
+
if (!rel) continue;
|
|
106
|
+
// git path is repo-root-relative → make it dir-relative to match finding.path.
|
|
107
|
+
const relToDir = path.relative(dir, path.resolve(repoRoot, rel)).replace(/\\/g, "/");
|
|
108
|
+
set.add(relToDir);
|
|
109
|
+
}
|
|
110
|
+
return set;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
const ann = (s) => String(s).replace(/%/g, "%25").replace(/\r/g, "%0D").replace(/\n/g, "%0A");
|
|
114
|
+
|
|
115
|
+
/** GitHub Actions workflow-command annotations (one per finding). Inline on the PR. */
|
|
116
|
+
export function annotationLines(findings, threshold) {
|
|
117
|
+
return findings.map((f) => {
|
|
118
|
+
const sev = HEALTH_RANK[f.health] >= threshold ? "error" : "warning";
|
|
119
|
+
const where = f.path ? `file=${ann(f.path)},line=${f.line || 1},` : "";
|
|
120
|
+
const msg = `${f.slug} is ${f.health}${f.retires ? ` (retires ${f.retires})` : ""}${f.replacement ? ` — switch to ${f.replacement}` : ""}`;
|
|
121
|
+
return `::${sev} ${where}title=AI model ${f.health}::${ann(msg)}`;
|
|
122
|
+
});
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/** Markdown for $GITHUB_STEP_SUMMARY (and a decent human report). */
|
|
126
|
+
export function summaryMarkdown(findings, { failing, failOn } = {}) {
|
|
127
|
+
const lines = ["## LLM Status — model lifecycle check", ""];
|
|
128
|
+
if (!findings.length) {
|
|
129
|
+
lines.push("✅ No deprecated, retiring, or retired AI models found.");
|
|
130
|
+
return lines.join("\n") + "\n";
|
|
131
|
+
}
|
|
132
|
+
const fc = failing ? failing.length : 0;
|
|
133
|
+
lines.push(fc ? `🔴 **${fc}** usage(s) at or past the \`${failOn}\` threshold — this check fails.` : `Found ${findings.length} aging model usage(s) (below the \`${failOn}\` fail threshold).`, "");
|
|
134
|
+
lines.push("| | Model | Where | Retires | Replacement |", "|---|---|---|---|---|");
|
|
135
|
+
for (const f of findings.slice(0, 100)) {
|
|
136
|
+
lines.push(`| ${BADGE[f.health]} ${f.health} | \`${f.slug}\` | \`${f.location || ""}\` | ${f.retires || "—"} | ${f.replacement ? `\`${f.replacement}\`` : "—"} |`);
|
|
137
|
+
}
|
|
138
|
+
if (findings.length > 100) lines.push(`\n…and ${findings.length - 100} more.`);
|
|
139
|
+
lines.push("", "_Powered by [LLM Status](https://llmstatus.ai)._");
|
|
140
|
+
return lines.join("\n") + "\n";
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
export { BADGE };
|
package/src/detect/core.js
CHANGED
|
@@ -3,15 +3,53 @@
|
|
|
3
3
|
* returns the model strings found per line. No I/O. */
|
|
4
4
|
|
|
5
5
|
// File extensions / TLDs the family globs accidentally swallow
|
|
6
|
-
// (e.g. "command-2.0.0.tgz", "grok-free.app"). Used to reject
|
|
6
|
+
// (e.g. "command-2.0.0.tgz", "grok-free.app", "llama-3.gguf"). Used to reject
|
|
7
|
+
// generic matches. Includes model-WEIGHT/data/media extensions so a weight-file
|
|
8
|
+
// reference (llama-3.safetensors) isn't mistaken for a model usage.
|
|
7
9
|
const BANNED_TAIL =
|
|
8
|
-
/\.(tgz|tar|gz|zip|js|ts|tsx|jsx|mjs|py|go|rb|json|md|lock|sh|css|html|txt|log|yaml|yml|toml|ini|conf|cfg|env|pem|crt|key|csv|xml|pdf|sql|app|com|net|io|dev|org|ai|co)\b/;
|
|
10
|
+
/\.(tgz|tar|gz|zip|js|ts|tsx|jsx|mjs|py|go|rb|json|md|lock|sh|css|html|txt|log|yaml|yml|toml|ini|conf|cfg|env|pem|crt|key|csv|xml|pdf|sql|gguf|safetensors|bin|onnx|pt|pth|ckpt|h5|npz|parquet|arrow|jpeg|jpg|png|gif|webp|bmp|svg|mp4|wav|app|com|net|io|dev|org|ai|co)\b/;
|
|
9
11
|
|
|
10
12
|
/** Trim leading/trailing separators a greedy family glob can capture. */
|
|
11
13
|
function cleanGeneric(s) {
|
|
12
14
|
return s.replace(/^[.\-_]+/, "").replace(/[.\-_]+$/, "");
|
|
13
15
|
}
|
|
14
16
|
|
|
17
|
+
/** A char is part of a model token if it's alnum or one of - _ . / : (slug-ish).
|
|
18
|
+
* Mirrors the server PR scanner (apps/web/lib/github/scan-pr.ts) so the CLI and
|
|
19
|
+
* the GitHub check agree on what counts as a boundary. */
|
|
20
|
+
function isTokenChar(ch) {
|
|
21
|
+
return /[A-Za-z0-9._/:-]/.test(ch);
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
// Provider prefixes legitimately precede an id ("anthropic.claude-…", "ft:gpt-…",
|
|
25
|
+
// "us.anthropic.…", "openrouter/…"), so '.' ':' '/' on the LEFT is still a boundary.
|
|
26
|
+
function isPrefixSep(ch) {
|
|
27
|
+
return ch === "." || ch === ":" || ch === "/";
|
|
28
|
+
}
|
|
29
|
+
// Known model-id SUFFIXES seen in real configs: Bedrock ':0'/'-v1', dated
|
|
30
|
+
// '-20250514' snapshots, '@version'. The remainder starting with one is still a
|
|
31
|
+
// boundary — so "claude-opus-4-20250514" resolves inside a Bedrock ARN, while
|
|
32
|
+
// "gpt-4" still does NOT match inside "gpt-4o". Kept identical to scan-pr.ts.
|
|
33
|
+
const MODEL_SUFFIX = /^(:|-v[0-9]|-[0-9]{6,}|@)/;
|
|
34
|
+
|
|
35
|
+
/** True when `term` occurs in `haystack` at a model-id boundary — tolerating
|
|
36
|
+
* provider prefixes + known version/region/snapshot suffixes, but NOT a plain
|
|
37
|
+
* embedded match. Both are already lower-cased. */
|
|
38
|
+
function matchesAtBoundary(haystack, term) {
|
|
39
|
+
let from = 0;
|
|
40
|
+
for (;;) {
|
|
41
|
+
const at = haystack.indexOf(term, from);
|
|
42
|
+
if (at < 0) return false;
|
|
43
|
+
const before = at > 0 ? haystack[at - 1] : "";
|
|
44
|
+
const rest = haystack.slice(at + term.length);
|
|
45
|
+
const after = rest[0] ?? "";
|
|
46
|
+
const boundedLeft = before === "" || !isTokenChar(before) || isPrefixSep(before);
|
|
47
|
+
const boundedRight = after === "" || !isTokenChar(after) || MODEL_SUFFIX.test(rest);
|
|
48
|
+
if (boundedLeft && boundedRight) return true;
|
|
49
|
+
from = at + 1; // a later occurrence may be bounded
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
|
|
15
53
|
/** Family globs catch brand-NEW versioned models before they're in the
|
|
16
54
|
* registry, so a real hit virtually always carries a version digit. Requiring
|
|
17
55
|
* one (and rejecting filename/domain tails) kills the bulk of false positives. */
|
|
@@ -23,7 +61,10 @@ function looksLikeModel(s) {
|
|
|
23
61
|
export function compilePatterns(patterns) {
|
|
24
62
|
const exact = [];
|
|
25
63
|
for (const ms of patterns.model_strings || []) {
|
|
26
|
-
|
|
64
|
+
// Registry strings are curated, and the boundary matcher prevents embedded
|
|
65
|
+
// matches — so a low floor is safe and lets short real ids (o1, o3) resolve.
|
|
66
|
+
// (The old >=4 floor silently dropped the entire OpenAI o-series.)
|
|
67
|
+
if (ms.match && ms.match.length >= 2) exact.push(ms.match.toLowerCase());
|
|
27
68
|
}
|
|
28
69
|
const generic = (patterns.generic_model_regexes || []).map((r) => new RegExp(r, "gi"));
|
|
29
70
|
return { exact, generic };
|
|
@@ -33,7 +74,10 @@ export function compilePatterns(patterns) {
|
|
|
33
74
|
export function detectInLine(line, compiled) {
|
|
34
75
|
const lower = line.toLowerCase();
|
|
35
76
|
const found = new Set();
|
|
36
|
-
|
|
77
|
+
// Exact registry strings must match at a token boundary, not as a raw substring
|
|
78
|
+
// — otherwise a short alias ("gpt-4") resolves inside a longer id ("gpt-4o-mini")
|
|
79
|
+
// to the wrong model. Matches the server PR scanner's boundary semantics.
|
|
80
|
+
for (const s of compiled.exact) if (matchesAtBoundary(lower, s)) found.add(s);
|
|
37
81
|
for (const re of compiled.generic) {
|
|
38
82
|
re.lastIndex = 0;
|
|
39
83
|
let m;
|
package/src/index.js
CHANGED
|
@@ -1,15 +1,20 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
+
import fs from "node:fs";
|
|
2
3
|
import path from "node:path";
|
|
3
4
|
import { resolveAuth, loadConfig, saveConfig, clearAuth, configFilePath } from "./config.js";
|
|
4
5
|
import { createClient } from "./api.js";
|
|
5
6
|
import { collectFrom, availability, ALL_SOURCE_IDS } from "./sources/index.js";
|
|
7
|
+
import { redactValue } from "./redact.js";
|
|
6
8
|
import { loginViaBrowser } from "./auth.js";
|
|
9
|
+
import { maybeCheckForUpdate } from "./updater.js";
|
|
10
|
+
import { track, maybeAnalyticsNotice } from "./telemetry.js";
|
|
11
|
+
import { BUILD_VERSION } from "./version.js";
|
|
7
12
|
|
|
8
13
|
function parseArgs(argv) {
|
|
9
14
|
const flags = {};
|
|
10
15
|
const positional = [];
|
|
11
16
|
const valueFlags = new Set([
|
|
12
|
-
"api", "key", "project", "dir",
|
|
17
|
+
"api", "key", "project", "dir", "fail-on", "diff", "json-out",
|
|
13
18
|
"sources", "region", "namespace", "kube-context", "db", "sql-table", "env",
|
|
14
19
|
]);
|
|
15
20
|
for (let i = 0; i < argv.length; i++) {
|
|
@@ -29,6 +34,12 @@ function parseArgs(argv) {
|
|
|
29
34
|
|
|
30
35
|
const uuidish = (s) => /^[0-9a-f]{8}-[0-9a-f]{4}-/i.test(s || "");
|
|
31
36
|
|
|
37
|
+
/** The "owner/name" repo slug for source deep-links. In GitHub Actions
|
|
38
|
+
* GITHUB_REPOSITORY is exactly that and the checkout is repo-root-relative (so
|
|
39
|
+
* our source_path lines up). Outside CI we return "" (omit it) rather than guess
|
|
40
|
+
* from a git remote, since a local scan root may not be the repo root. */
|
|
41
|
+
const ghRepoSlug = () => (process.env.GITHUB_REPOSITORY || "").trim();
|
|
42
|
+
|
|
32
43
|
/** Resolve the requested sources: default filesystem, "all", or a comma list. */
|
|
33
44
|
function parseSources(flags) {
|
|
34
45
|
const raw = (flags.sources || "").trim();
|
|
@@ -95,14 +106,16 @@ async function cmdUpgrade(_positional, flags) {
|
|
|
95
106
|
}
|
|
96
107
|
|
|
97
108
|
async function launchTui(initialView, flags) {
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
}
|
|
109
|
+
// Pass apiKey straight through (may be null). The TUI's Bootstrap wrapper
|
|
110
|
+
// renders an interactive SignIn screen when there's no key, then swaps to
|
|
111
|
+
// the main App on success — no separate browser-login polling phase that the
|
|
112
|
+
// user has to wait through without seeing anything interactive.
|
|
113
|
+
const { apiBase, apiKey } = resolveAuth(flags);
|
|
103
114
|
const dir = path.resolve(flags.dir || ".");
|
|
104
115
|
const { runApp } = await import("./tui/app.js");
|
|
105
|
-
|
|
116
|
+
// --scan / --rescan / --fresh force a fresh filesystem walk instead of loading
|
|
117
|
+
// the last persisted scan.
|
|
118
|
+
await runApp({ apiBase, apiKey, dir, initialView, fresh: !!(flags.scan || flags.rescan || flags.fresh) });
|
|
106
119
|
}
|
|
107
120
|
|
|
108
121
|
async function cmdScan(positional, flags) {
|
|
@@ -158,9 +171,12 @@ async function cmdScan(positional, flags) {
|
|
|
158
171
|
|
|
159
172
|
const usages = rows.map((r) => ({
|
|
160
173
|
model_id: r.model_id ?? undefined,
|
|
161
|
-
|
|
174
|
+
// Redact + bound the custom id: a generic-glob hit on an .env line can over-
|
|
175
|
+
// capture a secret-ish fragment, and only the snippet was being redacted.
|
|
176
|
+
custom_model_name: r.model_id ? undefined : redactValue(r.model_string).slice(0, 120),
|
|
162
177
|
environment: r.environment,
|
|
163
178
|
location_label: r.location_label,
|
|
179
|
+
source_repo: ghRepoSlug() || undefined,
|
|
164
180
|
source_path: r.source_path,
|
|
165
181
|
source_line: r.source_line ?? undefined,
|
|
166
182
|
}));
|
|
@@ -190,6 +206,7 @@ async function cmdScan(positional, flags) {
|
|
|
190
206
|
}
|
|
191
207
|
|
|
192
208
|
const res = await client.bulkUpload(projectId, usages);
|
|
209
|
+
track("usages_uploaded", { count: usages.length, created: res.created, updated: res.updated, source: "cli" });
|
|
193
210
|
if (flags.json) {
|
|
194
211
|
console.log(
|
|
195
212
|
JSON.stringify(
|
|
@@ -205,6 +222,144 @@ async function cmdScan(positional, flags) {
|
|
|
205
222
|
}
|
|
206
223
|
}
|
|
207
224
|
|
|
225
|
+
/** Pro cloud sync for `mm ci --report`: push this run's usages to the account so
|
|
226
|
+
* the dashboard tracks per-commit drift + alerts can fire. Plan-gated server-side
|
|
227
|
+
* (402 → skipped, the local check still ran). Best-effort: never fails the build. */
|
|
228
|
+
async function ciReport(dir, flags, res) {
|
|
229
|
+
const { apiBase, apiKey } = resolveAuth(flags);
|
|
230
|
+
if (!apiKey) {
|
|
231
|
+
process.stderr.write("! --report needs an API key (set LLMSTATUS_API_KEY) — skipping cloud sync.\n");
|
|
232
|
+
return null;
|
|
233
|
+
}
|
|
234
|
+
const client = createClient({ apiBase, apiKey });
|
|
235
|
+
const uniq = [...new Set(res.candidates.map((c) => c.model_string))];
|
|
236
|
+
const resolved = uniq.length ? (await client.resolve(uniq)).data || [] : [];
|
|
237
|
+
const byStr = new Map(resolved.map((r) => [r.input.toLowerCase(), r]));
|
|
238
|
+
const seen = new Set();
|
|
239
|
+
const usages = [];
|
|
240
|
+
for (const c of res.candidates) {
|
|
241
|
+
const r = byStr.get(c.model_string.toLowerCase());
|
|
242
|
+
const k = `${r?.model_id ?? "custom:" + c.model_string}|${c.location_label}`;
|
|
243
|
+
if (seen.has(k)) continue;
|
|
244
|
+
seen.add(k);
|
|
245
|
+
usages.push({
|
|
246
|
+
model_id: r?.model_id ?? undefined,
|
|
247
|
+
custom_model_name: r?.model_id ? undefined : redactValue(c.model_string).slice(0, 120),
|
|
248
|
+
environment: c.environment,
|
|
249
|
+
location_label: c.location_label,
|
|
250
|
+
source_path: c.source_path,
|
|
251
|
+
source_line: c.source_line ?? undefined,
|
|
252
|
+
});
|
|
253
|
+
}
|
|
254
|
+
try {
|
|
255
|
+
const projectName = flags.project || (process.env.GITHUB_REPOSITORY || "").split("/").pop() || path.basename(dir);
|
|
256
|
+
const projects = (await client.listProjects()).data || [];
|
|
257
|
+
const projectId = projects.find((p) => p.name === projectName)?.id || (await client.createProject(projectName)).id;
|
|
258
|
+
const r = await client.bulkUpload(projectId, usages);
|
|
259
|
+
process.stderr.write(`✓ Reported ${usages.length} usage(s) to LLM Status (project "${projectName}").\n`);
|
|
260
|
+
return { project: projectName, ...r };
|
|
261
|
+
} catch (e) {
|
|
262
|
+
if (e.status === 402) process.stderr.write("! Cloud CI reporting is a Pro feature — the local check still ran. Upgrade with: mm upgrade\n");
|
|
263
|
+
else process.stderr.write(`! Cloud report skipped: ${e.message}\n`);
|
|
264
|
+
return null;
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
/** Record this run's summary to /ci/runs (the dashboard's source). Pulls commit/
|
|
269
|
+
* branch/repo from the GitHub Actions env (sensible local fallbacks). Best-effort,
|
|
270
|
+
* Pro-gated server-side (402 → skipped). Never throws into the CI gate. */
|
|
271
|
+
async function postCiRun(dir, flags, res, failOn) {
|
|
272
|
+
const { apiBase, apiKey } = resolveAuth(flags);
|
|
273
|
+
if (!apiKey) return null;
|
|
274
|
+
const env = process.env;
|
|
275
|
+
const client = createClient({ apiBase, apiKey });
|
|
276
|
+
try {
|
|
277
|
+
await client.ciRun({
|
|
278
|
+
commit_sha: env.GITHUB_SHA || "local",
|
|
279
|
+
branch: env.GITHUB_REF_NAME || "",
|
|
280
|
+
ref: env.GITHUB_REF || "",
|
|
281
|
+
repo: env.GITHUB_REPOSITORY || path.basename(path.resolve(dir)),
|
|
282
|
+
counts: res.counts,
|
|
283
|
+
fail_on: failOn,
|
|
284
|
+
failing_count: res.failing.length,
|
|
285
|
+
findings: res.findings.slice(0, 200),
|
|
286
|
+
});
|
|
287
|
+
return true;
|
|
288
|
+
} catch (e) {
|
|
289
|
+
if (e.status === 402) process.stderr.write("! CI run recording is a Pro feature — the local check still ran. Upgrade: mm upgrade\n");
|
|
290
|
+
return null;
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
/** `mm ci [dir]` — evaluate the repo for deprecated/retiring models. Emits GitHub
|
|
295
|
+
* annotations + a step summary under GITHUB_ACTIONS; exits non-zero per --fail-on
|
|
296
|
+
* so it gates merges. Offline (public registry) by default; --report syncs (Pro). */
|
|
297
|
+
async function cmdCi(positional, flags) {
|
|
298
|
+
const dir = path.resolve(positional[1] || flags.dir || ".");
|
|
299
|
+
const failOn = String(flags["fail-on"] || "retired").toLowerCase();
|
|
300
|
+
const { evaluateCi, annotationLines, summaryMarkdown, filterToChangedFiles, getChangedFiles, HEALTH_RANK } = await import("./ci.js");
|
|
301
|
+
const res = await evaluateCi({
|
|
302
|
+
dir,
|
|
303
|
+
sources: parseSources(flags),
|
|
304
|
+
scanOpts: scanOpts(flags, dir),
|
|
305
|
+
failOn,
|
|
306
|
+
offline: !!flags.offline,
|
|
307
|
+
log: (m) => process.stderr.write(`! ${m}`),
|
|
308
|
+
});
|
|
309
|
+
let { findings, failing, threshold, counts } = res;
|
|
310
|
+
|
|
311
|
+
// --diff <base> (or a GitHub PR: GITHUB_BASE_REF as base, HEAD as head) restricts
|
|
312
|
+
// findings to files CHANGED vs base — we still scan the whole repo, then drop findings
|
|
313
|
+
// whose source_path isn't in the changed set. getChangedFiles returns null on any git
|
|
314
|
+
// failure (missing git / not a repo / bad base), and filterToChangedFiles treats null
|
|
315
|
+
// as "don't filter", so we transparently fall back to the full set with a stderr note.
|
|
316
|
+
const diffBase = flags.diff && flags.diff !== true ? flags.diff : process.env.GITHUB_BASE_REF || null;
|
|
317
|
+
if (diffBase) {
|
|
318
|
+
const changed = getChangedFiles(dir, diffBase, { log: (m) => process.stderr.write(`! ${m}`) });
|
|
319
|
+
if (changed) {
|
|
320
|
+
findings = filterToChangedFiles(findings, changed);
|
|
321
|
+
failing = findings.filter((f) => HEALTH_RANK[f.health] >= threshold);
|
|
322
|
+
counts = { ok: 0, deprecating: 0, retiring: 0, retired: 0 };
|
|
323
|
+
for (const f of findings) counts[f.health]++;
|
|
324
|
+
process.stderr.write(`! --diff ${diffBase}: ${findings.length} finding(s) in ${changed.size} changed file(s).\n`);
|
|
325
|
+
}
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
if (process.env.GITHUB_ACTIONS) {
|
|
329
|
+
for (const a of annotationLines(findings, threshold)) console.log(a);
|
|
330
|
+
if (process.env.GITHUB_STEP_SUMMARY) {
|
|
331
|
+
try { fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, summaryMarkdown(findings, { failing, failOn })); } catch { /* best effort */ }
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
// Record the run to the account (Pro): inventory drift (bulkUpload) + a CI-run
|
|
336
|
+
// row for the dashboard. Best-effort, plan-gated server-side, never fails CI.
|
|
337
|
+
const reported = flags.report ? await ciReport(dir, flags, res).catch(() => null) : null;
|
|
338
|
+
if (flags.report) await postCiRun(dir, flags, res, failOn).catch(() => null);
|
|
339
|
+
|
|
340
|
+
const report = { scanned: dir, fail_on: failOn, counts, findings, failing: failing.length, reported };
|
|
341
|
+
// --json-out writes CLEAN JSON to a file (annotations still go to stdout for
|
|
342
|
+
// GitHub) so a CI step can parse it without stdout pollution.
|
|
343
|
+
if (flags["json-out"]) { try { fs.writeFileSync(flags["json-out"], JSON.stringify(report, null, 2)); } catch { /* best effort */ } }
|
|
344
|
+
|
|
345
|
+
if (flags.json) {
|
|
346
|
+
console.log(JSON.stringify(report, null, 2));
|
|
347
|
+
} else {
|
|
348
|
+
const ICON = { ok: "🟢", deprecating: "🟡", retiring: "🟠", retired: "🔴" };
|
|
349
|
+
console.log(`LLM Status CI — scanned ${dir} (fail-on: ${failOn})`);
|
|
350
|
+
if (!findings.length) {
|
|
351
|
+
console.log("✓ No deprecated, retiring, or retired AI models found.");
|
|
352
|
+
} else {
|
|
353
|
+
for (const f of findings) {
|
|
354
|
+
console.log(` ${ICON[f.health]} ${f.health.padEnd(11)} ${f.slug.padEnd(30)} ${String(f.location || "").padEnd(28)}${f.retires ? ` retires ${f.retires}` : ""}${f.replacement ? ` → ${f.replacement}` : ""}`);
|
|
355
|
+
}
|
|
356
|
+
console.log("");
|
|
357
|
+
console.log(failing.length ? `✗ ${failing.length} usage(s) at or past "${failOn}" — failing the build.` : `✓ ${findings.length} aging usage(s), none past "${failOn}".`);
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
if (failing.length) process.exit(1);
|
|
361
|
+
}
|
|
362
|
+
|
|
208
363
|
/** List detection sources and whether each can run right now. */
|
|
209
364
|
async function cmdSources(_positional, flags) {
|
|
210
365
|
const dir = path.resolve(flags.dir || ".");
|
|
@@ -292,15 +447,17 @@ async function cmdStatus(positional, flags) {
|
|
|
292
447
|
const HELP = `LLM Status CLI
|
|
293
448
|
|
|
294
449
|
Usage:
|
|
295
|
-
mm Launch the TUI (
|
|
450
|
+
mm Launch the TUI (signs you in via in-TUI device flow if needed)
|
|
296
451
|
mm status [dir] Offline model-health check for a dir — no account needed
|
|
297
452
|
mm login [api_key] Browser sign-in with polling (or paste a key)
|
|
298
453
|
mm signup Create an account in the browser, then poll
|
|
299
454
|
mm logout Forget the saved API key
|
|
300
455
|
mm scan [dir] Scan for model usage; interactive TUI, or --ci/--json for pipelines
|
|
456
|
+
mm ci [dir] CI gate: fail the build on deprecated/retiring models (GitHub annotations)
|
|
457
|
+
(--diff <base> limits findings to files changed vs base; auto on PRs via GITHUB_BASE_REF)
|
|
301
458
|
mm sources List detection sources and whether each can run here
|
|
302
459
|
mm upgrade Open Stripe checkout and poll until Pro is active
|
|
303
|
-
mm tui
|
|
460
|
+
mm tui Force-launch the TUI (logs you in first if needed)
|
|
304
461
|
|
|
305
462
|
Scan sources (--sources, default filesystem; "all" for everything):
|
|
306
463
|
filesystem repo files aws-secrets AWS Secrets Manager + SSM
|
|
@@ -314,24 +471,60 @@ Flags: --api <url> · --key <key> · --project <id|name> · --yes · --json · -
|
|
|
314
471
|
|
|
315
472
|
Get started: \`mm login\` (opens your browser).`;
|
|
316
473
|
|
|
474
|
+
/** Awaits the updater promise; prints a one-liner if an update completed. Never throws. */
|
|
475
|
+
async function maybePrintUpdate(promise) {
|
|
476
|
+
try {
|
|
477
|
+
const r = await promise;
|
|
478
|
+
if (!r) return;
|
|
479
|
+
if (r.manual) {
|
|
480
|
+
// Can't auto-update in place (root-owned dir like /usr/local/bin) — would
|
|
481
|
+
// be unsafe. Point the user at a one-line reinstall to a user-owned dir.
|
|
482
|
+
process.stderr.write(`\n✦ mm ${r.to} is available (you're on ${r.from}). Auto-update can't write ${process.execPath} — reinstall:\n curl -fsSL https://llmstatus.ai/install.sh | bash\n`);
|
|
483
|
+
} else {
|
|
484
|
+
process.stderr.write(`\n✓ Updated mm ${r.from} → ${r.to} — your next run uses the new version.\n`);
|
|
485
|
+
}
|
|
486
|
+
} catch {
|
|
487
|
+
/* swallow */
|
|
488
|
+
}
|
|
489
|
+
}
|
|
490
|
+
|
|
317
491
|
async function main() {
|
|
318
492
|
const { positional, flags } = parseArgs(process.argv.slice(2));
|
|
319
493
|
const cmd = positional[0];
|
|
494
|
+
|
|
495
|
+
// --version / -v: print + exit (skip dispatch + updater).
|
|
496
|
+
if (cmd === "version" || flags.version || flags.v) {
|
|
497
|
+
console.log(BUILD_VERSION);
|
|
498
|
+
return;
|
|
499
|
+
}
|
|
500
|
+
|
|
501
|
+
// Anonymous, opt-out usage analytics (one-time disclosure, then a single
|
|
502
|
+
// event per invocation). No-op without a baked key / when opted out.
|
|
503
|
+
maybeAnalyticsNotice();
|
|
504
|
+
track("cli_command", { command: cmd || "tui" });
|
|
505
|
+
|
|
506
|
+
// Kick off the background self-update check — runs in parallel with the
|
|
507
|
+
// user's command, capped at 1/24h, only for shell-installed binaries.
|
|
508
|
+
const updatePromise = maybeCheckForUpdate(flags);
|
|
509
|
+
|
|
320
510
|
try {
|
|
321
511
|
if (cmd === "login") await cmdLogin(positional, flags);
|
|
322
512
|
else if (cmd === "signup") await cmdSignup(positional, flags);
|
|
323
513
|
else if (cmd === "logout") cmdLogout();
|
|
324
514
|
else if (cmd === "scan") await cmdScan(positional, flags);
|
|
515
|
+
else if (cmd === "ci") await cmdCi(positional, flags);
|
|
325
516
|
else if (cmd === "status") await cmdStatus(positional, flags);
|
|
326
517
|
else if (cmd === "sources") await cmdSources(positional, flags);
|
|
327
518
|
else if (cmd === "upgrade") await cmdUpgrade(positional, flags);
|
|
328
|
-
else if (cmd === "tui" || !cmd) await launchTui(positional[1]
|
|
519
|
+
else if (cmd === "tui" || !cmd) await launchTui(positional[1], flags);
|
|
329
520
|
else if (cmd === "help" || flags.help) console.log(HELP);
|
|
330
521
|
else console.log(HELP);
|
|
331
522
|
} catch (e) {
|
|
332
523
|
console.error(`Error: ${e?.message ?? e}`);
|
|
524
|
+
await maybePrintUpdate(updatePromise);
|
|
333
525
|
process.exit(1);
|
|
334
526
|
}
|
|
527
|
+
await maybePrintUpdate(updatePromise);
|
|
335
528
|
}
|
|
336
529
|
|
|
337
530
|
main();
|
package/src/openUrl.js
CHANGED
|
@@ -1,4 +1,46 @@
|
|
|
1
|
-
import { spawn } from "node:child_process";
|
|
1
|
+
import { spawn, spawnSync } from "node:child_process";
|
|
2
|
+
|
|
3
|
+
function hasCmd(c) {
|
|
4
|
+
try {
|
|
5
|
+
return spawnSync(process.platform === "win32" ? "where" : "which", [c], { stdio: "ignore" }).status === 0;
|
|
6
|
+
} catch {
|
|
7
|
+
return false;
|
|
8
|
+
}
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
/** Open a source location in the user's editor. Prefers a line-aware editor
|
|
12
|
+
* ($MM_EDITOR template, then `code -g`/`cursor -g`); falls back to the OS opener
|
|
13
|
+
* (no line jump). Non-blocking, best-effort, never throws. Test-gated by
|
|
14
|
+
* LLMSTATUS_NO_OPEN. Returns true if something was launched. */
|
|
15
|
+
export function openLocation(absPath, line) {
|
|
16
|
+
if (process.env.LLMSTATUS_NO_OPEN) return false;
|
|
17
|
+
const target = line ? `${absPath}:${line}` : absPath;
|
|
18
|
+
const editorEnv = (process.env.MM_EDITOR || "").trim();
|
|
19
|
+
let cmd, args;
|
|
20
|
+
if (editorEnv) {
|
|
21
|
+
const parts = editorEnv.split(/\s+/);
|
|
22
|
+
cmd = parts[0];
|
|
23
|
+
args = [...parts.slice(1), target];
|
|
24
|
+
} else if (hasCmd("code")) {
|
|
25
|
+
cmd = "code"; args = ["-g", target];
|
|
26
|
+
} else if (hasCmd("cursor")) {
|
|
27
|
+
cmd = "cursor"; args = ["-g", target];
|
|
28
|
+
} else if (process.platform === "darwin") {
|
|
29
|
+
cmd = "open"; args = [absPath];
|
|
30
|
+
} else if (process.platform === "win32") {
|
|
31
|
+
cmd = "cmd"; args = ["/c", "start", "", absPath];
|
|
32
|
+
} else {
|
|
33
|
+
cmd = "xdg-open"; args = [absPath];
|
|
34
|
+
}
|
|
35
|
+
try {
|
|
36
|
+
const child = spawn(cmd, args, { stdio: "ignore", detached: true });
|
|
37
|
+
child.on("error", () => {});
|
|
38
|
+
child.unref();
|
|
39
|
+
return true;
|
|
40
|
+
} catch {
|
|
41
|
+
return false;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
2
44
|
|
|
3
45
|
/** Open a URL in the user's default browser, cross-platform. Best-effort: if it
|
|
4
46
|
* fails we still printed the URL, so the user can open it manually. */
|
package/src/registry/local.js
CHANGED
|
@@ -1,14 +1,33 @@
|
|
|
1
1
|
/* Offline resolution + health from a verified snapshot — mirrors the server's
|
|
2
2
|
* computeHealth (apps/web/lib/serialize.ts) so an offline run matches online. */
|
|
3
3
|
|
|
4
|
-
/** Resolve model strings against the snapshot's detection table
|
|
4
|
+
/** Resolve model strings against the snapshot's detection table.
|
|
5
|
+
*
|
|
6
|
+
* Exact match stays AUTHORITATIVE: only an exact hit sets `model` (and thus
|
|
7
|
+
* lifecycle/health), so a custom id never inherits a real model's retirement
|
|
8
|
+
* date by accident. For misses we attach a non-authoritative `suggestion` — the
|
|
9
|
+
* nearest known id by two-way substring containment (mirrors the server's 0.6
|
|
10
|
+
* fuzzy in apps/web/lib/services/registry.ts) — so callers can hint "≈ gpt-4?"
|
|
11
|
+
* on a typo without treating it as a confirmed match. `confidence`: 1 exact,
|
|
12
|
+
* 0.6 suggestion, 0 unknown. */
|
|
5
13
|
export function resolveLocal(snapshot, strings) {
|
|
6
14
|
const exact = new Map((snapshot.detection?.model_strings || []).map((s) => [s.match.toLowerCase(), s.model_slug]));
|
|
7
15
|
const bySlug = new Map((snapshot.models || []).map((m) => [m.slug, m]));
|
|
16
|
+
const keys = [...exact.keys()];
|
|
8
17
|
return strings.map((s) => {
|
|
9
|
-
const
|
|
10
|
-
const
|
|
11
|
-
|
|
18
|
+
const n = String(s).toLowerCase();
|
|
19
|
+
const slug = exact.get(n) || null;
|
|
20
|
+
if (slug) {
|
|
21
|
+
const model = bySlug.get(slug) || null;
|
|
22
|
+
return { input: s, model_slug: slug, display: model?.display ?? s, model, confidence: 1, suggestion: null };
|
|
23
|
+
}
|
|
24
|
+
// No exact hit — find the nearest known id (longest two-way substring overlap).
|
|
25
|
+
let near = null;
|
|
26
|
+
for (const k of keys) {
|
|
27
|
+
if (k.length >= 4 && (k.includes(n) || n.includes(k)) && (!near || k.length > near.length)) near = k;
|
|
28
|
+
}
|
|
29
|
+
const suggestion = near ? exact.get(near) || null : null;
|
|
30
|
+
return { input: s, model_slug: null, display: s, model: null, confidence: suggestion ? 0.6 : 0, suggestion };
|
|
12
31
|
});
|
|
13
32
|
}
|
|
14
33
|
|
|
Binary file
|