muaddib-scanner 2.11.89 → 2.11.91
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json
CHANGED
package/src/monitor/state.js
CHANGED
|
@@ -975,6 +975,14 @@ const SCAN_LEDGER_OUTCOMES = new Set([
|
|
|
975
975
|
'static_timeout', 'size_skip', 'dropped', 'error'
|
|
976
976
|
]);
|
|
977
977
|
|
|
978
|
+
// Benign terminal verdicts — the ledger-headline "clean" bucket. Mirrors the
|
|
979
|
+
// in-memory stats.clean semantics (every path that increments stats.clean writes
|
|
980
|
+
// one of these outcomes). sandbox_inconclusive/unconfirmed and size_skip are
|
|
981
|
+
// deliberately in neither bucket: scanned but not vouched-for.
|
|
982
|
+
const CLEAN_LEDGER_OUTCOMES = new Set([
|
|
983
|
+
'clean', 'clean_low_signal', 'clean_tooling', 'ml_clean', 'llm_benign'
|
|
984
|
+
]);
|
|
985
|
+
|
|
978
986
|
/**
|
|
979
987
|
* Append one per-scan ledger entry recording the terminal outcome of a dequeued
|
|
980
988
|
* package. Best-effort: NEVER throws (a ledger failure must not break scanning).
|
|
@@ -1112,6 +1120,12 @@ function computeLedgerRollup(sinceTs, opts = {}) {
|
|
|
1112
1120
|
const byOutcome = Object.create(null);
|
|
1113
1121
|
const byEcosystem = Object.create(null);
|
|
1114
1122
|
let total = 0, scanned = 0, dropped = 0, alerted = 0;
|
|
1123
|
+
// Headline counters (ledger-derived daily-report headline — restart-proof, unlike
|
|
1124
|
+
// the in-memory stats counters). clean buckets all the benign terminal verdicts;
|
|
1125
|
+
// errors only the ledgerized failure outcomes (HTTP/tar failures live in the
|
|
1126
|
+
// in-memory errorsByType breakdown, not the ledger).
|
|
1127
|
+
let hClean = 0, hErrors = 0;
|
|
1128
|
+
const hByTier = { t1: 0, t1a: 0, t1b: 0, t2: 0, t3: 0 };
|
|
1115
1129
|
let earliest = null, latest = null;
|
|
1116
1130
|
// Two sets so `vanished` is correct regardless of drop/scan ordering in the file.
|
|
1117
1131
|
// droppedKeys is small (drops only happen under queue-cap pressure); scannedKeys is
|
|
@@ -1153,10 +1167,24 @@ function computeLedgerRollup(sinceTs, opts = {}) {
|
|
|
1153
1167
|
if (underCap) { droppedKeys.add(key); allNames.add(e.name); } else exactVanished = false;
|
|
1154
1168
|
} else {
|
|
1155
1169
|
scanned++; ecoNode.scanned++;
|
|
1156
|
-
if (outcome === 'suspect' || outcome === 'confirmed') {
|
|
1170
|
+
if (outcome === 'suspect' || outcome === 'confirmed') {
|
|
1171
|
+
alerted++; ecoNode.alerted++;
|
|
1172
|
+
const t = e.tier !== undefined && e.tier !== null ? String(e.tier) : null;
|
|
1173
|
+
if (t === '1a') hByTier.t1a++;
|
|
1174
|
+
else if (t === '1b') hByTier.t1b++;
|
|
1175
|
+
else if (t === '1') hByTier.t1++;
|
|
1176
|
+
else if (t === '2') hByTier.t2++;
|
|
1177
|
+
else if (t === '3') hByTier.t3++;
|
|
1178
|
+
} else if (CLEAN_LEDGER_OUTCOMES.has(outcome)) {
|
|
1179
|
+
hClean++;
|
|
1180
|
+
} else if (outcome === 'error' || outcome === 'static_timeout') {
|
|
1181
|
+
hErrors++;
|
|
1182
|
+
}
|
|
1157
1183
|
if (underCap) { scannedKeys.add(key); allNames.add(e.name); scannedNames.add(e.name); } else exactVanished = false;
|
|
1158
1184
|
}
|
|
1159
1185
|
});
|
|
1186
|
+
// Match the in-memory suspectByTier semantics where t1 = t1a + t1b (+ legacy '1').
|
|
1187
|
+
hByTier.t1 += hByTier.t1a + hByTier.t1b;
|
|
1160
1188
|
|
|
1161
1189
|
let vanished = 0;
|
|
1162
1190
|
for (const k of droppedKeys) { if (!scannedKeys.has(k)) vanished++; }
|
|
@@ -1181,6 +1209,16 @@ function computeLedgerRollup(sinceTs, opts = {}) {
|
|
|
1181
1209
|
distinctPackages: allNames.size,
|
|
1182
1210
|
distinctScanned: scannedNames.size,
|
|
1183
1211
|
distinctCoverage: allNames.size > 0 ? scannedNames.size / allNames.size : null,
|
|
1212
|
+
// Ledger-derived daily-report headline (window-exact, restart-proof). `suspect`
|
|
1213
|
+
// mirrors `alerted` (suspect+confirmed); `scanned` mirrors the non-dropped count
|
|
1214
|
+
// above. The in-memory counters remain the fallback when the ledger is unavailable.
|
|
1215
|
+
headline: {
|
|
1216
|
+
scanned,
|
|
1217
|
+
clean: hClean,
|
|
1218
|
+
suspect: alerted,
|
|
1219
|
+
errors: hErrors,
|
|
1220
|
+
byTier: hByTier
|
|
1221
|
+
},
|
|
1184
1222
|
byOutcome,
|
|
1185
1223
|
byEcosystem
|
|
1186
1224
|
};
|
|
@@ -1421,6 +1459,21 @@ function loadLastDailyReportDate() {
|
|
|
1421
1459
|
}
|
|
1422
1460
|
}
|
|
1423
1461
|
|
|
1462
|
+
/**
|
|
1463
|
+
* Load the exact ISO timestamp of the last daily report send (the start of the
|
|
1464
|
+
* current reporting window). Returns null when absent (pre-upgrade file, first
|
|
1465
|
+
* report ever, corrupt file) — callers fall back to a fixed 24h window.
|
|
1466
|
+
*/
|
|
1467
|
+
function loadLastDailyReportTs() {
|
|
1468
|
+
try {
|
|
1469
|
+
const raw = fs.readFileSync(LAST_DAILY_REPORT_FILE, 'utf8');
|
|
1470
|
+
const data = JSON.parse(raw);
|
|
1471
|
+
return typeof data.lastReportTs === 'string' ? data.lastReportTs : null;
|
|
1472
|
+
} catch {
|
|
1473
|
+
return null;
|
|
1474
|
+
}
|
|
1475
|
+
}
|
|
1476
|
+
|
|
1424
1477
|
/**
|
|
1425
1478
|
* Persist the date of the last daily report sent (YYYY-MM-DD), and optionally the
|
|
1426
1479
|
* monotonic scan-stats baseline captured at that moment (used by the next report's
|
|
@@ -1430,6 +1483,10 @@ function saveLastDailyReportDate(dateStr, scanStatsBaseline) {
|
|
|
1430
1483
|
try {
|
|
1431
1484
|
const payload = { lastReportDate: dateStr };
|
|
1432
1485
|
if (scanStatsBaseline) payload.scanStatsBaseline = scanStatsBaseline;
|
|
1486
|
+
// Exact send timestamp = start of the NEXT report's ledger window (8h→8h
|
|
1487
|
+
// semantics, restart-proof). Written in the same write-ahead as the date
|
|
1488
|
+
// stamp, so a mid-send kill can neither hole nor double-count the window.
|
|
1489
|
+
payload.lastReportTs = new Date().toISOString();
|
|
1433
1490
|
atomicWriteFileSync(LAST_DAILY_REPORT_FILE, JSON.stringify(payload, null, 2));
|
|
1434
1491
|
} catch (err) {
|
|
1435
1492
|
console.error(`[MONITOR] Failed to save last daily report date: ${err.message}`);
|
|
@@ -1735,6 +1792,7 @@ module.exports = {
|
|
|
1735
1792
|
captureScanStatsBaseline,
|
|
1736
1793
|
reconcileDailyHeadline,
|
|
1737
1794
|
loadLastDailyReportDate,
|
|
1795
|
+
loadLastDailyReportTs,
|
|
1738
1796
|
saveLastDailyReportDate,
|
|
1739
1797
|
hasReportBeenSentToday,
|
|
1740
1798
|
saveRecentlyScanned,
|
package/src/monitor/webhook.js
CHANGED
|
@@ -30,7 +30,8 @@ const {
|
|
|
30
30
|
loadStateRaw,
|
|
31
31
|
getScansSinceLastMemoryPersist,
|
|
32
32
|
setScansSinceLastMemoryPersist,
|
|
33
|
-
computeLedgerRollup
|
|
33
|
+
computeLedgerRollup,
|
|
34
|
+
loadLastDailyReportTs
|
|
34
35
|
} = require('./state.js');
|
|
35
36
|
const {
|
|
36
37
|
HIGH_CONFIDENCE_MALICE_TYPES,
|
|
@@ -1049,24 +1050,59 @@ function formatDelta(current, previous) {
|
|
|
1049
1050
|
return '=0';
|
|
1050
1051
|
}
|
|
1051
1052
|
|
|
1052
|
-
// Phase 0b:
|
|
1053
|
-
//
|
|
1054
|
-
//
|
|
1053
|
+
// Phase 0b: fallback window for the daily report's ledger section when no
|
|
1054
|
+
// last-report timestamp exists yet (first report ever / pre-upgrade stamp file).
|
|
1055
|
+
// Normal operation derives the window from lastReportTs instead (8h→8h Paris,
|
|
1056
|
+
// restart-proof). Env-tunable.
|
|
1055
1057
|
const LEDGER_ROLLUP_WINDOW_MS = (() => {
|
|
1056
1058
|
const v = parseInt(process.env.MUADDIB_LEDGER_ROLLUP_WINDOW_MS, 10);
|
|
1057
1059
|
return Number.isFinite(v) && v > 0 ? v : 24 * 60 * 60 * 1000;
|
|
1058
1060
|
})();
|
|
1059
1061
|
|
|
1062
|
+
// Hard ceiling on the report window. A multi-day daemon outage would otherwise make
|
|
1063
|
+
// the next report's window (and the rollup's distinct-key sets) span the whole gap;
|
|
1064
|
+
// clamp to 48h and flag it so the report stays honest about the truncation.
|
|
1065
|
+
const LEDGER_ROLLUP_MAX_WINDOW_MS = 48 * 60 * 60 * 1000;
|
|
1066
|
+
|
|
1060
1067
|
/**
|
|
1061
|
-
* Compute the per-scan ledger rollup for the daily-report window.
|
|
1062
|
-
*
|
|
1063
|
-
*
|
|
1064
|
-
*
|
|
1068
|
+
* Compute the per-scan ledger rollup for the daily-report window. The window is
|
|
1069
|
+
* [last report send → now] (8h→8h Paris semantics, exact across restarts) when the
|
|
1070
|
+
* lastReportTs stamp exists, else the fixed fallback window. Best-effort: a rollup
|
|
1071
|
+
* failure (corrupt ledger, I/O) must NEVER break the daily report, so this swallows
|
|
1072
|
+
* errors and returns null. Also returns null when the ledger is empty so the report
|
|
1073
|
+
* omits the section instead of showing a noise row of zeros.
|
|
1065
1074
|
*/
|
|
1066
1075
|
function safeLedgerRollup() {
|
|
1067
1076
|
try {
|
|
1068
|
-
const
|
|
1069
|
-
|
|
1077
|
+
const now = Date.now();
|
|
1078
|
+
let sinceMs = now - LEDGER_ROLLUP_WINDOW_MS;
|
|
1079
|
+
let windowClamped = false;
|
|
1080
|
+
let windowSource = 'fallback_24h';
|
|
1081
|
+
const lastTs = loadLastDailyReportTs();
|
|
1082
|
+
if (lastTs) {
|
|
1083
|
+
const p = Date.parse(lastTs);
|
|
1084
|
+
// Guard against clock skew (stamp in the future) — fall back to 24h.
|
|
1085
|
+
if (!Number.isNaN(p) && p <= now) {
|
|
1086
|
+
if (p < now - LEDGER_ROLLUP_MAX_WINDOW_MS) {
|
|
1087
|
+
sinceMs = now - LEDGER_ROLLUP_MAX_WINDOW_MS;
|
|
1088
|
+
windowClamped = true;
|
|
1089
|
+
} else {
|
|
1090
|
+
sinceMs = p;
|
|
1091
|
+
}
|
|
1092
|
+
windowSource = 'last_report';
|
|
1093
|
+
}
|
|
1094
|
+
}
|
|
1095
|
+
// Ledger source resolved at CALL time (not module load) so tests can point the
|
|
1096
|
+
// rollup at a synthetic/empty ledger after the module graph is already loaded.
|
|
1097
|
+
// Unset env → computeLedgerRollup falls back to its SCAN_LEDGER_FILE default.
|
|
1098
|
+
const fileOverride = process.env.MUADDIB_SCAN_LEDGER_FILE;
|
|
1099
|
+
const rollup = computeLedgerRollup(sinceMs, fileOverride ? { file: fileOverride } : {});
|
|
1100
|
+
if (rollup && rollup.total > 0) {
|
|
1101
|
+
rollup.windowClamped = windowClamped;
|
|
1102
|
+
rollup.windowSource = windowSource;
|
|
1103
|
+
return rollup;
|
|
1104
|
+
}
|
|
1105
|
+
return null;
|
|
1070
1106
|
} catch {
|
|
1071
1107
|
return null;
|
|
1072
1108
|
}
|
|
@@ -1091,7 +1127,10 @@ function formatLedgerField(rollup) {
|
|
|
1091
1127
|
if (ecos.length > 0) {
|
|
1092
1128
|
lines.push(ecos.slice(0, 4).map(e => `${e} ${rollup.byEcosystem[e].total}`).join(' · '));
|
|
1093
1129
|
}
|
|
1094
|
-
|
|
1130
|
+
const label = rollup.windowSource === 'last_report'
|
|
1131
|
+
? `Ledger (since last report${rollup.windowClamped ? ', clamped 48h' : ''})`
|
|
1132
|
+
: 'Ledger (24h)';
|
|
1133
|
+
return { name: label, value: lines.join('\n'), inline: false };
|
|
1095
1134
|
}
|
|
1096
1135
|
|
|
1097
1136
|
// AUDIT-C: MCP self-identity by package name (matches the F9/F15 MCP_NAME_RE family in
|
|
@@ -1115,6 +1154,22 @@ function buildDailyReportEmbed(stats, dailyAlerts, ledgerRollup) {
|
|
|
1115
1154
|
// instead of disk-based daily entries which can undercount due to UTC/Paris date mismatch
|
|
1116
1155
|
const { top3: diskTop3 } = buildReportFromDisk();
|
|
1117
1156
|
|
|
1157
|
+
// --- Phase 0b: per-scan ledger rollup (resolved early so the headline can use it) ---
|
|
1158
|
+
// Caller may pass a precomputed rollup (sendDailyReport does, to persist the same
|
|
1159
|
+
// numbers it displays); undefined → compute here; explicit null → omit the section.
|
|
1160
|
+
const ledger = ledgerRollup !== undefined ? ledgerRollup : safeLedgerRollup();
|
|
1161
|
+
|
|
1162
|
+
// HEADLINE BOUNDARY — scanned/clean/suspect come from the ledger window
|
|
1163
|
+
// [last report → now] when available: window-exact and restart-proof, unlike the
|
|
1164
|
+
// in-memory counters (reset-restore cycles can under-count after a restart storm).
|
|
1165
|
+
// Everything NOT in the ledger (errorsByType breakdown, changes-stream/publish-event
|
|
1166
|
+
// counts, pypi*, avg scan time) stays on the in-memory counters + daily-stats.json:
|
|
1167
|
+
// best-effort since the last reset, may under-count after a restart.
|
|
1168
|
+
const headline = (ledger && ledger.headline && ledger.headline.scanned > 0) ? ledger.headline : null;
|
|
1169
|
+
const hScanned = headline ? headline.scanned : stats.scanned;
|
|
1170
|
+
const hClean = headline ? headline.clean : stats.clean;
|
|
1171
|
+
const hSuspect = headline ? headline.suspect : stats.suspect;
|
|
1172
|
+
|
|
1118
1173
|
// Prefer in-memory dailyAlerts for top suspects (richer data), fallback to disk
|
|
1119
1174
|
const top3 = dailyAlerts.length > 0
|
|
1120
1175
|
? dailyAlerts.slice().sort((a, b) => (b.score || 0) - (a.score || 0) || b.findingsCount - a.findingsCount).slice(0, 3)
|
|
@@ -1133,14 +1188,9 @@ function buildDailyReportEmbed(stats, dailyAlerts, ledgerRollup) {
|
|
|
1133
1188
|
}).join('\n')
|
|
1134
1189
|
: 'None';
|
|
1135
1190
|
|
|
1136
|
-
// Avg scan time from in-memory stats
|
|
1191
|
+
// Avg scan time from in-memory stats (totalTimeMs is not ledgerized — best-effort)
|
|
1137
1192
|
const avg = stats.scanned > 0 ? (stats.totalTimeMs / stats.scanned / 1000).toFixed(1) : '0.0';
|
|
1138
1193
|
|
|
1139
|
-
// --- Phase 0b: per-scan ledger rollup (resolved early so Coverage can use it) ---
|
|
1140
|
-
// Caller may pass a precomputed rollup (sendDailyReport does, to persist the same
|
|
1141
|
-
// numbers it displays); undefined → compute here; explicit null → omit the section.
|
|
1142
|
-
const ledger = ledgerRollup !== undefined ? ledgerRollup : safeLedgerRollup();
|
|
1143
|
-
|
|
1144
1194
|
// --- Coverage ---
|
|
1145
1195
|
// HEADLINE: honest, version-collapsed coverage from the scan-ledger — distinct
|
|
1146
1196
|
// package NAMES actually scanned vs distinct names seen (scanned + dropped) in
|
|
@@ -1155,8 +1205,8 @@ function buildDailyReportEmbed(stats, dailyAlerts, ledgerRollup) {
|
|
|
1155
1205
|
const published = npmPub + pypiPub;
|
|
1156
1206
|
const catchupSkipped = (stats.npmCatchupSkippedSeqs || 0) + (stats.pypiCatchupSkippedEvents || 0);
|
|
1157
1207
|
const opsSuffix = catchupSkipped > 0
|
|
1158
|
-
? `\nOps: ${
|
|
1159
|
-
: `\nOps: ${
|
|
1208
|
+
? `\nOps: ${hScanned} | Catch-up skip: ${catchupSkipped}`
|
|
1209
|
+
: `\nOps: ${hScanned}`;
|
|
1160
1210
|
let coverageText;
|
|
1161
1211
|
if (ledger && ledger.distinctPackages > 0 && ledger.distinctCoverage != null) {
|
|
1162
1212
|
const pct = (ledger.distinctCoverage * 100).toFixed(0);
|
|
@@ -1183,8 +1233,8 @@ function buildDailyReportEmbed(stats, dailyAlerts, ledgerRollup) {
|
|
|
1183
1233
|
const yesterday = loadYesterdayMetrics();
|
|
1184
1234
|
let trendsText = 'No data (first day or missing)';
|
|
1185
1235
|
if (yesterday) {
|
|
1186
|
-
const dScanned = formatDelta(
|
|
1187
|
-
const dSuspect = formatDelta(
|
|
1236
|
+
const dScanned = formatDelta(hScanned, yesterday.scanned || 0);
|
|
1237
|
+
const dSuspect = formatDelta(hSuspect, yesterday.suspect || 0);
|
|
1188
1238
|
const dErrors = formatDelta(stats.errors, yesterday.errors || 0);
|
|
1189
1239
|
trendsText = `${dScanned} scanned, ${dSuspect} suspects, ${dErrors} errors`;
|
|
1190
1240
|
}
|
|
@@ -1245,8 +1295,8 @@ function buildDailyReportEmbed(stats, dailyAlerts, ledgerRollup) {
|
|
|
1245
1295
|
color: 0x3498db,
|
|
1246
1296
|
fields: [
|
|
1247
1297
|
{ name: 'Coverage', value: coverageText, inline: true },
|
|
1248
|
-
{ name: 'Clean', value: `${
|
|
1249
|
-
{ name: 'Suspects', value: `${
|
|
1298
|
+
{ name: 'Clean', value: `${hClean}`, inline: true },
|
|
1299
|
+
{ name: 'Suspects', value: `${hSuspect}`, inline: true },
|
|
1250
1300
|
{ name: 'Errors', value: formatErrorBreakdown(stats.errors, stats.errorsByType), inline: true },
|
|
1251
1301
|
{ name: 'Avg Scan Time', value: `${avg}s/pkg`, inline: true },
|
|
1252
1302
|
{ name: 'Timeouts', value: timeoutText, inline: true },
|
|
@@ -1262,7 +1312,9 @@ function buildDailyReportEmbed(stats, dailyAlerts, ledgerRollup) {
|
|
|
1262
1312
|
{ name: 'System', value: healthText, inline: false }
|
|
1263
1313
|
],
|
|
1264
1314
|
footer: {
|
|
1265
|
-
|
|
1315
|
+
// Headline-source annotation: 'ledger' = window-exact [last report → now],
|
|
1316
|
+
// 'counters' = in-memory fallback (ledger unavailable — pre-upgrade behavior).
|
|
1317
|
+
text: `MUAD'DIB - Daily summary | headline: ${headline ? 'ledger (since last report)' : 'counters'} | ${readableTime}`
|
|
1266
1318
|
},
|
|
1267
1319
|
timestamp: now.toISOString()
|
|
1268
1320
|
}]
|
|
@@ -1285,20 +1337,34 @@ async function sendDailyReport(stats, dailyAlerts, recentlyScanned, downloadsCac
|
|
|
1285
1337
|
console.log(`[MONITOR] Daily report suppressed: before ${DAILY_REPORT_HOUR}:00 Paris (hour=${getParisHour()})`);
|
|
1286
1338
|
return;
|
|
1287
1339
|
}
|
|
1288
|
-
//
|
|
1289
|
-
//
|
|
1290
|
-
//
|
|
1291
|
-
|
|
1340
|
+
// Phase 0b: compute the ledger rollup ONCE so the embed shows exactly the numbers
|
|
1341
|
+
// we persist (no double-scan, no drift between Discord and the on-disk metrics).
|
|
1342
|
+
// Resolved BEFORE the empty-skip and the reconcile: when the ledger headline is
|
|
1343
|
+
// available it IS the published number (window [last report → now], restart-proof),
|
|
1344
|
+
// and the counter-based machinery below only runs as fallback.
|
|
1345
|
+
const ledgerRollup = safeLedgerRollup();
|
|
1346
|
+
const headline = (ledgerRollup && ledgerRollup.headline && ledgerRollup.headline.scanned > 0)
|
|
1347
|
+
? ledgerRollup.headline : null;
|
|
1348
|
+
|
|
1349
|
+
if (!headline) {
|
|
1350
|
+
// Crash-safe FALLBACK headline: a restart-storm around report time can zero the
|
|
1351
|
+
// in-memory counter (the monitor OOM-restarts ~10×/day). Floor scanned/clean/suspect
|
|
1352
|
+
// at the durable scan-stats delta so we never publish "5" when ~44k were really
|
|
1353
|
+
// scanned. Not applied when the ledger headline is used — that one is window-exact.
|
|
1354
|
+
reconcileDailyHeadline(stats);
|
|
1355
|
+
}
|
|
1292
1356
|
|
|
1293
1357
|
// Never send an empty report (0 scanned — restart with no work done)
|
|
1294
|
-
|
|
1358
|
+
const publishedScanned = headline ? headline.scanned : stats.scanned;
|
|
1359
|
+
if (publishedScanned === 0) {
|
|
1295
1360
|
console.log('[MONITOR] Daily report skipped (0 packages scanned)');
|
|
1296
1361
|
return;
|
|
1297
1362
|
}
|
|
1298
1363
|
|
|
1299
1364
|
// Write-ahead: mark today's report as sent BEFORE the webhook HTTP request.
|
|
1300
1365
|
// If the process is killed (SIGKILL) during sendWebhook, the date is already
|
|
1301
|
-
// recorded on disk and prevents duplicate reports on next startup.
|
|
1366
|
+
// recorded on disk and prevents duplicate reports on next startup. The same
|
|
1367
|
+
// write-ahead stamps lastReportTs = start of the next report's ledger window.
|
|
1302
1368
|
const today = getParisDateString();
|
|
1303
1369
|
stats.lastDailyReportDate = today;
|
|
1304
1370
|
// Persist the monotonic scan-stats counter as the baseline for the NEXT report's
|
|
@@ -1306,23 +1372,23 @@ async function sendDailyReport(stats, dailyAlerts, recentlyScanned, downloadsCac
|
|
|
1306
1372
|
saveLastDailyReportDate(today, captureScanStatsBaseline());
|
|
1307
1373
|
// Observability: the success path previously logged nothing, which made the late-fire bug
|
|
1308
1374
|
// invisible in the journal. Log the stamped date + the actual Paris hour (an on-time 08:00
|
|
1309
|
-
// fire vs a catch-up at hour 14 are now distinguishable) + the headline count.
|
|
1310
|
-
console.log(`[MONITOR] Daily report firing for ${today} (hour=${getParisHour()} Paris, scanned=${
|
|
1375
|
+
// fire vs a catch-up at hour 14 are now distinguishable) + the headline count + source.
|
|
1376
|
+
console.log(`[MONITOR] Daily report firing for ${today} (hour=${getParisHour()} Paris, scanned=${publishedScanned}, headline=${headline ? 'ledger' : 'counters'})`);
|
|
1311
1377
|
|
|
1312
|
-
// Phase 0b: compute the ledger rollup ONCE so the embed shows exactly the numbers
|
|
1313
|
-
// we persist (no double-scan, no drift between Discord and the on-disk metrics).
|
|
1314
|
-
const ledgerRollup = safeLedgerRollup();
|
|
1315
1378
|
const payload = buildDailyReportEmbed(stats, dailyAlerts, ledgerRollup);
|
|
1316
1379
|
|
|
1317
|
-
// Persist locally with full raw metrics (independent of webhook — enables trend analysis)
|
|
1380
|
+
// Persist locally with full raw metrics (independent of webhook — enables trend analysis).
|
|
1381
|
+
// Headline (scanned/clean/suspect/byTier) follows the same source as the embed: ledger
|
|
1382
|
+
// window when available, in-memory counters otherwise. headlineSource records which.
|
|
1318
1383
|
persistDailyReport(payload, {
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1384
|
+
headlineSource: headline ? 'ledger' : 'counters',
|
|
1385
|
+
scanned: publishedScanned,
|
|
1386
|
+
clean: headline ? headline.clean : stats.clean,
|
|
1387
|
+
suspect: headline ? headline.suspect : stats.suspect,
|
|
1322
1388
|
errors: stats.errors,
|
|
1323
1389
|
errorsByType: { ...stats.errorsByType },
|
|
1324
1390
|
avgScanTimeMs: stats.scanned > 0 ? Math.round(stats.totalTimeMs / stats.scanned) : 0,
|
|
1325
|
-
suspectByTier: { ...stats.suspectByTier },
|
|
1391
|
+
suspectByTier: headline ? { ...headline.byTier } : { ...stats.suspectByTier },
|
|
1326
1392
|
mlFiltered: stats.mlFiltered || 0,
|
|
1327
1393
|
llmAnalyzed: stats.llmAnalyzed || 0,
|
|
1328
1394
|
llmSuppressed: stats.llmSuppressed || 0,
|
|
@@ -217,6 +217,10 @@ async function getPackageMetadata(packageName) {
|
|
|
217
217
|
// / pinned-old / vendored versions bypass the cap so we don't mask attacks
|
|
218
218
|
// captured in static fixtures (e.g. eslint-scope 3.7.2, chalk 5.6.1).
|
|
219
219
|
latest_version: latestVersion || null,
|
|
220
|
+
// P4 : full dist-tags map ({ latest, next, canary, ... }) so scoring can tell a
|
|
221
|
+
// maintainer-controlled pre-release channel version (inherits partial reputation)
|
|
222
|
+
// from a historical / pinned-old version (no reputation).
|
|
223
|
+
dist_tags: (meta['dist-tags'] && typeof meta['dist-tags'] === 'object') ? meta['dist-tags'] : null,
|
|
220
224
|
// F3 : list of maintainer email addresses (lowercased, unique) for DNS
|
|
221
225
|
// MX / RDAP downstream checks. Empty array if no emails published.
|
|
222
226
|
maintainer_emails: maintainerEmails,
|
package/src/scoring.js
CHANGED
|
@@ -1874,6 +1874,23 @@ function _hasMaliceSignal(threats) {
|
|
|
1874
1874
|
return threats.some(t => t.severity === 'HIGH' || t.severity === 'CRITICAL');
|
|
1875
1875
|
}
|
|
1876
1876
|
|
|
1877
|
+
// P4 (pre-release reputation): known maintainer-controlled pre-release dist-tag names.
|
|
1878
|
+
const _PRERELEASE_TAG_RE = /^(next|canary|nightly|rc|beta|alpha|dev|experimental|preview|snapshot|edge|insider|insiders)$/i;
|
|
1879
|
+
|
|
1880
|
+
// True when the scanned version is published on a NON-latest pre-release dist-tag
|
|
1881
|
+
// (e.g. dist-tags = { latest: "3.19.1", next: "3.19.0-nightly-..." } and we scan the
|
|
1882
|
+
// nightly). Maintainer-controlled, so an attacker cannot put a payload on `next`
|
|
1883
|
+
// without the account — and Track R still floors confirmed malice regardless.
|
|
1884
|
+
function _isPrereleaseChannelVersion(metadata) {
|
|
1885
|
+
const dt = metadata && metadata.dist_tags;
|
|
1886
|
+
const sv = metadata && metadata.scan_version;
|
|
1887
|
+
if (!dt || typeof dt !== 'object' || typeof sv !== 'string') return false;
|
|
1888
|
+
for (const [tag, ver] of Object.entries(dt)) {
|
|
1889
|
+
if (ver === sv && tag !== 'latest' && _PRERELEASE_TAG_RE.test(tag)) return true;
|
|
1890
|
+
}
|
|
1891
|
+
return false;
|
|
1892
|
+
}
|
|
1893
|
+
|
|
1877
1894
|
function applyReputationFactor(result, metadata) {
|
|
1878
1895
|
if (!result || !result.summary || !metadata) return null;
|
|
1879
1896
|
// FPR plan : the reputation factor describes "how trustworthy this package
|
|
@@ -1884,12 +1901,21 @@ function applyReputationFactor(result, metadata) {
|
|
|
1884
1901
|
// scan version is unknown (CLI scanning a directory without version field),
|
|
1885
1902
|
// we fail open : skip the factor entirely rather than apply a multiplier
|
|
1886
1903
|
// we cannot situate in time.
|
|
1904
|
+
// P4 (pre-release reputation): a canary/nightly/rc of an established package is a NEW
|
|
1905
|
+
// (not historical) version on a maintainer-controlled pre-release dist-tag, so it should
|
|
1906
|
+
// inherit the package's reputation — but only PARTIALLY (it is not the verified latest).
|
|
1907
|
+
// Any OTHER non-latest version (historical / pinned-old / vendored) keeps the full skip.
|
|
1908
|
+
let _prereleaseAttenuation = 1.0;
|
|
1887
1909
|
if (
|
|
1888
1910
|
typeof metadata.latest_version === 'string' &&
|
|
1889
1911
|
typeof metadata.scan_version === 'string' &&
|
|
1890
1912
|
metadata.latest_version !== metadata.scan_version
|
|
1891
1913
|
) {
|
|
1892
|
-
|
|
1914
|
+
if (_isPrereleaseChannelVersion(metadata)) {
|
|
1915
|
+
_prereleaseAttenuation = 0.85;
|
|
1916
|
+
} else {
|
|
1917
|
+
return null;
|
|
1918
|
+
}
|
|
1893
1919
|
}
|
|
1894
1920
|
if (
|
|
1895
1921
|
typeof metadata.latest_version === 'string' &&
|
|
@@ -1900,9 +1926,14 @@ function applyReputationFactor(result, metadata) {
|
|
|
1900
1926
|
// P3 hardening: a valid attestation must NOT earn a trust bonus on a package that
|
|
1901
1927
|
// also shows malice (TeamPCP attested-malware scenario). Withhold it here, where
|
|
1902
1928
|
// the threat list is available.
|
|
1903
|
-
|
|
1929
|
+
let factor = _factorFromMetadata(metadata, {
|
|
1904
1930
|
allowProvenanceBonus: !_hasMaliceSignal(result.threats)
|
|
1905
1931
|
});
|
|
1932
|
+
// P4: blend the factor toward neutral (1.0) for pre-release channel versions — they
|
|
1933
|
+
// get most of the reputation suppression but not 100% (they are not the verified latest).
|
|
1934
|
+
if (_prereleaseAttenuation !== 1.0) {
|
|
1935
|
+
factor = 1.0 + (factor - 1.0) * _prereleaseAttenuation;
|
|
1936
|
+
}
|
|
1906
1937
|
if (factor === 1.0) {
|
|
1907
1938
|
result.summary.reputationFactor = 1.0;
|
|
1908
1939
|
return null;
|