@mcptoolshop/promo-kit 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -0
- package/LICENSE +21 -0
- package/README.md +132 -0
- package/bin/promo-kit.mjs +150 -0
- package/index.mjs +9 -0
- package/kit.config.example.json +19 -0
- package/package.json +45 -0
- package/scripts/apply-control-patch.mjs +205 -0
- package/scripts/apply-submission-status.mjs +225 -0
- package/scripts/gen-baseline.mjs +402 -0
- package/scripts/gen-decision-drift.mjs +253 -0
- package/scripts/gen-experiment-decisions.mjs +282 -0
- package/scripts/gen-feedback-summary.mjs +278 -0
- package/scripts/gen-promo-decisions.mjs +507 -0
- package/scripts/gen-queue-health.mjs +223 -0
- package/scripts/gen-recommendation-patch.mjs +352 -0
- package/scripts/gen-recommendations.mjs +409 -0
- package/scripts/gen-telemetry-aggregate.mjs +266 -0
- package/scripts/gen-trust-receipt.mjs +184 -0
- package/scripts/kit-bootstrap.mjs +246 -0
- package/scripts/kit-migrate.mjs +111 -0
- package/scripts/kit-selftest.mjs +207 -0
- package/scripts/lib/config.mjs +124 -0
|
@@ -0,0 +1,225 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Apply Submission Status
|
|
5
|
+
*
|
|
6
|
+
* Validates and applies a status patch to a submission in submissions.json.
|
|
7
|
+
* Called from the apply-submission-status workflow.
|
|
8
|
+
*
|
|
9
|
+
* Usage:
|
|
10
|
+
* node scripts/apply-submission-status.mjs '{"slug":"my-tool","status":"needs-info","reviewNotes":"Please add a demo"}'
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import { readFileSync, writeFileSync } from "node:fs";
|
|
14
|
+
import { resolve, join } from "node:path";
|
|
15
|
+
import { getConfig, getRoot } from "./lib/config.mjs";
|
|
16
|
+
|
|
17
|
+
const ROOT = getRoot();
|
|
18
|
+
const config = getConfig();
|
|
19
|
+
const DATA_DIR = join(ROOT, config.paths.dataDir);
|
|
20
|
+
|
|
21
|
+
// ── Constants ─────────────────────────────────────────────────
|
|
22
|
+
|
|
23
|
+
export const VALID_STATUSES = [
|
|
24
|
+
"pending", "accepted", "rejected", "withdrawn", "needs-info",
|
|
25
|
+
];
|
|
26
|
+
|
|
27
|
+
const PATCHABLE_FIELDS = new Set([
|
|
28
|
+
"status", "reviewNotes", "lastReviewedAt", "sourcePr", "updatedAt", "reason",
|
|
29
|
+
]);
|
|
30
|
+
|
|
31
|
+
const PROTECTED_FIELDS = new Set([
|
|
32
|
+
"slug", "submittedAt", "tool", "lane",
|
|
33
|
+
]);
|
|
34
|
+
|
|
35
|
+
const ISO_DATE_RE = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}/;
|
|
36
|
+
|
|
37
|
+
function isHttpsUrl(str) {
|
|
38
|
+
try {
|
|
39
|
+
const url = new URL(str);
|
|
40
|
+
return url.protocol === "https:";
|
|
41
|
+
} catch {
|
|
42
|
+
return false;
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
// ── Field validators ──────────────────────────────────────────
|
|
47
|
+
|
|
48
|
+
const FIELD_VALIDATORS = {
|
|
49
|
+
status: (v) => typeof v === "string" && VALID_STATUSES.includes(v),
|
|
50
|
+
reviewNotes: (v) => typeof v === "string" && v.length <= 500,
|
|
51
|
+
lastReviewedAt: (v) => typeof v === "string" && ISO_DATE_RE.test(v),
|
|
52
|
+
sourcePr: (v) => typeof v === "string" && isHttpsUrl(v),
|
|
53
|
+
updatedAt: (v) => typeof v === "string" && ISO_DATE_RE.test(v),
|
|
54
|
+
reason: (v) => typeof v === "string" && v.length <= 300,
|
|
55
|
+
};
|
|
56
|
+
|
|
57
|
+
// ── Risk notes ────────────────────────────────────────────────
|
|
58
|
+
|
|
59
|
+
const RISK_NOTES = {
|
|
60
|
+
status: (v) => {
|
|
61
|
+
const notes = {
|
|
62
|
+
"needs-info": "Submission moved to needs-info — submitter should check queue",
|
|
63
|
+
accepted: "Submission accepted — will appear in catalog pipeline",
|
|
64
|
+
rejected: "Submission rejected — reason should be provided",
|
|
65
|
+
pending: "Submission moved back to pending",
|
|
66
|
+
withdrawn: "Submission withdrawn",
|
|
67
|
+
};
|
|
68
|
+
return notes[v] || `Status changed to "${v}"`;
|
|
69
|
+
},
|
|
70
|
+
reviewNotes: () => "Review notes updated",
|
|
71
|
+
reason: () => "Rejection/status reason updated",
|
|
72
|
+
};
|
|
73
|
+
|
|
74
|
+
// ── Core functions (exported for testing) ────────────────────
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* Validate a status patch for a given slug.
|
|
78
|
+
* @param {string} slug
|
|
79
|
+
* @param {Record<string, unknown>} fields
|
|
80
|
+
* @returns {{ valid: boolean, errors: string[] }}
|
|
81
|
+
*/
|
|
82
|
+
export function validateStatusPatch(slug, fields) {
|
|
83
|
+
const errors = [];
|
|
84
|
+
|
|
85
|
+
if (!slug || typeof slug !== "string") {
|
|
86
|
+
errors.push("slug: required non-empty string");
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
if (!fields || typeof fields !== "object") {
|
|
90
|
+
return { valid: false, errors: ["fields must be a non-null object"] };
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
for (const [field, value] of Object.entries(fields)) {
|
|
94
|
+
if (PROTECTED_FIELDS.has(field)) {
|
|
95
|
+
errors.push(`"${field}" is protected and cannot be patched`);
|
|
96
|
+
continue;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
if (!PATCHABLE_FIELDS.has(field)) {
|
|
100
|
+
errors.push(`"${field}" is not a recognized patchable field`);
|
|
101
|
+
continue;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
const validator = FIELD_VALIDATORS[field];
|
|
105
|
+
if (validator && !validator(value)) {
|
|
106
|
+
errors.push(`Invalid value for "${field}": ${JSON.stringify(value)}`);
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
return { valid: errors.length === 0, errors };
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* Apply a validated status patch to a submission.
|
|
115
|
+
* @param {string} slug
|
|
116
|
+
* @param {Record<string, unknown>} fields
|
|
117
|
+
* @param {{ dataDir?: string }} opts
|
|
118
|
+
* @returns {{ applied: boolean, riskNotes: string[], submission: object|null, error?: string }}
|
|
119
|
+
*/
|
|
120
|
+
export function applyStatusPatch(slug, fields, opts = {}) {
|
|
121
|
+
const { dataDir = DATA_DIR } = opts;
|
|
122
|
+
const filePath = join(dataDir, "submissions.json");
|
|
123
|
+
const riskNotes = [];
|
|
124
|
+
|
|
125
|
+
let data;
|
|
126
|
+
try {
|
|
127
|
+
data = JSON.parse(readFileSync(filePath, "utf8"));
|
|
128
|
+
} catch {
|
|
129
|
+
return { applied: false, riskNotes: [], submission: null, error: "Failed to read submissions.json" };
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
if (!Array.isArray(data.submissions)) {
|
|
133
|
+
return { applied: false, riskNotes: [], submission: null, error: "submissions.json has no submissions array" };
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
const idx = data.submissions.findIndex((s) => s.slug === slug);
|
|
137
|
+
if (idx === -1) {
|
|
138
|
+
return { applied: false, riskNotes: [], submission: null, error: `Slug "${slug}" not found in submissions.json` };
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
// Merge fields
|
|
142
|
+
const submission = data.submissions[idx];
|
|
143
|
+
for (const [field, value] of Object.entries(fields)) {
|
|
144
|
+
submission[field] = value;
|
|
145
|
+
|
|
146
|
+
// Generate risk notes
|
|
147
|
+
if (RISK_NOTES[field]) {
|
|
148
|
+
riskNotes.push(RISK_NOTES[field](value));
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
// Auto-set updatedAt if not explicitly provided
|
|
153
|
+
if (!fields.updatedAt) {
|
|
154
|
+
submission.updatedAt = new Date().toISOString();
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
data.submissions[idx] = submission;
|
|
158
|
+
writeFileSync(filePath, JSON.stringify(data, null, 2) + "\n", "utf8");
|
|
159
|
+
|
|
160
|
+
return { applied: true, riskNotes, submission };
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
/**
|
|
164
|
+
* Full pipeline: parse, validate, apply.
|
|
165
|
+
* @param {string} patchJson - JSON string from CLI arg
|
|
166
|
+
* @param {{ dataDir?: string, riskNotesPath?: string }} opts
|
|
167
|
+
*/
|
|
168
|
+
export function applySubmissionStatus(patchJson, opts = {}) {
|
|
169
|
+
const { dataDir = DATA_DIR, riskNotesPath = "/tmp/submission-status-risk-notes.txt" } = opts;
|
|
170
|
+
|
|
171
|
+
let patch;
|
|
172
|
+
try {
|
|
173
|
+
patch = JSON.parse(patchJson);
|
|
174
|
+
} catch (e) {
|
|
175
|
+
console.error(` Error: Invalid JSON — ${e.message}`);
|
|
176
|
+
process.exitCode = 1;
|
|
177
|
+
return { success: false, error: "Invalid JSON" };
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
const { slug, ...fields } = patch;
|
|
181
|
+
|
|
182
|
+
const validation = validateStatusPatch(slug, fields);
|
|
183
|
+
if (!validation.valid) {
|
|
184
|
+
console.error(" Validation errors:");
|
|
185
|
+
for (const err of validation.errors) {
|
|
186
|
+
console.error(` - ${err}`);
|
|
187
|
+
}
|
|
188
|
+
process.exitCode = 1;
|
|
189
|
+
return { success: false, errors: validation.errors };
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
const result = applyStatusPatch(slug, fields, { dataDir });
|
|
193
|
+
if (!result.applied) {
|
|
194
|
+
console.error(` Error: ${result.error}`);
|
|
195
|
+
process.exitCode = 1;
|
|
196
|
+
return { success: false, error: result.error };
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
console.log(` Applied status patch to "${slug}"`);
|
|
200
|
+
if (result.riskNotes.length > 0) {
|
|
201
|
+
console.log(" Risk notes:");
|
|
202
|
+
for (const note of result.riskNotes) {
|
|
203
|
+
console.log(` - ${note}`);
|
|
204
|
+
}
|
|
205
|
+
try {
|
|
206
|
+
writeFileSync(riskNotesPath, result.riskNotes.join("\n") + "\n", "utf8");
|
|
207
|
+
} catch { /* fail soft in non-CI env */ }
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
return { success: true, ...result };
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
// ── Entry point ──────────────────────────────────────────────
|
|
214
|
+
|
|
215
|
+
const isMain = process.argv[1] && resolve(process.argv[1]).endsWith("apply-submission-status.mjs");
|
|
216
|
+
if (isMain) {
|
|
217
|
+
const patchJson = process.argv[2];
|
|
218
|
+
if (!patchJson) {
|
|
219
|
+
console.error("Usage: node scripts/apply-submission-status.mjs '<patch-json>'");
|
|
220
|
+
process.exitCode = 1;
|
|
221
|
+
} else {
|
|
222
|
+
console.log("Applying submission status...");
|
|
223
|
+
applySubmissionStatus(patchJson);
|
|
224
|
+
}
|
|
225
|
+
}
|
|
@@ -0,0 +1,402 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Baseline Generator
|
|
5
|
+
*
|
|
6
|
+
* Reads ops-history.json, computes statistics, projects costs.
|
|
7
|
+
* Produces baseline.json for the dashboard and baseline.md for reference.
|
|
8
|
+
*
|
|
9
|
+
* Usage:
|
|
10
|
+
* node scripts/gen-baseline.mjs [--dry-run]
|
|
11
|
+
*
|
|
12
|
+
* Reads:
|
|
13
|
+
* site/src/data/ops-history.json
|
|
14
|
+
*
|
|
15
|
+
* Writes:
|
|
16
|
+
* site/src/data/baseline.json
|
|
17
|
+
* site/public/lab/baseline/baseline.md
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
import { readFileSync, writeFileSync, mkdirSync } from "node:fs";
|
|
21
|
+
import { resolve, join } from "node:path";
|
|
22
|
+
import { getConfig, getRoot } from "./lib/config.mjs";
|
|
23
|
+
|
|
24
|
+
const ROOT = getRoot();
|
|
25
|
+
const config = getConfig();
|
|
26
|
+
const DATA_DIR = join(ROOT, config.paths.dataDir);
|
|
27
|
+
const PUBLIC_DIR = join(ROOT, config.paths.publicDir, "lab", "baseline");
|
|
28
|
+
|
|
29
|
+
// ── Cost constants ──────────────────────────────────────────
|
|
30
|
+
|
|
31
|
+
const LINUX_RUNNER_RATE = 0.006; // $/minute for ubuntu-latest
|
|
32
|
+
|
|
33
|
+
// ── Helpers ─────────────────────────────────────────────────
|
|
34
|
+
|
|
35
|
+
function safeParseJson(filePath, fallback = null) {
|
|
36
|
+
try {
|
|
37
|
+
return JSON.parse(readFileSync(filePath, "utf8"));
|
|
38
|
+
} catch {
|
|
39
|
+
return fallback;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
// ── Minute Budgets ──────────────────────────────────────────
|
|
44
|
+
|
|
45
|
+
const BUDGET_TIERS = [200, 500, 1000];
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Compute minute-budget reality checks for each tier.
|
|
49
|
+
*
|
|
50
|
+
* @param {number} avgMinutesPerRun
|
|
51
|
+
* @param {object} schedulePresets - { conservative, standard, aggressive }
|
|
52
|
+
* @param {object} adapterStats - per-adapter stats
|
|
53
|
+
* @returns {object} Keyed by tier (200, 500, 1000)
|
|
54
|
+
*/
|
|
55
|
+
export function computeMinuteBudgets(avgMinutesPerRun, schedulePresets, adapterStats) {
|
|
56
|
+
const budgets = {};
|
|
57
|
+
|
|
58
|
+
// Ordered from most aggressive to most conservative
|
|
59
|
+
const presetOrder = [
|
|
60
|
+
{ name: "aggressive", preset: schedulePresets.aggressive },
|
|
61
|
+
{ name: "standard", preset: schedulePresets.standard },
|
|
62
|
+
{ name: "conservative", preset: schedulePresets.conservative },
|
|
63
|
+
];
|
|
64
|
+
|
|
65
|
+
for (const tier of BUDGET_TIERS) {
|
|
66
|
+
const maxRunsPerMonth = avgMinutesPerRun > 0 ? Math.floor(tier / avgMinutesPerRun) : 0;
|
|
67
|
+
|
|
68
|
+
// Pick the most aggressive preset that fits with 20% headroom
|
|
69
|
+
let recommendedPreset = "conservative";
|
|
70
|
+
for (const { name, preset } of presetOrder) {
|
|
71
|
+
if (preset.estimatedMinutes <= tier * 0.80) {
|
|
72
|
+
recommendedPreset = name;
|
|
73
|
+
break;
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
const recommended = schedulePresets[recommendedPreset];
|
|
78
|
+
const headroom = Math.round((tier - (recommended?.estimatedMinutes || 0)) * 10) / 10;
|
|
79
|
+
|
|
80
|
+
const whatStops = [];
|
|
81
|
+
if (schedulePresets.aggressive.estimatedMinutes > tier) {
|
|
82
|
+
whatStops.push("Aggressive (2x-weekly) not viable");
|
|
83
|
+
}
|
|
84
|
+
if (schedulePresets.standard.estimatedMinutes > tier) {
|
|
85
|
+
whatStops.push("Weekly schedule not viable \u2014 must run biweekly or less");
|
|
86
|
+
}
|
|
87
|
+
if (schedulePresets.conservative.estimatedMinutes > tier) {
|
|
88
|
+
whatStops.push("Even biweekly exceeds budget \u2014 manual runs only");
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
// Flag uncached adapters
|
|
92
|
+
for (const [ns, stats] of Object.entries(adapterStats || {})) {
|
|
93
|
+
if (stats.hitRate === 0 && stats.avgCalls > 0) {
|
|
94
|
+
whatStops.push(`Uncached adapter "${ns}" adds cost`);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
budgets[tier] = { maxRunsPerMonth, recommendedPreset, headroom, whatStops };
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
return budgets;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
// ── Core ────────────────────────────────────────────────────
|
|
105
|
+
|
|
106
|
+
/**
|
|
107
|
+
* Compute baseline statistics from ops history.
|
|
108
|
+
*
|
|
109
|
+
* @param {Array} history - ops-history.json entries (newest first)
|
|
110
|
+
* @param {{ period?: string }} opts
|
|
111
|
+
* @returns {object} Baseline object
|
|
112
|
+
*/
|
|
113
|
+
export function computeBaseline(history, opts = {}) {
|
|
114
|
+
const { period = "weekly" } = opts;
|
|
115
|
+
|
|
116
|
+
if (!history || history.length === 0) {
|
|
117
|
+
return {
|
|
118
|
+
runCount: 0,
|
|
119
|
+
period: { start: null, end: null, cadence: period },
|
|
120
|
+
avgRuntimeMs: 0,
|
|
121
|
+
p95RuntimeMs: 0,
|
|
122
|
+
stddevRuntimeMs: 0,
|
|
123
|
+
confidenceLabel: "Low",
|
|
124
|
+
avgCacheHitRate: 0,
|
|
125
|
+
avgMinutesPerRun: 0,
|
|
126
|
+
failureRate: 0,
|
|
127
|
+
adapterStats: {},
|
|
128
|
+
schedulePresets: {
|
|
129
|
+
conservative: { cadence: "biweekly", monthlyRuns: 2, estimatedMinutes: 0, estimatedCost: 0 },
|
|
130
|
+
standard: { cadence: "weekly", monthlyRuns: 4, estimatedMinutes: 0, estimatedCost: 0 },
|
|
131
|
+
aggressive: { cadence: "2x-weekly", monthlyRuns: 8, estimatedMinutes: 0, estimatedCost: 0 },
|
|
132
|
+
},
|
|
133
|
+
projection: {
|
|
134
|
+
monthlyRunCount: period === "weekly" ? 4 : 2,
|
|
135
|
+
estimatedMinutes: 0,
|
|
136
|
+
estimatedCost: 0,
|
|
137
|
+
riskItems: ["No run data available — baseline cannot be computed"],
|
|
138
|
+
},
|
|
139
|
+
minuteBudgets: computeMinuteBudgets(0, {
|
|
140
|
+
conservative: { cadence: "biweekly", monthlyRuns: 2, estimatedMinutes: 0, estimatedCost: 0 },
|
|
141
|
+
standard: { cadence: "weekly", monthlyRuns: 4, estimatedMinutes: 0, estimatedCost: 0 },
|
|
142
|
+
aggressive: { cadence: "2x-weekly", monthlyRuns: 8, estimatedMinutes: 0, estimatedCost: 0 },
|
|
143
|
+
}, {}),
|
|
144
|
+
};
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
const runCount = history.length;
|
|
148
|
+
|
|
149
|
+
// Period
|
|
150
|
+
const dates = history.map((r) => r.date).filter(Boolean).sort();
|
|
151
|
+
const start = dates[0] || null;
|
|
152
|
+
const end = dates[dates.length - 1] || null;
|
|
153
|
+
|
|
154
|
+
// Duration stats
|
|
155
|
+
const durations = history.map((r) => r.totalDurationMs || 0);
|
|
156
|
+
const avgRuntimeMs = Math.round(durations.reduce((s, d) => s + d, 0) / runCount);
|
|
157
|
+
|
|
158
|
+
// P95: sort ascending, pick index at ceil(0.95 * n) - 1
|
|
159
|
+
const sorted = [...durations].sort((a, b) => a - b);
|
|
160
|
+
const p95Idx = Math.max(0, Math.ceil(0.95 * sorted.length) - 1);
|
|
161
|
+
const p95RuntimeMs = sorted[p95Idx];
|
|
162
|
+
|
|
163
|
+
// Standard deviation
|
|
164
|
+
const variance = durations.reduce((s, d) => s + Math.pow(d - avgRuntimeMs, 2), 0) / runCount;
|
|
165
|
+
const stddevRuntimeMs = Math.round(Math.sqrt(variance));
|
|
166
|
+
|
|
167
|
+
// Confidence label
|
|
168
|
+
const confidenceLabel = runCount < 4 ? "Low" : runCount <= 12 ? "Medium" : "High";
|
|
169
|
+
|
|
170
|
+
// Cache hit rate
|
|
171
|
+
const cacheRates = history.map((r) => r.costStats?.cacheHitRate || 0);
|
|
172
|
+
const avgCacheHitRate = Math.round(cacheRates.reduce((s, r) => s + r, 0) / runCount * 100) / 100;
|
|
173
|
+
|
|
174
|
+
// Minutes per run
|
|
175
|
+
const minutes = history.map((r) => r.minutesEstimate || Math.max(1, Math.ceil((r.totalDurationMs || 0) / 60000)));
|
|
176
|
+
const avgMinutesPerRun = Math.round(minutes.reduce((s, m) => s + m, 0) / runCount * 10) / 10;
|
|
177
|
+
|
|
178
|
+
// Failure rate
|
|
179
|
+
const failures = history.filter((r) => !r.batchOk || (r.publishErrors || 0) > 0).length;
|
|
180
|
+
const failureRate = Math.round(failures / runCount * 100) / 100;
|
|
181
|
+
|
|
182
|
+
// Adapter stats (aggregated averages)
|
|
183
|
+
const adapterAgg = {};
|
|
184
|
+
for (const run of history) {
|
|
185
|
+
if (!run.costStats?.adapterBreakdown) continue;
|
|
186
|
+
for (const [ns, stats] of Object.entries(run.costStats.adapterBreakdown)) {
|
|
187
|
+
if (!adapterAgg[ns]) adapterAgg[ns] = { totalCalls: 0, totalCached: 0, runCount: 0 };
|
|
188
|
+
adapterAgg[ns].totalCalls += stats.calls || 0;
|
|
189
|
+
adapterAgg[ns].totalCached += stats.cached || 0;
|
|
190
|
+
adapterAgg[ns].runCount++;
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
const adapterStats = {};
|
|
195
|
+
for (const [ns, agg] of Object.entries(adapterAgg)) {
|
|
196
|
+
adapterStats[ns] = {
|
|
197
|
+
avgCalls: Math.round(agg.totalCalls / agg.runCount * 10) / 10,
|
|
198
|
+
avgCached: Math.round(agg.totalCached / agg.runCount * 10) / 10,
|
|
199
|
+
hitRate: agg.totalCalls > 0 ? Math.round(agg.totalCached / agg.totalCalls * 100) / 100 : 0,
|
|
200
|
+
};
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
// Projection
|
|
204
|
+
const monthlyRunCount = period === "weekly" ? 4 : 2;
|
|
205
|
+
const estimatedMinutes = Math.round(avgMinutesPerRun * monthlyRunCount * 10) / 10;
|
|
206
|
+
const estimatedCost = Math.round(estimatedMinutes * LINUX_RUNNER_RATE * 1000) / 1000;
|
|
207
|
+
|
|
208
|
+
// Schedule presets
|
|
209
|
+
const schedulePresets = {
|
|
210
|
+
conservative: {
|
|
211
|
+
cadence: "biweekly",
|
|
212
|
+
monthlyRuns: 2,
|
|
213
|
+
estimatedMinutes: Math.round(avgMinutesPerRun * 2 * 10) / 10,
|
|
214
|
+
estimatedCost: Math.round(avgMinutesPerRun * 2 * LINUX_RUNNER_RATE * 1000) / 1000,
|
|
215
|
+
},
|
|
216
|
+
standard: {
|
|
217
|
+
cadence: "weekly",
|
|
218
|
+
monthlyRuns: 4,
|
|
219
|
+
estimatedMinutes: Math.round(avgMinutesPerRun * 4 * 10) / 10,
|
|
220
|
+
estimatedCost: Math.round(avgMinutesPerRun * 4 * LINUX_RUNNER_RATE * 1000) / 1000,
|
|
221
|
+
},
|
|
222
|
+
aggressive: {
|
|
223
|
+
cadence: "2x-weekly",
|
|
224
|
+
monthlyRuns: 8,
|
|
225
|
+
estimatedMinutes: Math.round(avgMinutesPerRun * 8 * 10) / 10,
|
|
226
|
+
estimatedCost: Math.round(avgMinutesPerRun * 8 * LINUX_RUNNER_RATE * 1000) / 1000,
|
|
227
|
+
},
|
|
228
|
+
};
|
|
229
|
+
|
|
230
|
+
// Risk detection
|
|
231
|
+
const riskItems = [];
|
|
232
|
+
if (runCount < 4) {
|
|
233
|
+
riskItems.push("Low run count \u2014 baseline may not be representative");
|
|
234
|
+
}
|
|
235
|
+
if (avgCacheHitRate < 0.50) {
|
|
236
|
+
riskItems.push("Low cache efficiency \u2014 consider increasing maxAgeHours");
|
|
237
|
+
}
|
|
238
|
+
if (failureRate > 0.10) {
|
|
239
|
+
riskItems.push("High failure rate \u2014 review error codes");
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
// Check for adapters with 0% cache
|
|
243
|
+
for (const [ns, stats] of Object.entries(adapterStats)) {
|
|
244
|
+
if (stats.hitRate === 0 && adapterAgg[ns].totalCalls > 0) {
|
|
245
|
+
riskItems.push(`Adapter "${ns}" has no cache hits`);
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
// Minute budgets
|
|
250
|
+
const minuteBudgets = computeMinuteBudgets(avgMinutesPerRun, schedulePresets, adapterStats);
|
|
251
|
+
|
|
252
|
+
return {
|
|
253
|
+
runCount,
|
|
254
|
+
period: { start, end, cadence: period },
|
|
255
|
+
avgRuntimeMs,
|
|
256
|
+
p95RuntimeMs,
|
|
257
|
+
stddevRuntimeMs,
|
|
258
|
+
confidenceLabel,
|
|
259
|
+
avgCacheHitRate,
|
|
260
|
+
avgMinutesPerRun,
|
|
261
|
+
failureRate,
|
|
262
|
+
adapterStats,
|
|
263
|
+
schedulePresets,
|
|
264
|
+
projection: {
|
|
265
|
+
monthlyRunCount,
|
|
266
|
+
estimatedMinutes,
|
|
267
|
+
estimatedCost,
|
|
268
|
+
riskItems,
|
|
269
|
+
},
|
|
270
|
+
minuteBudgets,
|
|
271
|
+
};
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
/**
|
|
275
|
+
* Generate baseline markdown summary.
|
|
276
|
+
*
|
|
277
|
+
* @param {object} baseline - Baseline object from computeBaseline
|
|
278
|
+
* @returns {string} Markdown string
|
|
279
|
+
*/
|
|
280
|
+
export function generateBaselineMd(baseline) {
|
|
281
|
+
const lines = [];
|
|
282
|
+
lines.push("# NameOps Baseline Report");
|
|
283
|
+
lines.push("");
|
|
284
|
+
lines.push(`**Run count:** ${baseline.runCount}`);
|
|
285
|
+
lines.push(`**Period:** ${baseline.period.start || "N/A"} \u2013 ${baseline.period.end || "N/A"} (${baseline.period.cadence})`);
|
|
286
|
+
lines.push("");
|
|
287
|
+
|
|
288
|
+
lines.push("## Performance");
|
|
289
|
+
lines.push(`- Avg runtime: ${Math.round(baseline.avgRuntimeMs / 1000)}s`);
|
|
290
|
+
lines.push(`- P95 runtime: ${Math.round(baseline.p95RuntimeMs / 1000)}s`);
|
|
291
|
+
lines.push(`- Stddev runtime: ${Math.round((baseline.stddevRuntimeMs || 0) / 1000)}s`);
|
|
292
|
+
lines.push(`- Confidence: ${baseline.confidenceLabel || "N/A"}`);
|
|
293
|
+
lines.push(`- Avg cache hit rate: ${Math.round(baseline.avgCacheHitRate * 100)}%`);
|
|
294
|
+
lines.push(`- Failure rate: ${Math.round(baseline.failureRate * 100)}%`);
|
|
295
|
+
lines.push("");
|
|
296
|
+
|
|
297
|
+
if (Object.keys(baseline.adapterStats).length > 0) {
|
|
298
|
+
lines.push("## Adapter Performance");
|
|
299
|
+
lines.push("| Adapter | Avg Calls | Avg Cached | Hit Rate |");
|
|
300
|
+
lines.push("|---------|-----------|------------|----------|");
|
|
301
|
+
for (const [ns, stats] of Object.entries(baseline.adapterStats)) {
|
|
302
|
+
lines.push(`| ${ns} | ${stats.avgCalls} | ${stats.avgCached} | ${Math.round(stats.hitRate * 100)}% |`);
|
|
303
|
+
}
|
|
304
|
+
lines.push("");
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
lines.push("## Cost Projection (Monthly)");
|
|
308
|
+
lines.push(`- Estimated runs: ${baseline.projection.monthlyRunCount}`);
|
|
309
|
+
lines.push(`- Estimated minutes: ${baseline.projection.estimatedMinutes}`);
|
|
310
|
+
lines.push(`- Estimated cost: $${baseline.projection.estimatedCost.toFixed(3)}`);
|
|
311
|
+
lines.push(`- Rate: $${LINUX_RUNNER_RATE}/min (ubuntu-latest)`);
|
|
312
|
+
lines.push("");
|
|
313
|
+
|
|
314
|
+
if (baseline.projection.riskItems.length > 0) {
|
|
315
|
+
lines.push("## Risks");
|
|
316
|
+
for (const risk of baseline.projection.riskItems) {
|
|
317
|
+
lines.push(`- \u26a0\ufe0f ${risk}`);
|
|
318
|
+
}
|
|
319
|
+
lines.push("");
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
// Minute budgets
|
|
323
|
+
if (baseline.minuteBudgets) {
|
|
324
|
+
lines.push("## Minutes Budget Reality Check");
|
|
325
|
+
lines.push("| Budget (min/mo) | Max Runs | Recommended Preset | Headroom |");
|
|
326
|
+
lines.push("|-----------------|----------|-------------------|----------|");
|
|
327
|
+
for (const tier of [200, 500, 1000]) {
|
|
328
|
+
const mb = baseline.minuteBudgets[tier];
|
|
329
|
+
if (mb) {
|
|
330
|
+
lines.push(`| ${tier} | ${mb.maxRunsPerMonth} | ${mb.recommendedPreset} | ${mb.headroom} min |`);
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
lines.push("");
|
|
334
|
+
|
|
335
|
+
// What stops section for tightest tier
|
|
336
|
+
const tightest = baseline.minuteBudgets[200];
|
|
337
|
+
if (tightest && tightest.whatStops.length > 0) {
|
|
338
|
+
lines.push("## What Stops If Minutes Ran Out? (200 min budget)");
|
|
339
|
+
for (const item of tightest.whatStops) {
|
|
340
|
+
lines.push(`- \u26a0\ufe0f ${item}`);
|
|
341
|
+
}
|
|
342
|
+
lines.push("");
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
lines.push(`*Generated: ${new Date().toISOString().slice(0, 10)}*`);
|
|
347
|
+
lines.push("");
|
|
348
|
+
|
|
349
|
+
return lines.join("\n");
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
/**
|
|
353
|
+
* Full pipeline: load data, compute baseline, write outputs.
|
|
354
|
+
*
|
|
355
|
+
* @param {{ dataDir?: string, publicDir?: string, dryRun?: boolean }} opts
|
|
356
|
+
* @returns {object} Baseline object
|
|
357
|
+
*/
|
|
358
|
+
export function generateBaseline(opts = {}) {
|
|
359
|
+
const { dataDir = DATA_DIR, publicDir = PUBLIC_DIR, dryRun = false } = opts;
|
|
360
|
+
|
|
361
|
+
const rawHistory = safeParseJson(join(dataDir, "ops-history.json"), []);
|
|
362
|
+
const history = Array.isArray(rawHistory) ? rawHistory : (rawHistory.runs || []);
|
|
363
|
+
const baseline = computeBaseline(history);
|
|
364
|
+
|
|
365
|
+
if (dryRun) {
|
|
366
|
+
console.log(` [dry-run] Would write baseline (${baseline.runCount} runs)`);
|
|
367
|
+
console.log(` [dry-run] Avg runtime: ${Math.round(baseline.avgRuntimeMs / 1000)}s`);
|
|
368
|
+
console.log(` [dry-run] Projected monthly cost: $${baseline.projection.estimatedCost.toFixed(3)}`);
|
|
369
|
+
return baseline;
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
// Write baseline.json
|
|
373
|
+
const jsonOut = {
|
|
374
|
+
generatedAt: new Date().toISOString(),
|
|
375
|
+
...baseline,
|
|
376
|
+
};
|
|
377
|
+
writeFileSync(join(dataDir, "baseline.json"), JSON.stringify(jsonOut, null, 2) + "\n", "utf8");
|
|
378
|
+
console.log(` Wrote baseline.json (${baseline.runCount} runs)`);
|
|
379
|
+
|
|
380
|
+
// Write baseline.md
|
|
381
|
+
mkdirSync(publicDir, { recursive: true });
|
|
382
|
+
const md = generateBaselineMd(baseline);
|
|
383
|
+
writeFileSync(join(publicDir, "baseline.md"), md, "utf8");
|
|
384
|
+
console.log(` Wrote baseline.md`);
|
|
385
|
+
|
|
386
|
+
return baseline;
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
// ── Entry point ─────────────────────────────────────────────
|
|
390
|
+
|
|
391
|
+
const isMain = process.argv[1] &&
|
|
392
|
+
resolve(process.argv[1]).endsWith("gen-baseline.mjs");
|
|
393
|
+
|
|
394
|
+
if (isMain) {
|
|
395
|
+
const dryRun = process.argv.includes("--dry-run");
|
|
396
|
+
console.log("Generating baseline...");
|
|
397
|
+
if (dryRun) console.log(" Mode: DRY RUN");
|
|
398
|
+
|
|
399
|
+
const baseline = generateBaseline({ dryRun });
|
|
400
|
+
console.log(` Runs: ${baseline.runCount}`);
|
|
401
|
+
console.log(` Risks: ${baseline.projection.riskItems.length}`);
|
|
402
|
+
}
|