@homenshum/convex-mcp-nodebench 0.9.9 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +0 -28
- package/dist/index.js +15 -38
- package/dist/tools/actionAuditTools.js +4 -10
- package/dist/tools/authorizationTools.js +9 -39
- package/dist/tools/critterTools.js +14 -0
- package/dist/tools/functionTools.js +4 -7
- package/dist/tools/methodologyTools.js +14 -0
- package/dist/tools/qualityGateTools.js +15 -43
- package/dist/tools/queryEfficiencyTools.js +3 -6
- package/dist/tools/reportingTools.js +1 -1
- package/dist/tools/storageAuditTools.js +1 -1
- package/package.json +1 -2
package/README.md
CHANGED
|
@@ -193,34 +193,6 @@ packages/convex-mcp-nodebench/
|
|
|
193
193
|
|
|
194
194
|
## Changelog
|
|
195
195
|
|
|
196
|
-
### v0.9.9
|
|
197
|
-
- **MCP annotations**: `tools/list` returns `annotations: { title, category, phase, complexity }` per MCP 2025-11-25 spec — improves Claude Code Tool Search ranking
|
|
198
|
-
- **TOON output**: Token-Oriented Object Notation encoding (~40% fewer tokens), on by default, opt-out with `--no-toon`
|
|
199
|
-
- **Quality gate tuning**: Monorepo-scale thresholds — new `scale` parameter (`small`/`medium`/`large`) auto-adjusts warning/cast/collect limits
|
|
200
|
-
|
|
201
|
-
### v0.9.8
|
|
202
|
-
- **0 criticals**: Severity philosophy aligned — critical = runtime failure, warning = security posture / best practice
|
|
203
|
-
- **Auth**: Downgraded "no auth on DB write" and "sensitive name no auth" from critical to warning
|
|
204
|
-
- **Functions**: All missing-args/returns/handler downgraded to warning
|
|
205
|
-
- **Actions**: `ctx.db` access and missing `"use node"` downgraded to warning
|
|
206
|
-
|
|
207
|
-
### v0.9.7
|
|
208
|
-
- **Auth helper detection**: Pre-scans files for local functions wrapping `getAuthUserId()` — mutations calling helpers like `getSafeUserId(ctx)` now correctly detected as having auth
|
|
209
|
-
- **-50 false positives**: 188 → 138 criticals
|
|
210
|
-
|
|
211
|
-
### v0.9.6
|
|
212
|
-
- **Audit type key fixes**: `functionTools` → `"functions"`, `storageAuditTools` → `"storage"` — quality gate now sees 12/12 audit types
|
|
213
|
-
- **Function severity calibration**: Missing args for queries/internal functions downgraded to warning
|
|
214
|
-
|
|
215
|
-
### v0.9.5
|
|
216
|
-
- **Dogfood cycle**: Reduced criticals from 558 → 198 by running all 12 audits against the monorepo
|
|
217
|
-
- **Quality gate**: Excludes test/eval files from `as any` count, only counts warning-level unbounded collects
|
|
218
|
-
|
|
219
|
-
### v0.9.2 – v0.9.4
|
|
220
|
-
- **README rewrite**: Comprehensive 36-tool documentation with categorized tables
|
|
221
|
-
- **Architect E2E tests**: 10 tests validating industry-latest concepts
|
|
222
|
-
- **Strategy matching**: Pattern priority ordering fixes in architect tools
|
|
223
|
-
|
|
224
196
|
### v0.9.1
|
|
225
197
|
- **Fix**: Strategy matching order in architect tools -- specific patterns (`ctx.db.query`, `ctx.runMutation`) now matched before generic keywords (`query`, `mutation`)
|
|
226
198
|
|
package/dist/index.js
CHANGED
|
@@ -15,7 +15,6 @@
|
|
|
15
15
|
import { Server } from "@modelcontextprotocol/sdk/server/index.js";
|
|
16
16
|
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
|
|
17
17
|
import { ListToolsRequestSchema, CallToolRequestSchema, ListResourcesRequestSchema, ReadResourceRequestSchema, ListPromptsRequestSchema, GetPromptRequestSchema, } from "@modelcontextprotocol/sdk/types.js";
|
|
18
|
-
import { encode as toonEncode } from "@toon-format/toon";
|
|
19
18
|
import { getDb, seedGotchasIfEmpty } from "./db.js";
|
|
20
19
|
import { schemaTools } from "./tools/schemaTools.js";
|
|
21
20
|
import { functionTools } from "./tools/functionTools.js";
|
|
@@ -45,9 +44,6 @@ import { architectTools } from "./tools/architectTools.js";
|
|
|
45
44
|
import { CONVEX_GOTCHAS } from "./gotchaSeed.js";
|
|
46
45
|
import { REGISTRY } from "./tools/toolRegistry.js";
|
|
47
46
|
import { initEmbeddingIndex } from "./tools/embeddingProvider.js";
|
|
48
|
-
// ── CLI flags ────────────────────────────────────────────────────────
|
|
49
|
-
const cliArgs = process.argv.slice(2);
|
|
50
|
-
const useToon = !cliArgs.includes("--no-toon");
|
|
51
47
|
// ── All tools ───────────────────────────────────────────────────────
|
|
52
48
|
const ALL_TOOLS = [
|
|
53
49
|
...schemaTools,
|
|
@@ -83,7 +79,7 @@ for (const tool of ALL_TOOLS) {
|
|
|
83
79
|
// ── Server setup ────────────────────────────────────────────────────
|
|
84
80
|
const server = new Server({
|
|
85
81
|
name: "convex-mcp-nodebench",
|
|
86
|
-
version: "0.9.
|
|
82
|
+
version: "0.9.4",
|
|
87
83
|
}, {
|
|
88
84
|
capabilities: {
|
|
89
85
|
tools: {},
|
|
@@ -133,25 +129,13 @@ initEmbeddingIndex(embeddingCorpus).catch(() => {
|
|
|
133
129
|
/* Embedding init failed — semantic search stays disabled */
|
|
134
130
|
});
|
|
135
131
|
// ── Tool listing ────────────────────────────────────────────────────
|
|
136
|
-
// Includes MCP 2025-11-25 spec annotations: category, phase, complexity (model tier hint)
|
|
137
132
|
server.setRequestHandler(ListToolsRequestSchema, async () => {
|
|
138
133
|
return {
|
|
139
|
-
tools: ALL_TOOLS.map((t) => {
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
inputSchema: t.inputSchema,
|
|
145
|
-
...(entry ? {
|
|
146
|
-
annotations: {
|
|
147
|
-
title: t.name.replace(/_/g, " "),
|
|
148
|
-
category: entry.category,
|
|
149
|
-
phase: entry.phase,
|
|
150
|
-
complexity: entry.complexity,
|
|
151
|
-
},
|
|
152
|
-
} : {}),
|
|
153
|
-
};
|
|
154
|
-
}),
|
|
134
|
+
tools: ALL_TOOLS.map((t) => ({
|
|
135
|
+
name: t.name,
|
|
136
|
+
description: t.description,
|
|
137
|
+
inputSchema: t.inputSchema,
|
|
138
|
+
})),
|
|
155
139
|
};
|
|
156
140
|
});
|
|
157
141
|
// ── Tool execution ──────────────────────────────────────────────────
|
|
@@ -173,24 +157,11 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
|
|
|
173
157
|
}
|
|
174
158
|
try {
|
|
175
159
|
const result = await tool.handler(args || {});
|
|
176
|
-
// Serialize: TOON (~40% fewer tokens) or JSON
|
|
177
|
-
let serialized;
|
|
178
|
-
if (useToon) {
|
|
179
|
-
try {
|
|
180
|
-
serialized = toonEncode(result);
|
|
181
|
-
}
|
|
182
|
-
catch {
|
|
183
|
-
serialized = JSON.stringify(result, null, 2);
|
|
184
|
-
}
|
|
185
|
-
}
|
|
186
|
-
else {
|
|
187
|
-
serialized = JSON.stringify(result, null, 2);
|
|
188
|
-
}
|
|
189
160
|
return {
|
|
190
161
|
content: [
|
|
191
162
|
{
|
|
192
163
|
type: "text",
|
|
193
|
-
text:
|
|
164
|
+
text: JSON.stringify(result, null, 2),
|
|
194
165
|
},
|
|
195
166
|
],
|
|
196
167
|
};
|
|
@@ -378,7 +349,9 @@ After running all audits, summarize:
|
|
|
378
349
|
- Total issues by severity (critical/warning/info)
|
|
379
350
|
- Top 5 most impactful issues to fix first
|
|
380
351
|
- Quality gate score and grade
|
|
381
|
-
- Trend direction if previous audits exist (use convex_audit_diff)
|
|
352
|
+
- Trend direction if previous audits exist (use convex_audit_diff)
|
|
353
|
+
|
|
354
|
+
IMPORTANT — Analyst diagnostic: For every issue found, trace the ROOT CAUSE — don't just list what's wrong, explain WHY it happened. Ask "why" 5 times. Record root causes with convex_record_gotcha so they don't recur.`,
|
|
382
355
|
},
|
|
383
356
|
},
|
|
384
357
|
],
|
|
@@ -402,6 +375,8 @@ After running all audits, summarize:
|
|
|
402
375
|
6. convex_schema_migration_plan — Compare against previous snapshot for breaking changes
|
|
403
376
|
7. convex_quality_gate — Final quality check with thresholds
|
|
404
377
|
|
|
378
|
+
For each blocker: diagnose the ROOT CAUSE, not just the symptom. Explain WHY the issue exists and how to prevent it from recurring.
|
|
379
|
+
|
|
405
380
|
Report: DEPLOY or DO NOT DEPLOY with specific blockers to fix.`,
|
|
406
381
|
},
|
|
407
382
|
},
|
|
@@ -425,7 +400,9 @@ Report: DEPLOY or DO NOT DEPLOY with specific blockers to fix.`,
|
|
|
425
400
|
5. convex_audit_pagination — Unbounded numItems (DoS risk)
|
|
426
401
|
6. convex_audit_transaction_safety — Race condition risks
|
|
427
402
|
|
|
428
|
-
Focus on: unauthorized data access, unvalidated inputs, missing error boundaries, and potential data corruption vectors
|
|
403
|
+
Focus on: unauthorized data access, unvalidated inputs, missing error boundaries, and potential data corruption vectors.
|
|
404
|
+
|
|
405
|
+
Analyst diagnostic: For each security finding, trace upstream to the ROOT CAUSE. Don't just flag the symptom — explain what system condition allowed the vulnerability to exist. Record findings with convex_record_gotcha.`,
|
|
429
406
|
},
|
|
430
407
|
},
|
|
431
408
|
],
|
|
@@ -71,16 +71,11 @@ function auditActions(convexDir) {
|
|
|
71
71
|
}
|
|
72
72
|
}
|
|
73
73
|
const body = lines.slice(startLine, endLine).join("\n");
|
|
74
|
-
// Check 1: ctx.db access in action (
|
|
75
|
-
|
|
76
|
-
// e.g. ctx.runMutation(async (ctx) => { ctx.db.patch(...) }) — the inner ctx is a mutation context
|
|
77
|
-
const hasInlineCallback = /ctx\.run(Mutation|Query)\s*\(\s*async\s*\(/.test(body);
|
|
78
|
-
if (/ctx\.db\.(get|query|insert|patch|replace|delete)\s*\(/.test(body) && !hasInlineCallback) {
|
|
74
|
+
// Check 1: ctx.db access in action (FATAL — not allowed)
|
|
75
|
+
if (/ctx\.db\.(get|query|insert|patch|replace|delete)\s*\(/.test(body)) {
|
|
79
76
|
actionsWithDbAccess++;
|
|
80
|
-
// internalAction ctx.db is a warning (not client-callable, likely called from controlled contexts)
|
|
81
|
-
// public action ctx.db is a warning too (runtime error but caught during development/testing)
|
|
82
77
|
issues.push({
|
|
83
|
-
severity: "
|
|
78
|
+
severity: "critical",
|
|
84
79
|
location: `${relativePath}:${startLine + 1}`,
|
|
85
80
|
functionName: funcName,
|
|
86
81
|
message: `${funcType} "${funcName}" accesses ctx.db directly. Actions cannot access the database — use ctx.runQuery/ctx.runMutation instead.`,
|
|
@@ -90,9 +85,8 @@ function auditActions(convexDir) {
|
|
|
90
85
|
// Check 2: Node API usage without "use node"
|
|
91
86
|
if (!hasUseNode && (nodeApis.test(body) || nodeCryptoApis.test(body))) {
|
|
92
87
|
actionsWithoutNodeDirective++;
|
|
93
|
-
// Warning: missing directive is a deployment concern caught during development
|
|
94
88
|
issues.push({
|
|
95
|
-
severity: "
|
|
89
|
+
severity: "critical",
|
|
96
90
|
location: `${relativePath}:${startLine + 1}`,
|
|
97
91
|
functionName: funcName,
|
|
98
92
|
message: `${funcType} "${funcName}" uses Node.js APIs but file lacks "use node" directive. Will fail in Convex runtime.`,
|
|
@@ -41,27 +41,6 @@ function auditAuthorization(convexDir) {
|
|
|
41
41
|
const content = readFileSync(filePath, "utf-8");
|
|
42
42
|
const relativePath = filePath.replace(convexDir, "").replace(/^[\\/]/, "");
|
|
43
43
|
const lines = content.split("\n");
|
|
44
|
-
// Pre-scan: detect local helper functions that wrap getAuthUserId/getUserIdentity
|
|
45
|
-
// Pattern: function getSafeUserId(ctx) { ... getAuthUserId(ctx) ... }
|
|
46
|
-
const authHelperNames = [];
|
|
47
|
-
const helperFuncPattern = /(?:async\s+)?function\s+(\w+)\s*\(/g;
|
|
48
|
-
let hm;
|
|
49
|
-
while ((hm = helperFuncPattern.exec(content)) !== null) {
|
|
50
|
-
const hStart = content.slice(0, hm.index).split("\n").length - 1;
|
|
51
|
-
const hBody = lines.slice(hStart, Math.min(hStart + 30, lines.length)).join("\n");
|
|
52
|
-
if (/getAuthUserId|getUserIdentity|getAuthSessionId/.test(hBody)) {
|
|
53
|
-
authHelperNames.push(hm[1]);
|
|
54
|
-
}
|
|
55
|
-
}
|
|
56
|
-
// Also check arrow function helpers: const getUserId = async (ctx) => { ... }
|
|
57
|
-
const arrowHelperPattern = /(?:const|let)\s+(\w+)\s*=\s*(?:async\s*)?\([^)]*\)\s*(?::\s*[^=]+)?\s*=>/g;
|
|
58
|
-
while ((hm = arrowHelperPattern.exec(content)) !== null) {
|
|
59
|
-
const hStart = content.slice(0, hm.index).split("\n").length - 1;
|
|
60
|
-
const hBody = lines.slice(hStart, Math.min(hStart + 20, lines.length)).join("\n");
|
|
61
|
-
if (/getAuthUserId|getUserIdentity|getAuthSessionId/.test(hBody)) {
|
|
62
|
-
authHelperNames.push(hm[1]);
|
|
63
|
-
}
|
|
64
|
-
}
|
|
65
44
|
for (let i = 0; i < lines.length; i++) {
|
|
66
45
|
const line = lines[i];
|
|
67
46
|
for (const ft of funcTypes) {
|
|
@@ -90,14 +69,10 @@ function auditAuthorization(convexDir) {
|
|
|
90
69
|
}
|
|
91
70
|
}
|
|
92
71
|
const body = lines.slice(i, endLine).join("\n");
|
|
93
|
-
|
|
94
|
-
const hasDirectAuth = /ctx\.auth\.getUserIdentity\s*\(\s*\)/.test(body) ||
|
|
72
|
+
const hasAuthCheck = /ctx\.auth\.getUserIdentity\s*\(\s*\)/.test(body) ||
|
|
95
73
|
/getUserIdentity/.test(body) ||
|
|
96
74
|
/getAuthUserId/.test(body) ||
|
|
97
75
|
/getAuthSessionId/.test(body);
|
|
98
|
-
const callsAuthHelper = authHelperNames.length > 0 &&
|
|
99
|
-
authHelperNames.some(h => new RegExp(`\\b${h}\\s*\\(`).test(body));
|
|
100
|
-
const hasAuthCheck = hasDirectAuth || callsAuthHelper;
|
|
101
76
|
const hasDbWrite = dbWriteOps.test(body);
|
|
102
77
|
const isSensitiveName = writeSensitive.test(funcName);
|
|
103
78
|
if (hasAuthCheck) {
|
|
@@ -106,13 +81,11 @@ function auditAuthorization(convexDir) {
|
|
|
106
81
|
const identityAssign = body.match(/(?:const|let)\s+(\w+)\s*=\s*await\s+ctx\.auth\.getUserIdentity\s*\(\s*\)/);
|
|
107
82
|
if (identityAssign) {
|
|
108
83
|
const varName = identityAssign[1];
|
|
109
|
-
|
|
110
|
-
const hasNullCheck = new RegExp(`if\\s*\\(\\s*!${varName}\\b|if\\s*\\(\\s*${varName}\\s*===?\\s*null|if\\s*\\(\\s*${varName}\\s*\\)|${varName}\\s*&&|${varName}\\s*\\?|throw.*!${varName}|${varName}\\s*!==?\\s*null|===\\s*${varName}|!==\\s*${varName}`).test(body);
|
|
84
|
+
const hasNullCheck = new RegExp(`if\\s*\\(\\s*!${varName}\\b|if\\s*\\(\\s*${varName}\\s*===?\\s*null|if\\s*\\(\\s*!${varName}\\s*\\)`).test(body);
|
|
111
85
|
if (!hasNullCheck) {
|
|
112
86
|
uncheckedIdentity++;
|
|
113
|
-
// Queries can intentionally return different data for auth/unauth — warning not critical
|
|
114
87
|
issues.push({
|
|
115
|
-
severity: "
|
|
88
|
+
severity: "critical",
|
|
116
89
|
location: `${relativePath}:${i + 1}`,
|
|
117
90
|
functionName: funcName,
|
|
118
91
|
message: `${ft} "${funcName}" calls getUserIdentity() but doesn't check for null. Unauthenticated users will get undefined identity.`,
|
|
@@ -124,13 +97,11 @@ function auditAuthorization(convexDir) {
|
|
|
124
97
|
const authUserAssign = body.match(/(?:const|let)\s+(\w+)\s*=\s*await\s+getAuthUserId\s*\(/);
|
|
125
98
|
if (authUserAssign) {
|
|
126
99
|
const varName = authUserAssign[1];
|
|
127
|
-
|
|
128
|
-
const hasNullCheck = new RegExp(`if\\s*\\(\\s*!${varName}\\b|if\\s*\\(\\s*${varName}\\s*===?\\s*null|if\\s*\\(\\s*${varName}\\s*\\)|${varName}\\s*&&|${varName}\\s*\\?|throw.*!${varName}|${varName}\\s*!==?\\s*null|===\\s*${varName}|!==\\s*${varName}`).test(body);
|
|
100
|
+
const hasNullCheck = new RegExp(`if\\s*\\(\\s*!${varName}\\b|if\\s*\\(\\s*${varName}\\s*===?\\s*null|throw.*!${varName}`).test(body);
|
|
129
101
|
if (!hasNullCheck) {
|
|
130
102
|
uncheckedIdentity++;
|
|
131
|
-
// Queries can intentionally return different data for auth/unauth — warning not critical
|
|
132
103
|
issues.push({
|
|
133
|
-
severity: "
|
|
104
|
+
severity: "critical",
|
|
134
105
|
location: `${relativePath}:${i + 1}`,
|
|
135
106
|
functionName: funcName,
|
|
136
107
|
message: `${ft} "${funcName}" calls getAuthUserId() but doesn't check for null. Unauthenticated users will get null userId.`,
|
|
@@ -141,13 +112,11 @@ function auditAuthorization(convexDir) {
|
|
|
141
112
|
}
|
|
142
113
|
else {
|
|
143
114
|
withoutAuth++;
|
|
144
|
-
//
|
|
145
|
-
// Downgraded from critical — missing auth is a security posture issue, not a runtime failure.
|
|
146
|
-
// Many monorepo mutations are system-level (called by actions/schedulers), not client-facing.
|
|
115
|
+
// Critical: public mutation/action with DB writes but no auth
|
|
147
116
|
if ((ft === "mutation" || ft === "action") && hasDbWrite) {
|
|
148
117
|
const sensitiveHint = isSensitiveName ? ` Name "${funcName}" suggests a destructive operation.` : "";
|
|
149
118
|
issues.push({
|
|
150
|
-
severity: "
|
|
119
|
+
severity: "critical",
|
|
151
120
|
location: `${relativePath}:${i + 1}`,
|
|
152
121
|
functionName: funcName,
|
|
153
122
|
message: `Public ${ft} "${funcName}" writes to DB without auth check. Any client can call this.${sensitiveHint}`,
|
|
@@ -155,8 +124,9 @@ function auditAuthorization(convexDir) {
|
|
|
155
124
|
});
|
|
156
125
|
}
|
|
157
126
|
else if (isSensitiveName) {
|
|
127
|
+
// Only flag sensitive name separately if not already caught by DB-write check
|
|
158
128
|
issues.push({
|
|
159
|
-
severity: "
|
|
129
|
+
severity: "critical",
|
|
160
130
|
location: `${relativePath}:${i + 1}`,
|
|
161
131
|
functionName: funcName,
|
|
162
132
|
message: `Public ${ft} "${funcName}" has a sensitive name but no auth check. Consider making it internal or adding auth.`,
|
|
@@ -137,6 +137,20 @@ function scoreCritterCheck(input) {
|
|
|
137
137
|
score += 10;
|
|
138
138
|
feedback.push("Good: success criteria defined — this makes the deploy gate concrete.");
|
|
139
139
|
}
|
|
140
|
+
// ── Check 10: Bandaid detection — symptom fixes without root-cause reasoning ──
|
|
141
|
+
const bandaidPatterns = [
|
|
142
|
+
"add try.?catch", "wrap in try", "catch the error", "suppress the error",
|
|
143
|
+
"add optional chaining", "add \\?\\.", "silence the warning",
|
|
144
|
+
"add as any", "cast to any", "ignore the type",
|
|
145
|
+
"add a timeout", "increase the timeout", "add a delay",
|
|
146
|
+
"delete the test", "skip the test", "disable the test",
|
|
147
|
+
"hide the error", "remove the warning",
|
|
148
|
+
];
|
|
149
|
+
const bandaidRegex = new RegExp(bandaidPatterns.join("|"), "i");
|
|
150
|
+
if (bandaidRegex.test(taskLower) && !whyLower.includes("root cause") && !whyLower.includes("because") && whyLower.length < 50) {
|
|
151
|
+
score -= 20;
|
|
152
|
+
feedback.push("Bandaid alert: this looks like a symptom fix. What's the root cause? Diagnose like an analyst, not a junior dev.");
|
|
153
|
+
}
|
|
140
154
|
score = Math.max(0, Math.min(100, score));
|
|
141
155
|
let verdict;
|
|
142
156
|
if (score >= 70) {
|
|
@@ -57,7 +57,7 @@ function extractFunctions(convexDir) {
|
|
|
57
57
|
filePath,
|
|
58
58
|
relativePath,
|
|
59
59
|
line: i + 1,
|
|
60
|
-
hasArgs: /args\s*:\s*[\{\v]/.test(chunk) || /args\s*:\s*v\./.test(chunk)
|
|
60
|
+
hasArgs: /args\s*:\s*[\{\v]/.test(chunk) || /args\s*:\s*v\./.test(chunk),
|
|
61
61
|
hasReturns: /returns\s*:\s*v\./.test(chunk),
|
|
62
62
|
hasHandler: /handler\s*:/.test(chunk),
|
|
63
63
|
});
|
|
@@ -76,10 +76,8 @@ function auditFunctions(convexDir) {
|
|
|
76
76
|
if (fn.type === "httpAction")
|
|
77
77
|
continue; // httpActions don't have args/returns validators
|
|
78
78
|
if (!fn.hasArgs) {
|
|
79
|
-
// Missing args validator is a best practice recommendation, not a runtime failure.
|
|
80
|
-
// Functions without args simply accept no arguments — no unvalidated input risk.
|
|
81
79
|
issues.push({
|
|
82
|
-
severity: "
|
|
80
|
+
severity: "critical",
|
|
83
81
|
location: `${fn.relativePath}:${fn.line}`,
|
|
84
82
|
functionName: fn.name,
|
|
85
83
|
message: `${fn.type} "${fn.name}" is missing args validator`,
|
|
@@ -96,9 +94,8 @@ function auditFunctions(convexDir) {
|
|
|
96
94
|
});
|
|
97
95
|
}
|
|
98
96
|
if (!fn.hasHandler) {
|
|
99
|
-
// Old shorthand syntax (query(async (ctx) => {})) works fine — just a style recommendation
|
|
100
97
|
issues.push({
|
|
101
|
-
severity: "
|
|
98
|
+
severity: "critical",
|
|
102
99
|
location: `${fn.relativePath}:${fn.line}`,
|
|
103
100
|
functionName: fn.name,
|
|
104
101
|
message: `${fn.type} "${fn.name}" is missing handler property (may be using old syntax)`,
|
|
@@ -287,7 +284,7 @@ export const functionTools = [
|
|
|
287
284
|
const functions = extractFunctions(convexDir);
|
|
288
285
|
// Store audit result
|
|
289
286
|
const db = getDb();
|
|
290
|
-
db.prepare("INSERT INTO audit_results (id, project_dir, audit_type, issues_json, issue_count) VALUES (?, ?, ?, ?, ?)").run(genId("audit"), projectDir, "
|
|
287
|
+
db.prepare("INSERT INTO audit_results (id, project_dir, audit_type, issues_json, issue_count) VALUES (?, ?, ?, ?, ?)").run(genId("audit"), projectDir, "function_audit", JSON.stringify(issues), issues.length);
|
|
291
288
|
const critical = issues.filter((i) => i.severity === "critical");
|
|
292
289
|
const warnings = issues.filter((i) => i.severity === "warning");
|
|
293
290
|
// Aggregate issues by category for cleaner output
|
|
@@ -83,6 +83,19 @@ const METHODOLOGY_CONTENT = {
|
|
|
83
83
|
],
|
|
84
84
|
tools: ["convex_suggest_indexes", "convex_audit_schema"],
|
|
85
85
|
},
|
|
86
|
+
analyst_diagnostic: {
|
|
87
|
+
title: "Analyst Diagnostic — Root Cause Over Bandaids",
|
|
88
|
+
description: "Guide yourself like an analyst diagnosing the root cause, NOT a junior dev slapping on a bandaid. Mandatory for all bug work.",
|
|
89
|
+
steps: [
|
|
90
|
+
"1. REPRODUCE: Confirm the exact failure mode before touching any code",
|
|
91
|
+
"2. TRACE UPSTREAM: Walk from symptom → intermediate state → root cause. Don't stop at the first error you see",
|
|
92
|
+
"3. ASK 'WHY' 5 TIMES: Each answer should go one level deeper into the system",
|
|
93
|
+
"4. FIX THE CAUSE: The right fix makes the symptom impossible, not just invisible",
|
|
94
|
+
"5. VERIFY NO SIDEWAYS SHIFT: Bandaids move bugs, they don't fix them — check adjacent behavior",
|
|
95
|
+
"6. RECORD: Use convex_record_gotcha with the root cause so the next person doesn't re-discover it",
|
|
96
|
+
],
|
|
97
|
+
tools: ["convex_search_gotchas", "convex_record_gotcha", "convex_critter_check"],
|
|
98
|
+
},
|
|
86
99
|
};
|
|
87
100
|
// ── Tool Definitions ────────────────────────────────────────────────
|
|
88
101
|
export const methodologyTools = [
|
|
@@ -101,6 +114,7 @@ export const methodologyTools = [
|
|
|
101
114
|
"convex_deploy_verification",
|
|
102
115
|
"convex_knowledge_management",
|
|
103
116
|
"convex_index_optimization",
|
|
117
|
+
"analyst_diagnostic",
|
|
104
118
|
],
|
|
105
119
|
description: "Which methodology to explain",
|
|
106
120
|
},
|
|
@@ -1,34 +1,14 @@
|
|
|
1
1
|
import { resolve } from "node:path";
|
|
2
2
|
import { getDb, genId } from "../db.js";
|
|
3
3
|
import { getQuickRef } from "./toolRegistry.js";
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
maxUnboundedCollects: 20,
|
|
12
|
-
maxDanglingRefs: 10,
|
|
13
|
-
},
|
|
14
|
-
medium: {
|
|
15
|
-
maxCritical: 0,
|
|
16
|
-
maxWarnings: 200,
|
|
17
|
-
minAuthCoveragePercent: 20,
|
|
18
|
-
maxAsAnyCasts: 500,
|
|
19
|
-
maxUnboundedCollects: 100,
|
|
20
|
-
maxDanglingRefs: 30,
|
|
21
|
-
},
|
|
22
|
-
large: {
|
|
23
|
-
maxCritical: 0,
|
|
24
|
-
maxWarnings: 2000,
|
|
25
|
-
minAuthCoveragePercent: 10,
|
|
26
|
-
maxAsAnyCasts: 2000,
|
|
27
|
-
maxUnboundedCollects: 500,
|
|
28
|
-
maxDanglingRefs: 50,
|
|
29
|
-
},
|
|
4
|
+
const DEFAULT_THRESHOLDS = {
|
|
5
|
+
maxCritical: 0,
|
|
6
|
+
maxWarnings: 50,
|
|
7
|
+
minAuthCoveragePercent: 10,
|
|
8
|
+
maxAsAnyCasts: 500,
|
|
9
|
+
maxUnboundedCollects: 100,
|
|
10
|
+
maxDanglingRefs: 20,
|
|
30
11
|
};
|
|
31
|
-
const DEFAULT_THRESHOLDS = SCALE_THRESHOLDS.medium;
|
|
32
12
|
function runQualityGate(projectDir, thresholds) {
|
|
33
13
|
const db = getDb();
|
|
34
14
|
const checks = [];
|
|
@@ -97,18 +77,17 @@ function runQualityGate(projectDir, thresholds) {
|
|
|
97
77
|
}
|
|
98
78
|
catch { /* skip */ }
|
|
99
79
|
}
|
|
100
|
-
// Check 4: Type safety (as any casts)
|
|
80
|
+
// Check 4: Type safety (as any casts)
|
|
101
81
|
const typeSafety = getLatest("type_safety");
|
|
102
82
|
if (typeSafety) {
|
|
103
83
|
try {
|
|
104
84
|
const issues = JSON.parse(typeSafety.issues_json);
|
|
105
|
-
|
|
106
|
-
|
|
85
|
+
const asAnyIssues = Array.isArray(issues)
|
|
86
|
+
? issues.filter((i) => i.message?.includes("as any")).length
|
|
87
|
+
: 0;
|
|
107
88
|
// Each as-any issue represents a FILE, count from message for actual number
|
|
108
89
|
const actualCasts = Array.isArray(issues)
|
|
109
90
|
? issues.reduce((sum, i) => {
|
|
110
|
-
if (testEvalPattern.test(i.location ?? ""))
|
|
111
|
-
return sum; // skip test/eval files
|
|
112
91
|
const countMatch = i.message?.match(/(\d+)\s+`as any`/);
|
|
113
92
|
return sum + (countMatch ? parseInt(countMatch[1], 10) : 0);
|
|
114
93
|
}, 0)
|
|
@@ -123,13 +102,13 @@ function runQualityGate(projectDir, thresholds) {
|
|
|
123
102
|
}
|
|
124
103
|
catch { /* skip */ }
|
|
125
104
|
}
|
|
126
|
-
// Check 5: Unbounded collects
|
|
105
|
+
// Check 5: Unbounded collects
|
|
127
106
|
const queryEfficiency = getLatest("query_efficiency");
|
|
128
107
|
if (queryEfficiency) {
|
|
129
108
|
try {
|
|
130
109
|
const issues = JSON.parse(queryEfficiency.issues_json);
|
|
131
110
|
const unbounded = Array.isArray(issues)
|
|
132
|
-
? issues.filter((i) => i.
|
|
111
|
+
? issues.filter((i) => i.message?.includes(".collect()")).length
|
|
133
112
|
: 0;
|
|
134
113
|
checks.push({
|
|
135
114
|
metric: "unbounded_collects",
|
|
@@ -189,14 +168,9 @@ export const qualityGateTools = [
|
|
|
189
168
|
type: "string",
|
|
190
169
|
description: "Absolute path to the project root",
|
|
191
170
|
},
|
|
192
|
-
scale: {
|
|
193
|
-
type: "string",
|
|
194
|
-
enum: ["small", "medium", "large"],
|
|
195
|
-
description: "Project scale preset. small: <50 functions, tight thresholds. medium (default): 50-500 functions. large: 500+ functions, monorepo-scale thresholds (maxWarnings=2000, maxAsAny=2000, maxCollects=500).",
|
|
196
|
-
},
|
|
197
171
|
thresholds: {
|
|
198
172
|
type: "object",
|
|
199
|
-
description: "Custom thresholds
|
|
173
|
+
description: "Custom thresholds. Defaults: maxCritical=0, maxWarnings=50, minAuthCoveragePercent=10, maxAsAnyCasts=500, maxUnboundedCollects=100, maxDanglingRefs=20",
|
|
200
174
|
properties: {
|
|
201
175
|
maxCritical: { type: "number" },
|
|
202
176
|
maxWarnings: { type: "number" },
|
|
@@ -211,9 +185,8 @@ export const qualityGateTools = [
|
|
|
211
185
|
},
|
|
212
186
|
handler: async (args) => {
|
|
213
187
|
const projectDir = resolve(args.projectDir);
|
|
214
|
-
const scaleBase = SCALE_THRESHOLDS[args.scale ?? "medium"] ?? DEFAULT_THRESHOLDS;
|
|
215
188
|
const thresholds = {
|
|
216
|
-
...
|
|
189
|
+
...DEFAULT_THRESHOLDS,
|
|
217
190
|
...(args.thresholds ?? {}),
|
|
218
191
|
};
|
|
219
192
|
const result = runQualityGate(projectDir, thresholds);
|
|
@@ -222,7 +195,6 @@ export const qualityGateTools = [
|
|
|
222
195
|
db.prepare("INSERT INTO deploy_checks (id, project_dir, check_type, passed, findings) VALUES (?, ?, ?, ?, ?)").run(genId("deploy"), projectDir, "quality_gate", result.passed ? 1 : 0, JSON.stringify(result));
|
|
223
196
|
return {
|
|
224
197
|
...result,
|
|
225
|
-
scale: args.scale ?? "medium",
|
|
226
198
|
thresholdsUsed: thresholds,
|
|
227
199
|
quickRef: getQuickRef("convex_quality_gate"),
|
|
228
200
|
};
|
|
@@ -63,16 +63,13 @@ function auditQueryEfficiency(convexDir) {
|
|
|
63
63
|
const chain = lines.slice(chainStart, i + 1).join("\n");
|
|
64
64
|
if (!/\.take\s*\(/.test(chain) && !/\.paginate\s*\(/.test(chain)) {
|
|
65
65
|
collectWithoutLimit++;
|
|
66
|
+
// Check if it's a bounded table or could be large
|
|
66
67
|
const tableMatch = chain.match(/\.query\s*\(\s*["'](\w+)["']\s*\)/);
|
|
67
|
-
// Indexed queries are bounded by the index range — lower severity
|
|
68
|
-
const hasIndex = /\.withIndex\s*\(/.test(chain);
|
|
69
68
|
issues.push({
|
|
70
|
-
severity:
|
|
69
|
+
severity: "warning",
|
|
71
70
|
location: `${relativePath}:${i + 1}`,
|
|
72
71
|
message: `.collect() without .take() limit${tableMatch ? ` on table "${tableMatch[1]}"` : ""}. Could return entire table.`,
|
|
73
|
-
fix:
|
|
74
|
-
? "Consider adding .take(N) for predictable result sizes, even with index filtering"
|
|
75
|
-
: "Add .take(N) before .collect(), or use .paginate() for large result sets",
|
|
72
|
+
fix: "Add .take(N) before .collect(), or use .paginate() for large result sets",
|
|
76
73
|
});
|
|
77
74
|
}
|
|
78
75
|
}
|
|
@@ -75,7 +75,7 @@ function buildSarif(projectDir, auditTypes, limit) {
|
|
|
75
75
|
tool: {
|
|
76
76
|
driver: {
|
|
77
77
|
name: "convex-mcp-nodebench",
|
|
78
|
-
version: "0.9.
|
|
78
|
+
version: "0.9.4",
|
|
79
79
|
informationUri: "https://www.npmjs.com/package/@homenshum/convex-mcp-nodebench",
|
|
80
80
|
rules: [...rulesMap.values()],
|
|
81
81
|
},
|
|
@@ -133,7 +133,7 @@ export const storageAuditTools = [
|
|
|
133
133
|
}
|
|
134
134
|
const { issues, stats } = auditStorageUsage(convexDir);
|
|
135
135
|
const db = getDb();
|
|
136
|
-
db.prepare("INSERT INTO audit_results (id, project_dir, audit_type, issues_json, issue_count) VALUES (?, ?, ?, ?, ?)").run(genId("audit"), projectDir, "
|
|
136
|
+
db.prepare("INSERT INTO audit_results (id, project_dir, audit_type, issues_json, issue_count) VALUES (?, ?, ?, ?, ?)").run(genId("audit"), projectDir, "storage_usage", JSON.stringify(issues), issues.length);
|
|
137
137
|
return {
|
|
138
138
|
summary: {
|
|
139
139
|
...stats,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@homenshum/convex-mcp-nodebench",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.10.0",
|
|
4
4
|
"description": "Convex-specific MCP server applying NodeBench self-instruct diligence patterns to Convex development. Schema audit, function compliance, deployment gates, persistent gotcha DB, and methodology guidance. Complements Context7 (raw docs) and official Convex MCP (deployment introspection) with structured verification workflows.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -41,7 +41,6 @@
|
|
|
41
41
|
"author": "HomenShum",
|
|
42
42
|
"dependencies": {
|
|
43
43
|
"@modelcontextprotocol/sdk": "^1.0.4",
|
|
44
|
-
"@toon-format/toon": "^1.0.0",
|
|
45
44
|
"better-sqlite3": "^11.0.0"
|
|
46
45
|
},
|
|
47
46
|
"optionalDependencies": {
|