@doist/todoist-mcp 14.0.2 → 14.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +3 -3
- package/dist/main-http.js +2 -2
- package/dist/main.js +1 -1
- package/dist/{mcp-server-67PYKKIe.js → mcp-server-CyiS7tm_.js} +1 -1
- package/dist/{require-valid-todoist-token-D8WnAMKI.js → require-valid-todoist-token-CYTMaOLR.js} +1 -1
- package/package.json +2 -3
- package/scripts/bump-plugin-version.mjs +0 -24
- package/scripts/eval-instructions.ts +0 -541
- package/scripts/run-tool.ts +0 -160
- package/scripts/test-executable.cjs +0 -69
- package/scripts/validate-schemas.ts +0 -285
package/dist/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { t as s } from "./mcp-server-
|
|
2
|
-
import { F as r, a as d, b as i, c as n, d as c, e as l, f as m, h as p, i as f, j as T, k as j, l as u, m as g, n as k, o as v, p as P, q as S, r as h, s as b, u as y, v as A, w as C, x as R, y as x, z as F, g as M, A as O, B as E, C as L, D as U, E as w, G as H, H as I, I as W, J as q, K as z, L as B, M as D, N, O as V, P as _, Q as G, R as J, S as K, T as Q, U as X, V as Y, W as Z, X as $, Y as ss, Z as as, _ as es } from "./mcp-server-
|
|
3
|
-
import { r as os, v as rs } from "./require-valid-todoist-token-
|
|
1
|
+
import { t as s } from "./mcp-server-CyiS7tm_.js";
|
|
2
|
+
import { F as r, a as d, b as i, c as n, d as c, e as l, f as m, h as p, i as f, j as T, k as j, l as u, m as g, n as k, o as v, p as P, q as S, r as h, s as b, u as y, v as A, w as C, x as R, y as x, z as F, g as M, A as O, B as E, C as L, D as U, E as w, G as H, H as I, I as W, J as q, K as z, L as B, M as D, N, O as V, P as _, Q as G, R as J, S as K, T as Q, U as X, V as Y, W as Z, X as $, Y as ss, Z as as, _ as es } from "./mcp-server-CyiS7tm_.js";
|
|
3
|
+
import { r as os, v as rs } from "./require-valid-todoist-token-CYTMaOLR.js";
|
|
4
4
|
const e = s;
|
|
5
5
|
export {
|
|
6
6
|
r as FEATURE_NAMES,
|
package/dist/main-http.js
CHANGED
|
@@ -4,8 +4,8 @@ import S from "dotenv";
|
|
|
4
4
|
import { toNodeHandler as w } from "@modelcontextprotocol/node";
|
|
5
5
|
import { createMcpHandler as v } from "@modelcontextprotocol/server";
|
|
6
6
|
import h from "express";
|
|
7
|
-
import { g as I } from "./mcp-server-
|
|
8
|
-
import { r as g } from "./require-valid-todoist-token-
|
|
7
|
+
import { g as I } from "./mcp-server-CyiS7tm_.js";
|
|
8
|
+
import { r as g } from "./require-valid-todoist-token-CYTMaOLR.js";
|
|
9
9
|
function p(e) {
|
|
10
10
|
return e.startsWith("[") && e.endsWith("]") ? e.slice(1, -1) : e;
|
|
11
11
|
}
|
package/dist/main.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import { serveStdio as t } from "@modelcontextprotocol/server/stdio";
|
|
3
3
|
import n from "dotenv";
|
|
4
|
-
import { g as s } from "./mcp-server-
|
|
4
|
+
import { g as s } from "./mcp-server-CyiS7tm_.js";
|
|
5
5
|
function i() {
|
|
6
6
|
const o = process.env.TODOIST_BASE_URL, r = process.env.TODOIST_API_KEY;
|
|
7
7
|
if (!r)
|
package/dist/{require-valid-todoist-token-D8WnAMKI.js → require-valid-todoist-token-CYTMaOLR.js}
RENAMED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { validateBearerToken as _, createTokenVerdictCache as p, buildBearerChallenge as u } from "@doist/mcp-core/host";
|
|
2
|
-
import { $ as v, a0 as A } from "./mcp-server-
|
|
2
|
+
import { $ as v, a0 as A } from "./mcp-server-CyiS7tm_.js";
|
|
3
3
|
const k = "https://api.todoist.com", U = (e, t) => _({
|
|
4
4
|
// Resolved the way the SDK resolves its own requests against a custom base URL. An empty
|
|
5
5
|
// base (`TODOIST_BASE_URL=`) means the default, as it does for the SDK client.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@doist/todoist-mcp",
|
|
3
|
-
"version": "14.0.
|
|
3
|
+
"version": "14.0.4",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"main": "./dist/index.js",
|
|
6
6
|
"types": "./dist/index.d.ts",
|
|
@@ -12,7 +12,6 @@
|
|
|
12
12
|
"files": [
|
|
13
13
|
"dist",
|
|
14
14
|
"!dist/dev",
|
|
15
|
-
"scripts",
|
|
16
15
|
"package.json",
|
|
17
16
|
"LICENSE",
|
|
18
17
|
"README.md"
|
|
@@ -68,7 +67,7 @@
|
|
|
68
67
|
},
|
|
69
68
|
"dependencies": {
|
|
70
69
|
"@doist/mcp-core": "0.4.0",
|
|
71
|
-
"@doist/todoist-sdk": "
|
|
70
|
+
"@doist/todoist-sdk": "17.0.0",
|
|
72
71
|
"@modelcontextprotocol/ext-apps": "2.0.4",
|
|
73
72
|
"date-fns": "4.1.0",
|
|
74
73
|
"dompurify": "3.3.3",
|
|
@@ -1,24 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
2
|
-
import { readFileSync, writeFileSync } from 'node:fs'
|
|
3
|
-
|
|
4
|
-
const [version, manifestPath] = process.argv.slice(2)
|
|
5
|
-
|
|
6
|
-
if (!version) {
|
|
7
|
-
console.error('Usage: bump-plugin-version.mjs <version> [manifestPath]')
|
|
8
|
-
process.exit(1)
|
|
9
|
-
}
|
|
10
|
-
|
|
11
|
-
const paths = manifestPath
|
|
12
|
-
? [manifestPath]
|
|
13
|
-
: [
|
|
14
|
-
'.claude-plugin/plugin.json',
|
|
15
|
-
'plugins/todoist/.claude-plugin/plugin.json',
|
|
16
|
-
'gemini-extension.json',
|
|
17
|
-
]
|
|
18
|
-
|
|
19
|
-
for (const path of paths) {
|
|
20
|
-
const manifest = JSON.parse(readFileSync(path, 'utf8'))
|
|
21
|
-
manifest.version = version
|
|
22
|
-
writeFileSync(path, `${JSON.stringify(manifest, null, 4)}\n`)
|
|
23
|
-
console.log(`Updated ${path} to ${version}`)
|
|
24
|
-
}
|
|
@@ -1,541 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env npx tsx
|
|
2
|
-
/**
|
|
3
|
-
* Measure whether the server's tool surface leads a model to the right tool.
|
|
4
|
-
*
|
|
5
|
-
* The token-footprint test says what the surface costs; it cannot say whether
|
|
6
|
-
* the surface still works. This does: it sends the same tool definitions and
|
|
7
|
-
* instructions a real client sends, gives the model a prompt, and checks which
|
|
8
|
-
* tool it reaches for and how it fills the arguments.
|
|
9
|
-
*
|
|
10
|
-
* It reads the surface from the working tree, so the way to compare two
|
|
11
|
-
* versions is to run it on each and diff the pass rates:
|
|
12
|
-
*
|
|
13
|
-
* npx tsx scripts/eval-instructions.ts --label before
|
|
14
|
-
* git stash pop
|
|
15
|
-
* npx tsx scripts/eval-instructions.ts --label after
|
|
16
|
-
*
|
|
17
|
-
* Tool calls are never executed against Todoist, so this touches no Todoist
|
|
18
|
-
* data and needs no Todoist token. Judging looks at one call per attempt: the
|
|
19
|
-
* first, unless it is a context tool (see CONTEXT_TOOLS), in which case a
|
|
20
|
-
* canned result is fed back and the next call is judged. It does spend money
|
|
21
|
-
* on model calls; see --repeats and --models.
|
|
22
|
-
*
|
|
23
|
-
* Auth uses the Anthropic SDK's standard credential chain: either
|
|
24
|
-
* ANTHROPIC_API_KEY (a key from the Anthropic Console at platform.claude.com),
|
|
25
|
-
* or an OAuth profile created with `ant auth login` — `ant` being Anthropic's
|
|
26
|
-
* CLI, from https://github.com/anthropics/anthropic-cli. A set
|
|
27
|
-
* ANTHROPIC_API_KEY silently overrides a profile; `ant auth status` shows which
|
|
28
|
-
* one is in use. See AGENTS.md for install steps.
|
|
29
|
-
*
|
|
30
|
-
* Usage:
|
|
31
|
-
* npx tsx scripts/eval-instructions.ts [--label NAME] [--repeats N]
|
|
32
|
-
* [--models a,b] [--scenario ID]
|
|
33
|
-
*/
|
|
34
|
-
import { mkdirSync, writeFileSync } from 'node:fs'
|
|
35
|
-
import Anthropic from '@anthropic-ai/sdk'
|
|
36
|
-
import { z } from 'zod'
|
|
37
|
-
import { instructions } from '../src/mcp-server.js'
|
|
38
|
-
import { registeredTools } from '../src/tool-registry.js'
|
|
39
|
-
import type { UserInfoStructured } from '../src/tools/user-info.js'
|
|
40
|
-
import { createLimiter } from '../src/utils/concurrency.js'
|
|
41
|
-
import { ToolNames } from '../src/utils/tool-names.js'
|
|
42
|
-
|
|
43
|
-
/**
|
|
44
|
-
* The user ID the stubbed `user-info` reports. A scenario asking a first-person
|
|
45
|
-
* question can assert the model carried it into a filter argument rather than
|
|
46
|
-
* querying everyone.
|
|
47
|
-
*/
|
|
48
|
-
const EVAL_USER_ID = '2671355'
|
|
49
|
-
|
|
50
|
-
type Check = (input: Record<string, unknown>) => string | null
|
|
51
|
-
|
|
52
|
-
/** First item of a batch tool's array argument, e.g. add-tasks' `tasks`. */
|
|
53
|
-
function firstItem(input: Record<string, unknown>, key: string): Record<string, unknown> | null {
|
|
54
|
-
const list = input[key]
|
|
55
|
-
if (!Array.isArray(list) || list.length === 0) return null
|
|
56
|
-
const item = list[0]
|
|
57
|
-
return item && typeof item === 'object' ? (item as Record<string, unknown>) : null
|
|
58
|
-
}
|
|
59
|
-
|
|
60
|
-
type Scenario = {
|
|
61
|
-
id: string
|
|
62
|
-
prompt: string
|
|
63
|
-
/**
|
|
64
|
-
* Any of these counts as the right tool. Use for a rule that names the
|
|
65
|
-
* tool to reach for. Omit when the rule is "don't do X" — an allowlist
|
|
66
|
-
* then fails legitimate lookup steps it did not anticipate, which is
|
|
67
|
-
* noise, not signal.
|
|
68
|
-
*/
|
|
69
|
-
expect?: string[]
|
|
70
|
-
/** Calling any of these fails the scenario. Use for "don't do X" rules. */
|
|
71
|
-
forbid?: string[]
|
|
72
|
-
/** Extra assertion on the arguments; return a reason on failure. */
|
|
73
|
-
check?: Check
|
|
74
|
-
/** What this scenario is protecting. */
|
|
75
|
-
guards: string
|
|
76
|
-
}
|
|
77
|
-
|
|
78
|
-
const SCENARIOS: Scenario[] = [
|
|
79
|
-
{
|
|
80
|
-
// The ID is in the prompt on purpose. Without one the model has to
|
|
81
|
-
// search first, which is correct behaviour and tells us nothing about
|
|
82
|
-
// the rule under test.
|
|
83
|
-
id: 'reschedule-not-update',
|
|
84
|
-
prompt: 'Move task 6XG4Vw2c9J ("Weekly review", repeats every Monday) to next Tuesday.',
|
|
85
|
-
// The rule is "not update-tasks", so anything else -- including a
|
|
86
|
-
// preliminary lookup -- is acceptable.
|
|
87
|
-
forbid: [ToolNames.UPDATE_TASKS],
|
|
88
|
-
guards: 'instructions: reschedule-tasks vs update-tasks (recurrence loss)',
|
|
89
|
-
},
|
|
90
|
-
{
|
|
91
|
-
// Doist/Issues#21505: time-blocking needs a time and a duration, and
|
|
92
|
-
// update-tasks used to be the only tool that set both.
|
|
93
|
-
id: 'time-block-recurring',
|
|
94
|
-
prompt: 'Block 10:00-10:15 today for task 6XG4Vw2c9J ("Give meds", repeats every day at 9:00).',
|
|
95
|
-
forbid: [ToolNames.UPDATE_TASKS],
|
|
96
|
-
check: (input) => {
|
|
97
|
-
const task = firstItem(input, 'tasks')
|
|
98
|
-
// Only the reschedule call carries a date; a lookup is fine.
|
|
99
|
-
if (!task || !('date' in task)) return null
|
|
100
|
-
return task.duration ? null : `rescheduled without a duration: ${JSON.stringify(task)}`
|
|
101
|
-
},
|
|
102
|
-
guards: 'reschedule-tasks duration: time-blocking a recurring task',
|
|
103
|
-
},
|
|
104
|
-
{
|
|
105
|
-
id: 'completed-via-activity',
|
|
106
|
-
prompt: 'What did I actually get done last week?',
|
|
107
|
-
expect: [ToolNames.FIND_ACTIVITY],
|
|
108
|
-
check: (input) => {
|
|
109
|
-
// Exact match: "uncompleted" contains "completed".
|
|
110
|
-
if (input.eventType !== 'completed') {
|
|
111
|
-
return `eventType is ${JSON.stringify(input.eventType)}, not "completed"`
|
|
112
|
-
}
|
|
113
|
-
if (!input.dateFrom || !input.dateTo) {
|
|
114
|
-
return 'no dateFrom/dateTo, so this searches all history rather than last week'
|
|
115
|
-
}
|
|
116
|
-
// find-activity reports every user's events by default, so "what did
|
|
117
|
-
// I get done" answered without this includes collaborators' work.
|
|
118
|
-
if (input.initiatorId !== EVAL_USER_ID) {
|
|
119
|
-
return `initiatorId is ${JSON.stringify(input.initiatorId)}, so this reports every collaborator's completions, not "I"`
|
|
120
|
-
}
|
|
121
|
-
return null
|
|
122
|
-
},
|
|
123
|
-
guards: 'instructions: find-activity vs find-completed-tasks, over a bounded range, filtered to the asker',
|
|
124
|
-
},
|
|
125
|
-
{
|
|
126
|
-
id: 'resolve-person',
|
|
127
|
-
prompt: 'Who is Sarah?',
|
|
128
|
-
expect: [ToolNames.FIND_PROJECT_COLLABORATORS],
|
|
129
|
-
guards: 'instructions: resolving a name to a user ID',
|
|
130
|
-
},
|
|
131
|
-
{
|
|
132
|
-
id: 'subtask-check',
|
|
133
|
-
prompt: 'Does task 6XG4Vw2c9J have any subtasks?',
|
|
134
|
-
expect: [ToolNames.FETCH_OBJECT],
|
|
135
|
-
check: (input) =>
|
|
136
|
-
input.includeChildren === true ? null : 'includeChildren not set to true',
|
|
137
|
-
guards: 'instructions: fetch-object over a speculative find-tasks',
|
|
138
|
-
},
|
|
139
|
-
{
|
|
140
|
-
id: 'label-by-name',
|
|
141
|
-
prompt: 'Show me my tasks labelled urgent.',
|
|
142
|
-
expect: [ToolNames.FIND_TASKS, ToolNames.FIND_LABELS],
|
|
143
|
-
check: (input) => {
|
|
144
|
-
const labels = input.labels
|
|
145
|
-
const used = Array.isArray(labels) ? labels.map(String) : []
|
|
146
|
-
if (used.some((l) => l.toLowerCase() === 'urgent')) return null
|
|
147
|
-
// find-labels legitimately takes a search term rather than a labels array.
|
|
148
|
-
const search = String(input.searchTerm ?? input.name ?? '').toLowerCase()
|
|
149
|
-
return search.includes('urgent')
|
|
150
|
-
? null
|
|
151
|
-
: `label name not used: ${JSON.stringify(input)}`
|
|
152
|
-
},
|
|
153
|
-
guards: 'instructions: filter by label name, not ID',
|
|
154
|
-
},
|
|
155
|
-
{
|
|
156
|
-
id: 'archive-before-delete',
|
|
157
|
-
prompt: 'Delete workspace project 6XQ3Plan99 ("Q3 Planning").',
|
|
158
|
-
// The rule is "archive first", so the only wrong first move is the
|
|
159
|
-
// delete itself. Archiving and any lookup (find-projects, get-overview,
|
|
160
|
-
// fetch-object) are all legitimate openers.
|
|
161
|
-
forbid: [ToolNames.DELETE_OBJECT],
|
|
162
|
-
guards: 'instructions: workspace projects archive before delete',
|
|
163
|
-
},
|
|
164
|
-
{
|
|
165
|
-
id: 'priority-string',
|
|
166
|
-
prompt: 'Set task 6XG4Vw2c9J to high priority.',
|
|
167
|
-
expect: [ToolNames.UPDATE_TASKS],
|
|
168
|
-
check: (input) => {
|
|
169
|
-
const json = JSON.stringify(input)
|
|
170
|
-
if (/"priority"\s*:\s*"p[1-4]"/.test(json)) return null
|
|
171
|
-
if (/"priority"\s*:\s*\d/.test(json)) return `priority sent as an integer: ${json}`
|
|
172
|
-
return `no p1-p4 priority found: ${json}`
|
|
173
|
-
},
|
|
174
|
-
guards: 'input field description: priority is "p1".."p4", never an integer',
|
|
175
|
-
},
|
|
176
|
-
{
|
|
177
|
-
id: 'recurring-due-string',
|
|
178
|
-
prompt: 'Add a task "Water the plants" due every Monday.',
|
|
179
|
-
expect: [ToolNames.ADD_TASKS],
|
|
180
|
-
check: (input) => {
|
|
181
|
-
const task = firstItem(input, 'tasks')
|
|
182
|
-
if (!task) return `no tasks array: ${JSON.stringify(input)}`
|
|
183
|
-
const dueString = String(task.dueString ?? '')
|
|
184
|
-
// The recurrence has to be in dueString, not smuggled into content.
|
|
185
|
-
if (!dueString.toLowerCase().includes('monday')) {
|
|
186
|
-
return `recurrence not in dueString: ${JSON.stringify(task)}`
|
|
187
|
-
}
|
|
188
|
-
return /^\s*recurring/i.test(dueString)
|
|
189
|
-
? `dueString prefixed with "recurring": ${dueString}`
|
|
190
|
-
: null
|
|
191
|
-
},
|
|
192
|
-
guards: 'input field description: no "recurring" prefix on dueString',
|
|
193
|
-
},
|
|
194
|
-
{
|
|
195
|
-
id: 'today-includes-overdue',
|
|
196
|
-
prompt: 'Show me the tasks due today.',
|
|
197
|
-
expect: [ToolNames.FIND_TASKS_BY_DATE, ToolNames.GET_OVERVIEW],
|
|
198
|
-
check: (input) => {
|
|
199
|
-
// get-overview takes no date argument; only assert on the dated tool.
|
|
200
|
-
if (!('startDate' in input)) return null
|
|
201
|
-
// A concrete YYYY-MM-DD satisfies the schema but loses the overdue
|
|
202
|
-
// behaviour the keyword carries.
|
|
203
|
-
return input.startDate === 'today'
|
|
204
|
-
? null
|
|
205
|
-
: `startDate is ${JSON.stringify(input.startDate)}, not the 'today' keyword`
|
|
206
|
-
},
|
|
207
|
-
guards: "input field description: startDate 'today' and its overdue behaviour",
|
|
208
|
-
},
|
|
209
|
-
{
|
|
210
|
-
id: 'no-container-echo',
|
|
211
|
-
prompt: 'Rename task 6XG4Vw2c9J to "Draft the Q4 report".',
|
|
212
|
-
expect: [ToolNames.UPDATE_TASKS],
|
|
213
|
-
check: (input) => {
|
|
214
|
-
const json = JSON.stringify(input)
|
|
215
|
-
return /"(projectId|sectionId|parentId)"/.test(json)
|
|
216
|
-
? `echoed a container field, which is treated as a move: ${json}`
|
|
217
|
-
: null
|
|
218
|
-
},
|
|
219
|
-
guards: 'instructions: never echo projectId/sectionId/parentId back',
|
|
220
|
-
},
|
|
221
|
-
]
|
|
222
|
-
|
|
223
|
-
const DEFAULT_MODELS = ['claude-haiku-4-5', 'claude-sonnet-5']
|
|
224
|
-
const DEFAULT_REPEATS = 5
|
|
225
|
-
const MAX_CONCURRENCY = 4
|
|
226
|
-
|
|
227
|
-
/**
|
|
228
|
-
* Tools a model may legitimately call before the one a scenario is testing, to
|
|
229
|
-
* establish context it has no other way to get — today's date, the user's
|
|
230
|
-
* timezone. "What did I get done last week?" cannot be turned into a date range
|
|
231
|
-
* without them.
|
|
232
|
-
*
|
|
233
|
-
* Judging the very first call outright scores that correct behaviour as a
|
|
234
|
-
* failure: an `expect` allowlist naming the tool under test cannot also name
|
|
235
|
-
* every reasonable lookup that precedes it. So a call to one of these is
|
|
236
|
-
* answered with the canned result below and judging moves to the next call.
|
|
237
|
-
*
|
|
238
|
-
* The result is fiction, but its shape has to be real: a scenario may assert
|
|
239
|
-
* that the model carried a value through (see EVAL_USER_ID), and a model reads
|
|
240
|
-
* a malformed field the way it would read any other bad data. `satisfies` ties
|
|
241
|
-
* it to the tool's own output type, so a renamed or added field breaks here
|
|
242
|
-
* rather than drifting quietly. Formats must match too — `currentLocalTime`
|
|
243
|
-
* is what `toLocaleString('en-US', …)` produces, not ISO.
|
|
244
|
-
*/
|
|
245
|
-
const CONTEXT_TOOLS: Record<string, unknown> = {
|
|
246
|
-
[ToolNames.USER_INFO]: {
|
|
247
|
-
type: 'user_info',
|
|
248
|
-
userId: EVAL_USER_ID,
|
|
249
|
-
fullName: 'Eval User',
|
|
250
|
-
timezone: 'Europe/London',
|
|
251
|
-
currentLocalTime: '08/11/2026, 09:30:00',
|
|
252
|
-
startDay: 1,
|
|
253
|
-
startDayName: 'Monday',
|
|
254
|
-
weekStartDate: '2026-08-10',
|
|
255
|
-
weekEndDate: '2026-08-16',
|
|
256
|
-
currentWeekNumber: 33,
|
|
257
|
-
completedToday: 3,
|
|
258
|
-
dailyGoal: 5,
|
|
259
|
-
weeklyGoal: 30,
|
|
260
|
-
email: 'eval@example.com',
|
|
261
|
-
plan: 'Todoist Pro',
|
|
262
|
-
} satisfies UserInfoStructured,
|
|
263
|
-
}
|
|
264
|
-
|
|
265
|
-
/**
|
|
266
|
-
* How many context calls an attempt may make before it is judged a failure.
|
|
267
|
-
* A model that keeps gathering context is not answering the question, and
|
|
268
|
-
* without a cap a loop would bill for turns forever.
|
|
269
|
-
*/
|
|
270
|
-
const MAX_CONTEXT_HOPS = 3
|
|
271
|
-
|
|
272
|
-
function parseArgs() {
|
|
273
|
-
const args = process.argv.slice(2)
|
|
274
|
-
const get = (flag: string) => {
|
|
275
|
-
const i = args.indexOf(flag)
|
|
276
|
-
return i === -1 ? undefined : args[i + 1]
|
|
277
|
-
}
|
|
278
|
-
const label = get('--label') ?? 'run'
|
|
279
|
-
// The label becomes a filename, so keep it to characters that cannot escape
|
|
280
|
-
// the output directory.
|
|
281
|
-
if (!/^[A-Za-z0-9_-]+$/.test(label)) {
|
|
282
|
-
console.error(`--label must match [A-Za-z0-9_-]+, got "${label}"`)
|
|
283
|
-
process.exit(1)
|
|
284
|
-
}
|
|
285
|
-
return {
|
|
286
|
-
label,
|
|
287
|
-
repeats: Number(get('--repeats') ?? DEFAULT_REPEATS),
|
|
288
|
-
models: (get('--models') ?? DEFAULT_MODELS.join(','))
|
|
289
|
-
.split(',')
|
|
290
|
-
.map((m) => m.trim())
|
|
291
|
-
.filter(Boolean),
|
|
292
|
-
scenario: get('--scenario'),
|
|
293
|
-
}
|
|
294
|
-
}
|
|
295
|
-
|
|
296
|
-
/**
|
|
297
|
-
* The tool definitions as an MCP client forwards them to the Messages API.
|
|
298
|
-
*
|
|
299
|
-
* Note the Messages API tool shape has no output-schema field, so a tool's
|
|
300
|
-
* `outputSchema` never reaches the model here however the server advertises it.
|
|
301
|
-
*/
|
|
302
|
-
function buildTools(): Anthropic.Tool[] {
|
|
303
|
-
return registeredTools.map((tool) => ({
|
|
304
|
-
name: tool.name,
|
|
305
|
-
description: tool.description,
|
|
306
|
-
// `InputSchema` carries an index signature, so the extra JSON Schema
|
|
307
|
-
// keys Zod emits pass through rather than being rejected by the cast.
|
|
308
|
-
input_schema: z.toJSONSchema(z.object(tool.parameters), {
|
|
309
|
-
unrepresentable: 'any',
|
|
310
|
-
io: 'input',
|
|
311
|
-
target: 'draft-7',
|
|
312
|
-
}) as Anthropic.Tool.InputSchema,
|
|
313
|
-
}))
|
|
314
|
-
}
|
|
315
|
-
|
|
316
|
-
type Usage = { input: number; output: number; cacheWrite: number; cacheRead: number }
|
|
317
|
-
|
|
318
|
-
type Attempt = {
|
|
319
|
-
scenario: string
|
|
320
|
-
model: string
|
|
321
|
-
calledTool: string | null
|
|
322
|
-
pass: boolean
|
|
323
|
-
reason: string | null
|
|
324
|
-
/**
|
|
325
|
-
* The request itself failed (auth, rate limit, transport). Distinct from a
|
|
326
|
-
* model that answered but chose wrong — counting these as routing failures
|
|
327
|
-
* would let a run with bad credentials report 0% and be saved as a result.
|
|
328
|
-
*/
|
|
329
|
-
errored: boolean
|
|
330
|
-
usage: Usage
|
|
331
|
-
/** Context calls answered with a stub before the judged one. */
|
|
332
|
-
contextHops: number
|
|
333
|
-
}
|
|
334
|
-
|
|
335
|
-
/** Verdict on the one call a scenario is judged by. */
|
|
336
|
-
function judge(
|
|
337
|
-
scenario: Scenario,
|
|
338
|
-
call: Anthropic.ToolUseBlock,
|
|
339
|
-
): { pass: boolean; reason: string } {
|
|
340
|
-
if (scenario.forbid?.includes(call.name)) {
|
|
341
|
-
return { pass: false, reason: `called ${call.name}, which this rule forbids` }
|
|
342
|
-
}
|
|
343
|
-
if (scenario.expect && !scenario.expect.includes(call.name)) {
|
|
344
|
-
return { pass: false, reason: `expected ${scenario.expect.join(' or ')}` }
|
|
345
|
-
}
|
|
346
|
-
// An argument check written for a specific tool must not run against a
|
|
347
|
-
// different one a forbid-only scenario legitimately allows.
|
|
348
|
-
if (scenario.forbid && !scenario.expect) {
|
|
349
|
-
return { pass: true, reason: '' }
|
|
350
|
-
}
|
|
351
|
-
const reason = scenario.check?.(call.input as Record<string, unknown>) ?? null
|
|
352
|
-
return { pass: reason === null, reason: reason ?? '' }
|
|
353
|
-
}
|
|
354
|
-
|
|
355
|
-
async function runAttempt(
|
|
356
|
-
client: Anthropic,
|
|
357
|
-
model: string,
|
|
358
|
-
scenario: Scenario,
|
|
359
|
-
tools: Anthropic.Tool[],
|
|
360
|
-
): Promise<Attempt> {
|
|
361
|
-
const base = {
|
|
362
|
-
scenario: scenario.id,
|
|
363
|
-
model,
|
|
364
|
-
errored: false,
|
|
365
|
-
usage: { input: 0, output: 0, cacheWrite: 0, cacheRead: 0 },
|
|
366
|
-
contextHops: 0,
|
|
367
|
-
}
|
|
368
|
-
const messages: Anthropic.MessageParam[] = [{ role: 'user', content: scenario.prompt }]
|
|
369
|
-
try {
|
|
370
|
-
for (let hop = 0; ; hop++) {
|
|
371
|
-
const response = await client.messages.create({
|
|
372
|
-
model,
|
|
373
|
-
max_tokens: 4096,
|
|
374
|
-
// Tools render before system, so one breakpoint here caches both.
|
|
375
|
-
system: [
|
|
376
|
-
{ type: 'text', text: instructions, cache_control: { type: 'ephemeral' } },
|
|
377
|
-
],
|
|
378
|
-
tools,
|
|
379
|
-
messages,
|
|
380
|
-
})
|
|
381
|
-
|
|
382
|
-
base.usage = {
|
|
383
|
-
input: base.usage.input + response.usage.input_tokens,
|
|
384
|
-
output: base.usage.output + response.usage.output_tokens,
|
|
385
|
-
cacheWrite:
|
|
386
|
-
base.usage.cacheWrite + (response.usage.cache_creation_input_tokens ?? 0),
|
|
387
|
-
cacheRead: base.usage.cacheRead + (response.usage.cache_read_input_tokens ?? 0),
|
|
388
|
-
}
|
|
389
|
-
|
|
390
|
-
const calls = response.content.filter((b) => b.type === 'tool_use')
|
|
391
|
-
const call = calls[0]
|
|
392
|
-
if (!call) {
|
|
393
|
-
return { ...base, calledTool: null, pass: false, reason: 'no tool call' }
|
|
394
|
-
}
|
|
395
|
-
|
|
396
|
-
// A turn can carry several calls at once. Judge the substantive one
|
|
397
|
-
// rather than whichever block came first: a model that asks for the
|
|
398
|
-
// date and queries the activity log in the same turn has made its
|
|
399
|
-
// choice, and the order between the two is arbitrary. A forbidden
|
|
400
|
-
// call outranks that — catching it is the point of a forbid rule.
|
|
401
|
-
//
|
|
402
|
-
// A tool a scenario expects is never treated as context, or a
|
|
403
|
-
// scenario testing the route *to* user-info could never pass: its
|
|
404
|
-
// expected call would be stubbed instead of judged.
|
|
405
|
-
const isContext = (c: Anthropic.ToolUseBlock) =>
|
|
406
|
-
Object.hasOwn(CONTEXT_TOOLS, c.name) && !scenario.expect?.includes(c.name)
|
|
407
|
-
const substantive =
|
|
408
|
-
calls.find((c) => scenario.forbid?.includes(c.name)) ??
|
|
409
|
-
calls.find((c) => !isContext(c))
|
|
410
|
-
if (substantive) {
|
|
411
|
-
const { pass, reason } = judge(scenario, substantive)
|
|
412
|
-
return { ...base, calledTool: substantive.name, pass, reason: reason || null }
|
|
413
|
-
}
|
|
414
|
-
|
|
415
|
-
// Nothing but context gathering, so answer it and judge what the
|
|
416
|
-
// model reaches for next.
|
|
417
|
-
if (hop >= MAX_CONTEXT_HOPS) {
|
|
418
|
-
return {
|
|
419
|
-
...base,
|
|
420
|
-
calledTool: call.name,
|
|
421
|
-
pass: false,
|
|
422
|
-
reason: `still gathering context after ${MAX_CONTEXT_HOPS} calls`,
|
|
423
|
-
}
|
|
424
|
-
}
|
|
425
|
-
|
|
426
|
-
base.contextHops = hop + 1
|
|
427
|
-
messages.push({ role: 'assistant', content: response.content })
|
|
428
|
-
messages.push({
|
|
429
|
-
role: 'user',
|
|
430
|
-
content: calls.map((c) => ({
|
|
431
|
-
type: 'tool_result' as const,
|
|
432
|
-
tool_use_id: c.id,
|
|
433
|
-
content: JSON.stringify(CONTEXT_TOOLS[c.name]),
|
|
434
|
-
})),
|
|
435
|
-
})
|
|
436
|
-
}
|
|
437
|
-
} catch (error) {
|
|
438
|
-
const message = error instanceof Error ? error.message : String(error)
|
|
439
|
-
return {
|
|
440
|
-
...base,
|
|
441
|
-
calledTool: null,
|
|
442
|
-
pass: false,
|
|
443
|
-
errored: true,
|
|
444
|
-
reason: `request failed: ${message}`,
|
|
445
|
-
}
|
|
446
|
-
}
|
|
447
|
-
}
|
|
448
|
-
|
|
449
|
-
async function main() {
|
|
450
|
-
const { label, repeats, models, scenario: only } = parseArgs()
|
|
451
|
-
const scenarios = only ? SCENARIOS.filter((s) => s.id === only) : SCENARIOS
|
|
452
|
-
if (scenarios.length === 0) {
|
|
453
|
-
console.error(`No scenario matching "${only}"`)
|
|
454
|
-
process.exit(1)
|
|
455
|
-
}
|
|
456
|
-
|
|
457
|
-
const client = new Anthropic()
|
|
458
|
-
const tools = buildTools()
|
|
459
|
-
// queueTimeoutMs: 0 disables the default queue deadline, which exists for
|
|
460
|
-
// request-scoped work and does not apply to a batch script.
|
|
461
|
-
const limit = createLimiter(MAX_CONCURRENCY, { queueTimeoutMs: 0 })
|
|
462
|
-
|
|
463
|
-
console.log(`label: ${label}`)
|
|
464
|
-
console.log(`tools: ${tools.length}`)
|
|
465
|
-
console.log(`scenarios: ${scenarios.length} x ${repeats} repeats x ${models.length} models`)
|
|
466
|
-
console.log(`models: ${models.join(', ')}\n`)
|
|
467
|
-
|
|
468
|
-
const attempts: Attempt[] = []
|
|
469
|
-
for (const model of models) {
|
|
470
|
-
const jobs = scenarios.flatMap((s) => Array.from({ length: repeats }, () => s))
|
|
471
|
-
// Run one first so it writes the shared prefix to cache; the rest read it.
|
|
472
|
-
const first = jobs[0]
|
|
473
|
-
if (!first) continue
|
|
474
|
-
attempts.push(await runAttempt(client, model, first, tools))
|
|
475
|
-
attempts.push(
|
|
476
|
-
...(await Promise.all(
|
|
477
|
-
jobs.slice(1).map((s) => limit(() => runAttempt(client, model, s, tools))),
|
|
478
|
-
)),
|
|
479
|
-
)
|
|
480
|
-
console.log(`${model}: done`)
|
|
481
|
-
}
|
|
482
|
-
|
|
483
|
-
console.log(`\n=== ${label} ===`)
|
|
484
|
-
for (const model of models) {
|
|
485
|
-
console.log(`\n${model}`)
|
|
486
|
-
for (const s of scenarios) {
|
|
487
|
-
const all = attempts.filter((a) => a.model === model && a.scenario === s.id)
|
|
488
|
-
const rows = all.filter((a) => !a.errored)
|
|
489
|
-
const passed = rows.filter((a) => a.pass).length
|
|
490
|
-
const rate = rows.length ? Math.round((passed / rows.length) * 100) : 0
|
|
491
|
-
const mark = rate === 100 ? '✓' : rate >= 60 ? '~' : '✗'
|
|
492
|
-
console.log(` ${mark} ${String(rate).padStart(3)}% ${s.id}`)
|
|
493
|
-
const failure = rows.find((a) => !a.pass)
|
|
494
|
-
if (failure) {
|
|
495
|
-
console.log(
|
|
496
|
-
` ${failure.calledTool ?? 'no call'} — ${failure.reason ?? ''}`,
|
|
497
|
-
)
|
|
498
|
-
}
|
|
499
|
-
}
|
|
500
|
-
const rows = attempts.filter((a) => a.model === model && !a.errored)
|
|
501
|
-
const passed = rows.filter((a) => a.pass).length
|
|
502
|
-
console.log(` overall: ${passed}/${rows.length}`)
|
|
503
|
-
}
|
|
504
|
-
|
|
505
|
-
const errored = attempts.filter((a) => a.errored)
|
|
506
|
-
if (errored.length > 0) {
|
|
507
|
-
console.error(
|
|
508
|
-
`\n${errored.length}/${attempts.length} requests failed outright ` +
|
|
509
|
-
`(not counted as routing failures). First: ${errored[0]?.reason}`,
|
|
510
|
-
)
|
|
511
|
-
}
|
|
512
|
-
|
|
513
|
-
const total = attempts.reduce(
|
|
514
|
-
(acc, a) => ({
|
|
515
|
-
input: acc.input + a.usage.input,
|
|
516
|
-
output: acc.output + a.usage.output,
|
|
517
|
-
cacheWrite: acc.cacheWrite + a.usage.cacheWrite,
|
|
518
|
-
cacheRead: acc.cacheRead + a.usage.cacheRead,
|
|
519
|
-
}),
|
|
520
|
-
{ input: 0, output: 0, cacheWrite: 0, cacheRead: 0 },
|
|
521
|
-
)
|
|
522
|
-
console.log(
|
|
523
|
-
`\ntokens: ${total.input} uncached in, ${total.cacheWrite} cache write, ` +
|
|
524
|
-
`${total.cacheRead} cache read, ${total.output} out`,
|
|
525
|
-
)
|
|
526
|
-
|
|
527
|
-
mkdirSync('tmp/eval', { recursive: true })
|
|
528
|
-
const out = `tmp/eval/${label}.json`
|
|
529
|
-
writeFileSync(out, JSON.stringify({ label, models, repeats, attempts }, null, 2))
|
|
530
|
-
console.log(`\nwrote ${out}`)
|
|
531
|
-
|
|
532
|
-
// A run that could not obtain its samples is not a result to compare against.
|
|
533
|
-
if (errored.length > 0) {
|
|
534
|
-
process.exitCode = 1
|
|
535
|
-
}
|
|
536
|
-
}
|
|
537
|
-
|
|
538
|
-
main().catch((error) => {
|
|
539
|
-
console.error(error)
|
|
540
|
-
process.exit(1)
|
|
541
|
-
})
|
package/scripts/run-tool.ts
DELETED
|
@@ -1,160 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env npx tsx
|
|
2
|
-
/**
|
|
3
|
-
* Run any Todoist tool directly without going through MCP.
|
|
4
|
-
*
|
|
5
|
-
* Usage:
|
|
6
|
-
* npx tsx scripts/run-tool.ts <tool-name> '<json-args>'
|
|
7
|
-
* npx tsx scripts/run-tool.ts <tool-name> --file <args.json>
|
|
8
|
-
* npx tsx scripts/run-tool.ts --list
|
|
9
|
-
*
|
|
10
|
-
* Examples:
|
|
11
|
-
* npx tsx scripts/run-tool.ts add-tasks '{"tasks":[{"content":"Test task","order":1}]}'
|
|
12
|
-
* npx tsx scripts/run-tool.ts find-tasks '{"searchText":"meeting"}'
|
|
13
|
-
* npx tsx scripts/run-tool.ts get-overview '{}'
|
|
14
|
-
*
|
|
15
|
-
* Requires TODOIST_API_KEY in .env file (and optionally TODOIST_BASE_URL).
|
|
16
|
-
*/
|
|
17
|
-
import { readFileSync } from 'node:fs'
|
|
18
|
-
import { TodoistApi } from '@doist/todoist-sdk'
|
|
19
|
-
import { config } from 'dotenv'
|
|
20
|
-
import { registeredTools } from '../src/tool-registry.js'
|
|
21
|
-
import { createTodoistClient, runWithUsageTrackingContext } from '../src/usage-tracking.js'
|
|
22
|
-
|
|
23
|
-
config()
|
|
24
|
-
|
|
25
|
-
// Define a minimal type for tool execution that works with any tool
|
|
26
|
-
type ExecutableTool = {
|
|
27
|
-
name: string
|
|
28
|
-
description: string
|
|
29
|
-
execute: (
|
|
30
|
-
// oxlint-disable-next-line @typescript-eslint/no-explicit-any -- tools have varying parameter schemas
|
|
31
|
-
args: any,
|
|
32
|
-
client: TodoistApi,
|
|
33
|
-
) => Promise<{ textContent?: string; structuredContent?: unknown; contentItems?: unknown[] }>
|
|
34
|
-
}
|
|
35
|
-
|
|
36
|
-
const tools: Record<string, ExecutableTool> = Object.fromEntries(
|
|
37
|
-
registeredTools.map((tool) => [tool.name, tool as ExecutableTool]),
|
|
38
|
-
)
|
|
39
|
-
|
|
40
|
-
function printUsage() {
|
|
41
|
-
console.log(`
|
|
42
|
-
Usage:
|
|
43
|
-
npx tsx scripts/run-tool.ts <tool-name> '<json-args>'
|
|
44
|
-
npx tsx scripts/run-tool.ts <tool-name> --file <args.json>
|
|
45
|
-
npx tsx scripts/run-tool.ts --list
|
|
46
|
-
|
|
47
|
-
Available tools:
|
|
48
|
-
${Object.keys(tools)
|
|
49
|
-
.sort()
|
|
50
|
-
.map((name) => ` - ${name}`)
|
|
51
|
-
.join('\n')}
|
|
52
|
-
`)
|
|
53
|
-
}
|
|
54
|
-
|
|
55
|
-
async function main() {
|
|
56
|
-
const args = process.argv.slice(2)
|
|
57
|
-
|
|
58
|
-
if (args.length === 0 || args[0] === '--help' || args[0] === '-h') {
|
|
59
|
-
printUsage()
|
|
60
|
-
process.exit(0)
|
|
61
|
-
}
|
|
62
|
-
|
|
63
|
-
if (args[0] === '--list') {
|
|
64
|
-
console.log('Available tools:')
|
|
65
|
-
for (const name of Object.keys(tools).sort()) {
|
|
66
|
-
const tool = tools[name]
|
|
67
|
-
console.log(`\n${name}:`)
|
|
68
|
-
console.log(` ${tool.description}`)
|
|
69
|
-
}
|
|
70
|
-
process.exit(0)
|
|
71
|
-
}
|
|
72
|
-
|
|
73
|
-
const toolName = args[0]
|
|
74
|
-
const tool = tools[toolName]
|
|
75
|
-
|
|
76
|
-
if (!tool) {
|
|
77
|
-
console.error(`Unknown tool: ${toolName}`)
|
|
78
|
-
console.error(`Available tools: ${Object.keys(tools).sort().join(', ')}`)
|
|
79
|
-
process.exit(1)
|
|
80
|
-
}
|
|
81
|
-
|
|
82
|
-
let jsonArgs: string
|
|
83
|
-
if (args[1] === '--file') {
|
|
84
|
-
if (!args[2]) {
|
|
85
|
-
console.error('--file requires a path argument')
|
|
86
|
-
process.exit(1)
|
|
87
|
-
}
|
|
88
|
-
jsonArgs = readFileSync(args[2], 'utf-8')
|
|
89
|
-
} else {
|
|
90
|
-
jsonArgs = args[1] || '{}'
|
|
91
|
-
}
|
|
92
|
-
|
|
93
|
-
let parsedArgs: unknown
|
|
94
|
-
try {
|
|
95
|
-
parsedArgs = JSON.parse(jsonArgs)
|
|
96
|
-
} catch (e) {
|
|
97
|
-
console.error('Invalid JSON args:', e)
|
|
98
|
-
process.exit(1)
|
|
99
|
-
}
|
|
100
|
-
|
|
101
|
-
const apiKey = process.env.TODOIST_API_KEY
|
|
102
|
-
if (!apiKey) {
|
|
103
|
-
console.error('TODOIST_API_KEY not found in environment or .env file')
|
|
104
|
-
process.exit(1)
|
|
105
|
-
}
|
|
106
|
-
|
|
107
|
-
const baseUrl = process.env.TODOIST_BASE_URL
|
|
108
|
-
const client = createTodoistClient(apiKey, {
|
|
109
|
-
baseUrl,
|
|
110
|
-
// Local direct runs are a dev helper, not real MCP traffic.
|
|
111
|
-
tracking: { enabled: false },
|
|
112
|
-
})
|
|
113
|
-
|
|
114
|
-
console.log(`Running ${toolName} with args:`)
|
|
115
|
-
console.log(JSON.stringify(parsedArgs, null, 2))
|
|
116
|
-
console.log('---')
|
|
117
|
-
|
|
118
|
-
try {
|
|
119
|
-
const result = await runWithUsageTrackingContext(tool.name, () =>
|
|
120
|
-
tool.execute(parsedArgs, client),
|
|
121
|
-
)
|
|
122
|
-
|
|
123
|
-
if (result.textContent) {
|
|
124
|
-
console.log('\nText output:')
|
|
125
|
-
console.log(result.textContent)
|
|
126
|
-
}
|
|
127
|
-
|
|
128
|
-
if (result.structuredContent) {
|
|
129
|
-
console.log('\nStructured output:')
|
|
130
|
-
console.log(JSON.stringify(result.structuredContent, null, 2))
|
|
131
|
-
}
|
|
132
|
-
|
|
133
|
-
if (result.contentItems?.length) {
|
|
134
|
-
console.log(`\nContent items: ${result.contentItems.length}`)
|
|
135
|
-
for (const item of result.contentItems) {
|
|
136
|
-
const entry = item as Record<string, unknown>
|
|
137
|
-
if (entry.type === 'image') {
|
|
138
|
-
const data = entry.data as string
|
|
139
|
-
console.log(
|
|
140
|
-
` [image] ${entry.mimeType} (${Math.round((data.length * 0.75) / 1024)}KB base64)`,
|
|
141
|
-
)
|
|
142
|
-
} else if (entry.type === 'text') {
|
|
143
|
-
const text = entry.text as string
|
|
144
|
-
console.log(` [text] ${text.length > 200 ? `${text.slice(0, 200)}...` : text}`)
|
|
145
|
-
} else if (entry.type === 'resource') {
|
|
146
|
-
const resource = entry.resource as Record<string, unknown>
|
|
147
|
-
const blob = resource.blob as string
|
|
148
|
-
console.log(
|
|
149
|
-
` [resource] ${resource.mimeType} (${Math.round((blob.length * 0.75) / 1024)}KB base64)`,
|
|
150
|
-
)
|
|
151
|
-
}
|
|
152
|
-
}
|
|
153
|
-
}
|
|
154
|
-
} catch (error) {
|
|
155
|
-
console.error('Tool execution failed:', error)
|
|
156
|
-
process.exit(1)
|
|
157
|
-
}
|
|
158
|
-
}
|
|
159
|
-
|
|
160
|
-
main()
|
|
@@ -1,69 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
2
|
-
|
|
3
|
-
const { spawn } = require('node:child_process')
|
|
4
|
-
const path = require('node:path')
|
|
5
|
-
|
|
6
|
-
console.log('Testing MCP server executable...')
|
|
7
|
-
|
|
8
|
-
const mainJs = path.join(__dirname, '..', 'dist', 'main.js')
|
|
9
|
-
const child = spawn('node', [mainJs], {
|
|
10
|
-
stdio: ['pipe', 'pipe', 'pipe'],
|
|
11
|
-
})
|
|
12
|
-
|
|
13
|
-
let _stdoutOutput = ''
|
|
14
|
-
let stderrOutput = ''
|
|
15
|
-
let hasError = false
|
|
16
|
-
|
|
17
|
-
child.stdout.on('data', (data) => {
|
|
18
|
-
_stdoutOutput += data.toString()
|
|
19
|
-
})
|
|
20
|
-
|
|
21
|
-
child.stderr.on('data', (data) => {
|
|
22
|
-
const output = data.toString()
|
|
23
|
-
stderrOutput += output
|
|
24
|
-
|
|
25
|
-
// Only consider it an error if it's not related to graceful shutdown
|
|
26
|
-
if (output.includes('Error:') && !output.includes('SIGTERM') && !output.includes('SIGKILL')) {
|
|
27
|
-
console.error('Server startup error detected:', output)
|
|
28
|
-
hasError = true
|
|
29
|
-
}
|
|
30
|
-
})
|
|
31
|
-
|
|
32
|
-
child.on('error', (error) => {
|
|
33
|
-
console.error('Failed to start MCP server:', error.message)
|
|
34
|
-
hasError = true
|
|
35
|
-
process.exit(1)
|
|
36
|
-
})
|
|
37
|
-
|
|
38
|
-
child.on('exit', (code, signal) => {
|
|
39
|
-
// Expected signals when we kill the process
|
|
40
|
-
if (signal === 'SIGTERM' || signal === 'SIGKILL') {
|
|
41
|
-
return // This is expected
|
|
42
|
-
}
|
|
43
|
-
|
|
44
|
-
// Unexpected exit codes during startup
|
|
45
|
-
if (code !== null && code !== 0) {
|
|
46
|
-
console.error(`Server exited unexpectedly with code ${code}`)
|
|
47
|
-
hasError = true
|
|
48
|
-
}
|
|
49
|
-
})
|
|
50
|
-
|
|
51
|
-
// Kill the process after 2 seconds (MCP server should start successfully)
|
|
52
|
-
setTimeout(() => {
|
|
53
|
-
if (hasError) {
|
|
54
|
-
console.error('❌ MCP server failed to start properly')
|
|
55
|
-
if (stderrOutput.trim()) {
|
|
56
|
-
console.error('Error output:', stderrOutput.trim())
|
|
57
|
-
}
|
|
58
|
-
process.exit(1)
|
|
59
|
-
}
|
|
60
|
-
|
|
61
|
-
// Gracefully terminate
|
|
62
|
-
child.kill('SIGTERM')
|
|
63
|
-
|
|
64
|
-
setTimeout(() => {
|
|
65
|
-
console.log('✅ MCP server executable test passed')
|
|
66
|
-
console.log('Server started successfully and is ready to accept connections')
|
|
67
|
-
process.exit(0)
|
|
68
|
-
}, 200) // Give it a moment to clean up
|
|
69
|
-
}, 2000)
|
|
@@ -1,285 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
2
|
-
|
|
3
|
-
/**
|
|
4
|
-
* Schema Validation Script for Todoist MCP Server
|
|
5
|
-
*
|
|
6
|
-
* This script validates that all tool parameter schemas follow Gemini API compatibility rules.
|
|
7
|
-
* Specifically, it checks that no Zod string schemas use both .nullable() and .optional().
|
|
8
|
-
*
|
|
9
|
-
* This version imports the actual tools from the compiled index.js and validates their
|
|
10
|
-
* runtime Zod schemas for maximum accuracy.
|
|
11
|
-
*
|
|
12
|
-
* Usage:
|
|
13
|
-
* npm run build && node scripts/validate-schemas.js
|
|
14
|
-
* npm run build && node scripts/validate-schemas.js --verbose
|
|
15
|
-
* npm run build && node scripts/validate-schemas.js --json
|
|
16
|
-
*/
|
|
17
|
-
|
|
18
|
-
import { z } from 'zod'
|
|
19
|
-
|
|
20
|
-
type ValidationIssue = {
|
|
21
|
-
toolName: string
|
|
22
|
-
parameterPath: string
|
|
23
|
-
issue: string
|
|
24
|
-
suggestion: string
|
|
25
|
-
}
|
|
26
|
-
|
|
27
|
-
type ValidationResult = {
|
|
28
|
-
success: boolean
|
|
29
|
-
issues: ValidationIssue[]
|
|
30
|
-
toolsChecked: number
|
|
31
|
-
parametersChecked: number
|
|
32
|
-
}
|
|
33
|
-
|
|
34
|
-
type AnyZodSchema = z.ZodTypeAny | { _zod: { def: unknown } }
|
|
35
|
-
|
|
36
|
-
/**
|
|
37
|
-
* Recursively walk a Zod schema and detect problematic patterns
|
|
38
|
-
*/
|
|
39
|
-
function walkZodSchema(
|
|
40
|
-
schema: AnyZodSchema,
|
|
41
|
-
path: string,
|
|
42
|
-
issues: ValidationIssue[],
|
|
43
|
-
toolName: string,
|
|
44
|
-
): void {
|
|
45
|
-
// Check for ZodOptional containing a ZodNullable ZodString
|
|
46
|
-
if (schema instanceof z.ZodOptional) {
|
|
47
|
-
const innerSchema = schema.unwrap()
|
|
48
|
-
if (innerSchema instanceof z.ZodNullable) {
|
|
49
|
-
const nullableInner = innerSchema.unwrap()
|
|
50
|
-
if (nullableInner instanceof z.ZodString) {
|
|
51
|
-
issues.push({
|
|
52
|
-
toolName,
|
|
53
|
-
parameterPath: path,
|
|
54
|
-
issue: 'GEMINI_API_INCOMPATIBLE: z.string().nullable().optional() pattern detected',
|
|
55
|
-
suggestion:
|
|
56
|
-
'REQUIRED FIX: Change "z.string().nullable().optional()" to "z.string().optional()" and use special strings like "remove" or "unassign" in description to handle clearing. This pattern causes HTTP 400 errors in Google Gemini API due to OpenAPI 3.1 nullable type incompatibility.',
|
|
57
|
-
})
|
|
58
|
-
}
|
|
59
|
-
}
|
|
60
|
-
}
|
|
61
|
-
|
|
62
|
-
// Check for ZodNullable containing a ZodOptional ZodString
|
|
63
|
-
if (schema instanceof z.ZodNullable) {
|
|
64
|
-
const innerSchema = schema.unwrap()
|
|
65
|
-
if (innerSchema instanceof z.ZodOptional) {
|
|
66
|
-
const optionalInner = innerSchema.unwrap()
|
|
67
|
-
if (optionalInner instanceof z.ZodString) {
|
|
68
|
-
issues.push({
|
|
69
|
-
toolName,
|
|
70
|
-
parameterPath: path,
|
|
71
|
-
issue: 'GEMINI_API_INCOMPATIBLE: z.string().optional().nullable() pattern detected',
|
|
72
|
-
suggestion:
|
|
73
|
-
'REQUIRED FIX: Change "z.string().optional().nullable()" to "z.string().optional()" and use special strings like "remove" or "unassign" in description to handle clearing. This pattern causes HTTP 400 errors in Google Gemini API due to OpenAPI 3.1 nullable type incompatibility.',
|
|
74
|
-
})
|
|
75
|
-
}
|
|
76
|
-
}
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
// Recursively check nested schemas
|
|
80
|
-
if (schema instanceof z.ZodObject) {
|
|
81
|
-
const shape = schema.shape
|
|
82
|
-
for (const [key, value] of Object.entries(shape)) {
|
|
83
|
-
const newPath = path ? `${path}.${key}` : key
|
|
84
|
-
walkZodSchema(value as AnyZodSchema, newPath, issues, toolName)
|
|
85
|
-
}
|
|
86
|
-
} else if (schema instanceof z.ZodArray) {
|
|
87
|
-
const element = (schema as unknown as { _zod: { def: { element: AnyZodSchema } } })._zod.def
|
|
88
|
-
.element
|
|
89
|
-
walkZodSchema(element, `${path}[]`, issues, toolName)
|
|
90
|
-
} else if (
|
|
91
|
-
schema instanceof z.ZodOptional ||
|
|
92
|
-
schema instanceof z.ZodNullable ||
|
|
93
|
-
schema instanceof z.ZodDefault
|
|
94
|
-
) {
|
|
95
|
-
walkZodSchema(schema.unwrap() as AnyZodSchema, path, issues, toolName)
|
|
96
|
-
} else if (schema instanceof z.ZodUnion) {
|
|
97
|
-
const options = (schema as unknown as { _zod: { def: { options: AnyZodSchema[] } } })._zod
|
|
98
|
-
.def.options
|
|
99
|
-
options.forEach((option: AnyZodSchema, index: number) => {
|
|
100
|
-
walkZodSchema(option, `${path}[union:${index}]`, issues, toolName)
|
|
101
|
-
})
|
|
102
|
-
} else if (schema instanceof z.ZodDiscriminatedUnion) {
|
|
103
|
-
const options = (schema as unknown as { _zod: { def: { options: AnyZodSchema[] } } })._zod
|
|
104
|
-
.def.options
|
|
105
|
-
options.forEach((option: AnyZodSchema, index: number) => {
|
|
106
|
-
walkZodSchema(option, `${path}[union:${index}]`, issues, toolName)
|
|
107
|
-
})
|
|
108
|
-
} else if (schema instanceof z.ZodIntersection) {
|
|
109
|
-
const left = (schema as unknown as { _zod: { def: { left: AnyZodSchema } } })._zod.def.left
|
|
110
|
-
const right = (schema as unknown as { _zod: { def: { right: AnyZodSchema } } })._zod.def
|
|
111
|
-
.right
|
|
112
|
-
walkZodSchema(left, `${path}[left]`, issues, toolName)
|
|
113
|
-
walkZodSchema(right, `${path}[right]`, issues, toolName)
|
|
114
|
-
} else if (schema instanceof z.ZodRecord) {
|
|
115
|
-
const valueType = (schema as unknown as { _zod: { def: { valueType: AnyZodSchema } } })._zod
|
|
116
|
-
.def.valueType
|
|
117
|
-
walkZodSchema(valueType, `${path}[value]`, issues, toolName)
|
|
118
|
-
} else if (schema instanceof z.ZodTuple) {
|
|
119
|
-
const items = (schema as unknown as { _zod: { def: { items: AnyZodSchema[] } } })._zod.def
|
|
120
|
-
.items
|
|
121
|
-
items.forEach((item: AnyZodSchema, index: number) => {
|
|
122
|
-
walkZodSchema(item, `${path}[${index}]`, issues, toolName)
|
|
123
|
-
})
|
|
124
|
-
}
|
|
125
|
-
}
|
|
126
|
-
|
|
127
|
-
/**
|
|
128
|
-
* Validate a single tool's parameter schema
|
|
129
|
-
*/
|
|
130
|
-
function validateToolSchema(tool: {
|
|
131
|
-
name?: string
|
|
132
|
-
parameters?: Record<string, z.ZodTypeAny>
|
|
133
|
-
}): ValidationIssue[] {
|
|
134
|
-
const issues: ValidationIssue[] = []
|
|
135
|
-
const toolName = tool.name || 'unknown'
|
|
136
|
-
|
|
137
|
-
if (!tool.parameters) {
|
|
138
|
-
return issues
|
|
139
|
-
}
|
|
140
|
-
|
|
141
|
-
try {
|
|
142
|
-
const schema = z.object(tool.parameters)
|
|
143
|
-
walkZodSchema(schema, '', issues, toolName)
|
|
144
|
-
} catch (error) {
|
|
145
|
-
issues.push({
|
|
146
|
-
toolName,
|
|
147
|
-
parameterPath: 'root',
|
|
148
|
-
issue: `Failed to analyze schema: ${error}`,
|
|
149
|
-
suggestion: 'Check that the tool parameters are valid Zod schemas',
|
|
150
|
-
})
|
|
151
|
-
}
|
|
152
|
-
|
|
153
|
-
return issues
|
|
154
|
-
}
|
|
155
|
-
|
|
156
|
-
/**
|
|
157
|
-
* Main validation function using runtime schema analysis
|
|
158
|
-
*/
|
|
159
|
-
async function validateAllSchemas(verbose: boolean = false): Promise<ValidationResult> {
|
|
160
|
-
try {
|
|
161
|
-
const { registeredTools } = await import(`${process.cwd()}/dist/index.js`)
|
|
162
|
-
|
|
163
|
-
const allTools = registeredTools
|
|
164
|
-
|
|
165
|
-
const allIssues: ValidationIssue[] = []
|
|
166
|
-
let totalParameters = 0
|
|
167
|
-
|
|
168
|
-
for (const tool of allTools) {
|
|
169
|
-
const toolName = tool.name
|
|
170
|
-
const toolIssues = validateToolSchema(tool)
|
|
171
|
-
allIssues.push(...toolIssues)
|
|
172
|
-
|
|
173
|
-
// Count parameters for stats
|
|
174
|
-
if (tool.parameters) {
|
|
175
|
-
try {
|
|
176
|
-
const schema = z.object(tool.parameters)
|
|
177
|
-
const shape = schema.shape
|
|
178
|
-
if (shape) {
|
|
179
|
-
totalParameters += Object.keys(shape).length
|
|
180
|
-
}
|
|
181
|
-
} catch {
|
|
182
|
-
// Skip counting if schema is invalid
|
|
183
|
-
}
|
|
184
|
-
}
|
|
185
|
-
|
|
186
|
-
if (verbose) {
|
|
187
|
-
const issueCount = toolIssues.length
|
|
188
|
-
const status = issueCount === 0 ? '✅' : `❌ (${issueCount} issues)`
|
|
189
|
-
const paramCount = tool.parameters ? Object.keys(tool.parameters).length : 0
|
|
190
|
-
console.log(`${status} ${toolName} (${paramCount} parameters)`)
|
|
191
|
-
|
|
192
|
-
if (issueCount > 0) {
|
|
193
|
-
toolIssues.forEach((issue) => {
|
|
194
|
-
console.log(` ${issue.parameterPath}: ${issue.issue}`)
|
|
195
|
-
})
|
|
196
|
-
}
|
|
197
|
-
}
|
|
198
|
-
}
|
|
199
|
-
|
|
200
|
-
return {
|
|
201
|
-
success: allIssues.length === 0,
|
|
202
|
-
issues: allIssues,
|
|
203
|
-
toolsChecked: allTools.length,
|
|
204
|
-
parametersChecked: totalParameters,
|
|
205
|
-
}
|
|
206
|
-
} catch (error) {
|
|
207
|
-
return {
|
|
208
|
-
success: false,
|
|
209
|
-
issues: [
|
|
210
|
-
{
|
|
211
|
-
toolName: 'system',
|
|
212
|
-
parameterPath: 'import',
|
|
213
|
-
issue: `Failed to import tools: ${error}`,
|
|
214
|
-
suggestion: 'Ensure the project is built and dist/index.js exists',
|
|
215
|
-
},
|
|
216
|
-
],
|
|
217
|
-
toolsChecked: 0,
|
|
218
|
-
parametersChecked: 0,
|
|
219
|
-
}
|
|
220
|
-
}
|
|
221
|
-
}
|
|
222
|
-
|
|
223
|
-
/**
|
|
224
|
-
* CLI interface
|
|
225
|
-
*/
|
|
226
|
-
async function main() {
|
|
227
|
-
const args = process.argv.slice(2)
|
|
228
|
-
const verbose = args.includes('--verbose')
|
|
229
|
-
const jsonOutput = args.includes('--json')
|
|
230
|
-
|
|
231
|
-
try {
|
|
232
|
-
const result = await validateAllSchemas(verbose)
|
|
233
|
-
|
|
234
|
-
if (jsonOutput) {
|
|
235
|
-
console.log(JSON.stringify(result, null, 2))
|
|
236
|
-
} else {
|
|
237
|
-
if (result.success) {
|
|
238
|
-
console.log('✅ Schema validation passed!')
|
|
239
|
-
console.log(
|
|
240
|
-
` Checked ${result.toolsChecked} tools with ${result.parametersChecked} parameters`,
|
|
241
|
-
)
|
|
242
|
-
console.log(
|
|
243
|
-
' All schemas are Gemini API compatible (no .nullable() on optional strings)',
|
|
244
|
-
)
|
|
245
|
-
} else {
|
|
246
|
-
console.log('❌ Schema validation failed!')
|
|
247
|
-
console.log(
|
|
248
|
-
` Found ${result.issues.length} issue(s) in ${result.toolsChecked} tools:\n`,
|
|
249
|
-
)
|
|
250
|
-
|
|
251
|
-
result.issues.forEach((issue, index) => {
|
|
252
|
-
console.log(`\n${index + 1}. 🚫 VALIDATION FAILURE`)
|
|
253
|
-
console.log(` Tool: ${issue.toolName}`)
|
|
254
|
-
console.log(` Parameter: ${issue.parameterPath}`)
|
|
255
|
-
console.log(` Issue: ${issue.issue}`)
|
|
256
|
-
console.log(` Action Required: ${issue.suggestion}`)
|
|
257
|
-
console.log(` File Location: src/tools/${issue.toolName}.ts`)
|
|
258
|
-
console.log(` Fix Pattern: Remove .nullable() from the parameter schema`)
|
|
259
|
-
console.log(
|
|
260
|
-
` Example Fix: Change z.string().nullable().optional() → z.string().optional()`,
|
|
261
|
-
)
|
|
262
|
-
console.log(
|
|
263
|
-
` ⚠️ This validation failure will cause Gemini API HTTP 400 errors\n`,
|
|
264
|
-
)
|
|
265
|
-
})
|
|
266
|
-
}
|
|
267
|
-
}
|
|
268
|
-
|
|
269
|
-
process.exit(result.success ? 0 : 1)
|
|
270
|
-
} catch (error) {
|
|
271
|
-
console.error('Fatal error during schema validation:', error)
|
|
272
|
-
process.exit(1)
|
|
273
|
-
}
|
|
274
|
-
}
|
|
275
|
-
|
|
276
|
-
// Run if this script is executed directly
|
|
277
|
-
if (
|
|
278
|
-
process.argv[1]?.endsWith('validate-schemas.ts') ||
|
|
279
|
-
process.argv[1]?.endsWith('validate-schemas.js')
|
|
280
|
-
) {
|
|
281
|
-
main()
|
|
282
|
-
}
|
|
283
|
-
|
|
284
|
-
export type { ValidationIssue, ValidationResult }
|
|
285
|
-
export { validateAllSchemas }
|