mod8-cli 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +87 -0
- package/LICENSE +21 -0
- package/README.md +239 -0
- package/bin/mod8.js +2 -0
- package/dist/cli.js +302 -0
- package/dist/commands/addProvider.js +105 -0
- package/dist/commands/all.js +158 -0
- package/dist/commands/chat.js +855 -0
- package/dist/commands/config.js +29 -0
- package/dist/commands/devAuthStatus.js +34 -0
- package/dist/commands/devHostAsk.js +51 -0
- package/dist/commands/devHostSystem.js +15 -0
- package/dist/commands/devResolve.js +54 -0
- package/dist/commands/devSimulate.js +235 -0
- package/dist/commands/devWorkAsk.js +55 -0
- package/dist/commands/intentRouting.js +280 -0
- package/dist/commands/keys.js +55 -0
- package/dist/commands/list.js +27 -0
- package/dist/commands/login.js +147 -0
- package/dist/commands/logout.js +17 -0
- package/dist/commands/prompt.js +63 -0
- package/dist/commands/providers.js +30 -0
- package/dist/commands/verify.js +5 -0
- package/dist/input/compose.js +37 -0
- package/dist/input/files.js +49 -0
- package/dist/input/stdin.js +14 -0
- package/dist/providers/anthropic.js +115 -0
- package/dist/providers/displayName.js +25 -0
- package/dist/providers/errorHints.js +175 -0
- package/dist/providers/generic.js +331 -0
- package/dist/providers/genericChat.js +265 -0
- package/dist/providers/google.js +63 -0
- package/dist/providers/hostSystem.js +173 -0
- package/dist/providers/index.js +38 -0
- package/dist/providers/mock.js +87 -0
- package/dist/providers/modelResolution.js +42 -0
- package/dist/providers/openai.js +75 -0
- package/dist/providers/pricing.js +47 -0
- package/dist/providers/proxy.js +148 -0
- package/dist/providers/registry.js +196 -0
- package/dist/providers/types.js +1 -0
- package/dist/providers/workSystem.js +33 -0
- package/dist/storage/auth.js +65 -0
- package/dist/storage/config.js +35 -0
- package/dist/storage/keys.js +59 -0
- package/dist/storage/providers.js +337 -0
- package/dist/storage/sessions.js +150 -0
- package/dist/types.js +9 -0
- package/dist/util/debug.js +79 -0
- package/dist/util/errors.js +157 -0
- package/dist/util/prompt.js +111 -0
- package/dist/util/secrets.js +110 -0
- package/dist/util/text.js +53 -0
- package/dist/util/time.js +25 -0
- package/dist/verify/runner.js +437 -0
- package/package.json +69 -0
- package/specs/all-mode.yaml +44 -0
- package/specs/behavior/auto-fallback.yaml +49 -0
- package/specs/behavior/bare-name-routing.yaml +223 -0
- package/specs/behavior/bare-paste-confirm.yaml +125 -0
- package/specs/behavior/env-var-respected.yaml +108 -0
- package/specs/behavior/error-fidelity.yaml +92 -0
- package/specs/behavior/error-hints.yaml +160 -0
- package/specs/behavior/fresh-vs-resume.yaml +94 -0
- package/specs/behavior/fuzzy-match.yaml +208 -0
- package/specs/behavior/host-self-knowledge-fresh.yaml +66 -0
- package/specs/behavior/intent-no-mismatch.yaml +115 -0
- package/specs/behavior/login-logout.yaml +97 -0
- package/specs/behavior/no-model-allowlist.yaml +80 -0
- package/specs/behavior/paste-key.yaml +342 -0
- package/specs/behavior/provider-switching.yaml +186 -0
- package/specs/behavior/providers-json-respected.yaml +106 -0
- package/specs/behavior/self-knowledge.yaml +119 -0
- package/specs/behavior/stress-session.yaml +226 -0
- package/specs/behavior/switch-back-when-failing.yaml +90 -0
- package/specs/behavior/work-character.yaml +109 -0
- package/specs/chat-meta.yaml +349 -0
- package/specs/chat-startup.yaml +148 -0
- package/specs/chat.yaml +91 -0
- package/specs/config.yaml +42 -0
- package/specs/install.yaml +112 -0
- package/specs/keys.yaml +81 -0
- package/specs/one-shot.yaml +65 -0
- package/specs/pipe-and-files.yaml +40 -0
- package/specs/providers.yaml +172 -0
- package/specs/sessions.yaml +115 -0
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
name: behavior — per-kind structured error diagnosis
|
|
2
|
+
description: |
|
|
3
|
+
Bug: when a provider returned a SPECIFIC error ("[403] project denied
|
|
4
|
+
access"), mod8 dropped the message and showed a generic provider hint
|
|
5
|
+
("free-tier quota / billing / regions") that didn't match what was wrong.
|
|
6
|
+
|
|
7
|
+
Fix: every error is now diagnosed by KIND (auth / forbidden / rate-limit /
|
|
8
|
+
no-credit / server / network / timeout / model / other). The user-facing
|
|
9
|
+
output:
|
|
10
|
+
short — single line: "<provider> <verb> (HTTP <code>): '<raw>'"
|
|
11
|
+
long — multi-line explanation + provider-specific fix bullets
|
|
12
|
+
suggestion — short tip used by the warn / auto-fallback banners,
|
|
13
|
+
customized per kind ("paste a new key" for auth,
|
|
14
|
+
"switch back, project-level block" for forbidden, etc.)
|
|
15
|
+
|
|
16
|
+
These specs feed synthetic errors through `mod8 dev:explain-error` and
|
|
17
|
+
assert the diagnosis pipeline produces the right kind + content.
|
|
18
|
+
|
|
19
|
+
tests:
|
|
20
|
+
- name: '403 denied → kind=forbidden, quotes raw, suggests project-level fix (NOT generic billing)'
|
|
21
|
+
shell: |
|
|
22
|
+
mod8 dev:explain-error google "[403 Forbidden] Your project has been denied access. Please contact support."
|
|
23
|
+
expect:
|
|
24
|
+
stdout_contains:
|
|
25
|
+
- 'kind=forbidden'
|
|
26
|
+
- 'google rejected the request (HTTP 403)'
|
|
27
|
+
- "'Your project has been denied access. Please contact support.'"
|
|
28
|
+
- 'project-level block'
|
|
29
|
+
- 'Switch back to mod8'
|
|
30
|
+
- 'project / account level'
|
|
31
|
+
- 'Wait 24h for auto-review'
|
|
32
|
+
stdout_omits:
|
|
33
|
+
# The generic Google quota-latency hint must NOT show up — that's
|
|
34
|
+
# the regression we're fixing.
|
|
35
|
+
- 'free-tier quota'
|
|
36
|
+
- '10 min after creating a key'
|
|
37
|
+
|
|
38
|
+
- name: '401 invalid key → kind=auth, suggests paste-a-new-key inline'
|
|
39
|
+
shell: |
|
|
40
|
+
mod8 dev:explain-error openai "401 Unauthorized: invalid_api_key"
|
|
41
|
+
expect:
|
|
42
|
+
stdout_contains:
|
|
43
|
+
- 'kind=auth'
|
|
44
|
+
- 'openai rejected the API key (HTTP 401)'
|
|
45
|
+
- 'invalid_api_key'
|
|
46
|
+
# Specific OpenAI-vs-ChatGPT-subscription nuance is included.
|
|
47
|
+
- 'ChatGPT subscription is NOT an OpenAI API key'
|
|
48
|
+
- 'paste a new key'
|
|
49
|
+
# Suggestion routes the user toward replacing the key, not retrying.
|
|
50
|
+
- 'paste a new key to replace this one'
|
|
51
|
+
|
|
52
|
+
- name: '429 with retry-after extracts the delay'
|
|
53
|
+
shell: |
|
|
54
|
+
mod8 dev:explain-error anthropic "rate_limit_exceeded: please retry after 30 seconds"
|
|
55
|
+
expect:
|
|
56
|
+
stdout_contains:
|
|
57
|
+
- 'kind=rate-limit'
|
|
58
|
+
- 'anthropic rate-limited'
|
|
59
|
+
- 'retry in 30s'
|
|
60
|
+
# The suggestion echoes the SAME delay the provider gave us.
|
|
61
|
+
- 'wait 30s and retry'
|
|
62
|
+
|
|
63
|
+
- name: '429 without retry-after still works (no fake delay invented)'
|
|
64
|
+
shell: |
|
|
65
|
+
mod8 dev:explain-error openai "429 Too Many Requests"
|
|
66
|
+
expect:
|
|
67
|
+
stdout_contains:
|
|
68
|
+
- 'kind=rate-limit'
|
|
69
|
+
- 'openai rate-limited'
|
|
70
|
+
- 'wait a few seconds and retry'
|
|
71
|
+
stdout_omits:
|
|
72
|
+
- 'retry in 0s'
|
|
73
|
+
- 'wait 0s'
|
|
74
|
+
|
|
75
|
+
- name: '402 / insufficient_balance → kind=no-credit + provider billing URL'
|
|
76
|
+
shell: |
|
|
77
|
+
mod8 dev:explain-error deepseek "402 Payment Required: insufficient_balance"
|
|
78
|
+
expect:
|
|
79
|
+
stdout_contains:
|
|
80
|
+
- 'kind=no-credit'
|
|
81
|
+
- 'deepseek reports insufficient credit'
|
|
82
|
+
- 'insufficient_balance'
|
|
83
|
+
- 'platform.deepseek.com'
|
|
84
|
+
# Mentions the JUST-CREATED-KEY footnote so users don't churn on quota latency.
|
|
85
|
+
- '~10 minutes'
|
|
86
|
+
|
|
87
|
+
- name: '5xx server error → kind=server, suggests retry shortly'
|
|
88
|
+
shell: |
|
|
89
|
+
mod8 dev:explain-error groq "503 Service Unavailable: server overloaded"
|
|
90
|
+
expect:
|
|
91
|
+
stdout_contains:
|
|
92
|
+
- 'kind=server'
|
|
93
|
+
- 'groq returned a server error (HTTP 503)'
|
|
94
|
+
- 'server overloaded'
|
|
95
|
+
- 'having issues'
|
|
96
|
+
- 'retry shortly'
|
|
97
|
+
|
|
98
|
+
- name: 'network error → kind=network, switch back + check connection'
|
|
99
|
+
shell: |
|
|
100
|
+
mod8 dev:explain-error openrouter "fetch failed: ENOTFOUND openrouter.ai"
|
|
101
|
+
expect:
|
|
102
|
+
stdout_contains:
|
|
103
|
+
- 'kind=network'
|
|
104
|
+
- "couldn't reach openrouter"
|
|
105
|
+
- 'ENOTFOUND'
|
|
106
|
+
- 'Check your connection'
|
|
107
|
+
|
|
108
|
+
- name: 'timeout → kind=timeout, retry shortly'
|
|
109
|
+
shell: |
|
|
110
|
+
mod8 dev:explain-error xai "Request timed out after 60s"
|
|
111
|
+
expect:
|
|
112
|
+
stdout_contains:
|
|
113
|
+
- 'kind=timeout'
|
|
114
|
+
- 'request to xai timed out'
|
|
115
|
+
- 'retry shortly'
|
|
116
|
+
|
|
117
|
+
- name: 'model unavailable → kind=model, override env var hint'
|
|
118
|
+
shell: |
|
|
119
|
+
mod8 dev:explain-error mistral "model \`bogus-1\` does not exist"
|
|
120
|
+
expect:
|
|
121
|
+
stdout_contains:
|
|
122
|
+
- 'kind=model'
|
|
123
|
+
- 'mistral model not available'
|
|
124
|
+
- 'MOD8_MISTRAL_MODEL'
|
|
125
|
+
|
|
126
|
+
- name: 'other / unknown error → kind=other, raw message preserved (no fake diagnosis)'
|
|
127
|
+
shell: |
|
|
128
|
+
mod8 dev:explain-error openai "completely surprising message we have never seen"
|
|
129
|
+
expect:
|
|
130
|
+
stdout_contains:
|
|
131
|
+
- 'kind=other'
|
|
132
|
+
- 'openai failed'
|
|
133
|
+
- 'completely surprising message we have never seen'
|
|
134
|
+
- "Type 'mod8' to switch back"
|
|
135
|
+
stdout_omits:
|
|
136
|
+
# Never replace an unknown error with a fabricated diagnosis.
|
|
137
|
+
- 'kind=auth'
|
|
138
|
+
- 'kind=forbidden'
|
|
139
|
+
- 'kind=rate-limit'
|
|
140
|
+
|
|
141
|
+
- name: 'raw provider message is preserved verbatim across kinds (search-friendly)'
|
|
142
|
+
shell: |
|
|
143
|
+
mod8 dev:explain-error google "[403 Forbidden] Project 12345 has been denied access. Please contact support."
|
|
144
|
+
expect:
|
|
145
|
+
stdout_contains:
|
|
146
|
+
# Exact quote of the provider's wording — user can copy and search.
|
|
147
|
+
- "'Project 12345 has been denied access. Please contact support.'"
|
|
148
|
+
|
|
149
|
+
- name: '403 NOT confused with billing/quota — distinct fix path'
|
|
150
|
+
shell: |
|
|
151
|
+
mod8 dev:explain-error google "[403 Forbidden] Your project has been denied access. Please contact support."
|
|
152
|
+
expect:
|
|
153
|
+
stdout_contains:
|
|
154
|
+
- 'kind=forbidden'
|
|
155
|
+
- 'project-level block'
|
|
156
|
+
stdout_omits:
|
|
157
|
+
# No "out of credit" / "top up" diversions.
|
|
158
|
+
- 'kind=no-credit'
|
|
159
|
+
- 'Top up at'
|
|
160
|
+
- 'insufficient credit'
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
name: behavior — fresh vs resume startup
|
|
2
|
+
description: |
|
|
3
|
+
Behavioral coverage for the chat-startup contract:
|
|
4
|
+
- bare `mod8` opens a NEW session (matches ChatGPT-style "open = new")
|
|
5
|
+
- `mod8 resume` (no id) brings the most recent session back
|
|
6
|
+
- `mod8 resume <id>` targets a specific session
|
|
7
|
+
These tests assert on what the user OBSERVES (the "resuming session …" line
|
|
8
|
+
printed before Ink starts) for each combination of stored state + command.
|
|
9
|
+
|
|
10
|
+
tests:
|
|
11
|
+
- name: prior sessions exist, but bare `mod8` does NOT resume
|
|
12
|
+
requires_api_key: true
|
|
13
|
+
setup:
|
|
14
|
+
- shell: |
|
|
15
|
+
mkdir -p $MOD8_CONFIG_DIR/sessions
|
|
16
|
+
cat > $MOD8_CONFIG_DIR/sessions/2026-05-07-prev.json <<JSON
|
|
17
|
+
{
|
|
18
|
+
"version": 1,
|
|
19
|
+
"id": "2026-05-07-prev",
|
|
20
|
+
"title": "old chat",
|
|
21
|
+
"createdAt": 1715000000000,
|
|
22
|
+
"lastActivity": 1715000000000,
|
|
23
|
+
"messages": [
|
|
24
|
+
{"role": "user", "content": "remember the codeword fuchsia", "mode": "host"},
|
|
25
|
+
{"role": "assistant", "content": "got it.", "mode": "host"}
|
|
26
|
+
]
|
|
27
|
+
}
|
|
28
|
+
JSON
|
|
29
|
+
chmod 600 $MOD8_CONFIG_DIR/sessions/2026-05-07-prev.json
|
|
30
|
+
repl:
|
|
31
|
+
run: mod8
|
|
32
|
+
timeout_ms: 5000
|
|
33
|
+
inputs:
|
|
34
|
+
- send: ""
|
|
35
|
+
delay_ms: 1500
|
|
36
|
+
expect:
|
|
37
|
+
stdout_omits:
|
|
38
|
+
- "resuming session"
|
|
39
|
+
- "fuchsia"
|
|
40
|
+
- "old chat"
|
|
41
|
+
|
|
42
|
+
- name: bare `mod8` shows the welcome banner
|
|
43
|
+
requires_api_key: true
|
|
44
|
+
repl:
|
|
45
|
+
run: mod8
|
|
46
|
+
timeout_ms: 5000
|
|
47
|
+
inputs:
|
|
48
|
+
- send: ""
|
|
49
|
+
delay_ms: 1500
|
|
50
|
+
expect:
|
|
51
|
+
stdout_contains:
|
|
52
|
+
- "switch to claude"
|
|
53
|
+
- "use any provider"
|
|
54
|
+
|
|
55
|
+
- name: '`mod8 resume` (no id) prints "resuming session …" with the right id'
|
|
56
|
+
requires_api_key: true
|
|
57
|
+
setup:
|
|
58
|
+
- shell: |
|
|
59
|
+
mkdir -p $MOD8_CONFIG_DIR/sessions
|
|
60
|
+
cat > $MOD8_CONFIG_DIR/sessions/2026-05-07-rsme.json <<JSON
|
|
61
|
+
{
|
|
62
|
+
"version": 1,
|
|
63
|
+
"id": "2026-05-07-rsme",
|
|
64
|
+
"title": "recent",
|
|
65
|
+
"createdAt": 1715000000000,
|
|
66
|
+
"lastActivity": 1715000000000,
|
|
67
|
+
"messages": [
|
|
68
|
+
{"role": "user", "content": "x", "mode": "host"},
|
|
69
|
+
{"role": "assistant", "content": "y", "mode": "host"}
|
|
70
|
+
]
|
|
71
|
+
}
|
|
72
|
+
JSON
|
|
73
|
+
chmod 600 $MOD8_CONFIG_DIR/sessions/2026-05-07-rsme.json
|
|
74
|
+
repl:
|
|
75
|
+
run: mod8 resume
|
|
76
|
+
timeout_ms: 5000
|
|
77
|
+
inputs:
|
|
78
|
+
- send: ""
|
|
79
|
+
delay_ms: 1500
|
|
80
|
+
expect:
|
|
81
|
+
stdout_contains: "resuming session 2026-05-07-rsme"
|
|
82
|
+
|
|
83
|
+
- name: '`mod8 resume` with no sessions errors clearly'
|
|
84
|
+
repl:
|
|
85
|
+
run: mod8 resume
|
|
86
|
+
timeout_ms: 5000
|
|
87
|
+
inputs:
|
|
88
|
+
- send: ""
|
|
89
|
+
delay_ms: 1500
|
|
90
|
+
expect:
|
|
91
|
+
stderr_contains:
|
|
92
|
+
- "no sessions to resume"
|
|
93
|
+
- "mod8 list"
|
|
94
|
+
exit_code: 1
|
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
name: behavior — fuzzy-match provider typos
|
|
2
|
+
description: |
|
|
3
|
+
When the user types a near-typo of a configured provider, mod8 fuzzy-
|
|
4
|
+
matches it instead of erroring out. Levenshtein distance ≤ 2 against
|
|
5
|
+
configured ids, display names, built-in template ids, and the curated
|
|
6
|
+
HIGH_CONFIDENCE_BRAND_ALIASES.
|
|
7
|
+
|
|
8
|
+
Distance rules (set to keep the false-positive rate low):
|
|
9
|
+
- distance ≤ 1 → auto-route
|
|
10
|
+
- distance == 2 → auto-route, UNLESS the input is exactly 4 chars,
|
|
11
|
+
in which case ASK FIRST ("did you mean X?")
|
|
12
|
+
- input length < 4 → no fuzzy at all (avoids "go" / "ok" / "xyz" misfires)
|
|
13
|
+
- affirmative/negative tokens ("yes", "no", "go") → no fuzzy
|
|
14
|
+
|
|
15
|
+
Multiple matches → list and ask. Zero matches → fall through silently.
|
|
16
|
+
|
|
17
|
+
tests:
|
|
18
|
+
- name: 'distance-1 typo "gimini" auto-routes to google'
|
|
19
|
+
setup:
|
|
20
|
+
- shell: |
|
|
21
|
+
mkdir -p $MOD8_CONFIG_DIR
|
|
22
|
+
cat > $MOD8_CONFIG_DIR/providers.json <<JSON
|
|
23
|
+
{
|
|
24
|
+
"google": {"apiKey":"x","apiType":"gemini","name":"Google (Gemini)","defaultModel":"gemini-2.0-flash","color":"#06B6D4"},
|
|
25
|
+
"anthropic": {"apiKey":"x","apiType":"anthropic","name":"Anthropic (Claude)","defaultModel":"claude-sonnet-4-6","color":"#A78BFA"}
|
|
26
|
+
}
|
|
27
|
+
JSON
|
|
28
|
+
chmod 600 $MOD8_CONFIG_DIR/providers.json
|
|
29
|
+
shell: |
|
|
30
|
+
mod8 dev:simulate <<'INPUTS'
|
|
31
|
+
gimini
|
|
32
|
+
INPUTS
|
|
33
|
+
expect:
|
|
34
|
+
stdout_contains:
|
|
35
|
+
- 'step=1 input="gimini" mode=work provider=google action=fuzzy-route id=google distance=1'
|
|
36
|
+
|
|
37
|
+
- name: 'distance-2 typo "gomoni" still auto-routes (length 6 > 4)'
|
|
38
|
+
setup:
|
|
39
|
+
- shell: |
|
|
40
|
+
mkdir -p $MOD8_CONFIG_DIR
|
|
41
|
+
cat > $MOD8_CONFIG_DIR/providers.json <<JSON
|
|
42
|
+
{
|
|
43
|
+
"google": {"apiKey":"x","apiType":"gemini","name":"Google (Gemini)","defaultModel":"gemini-2.0-flash","color":"#06B6D4"},
|
|
44
|
+
"anthropic": {"apiKey":"x","apiType":"anthropic","name":"Anthropic (Claude)","defaultModel":"claude-sonnet-4-6","color":"#A78BFA"}
|
|
45
|
+
}
|
|
46
|
+
JSON
|
|
47
|
+
chmod 600 $MOD8_CONFIG_DIR/providers.json
|
|
48
|
+
shell: |
|
|
49
|
+
mod8 dev:simulate <<'INPUTS'
|
|
50
|
+
gomoni
|
|
51
|
+
INPUTS
|
|
52
|
+
expect:
|
|
53
|
+
stdout_contains:
|
|
54
|
+
- 'step=1 input="gomoni" mode=work provider=google action=fuzzy-route id=google distance=2'
|
|
55
|
+
|
|
56
|
+
- name: 'verb-form typo "use anthopic" auto-routes'
|
|
57
|
+
setup:
|
|
58
|
+
- shell: |
|
|
59
|
+
mkdir -p $MOD8_CONFIG_DIR
|
|
60
|
+
cat > $MOD8_CONFIG_DIR/providers.json <<JSON
|
|
61
|
+
{
|
|
62
|
+
"anthropic": {"apiKey":"x","apiType":"anthropic","name":"Anthropic (Claude)","defaultModel":"claude-sonnet-4-6","color":"#A78BFA"}
|
|
63
|
+
}
|
|
64
|
+
JSON
|
|
65
|
+
chmod 600 $MOD8_CONFIG_DIR/providers.json
|
|
66
|
+
shell: |
|
|
67
|
+
mod8 dev:simulate <<'INPUTS'
|
|
68
|
+
use anthopic
|
|
69
|
+
INPUTS
|
|
70
|
+
expect:
|
|
71
|
+
stdout_contains:
|
|
72
|
+
- 'step=1 input="use anthopic" mode=work provider=anthropic action=fuzzy-route id=anthropic distance=1'
|
|
73
|
+
|
|
74
|
+
- name: 'short distance-2 typo "clad" ASKs first instead of routing'
|
|
75
|
+
setup:
|
|
76
|
+
- shell: |
|
|
77
|
+
mkdir -p $MOD8_CONFIG_DIR
|
|
78
|
+
cat > $MOD8_CONFIG_DIR/providers.json <<JSON
|
|
79
|
+
{
|
|
80
|
+
"anthropic": {"apiKey":"x","apiType":"anthropic","name":"Anthropic (Claude)","defaultModel":"claude-sonnet-4-6","color":"#A78BFA"}
|
|
81
|
+
}
|
|
82
|
+
JSON
|
|
83
|
+
chmod 600 $MOD8_CONFIG_DIR/providers.json
|
|
84
|
+
shell: |
|
|
85
|
+
mod8 dev:simulate <<'INPUTS'
|
|
86
|
+
clad
|
|
87
|
+
yes
|
|
88
|
+
INPUTS
|
|
89
|
+
expect:
|
|
90
|
+
stdout_contains:
|
|
91
|
+
- 'step=1 input="clad" mode=host provider=host action=fuzzy-ask id=anthropic'
|
|
92
|
+
# User confirms → switches.
|
|
93
|
+
- 'step=2 input="yes" mode=work provider=anthropic action=fuzzy-confirmed id=anthropic'
|
|
94
|
+
|
|
95
|
+
- name: 'fuzzy-ask cancellation via "no" leaves us in host mode'
|
|
96
|
+
setup:
|
|
97
|
+
- shell: |
|
|
98
|
+
mkdir -p $MOD8_CONFIG_DIR
|
|
99
|
+
cat > $MOD8_CONFIG_DIR/providers.json <<JSON
|
|
100
|
+
{
|
|
101
|
+
"anthropic": {"apiKey":"x","apiType":"anthropic","name":"Anthropic (Claude)","defaultModel":"claude-sonnet-4-6","color":"#A78BFA"}
|
|
102
|
+
}
|
|
103
|
+
JSON
|
|
104
|
+
chmod 600 $MOD8_CONFIG_DIR/providers.json
|
|
105
|
+
shell: |
|
|
106
|
+
mod8 dev:simulate <<'INPUTS'
|
|
107
|
+
clad
|
|
108
|
+
no
|
|
109
|
+
INPUTS
|
|
110
|
+
expect:
|
|
111
|
+
stdout_contains:
|
|
112
|
+
- 'step=1 input="clad" mode=host provider=host action=fuzzy-ask id=anthropic'
|
|
113
|
+
- 'step=2 input="no" mode=host provider=host action=fuzzy-cancelled'
|
|
114
|
+
|
|
115
|
+
- name: 'short input ("go" / "ok" / "xyz" / "ant") never triggers fuzzy'
|
|
116
|
+
setup:
|
|
117
|
+
- shell: |
|
|
118
|
+
mkdir -p $MOD8_CONFIG_DIR
|
|
119
|
+
cat > $MOD8_CONFIG_DIR/providers.json <<JSON
|
|
120
|
+
{
|
|
121
|
+
"anthropic": {"apiKey":"x","apiType":"anthropic","name":"Anthropic (Claude)","defaultModel":"claude-sonnet-4-6","color":"#A78BFA"},
|
|
122
|
+
"google": {"apiKey":"x","apiType":"gemini","name":"Google (Gemini)","defaultModel":"gemini-2.0-flash","color":"#06B6D4"},
|
|
123
|
+
"xai": {"apiKey":"x","apiType":"openai-compat","name":"xAI (Grok)","baseUrl":"https://api.x.ai/v1","defaultModel":"grok-2-latest","color":"#6B7280"}
|
|
124
|
+
}
|
|
125
|
+
JSON
|
|
126
|
+
chmod 600 $MOD8_CONFIG_DIR/providers.json
|
|
127
|
+
shell: |
|
|
128
|
+
mod8 dev:simulate <<'INPUTS'
|
|
129
|
+
go
|
|
130
|
+
ok
|
|
131
|
+
xyz
|
|
132
|
+
ant
|
|
133
|
+
yo
|
|
134
|
+
INPUTS
|
|
135
|
+
expect:
|
|
136
|
+
stdout_contains:
|
|
137
|
+
- 'step=1 input="go" mode=host provider=host action=send'
|
|
138
|
+
- 'step=2 input="ok" mode=host provider=host action=send'
|
|
139
|
+
- 'step=3 input="xyz" mode=host provider=host action=send'
|
|
140
|
+
- 'step=4 input="ant" mode=host provider=host action=send'
|
|
141
|
+
- 'step=5 input="yo" mode=host provider=host action=send'
|
|
142
|
+
stdout_omits:
|
|
143
|
+
- 'fuzzy-route'
|
|
144
|
+
- 'fuzzy-ask'
|
|
145
|
+
|
|
146
|
+
- name: 'common english words near providers do not trigger fuzzy'
|
|
147
|
+
setup:
|
|
148
|
+
- shell: |
|
|
149
|
+
mkdir -p $MOD8_CONFIG_DIR
|
|
150
|
+
cat > $MOD8_CONFIG_DIR/providers.json <<JSON
|
|
151
|
+
{
|
|
152
|
+
"anthropic": {"apiKey":"x","apiType":"anthropic","name":"Anthropic (Claude)","defaultModel":"claude-sonnet-4-6","color":"#A78BFA"}
|
|
153
|
+
}
|
|
154
|
+
JSON
|
|
155
|
+
chmod 600 $MOD8_CONFIG_DIR/providers.json
|
|
156
|
+
shell: |
|
|
157
|
+
mod8 dev:simulate <<'INPUTS'
|
|
158
|
+
what
|
|
159
|
+
this
|
|
160
|
+
that
|
|
161
|
+
with
|
|
162
|
+
from
|
|
163
|
+
have
|
|
164
|
+
INPUTS
|
|
165
|
+
expect:
|
|
166
|
+
stdout_omits:
|
|
167
|
+
- 'fuzzy-route'
|
|
168
|
+
- 'fuzzy-ask'
|
|
169
|
+
|
|
170
|
+
- name: 'fuzzy on bare greeting "hi gimini" routes + carries "hi" as rest'
|
|
171
|
+
setup:
|
|
172
|
+
- shell: |
|
|
173
|
+
mkdir -p $MOD8_CONFIG_DIR
|
|
174
|
+
cat > $MOD8_CONFIG_DIR/providers.json <<JSON
|
|
175
|
+
{
|
|
176
|
+
"google": {"apiKey":"x","apiType":"gemini","name":"Google (Gemini)","defaultModel":"gemini-2.0-flash","color":"#06B6D4"}
|
|
177
|
+
}
|
|
178
|
+
JSON
|
|
179
|
+
chmod 600 $MOD8_CONFIG_DIR/providers.json
|
|
180
|
+
shell: |
|
|
181
|
+
mod8 dev:simulate <<'INPUTS'
|
|
182
|
+
hi gimini
|
|
183
|
+
INPUTS
|
|
184
|
+
expect:
|
|
185
|
+
stdout_contains:
|
|
186
|
+
- 'step=1 input="hi gimini" mode=work provider=google action=fuzzy-route id=google distance=1 rest="hi"'
|
|
187
|
+
|
|
188
|
+
- name: 'no fuzzy match for word with no near-provider → falls through silently'
|
|
189
|
+
setup:
|
|
190
|
+
- shell: |
|
|
191
|
+
mkdir -p $MOD8_CONFIG_DIR
|
|
192
|
+
cat > $MOD8_CONFIG_DIR/providers.json <<JSON
|
|
193
|
+
{
|
|
194
|
+
"anthropic": {"apiKey":"x","apiType":"anthropic","name":"Anthropic (Claude)","defaultModel":"claude-sonnet-4-6","color":"#A78BFA"}
|
|
195
|
+
}
|
|
196
|
+
JSON
|
|
197
|
+
chmod 600 $MOD8_CONFIG_DIR/providers.json
|
|
198
|
+
shell: |
|
|
199
|
+
mod8 dev:simulate <<'INPUTS'
|
|
200
|
+
hippopotamus
|
|
201
|
+
refrigerator
|
|
202
|
+
INPUTS
|
|
203
|
+
expect:
|
|
204
|
+
stdout_contains:
|
|
205
|
+
- 'step=1 input="hippopotamus" mode=host provider=host action=send'
|
|
206
|
+
- 'step=2 input="refrigerator" mode=host provider=host action=send'
|
|
207
|
+
stdout_omits:
|
|
208
|
+
- 'fuzzy-'
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
name: behavior — host system prompt refreshes mid-session
|
|
2
|
+
description: |
|
|
3
|
+
Bug 1: when the user adds a provider via the inline paste-key flow during
|
|
4
|
+
a chat session, the host (mod8) system prompt was built once at startup
|
|
5
|
+
and never updated. The next host turn would say "you don't have google
|
|
6
|
+
configured" — even though we just saved a Google key.
|
|
7
|
+
|
|
8
|
+
Fix: rebuild buildHostSystem(await readHostContext()) before every host
|
|
9
|
+
turn. The cost is negligible (one providers.json read per turn).
|
|
10
|
+
|
|
11
|
+
These specs verify the system prompt's "Providers configured RIGHT NOW"
|
|
12
|
+
block changes to reflect the current providers.json — proving the
|
|
13
|
+
rebuild path is wired.
|
|
14
|
+
|
|
15
|
+
tests:
|
|
16
|
+
- name: 'empty providers.json → host prompt says "(none yet)"'
|
|
17
|
+
shell: 'mod8 dev:host-system'
|
|
18
|
+
expect:
|
|
19
|
+
stdout_contains:
|
|
20
|
+
- 'Providers configured RIGHT NOW (in this session) — 0 configured'
|
|
21
|
+
- '(none yet'
|
|
22
|
+
|
|
23
|
+
- name: 'after seeding a provider → host prompt enumerates it'
|
|
24
|
+
setup:
|
|
25
|
+
- shell: |
|
|
26
|
+
mkdir -p $MOD8_CONFIG_DIR
|
|
27
|
+
cat > $MOD8_CONFIG_DIR/providers.json <<JSON
|
|
28
|
+
{
|
|
29
|
+
"google": {
|
|
30
|
+
"apiKey": "x",
|
|
31
|
+
"apiType": "gemini",
|
|
32
|
+
"name": "Google (Gemini)",
|
|
33
|
+
"defaultModel": "gemini-2.0-flash",
|
|
34
|
+
"color": "#06B6D4"
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
JSON
|
|
38
|
+
chmod 600 $MOD8_CONFIG_DIR/providers.json
|
|
39
|
+
shell: 'mod8 dev:host-system'
|
|
40
|
+
expect:
|
|
41
|
+
stdout_contains:
|
|
42
|
+
- 'Providers configured RIGHT NOW (in this session) — 1 configured'
|
|
43
|
+
- 'id: "google"'
|
|
44
|
+
- 'Google (Gemini)'
|
|
45
|
+
|
|
46
|
+
- name: 'inline paste-key (dev:simulate) saves provider; next host-system reflects it'
|
|
47
|
+
shell: |
|
|
48
|
+
# 1. Empty providers.json initially.
|
|
49
|
+
mkdir -p $MOD8_CONFIG_DIR
|
|
50
|
+
# Confirm starting state.
|
|
51
|
+
mod8 dev:host-system | grep -F 'RIGHT NOW (in this session) — 0 configured'
|
|
52
|
+
# 2. User pastes a key inline (the same path the chat REPL uses).
|
|
53
|
+
mod8 dev:simulate <<'INPUTS'
|
|
54
|
+
i want to paste a key
|
|
55
|
+
AIzaSyFAKE-TEST-NOT-REAL-AAAAAAAAAAAAAAAAAAAAAA
|
|
56
|
+
INPUTS
|
|
57
|
+
# 3. Now the host system prompt MUST reflect the new provider — without
|
|
58
|
+
# needing a chat restart. This proves the per-turn rebuild works.
|
|
59
|
+
echo "---after-paste---"
|
|
60
|
+
mod8 dev:host-system
|
|
61
|
+
expect:
|
|
62
|
+
stdout_contains:
|
|
63
|
+
- '---after-paste---'
|
|
64
|
+
- 'Providers configured RIGHT NOW (in this session) — 1 configured'
|
|
65
|
+
- 'id: "google"'
|
|
66
|
+
- 'Google (Gemini)'
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
name: behavior — host doesn't lie about provider switches
|
|
2
|
+
description: |
|
|
3
|
+
The bug we shipped: host LLM said "switching you to codex" but the
|
|
4
|
+
banner correctly showed "switching to claude" because <SWITCH_TO_WORK>
|
|
5
|
+
always lands on the default work model (claude). The user saw two
|
|
6
|
+
contradictory things and lost trust.
|
|
7
|
+
|
|
8
|
+
Fix: tighten the host system prompt so it NEVER claims to switch to
|
|
9
|
+
a non-claude provider via the token. These tests assert the host
|
|
10
|
+
behavior on inputs that previously triggered the lie.
|
|
11
|
+
|
|
12
|
+
tests:
|
|
13
|
+
- name: host — naming a non-claude provider does NOT trigger fake "switching to X"
|
|
14
|
+
requires_api_key: true
|
|
15
|
+
setup:
|
|
16
|
+
- shell: |
|
|
17
|
+
mkdir -p $MOD8_CONFIG_DIR
|
|
18
|
+
cat > $MOD8_CONFIG_DIR/providers.json <<JSON
|
|
19
|
+
{
|
|
20
|
+
"anthropic": {
|
|
21
|
+
"apiKey": "$ANTHROPIC_API_KEY",
|
|
22
|
+
"apiType": "anthropic",
|
|
23
|
+
"name": "Anthropic (Claude)",
|
|
24
|
+
"defaultModel": "claude-sonnet-4-6",
|
|
25
|
+
"color": "#A78BFA"
|
|
26
|
+
},
|
|
27
|
+
"openai": {
|
|
28
|
+
"apiKey": "sk-fake",
|
|
29
|
+
"apiType": "openai-compat",
|
|
30
|
+
"name": "codex",
|
|
31
|
+
"baseUrl": "https://api.openai.com/v1",
|
|
32
|
+
"defaultModel": "gpt-4o",
|
|
33
|
+
"color": "#10B981"
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
JSON
|
|
37
|
+
chmod 600 $MOD8_CONFIG_DIR/providers.json
|
|
38
|
+
# Note: the CLI's intent matcher already routes "use codex" / "talk with codex"
|
|
39
|
+
# directly, so dev:host-ask only sees this prompt if we send it raw — which
|
|
40
|
+
# is what dev:host-ask does. The host MUST NOT claim to switch to codex.
|
|
41
|
+
shell: 'mod8 dev:host-ask "switching to codex now please"'
|
|
42
|
+
expect:
|
|
43
|
+
# Host either tells the user the right phrasing, or hands off to claude
|
|
44
|
+
# — but NEVER claims "switching to codex" itself.
|
|
45
|
+
stdout_omits:
|
|
46
|
+
- "Switching you to codex"
|
|
47
|
+
- "switching to codex now"
|
|
48
|
+
- "I'll switch you to codex"
|
|
49
|
+
|
|
50
|
+
- name: host — "i want gpt" tells user the right phrasing instead of fake-switching
|
|
51
|
+
requires_api_key: true
|
|
52
|
+
setup:
|
|
53
|
+
- shell: |
|
|
54
|
+
mkdir -p $MOD8_CONFIG_DIR
|
|
55
|
+
cat > $MOD8_CONFIG_DIR/providers.json <<JSON
|
|
56
|
+
{
|
|
57
|
+
"anthropic": {
|
|
58
|
+
"apiKey": "$ANTHROPIC_API_KEY",
|
|
59
|
+
"apiType": "anthropic",
|
|
60
|
+
"name": "Anthropic (Claude)",
|
|
61
|
+
"defaultModel": "claude-sonnet-4-6",
|
|
62
|
+
"color": "#A78BFA"
|
|
63
|
+
},
|
|
64
|
+
"openai": {
|
|
65
|
+
"apiKey": "sk-fake",
|
|
66
|
+
"apiType": "openai-compat",
|
|
67
|
+
"name": "codex",
|
|
68
|
+
"baseUrl": "https://api.openai.com/v1",
|
|
69
|
+
"defaultModel": "gpt-4o",
|
|
70
|
+
"color": "#10B981"
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
JSON
|
|
74
|
+
chmod 600 $MOD8_CONFIG_DIR/providers.json
|
|
75
|
+
shell: 'mod8 dev:host-ask "i want gpt"'
|
|
76
|
+
expect:
|
|
77
|
+
# Either suggests "use codex" / "use openai" / etc, OR hands off to claude.
|
|
78
|
+
stdout_matches: "(?i)(use\\s+(codex|openai|gpt)|talk\\s+(to|with)\\s+(codex|openai|gpt)|claude)"
|
|
79
|
+
stdout_omits:
|
|
80
|
+
- "Switching you to gpt"
|
|
81
|
+
- "Switching you to codex"
|
|
82
|
+
|
|
83
|
+
- name: host — generic "let's go" handoff is plain claude, no other provider names
|
|
84
|
+
requires_api_key: true
|
|
85
|
+
setup:
|
|
86
|
+
- shell: |
|
|
87
|
+
mkdir -p $MOD8_CONFIG_DIR
|
|
88
|
+
cat > $MOD8_CONFIG_DIR/providers.json <<JSON
|
|
89
|
+
{
|
|
90
|
+
"anthropic": {
|
|
91
|
+
"apiKey": "$ANTHROPIC_API_KEY",
|
|
92
|
+
"apiType": "anthropic",
|
|
93
|
+
"name": "Anthropic (Claude)",
|
|
94
|
+
"defaultModel": "claude-sonnet-4-6",
|
|
95
|
+
"color": "#A78BFA"
|
|
96
|
+
},
|
|
97
|
+
"openai": {
|
|
98
|
+
"apiKey": "sk-fake",
|
|
99
|
+
"apiType": "openai-compat",
|
|
100
|
+
"name": "codex",
|
|
101
|
+
"baseUrl": "https://api.openai.com/v1",
|
|
102
|
+
"defaultModel": "gpt-4o",
|
|
103
|
+
"color": "#10B981"
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
JSON
|
|
107
|
+
chmod 600 $MOD8_CONFIG_DIR/providers.json
|
|
108
|
+
shell: 'mod8 dev:host-ask "ok let''s go"'
|
|
109
|
+
expect:
|
|
110
|
+
# When a plain handoff fires, the spoken text shouldn't promise codex/openai.
|
|
111
|
+
stdout_omits:
|
|
112
|
+
- "to codex"
|
|
113
|
+
- "to openai"
|
|
114
|
+
- "to gpt"
|
|
115
|
+
- "to grok"
|