mod8-cli 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +87 -0
- package/LICENSE +21 -0
- package/README.md +239 -0
- package/bin/mod8.js +2 -0
- package/dist/cli.js +302 -0
- package/dist/commands/addProvider.js +105 -0
- package/dist/commands/all.js +158 -0
- package/dist/commands/chat.js +855 -0
- package/dist/commands/config.js +29 -0
- package/dist/commands/devAuthStatus.js +34 -0
- package/dist/commands/devHostAsk.js +51 -0
- package/dist/commands/devHostSystem.js +15 -0
- package/dist/commands/devResolve.js +54 -0
- package/dist/commands/devSimulate.js +235 -0
- package/dist/commands/devWorkAsk.js +55 -0
- package/dist/commands/intentRouting.js +280 -0
- package/dist/commands/keys.js +55 -0
- package/dist/commands/list.js +27 -0
- package/dist/commands/login.js +147 -0
- package/dist/commands/logout.js +17 -0
- package/dist/commands/prompt.js +63 -0
- package/dist/commands/providers.js +30 -0
- package/dist/commands/verify.js +5 -0
- package/dist/input/compose.js +37 -0
- package/dist/input/files.js +49 -0
- package/dist/input/stdin.js +14 -0
- package/dist/providers/anthropic.js +115 -0
- package/dist/providers/displayName.js +25 -0
- package/dist/providers/errorHints.js +175 -0
- package/dist/providers/generic.js +331 -0
- package/dist/providers/genericChat.js +265 -0
- package/dist/providers/google.js +63 -0
- package/dist/providers/hostSystem.js +173 -0
- package/dist/providers/index.js +38 -0
- package/dist/providers/mock.js +87 -0
- package/dist/providers/modelResolution.js +42 -0
- package/dist/providers/openai.js +75 -0
- package/dist/providers/pricing.js +47 -0
- package/dist/providers/proxy.js +148 -0
- package/dist/providers/registry.js +196 -0
- package/dist/providers/types.js +1 -0
- package/dist/providers/workSystem.js +33 -0
- package/dist/storage/auth.js +65 -0
- package/dist/storage/config.js +35 -0
- package/dist/storage/keys.js +59 -0
- package/dist/storage/providers.js +337 -0
- package/dist/storage/sessions.js +150 -0
- package/dist/types.js +9 -0
- package/dist/util/debug.js +79 -0
- package/dist/util/errors.js +157 -0
- package/dist/util/prompt.js +111 -0
- package/dist/util/secrets.js +110 -0
- package/dist/util/text.js +53 -0
- package/dist/util/time.js +25 -0
- package/dist/verify/runner.js +437 -0
- package/package.json +69 -0
- package/specs/all-mode.yaml +44 -0
- package/specs/behavior/auto-fallback.yaml +49 -0
- package/specs/behavior/bare-name-routing.yaml +223 -0
- package/specs/behavior/bare-paste-confirm.yaml +125 -0
- package/specs/behavior/env-var-respected.yaml +108 -0
- package/specs/behavior/error-fidelity.yaml +92 -0
- package/specs/behavior/error-hints.yaml +160 -0
- package/specs/behavior/fresh-vs-resume.yaml +94 -0
- package/specs/behavior/fuzzy-match.yaml +208 -0
- package/specs/behavior/host-self-knowledge-fresh.yaml +66 -0
- package/specs/behavior/intent-no-mismatch.yaml +115 -0
- package/specs/behavior/login-logout.yaml +97 -0
- package/specs/behavior/no-model-allowlist.yaml +80 -0
- package/specs/behavior/paste-key.yaml +342 -0
- package/specs/behavior/provider-switching.yaml +186 -0
- package/specs/behavior/providers-json-respected.yaml +106 -0
- package/specs/behavior/self-knowledge.yaml +119 -0
- package/specs/behavior/stress-session.yaml +226 -0
- package/specs/behavior/switch-back-when-failing.yaml +90 -0
- package/specs/behavior/work-character.yaml +109 -0
- package/specs/chat-meta.yaml +349 -0
- package/specs/chat-startup.yaml +148 -0
- package/specs/chat.yaml +91 -0
- package/specs/config.yaml +42 -0
- package/specs/install.yaml +112 -0
- package/specs/keys.yaml +81 -0
- package/specs/one-shot.yaml +65 -0
- package/specs/pipe-and-files.yaml +40 -0
- package/specs/providers.yaml +172 -0
- package/specs/sessions.yaml +115 -0
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
name: behavior — provider switching
|
|
2
|
+
description: |
|
|
3
|
+
Tests what the user OBSERVES when they ask to switch providers. Covers
|
|
4
|
+
routing (does "use codex" actually go to codex?), banner correctness (does
|
|
5
|
+
the banner reflect the real target?), and worker character (does the
|
|
6
|
+
switched-to provider stay in character or impersonate mod8 host?).
|
|
7
|
+
|
|
8
|
+
The chat REPL itself crashes on raw-mode in our sandbox, so these tests
|
|
9
|
+
exercise the same code paths via two non-Ink endpoints:
|
|
10
|
+
- mod8 dev:resolve <input> → routing intent (no LLM)
|
|
11
|
+
- mod8 dev:work-ask <id> <p> → work-mode system prompt + provider call
|
|
12
|
+
|
|
13
|
+
tests:
|
|
14
|
+
- name: routing — "use codex" (custom display name) maps to openai
|
|
15
|
+
setup:
|
|
16
|
+
- shell: |
|
|
17
|
+
mkdir -p $MOD8_CONFIG_DIR
|
|
18
|
+
cat > $MOD8_CONFIG_DIR/providers.json <<JSON
|
|
19
|
+
{
|
|
20
|
+
"openai": {
|
|
21
|
+
"apiKey": "sk-fake",
|
|
22
|
+
"apiType": "openai-compat",
|
|
23
|
+
"name": "codex",
|
|
24
|
+
"baseUrl": "https://api.openai.com/v1",
|
|
25
|
+
"defaultModel": "gpt-4o",
|
|
26
|
+
"color": "#10B981"
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
JSON
|
|
30
|
+
chmod 600 $MOD8_CONFIG_DIR/providers.json
|
|
31
|
+
shell: 'mod8 dev:resolve "use codex"'
|
|
32
|
+
expect:
|
|
33
|
+
stdout_contains: "route id=codex resolved=openai"
|
|
34
|
+
|
|
35
|
+
- name: routing — "i want to talk with codex" (the exact phrasing that broke prod) routes
|
|
36
|
+
setup:
|
|
37
|
+
- shell: |
|
|
38
|
+
mkdir -p $MOD8_CONFIG_DIR
|
|
39
|
+
cat > $MOD8_CONFIG_DIR/providers.json <<JSON
|
|
40
|
+
{
|
|
41
|
+
"openai": {
|
|
42
|
+
"apiKey": "sk-fake",
|
|
43
|
+
"apiType": "openai-compat",
|
|
44
|
+
"name": "codex",
|
|
45
|
+
"baseUrl": "https://api.openai.com/v1",
|
|
46
|
+
"defaultModel": "gpt-4o",
|
|
47
|
+
"color": "#10B981"
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
JSON
|
|
51
|
+
chmod 600 $MOD8_CONFIG_DIR/providers.json
|
|
52
|
+
shell: 'mod8 dev:resolve "i want to talk with codex"'
|
|
53
|
+
expect:
|
|
54
|
+
stdout_contains: "route id=codex resolved=openai"
|
|
55
|
+
|
|
56
|
+
- name: routing — "i wanna chat with gpt" maps to openai
|
|
57
|
+
shell: 'mod8 dev:resolve "i wanna chat with gpt"'
|
|
58
|
+
expect:
|
|
59
|
+
stdout_contains: "route id=gpt resolved=openai"
|
|
60
|
+
|
|
61
|
+
- name: routing — "speak with grok" routes to xai
|
|
62
|
+
shell: 'mod8 dev:resolve "speak with grok"'
|
|
63
|
+
expect:
|
|
64
|
+
stdout_contains: "route id=grok resolved=xai"
|
|
65
|
+
|
|
66
|
+
- name: routing — "talk to mistral" routes to mistral
|
|
67
|
+
shell: 'mod8 dev:resolve "talk to mistral"'
|
|
68
|
+
expect:
|
|
69
|
+
stdout_contains: "route id=mistral resolved=mistral"
|
|
70
|
+
|
|
71
|
+
- name: routing — unknown provider returns null with no false positive
|
|
72
|
+
shell: 'mod8 dev:resolve "use thisproviderdoesnotexist"'
|
|
73
|
+
expect:
|
|
74
|
+
stdout_contains: "route id=thisproviderdoesnotexist resolved=null"
|
|
75
|
+
|
|
76
|
+
- name: routing — non-route plain text falls through cleanly
|
|
77
|
+
shell: 'mod8 dev:resolve "what is the weather today"'
|
|
78
|
+
expect:
|
|
79
|
+
stdout_contains: "none"
|
|
80
|
+
|
|
81
|
+
- name: 'routing — "lets me talk to codex" (no-apostrophe typo) still routes'
|
|
82
|
+
setup:
|
|
83
|
+
- shell: |
|
|
84
|
+
mkdir -p $MOD8_CONFIG_DIR
|
|
85
|
+
cat > $MOD8_CONFIG_DIR/providers.json <<JSON
|
|
86
|
+
{
|
|
87
|
+
"openai": {
|
|
88
|
+
"apiKey": "sk-fake",
|
|
89
|
+
"apiType": "openai-compat",
|
|
90
|
+
"name": "codex",
|
|
91
|
+
"baseUrl": "https://api.openai.com/v1",
|
|
92
|
+
"defaultModel": "gpt-4o",
|
|
93
|
+
"color": "#10B981"
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
JSON
|
|
97
|
+
chmod 600 $MOD8_CONFIG_DIR/providers.json
|
|
98
|
+
shell: 'mod8 dev:resolve "lets me talk to codex"'
|
|
99
|
+
expect:
|
|
100
|
+
stdout_contains: "route id=codex resolved=openai"
|
|
101
|
+
|
|
102
|
+
- name: 'routing — "lets talk to grok" (lets without apostrophe) routes'
|
|
103
|
+
shell: 'mod8 dev:resolve "lets talk to grok"'
|
|
104
|
+
expect:
|
|
105
|
+
stdout_contains: "route id=grok resolved=xai"
|
|
106
|
+
|
|
107
|
+
- name: 'routing — "let me chat with claude" routes (canonical phrasing)'
|
|
108
|
+
shell: 'mod8 dev:resolve "let me chat with claude"'
|
|
109
|
+
expect:
|
|
110
|
+
stdout_contains: "route id=claude resolved=anthropic"
|
|
111
|
+
|
|
112
|
+
# --- WORK-MODE CHARACTER TESTS (LLM-driven) ---
|
|
113
|
+
|
|
114
|
+
- name: work — meta question is deferred back to host, not impersonated
|
|
115
|
+
requires_api_key: true
|
|
116
|
+
setup:
|
|
117
|
+
- shell: |
|
|
118
|
+
mkdir -p $MOD8_CONFIG_DIR
|
|
119
|
+
cat > $MOD8_CONFIG_DIR/providers.json <<JSON
|
|
120
|
+
{
|
|
121
|
+
"anthropic": {
|
|
122
|
+
"apiKey": "$ANTHROPIC_API_KEY",
|
|
123
|
+
"apiType": "anthropic",
|
|
124
|
+
"name": "Anthropic (Claude)",
|
|
125
|
+
"defaultModel": "claude-sonnet-4-6",
|
|
126
|
+
"color": "#A78BFA"
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
JSON
|
|
130
|
+
chmod 600 $MOD8_CONFIG_DIR/providers.json
|
|
131
|
+
shell: 'mod8 dev:work-ask anthropic "what providers do I have configured?"'
|
|
132
|
+
expect:
|
|
133
|
+
# Work mode must NOT pretend to know mod8 config — it should defer.
|
|
134
|
+
stdout_matches: "(?i)(handing back|hand.+back|mod8|host|/mod8)"
|
|
135
|
+
stdout_omits:
|
|
136
|
+
- "You have"
|
|
137
|
+
- "configured providers"
|
|
138
|
+
|
|
139
|
+
- name: work — concrete coding task is answered, not deferred
|
|
140
|
+
requires_api_key: true
|
|
141
|
+
setup:
|
|
142
|
+
- shell: |
|
|
143
|
+
mkdir -p $MOD8_CONFIG_DIR
|
|
144
|
+
cat > $MOD8_CONFIG_DIR/providers.json <<JSON
|
|
145
|
+
{
|
|
146
|
+
"anthropic": {
|
|
147
|
+
"apiKey": "$ANTHROPIC_API_KEY",
|
|
148
|
+
"apiType": "anthropic",
|
|
149
|
+
"name": "Anthropic (Claude)",
|
|
150
|
+
"defaultModel": "claude-sonnet-4-6",
|
|
151
|
+
"color": "#A78BFA"
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
JSON
|
|
155
|
+
chmod 600 $MOD8_CONFIG_DIR/providers.json
|
|
156
|
+
shell: 'mod8 dev:work-ask anthropic "write a one-line python lambda that doubles a number"'
|
|
157
|
+
expect:
|
|
158
|
+
# Real work happens — code shows up, no hand-off token.
|
|
159
|
+
stdout_contains: "lambda"
|
|
160
|
+
stdout_omits:
|
|
161
|
+
- "handing back"
|
|
162
|
+
- "<SWITCH_TO_HOST>"
|
|
163
|
+
|
|
164
|
+
- name: work — claude does not claim to be mod8 when asked
|
|
165
|
+
requires_api_key: true
|
|
166
|
+
setup:
|
|
167
|
+
- shell: |
|
|
168
|
+
mkdir -p $MOD8_CONFIG_DIR
|
|
169
|
+
cat > $MOD8_CONFIG_DIR/providers.json <<JSON
|
|
170
|
+
{
|
|
171
|
+
"anthropic": {
|
|
172
|
+
"apiKey": "$ANTHROPIC_API_KEY",
|
|
173
|
+
"apiType": "anthropic",
|
|
174
|
+
"name": "Anthropic (Claude)",
|
|
175
|
+
"defaultModel": "claude-sonnet-4-6",
|
|
176
|
+
"color": "#A78BFA"
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
JSON
|
|
180
|
+
chmod 600 $MOD8_CONFIG_DIR/providers.json
|
|
181
|
+
shell: 'mod8 dev:work-ask anthropic "who are you?"'
|
|
182
|
+
expect:
|
|
183
|
+
stdout_matches: "(?i)claude"
|
|
184
|
+
stdout_omits:
|
|
185
|
+
- "I am mod8"
|
|
186
|
+
- "I'm mod8"
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
name: behavior — providers.json defaultModel reaches the SDK URL
|
|
2
|
+
description: |
|
|
3
|
+
Bug confirmation: when the user wrote "gemini-2.5-flash" in
|
|
4
|
+
providers.json's defaultModel field, mod8 was supposed to pass that
|
|
5
|
+
exact value to the Google SDK. The dead code in src/providers/google.ts
|
|
6
|
+
hard-coded gemini-2.0-flash which made it look like there was a swap;
|
|
7
|
+
the real path in generic.ts already used entry.defaultModel, but no
|
|
8
|
+
tests proved it end-to-end.
|
|
9
|
+
|
|
10
|
+
These specs lock in the contract: whatever string the user puts in
|
|
11
|
+
providers.json is what reaches the provider URL. No allowlist, no
|
|
12
|
+
silent substitution, no fallback to a built-in default.
|
|
13
|
+
|
|
14
|
+
tests:
|
|
15
|
+
- name: 'providers.json defaultModel is used when no env override'
|
|
16
|
+
setup:
|
|
17
|
+
- shell: |
|
|
18
|
+
mkdir -p $MOD8_CONFIG_DIR
|
|
19
|
+
cat > $MOD8_CONFIG_DIR/providers.json <<JSON
|
|
20
|
+
{
|
|
21
|
+
"google": {
|
|
22
|
+
"apiKey": "x",
|
|
23
|
+
"apiType": "gemini",
|
|
24
|
+
"name": "Google (Gemini)",
|
|
25
|
+
"defaultModel": "gemini-2.5-flash",
|
|
26
|
+
"color": "#06B6D4"
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
JSON
|
|
30
|
+
chmod 600 $MOD8_CONFIG_DIR/providers.json
|
|
31
|
+
shell: 'mod8 dev:resolve-model google'
|
|
32
|
+
expect:
|
|
33
|
+
stdout_contains:
|
|
34
|
+
- 'model="gemini-2.5-flash"'
|
|
35
|
+
- 'source=providers.json'
|
|
36
|
+
|
|
37
|
+
- name: 'providers.json model passes through to the SDK URL'
|
|
38
|
+
setup:
|
|
39
|
+
- shell: |
|
|
40
|
+
mkdir -p $MOD8_CONFIG_DIR
|
|
41
|
+
cat > $MOD8_CONFIG_DIR/providers.json <<JSON
|
|
42
|
+
{
|
|
43
|
+
"google": {
|
|
44
|
+
"apiKey": "AIzaSy-FAKE-NOT-REAL-AAAAAAAAAAAAAAAAAAAAAAA",
|
|
45
|
+
"apiType": "gemini",
|
|
46
|
+
"name": "Google (Gemini)",
|
|
47
|
+
"defaultModel": "gemini-2.5-flash",
|
|
48
|
+
"color": "#06B6D4"
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
JSON
|
|
52
|
+
chmod 600 $MOD8_CONFIG_DIR/providers.json
|
|
53
|
+
shell: 'mod8 dev:debug-call google'
|
|
54
|
+
expect:
|
|
55
|
+
stdout_contains:
|
|
56
|
+
- 'model="gemini-2.5-flash"'
|
|
57
|
+
- 'modelSource=providers.json'
|
|
58
|
+
# The exact SDK URL — not a hint, not a placeholder.
|
|
59
|
+
- 'models/gemini-2.5-flash:streamGenerateContent'
|
|
60
|
+
|
|
61
|
+
- name: 'arbitrary user-chosen model name is preserved in the URL (no allowlist)'
|
|
62
|
+
setup:
|
|
63
|
+
- shell: |
|
|
64
|
+
mkdir -p $MOD8_CONFIG_DIR
|
|
65
|
+
cat > $MOD8_CONFIG_DIR/providers.json <<JSON
|
|
66
|
+
{
|
|
67
|
+
"google": {
|
|
68
|
+
"apiKey": "AIzaSy-FAKE-NOT-REAL-AAAAAAAAAAAAAAAAAAAAAAA",
|
|
69
|
+
"apiType": "gemini",
|
|
70
|
+
"name": "Google (Gemini)",
|
|
71
|
+
"defaultModel": "gemini-3.0-experimental-future-model",
|
|
72
|
+
"color": "#06B6D4"
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
JSON
|
|
76
|
+
chmod 600 $MOD8_CONFIG_DIR/providers.json
|
|
77
|
+
shell: 'mod8 dev:debug-call google'
|
|
78
|
+
expect:
|
|
79
|
+
stdout_contains:
|
|
80
|
+
# Whatever the user wrote — even something that doesn't exist yet —
|
|
81
|
+
# appears in the SDK URL. No allowlist, no substitution.
|
|
82
|
+
- 'models/gemini-3.0-experimental-future-model'
|
|
83
|
+
- 'model="gemini-3.0-experimental-future-model"'
|
|
84
|
+
|
|
85
|
+
- name: 'openai-compat custom baseUrl + custom model both reach the URL'
|
|
86
|
+
setup:
|
|
87
|
+
- shell: |
|
|
88
|
+
mkdir -p $MOD8_CONFIG_DIR
|
|
89
|
+
cat > $MOD8_CONFIG_DIR/providers.json <<JSON
|
|
90
|
+
{
|
|
91
|
+
"deepseek": {
|
|
92
|
+
"apiKey": "sk-fake",
|
|
93
|
+
"apiType": "openai-compat",
|
|
94
|
+
"name": "DeepSeek",
|
|
95
|
+
"baseUrl": "https://api.deepseek.com",
|
|
96
|
+
"defaultModel": "deepseek-reasoner-some-future-variant",
|
|
97
|
+
"color": "#3B82F6"
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
JSON
|
|
101
|
+
chmod 600 $MOD8_CONFIG_DIR/providers.json
|
|
102
|
+
shell: 'MOD8_DEBUG=1 mod8 dev:resolve-model deepseek'
|
|
103
|
+
expect:
|
|
104
|
+
stdout_contains:
|
|
105
|
+
- 'model="deepseek-reasoner-some-future-variant"'
|
|
106
|
+
- 'source=providers.json'
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
name: behavior — mod8 has self-knowledge from configured state
|
|
2
|
+
description: |
|
|
3
|
+
Tests that the host's answers to meta questions reference the user's
|
|
4
|
+
ACTUAL configured providers (not generic "I don't know" answers). This
|
|
5
|
+
is the regression suite for the "mod8 doesn't know mod8" bug.
|
|
6
|
+
|
|
7
|
+
Each test seeds providers.json with realistic state, asks a meta
|
|
8
|
+
question via mod8 dev:host-ask, and asserts the response references
|
|
9
|
+
the configured provider names / ids / models.
|
|
10
|
+
|
|
11
|
+
tests:
|
|
12
|
+
- name: with custom-named openai, "what providers" lists "codex"
|
|
13
|
+
requires_api_key: true
|
|
14
|
+
setup:
|
|
15
|
+
- shell: |
|
|
16
|
+
mkdir -p $MOD8_CONFIG_DIR
|
|
17
|
+
cat > $MOD8_CONFIG_DIR/providers.json <<JSON
|
|
18
|
+
{
|
|
19
|
+
"anthropic": {
|
|
20
|
+
"apiKey": "$ANTHROPIC_API_KEY",
|
|
21
|
+
"apiType": "anthropic",
|
|
22
|
+
"name": "Anthropic (Claude)",
|
|
23
|
+
"defaultModel": "claude-sonnet-4-6",
|
|
24
|
+
"color": "#A78BFA"
|
|
25
|
+
},
|
|
26
|
+
"openai": {
|
|
27
|
+
"apiKey": "sk-fake",
|
|
28
|
+
"apiType": "openai-compat",
|
|
29
|
+
"name": "codex",
|
|
30
|
+
"baseUrl": "https://api.openai.com/v1",
|
|
31
|
+
"defaultModel": "gpt-4o",
|
|
32
|
+
"color": "#10B981"
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
JSON
|
|
36
|
+
chmod 600 $MOD8_CONFIG_DIR/providers.json
|
|
37
|
+
shell: 'mod8 dev:host-ask "what providers do I have?"'
|
|
38
|
+
expect:
|
|
39
|
+
stdout_matches:
|
|
40
|
+
- "(?i)codex"
|
|
41
|
+
- "(?i)(anthropic|claude)"
|
|
42
|
+
|
|
43
|
+
- name: '"how many operators" answers with the right count (2)'
|
|
44
|
+
requires_api_key: true
|
|
45
|
+
setup:
|
|
46
|
+
- shell: |
|
|
47
|
+
mkdir -p $MOD8_CONFIG_DIR
|
|
48
|
+
cat > $MOD8_CONFIG_DIR/providers.json <<JSON
|
|
49
|
+
{
|
|
50
|
+
"anthropic": {
|
|
51
|
+
"apiKey": "$ANTHROPIC_API_KEY",
|
|
52
|
+
"apiType": "anthropic",
|
|
53
|
+
"name": "Anthropic (Claude)",
|
|
54
|
+
"defaultModel": "claude-sonnet-4-6",
|
|
55
|
+
"color": "#A78BFA"
|
|
56
|
+
},
|
|
57
|
+
"openai": {
|
|
58
|
+
"apiKey": "sk-fake",
|
|
59
|
+
"apiType": "openai-compat",
|
|
60
|
+
"name": "codex",
|
|
61
|
+
"baseUrl": "https://api.openai.com/v1",
|
|
62
|
+
"defaultModel": "gpt-4o",
|
|
63
|
+
"color": "#10B981"
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
JSON
|
|
67
|
+
chmod 600 $MOD8_CONFIG_DIR/providers.json
|
|
68
|
+
shell: 'mod8 dev:host-ask "how many operators are you connected to?"'
|
|
69
|
+
expect:
|
|
70
|
+
stdout_contains: "2"
|
|
71
|
+
stdout_omits:
|
|
72
|
+
- "I don't have"
|
|
73
|
+
- "tell me about your project"
|
|
74
|
+
|
|
75
|
+
- name: with 3 providers, host enumerates them all
|
|
76
|
+
requires_api_key: true
|
|
77
|
+
setup:
|
|
78
|
+
- shell: |
|
|
79
|
+
mkdir -p $MOD8_CONFIG_DIR
|
|
80
|
+
cat > $MOD8_CONFIG_DIR/providers.json <<JSON
|
|
81
|
+
{
|
|
82
|
+
"anthropic": {
|
|
83
|
+
"apiKey": "$ANTHROPIC_API_KEY",
|
|
84
|
+
"apiType": "anthropic",
|
|
85
|
+
"name": "Anthropic (Claude)",
|
|
86
|
+
"defaultModel": "claude-sonnet-4-6",
|
|
87
|
+
"color": "#A78BFA"
|
|
88
|
+
},
|
|
89
|
+
"openai": {
|
|
90
|
+
"apiKey": "sk-fake",
|
|
91
|
+
"apiType": "openai-compat",
|
|
92
|
+
"name": "codex",
|
|
93
|
+
"baseUrl": "https://api.openai.com/v1",
|
|
94
|
+
"defaultModel": "gpt-4o",
|
|
95
|
+
"color": "#10B981"
|
|
96
|
+
},
|
|
97
|
+
"deepseek": {
|
|
98
|
+
"apiKey": "sk-fake",
|
|
99
|
+
"apiType": "openai-compat",
|
|
100
|
+
"name": "DeepSeek",
|
|
101
|
+
"baseUrl": "https://api.deepseek.com",
|
|
102
|
+
"defaultModel": "deepseek-chat",
|
|
103
|
+
"color": "#3B82F6"
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
JSON
|
|
107
|
+
chmod 600 $MOD8_CONFIG_DIR/providers.json
|
|
108
|
+
shell: 'mod8 dev:host-ask "list all my providers"'
|
|
109
|
+
expect:
|
|
110
|
+
stdout_matches:
|
|
111
|
+
- "(?i)(anthropic|claude)"
|
|
112
|
+
- "(?i)codex"
|
|
113
|
+
- "(?i)deepseek"
|
|
114
|
+
|
|
115
|
+
- name: empty config — host says so, doesn't pretend
|
|
116
|
+
requires_api_key: true
|
|
117
|
+
shell: 'mod8 dev:host-ask "what platforms are connected?"'
|
|
118
|
+
expect:
|
|
119
|
+
stdout_matches: "(?i)(none|nothing|no providers|0 configured|not configured|empty|haven't)"
|
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
name: behavior — stress test (long realistic session)
|
|
2
|
+
description: |
|
|
3
|
+
Simulates a 25-input session that hits 5 configured providers (anthropic,
|
|
4
|
+
openai/codex, deepseek, mistral, xai/grok) using mixed natural phrasings
|
|
5
|
+
the user actually types: bare names, greetings, "use X", "talk with X",
|
|
6
|
+
"let me chat with X", "i wanna talk with X", first-word + rest, "back",
|
|
7
|
+
"mod8", "/mod8", "@mod8".
|
|
8
|
+
|
|
9
|
+
Each input is fed through dev:simulate which applies the same routing
|
|
10
|
+
state machine the chat REPL uses. We assert the action and final
|
|
11
|
+
(mode, provider) for every step, so any drift in routing — silent
|
|
12
|
+
no-routing, wrong provider, banner-vs-text mismatch via stuck workId,
|
|
13
|
+
or false positives — fails the suite.
|
|
14
|
+
|
|
15
|
+
tests:
|
|
16
|
+
- name: 25-input session — 5 providers, mixed phrasings, no stuck states
|
|
17
|
+
setup:
|
|
18
|
+
- shell: |
|
|
19
|
+
mkdir -p $MOD8_CONFIG_DIR
|
|
20
|
+
cat > $MOD8_CONFIG_DIR/providers.json <<JSON
|
|
21
|
+
{
|
|
22
|
+
"anthropic": {
|
|
23
|
+
"apiKey": "sk-fake-ant",
|
|
24
|
+
"apiType": "anthropic",
|
|
25
|
+
"name": "Anthropic (Claude)",
|
|
26
|
+
"defaultModel": "claude-sonnet-4-6",
|
|
27
|
+
"color": "#A78BFA"
|
|
28
|
+
},
|
|
29
|
+
"openai": {
|
|
30
|
+
"apiKey": "sk-fake-oai",
|
|
31
|
+
"apiType": "openai-compat",
|
|
32
|
+
"name": "codex",
|
|
33
|
+
"baseUrl": "https://api.openai.com/v1",
|
|
34
|
+
"defaultModel": "gpt-4o",
|
|
35
|
+
"color": "#10B981"
|
|
36
|
+
},
|
|
37
|
+
"deepseek": {
|
|
38
|
+
"apiKey": "sk-fake-ds",
|
|
39
|
+
"apiType": "openai-compat",
|
|
40
|
+
"name": "DeepSeek",
|
|
41
|
+
"baseUrl": "https://api.deepseek.com",
|
|
42
|
+
"defaultModel": "deepseek-chat",
|
|
43
|
+
"color": "#3B82F6"
|
|
44
|
+
},
|
|
45
|
+
"mistral": {
|
|
46
|
+
"apiKey": "sk-fake-mi",
|
|
47
|
+
"apiType": "openai-compat",
|
|
48
|
+
"name": "Mistral",
|
|
49
|
+
"baseUrl": "https://api.mistral.ai/v1",
|
|
50
|
+
"defaultModel": "mistral-large-latest",
|
|
51
|
+
"color": "#EF4444"
|
|
52
|
+
},
|
|
53
|
+
"xai": {
|
|
54
|
+
"apiKey": "sk-fake-xai",
|
|
55
|
+
"apiType": "openai-compat",
|
|
56
|
+
"name": "xAI (Grok)",
|
|
57
|
+
"baseUrl": "https://api.x.ai/v1",
|
|
58
|
+
"defaultModel": "grok-2-latest",
|
|
59
|
+
"color": "#6B7280"
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
JSON
|
|
63
|
+
chmod 600 $MOD8_CONFIG_DIR/providers.json
|
|
64
|
+
shell: |
|
|
65
|
+
mod8 dev:simulate <<'INPUTS'
|
|
66
|
+
hi
|
|
67
|
+
codex
|
|
68
|
+
hello
|
|
69
|
+
mod8
|
|
70
|
+
use grok
|
|
71
|
+
tell me about JS
|
|
72
|
+
back
|
|
73
|
+
let me chat with deepseek
|
|
74
|
+
hello
|
|
75
|
+
switch to mistral
|
|
76
|
+
please continue
|
|
77
|
+
/mod8
|
|
78
|
+
i wanna talk with codex
|
|
79
|
+
hi
|
|
80
|
+
back to mod8
|
|
81
|
+
claude please
|
|
82
|
+
thanks
|
|
83
|
+
/mod8
|
|
84
|
+
talk with grok
|
|
85
|
+
hello there
|
|
86
|
+
gpt help me
|
|
87
|
+
mod8
|
|
88
|
+
hi mistral
|
|
89
|
+
yo
|
|
90
|
+
switch back
|
|
91
|
+
INPUTS
|
|
92
|
+
expect:
|
|
93
|
+
stdout_contains:
|
|
94
|
+
# Step 1: bare "hi" in host → send (greeting host)
|
|
95
|
+
- 'step=1 input="hi" mode=host provider=host action=send'
|
|
96
|
+
# Step 2: "codex" routes via bare-name → openai (custom display)
|
|
97
|
+
- 'step=2 input="codex" mode=work provider=openai action=route-bare'
|
|
98
|
+
# Step 3: "hello" in work/codex → send (chat with codex)
|
|
99
|
+
- 'step=3 input="hello" mode=work provider=openai action=send'
|
|
100
|
+
# Step 4: "mod8" → host-back, workId reset to anthropic
|
|
101
|
+
- 'step=4 input="mod8" mode=host provider=host action=host-back'
|
|
102
|
+
# Step 5: "use grok" → xai (configured)
|
|
103
|
+
- 'step=5 input="use grok" mode=work provider=xai action=route'
|
|
104
|
+
# Step 6: "tell me about JS" in work/xai → send
|
|
105
|
+
- 'step=6 input="tell me about JS" mode=work provider=xai action=send'
|
|
106
|
+
# Step 7: "back" → host-back (resets workId)
|
|
107
|
+
- 'step=7 input="back" mode=host provider=host action=host-back'
|
|
108
|
+
# Step 8: "let me chat with deepseek" → deepseek
|
|
109
|
+
- 'step=8 input="let me chat with deepseek" mode=work provider=deepseek action=route'
|
|
110
|
+
# Step 9: "hello" → send to deepseek
|
|
111
|
+
- 'step=9 input="hello" mode=work provider=deepseek action=send'
|
|
112
|
+
# Step 10: "switch to mistral" → mistral
|
|
113
|
+
- 'step=10 input="switch to mistral" mode=work provider=mistral action=route'
|
|
114
|
+
# Step 11: "please continue" → send to mistral
|
|
115
|
+
- 'step=11 input="please continue" mode=work provider=mistral action=send'
|
|
116
|
+
# Step 12: "/mod8" → host-back from work
|
|
117
|
+
- 'step=12 input="/mod8" mode=host provider=host action=host-back'
|
|
118
|
+
# Step 13: "i wanna talk with codex" → openai (typo-tolerant + synonym/display-name)
|
|
119
|
+
- 'step=13 input="i wanna talk with codex" mode=work provider=openai action=route'
|
|
120
|
+
# Step 14: "hi" → send to codex
|
|
121
|
+
- 'step=14 input="hi" mode=work provider=openai action=send'
|
|
122
|
+
# Step 15: "back to mod8" → host-back
|
|
123
|
+
- 'step=15 input="back to mod8" mode=host provider=host action=host-back'
|
|
124
|
+
# Step 16: "claude please" → anthropic via high-confidence brand alias
|
|
125
|
+
- 'step=16 input="claude please" mode=work provider=anthropic action=route-bare'
|
|
126
|
+
# Step 17: "thanks" → send to anthropic
|
|
127
|
+
- 'step=17 input="thanks" mode=work provider=anthropic action=send'
|
|
128
|
+
# Step 18: "/mod8" → host-back from anthropic, workId reset
|
|
129
|
+
- 'step=18 input="/mod8" mode=host provider=host action=host-back'
|
|
130
|
+
# Step 19: "talk with grok" → xai
|
|
131
|
+
- 'step=19 input="talk with grok" mode=work provider=xai action=route'
|
|
132
|
+
# Step 20: "hello there" → send to xai (greeting + non-provider word, falls through)
|
|
133
|
+
- 'step=20 input="hello there" mode=work provider=xai action=send'
|
|
134
|
+
# Step 21: "gpt help me" → openai (high-confidence first-word)
|
|
135
|
+
- 'step=21 input="gpt help me" mode=work provider=openai action=route-bare'
|
|
136
|
+
# Step 22: "mod8" → host-back, workId reset
|
|
137
|
+
- 'step=22 input="mod8" mode=host provider=host action=host-back'
|
|
138
|
+
# Step 23: "hi mistral" → mistral via greeting (full resolution)
|
|
139
|
+
- 'step=23 input="hi mistral" mode=work provider=mistral action=route-greeting'
|
|
140
|
+
# Step 24: "yo" alone → send (yo isn't a provider)
|
|
141
|
+
- 'step=24 input="yo" mode=work provider=mistral action=send'
|
|
142
|
+
# Step 25: "switch back" → host-back, workId reset
|
|
143
|
+
- 'step=25 input="switch back" mode=host provider=host action=host-back'
|
|
144
|
+
|
|
145
|
+
- name: 'workProviderId resets after host-back, so SWITCH_TO_WORK lands on default'
|
|
146
|
+
setup:
|
|
147
|
+
- shell: |
|
|
148
|
+
mkdir -p $MOD8_CONFIG_DIR
|
|
149
|
+
cat > $MOD8_CONFIG_DIR/providers.json <<JSON
|
|
150
|
+
{
|
|
151
|
+
"anthropic": {"apiKey":"x","apiType":"anthropic","name":"Anthropic (Claude)","defaultModel":"claude-sonnet-4-6","color":"#A78BFA"},
|
|
152
|
+
"openai": {"apiKey":"x","apiType":"openai-compat","name":"codex","baseUrl":"https://api.openai.com/v1","defaultModel":"gpt-4o","color":"#10B981"}
|
|
153
|
+
}
|
|
154
|
+
JSON
|
|
155
|
+
chmod 600 $MOD8_CONFIG_DIR/providers.json
|
|
156
|
+
shell: |
|
|
157
|
+
mod8 dev:simulate <<'INPUTS'
|
|
158
|
+
use codex
|
|
159
|
+
hi
|
|
160
|
+
mod8
|
|
161
|
+
use codex
|
|
162
|
+
mod8
|
|
163
|
+
INPUTS
|
|
164
|
+
expect:
|
|
165
|
+
stdout_contains:
|
|
166
|
+
# After host-back, provider=host (workId reset to anthropic, but mode=host shows "host")
|
|
167
|
+
- 'step=3 input="mod8" mode=host provider=host action=host-back'
|
|
168
|
+
# Re-routing to codex still works
|
|
169
|
+
- 'step=4 input="use codex" mode=work provider=openai action=route'
|
|
170
|
+
# Final mod8 → back to host
|
|
171
|
+
- 'step=5 input="mod8" mode=host provider=host action=host-back'
|
|
172
|
+
|
|
173
|
+
- name: false-positive avoidance — common english words near providers
|
|
174
|
+
setup:
|
|
175
|
+
- shell: |
|
|
176
|
+
mkdir -p $MOD8_CONFIG_DIR
|
|
177
|
+
cat > $MOD8_CONFIG_DIR/providers.json <<JSON
|
|
178
|
+
{
|
|
179
|
+
"anthropic": {"apiKey":"x","apiType":"anthropic","name":"Anthropic (Claude)","defaultModel":"claude-sonnet-4-6","color":"#A78BFA"}
|
|
180
|
+
}
|
|
181
|
+
JSON
|
|
182
|
+
chmod 600 $MOD8_CONFIG_DIR/providers.json
|
|
183
|
+
shell: |
|
|
184
|
+
mod8 dev:simulate <<'INPUTS'
|
|
185
|
+
write me a haiku
|
|
186
|
+
hello world
|
|
187
|
+
tell me about sonnet form
|
|
188
|
+
can you write opus?
|
|
189
|
+
llama farm
|
|
190
|
+
INPUTS
|
|
191
|
+
expect:
|
|
192
|
+
stdout_contains:
|
|
193
|
+
# haiku/sonnet/opus/llama are ambiguous synonyms — strict mode
|
|
194
|
+
# rejects them, so all of these stay in host mode as send actions.
|
|
195
|
+
- 'step=1 input="write me a haiku" mode=host provider=host action=send'
|
|
196
|
+
- 'step=2 input="hello world" mode=host provider=host action=send'
|
|
197
|
+
- 'step=3 input="tell me about sonnet form" mode=host provider=host action=send'
|
|
198
|
+
- 'step=4 input="can you write opus?" mode=host provider=host action=send'
|
|
199
|
+
- 'step=5 input="llama farm" mode=host provider=host action=send'
|
|
200
|
+
|
|
201
|
+
- name: host-back no-op when already in host (does NOT send /mod8 to LLM)
|
|
202
|
+
setup:
|
|
203
|
+
- shell: |
|
|
204
|
+
mkdir -p $MOD8_CONFIG_DIR
|
|
205
|
+
cat > $MOD8_CONFIG_DIR/providers.json <<JSON
|
|
206
|
+
{
|
|
207
|
+
"anthropic": {"apiKey":"x","apiType":"anthropic","name":"Anthropic (Claude)","defaultModel":"claude-sonnet-4-6","color":"#A78BFA"}
|
|
208
|
+
}
|
|
209
|
+
JSON
|
|
210
|
+
chmod 600 $MOD8_CONFIG_DIR/providers.json
|
|
211
|
+
shell: |
|
|
212
|
+
mod8 dev:simulate <<'INPUTS'
|
|
213
|
+
/mod8
|
|
214
|
+
@mod8
|
|
215
|
+
back
|
|
216
|
+
back to mod8
|
|
217
|
+
switch back
|
|
218
|
+
INPUTS
|
|
219
|
+
expect:
|
|
220
|
+
stdout_contains:
|
|
221
|
+
# All host-back patterns in host mode → no-op (NOT send)
|
|
222
|
+
- 'step=1 input="/mod8" mode=host provider=host action=host-back-noop'
|
|
223
|
+
- 'step=2 input="@mod8" mode=host provider=host action=host-back-noop'
|
|
224
|
+
- 'step=3 input="back" mode=host provider=host action=host-back-noop'
|
|
225
|
+
- 'step=4 input="back to mod8" mode=host provider=host action=host-back-noop'
|
|
226
|
+
- 'step=5 input="switch back" mode=host provider=host action=host-back-noop'
|