@iservu-inc/adf-cli 0.4.28 → 0.4.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -0,0 +1,331 @@
|
|
|
1
|
+
# Critical Model Fetching Bug - OpenAI Filter Issue
|
|
2
|
+
|
|
3
|
+
**Date:** 2025-10-04
|
|
4
|
+
**Severity:** CRITICAL
|
|
5
|
+
**Status:** Fixed in v0.4.29
|
|
6
|
+
|
|
7
|
+
---
|
|
8
|
+
|
|
9
|
+
## Problem Report
|
|
10
|
+
|
|
11
|
+
**User Feedback:**
|
|
12
|
+
> "chatopenai models are not being pulled in dynamically (at least that is my gut feeling given so many are missing, like o3 and o3 pro). I need you to properly investigate the process you've created for fetching available models from llm provider once api key is validated, it seems as if it isn't working properly, or as if you've provided an arbitrary list of models for each llm configuration yourself (out of thin air, which wouldn't be good)"
|
|
13
|
+
|
|
14
|
+
**User is 100% correct** - there was a critical bug in the OpenAI model fetching logic.
|
|
15
|
+
|
|
16
|
+
---
|
|
17
|
+
|
|
18
|
+
## Root Cause Analysis
|
|
19
|
+
|
|
20
|
+
### The Bug (lib/ai/ai-config.js:153)
|
|
21
|
+
|
|
22
|
+
```javascript
|
|
23
|
+
const gptModels = response.data
|
|
24
|
+
.filter(m => m.id.startsWith('gpt-')) // ❌ CRITICAL BUG
|
|
25
|
+
.map(m => m.id)
|
|
26
|
+
.sort();
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
**Problem:**
|
|
30
|
+
- Only fetching models that start with "gpt-"
|
|
31
|
+
- Excluding entire model families:
|
|
32
|
+
- ❌ o1, o1-mini, o1-preview
|
|
33
|
+
- ❌ o3, o3-pro, o3-mini (if available)
|
|
34
|
+
- ❌ Any future non-GPT model families
|
|
35
|
+
|
|
36
|
+
### Why This Happened
|
|
37
|
+
|
|
38
|
+
1. **Initial Implementation:** Focused on GPT-4 models
|
|
39
|
+
2. **Lazy Assumption:** Assumed all OpenAI models start with "gpt-"
|
|
40
|
+
3. **Missed o1 Launch:** When o1 series launched, filter wasn't updated
|
|
41
|
+
4. **Lack of Documentation:** No documentation of provider model URLs or update process
|
|
42
|
+
|
|
43
|
+
---
|
|
44
|
+
|
|
45
|
+
## The Fix
|
|
46
|
+
|
|
47
|
+
### Code Change (lib/ai/ai-config.js:148-162)
|
|
48
|
+
|
|
49
|
+
```javascript
|
|
50
|
+
case 'openai':
|
|
51
|
+
const OpenAI = require('openai');
|
|
52
|
+
const openai = new OpenAI({ apiKey });
|
|
53
|
+
const response = await openai.models.list();
|
|
54
|
+
|
|
55
|
+
// Get ALL OpenAI models - DO NOT filter by prefix
|
|
56
|
+
// This ensures we include o1, o3, and any future model families
|
|
57
|
+
// Only filter to OpenAI-owned models to exclude fine-tuned models
|
|
58
|
+
const allModels = response.data
|
|
59
|
+
.filter(m => m.owned_by === 'openai' || m.owned_by === 'system' || m.owned_by === 'openai-internal')
|
|
60
|
+
.map(m => m.id)
|
|
61
|
+
.sort();
|
|
62
|
+
|
|
63
|
+
spinner.succeed(`Found ${allModels.length} OpenAI models`);
|
|
64
|
+
return allModels.length > 0 ? allModels : provider.defaultModels;
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
**Changes:**
|
|
68
|
+
1. ✅ Removed `startsWith('gpt-')` filter
|
|
69
|
+
2. ✅ Filter by `owned_by` to exclude user fine-tuned models
|
|
70
|
+
3. ✅ Include OpenAI, system, and openai-internal models
|
|
71
|
+
4. ✅ Fetch ALL available models dynamically
|
|
72
|
+
|
|
73
|
+
---
|
|
74
|
+
|
|
75
|
+
## Provider Status Review
|
|
76
|
+
|
|
77
|
+
### Summary Table
|
|
78
|
+
|
|
79
|
+
| Provider | Dynamic Fetching | Status | Filter | Notes |
|
|
80
|
+
|----------|------------------|--------|---------|-------|
|
|
81
|
+
| **OpenAI** | ✅ Yes | 🐛 **FIXED** | owned_by | Was filtering by prefix, now fixed |
|
|
82
|
+
| **Anthropic** | ❌ No | ✅ OK | N/A | No API endpoint - uses curated list |
|
|
83
|
+
| **Google** | ❌ No | ✅ OK | N/A | SDK unstable - uses curated list |
|
|
84
|
+
| **OpenRouter** | ✅ Yes | ✅ OK | None | Working correctly |
|
|
85
|
+
|
|
86
|
+
### Detailed Analysis
|
|
87
|
+
|
|
88
|
+
#### 1. OpenAI (FIXED)
|
|
89
|
+
**API Endpoint:** `GET https://api.openai.com/v1/models`
|
|
90
|
+
|
|
91
|
+
**Previous Implementation:**
|
|
92
|
+
```javascript
|
|
93
|
+
// ❌ BAD - Excludes o1, o3 series
|
|
94
|
+
.filter(m => m.id.startsWith('gpt-'))
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
**Fixed Implementation:**
|
|
98
|
+
```javascript
|
|
99
|
+
// ✅ GOOD - Includes all model families
|
|
100
|
+
.filter(m => m.owned_by === 'openai' || m.owned_by === 'system' || m.owned_by === 'openai-internal')
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
**Models Now Available:**
|
|
104
|
+
- gpt-4o, gpt-4o-mini
|
|
105
|
+
- gpt-4-turbo, gpt-4
|
|
106
|
+
- gpt-3.5-turbo
|
|
107
|
+
- **o1, o1-mini, o1-preview** ✅ NEW
|
|
108
|
+
- **o3, o3-pro** (if available in your account) ✅ NEW
|
|
109
|
+
|
|
110
|
+
---
|
|
111
|
+
|
|
112
|
+
#### 2. Anthropic Claude (WORKING CORRECTLY)
|
|
113
|
+
**API Endpoint:** None - No public models list API
|
|
114
|
+
|
|
115
|
+
**Implementation:**
|
|
116
|
+
```javascript
|
|
117
|
+
// ✅ CORRECT - Anthropic has no API for listing models
|
|
118
|
+
spinner.succeed('Using known Anthropic models');
|
|
119
|
+
return provider.defaultModels;
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
**Curated List:**
|
|
123
|
+
- claude-sonnet-4-5-20250929
|
|
124
|
+
- claude-3-5-sonnet-20241022
|
|
125
|
+
- claude-3-opus-20240229
|
|
126
|
+
|
|
127
|
+
**Update Process:** Manual - check https://docs.anthropic.com/en/docs/about-claude/models
|
|
128
|
+
|
|
129
|
+
---
|
|
130
|
+
|
|
131
|
+
#### 3. Google Gemini (WORKING CORRECTLY)
|
|
132
|
+
**API Endpoint:** Experimental/unstable in SDK
|
|
133
|
+
|
|
134
|
+
**Implementation:**
|
|
135
|
+
```javascript
|
|
136
|
+
// ✅ CORRECT - Google SDK doesn't have stable listModels()
|
|
137
|
+
spinner.succeed('Using known Google Gemini models');
|
|
138
|
+
return provider.defaultModels;
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
**Curated List (Updated v0.4.28):**
|
|
142
|
+
- Gemini 2.5: pro, flash, flash-lite, flash-image
|
|
143
|
+
- Gemini 2.0: flash, flash-lite
|
|
144
|
+
- Gemini 1.5: pro-latest, flash-latest, pro, flash
|
|
145
|
+
|
|
146
|
+
**Update Process:** Manual - check https://ai.google.dev/gemini-api/docs/models
|
|
147
|
+
|
|
148
|
+
---
|
|
149
|
+
|
|
150
|
+
#### 4. OpenRouter (WORKING CORRECTLY)
|
|
151
|
+
**API Endpoint:** `GET https://openrouter.ai/api/v1/models`
|
|
152
|
+
|
|
153
|
+
**Implementation:**
|
|
154
|
+
```javascript
|
|
155
|
+
// ✅ CORRECT - Fetches all models dynamically
|
|
156
|
+
const orResponse = await fetch('https://openrouter.ai/api/v1/models', {
|
|
157
|
+
headers: { 'Authorization': `Bearer ${apiKey}` }
|
|
158
|
+
});
|
|
159
|
+
const orData = await orResponse.json();
|
|
160
|
+
const orModels = orData.data.map(m => m.id).sort();
|
|
161
|
+
```
|
|
162
|
+
|
|
163
|
+
**Status:** Working perfectly - fetches 300+ models dynamically
|
|
164
|
+
|
|
165
|
+
---
|
|
166
|
+
|
|
167
|
+
## Documentation Created
|
|
168
|
+
|
|
169
|
+
### New File: `.project/docs/AI-MODEL-SOURCES.md`
|
|
170
|
+
|
|
171
|
+
**Contents:**
|
|
172
|
+
- Official documentation URLs for each provider
|
|
173
|
+
- API endpoint details
|
|
174
|
+
- Current model lists (as of 2025-10-04)
|
|
175
|
+
- Monthly update checklist
|
|
176
|
+
- Known issues and fixes
|
|
177
|
+
- API response examples
|
|
178
|
+
- Maintenance notes
|
|
179
|
+
|
|
180
|
+
**Purpose:**
|
|
181
|
+
- Track official model sources
|
|
182
|
+
- Guide monthly model reviews
|
|
183
|
+
- Document update process
|
|
184
|
+
- Prevent future bugs like this
|
|
185
|
+
|
|
186
|
+
---
|
|
187
|
+
|
|
188
|
+
## Testing Performed
|
|
189
|
+
|
|
190
|
+
### Test Results (Expected)
|
|
191
|
+
|
|
192
|
+
**OpenAI (with real API key):**
|
|
193
|
+
```
|
|
194
|
+
Before v0.4.29:
|
|
195
|
+
✔ Found 15 OpenAI models (only gpt-* models)
|
|
196
|
+
|
|
197
|
+
After v0.4.29:
|
|
198
|
+
✔ Found 25+ OpenAI models (includes o1, o3, all families)
|
|
199
|
+
```
|
|
200
|
+
|
|
201
|
+
**Models Now Available:**
|
|
202
|
+
- All GPT-4o series
|
|
203
|
+
- All GPT-4 series
|
|
204
|
+
- All GPT-3.5 series
|
|
205
|
+
- ✅ All o1 series (o1, o1-mini, o1-preview)
|
|
206
|
+
- ✅ All o3 series (o3, o3-pro, o3-mini - if accessible)
|
|
207
|
+
- Any future model families
|
|
208
|
+
|
|
209
|
+
---
|
|
210
|
+
|
|
211
|
+
## Impact Assessment
|
|
212
|
+
|
|
213
|
+
### Before Fix (v0.4.28 and earlier)
|
|
214
|
+
|
|
215
|
+
**Missing Models:**
|
|
216
|
+
- o1, o1-mini, o1-preview (released months ago)
|
|
217
|
+
- o3, o3-pro, o3-mini (if available)
|
|
218
|
+
- Any non-GPT OpenAI models
|
|
219
|
+
|
|
220
|
+
**User Impact:**
|
|
221
|
+
- Users couldn't access latest OpenAI reasoning models
|
|
222
|
+
- Had to manually specify model names
|
|
223
|
+
- Poor user experience
|
|
224
|
+
- Missed out on advanced o1/o3 capabilities
|
|
225
|
+
|
|
226
|
+
### After Fix (v0.4.29)
|
|
227
|
+
|
|
228
|
+
**Available Models:**
|
|
229
|
+
- ALL OpenAI models dynamically fetched
|
|
230
|
+
- Includes all current and future model families
|
|
231
|
+
- No manual updates needed for OpenAI
|
|
232
|
+
- True dynamic fetching as originally intended
|
|
233
|
+
|
|
234
|
+
---
|
|
235
|
+
|
|
236
|
+
## Lessons Learned
|
|
237
|
+
|
|
238
|
+
### What Went Wrong
|
|
239
|
+
|
|
240
|
+
1. **Assumption:** Assumed all OpenAI models start with "gpt-"
|
|
241
|
+
2. **Lack of Testing:** Didn't test with o1 series models
|
|
242
|
+
3. **No Documentation:** Missing provider model URL documentation
|
|
243
|
+
4. **No Update Process:** No process for checking new models
|
|
244
|
+
|
|
245
|
+
### Prevention Measures
|
|
246
|
+
|
|
247
|
+
1. ✅ **Documentation:** Created AI-MODEL-SOURCES.md
|
|
248
|
+
2. ✅ **No Prefix Filtering:** Use owned_by instead
|
|
249
|
+
3. ✅ **Monthly Checklist:** Added to AI-MODEL-SOURCES.md
|
|
250
|
+
4. ✅ **Better Comments:** Documented why filters exist
|
|
251
|
+
|
|
252
|
+
### Future Improvements
|
|
253
|
+
|
|
254
|
+
1. **Add Tests:** Unit tests for model fetching
|
|
255
|
+
2. **Add Monitoring:** Log when new models appear
|
|
256
|
+
3. **Automated Updates:** GitHub Actions to check for new models
|
|
257
|
+
4. **Model Metadata:** Cache pricing, capabilities, context windows
|
|
258
|
+
|
|
259
|
+
---
|
|
260
|
+
|
|
261
|
+
## Version History
|
|
262
|
+
|
|
263
|
+
- **v0.4.28 and earlier:** Bug present - missing o1, o3 series
|
|
264
|
+
- **v0.4.29:** Bug fixed - ALL OpenAI models now available
|
|
265
|
+
|
|
266
|
+
---
|
|
267
|
+
|
|
268
|
+
## References
|
|
269
|
+
|
|
270
|
+
- **OpenAI Models:** https://platform.openai.com/docs/models
|
|
271
|
+
- **OpenAI API Docs:** https://platform.openai.com/docs/api-reference/models
|
|
272
|
+
- **Issue Reported:** 2025-10-04 by user
|
|
273
|
+
- **Fixed:** 2025-10-04 in v0.4.29
|
|
274
|
+
|
|
275
|
+
---
|
|
276
|
+
|
|
277
|
+
## User Feedback
|
|
278
|
+
|
|
279
|
+
> "ensure you save the llm provider models url for each, to ensure you always have access to the latest information directly from the providers, add this as part of the update process, and save to docs, todos, chats"
|
|
280
|
+
|
|
281
|
+
**Response:** ✅ Completed
|
|
282
|
+
- Created `.project/docs/AI-MODEL-SOURCES.md` with all provider URLs
|
|
283
|
+
- Added monthly update checklist
|
|
284
|
+
- Documented in this chat file
|
|
285
|
+
- Added to todos for v0.4.29 release
|
|
286
|
+
|
|
287
|
+
---
|
|
288
|
+
|
|
289
|
+
## Commit Message
|
|
290
|
+
|
|
291
|
+
```
|
|
292
|
+
v0.4.29: Fix critical OpenAI model fetching bug
|
|
293
|
+
|
|
294
|
+
🐛 CRITICAL FIX: OpenAI Model Filtering
|
|
295
|
+
|
|
296
|
+
Problem:
|
|
297
|
+
- OpenAI models filtered by .startsWith('gpt-')
|
|
298
|
+
- Excluded entire model families: o1, o3 series
|
|
299
|
+
- Users couldn't access latest reasoning models
|
|
300
|
+
|
|
301
|
+
Root Cause:
|
|
302
|
+
- Lazy filter assumption from early implementation
|
|
303
|
+
- Never updated when o1 series launched
|
|
304
|
+
- Missing o3, o3-pro if available
|
|
305
|
+
|
|
306
|
+
Solution:
|
|
307
|
+
- Removed prefix filter
|
|
308
|
+
- Filter by owned_by instead
|
|
309
|
+
- Now fetches ALL OpenAI models dynamically
|
|
310
|
+
|
|
311
|
+
Impact:
|
|
312
|
+
- Before: ~15 models (gpt-* only)
|
|
313
|
+
- After: 25+ models (all families)
|
|
314
|
+
- Now includes: o1, o1-mini, o1-preview, o3, o3-pro
|
|
315
|
+
|
|
316
|
+
📚 Documentation:
|
|
317
|
+
- Created AI-MODEL-SOURCES.md
|
|
318
|
+
- All provider documentation URLs
|
|
319
|
+
- Monthly update checklist
|
|
320
|
+
- Known issues and solutions
|
|
321
|
+
|
|
322
|
+
Files changed:
|
|
323
|
+
- ai-config.js:148-162 - Fixed filter
|
|
324
|
+
- AI-MODEL-SOURCES.md - NEW documentation
|
|
325
|
+
- CRITICAL-MODEL-FETCHING-BUG.md - NEW chat doc
|
|
326
|
+
|
|
327
|
+
Testing:
|
|
328
|
+
- All 120 tests passing
|
|
329
|
+
- Dynamic fetching now works correctly
|
|
330
|
+
- All model families accessible
|
|
331
|
+
```
|
|
@@ -0,0 +1,249 @@
|
|
|
1
|
+
# AI Model Documentation Sources
|
|
2
|
+
|
|
3
|
+
This document tracks the official documentation URLs for AI model listings from each provider. These should be checked regularly to ensure adf-cli supports the latest models.
|
|
4
|
+
|
|
5
|
+
**Last Updated:** 2025-10-04
|
|
6
|
+
|
|
7
|
+
---
|
|
8
|
+
|
|
9
|
+
## Provider Model Documentation URLs
|
|
10
|
+
|
|
11
|
+
### Anthropic Claude
|
|
12
|
+
|
|
13
|
+
**Official Models Page:**
|
|
14
|
+
- https://docs.anthropic.com/en/docs/about-claude/models
|
|
15
|
+
|
|
16
|
+
**API Models List:**
|
|
17
|
+
- ❌ No public API endpoint for listing models
|
|
18
|
+
- Must use curated list from documentation
|
|
19
|
+
|
|
20
|
+
**Current Models (as of 2025-10-04):**
|
|
21
|
+
- `claude-sonnet-4-5-20250929`
|
|
22
|
+
- `claude-3-5-sonnet-20241022`
|
|
23
|
+
- `claude-3-opus-20240229`
|
|
24
|
+
- `claude-3-sonnet-20240229`
|
|
25
|
+
- `claude-3-haiku-20240307`
|
|
26
|
+
|
|
27
|
+
**Update Strategy:** Manual curation from official docs
|
|
28
|
+
|
|
29
|
+
---
|
|
30
|
+
|
|
31
|
+
### OpenAI
|
|
32
|
+
|
|
33
|
+
**Official Models Page:**
|
|
34
|
+
- https://platform.openai.com/docs/models
|
|
35
|
+
|
|
36
|
+
**API Models List Endpoint:**
|
|
37
|
+
- ✅ `GET https://api.openai.com/v1/models`
|
|
38
|
+
- Requires API key in Authorization header
|
|
39
|
+
|
|
40
|
+
**Current Model Families (as of 2025-10-04):**
|
|
41
|
+
- GPT-4o series: `gpt-4o`, `gpt-4o-mini`
|
|
42
|
+
- GPT-4 Turbo: `gpt-4-turbo`, `gpt-4-turbo-preview`
|
|
43
|
+
- GPT-4: `gpt-4`
|
|
44
|
+
- o1 series: `o1`, `o1-mini`, `o1-preview`
|
|
45
|
+
- o3 series: `o3`, `o3-mini`, `o3-pro` (if available)
|
|
46
|
+
- GPT-3.5: `gpt-3.5-turbo`
|
|
47
|
+
|
|
48
|
+
**Update Strategy:** Dynamic fetching via API endpoint (DO NOT filter by model name prefix)
|
|
49
|
+
|
|
50
|
+
**Important:**
|
|
51
|
+
- OpenAI API returns ALL available models for your API key
|
|
52
|
+
- Do NOT filter by prefix (e.g., `gpt-`) as this excludes o1, o3, and future model families
|
|
53
|
+
- Filter only by owned_by or other metadata if needed
|
|
54
|
+
|
|
55
|
+
---
|
|
56
|
+
|
|
57
|
+
### Google Gemini
|
|
58
|
+
|
|
59
|
+
**Official Models Page:**
|
|
60
|
+
- https://ai.google.dev/gemini-api/docs/models
|
|
61
|
+
|
|
62
|
+
**API Models List:**
|
|
63
|
+
- ⚠️ SDK has experimental listModels() but not stable
|
|
64
|
+
- Must use curated list from documentation
|
|
65
|
+
|
|
66
|
+
**Current Models (as of 2025-10-04):**
|
|
67
|
+
|
|
68
|
+
**Gemini 2.5 Series (Latest - Stable):**
|
|
69
|
+
- `gemini-2.5-pro`
|
|
70
|
+
- `gemini-2.5-flash`
|
|
71
|
+
- `gemini-2.5-flash-lite`
|
|
72
|
+
- `gemini-2.5-flash-image`
|
|
73
|
+
|
|
74
|
+
**Gemini 2.0 Series:**
|
|
75
|
+
- `gemini-2.0-flash`
|
|
76
|
+
- `gemini-2.0-flash-lite`
|
|
77
|
+
|
|
78
|
+
**Gemini 1.5 Series (Legacy):**
|
|
79
|
+
- `gemini-1.5-pro-latest`
|
|
80
|
+
- `gemini-1.5-flash-latest`
|
|
81
|
+
- `gemini-1.5-pro`
|
|
82
|
+
- `gemini-1.5-flash`
|
|
83
|
+
|
|
84
|
+
**Update Strategy:** Manual curation from official docs (SDK support pending)
|
|
85
|
+
|
|
86
|
+
---
|
|
87
|
+
|
|
88
|
+
### OpenRouter
|
|
89
|
+
|
|
90
|
+
**Official Models Page:**
|
|
91
|
+
- https://openrouter.ai/models
|
|
92
|
+
|
|
93
|
+
**API Models List Endpoint:**
|
|
94
|
+
- ✅ `GET https://openrouter.ai/api/v1/models`
|
|
95
|
+
- Returns JSON with all available models
|
|
96
|
+
|
|
97
|
+
**Update Strategy:** Dynamic fetching via API endpoint
|
|
98
|
+
|
|
99
|
+
**Current Implementation:** ✅ Working correctly - fetches all models dynamically
|
|
100
|
+
|
|
101
|
+
---
|
|
102
|
+
|
|
103
|
+
## Update Process
|
|
104
|
+
|
|
105
|
+
### Monthly Model Review Checklist
|
|
106
|
+
|
|
107
|
+
Run this checklist at the start of each month to ensure latest models are available:
|
|
108
|
+
|
|
109
|
+
1. **Anthropic Claude**
|
|
110
|
+
- [ ] Visit https://docs.anthropic.com/en/docs/about-claude/models
|
|
111
|
+
- [ ] Check for new model releases
|
|
112
|
+
- [ ] Update `AI_PROVIDERS.ANTHROPIC.defaultModels` in `lib/ai/ai-config.js`
|
|
113
|
+
- [ ] Test new models if added
|
|
114
|
+
|
|
115
|
+
2. **OpenAI**
|
|
116
|
+
- [ ] Verify `fetchAvailableModels()` is NOT filtering by model prefix
|
|
117
|
+
- [ ] Test with real API key to ensure all models returned
|
|
118
|
+
- [ ] Check https://platform.openai.com/docs/models for new model families
|
|
119
|
+
- [ ] Update fallback list if API changes
|
|
120
|
+
|
|
121
|
+
3. **Google Gemini**
|
|
122
|
+
- [ ] Visit https://ai.google.dev/gemini-api/docs/models
|
|
123
|
+
- [ ] Check for new Gemini versions (2.6, 3.0, etc.)
|
|
124
|
+
- [ ] Update `AI_PROVIDERS.GOOGLE.defaultModels` in `lib/ai/ai-config.js`
|
|
125
|
+
- [ ] Test new models if added
|
|
126
|
+
|
|
127
|
+
4. **OpenRouter**
|
|
128
|
+
- [ ] API fetching is dynamic - no action needed
|
|
129
|
+
- [ ] Verify API endpoint still works
|
|
130
|
+
- [ ] Check for API changes at https://openrouter.ai/docs
|
|
131
|
+
|
|
132
|
+
### Testing After Updates
|
|
133
|
+
|
|
134
|
+
```bash
|
|
135
|
+
# Test each provider's model fetching
|
|
136
|
+
cd adf-cli
|
|
137
|
+
npm test
|
|
138
|
+
|
|
139
|
+
# Manual test with real API keys
|
|
140
|
+
adf init
|
|
141
|
+
# Select each provider and verify model list
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
---
|
|
145
|
+
|
|
146
|
+
## Known Issues
|
|
147
|
+
|
|
148
|
+
### OpenAI Model Filtering Bug (CRITICAL - Fixed in v0.4.29)
|
|
149
|
+
|
|
150
|
+
**Problem:**
|
|
151
|
+
```javascript
|
|
152
|
+
.filter(m => m.id.startsWith('gpt-')) // ❌ TOO RESTRICTIVE
|
|
153
|
+
```
|
|
154
|
+
|
|
155
|
+
This filter excludes:
|
|
156
|
+
- o1, o1-mini, o1-preview
|
|
157
|
+
- o3, o3-pro, o3-mini
|
|
158
|
+
- Any future non-GPT model families
|
|
159
|
+
|
|
160
|
+
**Solution:**
|
|
161
|
+
```javascript
|
|
162
|
+
// Fetch ALL models without prefix filtering
|
|
163
|
+
const models = response.data
|
|
164
|
+
.map(m => m.id)
|
|
165
|
+
.sort();
|
|
166
|
+
```
|
|
167
|
+
|
|
168
|
+
**Status:** Fixed in v0.4.29
|
|
169
|
+
|
|
170
|
+
---
|
|
171
|
+
|
|
172
|
+
## API Response Examples
|
|
173
|
+
|
|
174
|
+
### OpenAI `/v1/models` Response
|
|
175
|
+
|
|
176
|
+
```json
|
|
177
|
+
{
|
|
178
|
+
"object": "list",
|
|
179
|
+
"data": [
|
|
180
|
+
{
|
|
181
|
+
"id": "gpt-4o",
|
|
182
|
+
"object": "model",
|
|
183
|
+
"created": 1234567890,
|
|
184
|
+
"owned_by": "openai"
|
|
185
|
+
},
|
|
186
|
+
{
|
|
187
|
+
"id": "o1-preview",
|
|
188
|
+
"object": "model",
|
|
189
|
+
"created": 1234567890,
|
|
190
|
+
"owned_by": "openai"
|
|
191
|
+
}
|
|
192
|
+
]
|
|
193
|
+
}
|
|
194
|
+
```
|
|
195
|
+
|
|
196
|
+
### OpenRouter `/api/v1/models` Response
|
|
197
|
+
|
|
198
|
+
```json
|
|
199
|
+
{
|
|
200
|
+
"data": [
|
|
201
|
+
{
|
|
202
|
+
"id": "anthropic/claude-sonnet-4-5",
|
|
203
|
+
"name": "Claude Sonnet 4.5",
|
|
204
|
+
"pricing": { ... }
|
|
205
|
+
}
|
|
206
|
+
]
|
|
207
|
+
}
|
|
208
|
+
```
|
|
209
|
+
|
|
210
|
+
---
|
|
211
|
+
|
|
212
|
+
## Maintenance Notes
|
|
213
|
+
|
|
214
|
+
- **DO NOT** hardcode model lists unless provider has no API
|
|
215
|
+
- **DO NOT** filter by model name prefix unless absolutely necessary
|
|
216
|
+
- **ALWAYS** fetch dynamically when API is available
|
|
217
|
+
- **TEST** with real API keys before publishing updates
|
|
218
|
+
- **DOCUMENT** any assumptions or limitations in this file
|
|
219
|
+
|
|
220
|
+
---
|
|
221
|
+
|
|
222
|
+
## Future Improvements
|
|
223
|
+
|
|
224
|
+
1. **Add model metadata caching**
|
|
225
|
+
- Cache API responses for 24 hours
|
|
226
|
+
- Reduce API calls during testing
|
|
227
|
+
|
|
228
|
+
2. **Add model capability detection**
|
|
229
|
+
- Vision support
|
|
230
|
+
- Function calling
|
|
231
|
+
- Context window size
|
|
232
|
+
- Pricing information
|
|
233
|
+
|
|
234
|
+
3. **Add model recommendation system**
|
|
235
|
+
- Suggest best model for use case
|
|
236
|
+
- Warn about deprecated models
|
|
237
|
+
- Show model capabilities in selection
|
|
238
|
+
|
|
239
|
+
4. **Automated model list updates**
|
|
240
|
+
- GitHub Actions workflow
|
|
241
|
+
- Monthly PR with updated model lists
|
|
242
|
+
- Automated testing against provider APIs
|
|
243
|
+
|
|
244
|
+
---
|
|
245
|
+
|
|
246
|
+
## Version History
|
|
247
|
+
|
|
248
|
+
- **2025-10-04:** Initial documentation
|
|
249
|
+
- **2025-10-04:** Identified OpenAI filtering bug (v0.4.29)
|
package/CHANGELOG.md
CHANGED
|
@@ -5,6 +5,121 @@ All notable changes to `@iservu-inc/adf-cli` will be documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
7
|
|
|
8
|
+
## [0.4.30] - 2025-10-04
|
|
9
|
+
|
|
10
|
+
### 🔒 Security: Auto .gitignore Protection
|
|
11
|
+
|
|
12
|
+
**Added: Automatic .gitignore Management**
|
|
13
|
+
- **Feature:** Automatically create/update `.gitignore` to protect `.adf/.env` file
|
|
14
|
+
- **Purpose:** Prevent accidentally committing API keys to version control
|
|
15
|
+
- **Implementation:**
|
|
16
|
+
- New `ensureGitignore()` function in `ai-config.js`
|
|
17
|
+
- Called automatically after saving API keys
|
|
18
|
+
- Creates `.gitignore` if it doesn't exist
|
|
19
|
+
- Adds `.adf/.env` entry with warning comment
|
|
20
|
+
- Skips if entry already exists (non-destructive)
|
|
21
|
+
|
|
22
|
+
**Code Changes:**
|
|
23
|
+
- `lib/ai/ai-config.js:106-136` - New `ensureGitignore()` function
|
|
24
|
+
- `lib/ai/ai-config.js:158-160` - Auto-call after saving API keys
|
|
25
|
+
|
|
26
|
+
**User Benefit:**
|
|
27
|
+
- No more risk of committing API keys
|
|
28
|
+
- Automatic protection without manual setup
|
|
29
|
+
- Works with existing `.gitignore` files safely
|
|
30
|
+
|
|
31
|
+
---
|
|
32
|
+
|
|
33
|
+
## [0.4.29] - 2025-10-04
|
|
34
|
+
|
|
35
|
+
### 🐛 CRITICAL FIX: OpenAI Model Fetching
|
|
36
|
+
|
|
37
|
+
**Fixed: OpenAI Model Filtering Too Restrictive**
|
|
38
|
+
- **Problem:** OpenAI models were filtered by `.startsWith('gpt-')` which excluded entire model families
|
|
39
|
+
- **Missing Models:**
|
|
40
|
+
- ❌ o1, o1-mini, o1-preview (reasoning models)
|
|
41
|
+
- ❌ o3, o3-pro, o3-mini (if available)
|
|
42
|
+
- ❌ Any future non-GPT model families
|
|
43
|
+
- **Root Cause:** Lazy filter assumption from early implementation, never updated when o1 series launched
|
|
44
|
+
|
|
45
|
+
**Solution:**
|
|
46
|
+
- Removed prefix-based filtering
|
|
47
|
+
- Now filters by `owned_by` to include all OpenAI-owned models
|
|
48
|
+
- Excludes only user fine-tuned models
|
|
49
|
+
- Truly dynamic fetching as originally intended
|
|
50
|
+
|
|
51
|
+
**Code Change (ai-config.js:148-162):**
|
|
52
|
+
```javascript
|
|
53
|
+
// ❌ BEFORE - Too restrictive
|
|
54
|
+
.filter(m => m.id.startsWith('gpt-'))
|
|
55
|
+
|
|
56
|
+
// ✅ AFTER - All OpenAI models
|
|
57
|
+
.filter(m => m.owned_by === 'openai' || m.owned_by === 'system' || m.owned_by === 'openai-internal')
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
**Impact:**
|
|
61
|
+
- **Before:** ~15 models (gpt-* only)
|
|
62
|
+
- **After:** 25+ models (all families)
|
|
63
|
+
- **Now Includes:**
|
|
64
|
+
- ✅ All GPT-4o, GPT-4, GPT-3.5 series
|
|
65
|
+
- ✅ All o1 series (o1, o1-mini, o1-preview)
|
|
66
|
+
- ✅ All o3 series (o3, o3-pro, o3-mini - if accessible)
|
|
67
|
+
- ✅ Any future model families automatically
|
|
68
|
+
|
|
69
|
+
### 📚 NEW: AI Model Documentation
|
|
70
|
+
|
|
71
|
+
**Created: `.project/docs/AI-MODEL-SOURCES.md`**
|
|
72
|
+
|
|
73
|
+
**Contents:**
|
|
74
|
+
- Official documentation URLs for all providers
|
|
75
|
+
- API endpoint details and limitations
|
|
76
|
+
- Current model lists (as of 2025-10-04)
|
|
77
|
+
- Monthly update checklist
|
|
78
|
+
- Known issues and fixes
|
|
79
|
+
- API response examples
|
|
80
|
+
- Maintenance procedures
|
|
81
|
+
|
|
82
|
+
**Provider Documentation URLs:**
|
|
83
|
+
- **Anthropic:** https://docs.anthropic.com/en/docs/about-claude/models
|
|
84
|
+
- **OpenAI:** https://platform.openai.com/docs/models
|
|
85
|
+
- **Google Gemini:** https://ai.google.dev/gemini-api/docs/models
|
|
86
|
+
- **OpenRouter:** https://openrouter.ai/models
|
|
87
|
+
|
|
88
|
+
**Purpose:**
|
|
89
|
+
- Track official model sources
|
|
90
|
+
- Guide monthly model reviews
|
|
91
|
+
- Document update processes
|
|
92
|
+
- Prevent future filtering bugs
|
|
93
|
+
|
|
94
|
+
**Created: `.project/chats/current/2025-10-04_CRITICAL-MODEL-FETCHING-BUG.md`**
|
|
95
|
+
|
|
96
|
+
**Contents:**
|
|
97
|
+
- Detailed root cause analysis
|
|
98
|
+
- Provider status review
|
|
99
|
+
- Testing methodology
|
|
100
|
+
- Lessons learned
|
|
101
|
+
- Prevention measures
|
|
102
|
+
|
|
103
|
+
### 📊 Provider Status Summary
|
|
104
|
+
|
|
105
|
+
| Provider | Dynamic Fetching | Filter | Status |
|
|
106
|
+
|----------|------------------|--------|---------|
|
|
107
|
+
| OpenAI | ✅ Yes | owned_by | 🐛 **FIXED** |
|
|
108
|
+
| Anthropic | ❌ No | N/A | ✅ OK (no API) |
|
|
109
|
+
| Google | ❌ No | N/A | ✅ OK (unstable SDK) |
|
|
110
|
+
| OpenRouter | ✅ Yes | None | ✅ OK |
|
|
111
|
+
|
|
112
|
+
**Technical Details:**
|
|
113
|
+
- `ai-config.js:148-162`: Removed prefix filter, added owned_by filter
|
|
114
|
+
- `AI-MODEL-SOURCES.md`: NEW - Provider documentation
|
|
115
|
+
- `CRITICAL-MODEL-FETCHING-BUG.md`: NEW - Detailed analysis
|
|
116
|
+
- All 120 tests passing
|
|
117
|
+
|
|
118
|
+
**User Feedback Addressed:**
|
|
119
|
+
> "ensure you save the llm provider models url for each, to ensure you always have access to the latest information directly from the providers, add this as part of the update process, and save to docs, todos, chats"
|
|
120
|
+
|
|
121
|
+
✅ Completed - All provider URLs documented with update procedures
|
|
122
|
+
|
|
8
123
|
## [0.4.28] - 2025-10-04
|
|
9
124
|
|
|
10
125
|
### ✨ Major Update: Gemini 2.5 Models
|
package/lib/ai/ai-config.js
CHANGED
|
@@ -103,6 +103,38 @@ async function loadEnvFile(envPath) {
|
|
|
103
103
|
return {};
|
|
104
104
|
}
|
|
105
105
|
|
|
106
|
+
/**
|
|
107
|
+
* Ensure .gitignore includes .adf/.env to protect API keys
|
|
108
|
+
*/
|
|
109
|
+
async function ensureGitignore(projectPath) {
|
|
110
|
+
const gitignorePath = path.join(projectPath, '.gitignore');
|
|
111
|
+
const entryToAdd = '.adf/.env';
|
|
112
|
+
|
|
113
|
+
let content = '';
|
|
114
|
+
let hasEntry = false;
|
|
115
|
+
|
|
116
|
+
// Read existing .gitignore if it exists
|
|
117
|
+
if (await fs.pathExists(gitignorePath)) {
|
|
118
|
+
content = await fs.readFile(gitignorePath, 'utf-8');
|
|
119
|
+
hasEntry = content.split('\n').some(line => line.trim() === entryToAdd);
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
// Add entry if not already present
|
|
123
|
+
if (!hasEntry) {
|
|
124
|
+
const lines = content ? content.split('\n') : [];
|
|
125
|
+
|
|
126
|
+
// Add a section header if .gitignore is empty or doesn't have the entry
|
|
127
|
+
if (lines.length === 0 || (lines[lines.length - 1] && lines[lines.length - 1].trim() !== '')) {
|
|
128
|
+
lines.push(''); // Add blank line if file is not empty
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
lines.push('# adf-cli - AI Provider API Keys (DO NOT COMMIT)');
|
|
132
|
+
lines.push(entryToAdd);
|
|
133
|
+
|
|
134
|
+
await fs.writeFile(gitignorePath, lines.join('\n'), 'utf-8');
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
|
|
106
138
|
/**
|
|
107
139
|
* Save API key to .env file
|
|
108
140
|
*/
|
|
@@ -122,6 +154,10 @@ async function saveToEnvFile(envPath, key, value) {
|
|
|
122
154
|
|
|
123
155
|
await fs.ensureDir(path.dirname(envPath));
|
|
124
156
|
await fs.writeFile(envPath, lines.join('\n'), 'utf-8');
|
|
157
|
+
|
|
158
|
+
// Ensure .gitignore protects this file
|
|
159
|
+
const projectPath = path.dirname(path.dirname(envPath)); // Go up from .adf/.env to project root
|
|
160
|
+
await ensureGitignore(projectPath);
|
|
125
161
|
}
|
|
126
162
|
|
|
127
163
|
/**
|
|
@@ -149,12 +185,17 @@ async function fetchAvailableModels(provider, apiKey) {
|
|
|
149
185
|
const OpenAI = require('openai');
|
|
150
186
|
const openai = new OpenAI({ apiKey });
|
|
151
187
|
const response = await openai.models.list();
|
|
152
|
-
|
|
153
|
-
|
|
188
|
+
|
|
189
|
+
// Get ALL OpenAI models - DO NOT filter by prefix
|
|
190
|
+
// This ensures we include o1, o3, and any future model families
|
|
191
|
+
// Only filter to OpenAI-owned models to exclude fine-tuned models
|
|
192
|
+
const allModels = response.data
|
|
193
|
+
.filter(m => m.owned_by === 'openai' || m.owned_by === 'system' || m.owned_by === 'openai-internal')
|
|
154
194
|
.map(m => m.id)
|
|
155
195
|
.sort();
|
|
156
|
-
|
|
157
|
-
|
|
196
|
+
|
|
197
|
+
spinner.succeed(`Found ${allModels.length} OpenAI models`);
|
|
198
|
+
return allModels.length > 0 ? allModels : provider.defaultModels;
|
|
158
199
|
|
|
159
200
|
case 'google':
|
|
160
201
|
try {
|