@jsonstudio/rcc 0.89.942 → 0.89.1083

package/scripts/analyze-codex-error-failures.mjs

@@ -37,11 +37,25 @@ async function fileExists(p) {
   }
 }
 
-async function listJsonFiles(root) {
-  const entries = await fs.readdir(root);
-  return entries
-    .filter((name) => name.toLowerCase().endsWith('.json'))
-    .map((name) => path.join(root, name));
+async function* walkJsonFiles(root) {
+  const stack = [root];
+  while (stack.length) {
+    const current = stack.pop();
+    let entries;
+    try {
+      entries = await fs.readdir(current, { withFileTypes: true });
+    } catch {
+      continue;
+    }
+    for (const entry of entries) {
+      const full = path.join(current, entry.name);
+      if (entry.isDirectory()) {
+        stack.push(full);
+      } else if (entry.isFile() && entry.name.toLowerCase().endsWith('.json')) {
+        yield full;
+      }
+    }
+  }
 }
 
 async function analyzeFile(filePath) {
@@ -61,20 +75,13 @@ async function main() {
     process.exit(0);
   }
 
-  const files = await listJsonFiles(RESPONSES_DIR);
-  if (!files.length) {
-    console.log('[analyze-codex-error-failures] no JSON files under', RESPONSES_DIR);
-    process.exit(0);
-  }
-
-  console.log(`[analyze-codex-error-failures] scanning ${files.length} file(s) under ${RESPONSES_DIR}`);
-
   const summary = new Map();
   for (const p of PATTERNS) {
     summary.set(p, { count: 0, files: [] });
   }
 
-  for (const file of files) {
+  let scanned = 0;
+  for await (const file of walkJsonFiles(RESPONSES_DIR)) {
     let res;
     try {
       res = await analyzeFile(file);
@@ -90,8 +97,11 @@ async function main() {
         entry.files.push(path.basename(file));
       }
     }
+    scanned += 1;
   }
 
+  console.log(`[analyze-codex-error-failures] scanned ${scanned} file(s) under ${RESPONSES_DIR}`);
+
   for (const key of PATTERNS) {
     const { count, files } = summary.get(key);
     console.log(`\n=== Pattern: "${key}" ===`);

package/scripts/classify-codex-samples.mjs

@@ -30,7 +30,6 @@ const PROVIDER_KEYS = {
 const TOOL_TYPES = {
   'apply_patch': 'apply_patch',
   'shell': 'shell_command',
-  'toon': 'toon_tool',
   'submit_tool_outputs': 'tool_loop',
   'list_files': 'file_operation',
   'write_file': 'file_operation',
@@ -46,7 +45,6 @@ class SampleClassifier {
       byProvider: {},
       byToolType: {},
       withToolCalls: 0,
-      withToon: 0,
       errors: 0
     };
   }
@@ -68,9 +66,6 @@ class SampleClassifier {
   identifyToolType(toolCall) {
     const funcName = toolCall.function?.name || '';
     
-    // 检查 TOON 格式
-    if (this.isToonTool(toolCall)) return 'toon_tool';
-    
     // 检查已知工具名称
     for (const [pattern, type] of Object.entries(TOOL_TYPES)) {
       if (funcName.toLowerCase().includes(pattern)) return type;
@@ -84,19 +79,6 @@ class SampleClassifier {
     return 'unknown_tool';
   }
 
-  // 检查是否为 TOON 工具
-  isToonTool(toolCall) {
-    const args = toolCall.function?.arguments;
-    if (!args) return false;
-    
-    try {
-      const parsed = JSON.parse(args);
-      return parsed && typeof parsed.toon === 'string';
-    } catch {
-      return false;
-    }
-  }
-
   // 分析单个样本
   async analyzeSample(filePath) {
     try {
@@ -107,7 +89,6 @@ class SampleClassifier {
       const sampleId = basename(dirname(filePath)) + '/' + basename(filePath, '.json');
       
       let toolCalls = [];
-      let hasToon = false;
       
       // 提取 tool_calls
       if (data.tool_calls) {
@@ -128,8 +109,6 @@ class SampleClassifier {
         const toolType = this.identifyToolType(toolCall);
         toolTypes.push(toolType);
         
-        if (toolType === 'toon_tool') hasToon = true;
-        
         // 更新统计
         this.stats.byToolType[toolType] = (this.stats.byToolType[toolType] || 0) + 1;
       }
@@ -140,7 +119,6 @@ class SampleClassifier {
         filePath,
         hasToolCalls: toolCalls.length > 0,
         toolTypes,
-        hasToon,
         toolCallCount: toolCalls.length
       };
       
@@ -150,7 +128,6 @@ class SampleClassifier {
       this.stats.total++;
       this.stats.byProvider[providerKey] = (this.stats.byProvider[providerKey] || 0) + 1;
       if (toolCalls.length > 0) this.stats.withToolCalls++;
-      if (hasToon) this.stats.withToon++;
       
     } catch (error) {
       console.error(`Error analyzing ${filePath}:`, error.message);
@@ -196,7 +173,6 @@ class SampleClassifier {
     
     console.log(`总样本数: ${this.stats.total}`);
     console.log(`包含工具调用: ${this.stats.withToolCalls}`);
-    console.log(`包含 TOON: ${this.stats.withToon}`);
     console.log(`错误数: ${this.stats.errors}`);
     
     console.log('\n按 Provider 分布:');
@@ -224,21 +200,10 @@ class SampleClassifier {
     console.log('==================');
     
     const hasApplyPatch = this.stats.byToolType['apply_patch'] > 0;
-    const hasToon = this.stats.byToolType['toon_tool'] > 0;
     const hasShell = this.stats.byToolType['shell_command'] > 0;
     
     if (!hasApplyPatch) console.log('  - 缺少 apply_patch 样本');
-    if (!hasToon) console.log('  - 缺少 TOON 工具样本');
     if (!hasShell) console.log('  - 缺少 shell command 样本');
-    
-    // TOON 样本详情
-    if (this.stats.withToon > 0) {
-      console.log('\n🔧 TOON 工具样本:');
-      const toonSamples = this.samples.filter(s => s.hasToon);
-      for (const sample of toonSamples) {
-        console.log(`  - ${sample.provider}: ${sample.id}`);
-      }
-    }
   }
 }
 

package/scripts/copy-modules-config.mjs

@@ -6,6 +6,8 @@ async function copyModulesConfig() {
   const root = process.cwd();
   const srcModulesConfig = path.join(root, 'config', 'modules.json');
   const distModulesConfig = path.join(root, 'dist', 'config', 'modules.json');
+  const srcDaemonAdminUi = path.join(root, 'docs', 'daemon-admin-ui.html');
+  const distDaemonAdminUi = path.join(root, 'dist', 'docs', 'daemon-admin-ui.html');
 
   try {
     // 确保源文件存在
@@ -18,6 +20,20 @@ async function copyModulesConfig() {
     await fs.copyFile(srcModulesConfig, distModulesConfig);
 
     console.log('[copy-modules-config] copied modules.json to dist/config/modules.json');
+
+    try {
+      await fs.access(srcDaemonAdminUi);
+      await fs.mkdir(path.dirname(distDaemonAdminUi), { recursive: true });
+      await fs.copyFile(srcDaemonAdminUi, distDaemonAdminUi);
+      console.log('[copy-modules-config] copied daemon-admin-ui.html to dist/docs/daemon-admin-ui.html');
+    } catch (error) {
+      if (error.code === 'ENOENT') {
+        console.warn('[copy-modules-config] docs/daemon-admin-ui.html not found, skipping');
+      } else {
+        console.error('[copy-modules-config] failed to copy daemon admin ui:', error.message);
+        process.exit(1);
+      }
+    }
   } catch (error) {
     if (error.code === 'ENOENT') {
       console.warn('[copy-modules-config] source modules.json not found, skipping');
@@ -28,4 +44,4 @@ async function copyModulesConfig() {
   }
 }
 
-copyModulesConfig();
+copyModulesConfig();

package/scripts/generate-snapshot-data.mjs

@@ -148,21 +148,26 @@ class SnapshotDataGenerator {
     
     try {
       await fs.mkdir(outputDir, { recursive: true });
-      const files = await fs.readdir(responsesDir);
-      
-      const groups = this.groupFilesByRequestId(files);
       let count = 0;
-      
-      for (const [requestId, groupFiles] of Object.entries(groups)) {
+
+      const entries = await fs.readdir(responsesDir, { withFileTypes: true });
+
+      // New layout: openai-responses/<requestId>/*.json
+      for (const entry of entries) {
         if (count >= MAX_SAMPLES) break;
-        
+        if (!entry.isDirectory()) continue;
+        const requestId = entry.name;
+        if (!requestId.startsWith('req_') && !requestId.includes('responses')) continue;
+
+        const subdirPath = path.join(responsesDir, requestId);
+        const files = (await fs.readdir(subdirPath)).filter((f) => f.endsWith('.json'));
         const snapshot = await this.buildSnapshotFromFiles(
-          requestId, 
-          'openai-responses', 
-          groupFiles, 
-          responsesDir
+          requestId,
+          'openai-responses',
+          files,
+          subdirPath
         );
-        
+
         if (snapshot) {
           const outputPath = path.join(outputDir, `${requestId}.json`);
           await fs.writeFile(outputPath, JSON.stringify(snapshot, null, 2));
@@ -171,6 +176,31 @@ class SnapshotDataGenerator {
           this.samplesGenerated++;
         }
       }
+
+      // Legacy layout: openai-responses/*.json
+      if (count < MAX_SAMPLES) {
+        const files = entries.filter((e) => e.isFile()).map((e) => e.name);
+        const groups = this.groupFilesByRequestId(files);
+      
+        for (const [requestId, groupFiles] of Object.entries(groups)) {
+          if (count >= MAX_SAMPLES) break;
+
+          const snapshot = await this.buildSnapshotFromFiles(
+            requestId,
+            'openai-responses',
+            groupFiles,
+            responsesDir
+          );
+
+          if (snapshot) {
+            const outputPath = path.join(outputDir, `${requestId}.json`);
+            await fs.writeFile(outputPath, JSON.stringify(snapshot, null, 2));
+            console.log(`  ✅ 生成快照: ${requestId}`);
+            count++;
+            this.samplesGenerated++;
+          }
+        }
+      }
       
       console.log(`  📊 Responses: ${count} 个快照`);
     } catch (error) {

package/scripts/mock-provider/extract.mjs

@@ -105,8 +105,8 @@ function findProviderId(node) {
 }
 
 async function detectProviderId(entryDir, prefix, request) {
-  const semanticPath = path.join(entryDir, `${prefix}_semantic_map_from_chat.json`);
-  if (await fileExists(semanticPath)) {
+  const semanticPath = prefix ? path.join(entryDir, `${prefix}_semantic_map_from_chat.json`) : null;
+  if (semanticPath && (await fileExists(semanticPath))) {
     try {
       const semantic = JSON.parse(await fs.readFile(semanticPath, 'utf-8'));
       const detected = findProviderId(semantic);
@@ -119,24 +119,100 @@ async function detectProviderId(entryDir, prefix, request) {
   }
   const fallback =
     request?.providerId ||
-    request?.data?.providerId ||
-    request?.data?.meta?.providerId ||
+    request?.meta?.providerId ||
+    request?.body?.providerId ||
+    request?.body?.meta?.providerId ||
     'unknown';
   return sanitizeComponent(fallback, 'unknown');
 }
 
 function buildTags(response) {
   const tags = [];
-  const body = response?.data?.body;
-  if (body?.__sse_responses) {
+  const body = response?.body || response?.data?.body;
+  if (body?.__sse_responses || body?.mode === 'sse') {
     tags.push('sse');
   }
-  if (Array.isArray(body?.tool_calls) && body.tool_calls.length > 0) {
+  const toolCalls = body?.tool_calls || body?.output?.tool_calls;
+  if (Array.isArray(toolCalls) && toolCalls.length > 0) {
     tags.push('tool-call');
   }
   return tags;
 }
 
+function parseRequestTimestamp(requestId, request) {
+  if (typeof request?.timestamp === 'number' && Number.isFinite(request.timestamp)) {
+    return request.timestamp;
+  }
+  const buildTime = request?.meta?.buildTime;
+  if (typeof buildTime === 'string') {
+    const parsed = Date.parse(buildTime);
+    if (!Number.isNaN(parsed)) {
+      return parsed;
+    }
+  }
+  const match = String(requestId || '').match(/(\d{8})T(\d{6})(\d{3})?/);
+  if (match) {
+    const ymd = match[1];
+    const hms = match[2];
+    const ms = match[3] || '000';
+    const year = Number(ymd.slice(0, 4));
+    const month = Number(ymd.slice(4, 6));
+    const day = Number(ymd.slice(6, 8));
+    const hour = Number(hms.slice(0, 2));
+    const min = Number(hms.slice(2, 4));
+    const sec = Number(hms.slice(4, 6));
+    const msNum = Number(ms);
+    if ([year, month, day, hour, min, sec, msNum].every((v) => Number.isFinite(v))) {
+      return new Date(year, month - 1, day, hour, min, sec, msNum).getTime();
+    }
+  }
+  return Date.now();
+}
+
+function extractModelFromProviderRequest(request) {
+  const body = request?.body || request?.data?.body;
+  const raw =
+    body?.model ||
+    body?.data?.model ||
+    request?.model ||
+    request?.data?.model ||
+    undefined;
+  return typeof raw === 'string' && raw.trim() ? raw.trim() : 'unknown';
+}
+
+async function detectProviderIdFromRequestDir(requestDir, request) {
+  try {
+    const files = await fs.readdir(requestDir);
+    const semanticCandidates = files
+      .filter((name) => name.toLowerCase().endsWith('.json') && name.toLowerCase().includes('semantic_map'))
+      .slice(0, 10);
+    for (const file of semanticCandidates) {
+      try {
+        const semantic = JSON.parse(await fs.readFile(path.join(requestDir, file), 'utf-8'));
+        const detected = findProviderId(semantic);
+        if (detected) {
+          return sanitizeComponent(detected, 'unknown');
+        }
+      } catch {
+        // ignore semantic parse errors
+      }
+    }
+  } catch {
+    // ignore dir read errors
+  }
+  return await detectProviderId(requestDir, null, request);
+}
+
+function pickLatestStageFile(files, prefix) {
+  const candidates = files
+    .filter((name) => name.toLowerCase().endsWith('.json') && name.startsWith(prefix))
+    .sort((a, b) => a.localeCompare(b, 'en'));
+  if (!candidates.length) {
+    return null;
+  }
+  return candidates[candidates.length - 1];
+}
+
 async function extractPair(entryName, entryDir, file, registry, seqMap, filters = {}) {
   const prefix = file.replace('_provider-request.json', '');
   const requestPath = path.join(entryDir, file);
@@ -152,8 +228,8 @@ async function extractPair(entryName, entryDir, file, registry, seqMap, filters
   if (filters.provider && providerId !== sanitizeComponent(filters.provider, filters.provider)) {
     return;
   }
-  const model = sanitizeComponent(request?.data?.body?.model, 'unknown');
-  const tsToken = timestampToToken(request?.timestamp);
+  const model = sanitizeComponent(extractModelFromProviderRequest(request), 'unknown');
+  const tsToken = timestampToToken(parseRequestTimestamp(prefix, request));
   const seqKey = `${entryName}|${providerId}|${model}|${tsToken}`;
   const seq = (seqMap.get(seqKey) || 0) + 1;
   seqMap.set(seqKey, seq);
@@ -198,6 +274,66 @@ async function extractPair(entryName, entryDir, file, registry, seqMap, filters
   console.log(`✅ ${reqId}`);
 }
 
+async function extractPairFromRequestDir(entryName, requestDir, registry, seqMap, filters = {}) {
+  const files = await fs.readdir(requestDir);
+  const providerRequestFile = pickLatestStageFile(files, 'provider-request');
+  const providerResponseFile = pickLatestStageFile(files, 'provider-response');
+  if (!providerRequestFile || !providerResponseFile) {
+    return;
+  }
+
+  const requestPath = path.join(requestDir, providerRequestFile);
+  const responsePath = path.join(requestDir, providerResponseFile);
+  const request = JSON.parse(await fs.readFile(requestPath, 'utf-8'));
+  const response = JSON.parse(await fs.readFile(responsePath, 'utf-8'));
+
+  const requestId = path.basename(requestDir);
+  const providerId = await detectProviderIdFromRequestDir(requestDir, request);
+  if (filters.provider && providerId !== sanitizeComponent(filters.provider, filters.provider)) {
+    return;
+  }
+  const model = sanitizeComponent(extractModelFromProviderRequest(request), 'unknown');
+  const tsToken = timestampToToken(parseRequestTimestamp(requestId, request));
+  const seqKey = `${entryName}|${providerId}|${model}|${tsToken}`;
+  const seq = (seqMap.get(seqKey) || 0) + 1;
+  seqMap.set(seqKey, seq);
+  const seqStr = String(seq).padStart(3, '0');
+  const reqId = `${entryName}-${providerId}-${model}-${tsToken}-${seqStr}`;
+
+  const daySegment = tsToken.slice(0, 8);
+  const timeSegment = tsToken.slice(9);
+  const targetDir = path.join(MOCK_SAMPLES_DIR, entryName, providerId, model, daySegment, timeSegment, seqStr);
+  await ensureDir(targetDir);
+
+  const enrichedRequest = { ...request, reqId, entryEndpoint: request?.endpoint };
+  const enrichedResponse = { ...response, reqId, entryEndpoint: response?.endpoint };
+  await fs.writeFile(path.join(targetDir, 'request.json'), JSON.stringify(enrichedRequest, null, 2));
+  await fs.writeFile(path.join(targetDir, 'response.json'), JSON.stringify(enrichedResponse, null, 2));
+
+  const clientRequestFile = pickLatestStageFile(files, 'client-request');
+  if (clientRequestFile) {
+    try {
+      const client = JSON.parse(await fs.readFile(path.join(requestDir, clientRequestFile), 'utf-8'));
+      const enrichedClient = { ...client, reqId, entryEndpoint: client?.endpoint };
+      await fs.writeFile(path.join(targetDir, 'client-request.json'), JSON.stringify(enrichedClient, null, 2));
+    } catch {
+      // ignore client-request parse errors
+    }
+  }
+
+  registry.samples = registry.samples.filter((sample) => sample.reqId !== reqId);
+  registry.samples.push({
+    reqId,
+    entry: entryName,
+    providerId,
+    model,
+    timestamp: new Date(parseRequestTimestamp(requestId, request)).toISOString(),
+    path: path.relative(MOCK_SAMPLES_DIR, targetDir),
+    tags: buildTags(response)
+  });
+  console.log(`✅ ${reqId}`);
+}
+
 async function extractAll(options = {}) {
   console.log('[mock:extract] Preparing directories...');
   await ensureDir(MOCK_SAMPLES_DIR);
@@ -214,7 +350,41 @@ async function extractAll(options = {}) {
       continue;
     }
     const entryDir = path.join(CODEX_SAMPLES_DIR, dirEntry.name);
-    const files = await fs.readdir(entryDir);
+    const entries = await fs.readdir(entryDir, { withFileTypes: true });
+
+    // Newer layout: entry/<provider>/<requestId>/... (all stages in one directory)
+    // Previous layout: entry/<requestId>/... (single-level request directory)
+    for (const sub of entries) {
+      if (!sub.isDirectory()) continue;
+      const maybeProviderOrRequestDir = path.join(entryDir, sub.name);
+      let nested = [];
+      try {
+        nested = await fs.readdir(maybeProviderOrRequestDir, { withFileTypes: true });
+      } catch {
+        nested = [];
+      }
+      const hasJsonFiles = nested.some((e) => e.isFile() && e.name.toLowerCase().endsWith('.json'));
+      if (hasJsonFiles) {
+        try {
+          await extractPairFromRequestDir(dirEntry.name, maybeProviderOrRequestDir, registry, seqMap, { provider: providerFilter });
+        } catch (error) {
+          console.warn(`⚠️  Skipped ${sub.name}: ${error.message}`);
+        }
+        continue;
+      }
+      const requestDirs = nested.filter((e) => e.isDirectory());
+      for (const reqSub of requestDirs) {
+        const requestDir = path.join(maybeProviderOrRequestDir, reqSub.name);
+        try {
+          await extractPairFromRequestDir(dirEntry.name, requestDir, registry, seqMap, { provider: providerFilter });
+        } catch (error) {
+          console.warn(`⚠️  Skipped ${sub.name}/${reqSub.name}: ${error.message}`);
+        }
+      }
+    }
+
+    // Legacy layout: entry/*_provider-request.json
+    const files = entries.filter((e) => e.isFile()).map((e) => e.name);
     const requests = files.filter((name) => name.endsWith('_provider-request.json'));
     for (const file of requests) {
       try {
@@ -230,25 +400,73 @@ async function extractAll(options = {}) {
 }
 
 async function extractSingle(reqId) {
-  const parts = reqId.split('-');
-  if (parts.length < 5) {
-    throw new Error(`Invalid requestId format: ${reqId}`);
+  await ensureDir(MOCK_SAMPLES_DIR);
+  await ensureDir(path.join(MOCK_SAMPLES_DIR, '_registry'));
+  const registry = await loadRegistry();
+  const seqMap = buildSeqMapFromRegistry(registry);
+
+  const inferredEntry = inferEntryFolder(reqId);
+  const entryDir = path.join(CODEX_SAMPLES_DIR, inferredEntry);
+  const requestDir = path.join(entryDir, sanitizeRequestDirName(reqId));
+  try {
+    const stat = await fs.stat(requestDir);
+    if (stat.isDirectory()) {
+      await extractPairFromRequestDir(inferredEntry, requestDir, registry, seqMap);
+      await saveRegistry(registry);
+      return;
+    }
+  } catch {
+    // fall through
+  }
+
+  // Newer layout: entry/<provider>/<requestId>/...
+  try {
+    const providerDirs = await fs.readdir(entryDir, { withFileTypes: true });
+    for (const providerDir of providerDirs) {
+      if (!providerDir.isDirectory()) continue;
+      const candidate = path.join(entryDir, providerDir.name, sanitizeRequestDirName(reqId));
+      try {
+        const stat = await fs.stat(candidate);
+        if (stat.isDirectory()) {
+          await extractPairFromRequestDir(inferredEntry, candidate, registry, seqMap);
+          await saveRegistry(registry);
+          return;
+        }
+      } catch {
+        // continue
+      }
+    }
+  } catch {
+    // ignore entryDir scan errors
   }
-  const entry = parts.slice(0, parts.length - 4).join('-') || parts[0];
-  const entryDir = path.join(CODEX_SAMPLES_DIR, entry);
+
   const files = await fs.readdir(entryDir);
   const target = files.find((file) => file.includes(reqId) && file.endsWith('_provider-request.json'));
   if (!target) {
-    throw new Error(`Request ${reqId} not found under ${entry}`);
+    throw new Error(`Request ${reqId} not found under ${inferredEntry}`);
   }
-  await ensureDir(MOCK_SAMPLES_DIR);
-  await ensureDir(path.join(MOCK_SAMPLES_DIR, '_registry'));
-  const registry = await loadRegistry();
-  const seqMap = buildSeqMapFromRegistry(registry);
-  await extractPair(entry, entryDir, target, registry, seqMap);
+  await extractPair(inferredEntry, entryDir, target, registry, seqMap);
   await saveRegistry(registry);
 }
 
+function sanitizeRequestDirName(value) {
+  if (typeof value !== 'string' || !value.trim()) {
+    return `req_${Date.now()}`;
+  }
+  return value.trim().replace(/[^A-Za-z0-9_.-]/g, '_');
+}
+
+function inferEntryFolder(reqId) {
+  const lower = String(reqId || '').toLowerCase();
+  if (lower.includes('openai-responses') || lower.includes('responses')) {
+    return 'openai-responses';
+  }
+  if (lower.includes('anthropic')) {
+    return 'anthropic-messages';
+  }
+  return 'openai-chat';
+}
+
 function parseArguments(argv) {
   const opts = { mode: 'all', reqId: undefined, provider: undefined, entry: undefined };
   for (const arg of argv) {