@jsonstudio/rcc 0.89.932 → 0.89.942

package/scripts/mock-provider/run-regressions.mjs

@@ -300,6 +300,67 @@ function collectInvalidNames(payload) {
   return failures;
 }
 
+function validateToolCallIds(payload, sample, tagSet) {
+  const errors = [];
+  if (!payload || typeof payload !== 'object') {
+    return errors;
+  }
+  const wantsFcIds =
+    tagSet.has('require_fc_call_ids') ||
+    tagSet.has('missing_tool_call_id') ||
+    tagSet.has('regression');
+
+  const allToolCallIds = new Set();
+  if (Array.isArray(payload.output)) {
+    payload.output.forEach((entry, oi) => {
+      if (!entry || typeof entry !== 'object') return;
+      const toolCalls = Array.isArray(entry.tool_calls) ? entry.tool_calls : [];
+      toolCalls.forEach((tc, ti) => {
+        if (!tc || typeof tc !== 'object') return;
+        const rawId = typeof tc.id === 'string' ? tc.id.trim() : '';
+        if (!rawId) {
+          errors.push(`output[${oi}].tool_calls[${ti}].id missing`);
+          return;
+        }
+        allToolCallIds.add(rawId);
+        if (wantsFcIds && !/^call_[A-Za-z0-9]+$/.test(rawId)) {
+          errors.push(`output[${oi}].tool_calls[${ti}].id has invalid format: ${rawId}`);
+        }
+      });
+    });
+  }
+
+  const requiredAction = payload.required_action;
+  const submit = requiredAction && typeof requiredAction === 'object'
+    ? requiredAction.submit_tool_outputs
+    : undefined;
+  const submitCalls = submit && typeof submit === 'object'
+    ? submit.tool_calls
+    : undefined;
+  if (Array.isArray(submitCalls)) {
+    submitCalls.forEach((tc, i) => {
+      if (!tc || typeof tc !== 'object') return;
+      const rawId = typeof tc.tool_call_id === 'string' ? tc.tool_call_id.trim() : '';
+      if (!rawId) {
+        errors.push(`required_action.submit_tool_outputs.tool_calls[${i}].tool_call_id missing`);
+        return;
+      }
+      if (wantsFcIds && !/^call_[A-Za-z0-9]+$/.test(rawId)) {
+        errors.push(
+          `required_action.submit_tool_outputs.tool_calls[${i}].tool_call_id has invalid format: ${rawId}`
+        );
+      }
+      if (allToolCallIds.size > 0 && !allToolCallIds.has(rawId)) {
+        errors.push(
+          `required_action.submit_tool_outputs.tool_calls[${i}].tool_call_id=${rawId} has no matching output.tool_calls entry`
+        );
+      }
+    });
+  }
+
+  return errors;
+}
+
 function extractRequestBody(doc) {
   const body = doc && typeof doc === 'object' ? doc.body : undefined;
   if (!body || typeof body !== 'object') {
@@ -379,6 +440,7 @@ async function runSample(sample, index) {
         return undefined;
       }
     })();
+    const tags = new Set(Array.isArray(sample.tags) ? sample.tags : []);
     if (body && Array.isArray(body.output)) {
       const invalid = collectInvalidNames(body);
       if (invalid.length) {
@@ -388,6 +450,14 @@ async function runSample(sample, index) {
             .join('\n')}`
         );
       }
+      const idErrors = validateToolCallIds(body, sample, tags);
+      if (idErrors.length) {
+        throw new Error(
+          `[${sample.reqId}] tool_call_id invariants failed:\n${idErrors
+            .map((msg) => ` - ${msg}`)
+            .join('\n')}`
+        );
+      }
     }
   } catch (error) {
     const logs = server.logs();
@@ -412,6 +482,10 @@ async function main() {
     console.warn('[mock:regressions] No regression-tagged samples matched current filters; skipping mock replay.');
     return;
   }
+
+  const coverageByEntry = Object.create(null);
+  const coverageByProvider = Object.create(null);
+
   const tagCounter = Object.create(null);
   const incrementTag = (tag) => {
     if (!watchedTags.has(tag)) {
@@ -421,6 +495,12 @@ async function main() {
   };
   regressionSamples.forEach((sample) => {
     const matched = new Set();
+    const entry = typeof sample.entry === 'string' && sample.entry.trim().length ? sample.entry.trim() : 'unknown';
+    const providerId =
+      typeof sample.providerId === 'string' && sample.providerId.trim().length ? sample.providerId.trim() : 'unknown';
+    coverageByEntry[entry] = (coverageByEntry[entry] || 0) + 1;
+    coverageByProvider[providerId] = (coverageByProvider[providerId] || 0) + 1;
+
     (sample.tags || []).forEach((tag) => {
       if (watchedTags.has(tag) && !matched.has(tag)) {
         incrementTag(tag);
@@ -436,6 +516,14 @@ async function main() {
     .map((tag) => `${tag}=${tagCounter[tag] || 0}`)
     .join(', ');
   console.log(`✅ mock provider regressions passed (${regressionSamples.length} samples · ${summary})`);
+  const byEntry = Object.entries(coverageByEntry)
+    .map(([entry, count]) => `${entry}=${count}`)
+    .join(', ');
+  const byProvider = Object.entries(coverageByProvider)
+    .map(([pid, count]) => `${pid}=${count}`)
+    .join(', ');
+  console.log(`[mock:regressions] coverage by entry: ${byEntry}`);
+  console.log(`[mock:regressions] coverage by providerId: ${byProvider}`);
 }
 
 main().catch((error) => {