@jsonstudio/rcc 0.89.1136 → 0.89.1205

package/scripts/unified-hub-shadow-compare.mjs

@@ -0,0 +1,359 @@
+#!/usr/bin/env node
+/**
+ * Unified Hub Framework V1 – Black-box shadow compare
+ *
+ * Runs the llmswitch-core HubPipeline twice (baseline vs candidate) on the same
+ * input payload and diffs providerPayload + key metadata.
+ *
+ * This is the gating tool for gradual rollout:
+ *   off → shadow(observe) → enforce → widen
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+import url from 'node:url';
+
+function resolveErrorSamplesRoot() {
+  const envOverride =
+    process.env.ROUTECODEX_ERRORSAMPLES_DIR ||
+    process.env.ROUTECODEX_ERROR_SAMPLES_DIR;
+  if (envOverride && String(envOverride).trim()) {
+    return path.resolve(String(envOverride).trim());
+  }
+  const home = process.env.HOME || process.env.USERPROFILE || '';
+  return path.join(home, '.routecodex', 'errorsamples');
+}
+
+function ensureDirSync(dir) {
+  try {
+    fs.mkdirSync(dir, { recursive: true });
+  } catch {
+    // ignore
+  }
+}
+
+function usage() {
+  console.log(`Usage:
+  node scripts/unified-hub-shadow-compare.mjs --request <file.json> [options]
+
+Options:
+  --request <file.json>             can be a raw request body JSON OR a codex-samples client-request.json
+  --entry-endpoint <path>        default: /v1/chat/completions
+  --route-hint <name>            optional: select route (default/openai/responses/anthropic/gemini)
+  --baseline-mode <off|observe|enforce>  default: off
+  --candidate-mode <off|observe|enforce> default: observe
+  --help                         show help
+`);
+}
+
+function parseArgs() {
+  const args = process.argv.slice(2);
+  const opts = {
+    entryEndpoint: '/v1/chat/completions',
+    baselineMode: 'off',
+    candidateMode: 'observe',
+    routeHint: undefined
+  };
+  for (let i = 0; i < args.length; i += 1) {
+    const a = args[i];
+    if (a === '--request') opts.request = args[++i];
+    else if (a === '--entry-endpoint') opts.entryEndpoint = args[++i];
+    else if (a === '--route-hint') opts.routeHint = args[++i];
+    else if (a === '--baseline-mode') opts.baselineMode = args[++i];
+    else if (a === '--candidate-mode') opts.candidateMode = args[++i];
+    else if (a === '--help' || a === '-h') {
+      usage();
+      process.exit(0);
+    } else {
+      console.error(`Unknown arg: ${a}`);
+      usage();
+      process.exit(2);
+    }
+  }
+  if (!opts.request) {
+    usage();
+    process.exit(2);
+  }
+  return opts;
+}
+
+function normalizeEntryProviderProtocol(entryEndpoint) {
+  const lowered = String(entryEndpoint || '').toLowerCase();
+  if (lowered.includes('/v1/responses')) return 'openai-responses';
+  if (lowered.includes('/v1/messages')) return 'anthropic-messages';
+  return 'openai-chat';
+}
+
+function readJson(file) {
+  return JSON.parse(fs.readFileSync(path.resolve(file), 'utf8'));
+}
+
+function extractEndpointFromSample(doc) {
+  return doc?.data?.url || doc?.url || doc?.endpoint || undefined;
+}
+
+function extractRouteHintFromSample(doc) {
+  const headers = doc?.headers;
+  if (!headers || typeof headers !== 'object') return undefined;
+  const hint = headers['X-Route-Hint'] || headers['x-route-hint'];
+  return typeof hint === 'string' && hint.trim() ? hint.trim() : undefined;
+}
+
+function extractBodyFromSample(doc) {
+  // Keep in sync with scripts/replay-codex-sample.mjs
+  const bodyNode = doc?.data?.body || doc?.body;
+  if (!bodyNode) {
+    if (typeof doc?.data?.data === 'object') return doc.data.data;
+    if (typeof doc?.body?.data === 'object') return doc.body.data;
+    return undefined;
+  }
+  if (typeof bodyNode?.body === 'object') return bodyNode.body;
+  if (typeof bodyNode === 'object') return bodyNode;
+  if (typeof doc?.data?.data === 'object') return doc.data.data;
+  if (typeof doc?.body?.data === 'object') return doc.body.data;
+  return undefined;
+}
+
+function readRequestPayloadAndHints(file) {
+  const doc = readJson(file);
+  const extracted = extractBodyFromSample(doc);
+  return {
+    payload: extracted && typeof extracted === 'object' ? extracted : doc,
+    entryEndpoint: extractEndpointFromSample(doc),
+    routeHint: extractRouteHintFromSample(doc)
+  };
+}
+
+function stableStringify(value) {
+  return JSON.stringify(
+    value,
+    (key, val) => {
+      if (val && typeof val === 'object' && !Array.isArray(val)) {
+        const out = {};
+        for (const k of Object.keys(val).sort()) {
+          out[k] = val[k];
+        }
+        return out;
+      }
+      return val;
+    },
+    2
+  );
+}
+
+function diffPayloads(expected, actual, p = '<root>') {
+  if (Object.is(expected, actual)) return [];
+  if (typeof expected !== typeof actual) {
+    return [{ path: p, expected, actual }];
+  }
+  if (Array.isArray(expected) && Array.isArray(actual)) {
+    const max = Math.max(expected.length, actual.length);
+    const diffs = [];
+    for (let i = 0; i < max; i += 1) {
+      diffs.push(...diffPayloads(expected[i], actual[i], `${p}[${i}]`));
+    }
+    return diffs;
+  }
+  if (expected && typeof expected === 'object' && actual && typeof actual === 'object') {
+    const keys = new Set([...Object.keys(expected), ...Object.keys(actual)]);
+    const diffs = [];
+    for (const key of keys) {
+      const next = p === '<root>' ? key : `${p}.${key}`;
+      if (!(key in actual)) diffs.push({ path: next, expected: expected[key], actual: undefined });
+      else if (!(key in expected)) diffs.push({ path: next, expected: undefined, actual: actual[key] });
+      else diffs.push(...diffPayloads(expected[key], actual[key], next));
+    }
+    return diffs;
+  }
+  return [{ path: p, expected, actual }];
+}
+
+function buildVirtualRouterConfig() {
+  // Minimal deterministic routes. We only need routing to select an outbound providerType/protocol.
+  return {
+    providers: {
+      mockOpenai: {
+        id: 'mockOpenai',
+        enabled: true,
+        type: 'openai',
+        baseURL: 'mock://openai',
+        auth: { type: 'apikey', apiKey: 'mock' },
+        models: { 'gpt-test': {} }
+      },
+      mockResponses: {
+        id: 'mockResponses',
+        enabled: true,
+        type: 'responses',
+        baseURL: 'mock://responses',
+        auth: { type: 'apikey', apiKey: 'mock' },
+        models: { 'gpt-test': {} }
+      },
+      mockAnthropic: {
+        id: 'mockAnthropic',
+        enabled: true,
+        type: 'anthropic',
+        baseURL: 'mock://anthropic',
+        auth: { type: 'apikey', apiKey: 'mock' },
+        models: { 'gpt-test': {} }
+      },
+      mockGemini: {
+        id: 'mockGemini',
+        enabled: true,
+        type: 'gemini',
+        baseURL: 'mock://gemini',
+        auth: { type: 'apikey', apiKey: 'mock' },
+        models: { 'gpt-test': {} }
+      }
+    },
+    routing: {
+      default: [
+        { id: 'default-primary', targets: ['mockOpenai.gpt-test'] }
+      ],
+      openai: [
+        { id: 'openai-primary', targets: ['mockOpenai.gpt-test'] }
+      ],
+      responses: [
+        { id: 'responses-primary', targets: ['mockResponses.gpt-test'] }
+      ],
+      anthropic: [
+        { id: 'anthropic-primary', targets: ['mockAnthropic.gpt-test'] }
+      ],
+      gemini: [
+        { id: 'gemini-primary', targets: ['mockGemini.gpt-test'] }
+      ]
+    }
+  };
+}
+
+async function importHubPipelineCtor() {
+  // Use the locally linked @jsonstudio/llms (symlinked to sharedmodule/llmswitch-core).
+  const repoRoot = path.resolve(path.dirname(url.fileURLToPath(import.meta.url)), '..');
+  const hubPath = path.join(repoRoot, 'sharedmodule', 'llmswitch-core', 'dist', 'conversion', 'hub', 'pipeline', 'hub-pipeline.js');
+  const mod = await import(url.pathToFileURL(hubPath).href);
+  if (typeof mod.HubPipeline !== 'function') {
+    throw new Error('HubPipeline ctor not found in built llmswitch-core dist. Run `npm run llmswitch:build` or `cd sharedmodule/llmswitch-core && npm run build`.');
+  }
+  return mod.HubPipeline;
+}
+
+async function bootstrapVirtualRouterConfig(rawConfig) {
+  const repoRoot = path.resolve(path.dirname(url.fileURLToPath(import.meta.url)), '..');
+  const bootstrapPath = path.join(
+    repoRoot,
+    'sharedmodule',
+    'llmswitch-core',
+    'dist',
+    'router',
+    'virtual-router',
+    'bootstrap.js'
+  );
+  const mod = await import(url.pathToFileURL(bootstrapPath).href);
+  if (typeof mod.bootstrapVirtualRouterConfig !== 'function') {
+    throw new Error('bootstrapVirtualRouterConfig not found in built llmswitch-core dist.');
+  }
+  return mod.bootstrapVirtualRouterConfig(rawConfig);
+}
+
+async function runOnce({ requestId, mode, entryEndpoint, routeHint, payload }) {
+  const HubPipeline = await importHubPipelineCtor();
+  const artifacts = await bootstrapVirtualRouterConfig(buildVirtualRouterConfig());
+  const pipeline = new HubPipeline({
+    virtualRouter: artifacts.config,
+    policy: { mode }
+  });
+  const providerProtocol = normalizeEntryProviderProtocol(entryEndpoint);
+  const result = await pipeline.execute({
+    id: requestId,
+    endpoint: entryEndpoint,
+    payload,
+    metadata: {
+      entryEndpoint,
+      providerProtocol,
+      routeHint
+    }
+  });
+  return result;
+}
+
+function writeCompareErrorSample(opts) {
+  const root = resolveErrorSamplesRoot();
+  if (!root || !String(root).trim()) return;
+  const dir = path.join(root, 'unified-hub-shadow');
+  ensureDirSync(dir);
+  const stamp = new Date().toISOString().replace(/[:.]/g, '-');
+  const file = path.join(dir, `unified-hub-shadow-diff-${stamp}.json`);
+  try {
+    fs.writeFileSync(file, JSON.stringify(opts, null, 2), 'utf8');
+    console.error(`[unified-hub-shadow-compare] wrote errorsample: ${file}`);
+  } catch {
+    // ignore
+  }
+}
+
+async function main() {
+  const opts = parseArgs();
+  const loaded = readRequestPayloadAndHints(opts.request);
+  const payload = loaded.payload;
+  const entryEndpoint = String(opts.entryEndpoint || loaded.entryEndpoint || '/v1/chat/completions');
+  const routeHint = opts.routeHint ? String(opts.routeHint) : (loaded.routeHint || undefined);
+  const baselineMode = String(opts.baselineMode);
+  const candidateMode = String(opts.candidateMode);
+  const requestId = `shadow_unified_hub_${Date.now()}`;
+
+  const baseline = await runOnce({ requestId, mode: baselineMode, entryEndpoint, routeHint, payload });
+  const candidate = await runOnce({ requestId, mode: candidateMode, entryEndpoint, routeHint, payload });
+
+  const baselineOut = {
+    providerPayload: baseline.providerPayload,
+    target: baseline.target,
+    metadata: {
+      entryEndpoint: baseline.metadata?.entryEndpoint,
+      providerProtocol: baseline.metadata?.providerProtocol,
+      processMode: baseline.metadata?.processMode,
+      stream: baseline.metadata?.stream,
+      routeHint: baseline.metadata?.routeHint
+    }
+  };
+  const candidateOut = {
+    providerPayload: candidate.providerPayload,
+    target: candidate.target,
+    metadata: {
+      entryEndpoint: candidate.metadata?.entryEndpoint,
+      providerProtocol: candidate.metadata?.providerProtocol,
+      processMode: candidate.metadata?.processMode,
+      stream: candidate.metadata?.stream,
+      routeHint: candidate.metadata?.routeHint
+    }
+  };
+
+  const diffs = diffPayloads(baselineOut, candidateOut);
+  if (!diffs.length) {
+    console.log('[unified-hub-shadow-compare] OK diff=0');
+    return;
+  }
+  console.error(`[unified-hub-shadow-compare] DIFF count=${diffs.length} (showing first 80)`);
+  diffs.slice(0, 80).forEach((d) => {
+    console.error(`- ${d.path}`);
+  });
+  writeCompareErrorSample({
+    kind: 'unified-hub-shadow-diff',
+    timestamp: new Date().toISOString(),
+    requestFile: path.resolve(opts.request),
+    entryEndpoint,
+    routeHint,
+    baselineMode,
+    candidateMode,
+    diffCount: diffs.length,
+    diffPaths: diffs.slice(0, 200).map((d) => d.path),
+    baseline: baselineOut,
+    candidate: candidateOut
+  });
+  console.error('\n--- baseline (stable) ---\n' + stableStringify(baselineOut));
+  console.error('\n--- candidate (stable) ---\n' + stableStringify(candidateOut));
+  process.exitCode = 1;
+}
+
+main().catch((err) => {
+  console.error('[unified-hub-shadow-compare] failed:', err);
+  process.exit(1);
+});

package/scripts/verify-e2e-gemini-followup-sample.mjs

@@ -0,0 +1,269 @@
+#!/usr/bin/env node
+/**
+ * E2E (real upstream) regression:
+ * - Replays a known-large OpenAI Responses payload that previously triggered
+ *   Gemini `MALFORMED_FUNCTION_CALL` (history tool args not aligned to schema),
+ *   causing empty reply + SERVERTOOL_EMPTY_FOLLOWUP (502).
+ *
+ * Safety: this hits real Antigravity/Gemini upstream and requires local auth.
+ * Default: skipped unless ROUTECODEX_VERIFY_ANTIGRAVITY=1.
+ */
+
+import { spawn } from 'node:child_process';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+
+if (String(process.env.ROUTECODEX_VERIFY_ANTIGRAVITY || '').trim() !== '1') {
+  console.log('[verify:e2e-gemini-followup] skip (set ROUTECODEX_VERIFY_ANTIGRAVITY=1 to enable)');
+  process.exit(0);
+}
+
+const VERIFY_PORT = process.env.ROUTECODEX_VERIFY_PORT || '5582';
+const VERIFY_BASE = process.env.ROUTECODEX_VERIFY_BASE_URL || `http://127.0.0.1:${VERIFY_PORT}`;
+const VERIFY_CONFIG =
+  process.env.ROUTECODEX_VERIFY_CONFIG ||
+  process.env.ROUTECODEX_CONFIG_PATH ||
+  `${process.env.HOME || ''}/.routecodex/config.json`;
+
+const DEFAULT_SAMPLE = path.join(
+  os.homedir(),
+  '.routecodex',
+  'codex-samples',
+  'openai-responses',
+  'antigravity.geetasamodgeetasamoda.gemini-3-pro-high',
+  'req_1768707507351_30f4e503',
+  'client-request.json'
+);
+
+function readJson(file) {
+  return JSON.parse(fs.readFileSync(file, 'utf8'));
+}
+
+function readServerApiKeyFromConfig(configPath) {
+  try {
+    const raw = fs.readFileSync(configPath, 'utf8');
+    const json = raw && raw.trim() ? JSON.parse(raw) : {};
+    const apikey = json?.httpserver?.apikey;
+    return typeof apikey === 'string' && apikey.trim() ? apikey.trim() : '';
+  } catch {
+    return '';
+  }
+}
+
+function buildAuthHeaders(serverApiKey) {
+  if (!serverApiKey) return {};
+  // middleware accepts x-api-key and many aliases; keep it simple.
+  return { 'x-api-key': serverApiKey };
+}
+
+function extractResponsesBody(sampleDoc) {
+  // common snapshot shapes:
+  // - { body: { body: <responses payload>, metadata: ... }, headers: ..., meta: ... }
+  // - { data: { body: <responses payload> } }
+  // - <responses payload>
+  const bodyNode = sampleDoc?.data?.body ?? sampleDoc?.body ?? sampleDoc;
+  if (bodyNode && typeof bodyNode === 'object' && typeof bodyNode.body === 'object' && bodyNode.body) {
+    return bodyNode.body;
+  }
+  if (bodyNode && typeof bodyNode === 'object') {
+    return bodyNode;
+  }
+  return undefined;
+}
+
+async function waitForServer(timeoutMs = 45_000) {
+  const start = Date.now();
+  while (Date.now() - start < timeoutMs) {
+    try {
+      const res = await fetch(`${VERIFY_BASE}/health`);
+      if (res.ok) return;
+    } catch {
+      // ignore
+    }
+    await new Promise((resolve) => setTimeout(resolve, 500));
+  }
+  throw new Error(`[verify:e2e-gemini-followup] server health timeout: ${VERIFY_BASE}/health`);
+}
+
+async function readSse(response) {
+  const reader = response.body?.getReader();
+  if (!reader) throw new Error('Response is not streamable');
+  const decoder = new TextDecoder();
+  let buffer = '';
+  const frames = [];
+  while (true) {
+    const { done, value } = await reader.read();
+    if (done) break;
+    buffer += decoder.decode(value, { stream: true });
+    let idx;
+    while ((idx = buffer.indexOf('\n\n')) !== -1) {
+      const chunk = buffer.slice(0, idx).trim();
+      buffer = buffer.slice(idx + 2);
+      if (chunk) frames.push(chunk);
+    }
+  }
+  buffer += decoder.decode(new Uint8Array(), { stream: false });
+  if (buffer.trim()) frames.push(buffer.trim());
+  return frames;
+}
+
+async function waitForFile(filePath, timeoutMs = 60_000) {
+  const start = Date.now();
+  while (Date.now() - start < timeoutMs) {
+    if (fs.existsSync(filePath)) return;
+    await new Promise((resolve) => setTimeout(resolve, 500));
+  }
+  throw new Error(`[verify:e2e-gemini-followup] timeout waiting for ${filePath}`);
+}
+
+async function main() {
+  if (!VERIFY_CONFIG) {
+    throw new Error('Missing ROUTECODEX_VERIFY_CONFIG/ROUTECODEX_CONFIG_PATH');
+  }
+
+  const serverApiKey = readServerApiKeyFromConfig(VERIFY_CONFIG);
+  if (!serverApiKey) {
+    throw new Error(`Missing httpserver.apikey in config: ${VERIFY_CONFIG}`);
+  }
+  const authHeaders = buildAuthHeaders(serverApiKey);
+
+  const samplePath = process.env.ROUTECODEX_VERIFY_SAMPLE || DEFAULT_SAMPLE;
+  if (!fs.existsSync(samplePath)) {
+    throw new Error(`Sample file not found: ${samplePath}`);
+  }
+  const sampleDoc = readJson(samplePath);
+  const payload = extractResponsesBody(sampleDoc);
+  if (!payload || typeof payload !== 'object') {
+    throw new Error('Sample did not contain a JSON request body');
+  }
+
+  const requestId =
+    (typeof process.env.ROUTECODEX_VERIFY_REQUEST_ID === 'string' && process.env.ROUTECODEX_VERIFY_REQUEST_ID.trim()) ||
+    `req_e2e_gemini_followup_${Date.now()}`;
+
+  const serverEnv = {
+    ...process.env,
+    ROUTECODEX_CONFIG_PATH: VERIFY_CONFIG,
+    ROUTECODEX_PORT: VERIFY_PORT,
+    ROUTECODEX_V2_HOOKS: '0',
+    RCC_V2_HOOKS: '0'
+  };
+
+  const debugLogs = String(process.env.ROUTECODEX_VERIFY_DEBUG || '').trim() === '1';
+  const serverLogPath = path.join(
+    os.tmpdir(),
+    `routecodex-verify-e2e-gemini-followup-${Date.now()}.log`
+  );
+  const serverLogStream = fs.createWriteStream(serverLogPath, { flags: 'a' });
+
+  const server = spawn('node', ['dist/index.js'], {
+    env: serverEnv,
+    stdio: ['ignore', 'pipe', 'pipe']
+  });
+
+  if (server.stdout) {
+    server.stdout.on('data', (chunk) => {
+      serverLogStream.write(chunk);
+      if (debugLogs) process.stdout.write(chunk);
+    });
+  }
+  if (server.stderr) {
+    server.stderr.on('data', (chunk) => {
+      serverLogStream.write(chunk);
+      if (debugLogs) process.stderr.write(chunk);
+    });
+  }
+
+  const shutdown = () => {
+    if (!server.killed) server.kill('SIGTERM');
+  };
+  process.on('SIGINT', shutdown);
+  process.on('SIGTERM', shutdown);
+
+  try {
+    await waitForServer();
+
+    // Keep headers minimal (we observed certain extra “sample headers” can trigger malformed capture in replay tooling).
+    const wantsSse = payload.stream === true;
+    const res = await fetch(`${VERIFY_BASE}/v1/responses`, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        Accept: wantsSse ? 'text/event-stream' : 'application/json',
+        'OpenAI-Beta': 'responses-2024-12-17',
+        'x-request-id': requestId,
+        ...(authHeaders || {})
+      },
+      body: JSON.stringify(payload)
+    });
+
+    if (!res.ok) {
+      const text = await res.text();
+      throw new Error(`HTTP ${res.status}: ${text}`);
+    }
+
+    if (wantsSse) {
+      const frames = await readSse(res);
+      const hasErrorFrame = frames.some((frame) => /^event:\\s*error\\b/m.test(frame));
+      if (hasErrorFrame) {
+        throw new Error(`SSE error frame returned (frames=${frames.length})`);
+      }
+    } else {
+      // ensure it is parseable JSON
+      await res.json();
+    }
+
+    // Snapshot assertions (ensures we really hit Gemini upstream and validated the mapped payload).
+    const snapDir = path.join(
+      os.homedir(),
+      '.routecodex',
+      'codex-samples',
+      'openai-responses',
+      'antigravity.geetasamodgeetasamoda.gemini-3-pro-high',
+      requestId
+    );
+    const providerReq = path.join(snapDir, 'provider-request.json');
+    const providerResp = path.join(snapDir, 'provider-response.json');
+
+    await waitForFile(providerReq);
+    await waitForFile(providerResp);
+
+    const providerReqText = fs.readFileSync(providerReq, 'utf8');
+    if (/\"cmd\"\s*:/.test(providerReqText)) {
+      throw new Error('provider-request still contains legacy tool arg key "cmd"');
+    }
+    if (!/\"command\"\s*:/.test(providerReqText)) {
+      throw new Error('provider-request missing expected tool arg key "command"');
+    }
+    if (!/\"instructions\"\s*:/.test(providerReqText)) {
+      throw new Error('provider-request missing expected tool arg key "instructions"');
+    }
+    if (!/\"chars\"\s*:/.test(providerReqText)) {
+      throw new Error('provider-request missing expected tool arg key "chars"');
+    }
+
+    const providerRespJson = readJson(providerResp);
+    const raw = providerRespJson?.body?.raw;
+    if (typeof raw === 'string' && raw.includes('MALFORMED_FUNCTION_CALL')) {
+      throw new Error('provider-response still contains MALFORMED_FUNCTION_CALL');
+    }
+
+    console.log(`✅ [verify:e2e-gemini-followup] OK requestId=${requestId}`);
+    if (!debugLogs) {
+      console.log(`[verify:e2e-gemini-followup] server log: ${serverLogPath}`);
+    }
+  } finally {
+    shutdown();
+    try {
+      serverLogStream.end();
+    } catch {
+      // ignore
+    }
+  }
+}
+
+main().catch((err) => {
+  console.error('[verify:e2e-gemini-followup] failed:', err instanceof Error ? err.stack || err.message : String(err));
+  process.exit(1);
+});