@veewo/gitnexus 1.5.0 → 1.5.2

package/dist/benchmark/agent-context/runner.js

@@ -6,6 +6,9 @@ function buildToolInput(step, repo) {
     if (repo) {
         input.repo = repo;
     }
+    if ((step.tool === 'query' || step.tool === 'context') && !('response_profile' in input)) {
+        input.response_profile = 'full';
+    }
     // LocalBackend impact contract uses `target_uid`, while dataset rows may carry `uid`.
     if (step.tool === 'impact') {
         const uid = input.uid;

package/dist/benchmark/agent-context/runner.test.js

@@ -77,3 +77,25 @@ test('executeToolPlan maps impact uid to target_uid for backend impact contract'
     assert.equal(calls.length, 1);
     assert.equal(calls[0].target_uid, 'Class:Assets/NEON/Code/NetworkCode/NeonMgr/MirrorNetMgr.cs:MirrorNetMgr');
 });
+test('executeToolPlan injects response_profile=full for legacy query/context payloads', async () => {
+    const calls = [];
+    const fakeRunner = {
+        query: async (params) => {
+            calls.push({ tool: 'query', params });
+            return {};
+        },
+        context: async (params) => {
+            calls.push({ tool: 'context', params });
+            return {};
+        },
+        impact: async () => ({}),
+        cypher: async () => ({}),
+        close: async () => { },
+    };
+    await executeToolPlan([
+        { tool: 'query', input: { query: 'Target' } },
+        { tool: 'context', input: { name: 'Target' } },
+    ], fakeRunner, 'sample-repo');
+    assert.equal(calls[0].params.response_profile, 'full');
+    assert.equal(calls[1].params.response_profile, 'full');
+});

package/dist/benchmark/agent-context/tool-runner.d.ts

@@ -1,7 +1,8 @@
-export declare function createAgentContextToolRunner(): Promise<{
-    query: (params: any) => Promise<any>;
-    context: (params: any) => Promise<any>;
-    impact: (params: any) => Promise<any>;
-    cypher: (params: any) => Promise<any>;
+export interface AgentContextToolRunner {
+    query: (params: Record<string, unknown>) => Promise<any>;
+    context: (params: Record<string, unknown>) => Promise<any>;
+    impact: (params: Record<string, unknown>) => Promise<any>;
+    cypher: (params: Record<string, unknown>) => Promise<any>;
     close: () => Promise<void>;
-}>;
+}
+export declare function createAgentContextToolRunner(): Promise<AgentContextToolRunner>;

package/dist/benchmark/agent-safe-query-context/io.d.ts

@@ -0,0 +1,2 @@
+import type { AgentSafeBenchmarkSuite } from './types.js';
+export declare function loadAgentSafeQueryContextSuite(root: string): Promise<AgentSafeBenchmarkSuite>;

package/dist/benchmark/agent-safe-query-context/io.js

@@ -0,0 +1,86 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { AGENT_SAFE_CASE_KEYS } from './types.js';
+const PLACEHOLDER_RE = /TODO|TBD|placeholder|<resource>|<symbol>/i;
+export async function loadAgentSafeQueryContextSuite(root) {
+    const thresholds = JSON.parse(await fs.readFile(path.join(root, 'thresholds.json'), 'utf-8'));
+    const cases = JSON.parse(await fs.readFile(path.join(root, 'cases.json'), 'utf-8'));
+    for (const key of AGENT_SAFE_CASE_KEYS) {
+        assertCase(key, cases[key]);
+    }
+    return { thresholds, cases };
+}
+function assertCase(name, value) {
+    if (!value) {
+        throw new Error(`missing required case: ${name}`);
+    }
+    for (const field of ['label', 'start_query', 'retry_query', 'proof_cypher']) {
+        const candidate = value[field];
+        if (!candidate || typeof candidate !== 'string') {
+            throw new Error(`missing required field: ${name}.${field}`);
+        }
+        assertNoPlaceholder(`${name}.${field}`, candidate);
+    }
+    if (!Array.isArray(value.proof_contexts) || value.proof_contexts.length === 0) {
+        throw new Error(`missing required field: ${name}.proof_contexts`);
+    }
+    value.proof_contexts.forEach((entry, index) => assertNoPlaceholder(`${name}.proof_contexts[${index}]`, entry));
+    if (!Array.isArray(value.tool_plan) || value.tool_plan.length === 0) {
+        throw new Error(`missing required field: ${name}.tool_plan`);
+    }
+    assertLiveTask(name, value.live_task, value.semantic_tuple);
+    if (value.start_query_input && typeof value.start_query_input === 'object') {
+        for (const entry of Object.values(value.start_query_input)) {
+            if (typeof entry === 'string') {
+                assertNoPlaceholder(`${name}.start_query_input`, entry);
+            }
+        }
+    }
+    if (value.retry_query_input && typeof value.retry_query_input === 'object') {
+        for (const entry of Object.values(value.retry_query_input)) {
+            if (typeof entry === 'string') {
+                assertNoPlaceholder(`${name}.retry_query_input`, entry);
+            }
+        }
+    }
+    assertSemanticTuple(name, value.semantic_tuple);
+}
+function assertLiveTask(name, liveTask, tuple) {
+    if (!liveTask) {
+        throw new Error(`missing required field: ${name}.live_task`);
+    }
+    for (const field of ['objective', 'symbol_seed', 'resource_seed']) {
+        const candidate = liveTask[field];
+        if (!candidate || typeof candidate !== 'string') {
+            throw new Error(`missing required field: ${name}.live_task.${field}`);
+        }
+        assertNoPlaceholder(`${name}.live_task.${field}`, candidate);
+    }
+    if (tuple.proof_edge && liveTask.objective.includes(tuple.proof_edge)) {
+        throw new Error(`${name}.live_task.objective leaks canonical proof_edge`);
+    }
+    if (tuple.proof_edges?.every((edge) => liveTask.objective.includes(edge))) {
+        throw new Error(`${name}.live_task.objective leaks canonical proof_edges`);
+    }
+}
+function assertSemanticTuple(name, tuple) {
+    if (!tuple) {
+        throw new Error(`missing required field: ${name}.semantic_tuple`);
+    }
+    assertNoPlaceholder(`${name}.semantic_tuple.resource_anchor`, tuple.resource_anchor);
+    assertNoPlaceholder(`${name}.semantic_tuple.symbol_anchor`, tuple.symbol_anchor);
+    if (tuple.proof_edge) {
+        assertNoPlaceholder(`${name}.semantic_tuple.proof_edge`, tuple.proof_edge);
+    }
+    if (tuple.proof_edges) {
+        tuple.proof_edges.forEach((entry, index) => assertNoPlaceholder(`${name}.semantic_tuple.proof_edges[${index}]`, entry));
+    }
+    if (!tuple.proof_edge && (!tuple.proof_edges || tuple.proof_edges.length === 0)) {
+        throw new Error(`missing proof edge(s): ${name}.semantic_tuple`);
+    }
+}
+function assertNoPlaceholder(field, value) {
+    if (PLACEHOLDER_RE.test(value)) {
+        throw new Error(`${field} contains placeholder text`);
+    }
+}

package/dist/benchmark/agent-safe-query-context/io.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/benchmark/agent-safe-query-context/io.test.js

@@ -0,0 +1,13 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import path from 'node:path';
+import { loadAgentSafeQueryContextSuite } from './io.js';
+test('loads canonical benchmark cases without placeholders', async () => {
+    const suite = await loadAgentSafeQueryContextSuite(path.resolve('../benchmarks/agent-safe-query-context/neonspark-v1'));
+    assert.deepEqual(Object.keys(suite.cases).sort(), ['reload', 'weapon_powerup']);
+    assert.equal(suite.cases.weapon_powerup.semantic_tuple.resource_anchor, 'Assets/NEON/DataAssets/Powerups/1_newWeapon/0_pick/法器_Orb/1_weapon_orb_key.asset');
+    assert.equal(suite.cases.reload.semantic_tuple.proof_edge, 'ReloadBase.GetValue -> ReloadBase.CheckReload');
+    assert.equal(suite.cases.weapon_powerup.live_task.symbol_seed, 'WeaponPowerUp');
+    assert.equal(suite.cases.reload.live_task.resource_seed, 'Assets/NEON/Graphs/PlayerGun/Gungraph_use/1_weapon_orb_key.asset');
+    assert.equal(suite.cases.weapon_powerup.live_task.objective.includes('HoldPickup -> WeaponPowerUp.PickItUp'), false);
+});

package/dist/benchmark/agent-safe-query-context/report.d.ts

@@ -0,0 +1,57 @@
+import { executeToolPlan } from '../agent-context/runner.js';
+import { type AgentContextToolRunner } from '../agent-context/tool-runner.js';
+import { type WorkflowReplayResult } from './runner.js';
+import { loadSubagentLiveCaseResult, type SubagentLiveResult, type TelemetryStep } from './subagent-live.js';
+import type { AgentSafeBenchmarkCase, AgentSafeBenchmarkSuite, AgentSafeCaseKey, SemanticTuple } from './types.js';
+type CaseKey = AgentSafeCaseKey;
+export interface SameScriptCaseResult {
+    tool_plan: AgentSafeBenchmarkCase['tool_plan'];
+    steps: TelemetryStep[];
+    semantic_tuple: SemanticTuple;
+    semantic_tuple_pass: boolean;
+    tool_calls_to_completion: number;
+    tokens_to_completion: number;
+}
+export interface AgentSafeQueryContextBenchmarkReport {
+    generatedAt: string;
+    workflow_replay_full: Record<CaseKey, WorkflowReplayResult>;
+    workflow_replay_slim: Record<CaseKey, WorkflowReplayResult>;
+    same_script_full: Record<CaseKey, SameScriptCaseResult>;
+    same_script_slim: Record<CaseKey, SameScriptCaseResult>;
+    subagent_live: Record<CaseKey, SubagentLiveResult>;
+    acceptance: {
+        pass: boolean;
+        cases: Record<CaseKey, boolean>;
+    };
+    pass: boolean;
+    cases: Record<CaseKey, SubagentLiveResult>;
+    same_script: {
+        tool_plan: Record<CaseKey, AgentSafeBenchmarkCase['tool_plan']>;
+        cases: Record<CaseKey, SameScriptCaseResult>;
+    };
+    semantic_equivalence: {
+        pass: boolean;
+        cases: Record<CaseKey, boolean>;
+    };
+    token_summary: Record<CaseKey, {
+        before: number;
+        after: number;
+        saved: number;
+        reduction: number;
+    }>;
+    call_summary: Record<CaseKey, {
+        before: number;
+        after: number;
+        saved: number;
+    }>;
+}
+export declare function runAgentSafeQueryContextBenchmark(suite: AgentSafeBenchmarkSuite, options: {
+    repo?: string;
+    subagentRunsDir?: string;
+}, deps?: {
+    runner?: AgentContextToolRunner;
+    executeToolPlan?: typeof executeToolPlan;
+    loadSubagentLiveCaseResult?: typeof loadSubagentLiveCaseResult;
+}): Promise<AgentSafeQueryContextBenchmarkReport>;
+export declare function writeAgentSafeQueryContextReports(reportDir: string, report: AgentSafeQueryContextBenchmarkReport): Promise<void>;
+export {};

package/dist/benchmark/agent-safe-query-context/report.js

@@ -0,0 +1,159 @@
+import path from 'node:path';
+import { estimateTokens } from '../u2-e2e/metrics.js';
+import { writeReports } from '../report.js';
+import { executeToolPlan } from '../agent-context/runner.js';
+import { createAgentContextToolRunner } from '../agent-context/tool-runner.js';
+import { deriveSemanticTuple, semanticTuplePass } from './semantic-tuple.js';
+import { runWorkflowReplay } from './runner.js';
+import { loadSubagentLiveCaseResult } from './subagent-live.js';
+export async function runAgentSafeQueryContextBenchmark(suite, options, deps = {}) {
+    const runner = deps.runner || (await createAgentContextToolRunner());
+    const ownsRunner = !deps.runner;
+    const executeToolPlanImpl = deps.executeToolPlan || executeToolPlan;
+    const loadSubagentLiveCaseResultImpl = deps.loadSubagentLiveCaseResult || loadSubagentLiveCaseResult;
+    const workflowReplayFullCases = {};
+    const workflowReplaySlimCases = {};
+    const sameScriptFullCases = {};
+    const sameScriptSlimCases = {};
+    const subagentLiveCases = {};
+    const acceptanceCases = {};
+    const semanticEquivalenceCases = {};
+    const tokenSummary = {};
+    const callSummary = {};
+    if (!options.subagentRunsDir) {
+        throw new Error('subagentRunsDir is required for real subagent benchmark runs');
+    }
+    try {
+        for (const key of Object.keys(suite.cases)) {
+            const benchmarkCase = suite.cases[key];
+            const workflowReplayFull = await runWorkflowReplay(benchmarkCase, runner, {
+                repo: options.repo,
+                maxSteps: suite.thresholds.workflowReplay.maxSteps,
+                responseProfile: 'full',
+            });
+            const workflowReplaySlim = await runWorkflowReplay(benchmarkCase, runner, {
+                repo: options.repo,
+                maxSteps: suite.thresholds.workflowReplay.maxSteps,
+                responseProfile: 'slim',
+            });
+            const sameScriptFull = await runSameScriptCase(benchmarkCase, runner, executeToolPlanImpl, {
+                repo: options.repo,
+                responseProfile: 'full',
+            });
+            const sameScriptSlim = await runSameScriptCase(benchmarkCase, runner, executeToolPlanImpl, {
+                repo: options.repo,
+                responseProfile: 'slim',
+            });
+            const subagentLive = await loadSubagentLiveCaseResultImpl(path.join(options.subagentRunsDir, key), benchmarkCase);
+            workflowReplayFullCases[key] = workflowReplayFull;
+            workflowReplaySlimCases[key] = workflowReplaySlim;
+            sameScriptFullCases[key] = sameScriptFull;
+            sameScriptSlimCases[key] = sameScriptSlim;
+            subagentLiveCases[key] = subagentLive;
+            acceptanceCases[key] = workflowReplaySlim.semantic_tuple_pass
+                && workflowReplaySlim.post_narrowing_anchor_pass
+                && workflowReplaySlim.post_narrowing_follow_up_hit
+                && workflowReplaySlim.guid_invariance_pass
+                && workflowReplaySlim.live_tool_evidence_pass
+                && workflowReplaySlim.freeze_ready
+                && workflowReplaySlim.tier_envelope.facts_present
+                && workflowReplaySlim.tier_envelope.closure_present
+                && workflowReplaySlim.tier_envelope.clues_present
+                && workflowReplaySlim.tier_envelope.semantic_order_pass
+                && !workflowReplaySlim.placeholder_leak_detected
+                && !workflowReplaySlim.heuristic_top_summary_detected;
+            semanticEquivalenceCases[key] = sameScriptSlim.semantic_tuple_pass && subagentLive.semantic_tuple_pass;
+            const tokenSaved = sameScriptFull.tokens_to_completion - sameScriptSlim.tokens_to_completion;
+            tokenSummary[key] = {
+                before: sameScriptFull.tokens_to_completion,
+                after: sameScriptSlim.tokens_to_completion,
+                saved: tokenSaved,
+                reduction: sameScriptFull.tokens_to_completion > 0 ? Number((tokenSaved / sameScriptFull.tokens_to_completion).toFixed(3)) : 0,
+            };
+            callSummary[key] = {
+                before: sameScriptFull.tool_calls_to_completion,
+                after: sameScriptSlim.tool_calls_to_completion,
+                saved: sameScriptFull.tool_calls_to_completion - sameScriptSlim.tool_calls_to_completion,
+            };
+        }
+    }
+    finally {
+        if (ownsRunner) {
+            await runner.close();
+        }
+    }
+    const pass = Object.values(acceptanceCases).every(Boolean);
+    return {
+        generatedAt: new Date().toISOString(),
+        workflow_replay_full: workflowReplayFullCases,
+        workflow_replay_slim: workflowReplaySlimCases,
+        same_script_full: sameScriptFullCases,
+        same_script_slim: sameScriptSlimCases,
+        subagent_live: subagentLiveCases,
+        acceptance: {
+            pass,
+            cases: acceptanceCases,
+        },
+        pass,
+        cases: subagentLiveCases,
+        same_script: {
+            tool_plan: {
+                weapon_powerup: suite.cases.weapon_powerup.tool_plan,
+                reload: suite.cases.reload.tool_plan,
+            },
+            cases: sameScriptSlimCases,
+        },
+        semantic_equivalence: {
+            pass: Object.values(semanticEquivalenceCases).every(Boolean),
+            cases: semanticEquivalenceCases,
+        },
+        token_summary: tokenSummary,
+        call_summary: callSummary,
+    };
+}
+export async function writeAgentSafeQueryContextReports(reportDir, report) {
+    const markdown = [
+        '# Agent-Safe Query/Context Benchmark Summary',
+        '',
+        `- Pass: ${report.acceptance.pass ? 'YES' : 'NO'}`,
+        '',
+        '## Cases',
+        ...['weapon_powerup', 'reload'].map((key) => `- ${key}: live_pass=${report.subagent_live[key].semantic_tuple_pass}, token_saved=${report.token_summary[key].saved}, call_saved=${report.call_summary[key].saved}, anchor_top1_pass=${report.workflow_replay_slim[key].anchor_top1_pass}, recommended_follow_up_hit=${report.workflow_replay_slim[key].recommended_follow_up_hit}, post_narrowing_anchor_pass=${report.workflow_replay_slim[key].post_narrowing_anchor_pass}, post_narrowing_follow_up_hit=${report.workflow_replay_slim[key].post_narrowing_follow_up_hit}, guid_invariance_pass=${report.workflow_replay_slim[key].guid_invariance_pass}, live_tool_evidence_pass=${report.workflow_replay_slim[key].live_tool_evidence_pass}, freeze_ready=${report.workflow_replay_slim[key].freeze_ready}, confirmed_chain_steps=${report.workflow_replay_slim[key].confirmed_chain.steps.length}, tier_facts=${report.workflow_replay_slim[key].tier_envelope.facts_present}, tier_closure=${report.workflow_replay_slim[key].tier_envelope.closure_present}, tier_clues=${report.workflow_replay_slim[key].tier_envelope.clues_present}, tier_semantic_order=${report.workflow_replay_slim[key].tier_envelope.semantic_order_pass}, tier_summary_source=${report.workflow_replay_slim[key].tier_envelope.summary_source}, ambiguity_detour_count=${report.workflow_replay_slim[key].ambiguity_detour_count}, placeholder_leak_detected=${report.workflow_replay_slim[key].placeholder_leak_detected}, heuristic_top_summary_detected=${report.workflow_replay_slim[key].heuristic_top_summary_detected}`),
+    ].join('\n');
+    await writeReports(reportDir, report, markdown);
+}
+async function runSameScriptCase(benchmarkCase, runner, executeToolPlanImpl, options) {
+    const toolPlan = applyResponseProfileToToolPlan(benchmarkCase.tool_plan, options.responseProfile);
+    const outputs = await executeToolPlanImpl(toolPlan, runner, options.repo);
+    const steps = outputs.map((step) => ({
+        tool: step.tool,
+        input: step.input,
+        output: step.output,
+        durationMs: 0,
+        totalTokensEst: estimateTokens(JSON.stringify(step.input)) + estimateTokens(JSON.stringify(step.output)),
+        timestamp: new Date(0).toISOString(),
+    }));
+    const semanticTuple = deriveSemanticTuple(benchmarkCase.semantic_tuple, steps.map((step) => step.output));
+    return {
+        tool_plan: toolPlan,
+        steps,
+        semantic_tuple: semanticTuple,
+        semantic_tuple_pass: semanticTuplePass(semanticTuple, benchmarkCase.semantic_tuple),
+        tool_calls_to_completion: steps.length,
+        tokens_to_completion: steps.reduce((sum, step) => sum + step.totalTokensEst, 0),
+    };
+}
+function applyResponseProfileToToolPlan(toolPlan, responseProfile) {
+    return toolPlan.map((step) => {
+        if (step.tool !== 'query' && step.tool !== 'context') {
+            return step;
+        }
+        return {
+            ...step,
+            input: {
+                ...step.input,
+                response_profile: responseProfile,
+            },
+        };
+    });
+}

package/dist/benchmark/agent-safe-query-context/report.test.d.ts

@@ -0,0 +1 @@
+export {};