@devflow-tools/benchmark 0.16.27 → 0.16.29

package/fixtures/retrieval-quality-corpus.json

@@ -1,18 +1,36 @@
 {
   "schemaVersion": "devflow.retrieval-quality-corpus.v1",
   "maxTokenGrowthRatio": 0.5,
+  "trialCount": 3,
+  "requiredAcceptanceClasses": [
+    "exact-positive", "paraphrase", "mixed-language", "expected-empty", "out-of-domain",
+    "incompatible-version", "stale-file", "weak-graph", "embedding-degraded",
+    "reranker-unavailable", "policy-action", "session-end-reconciliation"
+  ],
   "projects": [
     {
       "id": "devflow",
       "projectRoot": "../../..",
       "fixturePath": "retrieval-quality-devflow.json",
-      "prepareCodeGraph": true
+      "prepareCodeGraph": true,
+      "acceptanceClasses": ["exact-positive", "policy-action", "session-end-reconciliation"]
     },
     {
       "id": "react-video",
       "projectRoot": "projects/react-video",
       "fixturePath": "retrieval-quality-react-video.json",
-      "prepareCodeGraph": true
+      "prepareCodeGraph": true,
+      "acceptanceClasses": [
+        "paraphrase", "mixed-language", "expected-empty", "out-of-domain",
+        "incompatible-version", "stale-file", "weak-graph", "embedding-degraded", "reranker-unavailable"
+      ]
+    },
+    {
+      "id": "rsbuild-react-quality",
+      "projectRoot": "projects/rsbuild-react-quality",
+      "fixturePath": "retrieval-quality-rsbuild-react.json",
+      "prepareCodeGraph": true,
+      "acceptanceClasses": ["exact-positive", "expected-empty", "policy-action", "embedding-degraded"]
     },
     {
       "id": "rsbuild-react",
@@ -21,5 +39,42 @@
       "optional": true,
       "deferred": true
     }
+  ],
+  "authoritativeScenarios": [
+    {
+      "id": "http-expected-empty",
+      "acceptanceClasses": ["expected-empty", "out-of-domain"],
+      "expected": { "memoryIds": [], "coverageGaps": ["knowledge_source_missing"], "codePaths": ["src/components/Login.tsx"], "codeRoles": ["target"], "domainAction": "devflow_unsupported_action", "verification": ["unknown"], "ledgerTransitions": ["selected", "exposed", "unknown"] }
+    },
+    {
+      "id": "prettier-explicit-immediate",
+      "acceptanceClasses": ["exact-positive", "policy-action"],
+      "expected": { "memoryIds": ["prettier-project-policy"], "memoryFacets": ["formatting"], "knowledgeSources": [], "codePaths": ["src/components/Login.tsx"], "codeRoles": ["target"], "domainAction": "react_refactor_component", "verification": ["formatter", "typecheck"], "ledgerTransitions": ["selected", "exposed", "adopted", "positive_outcome"] }
+    },
+    {
+      "id": "login-tailwind-shadcn-prettier",
+      "acceptanceClasses": ["mixed-language", "policy-action"],
+      "expected": { "memoryFacets": ["formatting", "styling", "component_library"], "knowledgeSources": ["react", "tailwindcss"], "codePaths": ["src/components/Login.tsx"], "codeRoles": ["target"], "domainAction": "react_new_component", "verification": ["formatter", "typecheck"], "ledgerTransitions": ["eligible", "candidate", "selected", "exposed", "adopted", "positive_outcome"] }
+    },
+    {
+      "id": "successful-grep-no-learning",
+      "acceptanceClasses": ["expected-empty"],
+      "expected": { "memoryIds": [], "coverageGaps": [], "codePaths": ["src/components/Login.tsx"], "codeRoles": ["target"], "domainAction": "react_audit_performance", "verification": ["unknown"], "ledgerTransitions": ["selected", "exposed", "unknown"] }
+    },
+    {
+      "id": "stale-component-exclusion",
+      "acceptanceClasses": ["stale-file", "weak-graph"],
+      "expected": { "memoryIds": [], "coverageGaps": [], "codePaths": ["src/components/Login.tsx"], "codeRoles": ["target"], "domainAction": "react_review_hooks", "verification": ["unknown"], "ledgerTransitions": ["rejected", "selected", "exposed", "unknown"] }
+    },
+    {
+      "id": "api-specific-knowledge-rejection",
+      "acceptanceClasses": ["incompatible-version", "embedding-degraded", "reranker-unavailable"],
+      "expected": { "memoryIds": [], "knowledgeSources": ["react"], "knowledgeChunks": ["api"], "codePaths": ["src/components/Login.tsx"], "codeRoles": ["target"], "domainAction": "react_review_hooks", "verification": ["unknown"], "ledgerTransitions": ["candidate", "rejected", "selected", "exposed", "unknown"] }
+    },
+    {
+      "id": "session-exit-closure",
+      "acceptanceClasses": ["session-end-reconciliation"],
+      "expected": { "memoryIds": [], "coverageGaps": [], "codePaths": ["src/components/Login.tsx"], "codeRoles": ["target"], "domainAction": "react_audit_performance", "verification": ["session_end_receipt"], "ledgerTransitions": ["selected", "exposed", "unknown"] }
+    }
   ]
 }

package/fixtures/retrieval-quality-devflow.json

@@ -2,15 +2,18 @@
   "schemaVersion": "devflow.retrieval-quality.v1",
   "thresholds": {
     "codeFileRecallAt5": 0.85,
-    "codeFilePrecisionAt5": 0.75,
+    "codeFilePrecisionAt5": 0.85,
     "symbolNdcgAt5": 0.8,
     "falseCrossFileEdgeRate": 0.01,
     "memoryPrecisionAt5": 0.8,
     "expectedEmptyMemoryAccuracy": 1,
     "knowledgeNdcgAt5": 0.85,
+    "knowledgePrecisionAt3": 0.8,
     "knowledgeVectorCoverage": 0.95,
     "knowledgeChunkP95": 3000,
     "knowledgeVersionMatch": 1,
+    "maxKnowledgeInjectionViolations": 0,
+    "maxStaleSelectedFiles": 0,
     "maxRetrievalCycle": 3
   },
   "queries": [
@@ -22,8 +25,7 @@
         { "path": "packages/database/src/database.ts", "role": "target" },
         { "path": "packages/database/__tests__/database.host-actions.test.ts", "role": "caller" },
         { "path": "packages/database/__tests__/database.memory-turn-receipts.test.ts", "role": "caller" },
-        { "path": "packages/database/src/retrieval-sessions.ts", "role": "dependency" },
-        { "path": "packages/database/src/types.ts", "role": "dependency" }
+        { "path": "packages/database/src/retrieval-sessions.ts", "role": "target" }
       ],
       "expectedEmptyMemory": true
     }

package/fixtures/retrieval-quality-react-video.json

@@ -2,15 +2,18 @@
   "schemaVersion": "devflow.retrieval-quality.v1",
   "thresholds": {
     "codeFileRecallAt5": 0.85,
-    "codeFilePrecisionAt5": 0.75,
+    "codeFilePrecisionAt5": 0.85,
     "symbolNdcgAt5": 0.8,
     "falseCrossFileEdgeRate": 0.01,
     "memoryPrecisionAt5": 0.8,
     "expectedEmptyMemoryAccuracy": 1,
     "knowledgeNdcgAt5": 0.85,
+    "knowledgePrecisionAt3": 0.8,
     "knowledgeVectorCoverage": 0.95,
     "knowledgeChunkP95": 3000,
     "knowledgeVersionMatch": 1,
+    "maxKnowledgeInjectionViolations": 0,
+    "maxStaleSelectedFiles": 0,
     "maxRetrievalCycle": 3
   },
   "queries": [
@@ -26,6 +29,65 @@
       ],
       "relevantSymbols": ["VideoPlayer"],
       "expectedEmptyMemory": true
+    },
+    {
+      "id": "video-player-performance-paraphrase-zh-en",
+      "query": "审查 src/components/video-player/VideoPlayer.tsx 的 React render 性能和调用方",
+      "intent": "performance",
+      "relevantCode": [
+        { "path": "src/components/video-player/VideoPlayer.tsx", "role": "target" },
+        { "path": "src/pages/VideoPlayerPage.tsx", "role": "caller" },
+        { "path": "src/components/video-player/VideoPlayer.css", "role": "resource" },
+        { "path": "src/routes/video-player.tsx", "role": "route" }
+      ],
+      "relevantSymbols": ["VideoPlayer"],
+      "expectedEmptyMemory": true,
+      "degradation": "reranker_unavailable"
+    },
+    {
+      "id": "out-of-domain-expected-empty",
+      "query": "explain Kubernetes ingress certificate rotation",
+      "intent": "understand",
+      "expectedEmptyMemory": true,
+      "expectedEmptyKnowledge": true
+    },
+    {
+      "id": "incompatible-version-no-substitution",
+      "query": "modify the React 18 useEffect API integration in src/components/video-player/VideoPlayer.tsx",
+      "intent": "feature",
+      "relevantCode": [
+        { "path": "src/components/video-player/VideoPlayer.tsx", "role": "target" },
+        { "path": "src/pages/VideoPlayerPage.tsx", "role": "caller" },
+        { "path": "src/components/video-player/VideoPlayer.css", "role": "resource" },
+        { "path": "src/routes/video-player.tsx", "role": "route" }
+      ],
+      "forbiddenKnowledgeSources": ["typescript", "browser"],
+      "forbiddenKnowledgeKinds": ["overview"],
+      "degradation": "embedding_unavailable"
+    },
+    {
+      "id": "create-rejects-generic-overview",
+      "query": "create a playback control in src/components/video-player/VideoPlayer.tsx",
+      "intent": "feature",
+      "relevantCode": [
+        { "path": "src/components/video-player/VideoPlayer.tsx", "role": "target" },
+        { "path": "src/pages/VideoPlayerPage.tsx", "role": "caller" },
+        { "path": "src/components/video-player/VideoPlayer.css", "role": "resource" },
+        { "path": "src/routes/video-player.tsx", "role": "route" }
+      ],
+      "forbiddenKnowledgeKinds": ["overview"]
+    },
+    {
+      "id": "stale-file-guard",
+      "query": "analyze the current VideoPlayer implementation in src/components/video-player/VideoPlayer.tsx",
+      "intent": "review",
+      "relevantCode": [
+        { "path": "src/components/video-player/VideoPlayer.tsx", "role": "target" },
+        { "path": "src/pages/VideoPlayerPage.tsx", "role": "caller" },
+        { "path": "src/components/video-player/VideoPlayer.css", "role": "resource" },
+        { "path": "src/routes/video-player.tsx", "role": "route" }
+      ],
+      "expectedEmptyMemory": true
     }
   ],
   "forbiddenCodeEdges": [

package/fixtures/retrieval-quality-rsbuild-react.json

@@ -0,0 +1,93 @@
+{
+  "schemaVersion": "devflow.retrieval-quality.v1",
+  "thresholds": {
+    "codeFileRecallAt5": 1,
+    "codeFilePrecisionAt5": 1,
+    "symbolNdcgAt5": 0.8,
+    "falseCrossFileEdgeRate": 0.01,
+    "memoryRecallAt5": 0.9,
+    "policyMemoryRecallAt5": 1,
+    "memoryPrecisionAt5": 0.95,
+    "expectedEmptyMemoryAccuracy": 1,
+    "knowledgeNdcgAt5": 0.85,
+    "knowledgePrecisionAt3": 0.8,
+    "knowledgeVectorCoverage": 0,
+    "knowledgeChunkP95": 3000,
+    "knowledgeVersionMatch": 1,
+    "maxKnowledgeInjectionViolations": 0,
+    "maxStaleSelectedFiles": 0,
+    "maxRetrievalCycle": 3,
+    "minPolicyActionAccuracy": 1,
+    "minLedgerCompleteness": 1,
+    "maxSameSessionMemoryReadyMs": 5000
+  },
+  "setup": {
+    "resetProjectState": true,
+    "explicitMemories": [
+      "In this project, format code with Prettier.",
+      "In this project, style components with Tailwind CSS.",
+      "In this project, use shadcn as the component library.",
+      "In this project, build components with React."
+    ],
+    "knowledgeSources": [
+      {
+        "name": "react",
+        "pluginName": "builtin",
+        "version": "18.3.0",
+        "chunks": [
+          {
+            "title": "React component rendering",
+            "content": "React components should keep render work focused. Profile component commits before changing update frequency. A Login component should use controlled form state and accessible labels.",
+            "sourceUrl": "https://react.dev/reference/react"
+          }
+        ]
+      }
+    ]
+  },
+  "queries": [
+    {
+      "id": "http-expected-empty",
+      "query": "audit HTTP request latency and connection reuse",
+      "intent": "performance",
+      "expectedEmptyMemory": true,
+      "expectedEmptyKnowledge": true
+    },
+    {
+      "id": "project-ui-conventions",
+      "query": "What Prettier Tailwind and shadcn conventions apply to src/components/Login.tsx?",
+      "intent": "understand",
+      "relevantCode": [{ "path": "src/components/Login.tsx", "role": "target" }],
+      "relevantMemoryTerms": ["prettier", "tailwind", "shadcn"]
+    },
+    {
+      "id": "login-tailwind-shadcn-prettier",
+      "query": "create Login component in src/components/Login.tsx",
+      "intent": "feature",
+      "activeSkill": "devflow:react",
+      "relevantCode": [{ "path": "src/components/Login.tsx", "role": "target" }],
+      "relevantMemoryTerms": ["prettier", "tailwind", "shadcn", "react"],
+      "relevantKnowledgeSources": ["react"],
+      "expectedPolicyFacets": ["formatting", "styling", "component_library", "framework"],
+      "expectedDomainAction": "react_new_component",
+      "expectedVerificationKinds": ["formatter", "typecheck"]
+    },
+    {
+      "id": "video-performance-domain-action",
+      "query": "analyze rendering performance in src/components/Login.tsx",
+      "intent": "performance",
+      "activeSkill": "devflow:react",
+      "relevantCode": [{ "path": "src/components/Login.tsx", "role": "target" }],
+      "relevantKnowledgeSources": ["react"],
+      "expectedDomainAction": "react_audit_performance"
+    },
+    {
+      "id": "api-specific-knowledge-rejection",
+      "query": "review React 19 useEffect API hooks in src/components/Login.tsx",
+      "intent": "review",
+      "activeSkill": "devflow:react",
+      "relevantCode": [{ "path": "src/components/Login.tsx", "role": "target" }],
+      "expectedEmptyKnowledge": true,
+      "expectedDomainAction": "react_review_hooks"
+    }
+  ]
+}

package/fixtures/retrieval-quality.json

@@ -2,15 +2,18 @@
   "schemaVersion": "devflow.retrieval-quality.v1",
   "thresholds": {
     "codeFileRecallAt5": 0.85,
-    "codeFilePrecisionAt5": 0.75,
+    "codeFilePrecisionAt5": 0.85,
     "symbolNdcgAt5": 0.85,
     "falseCrossFileEdgeRate": 0.01,
     "memoryPrecisionAt5": 0.8,
     "expectedEmptyMemoryAccuracy": 1,
     "knowledgeNdcgAt5": 0.85,
+    "knowledgePrecisionAt3": 0.8,
     "knowledgeVectorCoverage": 0.95,
     "knowledgeChunkP95": 3000,
-    "knowledgeVersionMatch": 1
+    "knowledgeVersionMatch": 1,
+    "maxKnowledgeInjectionViolations": 0,
+    "maxStaleSelectedFiles": 0
   },
   "queries": [
     {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@devflow-tools/benchmark",
-  "version": "0.16.27",
+  "version": "0.16.29",
   "type": "module",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -19,9 +19,9 @@
   },
   "dependencies": {
     "@colbymchenry/codegraph": "^1.1.1",
-    "@devflow-tools/context-engine": "0.16.27",
-    "@devflow-tools/database": "0.16.27",
-    "@devflow-tools/sdk": "0.16.27"
+    "@devflow-tools/context-engine": "0.16.29",
+    "@devflow-tools/database": "0.16.29",
+    "@devflow-tools/sdk": "0.16.29"
   },
   "devDependencies": {
     "typescript": "^5.5.0",
@@ -31,5 +31,5 @@
     "dist",
     "fixtures"
   ],
-  "gitHead": "964af33767bf232606e63d973b3a29d7d53c8119"
+  "gitHead": "7a0def7e75bdc9462308824e450f1a33e5e75b34"
 }