@devflow-tools/benchmark 0.17.2 → 0.17.4

package/fixtures/semantic-intent-corpus.json

@@ -0,0 +1,125 @@
+{
+  "schemaVersion": "devflow.semantic-intent-corpus.v1",
+  "fixtures": [
+    { "id": "debug-mutate-01", "family": "debug_mutate", "partition": "development", "prompt": "fix the login page crash", "expected": { "primaryIntent": "debug", "primaryAction": "debug", "mutationPermission": "explicitly_authorized", "memoryDirective": "none", "knowledgeKind": "error", "outOfScope": false } },
+    { "id": "debug-mutate-02", "family": "debug_mutate", "partition": "development", "prompt": "修复登录崩溃", "expected": { "primaryIntent": "debug", "primaryAction": "debug", "mutationPermission": "explicitly_authorized", "memoryDirective": "none", "knowledgeKind": "error", "outOfScope": false } },
+    { "id": "debug-mutate-03", "family": "debug_mutate", "partition": "development", "prompt": "debug the failing token refresh", "expected": { "primaryIntent": "debug", "primaryAction": "debug", "mutationPermission": "explicitly_authorized", "memoryDirective": "none", "knowledgeKind": "error", "outOfScope": false } },
+    { "id": "debug-mutate-04", "family": "debug_mutate", "partition": "development", "prompt": "解决支付回调异常", "expected": { "primaryIntent": "debug", "primaryAction": "debug", "mutationPermission": "explicitly_authorized", "memoryDirective": "none", "knowledgeKind": "error", "outOfScope": false } },
+    { "id": "debug-mutate-05", "family": "debug_mutate", "partition": "development", "prompt": "fix the useAuth hook in LoginPage", "expected": { "primaryIntent": "debug", "primaryAction": "debug", "mutationPermission": "explicitly_authorized", "memoryDirective": "none", "knowledgeKind": "error", "outOfScope": false } },
+    { "id": "debug-mutate-06", "family": "debug_mutate", "partition": "development", "prompt": "请直接修复 build failure", "expected": { "primaryIntent": "debug", "primaryAction": "debug", "mutationPermission": "explicitly_authorized", "memoryDirective": "none", "knowledgeKind": "error", "outOfScope": false } },
+
+    { "id": "feature-mutate-01", "family": "feature_mutate", "partition": "development", "prompt": "add a new search feature", "expected": { "primaryIntent": "feature", "primaryAction": "create", "mutationPermission": "explicitly_authorized", "memoryDirective": "none", "outOfScope": false } },
+    { "id": "feature-mutate-02", "family": "feature_mutate", "partition": "development", "prompt": "帮我生成一个登录组件", "expected": { "primaryIntent": "feature", "primaryAction": "create", "mutationPermission": "explicitly_authorized", "memoryDirective": "none", "outOfScope": false } },
+    { "id": "feature-mutate-03", "family": "feature_mutate", "partition": "development", "prompt": "create a Login component", "expected": { "primaryIntent": "feature", "primaryAction": "create", "mutationPermission": "explicitly_authorized", "memoryDirective": "none", "outOfScope": false } },
+    { "id": "feature-mutate-04", "family": "feature_mutate", "partition": "development", "prompt": "build a Login component", "expected": { "primaryIntent": "feature", "primaryAction": "create", "mutationPermission": "explicitly_authorized", "memoryDirective": "none", "outOfScope": false } },
+    { "id": "feature-mutate-05", "family": "feature_mutate", "partition": "development", "prompt": "编写登录组件", "expected": { "primaryIntent": "feature", "primaryAction": "create", "mutationPermission": "explicitly_authorized", "memoryDirective": "none", "outOfScope": false } },
+    { "id": "feature-mutate-06", "family": "feature_mutate", "partition": "development", "prompt": "创建组件 Login", "expected": { "primaryIntent": "feature", "primaryAction": "create", "mutationPermission": "explicitly_authorized", "memoryDirective": "none", "outOfScope": false } },
+
+    { "id": "analysis-only-01", "family": "analysis_only", "partition": "development", "prompt": "how does the auth system work", "expected": { "primaryIntent": "understand", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "none", "knowledgeKind": "behavior", "outOfScope": false } },
+    { "id": "analysis-only-02", "family": "analysis_only", "partition": "development", "prompt": "这个实现没有 bug，只想了解它为什么这样设计", "expected": { "primaryIntent": "understand", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "none", "knowledgeKind": "behavior", "outOfScope": false } },
+    { "id": "analysis-only-03", "family": "analysis_only", "partition": "development", "prompt": "analyze the current VideoPlayer implementation in src/components/video-player/VideoPlayer.tsx", "expected": { "primaryIntent": "review", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "none", "knowledgeKind": "behavior", "outOfScope": false } },
+    { "id": "analysis-only-04", "family": "analysis_only", "partition": "development", "prompt": "分析当前登录模块的实现逻辑", "expected": { "primaryIntent": "review", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "none", "knowledgeKind": "behavior", "outOfScope": false } },
+    { "id": "analysis-only-05", "family": "analysis_only", "partition": "development", "prompt": "how to implement a cache without changing the current code", "expected": { "primaryIntent": "understand", "primaryAction": "analyze", "mutationPermission": "explicitly_forbidden", "memoryDirective": "none", "knowledgeKind": "behavior", "outOfScope": false } },
+    { "id": "analysis-only-06", "family": "analysis_only", "partition": "development", "prompt": "查看现有组件实现方式", "expected": { "primaryIntent": "review", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "none", "knowledgeKind": "behavior", "outOfScope": false } },
+    { "id": "analysis-only-07", "family": "analysis_only", "partition": "development", "prompt": "review the existing authentication implementation", "expected": { "primaryIntent": "review", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "none", "knowledgeKind": "behavior", "outOfScope": false } },
+
+    { "id": "release-mutate-01", "family": "release_mutate", "partition": "development", "prompt": "publish version 0.18.0", "expected": { "primaryIntent": "feature", "primaryAction": "release", "mutationPermission": "explicitly_authorized", "memoryDirective": "none", "outOfScope": false } },
+    { "id": "release-mutate-02", "family": "release_mutate", "partition": "development", "prompt": "发布 0.18.0", "expected": { "primaryIntent": "feature", "primaryAction": "release", "mutationPermission": "explicitly_authorized", "memoryDirective": "none", "outOfScope": false } },
+    { "id": "release-mutate-03", "family": "release_mutate", "partition": "development", "prompt": "release the approved package", "expected": { "primaryIntent": "feature", "primaryAction": "release", "mutationPermission": "explicitly_authorized", "memoryDirective": "none", "outOfScope": false } },
+    { "id": "release-mutate-04", "family": "release_mutate", "partition": "development", "prompt": "开始发版 npm 包", "expected": { "primaryIntent": "feature", "primaryAction": "release", "mutationPermission": "explicitly_authorized", "memoryDirective": "none", "outOfScope": false } },
+    { "id": "release-mutate-05", "family": "release_mutate", "partition": "development", "prompt": "publish the current workspace as 0.18.1", "expected": { "primaryIntent": "feature", "primaryAction": "release", "mutationPermission": "explicitly_authorized", "memoryDirective": "none", "outOfScope": false } },
+    { "id": "release-mutate-06", "family": "release_mutate", "partition": "development", "prompt": "请发布新版本", "expected": { "primaryIntent": "feature", "primaryAction": "release", "mutationPermission": "explicitly_authorized", "memoryDirective": "none", "outOfScope": false } },
+
+    { "id": "memory-save-01", "family": "memory_save", "partition": "development", "prompt": "remember that this project uses pnpm", "expected": { "primaryIntent": "memory", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "save", "outOfScope": false } },
+    { "id": "memory-save-02", "family": "memory_save", "partition": "development", "prompt": "记住项目统一使用 pnpm", "expected": { "primaryIntent": "memory", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "save", "outOfScope": false } },
+    { "id": "memory-save-03", "family": "memory_save", "partition": "development", "prompt": "save this convention to project memory", "expected": { "primaryIntent": "memory", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "save", "outOfScope": false } },
+    { "id": "memory-save-04", "family": "memory_save", "partition": "development", "prompt": "请记住我们不使用 yarn", "expected": { "primaryIntent": "memory", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "save", "outOfScope": false } },
+    { "id": "memory-save-05", "family": "memory_save", "partition": "development", "prompt": "请记住项目使用 TanStack Router，不修改项目文件", "expected": { "primaryIntent": "memory", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "save", "outOfScope": false } },
+    { "id": "memory-save-06", "family": "memory_save", "partition": "development", "prompt": "store the API naming rule for later", "expected": { "primaryIntent": "memory", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "save", "outOfScope": false } },
+
+    { "id": "knowledge-api-01", "family": "knowledge_api", "partition": "development", "prompt": "query the React 19 use API", "expected": { "primaryIntent": "knowledge", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "none", "knowledgeKind": "api", "outOfScope": false } },
+    { "id": "knowledge-api-02", "family": "knowledge_api", "partition": "development", "prompt": "查询 React 19 use API", "expected": { "primaryIntent": "knowledge", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "none", "knowledgeKind": "api", "outOfScope": false } },
+    { "id": "knowledge-api-03", "family": "knowledge_api", "partition": "development", "prompt": "what is the signature of useActionState", "expected": { "primaryIntent": "knowledge", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "none", "knowledgeKind": "api", "outOfScope": false } },
+    { "id": "knowledge-api-04", "family": "knowledge_api", "partition": "development", "prompt": "查一下 BGE-M3 的调用参数", "expected": { "primaryIntent": "knowledge", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "none", "knowledgeKind": "api", "outOfScope": false } },
+    { "id": "knowledge-api-05", "family": "knowledge_api", "partition": "development", "prompt": "show the MCP sampling method reference", "expected": { "primaryIntent": "knowledge", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "none", "knowledgeKind": "api", "outOfScope": false } },
+    { "id": "knowledge-api-06", "family": "knowledge_api", "partition": "development", "prompt": "解释 React hooks 的原理", "expected": { "primaryIntent": "knowledge", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "none", "knowledgeKind": "overview", "outOfScope": false } },
+
+    { "id": "conditional-mutation-01", "family": "conditional_mutation", "partition": "development", "prompt": "analyze the issue and implement the approved solution", "expected": { "primaryIntent": "debug", "primaryAction": "create", "mutationPermission": "conditional", "memoryDirective": "none", "knowledgeKind": "error", "outOfScope": false } },
+    { "id": "conditional-mutation-02", "family": "conditional_mutation", "partition": "development", "prompt": "先分析登录崩溃，确认根因后再修复", "expected": { "primaryIntent": "debug", "primaryAction": "debug", "mutationPermission": "conditional", "memoryDirective": "none", "knowledgeKind": "error", "outOfScope": false } },
+    { "id": "conditional-mutation-03", "family": "conditional_mutation", "partition": "development", "prompt": "if the tests reproduce it, fix the cache race", "expected": { "primaryIntent": "debug", "primaryAction": "debug", "mutationPermission": "conditional", "memoryDirective": "none", "knowledgeKind": "error", "outOfScope": false } },
+    { "id": "conditional-mutation-04", "family": "conditional_mutation", "partition": "development", "prompt": "如果能确认类型错误，再修改接口", "expected": { "primaryIntent": "debug", "primaryAction": "modify", "mutationPermission": "conditional", "memoryDirective": "none", "knowledgeKind": "error", "outOfScope": false } },
+    { "id": "conditional-mutation-05", "family": "conditional_mutation", "partition": "development", "prompt": "分析完成后实现这个方案", "expected": { "primaryIntent": "feature", "primaryAction": "create", "mutationPermission": "conditional", "memoryDirective": "none", "outOfScope": false } },
+    { "id": "conditional-mutation-06", "family": "conditional_mutation", "partition": "development", "prompt": "once I approve the diff, apply the refactor", "expected": { "primaryIntent": "refactor", "primaryAction": "modify", "mutationPermission": "conditional", "memoryDirective": "none", "outOfScope": false } },
+
+    { "id": "negated-debug-01", "family": "negated_debug", "partition": "heldout", "prompt": "The build command is failing; explain why, do not execute anything", "expected": { "primaryIntent": "debug", "primaryAction": "analyze", "mutationPermission": "explicitly_forbidden", "memoryDirective": "none", "knowledgeKind": "error", "outOfScope": false } },
+    { "id": "negated-debug-02", "family": "negated_debug", "partition": "heldout", "prompt": "登录一直崩溃，只分析原因，不要修复", "expected": { "primaryIntent": "debug", "primaryAction": "analyze", "mutationPermission": "explicitly_forbidden", "memoryDirective": "none", "knowledgeKind": "error", "outOfScope": false } },
+    { "id": "negated-debug-03", "family": "negated_debug", "partition": "heldout", "prompt": "diagnose the failing test without changing files", "expected": { "primaryIntent": "debug", "primaryAction": "analyze", "mutationPermission": "explicitly_forbidden", "memoryDirective": "none", "knowledgeKind": "error", "outOfScope": false } },
+    { "id": "negated-debug-04", "family": "negated_debug", "partition": "heldout", "prompt": "解释缓存 race，不执行命令也不改代码", "expected": { "primaryIntent": "understand", "primaryAction": "analyze", "mutationPermission": "explicitly_forbidden", "memoryDirective": "none", "knowledgeKind": "error", "outOfScope": false } },
+    { "id": "negated-debug-05", "family": "negated_debug", "partition": "heldout", "prompt": "review the crash report; never run the fix", "expected": { "primaryIntent": "review", "primaryAction": "analyze", "mutationPermission": "explicitly_forbidden", "memoryDirective": "none", "knowledgeKind": "error", "outOfScope": false } },
+    { "id": "negated-debug-06", "family": "negated_debug", "partition": "heldout", "prompt": "分析构建失败，但不要重新构建", "expected": { "primaryIntent": "debug", "primaryAction": "analyze", "mutationPermission": "explicitly_forbidden", "memoryDirective": "none", "knowledgeKind": "error", "outOfScope": false } },
+
+    { "id": "negated-release-analysis", "family": "negated_release", "partition": "heldout", "prompt": "如果发布失败，只分析日志，不要重新发布", "expected": { "primaryIntent": "review", "primaryAction": "analyze", "mutationPermission": "explicitly_forbidden", "memoryDirective": "none", "knowledgeKind": "error", "outOfScope": false } },
+    { "id": "negated-release-02", "family": "negated_release", "partition": "heldout", "prompt": "analyze why 0.17.3 publish failed; do not republish", "expected": { "primaryIntent": "debug", "primaryAction": "analyze", "mutationPermission": "explicitly_forbidden", "memoryDirective": "none", "knowledgeKind": "error", "outOfScope": false } },
+    { "id": "negated-release-03", "family": "negated_release", "partition": "heldout", "prompt": "查看发版错误，不要运行 npm publish", "expected": { "primaryIntent": "review", "primaryAction": "analyze", "mutationPermission": "explicitly_forbidden", "memoryDirective": "none", "knowledgeKind": "error", "outOfScope": false } },
+    { "id": "negated-release-04", "family": "negated_release", "partition": "heldout", "prompt": "explain the release pipeline without publishing", "expected": { "primaryIntent": "understand", "primaryAction": "analyze", "mutationPermission": "explicitly_forbidden", "memoryDirective": "none", "knowledgeKind": "behavior", "outOfScope": false } },
+    { "id": "negated-release-05", "family": "negated_release", "partition": "heldout", "prompt": "审查发布脚本，禁止发版", "expected": { "primaryIntent": "review", "primaryAction": "analyze", "mutationPermission": "explicitly_forbidden", "memoryDirective": "none", "knowledgeKind": "behavior", "outOfScope": false } },
+    { "id": "negated-release-06", "family": "negated_release", "partition": "heldout", "prompt": "find the npm publish failure but make no release", "expected": { "primaryIntent": "debug", "primaryAction": "analyze", "mutationPermission": "explicitly_forbidden", "memoryDirective": "none", "knowledgeKind": "error", "outOfScope": false } },
+
+    { "id": "memory-forget-recall-01", "family": "memory_forget_recall", "partition": "heldout", "prompt": "删除“旧缓存策略”这条记忆", "expected": { "primaryIntent": "memory", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "forget", "outOfScope": false } },
+    { "id": "memory-forget-recall-02", "family": "memory_forget_recall", "partition": "heldout", "prompt": "forget the old cache policy", "expected": { "primaryIntent": "memory", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "forget", "outOfScope": false } },
+    { "id": "memory-forget-recall-03", "family": "memory_forget_recall", "partition": "heldout", "prompt": "回忆项目选择路由库的原因", "expected": { "primaryIntent": "memory", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "recall", "outOfScope": false } },
+    { "id": "memory-forget-recall-04", "family": "memory_forget_recall", "partition": "heldout", "prompt": "recall why we chose the router", "expected": { "primaryIntent": "memory", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "recall", "outOfScope": false } },
+    { "id": "memory-forget-recall-05", "family": "memory_forget_recall", "partition": "heldout", "prompt": "别记住这个临时 pnpm 命令", "expected": { "primaryIntent": "memory", "primaryAction": "analyze", "mutationPermission": "explicitly_forbidden", "memoryDirective": "none", "outOfScope": false } },
+    { "id": "memory-forget-recall-06", "family": "memory_forget_recall", "partition": "heldout", "prompt": "记一下还是算了", "expected": { "primaryIntent": "memory", "primaryAction": "analyze", "mutationPermission": "ambiguous", "memoryDirective": "ambiguous", "outOfScope": false } },
+
+    { "id": "knowledge-migration-01", "family": "knowledge_migration", "partition": "heldout", "prompt": "migrate React 18 to React 19", "expected": { "primaryIntent": "knowledge", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "none", "knowledgeKind": "migration", "outOfScope": false } },
+    { "id": "knowledge-migration-02", "family": "knowledge_migration", "partition": "heldout", "prompt": "迁移 React 18 到 React 19", "expected": { "primaryIntent": "knowledge", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "none", "knowledgeKind": "migration", "outOfScope": false } },
+    { "id": "knowledge-migration-03", "family": "knowledge_migration", "partition": "heldout", "prompt": "what are the breaking changes in Vue 4", "expected": { "primaryIntent": "knowledge", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "none", "knowledgeKind": "migration", "outOfScope": false } },
+    { "id": "knowledge-migration-04", "family": "knowledge_migration", "partition": "heldout", "prompt": "查询 Node 24 升级指南", "expected": { "primaryIntent": "knowledge", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "none", "knowledgeKind": "migration", "outOfScope": false } },
+    { "id": "knowledge-migration-05", "family": "knowledge_migration", "partition": "heldout", "prompt": "compare the migration notes for Vite 7", "expected": { "primaryIntent": "knowledge", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "none", "knowledgeKind": "migration", "outOfScope": false } },
+    { "id": "knowledge-migration-06", "family": "knowledge_migration", "partition": "heldout", "prompt": "了解 TypeScript 6 迁移风险", "expected": { "primaryIntent": "knowledge", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "none", "knowledgeKind": "migration", "outOfScope": false } },
+
+    { "id": "negative-performance-01", "family": "negative_performance", "partition": "heldout", "prompt": "not a performance problem; review naming only", "expected": { "primaryIntent": "review", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "none", "outOfScope": false } },
+    { "id": "negative-performance-02", "family": "negative_performance", "partition": "heldout", "prompt": "不是性能问题，只分析变量命名", "expected": { "primaryIntent": "review", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "none", "outOfScope": false } },
+    { "id": "negative-performance-03", "family": "negative_performance", "partition": "heldout", "prompt": "分析 React 组件渲染性能", "expected": { "primaryIntent": "performance", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "none", "knowledgeKind": "behavior", "outOfScope": false } },
+    { "id": "negative-performance-04", "family": "negative_performance", "partition": "heldout", "prompt": "不要优化性能，只解释渲染顺序", "expected": { "primaryIntent": "understand", "primaryAction": "analyze", "mutationPermission": "explicitly_forbidden", "memoryDirective": "none", "knowledgeKind": "behavior", "outOfScope": false } },
+    { "id": "negative-performance-05", "family": "negative_performance", "partition": "heldout", "prompt": "this is not a memory leak; explain ownership", "expected": { "primaryIntent": "understand", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "none", "knowledgeKind": "behavior", "outOfScope": false } },
+    { "id": "negative-performance-06", "family": "negative_performance", "partition": "heldout", "prompt": "不处理卡顿，审查组件职责即可", "expected": { "primaryIntent": "review", "primaryAction": "analyze", "mutationPermission": "explicitly_forbidden", "memoryDirective": "none", "outOfScope": false } },
+
+    { "id": "oos-01", "family": "out_of_scope", "partition": "heldout", "prompt": "天气怎么样", "expected": { "primaryIntent": "unknown", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "none", "outOfScope": true } },
+    { "id": "oos-02", "family": "out_of_scope", "partition": "heldout", "prompt": "write a poem about the ocean", "expected": { "primaryIntent": "unknown", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "none", "outOfScope": true } },
+    { "id": "oos-03", "family": "out_of_scope", "partition": "heldout", "prompt": "上海明天限行吗", "expected": { "primaryIntent": "unknown", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "none", "outOfScope": true } },
+    { "id": "oos-04", "family": "out_of_scope", "partition": "heldout", "prompt": "recommend a dinner recipe", "expected": { "primaryIntent": "unknown", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "none", "outOfScope": true } },
+    { "id": "oos-05", "family": "out_of_scope", "partition": "heldout", "prompt": "讲一个冷笑话", "expected": { "primaryIntent": "unknown", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "none", "outOfScope": true } },
+    { "id": "oos-06", "family": "out_of_scope", "partition": "heldout", "prompt": "translate this legal contract", "expected": { "primaryIntent": "unknown", "primaryAction": "analyze", "mutationPermission": "not_requested", "memoryDirective": "none", "outOfScope": true } },
+    { "id": "oos-07", "family": "out_of_scope", "partition": "heldout", "prompt": "hmm", "expected": { "primaryIntent": "unknown", "primaryAction": "analyze", "mutationPermission": "ambiguous", "memoryDirective": "none", "outOfScope": true } },
+
+    { "id": "review-nonmutating-01", "family": "review_nonmutating", "partition": "heldout", "prompt": "review LoginPage without modifying the project", "expected": { "primaryIntent": "review", "primaryAction": "analyze", "mutationPermission": "explicitly_forbidden", "memoryDirective": "none", "outOfScope": false } },
+    { "id": "review-nonmutating-02", "family": "review_nonmutating", "partition": "heldout", "prompt": "审查 LoginPage，不要创建或修改任何文件", "expected": { "primaryIntent": "review", "primaryAction": "analyze", "mutationPermission": "explicitly_forbidden", "memoryDirective": "none", "outOfScope": false } },
+    { "id": "review-nonmutating-03", "family": "review_nonmutating", "partition": "heldout", "prompt": "please critique the component; no file changes", "expected": { "primaryIntent": "review", "primaryAction": "analyze", "mutationPermission": "explicitly_forbidden", "memoryDirective": "none", "outOfScope": false } },
+    { "id": "review-nonmutating-04", "family": "review_nonmutating", "partition": "heldout", "prompt": "只做代码审查，禁止编辑", "expected": { "primaryIntent": "review", "primaryAction": "analyze", "mutationPermission": "explicitly_forbidden", "memoryDirective": "none", "outOfScope": false } },
+    { "id": "review-nonmutating-05", "family": "review_nonmutating", "partition": "heldout", "prompt": "assess the architecture and do not implement recommendations", "expected": { "primaryIntent": "review", "primaryAction": "analyze", "mutationPermission": "explicitly_forbidden", "memoryDirective": "none", "outOfScope": false } },
+    { "id": "review-nonmutating-06", "family": "review_nonmutating", "partition": "heldout", "prompt": "检查类型设计，不要提交修改", "expected": { "primaryIntent": "review", "primaryAction": "analyze", "mutationPermission": "explicitly_forbidden", "memoryDirective": "none", "outOfScope": false } },
+
+    { "id": "conditional-release-01", "family": "conditional_release", "partition": "heldout", "prompt": "if all tests pass, publish 0.18.0", "expected": { "primaryIntent": "feature", "primaryAction": "release", "mutationPermission": "conditional", "memoryDirective": "none", "outOfScope": false } },
+    { "id": "conditional-release-02", "family": "conditional_release", "partition": "heldout", "prompt": "测试全部通过后再发布 0.18.0", "expected": { "primaryIntent": "feature", "primaryAction": "release", "mutationPermission": "conditional", "memoryDirective": "none", "outOfScope": false } },
+    { "id": "conditional-release-03", "family": "conditional_release", "partition": "heldout", "prompt": "publish only after I approve the changelog", "expected": { "primaryIntent": "feature", "primaryAction": "release", "mutationPermission": "conditional", "memoryDirective": "none", "outOfScope": false } },
+    { "id": "conditional-release-04", "family": "conditional_release", "partition": "heldout", "prompt": "等我确认版本号之后发版", "expected": { "primaryIntent": "feature", "primaryAction": "release", "mutationPermission": "conditional", "memoryDirective": "none", "outOfScope": false } },
+    { "id": "conditional-release-05", "family": "conditional_release", "partition": "heldout", "prompt": "when CI is green, run npm publish", "expected": { "primaryIntent": "feature", "primaryAction": "release", "mutationPermission": "conditional", "memoryDirective": "none", "outOfScope": false } },
+    { "id": "conditional-release-06", "family": "conditional_release", "partition": "heldout", "prompt": "如果 tag 尚未存在，就发布新版本", "expected": { "primaryIntent": "feature", "primaryAction": "release", "mutationPermission": "conditional", "memoryDirective": "none", "outOfScope": false } },
+
+    { "id": "ambiguous-followup-01", "family": "ambiguous_followup", "partition": "heldout", "prompt": "继续", "expected": { "primaryIntent": "unknown", "primaryAction": "analyze", "mutationPermission": "ambiguous", "memoryDirective": "none", "outOfScope": true } },
+    { "id": "ambiguous-followup-02", "family": "ambiguous_followup", "partition": "heldout", "prompt": "按刚才批准的方案执行", "expected": { "primaryIntent": "unknown", "primaryAction": "analyze", "mutationPermission": "ambiguous", "memoryDirective": "none", "outOfScope": true } },
+    { "id": "ambiguous-followup-03", "family": "ambiguous_followup", "partition": "heldout", "prompt": "do it", "expected": { "primaryIntent": "unknown", "primaryAction": "analyze", "mutationPermission": "ambiguous", "memoryDirective": "none", "outOfScope": true } },
+    { "id": "ambiguous-followup-04", "family": "ambiguous_followup", "partition": "heldout", "prompt": "用上一个", "expected": { "primaryIntent": "unknown", "primaryAction": "analyze", "mutationPermission": "ambiguous", "memoryDirective": "none", "outOfScope": true } },
+    { "id": "ambiguous-followup-05", "family": "ambiguous_followup", "partition": "heldout", "prompt": "apply that change", "expected": { "primaryIntent": "unknown", "primaryAction": "analyze", "mutationPermission": "ambiguous", "memoryDirective": "none", "outOfScope": true } },
+    { "id": "ambiguous-followup-06", "family": "ambiguous_followup", "partition": "heldout", "prompt": "就这么办", "expected": { "primaryIntent": "unknown", "primaryAction": "analyze", "mutationPermission": "ambiguous", "memoryDirective": "none", "outOfScope": true } },
+
+    { "id": "explicit-implementation-01", "family": "explicit_implementation", "partition": "development", "prompt": "implement a Login component", "expected": { "primaryIntent": "feature", "primaryAction": "create", "mutationPermission": "explicitly_authorized", "memoryDirective": "none", "outOfScope": false } },
+    { "id": "explicit-implementation-02", "family": "explicit_implementation", "partition": "development", "prompt": "implement a cache for the authentication module", "expected": { "primaryIntent": "feature", "primaryAction": "create", "mutationPermission": "explicitly_authorized", "memoryDirective": "none", "outOfScope": false } },
+    { "id": "explicit-implementation-03", "family": "explicit_implementation", "partition": "development", "prompt": "帮我实现登录组件", "expected": { "primaryIntent": "feature", "primaryAction": "create", "mutationPermission": "explicitly_authorized", "memoryDirective": "none", "outOfScope": false } },
+    { "id": "explicit-implementation-04", "family": "explicit_implementation", "partition": "development", "prompt": "新增缓存模块", "expected": { "primaryIntent": "feature", "primaryAction": "create", "mutationPermission": "explicitly_authorized", "memoryDirective": "none", "outOfScope": false } },
+    { "id": "explicit-implementation-05", "family": "explicit_implementation", "partition": "development", "prompt": "scaffold a settings page", "expected": { "primaryIntent": "feature", "primaryAction": "create", "mutationPermission": "explicitly_authorized", "memoryDirective": "none", "outOfScope": false } },
+    { "id": "explicit-implementation-06", "family": "explicit_implementation", "partition": "development", "prompt": "开发一个用户搜索功能", "expected": { "primaryIntent": "feature", "primaryAction": "create", "mutationPermission": "explicitly_authorized", "memoryDirective": "none", "outOfScope": false } }
+  ]
+}

package/fixtures/semantic-regex-inventory.json

@@ -0,0 +1,20 @@
+{
+  "schemaVersion": "devflow.semantic-regex-inventory.v1",
+  "entries": [
+    { "file": "packages/context-engine/src/intent-parser.ts", "symbol": "INTENT_PATTERNS", "responsibility": "semantic_classifier", "disposition": "shadow_only", "replacementField": "TaskSemanticFrameV2.primaryIntent" },
+    { "file": "packages/context-engine/src/intent-parser.ts", "symbol": "CN_INTENT_MAP", "responsibility": "semantic_classifier", "disposition": "shadow_only", "replacementField": "TaskSemanticFrameV2.primaryIntent" },
+    { "file": "packages/context-engine/src/intent-parser.ts", "symbol": "ZH_EN_MAP", "responsibility": "lexical_feature", "disposition": "retain_for_retrieval_only" },
+    { "file": "packages/context-engine/src/retrieval-task-plan.ts", "symbol": "canonicalIntent", "responsibility": "semantic_classifier", "disposition": "shadow_only", "replacementField": "TaskSemanticFrameV2.primaryIntent" },
+    { "file": "packages/context-engine/src/retrieval-task-plan.ts", "symbol": "inferAction", "responsibility": "semantic_classifier", "disposition": "shadow_only", "replacementField": "TaskSemanticFrameV2.primaryAction" },
+    { "file": "packages/context-engine/src/retrieval-task-plan.ts", "symbol": "extractKnowledgeSymbols", "responsibility": "structural_authority", "disposition": "keep" },
+    { "file": "packages/context-engine/src/retrieval-task-plan.ts", "symbol": "extractEntityVersions", "responsibility": "structural_authority", "disposition": "keep" },
+    { "file": "packages/memory-engine/src/signal-analyzer.ts", "symbol": "detectCheap", "scope": "user_message", "responsibility": "semantic_classifier", "disposition": "shadow_only", "replacementField": "TaskSemanticFrameV2.memoryDirective" },
+    { "file": "packages/memory-engine/src/signal-analyzer.ts", "symbol": "detectCheap", "scope": "tool_event", "responsibility": "structural_authority", "disposition": "keep" },
+    { "file": "packages/memory-engine/src/explicit-memory-enricher.ts", "symbol": "classifyExplicitMemory", "responsibility": "semantic_classifier", "disposition": "shadow_only", "replacementField": "TaskSemanticFrameV2.memoryDirective" },
+    { "file": "packages/memory-engine/src/observation-quality.ts", "symbol": "classifyFailureCategory", "responsibility": "lexical_feature", "disposition": "retain_non_authoritative" },
+    { "file": "packages/knowledge-engine/src/source-selector.ts", "symbol": "SOURCE_FAMILIES", "scope": "query", "responsibility": "semantic_classifier", "disposition": "shadow_only", "replacementField": "TaskSemanticFrameV2.knowledgeNeeds" },
+    { "file": "packages/knowledge-engine/src/source-selector.ts", "symbol": "SOURCE_FAMILIES", "scope": "source/dependencies/frameworks/languages/files", "responsibility": "structural_authority", "disposition": "keep" },
+    { "file": "packages/devflow/src/hooks/evidence-contract.ts", "symbol": "validateEvidenceContract", "responsibility": "safety_defense", "disposition": "keep" },
+    { "file": "packages/devflow/src/hooks/slash-command.ts", "symbol": "parseSlashCommand", "responsibility": "structural_authority", "disposition": "keep" }
+  ]
+}

package/fixtures/semantic-thresholds.json

@@ -0,0 +1,10 @@
+{
+  "schemaVersion": "semantic-thresholds.v1",
+  "version": "semantic-intent-corpus.v1",
+  "modelId": "Xenova/all-MiniLM-L6-v2",
+  "modelRevision": "751bff37182d3f1213fa05d7196b954e230abad9",
+  "routeCatalogVersion": "semantic-route-catalog.v1",
+  "minimumScore": 0.55,
+  "minimumMargin": 0.08,
+  "fitPartition": "development"
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@devflow-tools/benchmark",
-  "version": "0.17.2",
+  "version": "0.17.4",
   "type": "module",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -19,9 +19,9 @@
   },
   "dependencies": {
     "@colbymchenry/codegraph": "^1.1.1",
-    "@devflow-tools/context-engine": "0.17.2",
-    "@devflow-tools/database": "0.17.2",
-    "@devflow-tools/sdk": "0.17.2"
+    "@devflow-tools/context-engine": "0.17.4",
+    "@devflow-tools/database": "0.17.4",
+    "@devflow-tools/sdk": "0.17.4"
   },
   "devDependencies": {
     "typescript": "^5.5.0",
@@ -31,5 +31,5 @@
     "dist",
     "fixtures"
   ],
-  "gitHead": "964af33767bf232606e63d973b3a29d7d53c8119"
+  "gitHead": "b880054f9cef4ed62b4d02544257c9f5cbd811bc"
 }