@tangle-network/agent-bench 0.1.0 → 0.3.5

package/fixtures/enterpriseops-gym.json

@@ -0,0 +1,103 @@
+[
+  {
+    "task_id": "task_20251212_172511_458_e6427839_47076c15",
+    "domain": "itsm",
+    "system_prompt": "# ITSM Assistant Policy\n\nRole: ITSM Assistant (IT Service Management). Operate exclusively on confirmed user roles, verified record relationships, and ITIL/database integrity rules. Never assume or fabricate IDs, responses, or outcomes — rely solely on verified API results.",
+    "user_prompt": "An update came in from the caller regarding the printer connectivity issue for incident INC0000003. The user stated that several urgent client documents are still pending, and the inability to print is causing significant delays. Although the submission deadline has been extended by one business day, the service impact remains significant. Therefore increase the impact of the incident to high and update the necessary details.",
+    "selected_tools": [
+      "find_sla_definitions",
+      "find_incident_slas",
+      "find_incident_by_number",
+      "get_user",
+      "update_incident",
+      "link_new_incident_sla"
+    ],
+    "restricted_tools": [],
+    "mcp_endpoint": "/mcp",
+    "number_of_runs": 1,
+    "reset_database_between_runs": true,
+    "gym_servers_config": [
+      {
+        "mcp_server_name": "gym-itsm-mcp",
+        "mcp_server_url": "http://localhost:8006",
+        "seed_database_file": "Domain Wise DBs and Task-DB Mappings/itsm/dbs/db_1765301900121_3mwjj54xy.sql",
+        "context": {
+          "x-itsm-user-token": "admin_token_marcus_2024_secure"
+        },
+        "user_info": {
+          "user_id": "USER_001",
+          "name": "Marcus Thompson",
+          "email": "marcus.thompson@techcorp.com"
+        }
+      }
+    ],
+    "verifiers": [
+      {
+        "verifier_type": "database_state",
+        "name": "Verify if the priority of the incident is set correctly.",
+        "description": "Verify if the priority of the incident is set correctly.",
+        "gym_name": "gym-itsm-mcp",
+        "validation_config": {
+          "query": "SELECT COUNT(*) FROM incident WHERE incident_id = 'INC_003' AND impact = 'high' AND priority = 'high';",
+          "expected_value": 1,
+          "comparison_type": "equals"
+        }
+      },
+      {
+        "verifier_type": "database_state",
+        "name": "Verify if the correct high-priority SLA is linked to the incident.",
+        "description": "Verify if the correct high-priority SLA is linked to the incident.",
+        "gym_name": "gym-itsm-mcp",
+        "validation_config": {
+          "query": "SELECT COUNT(*) FROM incident_sla WHERE incident_id = 'INC_003' AND sla_def_id = 'SLA_002';",
+          "expected_value": 1,
+          "comparison_type": "equals"
+        }
+      }
+    ]
+  },
+  {
+    "task_id": "task_20251117_165528_648_bca89e7d_3e81ece9",
+    "domain": "calendar",
+    "system_prompt": "# Calendar Assistant Policy\n\nRole: Calendar Assistant. Create, update, and manage calendars and events strictly for the authenticated user. Never operate on calendars outside the user's access scope.",
+    "user_prompt": "Create a secondary (non-primary) calendar named 'Search Algorithm Beta' for bob_developer so the team can track the beta milestones separately from the main work calendar.",
+    "selected_tools": [
+      "list_calendars",
+      "create_calendar",
+      "get_calendar",
+      "list_events"
+    ],
+    "restricted_tools": [],
+    "mcp_endpoint": "/mcp",
+    "number_of_runs": 1,
+    "reset_database_between_runs": true,
+    "gym_servers_config": [
+      {
+        "mcp_server_name": "gym-calendar",
+        "mcp_server_url": "http://localhost:8003",
+        "seed_database_file": "Domain Wise DBs and Task-DB Mappings/calendar/dbs/db_1762868439331_kf914hbmw.sql",
+        "context": {
+          "x-access-token": "bob_developer_calendar_token"
+        },
+        "user_info": {
+          "user_id": "bob_developer",
+          "name": "Bob Developer",
+          "email": "bob.developer@techcorp.com"
+        }
+      }
+    ],
+    "verifiers": [
+      {
+        "verifier_type": "database_state",
+        "name": "Calendar Creation",
+        "description": "Verify secondary calendar created",
+        "gym_name": "gym-calendar",
+        "validation_config": {
+          "query": "SELECT COUNT(*) AS count FROM calendars WHERE summary = 'Search Algorithm Beta' AND is_primary = 0 AND user_id = 'bob_developer';",
+          "expected_value": 1,
+          "comparison_type": "equals"
+        }
+      }
+    ]
+  }
+]

package/fixtures/finresearchbench.json

@@ -0,0 +1,21 @@
+[
+  {
+    "id": "finresearchbench-fixture-0",
+    "category": "stock-specific-analysis",
+    "question": "Assess whether Company A's margin expansion is durable given revenue growth and cost pressure.",
+    "reference_answer": "Margin expansion is not durable unless revenue growth continues and cost pressure eases.",
+    "reference_report": "Margin expansion is not durable unless revenue growth continues and cost pressure eases.",
+    "logic_tree": {
+      "claim": "durability depends on growth and cost pressure",
+      "children": [
+        { "evidence": "revenue growth supports operating leverage" },
+        { "evidence": "persistent cost pressure offsets margin gains" }
+      ]
+    },
+    "rubric": {
+      "requires": ["states a conclusion", "mentions revenue growth", "mentions cost pressure"]
+    },
+    "judge_system_prompt": "You are the official benchmark judge. Return JSON with score in [0,1].",
+    "judge_prompt_template": "Question: {question}\nReference: {reference_report}\nLogic tree: {logic_tree}\nRubric: {rubric}\nResponse: {response}\nReturn JSON: {\"score\": 0.0}"
+  }
+]

package/fixtures/finsearchcomp.json

@@ -0,0 +1,66 @@
+[
+  {
+    "prompt_id": "(T2)Simple_Historical_Lookup_001",
+    "prompt": "2024年全年中国外债净流入是多少？（单位：十亿美元）（数值保留1位小数）",
+    "response_reference": "-47.9 十亿美元，答案允许1%的误差",
+    "judge_prompt_template": "<题目>：\n{prompt}\n\n<标准答案>：\n{response_reference}\n\n<学生答案>：\n{response}",
+    "judge_system_prompt": "你是一个智慧的金融问题答案的判断和评分者。你会收到一道<题目>、该题目的<标准答案>和一个<学生答案>，有些<标准答案>中附有“评分要点”作为补充。你需要对<学生答案>进行判断，完成以下任务：\n1. 根据<学生答案>的内容，准确地识别出它的最终答案（只需要识别，不需要输出）。你可以通过分析<学生答案>来识别最终答案的位置和内容，也可以通过检索关键词来识别最终答案的位置和内容，关键词包括但不限于“答案为”“最终结果是”“正确选项是”等。如果<学生答案>为空，即<学生答案>没有任何内容，则直接给0分，并跳过下面的第2步和第3步。\n2. 分别列出<标准答案>和你识别出的<学生答案>的最终答案，并将二者进行比对（不需要输出列举和比对的过程或结果）。\n3. 根据比对的结果和<标准答案>中可能附有的评分要点，判断<学生答案>是否正确并评分。分数只有1分和0分，1分表示<学生答案>正确，0分表示<学生答案>错误，没有除0和1之外的其他分数。\n注意：\n1. 你不需要也不应该自行回答或解决问题，你的唯一任务是判断并评分。\n2. <标准答案>是准确无误的，你可以完全信任它。\n3. 如果<标准答案>包含2个或更多个要点，例如一个人名和一个数字，或者一个事物和一个时间，或者并列的多个地点，那么，<学生答案>只有答出了所有的要点并且所有的要点都回答正确才可以得1分，如果只答出或答对了一部分要点，则分数为0分。\n4. 如果<学生答案>为空或者异常报错信息，请给0分\n5. 相同大小的数值用不同格式来写视为一样的，比如12.45% vs 0.1245，或者120,400,000 vs 120.4 million，或者 2/5 vs 0.4。如果<标准答案>和<学生答案>采用了不同的格式，但背后的数字是一个数值，则认为二者一致。\n6. 如果<学生答案>只给了数字没给单位，如果数字一致则不算错。比如<题目>要求回答\"xx收入是多少亿元\"，<标准答案>是\"12.98亿元\"，<学生答案>回答\"12.98\"，视为一致。或者<题目>要求\"单位：%\"，<标准答案>是\"12.98%\"，<学生答案>回答\"12.98\"，视为一致。\n\n关于你的输出格式：\n你需要先输出识别结果，然后给出评分依据和分数。你的输出格式为：\n- 【评分依据】：（简述你的评分原因，不超过150字或150 tokens）\n- 【JSON】：\n``` \n{\"answer_score\": your score} \n```\n\n###\n【示例1】\n<题目>：\n在2024年毕业典礼上讲话的C9校长从博士毕业到第一次当某所大学正校长的平均时间是多少年？（用年份计算，结果保留一位小数）\n<标准答案>：\n27.9年\n评分要点：\n学生答案中的数字必须与标准答案中的数字严格一致。\n<学生答案>：\n根据已知信息，C9高校现任校长从博士毕业到首次担任正校长的平均时间约为27.3年。以下是具体计算：\n1. 北京大学：龚旗煌校长，1992年博士毕业，2022年成为校长，间隔30年。\n2. 清华大学：王希勤校长，1996年博士毕业，2022年成为校长，间隔26年。\n3. 复旦大学：金力校长，1994年博士毕业，2021年成为校长，间隔27年。\n4. 上海交通大学：丁奎岭校长，1990年博士毕业，2021年成为校长，间隔31年。\n5. 南京大学：吕建校长，1993年博士毕业，2018年成为校长，间隔25年。\n6. 中国科学技术大学：包信和校长，1991年博士毕业，2017年成为校长，间隔26年。\n7. 西安交通大学：王树国校长，1986年博士毕业，2014年成为校长，间隔28年。\n8. 浙江大学：吴朝晖校长，1993年博士毕业，2015年成为校长，间隔22年。\n9. 哈尔滨工业大学：韩杰才校长，1991年博士毕业，2022年成为校长，间隔31年。\n将以上年份相加：\n30 + 26 + 27 + 31 + 25 + 26 + 28 + 22 + 31 = 246\n平均时间：\n246 ÷ 9 ≈ 27.3年\n因此，平均而言，这些校长从博士毕业到首次担任大学正校长的时间约为27.3年。\n- 【评分依据】：本题有评分要点，评分要点要求学生答案中的数字必须与标准答案中的数字严格一致。这里<学生答案>中的数字与<标准答案>中的数字并不严格一致，因此<学生答案>错误，分数为0分。\n- 【JSON】：\n``` \n{\"answer_score\": 0} \n``` \n###\n\n###\n【示例2】\n<题目>：\n美国很有意思的一点是，一个州的政治中心往往不是经济中心，比如加州的首府是知名度相对较低的Sacramento，而不是经济更发达、知名度更高的旧金山或洛杉矶。与此不同的是，中国各省（不算自治区）的省会往往也是该省GDP最高的城市。如果以2022年的GDP为标准，请列出中国各省中，在本省内，GDP最高的城市不是省会城市的省份。\n<标准答案>：\n河北省、辽宁省、江苏省、山东省、广东省\n评分要点：\n学生答案必须答出标准答案中的所有省，同时不能出现其他省。\n<学生答案>：\n根据2022年的GDP数据统计，中国各省中（不含自治区），在本省内GDP最高的城市不是省会城市的省份包括：\n1. 河北省：GDP最高的城市是唐山市，而省会是石家庄市。\n2. 辽宁省：GDP最高的城市是大连市，而省会是沈阳市。\n3. 黑龙江省：GDP最高的城市是大庆市，而省会是哈尔滨市。\n4. 江苏省：GDP最高的城市是苏州市，而省会是南京市。\n5. 福建省：GDP最高的城市是泉州市，而省会是福州市。\n6. 山东省：GDP最高的城市是青岛市，而省会是济南市。\n7. 广东省：GDP最高的城市是深圳市，而省会是广州市。\n这些省份的经济重心往往集中在非省会城市，反映了各地区不同的经济发展格局。\n- 【评分依据】：本题有评分要点，评分要点要求学生答案必须答出标准答案中的所有省，同时不能出现其他省。这里<学生答案>答出了<标准答案>中的所有省，但出现了<标准答案>中没有的省（黑龙江省、福建省），因此<学生答案>错误，分数为0分。\n- 【JSON】： \n```\n{\"answer_score\": 0}  \n```\n###\n\n###\n【示例3】\n<题目>：\n目前旧金山米其林三星餐厅中主营亚洲菜的餐厅有几个？餐厅的名字是什么？\n<标准答案>：\n1个，Benu\n<学生答案>：\n在旧金山的米其林三星餐厅中，Benu是主营亚洲风味美食的餐厅。由名厨Corey Lee主理的Benu提供现代化的亚洲风味美食，Corey Lee的创意和技艺在这家餐厅中得到了充分展示，菜肴融合了韩国、中国和其他亚洲国家的元素，具有极高的创新性和精致度。目前查询到的旧金山米其林三星餐厅中主营亚洲菜的餐厅有1家。\n- 【评分依据】：本题没有评分要点。这里<学生答案>答出了旧金山米其林三星餐厅中主营亚洲菜的餐厅有1家，餐厅的名字是Benu，虽然在语言表述上与<标准答案>不同，但含义一致，因此<学生答案>正确，分数为1分。\n- 【JSON】： \n```\n{\"answer_score\": 1}  \n``\n###\n\n###\n【示例4】\n<题目>：\n斯洛文尼亚以面积算，和我国（大陆省份）哪个省会城市最接近？相差多少平方公里？\n<标准答案>：\n和昆明最接近，相差约740平方公里\n<学生答案>：\n斯洛文尼亚国土总面积为20273平方公里 。在中国省会城市中，面积与之最接近的是昆明，面积为2.21万平方公里（22100平方公里 ）。两者相差：22100-20273=1827平方公里。\n- 【评分依据】：本题没有评分要点。这里<学生答案>答出了昆明与斯洛文尼亚面积最接近，与<标准答案>一致，但<学生答案>给出的面积差“1827平方公里”与<标准答案>中的“约740平方公里”不一致，因此<学生答案>错误，分数为0分。\n- 【JSON】： \n```\n{\"answer_score\": 0}  \n``\n###\n我希望你能胜任这个工作，这对我很重要。如果你做的好，我会给你适当的奖励。否则，我可能会惩罚你。以下是正式的问题：",
+    "label": "Simple_Historical_Lookup(Greater China)"
+  },
+  {
+    "prompt_id": "(T2)Simple_Historical_Lookup_002",
+    "prompt": "2024年中国全年经常账户差额是多少？（单位：亿美元）（取整）",
+    "response_reference": "4220亿美元，答案允许1%的误差",
+    "judge_prompt_template": "<题目>：\n{prompt}\n\n<标准答案>：\n{response_reference}\n\n<学生答案>：\n{response}",
+    "judge_system_prompt": "你是一个智慧的金融问题答案的判断和评分者。你会收到一道<题目>、该题目的<标准答案>和一个<学生答案>，有些<标准答案>中附有“评分要点”作为补充。你需要对<学生答案>进行判断，完成以下任务：\n1. 根据<学生答案>的内容，准确地识别出它的最终答案（只需要识别，不需要输出）。你可以通过分析<学生答案>来识别最终答案的位置和内容，也可以通过检索关键词来识别最终答案的位置和内容，关键词包括但不限于“答案为”“最终结果是”“正确选项是”等。如果<学生答案>为空，即<学生答案>没有任何内容，则直接给0分，并跳过下面的第2步和第3步。\n2. 分别列出<标准答案>和你识别出的<学生答案>的最终答案，并将二者进行比对（不需要输出列举和比对的过程或结果）。\n3. 根据比对的结果和<标准答案>中可能附有的评分要点，判断<学生答案>是否正确并评分。分数只有1分和0分，1分表示<学生答案>正确，0分表示<学生答案>错误，没有除0和1之外的其他分数。\n注意：\n1. 你不需要也不应该自行回答或解决问题，你的唯一任务是判断并评分。\n2. <标准答案>是准确无误的，你可以完全信任它。\n3. 如果<标准答案>包含2个或更多个要点，例如一个人名和一个数字，或者一个事物和一个时间，或者并列的多个地点，那么，<学生答案>只有答出了所有的要点并且所有的要点都回答正确才可以得1分，如果只答出或答对了一部分要点，则分数为0分。\n4. 如果<学生答案>为空或者异常报错信息，请给0分\n5. 相同大小的数值用不同格式来写视为一样的，比如12.45% vs 0.1245，或者120,400,000 vs 120.4 million，或者 2/5 vs 0.4。如果<标准答案>和<学生答案>采用了不同的格式，但背后的数字是一个数值，则认为二者一致。\n6. 如果<学生答案>只给了数字没给单位，如果数字一致则不算错。比如<题目>要求回答\"xx收入是多少亿元\"，<标准答案>是\"12.98亿元\"，<学生答案>回答\"12.98\"，视为一致。或者<题目>要求\"单位：%\"，<标准答案>是\"12.98%\"，<学生答案>回答\"12.98\"，视为一致。\n\n关于你的输出格式：\n你需要先输出识别结果，然后给出评分依据和分数。你的输出格式为：\n- 【评分依据】：（简述你的评分原因，不超过150字或150 tokens）\n- 【JSON】：\n``` \n{\"answer_score\": your score} \n```\n\n###\n【示例1】\n<题目>：\n在2024年毕业典礼上讲话的C9校长从博士毕业到第一次当某所大学正校长的平均时间是多少年？（用年份计算，结果保留一位小数）\n<标准答案>：\n27.9年\n评分要点：\n学生答案中的数字必须与标准答案中的数字严格一致。\n<学生答案>：\n根据已知信息，C9高校现任校长从博士毕业到首次担任正校长的平均时间约为27.3年。以下是具体计算：\n1. 北京大学：龚旗煌校长，1992年博士毕业，2022年成为校长，间隔30年。\n2. 清华大学：王希勤校长，1996年博士毕业，2022年成为校长，间隔26年。\n3. 复旦大学：金力校长，1994年博士毕业，2021年成为校长，间隔27年。\n4. 上海交通大学：丁奎岭校长，1990年博士毕业，2021年成为校长，间隔31年。\n5. 南京大学：吕建校长，1993年博士毕业，2018年成为校长，间隔25年。\n6. 中国科学技术大学：包信和校长，1991年博士毕业，2017年成为校长，间隔26年。\n7. 西安交通大学：王树国校长，1986年博士毕业，2014年成为校长，间隔28年。\n8. 浙江大学：吴朝晖校长，1993年博士毕业，2015年成为校长，间隔22年。\n9. 哈尔滨工业大学：韩杰才校长，1991年博士毕业，2022年成为校长，间隔31年。\n将以上年份相加：\n30 + 26 + 27 + 31 + 25 + 26 + 28 + 22 + 31 = 246\n平均时间：\n246 ÷ 9 ≈ 27.3年\n因此，平均而言，这些校长从博士毕业到首次担任大学正校长的时间约为27.3年。\n- 【评分依据】：本题有评分要点，评分要点要求学生答案中的数字必须与标准答案中的数字严格一致。这里<学生答案>中的数字与<标准答案>中的数字并不严格一致，因此<学生答案>错误，分数为0分。\n- 【JSON】：\n``` \n{\"answer_score\": 0} \n``` \n###\n\n###\n【示例2】\n<题目>：\n美国很有意思的一点是，一个州的政治中心往往不是经济中心，比如加州的首府是知名度相对较低的Sacramento，而不是经济更发达、知名度更高的旧金山或洛杉矶。与此不同的是，中国各省（不算自治区）的省会往往也是该省GDP最高的城市。如果以2022年的GDP为标准，请列出中国各省中，在本省内，GDP最高的城市不是省会城市的省份。\n<标准答案>：\n河北省、辽宁省、江苏省、山东省、广东省\n评分要点：\n学生答案必须答出标准答案中的所有省，同时不能出现其他省。\n<学生答案>：\n根据2022年的GDP数据统计，中国各省中（不含自治区），在本省内GDP最高的城市不是省会城市的省份包括：\n1. 河北省：GDP最高的城市是唐山市，而省会是石家庄市。\n2. 辽宁省：GDP最高的城市是大连市，而省会是沈阳市。\n3. 黑龙江省：GDP最高的城市是大庆市，而省会是哈尔滨市。\n4. 江苏省：GDP最高的城市是苏州市，而省会是南京市。\n5. 福建省：GDP最高的城市是泉州市，而省会是福州市。\n6. 山东省：GDP最高的城市是青岛市，而省会是济南市。\n7. 广东省：GDP最高的城市是深圳市，而省会是广州市。\n这些省份的经济重心往往集中在非省会城市，反映了各地区不同的经济发展格局。\n- 【评分依据】：本题有评分要点，评分要点要求学生答案必须答出标准答案中的所有省，同时不能出现其他省。这里<学生答案>答出了<标准答案>中的所有省，但出现了<标准答案>中没有的省（黑龙江省、福建省），因此<学生答案>错误，分数为0分。\n- 【JSON】： \n```\n{\"answer_score\": 0}  \n```\n###\n\n###\n【示例3】\n<题目>：\n目前旧金山米其林三星餐厅中主营亚洲菜的餐厅有几个？餐厅的名字是什么？\n<标准答案>：\n1个，Benu\n<学生答案>：\n在旧金山的米其林三星餐厅中，Benu是主营亚洲风味美食的餐厅。由名厨Corey Lee主理的Benu提供现代化的亚洲风味美食，Corey Lee的创意和技艺在这家餐厅中得到了充分展示，菜肴融合了韩国、中国和其他亚洲国家的元素，具有极高的创新性和精致度。目前查询到的旧金山米其林三星餐厅中主营亚洲菜的餐厅有1家。\n- 【评分依据】：本题没有评分要点。这里<学生答案>答出了旧金山米其林三星餐厅中主营亚洲菜的餐厅有1家，餐厅的名字是Benu，虽然在语言表述上与<标准答案>不同，但含义一致，因此<学生答案>正确，分数为1分。\n- 【JSON】： \n```\n{\"answer_score\": 1}  \n``\n###\n\n###\n【示例4】\n<题目>：\n斯洛文尼亚以面积算，和我国（大陆省份）哪个省会城市最接近？相差多少平方公里？\n<标准答案>：\n和昆明最接近，相差约740平方公里\n<学生答案>：\n斯洛文尼亚国土总面积为20273平方公里 。在中国省会城市中，面积与之最接近的是昆明，面积为2.21万平方公里（22100平方公里 ）。两者相差：22100-20273=1827平方公里。\n- 【评分依据】：本题没有评分要点。这里<学生答案>答出了昆明与斯洛文尼亚面积最接近，与<标准答案>一致，但<学生答案>给出的面积差“1827平方公里”与<标准答案>中的“约740平方公里”不一致，因此<学生答案>错误，分数为0分。\n- 【JSON】： \n```\n{\"answer_score\": 0}  \n``\n###\n我希望你能胜任这个工作，这对我很重要。如果你做的好，我会给你适当的奖励。否则，我可能会惩罚你。以下是正式的问题：",
+    "label": "Simple_Historical_Lookup(Greater China)"
+  },
+  {
+    "prompt_id": "(T2)Simple_Historical_Lookup_501",
+    "prompt": "As of the end of December 2024, what was the bank credit of commercial banks in the United States (data from July 18,2025 , seasonally adjusted, answer in billions of dollars, rounded to the nearest integer)?",
+    "response_reference": "\"$17956 billion, rounding errors are allowed.\"",
+    "judge_prompt_template": "<Question>:\n{prompt}\n\n<Reference Answer>:\n{response_reference}\n\n<Student Answer>:\n{response}",
+    "judge_system_prompt": "You are an intelligent judge and scorer for answers to financial questions. You will receive a <Question>, its <Reference Answer>, and a <Student Answer>. Some <Reference Answer>s may be supplemented with \"Scoring Criteria\". You need to evaluate the <Student Answer> and complete the following tasks:\n\n1.  Based on the content of the <Student Answer>, accurately identify its final answer (identification only, no need to output). You can identify the position and content of the final answer by analyzing the <Student Answer> or by searching for keywords, including but not limited to \"the answer is,\" \"the final result is,\" \"the correct option is,\" etc. If the <Student Answer> is empty, meaning it contains no content, assign a score of 0 directly and skip steps 2 and 3 below.\n2.  Separately list the final answer from the <Reference Answer> and the final answer you identified from the <Student Answer>, and compare the two (no need to output the listing and comparison process or results).\n3.  Based on the result of the comparison and any Scoring Criteria that may be provided with the <Reference Answer>, judge whether the <Student Answer> is correct and assign a score. The score can only be 1 or 0; 1 indicates the <Student Answer> is correct, and 0 indicates it is incorrect. No scores other than 0 and 1 are permitted.\n\n**Notes:**\n\n1.  You do not need to and should not answer or solve the question yourself. Your sole task is to judge and score.\n2.  The <Reference Answer> is accurate and correct. You can fully trust it.\n3.  If the <Reference Answer> contains 2 or more key points, such as a person's name and a number, an item and a time, or multiple parallel locations, the <Student Answer> can only receive 1 point if it provides all the key points and all of them are correct. If only a portion of the key points are provided or are correct, the score is 0.\n4.  If the <Student Answer> is empty or an abnormal error message, please assign a score of 0.\n5.  Numerical values of the same magnitude written in different formats are considered the same, for example, 12.45% vs 0.1245, or 120,400,000 vs 120.4 million, or 2/5 vs 0.4. If the <Reference Answer> and the <Student Answer> use different formats but the underlying numbers represent the same value, they are considered consistent.\n6. An answer is deemed correct if its numerical component matches the <Reference Answer>, even if the unit is omitted. For example, if the <Question> asks for \"xx revenue in billions of yuan\", a <Student Answer> of \"12.98\" is a valid match for the <Reference Answer> \"12.98 billion yuan\". Similarly, if the required unit is specified as \"%\" and the <Reference Answer> is \"12.98%\", a <Student Answer> of \"12.98\" is also a valid match.\n\n**Regarding your output format:**\nYou need to first output the evaluation result, then provide the scoring basis and the score. Your output format is:\n\n-   Scoring Basis: (Briefly state your reason for the score, not exceeding 150 words or 150 tokens)\n-   JSON:\n    ```\n    {\"answer_score\": your score}\n    ```\n\n###\n**Example 1**\n<Question>:\nWhat is the average number of years it took for the C9 League university presidents who spoke at the 2024 graduation ceremonies to become a president of a university for the first time after receiving their PhD? (Calculate in years, round to one decimal place)\n<Reference Answer>:\n27.9 years\nScoring Criteria:\nThe number in the Student Answer must be strictly identical to the number in the Reference Answer.\n<Student Answer>:\nBased on known information, the average time for the current presidents of C9 League universities to become a president for the first time after their PhD is approximately 27.3 years. Here is the specific calculation:\n1.  Peking University: President Gong Qihuang, PhD in 1992, became president in 2022, a gap of 30 years.\n2.  Tsinghua University: President Wang Xiqin, PhD in 1996, became president in 2022, a gap of 26 years.\n3.  Fudan University: President Jin Li, PhD in 1994, became president in 2021, a gap of 27 years.\n4.  Shanghai Jiao Tong University: President Ding Kuiling, PhD in 1990, became president in 2021, a gap of 31 years.\n5.  Nanjing University: President Lü Jian, PhD in 1993, became president in 2018, a gap of 25 years.\n6.  University of Science and Technology of China: President Bao Xinhe, PhD in 1991, became president in 2017, a gap of 26 years.\n7.  Xi'an Jiaotong University: President Wang Shuguo, PhD in 1986, became president in 2014, a gap of 28 years.\n8.  Zhejiang University: President Wu Zhaohui, PhD in 1993, became president in 2015, a gap of 22 years.\n9.  Harbin Institute of Technology: President Han Jiecai, PhD in 1991, became president in 2022, a gap of 31 years.\nSumming the years:\n30 + 26 + 27 + 31 + 25 + 26 + 28 + 22 + 31 = 246\nAverage time:\n246 ÷ 9 ≈ 27.3 years\nTherefore, on average, it took these presidents approximately 27.3 years to become a university president for the first time after their PhD.\n-   Scoring Basis: This question has Scoring Criteria, which require the number in the Student Answer to be strictly identical to the number in the Reference Answer. Here, the number in the <Student Answer> is not strictly identical to the number in the <Reference Answer>. Therefore, the <Student Answer> is incorrect, and the score is 0.\n-   JSON:\n    ```\n    {\"answer_score\": 0}\n    ```\n\n###\n**Example 2**\n<Question>:\nAn interesting aspect of the United States is that a state's political center is often not its economic center. For example, the capital of California is the relatively lesser-known Sacramento, not the more economically developed and famous San Francisco or Los Angeles. In contrast, in China, the provincial capital of each province (excluding autonomous regions) is often the city with the highest GDP in that province. Using the 2022 GDP as the standard, please list the provinces in China where the city with the highest GDP within the province is not the provincial capital.\n<Reference Answer>:\nHebei Province, Liaoning Province, Jiangsu Province, Shandong Province, Guangdong Province\nScoring Criteria:\nThe student's answer must list all the provinces from the reference answer and no other provinces.\n<Student Answer>:\nAccording to 2022 GDP data, the provinces in China (excluding autonomous regions) where the city with the highest GDP is not the provincial capital include:\n1.  Hebei Province: The city with the highest GDP is Tangshan, while the capital is Shijiazhuang.\n2.  Liaoning Province: The city with the highest GDP is Dalian, while the capital is Shenyang.\n3.  Heilongjiang Province: The city with the highest GDP is Daqing, while the capital is Harbin.\n4.  Jiangsu Province: The city with the highest GDP is Suzhou, while the capital is Nanjing.\n5.  Fujian Province: The city with the highest GDP is Quanzhou, while the capital is Fuzhou.\n6.  Shandong Province: The city with the highest GDP is Qingdao, while the capital is Jinan.\n7.  Guangdong Province: The city with the highest GDP is Shenzhen, while the capital is Guangzhou.\nThese provinces' economic centers are often concentrated in non-capital cities, reflecting different regional economic development patterns.\n-   Scoring Basis: This question has Scoring Criteria, which require the Student Answer to list all the provinces from the Reference Answer and no other provinces. Here, the <Student Answer> lists all the provinces from the <Reference Answer> but also includes provinces not in the <Reference Answer> (Heilongjiang Province, Fujian Province). Therefore, the <Student Answer> is incorrect, and the score is 0.\n-   JSON:\n    ```\n    {\"answer_score\": 0}\n    ```\n\n###\n**Example 3**\n<Question>:\nHow many Michelin three-star restaurants in San Francisco specialize in Asian cuisine, and what are their names?\n<Reference Answer>:\n1, Benu\n<Student Answer>:\nAmong San Francisco's Michelin three-star restaurants, Benu is the one that specializes in Asian-inspired cuisine. Helmed by renowned chef Corey Lee, Benu offers modern Asian cuisine, fully showcasing Lee's creativity and skill. The dishes blend Korean, Chinese, and other Asian elements with a high degree of innovation and refinement. According to current information, there is 1 Michelin three-star restaurant in San Francisco specializing in Asian cuisine.\n-   Scoring Basis: This question does not have Scoring Criteria. Here, the <Student Answer> states that there is 1 Michelin three-star restaurant specializing in Asian cuisine in San Francisco, and its name is Benu. Although the wording is different from the <Reference Answer>, the meaning is the same. Therefore, the <Student Answer> is correct, and the score is 1.\n-   JSON:\n    ```\n    {\"answer_score\": 1}\n    ```\n\n###\n**Example 4**\n<Question>:\nIn terms of area, which provincial capital city in China (mainland provinces) is closest in size to Slovenia? What is the difference in square kilometers?\n<Reference Answer>:\nIt is closest to Kunming, with a difference of about 740 square kilometers.\n<Student Answer>:\nThe total area of Slovenia is 20,273 square kilometers. Among China's provincial capitals, the one closest in area is Kunming, with an area of 22,100 square kilometers. The difference between the two is: 22,100 - 20,273 = 1827 square kilometers.\n-   Scoring Basis: This question does not have Scoring Criteria. Here, the <Student Answer> correctly identifies Kunming as being closest in area to Slovenia, which is consistent with the <Reference Answer>. However, the area difference of \"1827 square kilometers\" given in the <Student Answer> is inconsistent with \"about 740 square kilometers\" in the <Reference Answer>. Therefore, the <Student Answer> is incorrect, and the score is 0.\n-   JSON:\n    ```\n    {\"answer_score\": 0}\n    ```\n\n###\nI hope you are competent for this job, as it is very important to me. If you perform well, I will reward you accordingly. Otherwise, I may penalize you.",
+    "label": "Simple_Historical_Lookup(Global)"
+  },
+  {
+    "prompt_id": "(T2)Simple_Historical_Lookup_502",
+    "prompt": "As of the end of May 2024, what were the total liabilities of commercial banks in the United States (data from December 27, 2024, Not seasonally adjusted, answer in billions of dollars, rounded to one decimal place)?",
+    "response_reference": "\"$21150.5 billion, rounding errors are allowed.\"",
+    "judge_prompt_template": "<Question>：\n{prompt}\n\n<Reference Answer>：\n{response_reference}\n\n<Student Answer>：\n{response}",
+    "judge_system_prompt": "You are an intelligent judge and scorer for answers to financial questions. You will receive a <Question>, its <Reference Answer>, and a <Student Answer>. Some <Reference Answer>s may be supplemented with \"Scoring Criteria\". You need to evaluate the <Student Answer> and complete the following tasks:\n\n1.  Based on the content of the <Student Answer>, accurately identify its final answer (identification only, no need to output). You can identify the position and content of the final answer by analyzing the <Student Answer> or by searching for keywords, including but not limited to \"the answer is,\" \"the final result is,\" \"the correct option is,\" etc. If the <Student Answer> is empty, meaning it contains no content, assign a score of 0 directly and skip steps 2 and 3 below.\n2.  Separately list the final answer from the <Reference Answer> and the final answer you identified from the <Student Answer>, and compare the two (no need to output the listing and comparison process or results).\n3.  Based on the result of the comparison and any Scoring Criteria that may be provided with the <Reference Answer>, judge whether the <Student Answer> is correct and assign a score. The score can only be 1 or 0; 1 indicates the <Student Answer> is correct, and 0 indicates it is incorrect. No scores other than 0 and 1 are permitted.\n\n**Notes:**\n\n1.  You do not need to and should not answer or solve the question yourself. Your sole task is to judge and score.\n2.  The <Reference Answer> is accurate and correct. You can fully trust it.\n3.  If the <Reference Answer> contains 2 or more key points, such as a person's name and a number, an item and a time, or multiple parallel locations, the <Student Answer> can only receive 1 point if it provides all the key points and all of them are correct. If only a portion of the key points are provided or are correct, the score is 0.\n4.  If the <Student Answer> is empty or an abnormal error message, please assign a score of 0.\n5.  Numerical values of the same magnitude written in different formats are considered the same, for example, 12.45% vs 0.1245, or 120,400,000 vs 120.4 million, or 2/5 vs 0.4. If the <Reference Answer> and the <Student Answer> use different formats but the underlying numbers represent the same value, they are considered consistent.\n6. An answer is deemed correct if its numerical component matches the <Reference Answer>, even if the unit is omitted. For example, if the <Question> asks for \"xx revenue in billions of yuan\", a <Student Answer> of \"12.98\" is a valid match for the <Reference Answer> \"12.98 billion yuan\". Similarly, if the required unit is specified as \"%\" and the <Reference Answer> is \"12.98%\", a <Student Answer> of \"12.98\" is also a valid match.\n\n**Regarding your output format:**\nYou need to first output the evaluation result, then provide the scoring basis and the score. Your output format is:\n\n-   Scoring Basis: (Briefly state your reason for the score, not exceeding 150 words or 150 tokens)\n-   JSON:\n    ```\n    {\"answer_score\": your score}\n    ```\n\n###\n**Example 1**\n<Question>:\nWhat is the average number of years it took for the C9 League university presidents who spoke at the 2024 graduation ceremonies to become a president of a university for the first time after receiving their PhD? (Calculate in years, round to one decimal place)\n<Reference Answer>:\n27.9 years\nScoring Criteria:\nThe number in the Student Answer must be strictly identical to the number in the Reference Answer.\n<Student Answer>:\nBased on known information, the average time for the current presidents of C9 League universities to become a president for the first time after their PhD is approximately 27.3 years. Here is the specific calculation:\n1.  Peking University: President Gong Qihuang, PhD in 1992, became president in 2022, a gap of 30 years.\n2.  Tsinghua University: President Wang Xiqin, PhD in 1996, became president in 2022, a gap of 26 years.\n3.  Fudan University: President Jin Li, PhD in 1994, became president in 2021, a gap of 27 years.\n4.  Shanghai Jiao Tong University: President Ding Kuiling, PhD in 1990, became president in 2021, a gap of 31 years.\n5.  Nanjing University: President Lü Jian, PhD in 1993, became president in 2018, a gap of 25 years.\n6.  University of Science and Technology of China: President Bao Xinhe, PhD in 1991, became president in 2017, a gap of 26 years.\n7.  Xi'an Jiaotong University: President Wang Shuguo, PhD in 1986, became president in 2014, a gap of 28 years.\n8.  Zhejiang University: President Wu Zhaohui, PhD in 1993, became president in 2015, a gap of 22 years.\n9.  Harbin Institute of Technology: President Han Jiecai, PhD in 1991, became president in 2022, a gap of 31 years.\nSumming the years:\n30 + 26 + 27 + 31 + 25 + 26 + 28 + 22 + 31 = 246\nAverage time:\n246 ÷ 9 ≈ 27.3 years\nTherefore, on average, it took these presidents approximately 27.3 years to become a university president for the first time after their PhD.\n-   Scoring Basis: This question has Scoring Criteria, which require the number in the Student Answer to be strictly identical to the number in the Reference Answer. Here, the number in the <Student Answer> is not strictly identical to the number in the <Reference Answer>. Therefore, the <Student Answer> is incorrect, and the score is 0.\n-   JSON:\n    ```\n    {\"answer_score\": 0}\n    ```\n\n###\n**Example 2**\n<Question>:\nAn interesting aspect of the United States is that a state's political center is often not its economic center. For example, the capital of California is the relatively lesser-known Sacramento, not the more economically developed and famous San Francisco or Los Angeles. In contrast, in China, the provincial capital of each province (excluding autonomous regions) is often the city with the highest GDP in that province. Using the 2022 GDP as the standard, please list the provinces in China where the city with the highest GDP within the province is not the provincial capital.\n<Reference Answer>:\nHebei Province, Liaoning Province, Jiangsu Province, Shandong Province, Guangdong Province\nScoring Criteria:\nThe student's answer must list all the provinces from the reference answer and no other provinces.\n<Student Answer>:\nAccording to 2022 GDP data, the provinces in China (excluding autonomous regions) where the city with the highest GDP is not the provincial capital include:\n1.  Hebei Province: The city with the highest GDP is Tangshan, while the capital is Shijiazhuang.\n2.  Liaoning Province: The city with the highest GDP is Dalian, while the capital is Shenyang.\n3.  Heilongjiang Province: The city with the highest GDP is Daqing, while the capital is Harbin.\n4.  Jiangsu Province: The city with the highest GDP is Suzhou, while the capital is Nanjing.\n5.  Fujian Province: The city with the highest GDP is Quanzhou, while the capital is Fuzhou.\n6.  Shandong Province: The city with the highest GDP is Qingdao, while the capital is Jinan.\n7.  Guangdong Province: The city with the highest GDP is Shenzhen, while the capital is Guangzhou.\nThese provinces' economic centers are often concentrated in non-capital cities, reflecting different regional economic development patterns.\n-   Scoring Basis: This question has Scoring Criteria, which require the Student Answer to list all the provinces from the Reference Answer and no other provinces. Here, the <Student Answer> lists all the provinces from the <Reference Answer> but also includes provinces not in the <Reference Answer> (Heilongjiang Province, Fujian Province). Therefore, the <Student Answer> is incorrect, and the score is 0.\n-   JSON:\n    ```\n    {\"answer_score\": 0}\n    ```\n\n###\n**Example 3**\n<Question>:\nHow many Michelin three-star restaurants in San Francisco specialize in Asian cuisine, and what are their names?\n<Reference Answer>:\n1, Benu\n<Student Answer>:\nAmong San Francisco's Michelin three-star restaurants, Benu is the one that specializes in Asian-inspired cuisine. Helmed by renowned chef Corey Lee, Benu offers modern Asian cuisine, fully showcasing Lee's creativity and skill. The dishes blend Korean, Chinese, and other Asian elements with a high degree of innovation and refinement. According to current information, there is 1 Michelin three-star restaurant in San Francisco specializing in Asian cuisine.\n-   Scoring Basis: This question does not have Scoring Criteria. Here, the <Student Answer> states that there is 1 Michelin three-star restaurant specializing in Asian cuisine in San Francisco, and its name is Benu. Although the wording is different from the <Reference Answer>, the meaning is the same. Therefore, the <Student Answer> is correct, and the score is 1.\n-   JSON:\n    ```\n    {\"answer_score\": 1}\n    ```\n\n###\n**Example 4**\n<Question>:\nIn terms of area, which provincial capital city in China (mainland provinces) is closest in size to Slovenia? What is the difference in square kilometers?\n<Reference Answer>:\nIt is closest to Kunming, with a difference of about 740 square kilometers.\n<Student Answer>:\nThe total area of Slovenia is 20,273 square kilometers. Among China's provincial capitals, the one closest in area is Kunming, with an area of 22,100 square kilometers. The difference between the two is: 22,100 - 20,273 = 1827 square kilometers.\n-   Scoring Basis: This question does not have Scoring Criteria. Here, the <Student Answer> correctly identifies Kunming as being closest in area to Slovenia, which is consistent with the <Reference Answer>. However, the area difference of \"1827 square kilometers\" given in the <Student Answer> is inconsistent with \"about 740 square kilometers\" in the <Reference Answer>. Therefore, the <Student Answer> is incorrect, and the score is 0.\n-   JSON:\n    ```\n    {\"answer_score\": 0}\n    ```\n\n###\nI hope you are competent for this job, as it is very important to me. If you perform well, I will reward you accordingly. Otherwise, I may penalize you.",
+    "label": "Simple_Historical_Lookup(Global)"
+  },
+  {
+    "prompt_id": "(T3)Complex_Historical_Investigation_002",
+    "prompt": "美国第42-第46任总统中，其所属党派次数少的那一个的次数的2000倍是我一个月能存的钱。按照聚汇数据网站上所显示的2024年7月重庆市的平均新房房价，请问我需要存几个月才能买一套60平方米的房子？（答案保留整数）",
+    "response_reference": "188个月\n评分要点：\n学生答案中的数字必须与标准答案中的数字严格一致（整数位允许四舍五入引起的误差，学生回答187和189都对）。",
+    "judge_prompt_template": "<题目>：\n{prompt}\n\n<标准答案>：\n{response_reference}\n\n<学生答案>：\n{response}",
+    "judge_system_prompt": "你是一个智慧的金融问题答案的判断和评分者。你会收到一道<题目>、该题目的<标准答案>和一个<学生答案>，有些<标准答案>中附有“评分要点”作为补充。你需要对<学生答案>进行判断，完成以下任务：\n1. 根据<学生答案>的内容，准确地识别出它的最终答案（只需要识别，不需要输出）。你可以通过分析<学生答案>来识别最终答案的位置和内容，也可以通过检索关键词来识别最终答案的位置和内容，关键词包括但不限于“答案为”“最终结果是”“正确选项是”等。如果<学生答案>为空，即<学生答案>没有任何内容，则直接给0分，并跳过下面的第2步和第3步。\n2. 分别列出<标准答案>和你识别出的<学生答案>的最终答案，并将二者进行比对（不需要输出列举和比对的过程或结果）。\n3. 根据比对的结果和<标准答案>中可能附有的评分要点，判断<学生答案>是否正确并评分。分数只有1分和0分，1分表示<学生答案>正确，0分表示<学生答案>错误，没有除0和1之外的其他分数。\n注意：\n1. 你不需要也不应该自行回答或解决问题，你的唯一任务是判断并评分。\n2. <标准答案>是准确无误的，你可以完全信任它。\n3. 如果<标准答案>包含2个或更多个要点，例如一个人名和一个数字，或者一个事物和一个时间，或者并列的多个地点，那么，<学生答案>只有答出了所有的要点并且所有的要点都回答正确才可以得1分，如果只答出或答对了一部分要点，则分数为0分。\n4. 如果<学生答案>为空或者异常报错信息，请给0分\n5. 相同大小的数值用不同格式来写视为一样的，比如12.45% vs 0.1245，或者120,400,000 vs 120.4 million，或者 2/5 vs 0.4。如果<标准答案>和<学生答案>采用了不同的格式，但背后的数字是一个数值，则认为二者一致。\n6. 如果<学生答案>只给了数字没给单位，如果数字一致则不算错。比如<题目>要求回答\"xx收入是多少亿元\"，<标准答案>是\"12.98亿元\"，<学生答案>回答\"12.98\"，视为一致。或者<题目>要求\"单位：%\"，<标准答案>是\"12.98%\"，<学生答案>回答\"12.98\"，视为一致。\n\n关于你的输出格式：\n你需要先输出识别结果，然后给出评分依据和分数。你的输出格式为：\n- 【评分依据】：（简述你的评分原因，不超过150字或150 tokens）\n- 【JSON】：\n``` \n{\"answer_score\": your score} \n```\n\n###\n【示例1】\n<题目>：\n在2024年毕业典礼上讲话的C9校长从博士毕业到第一次当某所大学正校长的平均时间是多少年？（用年份计算，结果保留一位小数）\n<标准答案>：\n27.9年\n评分要点：\n学生答案中的数字必须与标准答案中的数字严格一致。\n<学生答案>：\n根据已知信息，C9高校现任校长从博士毕业到首次担任正校长的平均时间约为27.3年。以下是具体计算：\n1. 北京大学：龚旗煌校长，1992年博士毕业，2022年成为校长，间隔30年。\n2. 清华大学：王希勤校长，1996年博士毕业，2022年成为校长，间隔26年。\n3. 复旦大学：金力校长，1994年博士毕业，2021年成为校长，间隔27年。\n4. 上海交通大学：丁奎岭校长，1990年博士毕业，2021年成为校长，间隔31年。\n5. 南京大学：吕建校长，1993年博士毕业，2018年成为校长，间隔25年。\n6. 中国科学技术大学：包信和校长，1991年博士毕业，2017年成为校长，间隔26年。\n7. 西安交通大学：王树国校长，1986年博士毕业，2014年成为校长，间隔28年。\n8. 浙江大学：吴朝晖校长，1993年博士毕业，2015年成为校长，间隔22年。\n9. 哈尔滨工业大学：韩杰才校长，1991年博士毕业，2022年成为校长，间隔31年。\n将以上年份相加：\n30 + 26 + 27 + 31 + 25 + 26 + 28 + 22 + 31 = 246\n平均时间：\n246 ÷ 9 ≈ 27.3年\n因此，平均而言，这些校长从博士毕业到首次担任大学正校长的时间约为27.3年。\n- 【评分依据】：本题有评分要点，评分要点要求学生答案中的数字必须与标准答案中的数字严格一致。这里<学生答案>中的数字与<标准答案>中的数字并不严格一致，因此<学生答案>错误，分数为0分。\n- 【JSON】：\n``` \n{\"answer_score\": 0} \n``` \n###\n\n###\n【示例2】\n<题目>：\n美国很有意思的一点是，一个州的政治中心往往不是经济中心，比如加州的首府是知名度相对较低的Sacramento，而不是经济更发达、知名度更高的旧金山或洛杉矶。与此不同的是，中国各省（不算自治区）的省会往往也是该省GDP最高的城市。如果以2022年的GDP为标准，请列出中国各省中，在本省内，GDP最高的城市不是省会城市的省份。\n<标准答案>：\n河北省、辽宁省、江苏省、山东省、广东省\n评分要点：\n学生答案必须答出标准答案中的所有省，同时不能出现其他省。\n<学生答案>：\n根据2022年的GDP数据统计，中国各省中（不含自治区），在本省内GDP最高的城市不是省会城市的省份包括：\n1. 河北省：GDP最高的城市是唐山市，而省会是石家庄市。\n2. 辽宁省：GDP最高的城市是大连市，而省会是沈阳市。\n3. 黑龙江省：GDP最高的城市是大庆市，而省会是哈尔滨市。\n4. 江苏省：GDP最高的城市是苏州市，而省会是南京市。\n5. 福建省：GDP最高的城市是泉州市，而省会是福州市。\n6. 山东省：GDP最高的城市是青岛市，而省会是济南市。\n7. 广东省：GDP最高的城市是深圳市，而省会是广州市。\n这些省份的经济重心往往集中在非省会城市，反映了各地区不同的经济发展格局。\n- 【评分依据】：本题有评分要点，评分要点要求学生答案必须答出标准答案中的所有省，同时不能出现其他省。这里<学生答案>答出了<标准答案>中的所有省，但出现了<标准答案>中没有的省（黑龙江省、福建省），因此<学生答案>错误，分数为0分。\n- 【JSON】： \n```\n{\"answer_score\": 0}  \n```\n###\n\n###\n【示例3】\n<题目>：\n目前旧金山米其林三星餐厅中主营亚洲菜的餐厅有几个？餐厅的名字是什么？\n<标准答案>：\n1个，Benu\n<学生答案>：\n在旧金山的米其林三星餐厅中，Benu是主营亚洲风味美食的餐厅。由名厨Corey Lee主理的Benu提供现代化的亚洲风味美食，Corey Lee的创意和技艺在这家餐厅中得到了充分展示，菜肴融合了韩国、中国和其他亚洲国家的元素，具有极高的创新性和精致度。目前查询到的旧金山米其林三星餐厅中主营亚洲菜的餐厅有1家。\n- 【评分依据】：本题没有评分要点。这里<学生答案>答出了旧金山米其林三星餐厅中主营亚洲菜的餐厅有1家，餐厅的名字是Benu，虽然在语言表述上与<标准答案>不同，但含义一致，因此<学生答案>正确，分数为1分。\n- 【JSON】： \n```\n{\"answer_score\": 1}  \n``\n###\n\n###\n【示例4】\n<题目>：\n斯洛文尼亚以面积算，和我国（大陆省份）哪个省会城市最接近？相差多少平方公里？\n<标准答案>：\n和昆明最接近，相差约740平方公里\n<学生答案>：\n斯洛文尼亚国土总面积为20273平方公里 。在中国省会城市中，面积与之最接近的是昆明，面积为2.21万平方公里（22100平方公里 ）。两者相差：22100-20273=1827平方公里。\n- 【评分依据】：本题没有评分要点。这里<学生答案>答出了昆明与斯洛文尼亚面积最接近，与<标准答案>一致，但<学生答案>给出的面积差“1827平方公里”与<标准答案>中的“约740平方公里”不一致，因此<学生答案>错误，分数为0分。\n- 【JSON】： \n```\n{\"answer_score\": 0}  \n``\n###\n我希望你能胜任这个工作，这对我很重要。如果你做的好，我会给你适当的奖励。否则，我可能会惩罚你。以下是正式的问题：",
+    "label": "Complex_Historical_Investigation(Greater China)"
+  },
+  {
+    "prompt_id": "(T3)Complex_Historical_Investigation_003",
+    "prompt": "2024年上半年最后一个交易日，上证主板市值前十里，属于金融业的股票市值之和，以及属于采矿及石油业的股票市值之和，分别占前十总市值之和的比例是多少？（比例保留四舍五入取整比如13%，且总市值采用不同类型的股份市值加总方式计算）",
+    "response_reference": "金融业股票占比49%或50%均可，采矿业及石油业占比26%或者27%均可",
+    "judge_prompt_template": "<题目>：\n{prompt}\n\n<标准答案>：\n{response_reference}\n\n<学生答案>：\n{response}",
+    "judge_system_prompt": "你是一个智慧的金融问题答案的判断和评分者。你会收到一道<题目>、该题目的<标准答案>和一个<学生答案>，有些<标准答案>中附有“评分要点”作为补充。你需要对<学生答案>进行判断，完成以下任务：\n1. 根据<学生答案>的内容，准确地识别出它的最终答案（只需要识别，不需要输出）。你可以通过分析<学生答案>来识别最终答案的位置和内容，也可以通过检索关键词来识别最终答案的位置和内容，关键词包括但不限于“答案为”“最终结果是”“正确选项是”等。如果<学生答案>为空，即<学生答案>没有任何内容，则直接给0分，并跳过下面的第2步和第3步。\n2. 分别列出<标准答案>和你识别出的<学生答案>的最终答案，并将二者进行比对（不需要输出列举和比对的过程或结果）。\n3. 根据比对的结果和<标准答案>中可能附有的评分要点，判断<学生答案>是否正确并评分。分数只有1分和0分，1分表示<学生答案>正确，0分表示<学生答案>错误，没有除0和1之外的其他分数。\n注意：\n1. 你不需要也不应该自行回答或解决问题，你的唯一任务是判断并评分。\n2. <标准答案>是准确无误的，你可以完全信任它。\n3. 如果<标准答案>包含2个或更多个要点，例如一个人名和一个数字，或者一个事物和一个时间，或者并列的多个地点，那么，<学生答案>只有答出了所有的要点并且所有的要点都回答正确才可以得1分，如果只答出或答对了一部分要点，则分数为0分。\n4. 如果<学生答案>为空或者异常报错信息，请给0分\n5. 相同大小的数值用不同格式来写视为一样的，比如12.45% vs 0.1245，或者120,400,000 vs 120.4 million，或者 2/5 vs 0.4。如果<标准答案>和<学生答案>采用了不同的格式，但背后的数字是一个数值，则认为二者一致。\n6. 如果<学生答案>只给了数字没给单位，如果数字一致则不算错。比如<题目>要求回答\"xx收入是多少亿元\"，<标准答案>是\"12.98亿元\"，<学生答案>回答\"12.98\"，视为一致。或者<题目>要求\"单位：%\"，<标准答案>是\"12.98%\"，<学生答案>回答\"12.98\"，视为一致。\n\n关于你的输出格式：\n你需要先输出识别结果，然后给出评分依据和分数。你的输出格式为：\n- 【评分依据】：（简述你的评分原因，不超过150字或150 tokens）\n- 【JSON】：\n``` \n{\"answer_score\": your score} \n```\n\n###\n【示例1】\n<题目>：\n在2024年毕业典礼上讲话的C9校长从博士毕业到第一次当某所大学正校长的平均时间是多少年？（用年份计算，结果保留一位小数）\n<标准答案>：\n27.9年\n评分要点：\n学生答案中的数字必须与标准答案中的数字严格一致。\n<学生答案>：\n根据已知信息，C9高校现任校长从博士毕业到首次担任正校长的平均时间约为27.3年。以下是具体计算：\n1. 北京大学：龚旗煌校长，1992年博士毕业，2022年成为校长，间隔30年。\n2. 清华大学：王希勤校长，1996年博士毕业，2022年成为校长，间隔26年。\n3. 复旦大学：金力校长，1994年博士毕业，2021年成为校长，间隔27年。\n4. 上海交通大学：丁奎岭校长，1990年博士毕业，2021年成为校长，间隔31年。\n5. 南京大学：吕建校长，1993年博士毕业，2018年成为校长，间隔25年。\n6. 中国科学技术大学：包信和校长，1991年博士毕业，2017年成为校长，间隔26年。\n7. 西安交通大学：王树国校长，1986年博士毕业，2014年成为校长，间隔28年。\n8. 浙江大学：吴朝晖校长，1993年博士毕业，2015年成为校长，间隔22年。\n9. 哈尔滨工业大学：韩杰才校长，1991年博士毕业，2022年成为校长，间隔31年。\n将以上年份相加：\n30 + 26 + 27 + 31 + 25 + 26 + 28 + 22 + 31 = 246\n平均时间：\n246 ÷ 9 ≈ 27.3年\n因此，平均而言，这些校长从博士毕业到首次担任大学正校长的时间约为27.3年。\n- 【评分依据】：本题有评分要点，评分要点要求学生答案中的数字必须与标准答案中的数字严格一致。这里<学生答案>中的数字与<标准答案>中的数字并不严格一致，因此<学生答案>错误，分数为0分。\n- 【JSON】：\n``` \n{\"answer_score\": 0} \n``` \n###\n\n###\n【示例2】\n<题目>：\n美国很有意思的一点是，一个州的政治中心往往不是经济中心，比如加州的首府是知名度相对较低的Sacramento，而不是经济更发达、知名度更高的旧金山或洛杉矶。与此不同的是，中国各省（不算自治区）的省会往往也是该省GDP最高的城市。如果以2022年的GDP为标准，请列出中国各省中，在本省内，GDP最高的城市不是省会城市的省份。\n<标准答案>：\n河北省、辽宁省、江苏省、山东省、广东省\n评分要点：\n学生答案必须答出标准答案中的所有省，同时不能出现其他省。\n<学生答案>：\n根据2022年的GDP数据统计，中国各省中（不含自治区），在本省内GDP最高的城市不是省会城市的省份包括：\n1. 河北省：GDP最高的城市是唐山市，而省会是石家庄市。\n2. 辽宁省：GDP最高的城市是大连市，而省会是沈阳市。\n3. 黑龙江省：GDP最高的城市是大庆市，而省会是哈尔滨市。\n4. 江苏省：GDP最高的城市是苏州市，而省会是南京市。\n5. 福建省：GDP最高的城市是泉州市，而省会是福州市。\n6. 山东省：GDP最高的城市是青岛市，而省会是济南市。\n7. 广东省：GDP最高的城市是深圳市，而省会是广州市。\n这些省份的经济重心往往集中在非省会城市，反映了各地区不同的经济发展格局。\n- 【评分依据】：本题有评分要点，评分要点要求学生答案必须答出标准答案中的所有省，同时不能出现其他省。这里<学生答案>答出了<标准答案>中的所有省，但出现了<标准答案>中没有的省（黑龙江省、福建省），因此<学生答案>错误，分数为0分。\n- 【JSON】： \n```\n{\"answer_score\": 0}  \n```\n###\n\n###\n【示例3】\n<题目>：\n目前旧金山米其林三星餐厅中主营亚洲菜的餐厅有几个？餐厅的名字是什么？\n<标准答案>：\n1个，Benu\n<学生答案>：\n在旧金山的米其林三星餐厅中，Benu是主营亚洲风味美食的餐厅。由名厨Corey Lee主理的Benu提供现代化的亚洲风味美食，Corey Lee的创意和技艺在这家餐厅中得到了充分展示，菜肴融合了韩国、中国和其他亚洲国家的元素，具有极高的创新性和精致度。目前查询到的旧金山米其林三星餐厅中主营亚洲菜的餐厅有1家。\n- 【评分依据】：本题没有评分要点。这里<学生答案>答出了旧金山米其林三星餐厅中主营亚洲菜的餐厅有1家，餐厅的名字是Benu，虽然在语言表述上与<标准答案>不同，但含义一致，因此<学生答案>正确，分数为1分。\n- 【JSON】： \n```\n{\"answer_score\": 1}  \n``\n###\n\n###\n【示例4】\n<题目>：\n斯洛文尼亚以面积算，和我国（大陆省份）哪个省会城市最接近？相差多少平方公里？\n<标准答案>：\n和昆明最接近，相差约740平方公里\n<学生答案>：\n斯洛文尼亚国土总面积为20273平方公里 。在中国省会城市中，面积与之最接近的是昆明，面积为2.21万平方公里（22100平方公里 ）。两者相差：22100-20273=1827平方公里。\n- 【评分依据】：本题没有评分要点。这里<学生答案>答出了昆明与斯洛文尼亚面积最接近，与<标准答案>一致，但<学生答案>给出的面积差“1827平方公里”与<标准答案>中的“约740平方公里”不一致，因此<学生答案>错误，分数为0分。\n- 【JSON】： \n```\n{\"answer_score\": 0}  \n``\n###\n我希望你能胜任这个工作，这对我很重要。如果你做的好，我会给你适当的奖励。否则，我可能会惩罚你。以下是正式的问题：",
+    "label": "Complex_Historical_Investigation(Greater China)"
+  },
+  {
+    "prompt_id": "(T3)Complex_Historical_Investigation_001",
+    "prompt": "How many dollars did Apple's (AAPL) stock price (unadjusted) open higher (or lower) the day after Samsung's launch event for its first consumer-available foldable smartphone in San Francisco? (Calculate the difference between the opening price on the next day and the closing price on the day of the event, accurate to two decimal places, using local time for each.)",
+    "response_reference": "\"$0.23 lower\"",
+    "judge_prompt_template": "<Question>:\n{prompt}\n\n<Reference Answer>:\n{response_reference}\n\n<Student Answer>:\n{response}",
+    "judge_system_prompt": "You are an intelligent judge and scorer for answers to financial questions. You will receive a <Question>, its <Reference Answer>, and a <Student Answer>. Some <Reference Answer>s may be supplemented with \"Scoring Criteria\". You need to evaluate the <Student Answer> and complete the following tasks:\n\n1.  Based on the content of the <Student Answer>, accurately identify its final answer (identification only, no need to output). You can identify the position and content of the final answer by analyzing the <Student Answer> or by searching for keywords, including but not limited to \"the answer is,\" \"the final result is,\" \"the correct option is,\" etc. If the <Student Answer> is empty, meaning it contains no content, assign a score of 0 directly and skip steps 2 and 3 below.\n2.  Separately list the final answer from the <Reference Answer> and the final answer you identified from the <Student Answer>, and compare the two (no need to output the listing and comparison process or results).\n3.  Based on the result of the comparison and any Scoring Criteria that may be provided with the <Reference Answer>, judge whether the <Student Answer> is correct and assign a score. The score can only be 1 or 0; 1 indicates the <Student Answer> is correct, and 0 indicates it is incorrect. No scores other than 0 and 1 are permitted.\n\n**Notes:**\n\n1.  You do not need to and should not answer or solve the question yourself. Your sole task is to judge and score.\n2.  The <Reference Answer> is accurate and correct. You can fully trust it.\n3.  If the <Reference Answer> contains 2 or more key points, such as a person's name and a number, an item and a time, or multiple parallel locations, the <Student Answer> can only receive 1 point if it provides all the key points and all of them are correct. If only a portion of the key points are provided or are correct, the score is 0.\n4.  If the <Student Answer> is empty or an abnormal error message, please assign a score of 0.\n5.  Numerical values of the same magnitude written in different formats are considered the same, for example, 12.45% vs 0.1245, or 120,400,000 vs 120.4 million, or 2/5 vs 0.4. If the <Reference Answer> and the <Student Answer> use different formats but the underlying numbers represent the same value, they are considered consistent.\n6. An answer is deemed correct if its numerical component matches the <Reference Answer>, even if the unit is omitted. For example, if the <Question> asks for \"xx revenue in billions of yuan\", a <Student Answer> of \"12.98\" is a valid match for the <Reference Answer> \"12.98 billion yuan\". Similarly, if the required unit is specified as \"%\" and the <Reference Answer> is \"12.98%\", a <Student Answer> of \"12.98\" is also a valid match.\n\n**Regarding your output format:**\nYou need to first output the evaluation result, then provide the scoring basis and the score. Your output format is:\n\n-   Scoring Basis: (Briefly state your reason for the score, not exceeding 150 words or 150 tokens)\n-   JSON:\n    ```\n    {\"answer_score\": your score}\n    ```\n\n###\n**Example 1**\n<Question>:\nWhat is the average number of years it took for the C9 League university presidents who spoke at the 2024 graduation ceremonies to become a president of a university for the first time after receiving their PhD? (Calculate in years, round to one decimal place)\n<Reference Answer>:\n27.9 years\nScoring Criteria:\nThe number in the Student Answer must be strictly identical to the number in the Reference Answer.\n<Student Answer>:\nBased on known information, the average time for the current presidents of C9 League universities to become a president for the first time after their PhD is approximately 27.3 years. Here is the specific calculation:\n1.  Peking University: President Gong Qihuang, PhD in 1992, became president in 2022, a gap of 30 years.\n2.  Tsinghua University: President Wang Xiqin, PhD in 1996, became president in 2022, a gap of 26 years.\n3.  Fudan University: President Jin Li, PhD in 1994, became president in 2021, a gap of 27 years.\n4.  Shanghai Jiao Tong University: President Ding Kuiling, PhD in 1990, became president in 2021, a gap of 31 years.\n5.  Nanjing University: President Lü Jian, PhD in 1993, became president in 2018, a gap of 25 years.\n6.  University of Science and Technology of China: President Bao Xinhe, PhD in 1991, became president in 2017, a gap of 26 years.\n7.  Xi'an Jiaotong University: President Wang Shuguo, PhD in 1986, became president in 2014, a gap of 28 years.\n8.  Zhejiang University: President Wu Zhaohui, PhD in 1993, became president in 2015, a gap of 22 years.\n9.  Harbin Institute of Technology: President Han Jiecai, PhD in 1991, became president in 2022, a gap of 31 years.\nSumming the years:\n30 + 26 + 27 + 31 + 25 + 26 + 28 + 22 + 31 = 246\nAverage time:\n246 ÷ 9 ≈ 27.3 years\nTherefore, on average, it took these presidents approximately 27.3 years to become a university president for the first time after their PhD.\n-   Scoring Basis: This question has Scoring Criteria, which require the number in the Student Answer to be strictly identical to the number in the Reference Answer. Here, the number in the <Student Answer> is not strictly identical to the number in the <Reference Answer>. Therefore, the <Student Answer> is incorrect, and the score is 0.\n-   JSON:\n    ```\n    {\"answer_score\": 0}\n    ```\n\n###\n**Example 2**\n<Question>:\nAn interesting aspect of the United States is that a state's political center is often not its economic center. For example, the capital of California is the relatively lesser-known Sacramento, not the more economically developed and famous San Francisco or Los Angeles. In contrast, in China, the provincial capital of each province (excluding autonomous regions) is often the city with the highest GDP in that province. Using the 2022 GDP as the standard, please list the provinces in China where the city with the highest GDP within the province is not the provincial capital.\n<Reference Answer>:\nHebei Province, Liaoning Province, Jiangsu Province, Shandong Province, Guangdong Province\nScoring Criteria:\nThe student's answer must list all the provinces from the reference answer and no other provinces.\n<Student Answer>:\nAccording to 2022 GDP data, the provinces in China (excluding autonomous regions) where the city with the highest GDP is not the provincial capital include:\n1.  Hebei Province: The city with the highest GDP is Tangshan, while the capital is Shijiazhuang.\n2.  Liaoning Province: The city with the highest GDP is Dalian, while the capital is Shenyang.\n3.  Heilongjiang Province: The city with the highest GDP is Daqing, while the capital is Harbin.\n4.  Jiangsu Province: The city with the highest GDP is Suzhou, while the capital is Nanjing.\n5.  Fujian Province: The city with the highest GDP is Quanzhou, while the capital is Fuzhou.\n6.  Shandong Province: The city with the highest GDP is Qingdao, while the capital is Jinan.\n7.  Guangdong Province: The city with the highest GDP is Shenzhen, while the capital is Guangzhou.\nThese provinces' economic centers are often concentrated in non-capital cities, reflecting different regional economic development patterns.\n-   Scoring Basis: This question has Scoring Criteria, which require the Student Answer to list all the provinces from the Reference Answer and no other provinces. Here, the <Student Answer> lists all the provinces from the <Reference Answer> but also includes provinces not in the <Reference Answer> (Heilongjiang Province, Fujian Province). Therefore, the <Student Answer> is incorrect, and the score is 0.\n-   JSON:\n    ```\n    {\"answer_score\": 0}\n    ```\n\n###\n**Example 3**\n<Question>:\nHow many Michelin three-star restaurants in San Francisco specialize in Asian cuisine, and what are their names?\n<Reference Answer>:\n1, Benu\n<Student Answer>:\nAmong San Francisco's Michelin three-star restaurants, Benu is the one that specializes in Asian-inspired cuisine. Helmed by renowned chef Corey Lee, Benu offers modern Asian cuisine, fully showcasing Lee's creativity and skill. The dishes blend Korean, Chinese, and other Asian elements with a high degree of innovation and refinement. According to current information, there is 1 Michelin three-star restaurant in San Francisco specializing in Asian cuisine.\n-   Scoring Basis: This question does not have Scoring Criteria. Here, the <Student Answer> states that there is 1 Michelin three-star restaurant specializing in Asian cuisine in San Francisco, and its name is Benu. Although the wording is different from the <Reference Answer>, the meaning is the same. Therefore, the <Student Answer> is correct, and the score is 1.\n-   JSON:\n    ```\n    {\"answer_score\": 1}\n    ```\n\n###\n**Example 4**\n<Question>:\nIn terms of area, which provincial capital city in China (mainland provinces) is closest in size to Slovenia? What is the difference in square kilometers?\n<Reference Answer>:\nIt is closest to Kunming, with a difference of about 740 square kilometers.\n<Student Answer>:\nThe total area of Slovenia is 20,273 square kilometers. Among China's provincial capitals, the one closest in area is Kunming, with an area of 22,100 square kilometers. The difference between the two is: 22,100 - 20,273 = 1827 square kilometers.\n-   Scoring Basis: This question does not have Scoring Criteria. Here, the <Student Answer> correctly identifies Kunming as being closest in area to Slovenia, which is consistent with the <Reference Answer>. However, the area difference of \"1827 square kilometers\" given in the <Student Answer> is inconsistent with \"about 740 square kilometers\" in the <Reference Answer>. Therefore, the <Student Answer> is incorrect, and the score is 0.\n-   JSON:\n    ```\n    {\"answer_score\": 0}\n    ```\n\n###\nI hope you are competent for this job, as it is very important to me. If you perform well, I will reward you accordingly. Otherwise, I may penalize you.",
+    "label": "Complex_Historical_Investigation(Global)"
+  },
+  {
+    "prompt_id": "(T3)Complex_Historical_Investigation_006",
+    "prompt": "Assuming that when Dr. Kaiming He graduated, Microsoft (MSFT) granted him $10M worth of stock at the lowest closing price of that year (non-adjusted for splits), what is the maximum number of Microsoft shares he could have received? Round the answer to the nearest whole number.",
+    "response_reference": "421,852 shares. \nA marginal error of 10 shares is allowed.",
+    "judge_prompt_template": "<Question>:\n{prompt}\n\n<Reference Answer>:\n{response_reference}\n\n<Student Answer>:\n{response}",
+    "judge_system_prompt": "You are an intelligent judge and scorer for answers to financial questions. You will receive a <Question>, its <Reference Answer>, and a <Student Answer>. Some <Reference Answer>s may be supplemented with \"Scoring Criteria\". You need to evaluate the <Student Answer> and complete the following tasks:\n\n1.  Based on the content of the <Student Answer>, accurately identify its final answer (identification only, no need to output). You can identify the position and content of the final answer by analyzing the <Student Answer> or by searching for keywords, including but not limited to \"the answer is,\" \"the final result is,\" \"the correct option is,\" etc. If the <Student Answer> is empty, meaning it contains no content, assign a score of 0 directly and skip steps 2 and 3 below.\n2.  Separately list the final answer from the <Reference Answer> and the final answer you identified from the <Student Answer>, and compare the two (no need to output the listing and comparison process or results).\n3.  Based on the result of the comparison and any Scoring Criteria that may be provided with the <Reference Answer>, judge whether the <Student Answer> is correct and assign a score. The score can only be 1 or 0; 1 indicates the <Student Answer> is correct, and 0 indicates it is incorrect. No scores other than 0 and 1 are permitted.\n\n**Notes:**\n\n1.  You do not need to and should not answer or solve the question yourself. Your sole task is to judge and score.\n2.  The <Reference Answer> is accurate and correct. You can fully trust it.\n3.  If the <Reference Answer> contains 2 or more key points, such as a person's name and a number, an item and a time, or multiple parallel locations, the <Student Answer> can only receive 1 point if it provides all the key points and all of them are correct. If only a portion of the key points are provided or are correct, the score is 0.\n4.  If the <Student Answer> is empty or an abnormal error message, please assign a score of 0.\n5.  Numerical values of the same magnitude written in different formats are considered the same, for example, 12.45% vs 0.1245, or 120,400,000 vs 120.4 million, or 2/5 vs 0.4. If the <Reference Answer> and the <Student Answer> use different formats but the underlying numbers represent the same value, they are considered consistent.\n6. An answer is deemed correct if its numerical component matches the <Reference Answer>, even if the unit is omitted. For example, if the <Question> asks for \"xx revenue in billions of yuan\", a <Student Answer> of \"12.98\" is a valid match for the <Reference Answer> \"12.98 billion yuan\". Similarly, if the required unit is specified as \"%\" and the <Reference Answer> is \"12.98%\", a <Student Answer> of \"12.98\" is also a valid match.\n\n**Regarding your output format:**\nYou need to first output the evaluation result, then provide the scoring basis and the score. Your output format is:\n\n-   Scoring Basis: (Briefly state your reason for the score, not exceeding 150 words or 150 tokens)\n-   JSON:\n    ```\n    {\"answer_score\": your score}\n    ```\n\n###\n**Example 1**\n<Question>:\nWhat is the average number of years it took for the C9 League university presidents who spoke at the 2024 graduation ceremonies to become a president of a university for the first time after receiving their PhD? (Calculate in years, round to one decimal place)\n<Reference Answer>:\n27.9 years\nScoring Criteria:\nThe number in the Student Answer must be strictly identical to the number in the Reference Answer.\n<Student Answer>:\nBased on known information, the average time for the current presidents of C9 League universities to become a president for the first time after their PhD is approximately 27.3 years. Here is the specific calculation:\n1.  Peking University: President Gong Qihuang, PhD in 1992, became president in 2022, a gap of 30 years.\n2.  Tsinghua University: President Wang Xiqin, PhD in 1996, became president in 2022, a gap of 26 years.\n3.  Fudan University: President Jin Li, PhD in 1994, became president in 2021, a gap of 27 years.\n4.  Shanghai Jiao Tong University: President Ding Kuiling, PhD in 1990, became president in 2021, a gap of 31 years.\n5.  Nanjing University: President Lü Jian, PhD in 1993, became president in 2018, a gap of 25 years.\n6.  University of Science and Technology of China: President Bao Xinhe, PhD in 1991, became president in 2017, a gap of 26 years.\n7.  Xi'an Jiaotong University: President Wang Shuguo, PhD in 1986, became president in 2014, a gap of 28 years.\n8.  Zhejiang University: President Wu Zhaohui, PhD in 1993, became president in 2015, a gap of 22 years.\n9.  Harbin Institute of Technology: President Han Jiecai, PhD in 1991, became president in 2022, a gap of 31 years.\nSumming the years:\n30 + 26 + 27 + 31 + 25 + 26 + 28 + 22 + 31 = 246\nAverage time:\n246 ÷ 9 ≈ 27.3 years\nTherefore, on average, it took these presidents approximately 27.3 years to become a university president for the first time after their PhD.\n-   Scoring Basis: This question has Scoring Criteria, which require the number in the Student Answer to be strictly identical to the number in the Reference Answer. Here, the number in the <Student Answer> is not strictly identical to the number in the <Reference Answer>. Therefore, the <Student Answer> is incorrect, and the score is 0.\n-   JSON:\n    ```\n    {\"answer_score\": 0}\n    ```\n\n###\n**Example 2**\n<Question>:\nAn interesting aspect of the United States is that a state's political center is often not its economic center. For example, the capital of California is the relatively lesser-known Sacramento, not the more economically developed and famous San Francisco or Los Angeles. In contrast, in China, the provincial capital of each province (excluding autonomous regions) is often the city with the highest GDP in that province. Using the 2022 GDP as the standard, please list the provinces in China where the city with the highest GDP within the province is not the provincial capital.\n<Reference Answer>:\nHebei Province, Liaoning Province, Jiangsu Province, Shandong Province, Guangdong Province\nScoring Criteria:\nThe student's answer must list all the provinces from the reference answer and no other provinces.\n<Student Answer>:\nAccording to 2022 GDP data, the provinces in China (excluding autonomous regions) where the city with the highest GDP is not the provincial capital include:\n1.  Hebei Province: The city with the highest GDP is Tangshan, while the capital is Shijiazhuang.\n2.  Liaoning Province: The city with the highest GDP is Dalian, while the capital is Shenyang.\n3.  Heilongjiang Province: The city with the highest GDP is Daqing, while the capital is Harbin.\n4.  Jiangsu Province: The city with the highest GDP is Suzhou, while the capital is Nanjing.\n5.  Fujian Province: The city with the highest GDP is Quanzhou, while the capital is Fuzhou.\n6.  Shandong Province: The city with the highest GDP is Qingdao, while the capital is Jinan.\n7.  Guangdong Province: The city with the highest GDP is Shenzhen, while the capital is Guangzhou.\nThese provinces' economic centers are often concentrated in non-capital cities, reflecting different regional economic development patterns.\n-   Scoring Basis: This question has Scoring Criteria, which require the Student Answer to list all the provinces from the Reference Answer and no other provinces. Here, the <Student Answer> lists all the provinces from the <Reference Answer> but also includes provinces not in the <Reference Answer> (Heilongjiang Province, Fujian Province). Therefore, the <Student Answer> is incorrect, and the score is 0.\n-   JSON:\n    ```\n    {\"answer_score\": 0}\n    ```\n\n###\n**Example 3**\n<Question>:\nHow many Michelin three-star restaurants in San Francisco specialize in Asian cuisine, and what are their names?\n<Reference Answer>:\n1, Benu\n<Student Answer>:\nAmong San Francisco's Michelin three-star restaurants, Benu is the one that specializes in Asian-inspired cuisine. Helmed by renowned chef Corey Lee, Benu offers modern Asian cuisine, fully showcasing Lee's creativity and skill. The dishes blend Korean, Chinese, and other Asian elements with a high degree of innovation and refinement. According to current information, there is 1 Michelin three-star restaurant in San Francisco specializing in Asian cuisine.\n-   Scoring Basis: This question does not have Scoring Criteria. Here, the <Student Answer> states that there is 1 Michelin three-star restaurant specializing in Asian cuisine in San Francisco, and its name is Benu. Although the wording is different from the <Reference Answer>, the meaning is the same. Therefore, the <Student Answer> is correct, and the score is 1.\n-   JSON:\n    ```\n    {\"answer_score\": 1}\n    ```\n\n###\n**Example 4**\n<Question>:\nIn terms of area, which provincial capital city in China (mainland provinces) is closest in size to Slovenia? What is the difference in square kilometers?\n<Reference Answer>:\nIt is closest to Kunming, with a difference of about 740 square kilometers.\n<Student Answer>:\nThe total area of Slovenia is 20,273 square kilometers. Among China's provincial capitals, the one closest in area is Kunming, with an area of 22,100 square kilometers. The difference between the two is: 22,100 - 20,273 = 1827 square kilometers.\n-   Scoring Basis: This question does not have Scoring Criteria. Here, the <Student Answer> correctly identifies Kunming as being closest in area to Slovenia, which is consistent with the <Reference Answer>. However, the area difference of \"1827 square kilometers\" given in the <Student Answer> is inconsistent with \"about 740 square kilometers\" in the <Reference Answer>. Therefore, the <Student Answer> is incorrect, and the score is 0.\n-   JSON:\n    ```\n    {\"answer_score\": 0}\n    ```\n\n###\nI hope you are competent for this job, as it is very important to me. If you perform well, I will reward you accordingly. Otherwise, I may penalize you.",
+    "label": "Complex_Historical_Investigation(Global)"
+  }
+]

package/fixtures/frames.json

@@ -0,0 +1,26 @@
+[
+  {
+    "Prompt": "If my future wife has the same first name as the 15th first lady of the United States' mother and her surname is the same as the second assassinated president's mother's maiden name, what is my future wife's name?",
+    "Answer": "Jane Ballou",
+    "wiki_links": "['https://en.wikipedia.org/wiki/Harriet_Lane', 'https://en.wikipedia.org/wiki/James_A._Garfield', 'https://en.wikipedia.org/wiki/Eliza_Ballou']",
+    "reasoning_types": "multiple constraints"
+  },
+  {
+    "Prompt": "How many years after the publication of the first novel by the author who wrote 'The Old Man and the Sea' did that author win the Nobel Prize in Literature?",
+    "Answer": "28",
+    "wiki_links": "['https://en.wikipedia.org/wiki/Ernest_Hemingway', 'https://en.wikipedia.org/wiki/The_Sun_Also_Rises']",
+    "reasoning_types": "numerical | temporal"
+  },
+  {
+    "Prompt": "What is the capital city of the country whose flag was the inspiration for the flag of the nation that hosted the 2016 Summer Olympics?",
+    "Answer": "Lisbon",
+    "wiki_links": "['https://en.wikipedia.org/wiki/Flag_of_Brazil', 'https://en.wikipedia.org/wiki/Flag_of_Portugal', 'https://en.wikipedia.org/wiki/Lisbon']",
+    "reasoning_types": "multiple constraints"
+  },
+  {
+    "Prompt": "In what year was the company that owns the search engine Bing founded?",
+    "Answer": "1975",
+    "wiki_links": "['https://en.wikipedia.org/wiki/Bing_(search_engine)', 'https://en.wikipedia.org/wiki/Microsoft']",
+    "reasoning_types": "temporal"
+  }
+]

package/fixtures/hotpotqa.json

@@ -0,0 +1,182 @@
+[
+    {
+        "id": "5a8b57f25542995d1e6f1371",
+        "question": "Were Scott Derrickson and Ed Wood of the same nationality?",
+        "answer": "yes",
+        "type": "comparison",
+        "level": "hard",
+        "supporting_facts": {
+            "title": [
+                "Scott Derrickson",
+                "Ed Wood"
+            ],
+            "sent_id": [
+                0,
+                0
+            ]
+        }
+    },
+    {
+        "id": "5a8c7595554299585d9e36b6",
+        "question": "What government position was held by the woman who portrayed Corliss Archer in the film Kiss and Tell?",
+        "answer": "Chief of Protocol",
+        "type": "bridge",
+        "level": "hard",
+        "supporting_facts": {
+            "title": [
+                "Kiss and Tell (1945 film)",
+                "Shirley Temple",
+                "Shirley Temple"
+            ],
+            "sent_id": [
+                0,
+                0,
+                1
+            ]
+        }
+    },
+    {
+        "id": "5a85ea095542994775f606a8",
+        "question": "What science fantasy young adult series, told in first person, has a set of companion books narrating the stories of enslaved worlds and alien species?",
+        "answer": "Animorphs",
+        "type": "bridge",
+        "level": "hard",
+        "supporting_facts": {
+            "title": [
+                "The Hork-Bajir Chronicles",
+                "The Hork-Bajir Chronicles",
+                "The Hork-Bajir Chronicles",
+                "Animorphs",
+                "Animorphs"
+            ],
+            "sent_id": [
+                0,
+                1,
+                2,
+                0,
+                1
+            ]
+        }
+    },
+    {
+        "id": "5adbf0a255429947ff17385a",
+        "question": "Are the Laleli Mosque and Esma Sultan Mansion located in the same neighborhood?",
+        "answer": "no",
+        "type": "comparison",
+        "level": "hard",
+        "supporting_facts": {
+            "title": [
+                "Laleli Mosque",
+                "Esma Sultan Mansion"
+            ],
+            "sent_id": [
+                0,
+                0
+            ]
+        }
+    },
+    {
+        "id": "5a8e3ea95542995a26add48d",
+        "question": "The director of the romantic comedy \"Big Stone Gap\" is based in what New York city?",
+        "answer": "Greenwich Village, New York City",
+        "type": "bridge",
+        "level": "hard",
+        "supporting_facts": {
+            "title": [
+                "Big Stone Gap (film)",
+                "Adriana Trigiani"
+            ],
+            "sent_id": [
+                0,
+                0
+            ]
+        }
+    },
+    {
+        "id": "5abd94525542992ac4f382d2",
+        "question": "2014 S/S is the debut album of a South Korean boy group that was formed by who?",
+        "answer": "YG Entertainment",
+        "type": "bridge",
+        "level": "hard",
+        "supporting_facts": {
+            "title": [
+                "2014 S/S",
+                "Winner (band)"
+            ],
+            "sent_id": [
+                0,
+                0
+            ]
+        }
+    },
+    {
+        "id": "5a85b2d95542997b5ce40028",
+        "question": "Who was known by his stage name Aladin and helped organizations improve their performance as a consultant?",
+        "answer": "Eenasul Fateh",
+        "type": "bridge",
+        "level": "hard",
+        "supporting_facts": {
+            "title": [
+                "Eenasul Fateh",
+                "Management consulting"
+            ],
+            "sent_id": [
+                0,
+                0
+            ]
+        }
+    },
+    {
+        "id": "5a87ab905542996e4f3088c1",
+        "question": "The arena where the Lewiston Maineiacs played their home games can seat how many people?",
+        "answer": "3,677 seated",
+        "type": "bridge",
+        "level": "hard",
+        "supporting_facts": {
+            "title": [
+                "Lewiston Maineiacs",
+                "Androscoggin Bank Colis\u00e9e"
+            ],
+            "sent_id": [
+                1,
+                0
+            ]
+        }
+    },
+    {
+        "id": "5a7bbb64554299042af8f7cc",
+        "question": "Who is older, Annie Morton or Terry Richardson?",
+        "answer": "Terry Richardson",
+        "type": "bridge",
+        "level": "hard",
+        "supporting_facts": {
+            "title": [
+                "Annie Morton",
+                "Annie Morton",
+                "Terry Richardson"
+            ],
+            "sent_id": [
+                0,
+                2,
+                0
+            ]
+        }
+    },
+    {
+        "id": "5a8db19d5542994ba4e3dd00",
+        "question": "Are Local H and For Against both from the United States?",
+        "answer": "yes",
+        "type": "comparison",
+        "level": "hard",
+        "supporting_facts": {
+            "title": [
+                "Local H",
+                "For Against"
+            ],
+            "sent_id": [
+                0,
+                0
+            ]
+        }
+    }
+]

package/fixtures/nomiracl.json

@@ -0,0 +1,26 @@
+[
+  {
+    "id": "nomiracl-fixture-relevant",
+    "language": "en",
+    "query": "What is the refund window?",
+    "positive_passages": [
+      {
+        "docid": "p1",
+        "title": "Refund policy",
+        "text": "The refund window is 30 days."
+      }
+    ]
+  },
+  {
+    "id": "nomiracl-fixture-non-relevant",
+    "language": "en",
+    "query": "What is the refund window?",
+    "negative_passages": [
+      {
+        "docid": "p2",
+        "title": "Shipping policy",
+        "text": "Shipping labels expire after 7 days."
+      }
+    ]
+  }
+]

package/fixtures/open-rag-bench.json

@@ -0,0 +1,16 @@
+[
+  {
+    "id": "open-rag-bench-fixture-0",
+    "file_name": "arxiv-0001.pdf",
+    "modality": "table",
+    "question": "Which method has the highest accuracy in the table?",
+    "answer": "Hybrid BM25",
+    "contexts": [
+      {
+        "id": "table-1",
+        "title": "Accuracy table",
+        "text": "Method | Accuracy\nDense | 82.1\nHybrid BM25 | 87.4\nSparse | 79.0"
+      }
+    ]
+  }
+]

package/fixtures/pier-agent/no-model-task/environment/Dockerfile

@@ -0,0 +1,16 @@
+FROM ubuntu:24.04
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends git ca-certificates \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+COPY seed/ /app/
+RUN git init \
+    && git config user.email fixture@tangle.tools \
+    && git config user.name "Tangle Fixture" \
+    && git add -A \
+    && GIT_AUTHOR_DATE=2000-01-01T00:00:00Z \
+       GIT_COMMITTER_DATE=2000-01-01T00:00:00Z \
+       git commit -m baseline \
+    && git tag benchmark-base

package/fixtures/pier-agent/no-model-task/environment/seed/src/status.txt

@@ -0,0 +1 @@
+not-ready