attention-manifest-protocol 0.4.2 → 0.4.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (827) hide show
  1. package/CHANGELOG.md +341 -293
  2. package/CHANGELOG.zh-CN.md +41 -8
  3. package/README.md +250 -171
  4. package/README.zh-CN.md +251 -171
  5. package/lib/assignment/assign.d.ts +102 -0
  6. package/lib/assignment/assign.js +556 -0
  7. package/lib/assignment/binding.d.ts +103 -0
  8. package/lib/assignment/binding.js +575 -0
  9. package/lib/assignment/errors.d.ts +14 -0
  10. package/lib/assignment/errors.js +16 -0
  11. package/lib/assignment/guards.d.ts +18 -0
  12. package/lib/assignment/guards.js +28 -0
  13. package/lib/assignment/index.d.ts +10 -0
  14. package/lib/assignment/index.js +10 -0
  15. package/lib/assignment/types.d.ts +169 -0
  16. package/lib/assignment/types.js +1 -0
  17. package/lib/attention/budget.d.ts +1 -1
  18. package/lib/attention/budget.js +31 -9
  19. package/lib/bayes/engine.js +2 -1
  20. package/lib/bayes/reliability.js +25 -10
  21. package/lib/boundary/frame.d.ts +10 -4
  22. package/lib/boundary/frame.js +346 -109
  23. package/lib/boundary/kinds.d.ts +11 -0
  24. package/lib/boundary/kinds.js +33 -0
  25. package/lib/boundary/refTable.d.ts +19 -9
  26. package/lib/boundary/refTable.js +128 -30
  27. package/lib/boundary/toolCall.d.ts +22 -10
  28. package/lib/boundary/toolCall.js +798 -97
  29. package/lib/concept/conceptBudget.js +10 -1
  30. package/lib/concurrency/idempotency.d.ts +4 -3
  31. package/lib/concurrency/idempotency.js +57 -13
  32. package/lib/concurrency/retry.js +18 -3
  33. package/lib/context/agentContext.d.ts +13 -0
  34. package/lib/context/agentContext.js +14 -1
  35. package/lib/context/ast.d.ts +4 -1
  36. package/lib/context/ast.js +13 -3
  37. package/lib/context/contentStore.js +22 -9
  38. package/lib/context/contextBuilder.d.ts +457 -0
  39. package/lib/context/contextBuilder.js +1395 -0
  40. package/lib/context/contextPackage.d.ts +214 -0
  41. package/lib/context/contextPackage.js +191 -0
  42. package/lib/context/index.d.ts +14 -3
  43. package/lib/context/index.js +9 -1
  44. package/lib/context/relevance.d.ts +171 -0
  45. package/lib/context/relevance.js +250 -0
  46. package/lib/context/workspaceDiff.d.ts +105 -0
  47. package/lib/context/workspaceDiff.js +238 -0
  48. package/lib/context/workspaceService.d.ts +236 -1
  49. package/lib/context/workspaceService.js +1501 -73
  50. package/lib/cooperation/collaboration.d.ts +9 -0
  51. package/lib/cooperation/session.d.ts +4 -4
  52. package/lib/cooperation/session.js +3 -3
  53. package/lib/database/adapter/filesystemStore.d.ts +145 -0
  54. package/lib/database/adapter/filesystemStore.js +321 -0
  55. package/lib/database/adapter/index.d.ts +2 -0
  56. package/lib/database/adapter/index.js +2 -0
  57. package/lib/database/adapter/workspaceSessionStore.d.ts +118 -0
  58. package/lib/database/adapter/workspaceSessionStore.js +120 -0
  59. package/lib/database/entity/fileGraph.d.ts +53 -0
  60. package/lib/database/entity/fileGraph.js +15 -0
  61. package/lib/database/entity/fileTree.d.ts +76 -0
  62. package/lib/database/entity/fileTree.js +39 -0
  63. package/lib/database/entity/hook.d.ts +29 -0
  64. package/lib/database/entity/hook.js +18 -0
  65. package/lib/database/entity/index.d.ts +4 -0
  66. package/lib/database/entity/index.js +4 -0
  67. package/lib/database/entity/workspaceSession.d.ts +53 -0
  68. package/lib/database/entity/workspaceSession.js +28 -0
  69. package/lib/database/repository/fileAnnotationRepository.d.ts +75 -0
  70. package/lib/database/repository/fileAnnotationRepository.js +109 -0
  71. package/lib/database/repository/fileGraphRepository.d.ts +188 -0
  72. package/lib/database/repository/fileGraphRepository.js +648 -0
  73. package/lib/database/repository/fileNodeRepository.d.ts +116 -0
  74. package/lib/database/repository/fileNodeRepository.js +263 -0
  75. package/lib/database/repository/fileOutlineRepository.d.ts +40 -0
  76. package/lib/database/repository/fileOutlineRepository.js +104 -0
  77. package/lib/database/repository/fileWriteRepository.d.ts +41 -0
  78. package/lib/database/repository/fileWriteRepository.js +93 -0
  79. package/lib/database/repository/hookRepository.d.ts +67 -0
  80. package/lib/database/repository/hookRepository.js +150 -0
  81. package/lib/database/repository/index.d.ts +7 -0
  82. package/lib/database/repository/index.js +7 -0
  83. package/lib/database/repository/workspaceSessionRepository.d.ts +134 -0
  84. package/lib/database/repository/workspaceSessionRepository.js +327 -0
  85. package/lib/database/service/database.d.ts +14 -0
  86. package/lib/database/service/database.js +21 -0
  87. package/lib/database/sql/schema.js +187 -0
  88. package/lib/database/store/database.js +6 -1
  89. package/lib/delegate/team.d.ts +5 -0
  90. package/lib/delegate/team.js +83 -20
  91. package/lib/discover/classify.d.ts +20 -0
  92. package/lib/discover/classify.js +85 -0
  93. package/lib/discover/host.d.ts +20 -0
  94. package/lib/discover/host.js +89 -0
  95. package/lib/discover/index.d.ts +11 -0
  96. package/lib/discover/index.js +11 -0
  97. package/lib/discover/kinds.d.ts +14 -0
  98. package/lib/discover/kinds.js +23 -0
  99. package/lib/discover/tree.d.ts +7 -0
  100. package/lib/discover/tree.js +432 -0
  101. package/lib/discover/types.d.ts +115 -0
  102. package/lib/discover/types.js +1 -0
  103. package/lib/error/boundaryRejection.d.ts +1 -1
  104. package/lib/evaluation/agreement.js +2 -2
  105. package/lib/execution/handles.js +6 -0
  106. package/lib/execution/pin.js +2 -2
  107. package/lib/file/cfb.d.ts +56 -0
  108. package/lib/file/cfb.js +396 -0
  109. package/lib/file/dependency.d.ts +151 -0
  110. package/lib/file/dependency.js +568 -0
  111. package/lib/file/directory.d.ts +34 -0
  112. package/lib/file/directory.js +141 -0
  113. package/lib/file/errors.d.ts +20 -0
  114. package/lib/file/errors.js +18 -0
  115. package/lib/file/exportGraph.d.ts +31 -0
  116. package/lib/file/exportGraph.js +554 -0
  117. package/lib/file/file.d.ts +40 -0
  118. package/lib/file/file.js +339 -0
  119. package/lib/file/importGraph.d.ts +121 -0
  120. package/lib/file/importGraph.js +1022 -0
  121. package/lib/file/index.d.ts +45 -0
  122. package/lib/file/index.js +30 -0
  123. package/lib/file/metadata.d.ts +66 -0
  124. package/lib/file/metadata.js +470 -0
  125. package/lib/file/path.d.ts +60 -0
  126. package/lib/file/path.js +259 -0
  127. package/lib/file/read.d.ts +168 -0
  128. package/lib/file/read.js +541 -0
  129. package/lib/file/runtime.d.ts +95 -0
  130. package/lib/file/runtime.js +150 -0
  131. package/lib/file/service.d.ts +227 -0
  132. package/lib/file/service.js +663 -0
  133. package/lib/file/syntaxTree.d.ts +7 -0
  134. package/lib/file/syntaxTree.js +38 -0
  135. package/lib/file/time.d.ts +38 -0
  136. package/lib/file/time.js +44 -0
  137. package/lib/file/tree.d.ts +53 -0
  138. package/lib/file/tree.js +417 -0
  139. package/lib/file/types.d.ts +371 -0
  140. package/lib/file/types.js +1 -0
  141. package/lib/file/write.d.ts +159 -0
  142. package/lib/file/write.js +1028 -0
  143. package/lib/file/zip.d.ts +35 -0
  144. package/lib/file/zip.js +349 -0
  145. package/lib/fingerprint/algorithm.d.ts +97 -0
  146. package/lib/fingerprint/algorithm.js +271 -0
  147. package/lib/fingerprint/bytes.d.ts +18 -0
  148. package/lib/fingerprint/bytes.js +31 -0
  149. package/lib/fingerprint/error.d.ts +73 -0
  150. package/lib/fingerprint/error.js +86 -0
  151. package/lib/fingerprint/file.d.ts +71 -0
  152. package/lib/fingerprint/file.js +254 -0
  153. package/lib/fingerprint/index.d.ts +37 -0
  154. package/lib/fingerprint/index.js +12 -0
  155. package/lib/fingerprint/json.d.ts +41 -0
  156. package/lib/fingerprint/json.js +94 -0
  157. package/lib/fingerprint/number.d.ts +32 -0
  158. package/lib/fingerprint/number.js +95 -0
  159. package/lib/fingerprint/policy.d.ts +54 -0
  160. package/lib/fingerprint/policy.js +50 -0
  161. package/lib/fingerprint/preimage.d.ts +84 -0
  162. package/lib/fingerprint/preimage.js +133 -0
  163. package/lib/fingerprint/raw.d.ts +26 -0
  164. package/lib/fingerprint/raw.js +45 -0
  165. package/lib/fingerprint/ref.d.ts +39 -0
  166. package/lib/fingerprint/ref.js +178 -0
  167. package/lib/fingerprint/text.d.ts +42 -0
  168. package/lib/fingerprint/text.js +155 -0
  169. package/lib/fingerprint/tree.d.ts +56 -0
  170. package/lib/fingerprint/tree.js +288 -0
  171. package/lib/fingerprint/verify.d.ts +20 -0
  172. package/lib/fingerprint/verify.js +99 -0
  173. package/lib/global/filesystem.d.ts +43 -0
  174. package/lib/global/filesystem.js +75 -0
  175. package/lib/global/index.d.ts +8 -0
  176. package/lib/global/index.js +8 -0
  177. package/lib/global/service.d.ts +51 -0
  178. package/lib/global/service.js +98 -0
  179. package/lib/governance/transport.js +18 -6
  180. package/lib/hook/ast.d.ts +29 -0
  181. package/lib/hook/ast.js +22 -0
  182. package/lib/hook/code.d.ts +55 -0
  183. package/lib/hook/code.js +47 -0
  184. package/lib/hook/dependency.d.ts +45 -0
  185. package/lib/hook/dependency.js +28 -0
  186. package/lib/hook/diff.d.ts +114 -0
  187. package/lib/hook/diff.js +327 -0
  188. package/lib/hook/exportation.d.ts +31 -0
  189. package/lib/hook/exportation.js +24 -0
  190. package/lib/hook/file.d.ts +44 -0
  191. package/lib/hook/file.js +154 -0
  192. package/lib/hook/importation.d.ts +33 -0
  193. package/lib/hook/importation.js +22 -0
  194. package/lib/hook/index.d.ts +30 -0
  195. package/lib/hook/index.js +23 -0
  196. package/lib/hook/outline.d.ts +35 -0
  197. package/lib/hook/outline.js +27 -0
  198. package/lib/hook/project.d.ts +65 -0
  199. package/lib/hook/project.js +217 -0
  200. package/lib/hook/types.d.ts +114 -0
  201. package/lib/hook/types.js +407 -0
  202. package/lib/index.d.ts +9 -0
  203. package/lib/index.js +9 -0
  204. package/lib/lock/lease.js +28 -3
  205. package/lib/manifest/cardSign.d.ts +1 -1
  206. package/lib/manifest/cardSign.js +37 -7
  207. package/lib/manifest/federation.js +13 -6
  208. package/lib/manifest/trust.d.ts +2 -0
  209. package/lib/manifest/trust.js +27 -6
  210. package/lib/module/index.d.ts +4 -0
  211. package/lib/module/index.js +4 -0
  212. package/lib/module/mcp/client.js +21 -2
  213. package/lib/module/mcp/filesystemServer.d.ts +24 -0
  214. package/lib/module/mcp/filesystemServer.js +102 -0
  215. package/lib/module/mcp/http.js +32 -19
  216. package/lib/module/mcp/workspaceServer.js +20 -3
  217. package/lib/module/tool/filesystemExecutor.d.ts +18 -0
  218. package/lib/module/tool/filesystemExecutor.js +49 -0
  219. package/lib/module/tool/globalExecutor.d.ts +17 -0
  220. package/lib/module/tool/globalExecutor.js +39 -0
  221. package/lib/module/tool/globalFilesystemExecutor.d.ts +18 -0
  222. package/lib/module/tool/globalFilesystemExecutor.js +49 -0
  223. package/lib/module/tool/workspaceExecutor.d.ts +9 -0
  224. package/lib/module/tool/workspaceExecutor.js +16 -3
  225. package/lib/naming/aliases.d.ts +48 -0
  226. package/lib/naming/aliases.js +79 -0
  227. package/lib/naming/catalog.d.ts +117 -0
  228. package/lib/naming/catalog.js +108 -0
  229. package/lib/naming/grammar.d.ts +67 -0
  230. package/lib/naming/grammar.js +212 -0
  231. package/lib/naming/ids.d.ts +157 -0
  232. package/lib/naming/ids.js +119 -0
  233. package/lib/naming/index.d.ts +13 -0
  234. package/lib/naming/index.js +13 -0
  235. package/lib/outline/index.d.ts +9 -0
  236. package/lib/outline/index.js +8 -0
  237. package/lib/outline/office.d.ts +160 -0
  238. package/lib/outline/office.js +2375 -0
  239. package/lib/outline/officeBridge.d.ts +32 -0
  240. package/lib/outline/officeBridge.js +27 -0
  241. package/lib/outline/outline.d.ts +159 -0
  242. package/lib/outline/outline.js +837 -0
  243. package/lib/outline/outlineIndex.d.ts +97 -0
  244. package/lib/outline/outlineIndex.js +325 -0
  245. package/lib/outline/outlineKeywords.d.ts +59 -0
  246. package/lib/outline/outlineKeywords.js +143 -0
  247. package/lib/outline/outlineStrategies.d.ts +414 -0
  248. package/lib/outline/outlineStrategies.js +8324 -0
  249. package/lib/outline/reactOutline.d.ts +37 -0
  250. package/lib/outline/reactOutline.js +1070 -0
  251. package/lib/outline/shellOutline.d.ts +65 -0
  252. package/lib/outline/shellOutline.js +873 -0
  253. package/lib/outline/sqlOutline.d.ts +170 -0
  254. package/lib/outline/sqlOutline.js +1536 -0
  255. package/lib/outline/strategy.d.ts +46 -0
  256. package/lib/outline/strategy.js +1 -0
  257. package/lib/outline/types.d.ts +95 -0
  258. package/lib/outline/types.js +1 -0
  259. package/lib/outline/visualBasicOutline.d.ts +27 -0
  260. package/lib/outline/visualBasicOutline.js +750 -0
  261. package/lib/protocol/filesystem.d.ts +55 -0
  262. package/lib/protocol/filesystem.js +105 -0
  263. package/lib/protocol/global.d.ts +42 -0
  264. package/lib/protocol/global.js +63 -0
  265. package/lib/protocol/globalFilesystem.d.ts +40 -0
  266. package/lib/protocol/globalFilesystem.js +59 -0
  267. package/lib/protocol/index.d.ts +4 -0
  268. package/lib/protocol/index.js +4 -0
  269. package/lib/protocol/protocol.d.ts +4 -0
  270. package/lib/protocol/wire.js +5 -2
  271. package/lib/protocol/workspace.d.ts +305 -0
  272. package/lib/protocol/workspace.js +258 -0
  273. package/lib/receipt/receipt.d.ts +8 -0
  274. package/lib/registry/content.js +9 -3
  275. package/lib/runtime/kernelRuntime.js +9 -1
  276. package/lib/scenario/scenario.js +5 -4
  277. package/lib/semantic/canonical.d.ts +61 -0
  278. package/lib/semantic/canonical.js +135 -8
  279. package/lib/semantic/jsonValue.d.ts +31 -0
  280. package/lib/semantic/jsonValue.js +122 -0
  281. package/lib/semantic/sha256.js +3 -0
  282. package/lib/semantic/time.d.ts +14 -0
  283. package/lib/semantic/time.js +56 -0
  284. package/lib/session/context.d.ts +76 -0
  285. package/lib/session/context.js +419 -0
  286. package/lib/session/errors.d.ts +24 -0
  287. package/lib/session/errors.js +22 -0
  288. package/lib/session/identity.d.ts +24 -0
  289. package/lib/session/identity.js +56 -0
  290. package/lib/session/index.d.ts +14 -0
  291. package/lib/session/index.js +12 -0
  292. package/lib/session/registry.d.ts +130 -0
  293. package/lib/session/registry.js +908 -0
  294. package/lib/session/types.d.ts +227 -0
  295. package/lib/session/types.js +14 -0
  296. package/lib/structure/index.d.ts +1 -0
  297. package/lib/structure/index.js +1 -0
  298. package/lib/structure/trie.d.ts +88 -0
  299. package/lib/structure/trie.js +183 -0
  300. package/lib/syntax/byteIndex.d.ts +44 -0
  301. package/lib/syntax/byteIndex.js +148 -0
  302. package/lib/syntax/charset.d.ts +88 -0
  303. package/lib/syntax/charset.js +276 -0
  304. package/lib/syntax/index.d.ts +6 -0
  305. package/lib/syntax/index.js +5 -0
  306. package/lib/syntax/syntaxRegistry.d.ts +150 -0
  307. package/lib/syntax/syntaxRegistry.js +314 -0
  308. package/lib/syntax/syntaxTree.d.ts +106 -0
  309. package/lib/syntax/syntaxTree.js +668 -0
  310. package/lib/syntax/types.d.ts +15 -0
  311. package/lib/syntax/types.js +1 -0
  312. package/lib/types/common.d.ts +1 -1
  313. package/package.json +72 -193
  314. package/.editorconfig +0 -12
  315. package/.prettierignore +0 -17
  316. package/MCP-Token/347/250/216/344/270/216/344/270/213/344/270/200/344/273/243/345/267/245/345/205/267/345/215/217/350/256/256.html +0 -1763
  317. package/docs/annotation-campaign-A.csv +0 -54
  318. package/docs/annotation-campaign-B.csv +0 -54
  319. package/docs/annotation-campaign.csv +0 -54
  320. package/docs/eval/annotations-A.jsonl +0 -16
  321. package/docs/eval/annotations.json +0 -122
  322. package/docs/eval/recorded-labels.json +0 -12
  323. package/docs/eval/replay-cases.json +0 -14
  324. package/docs/eval/replay-labels-b.json +0 -14
  325. package/docs/examples/attention-request.json +0 -16
  326. package/docs/examples/checkpoint.json +0 -15
  327. package/docs/examples/claim.json +0 -66
  328. package/docs/examples/collaboration.json +0 -66
  329. package/docs/examples/concept-checkpoint.json +0 -15
  330. package/docs/examples/concept-prune-plan.json +0 -45
  331. package/docs/examples/federation.json +0 -126
  332. package/docs/examples/frame-chain.json +0 -77
  333. package/docs/examples/ledger-snapshot.json +0 -108
  334. package/docs/examples/manifest.json +0 -77
  335. package/docs/examples/message.json +0 -19
  336. package/docs/examples/other-model.json +0 -45
  337. package/docs/examples/plan.json +0 -50
  338. package/docs/examples/prune-plan.json +0 -48
  339. package/docs/examples/receipt.json +0 -22
  340. package/docs/examples/runtime-run.json +0 -573
  341. package/docs/examples/team.json +0 -66
  342. package/docs/examples/verb-note.json +0 -11
  343. package/docs/probes/bayes-scores.json +0 -1452
  344. package/docs/probes/calibration-imported.json +0 -141
  345. package/docs/probes/context-ast-deepseek.json +0 -1577
  346. package/docs/probes/deepseek-probe.json +0 -917
  347. package/docs/probes/eval-recorded-replay.json +0 -597
  348. package/docs/probes/eval-replay.json +0 -124
  349. package/docs/probes/eval-summary.json +0 -42
  350. package/docs/probes/runtime-eval.json +0 -4230
  351. package/docs/schemas/collaboration.schema.json +0 -73
  352. package/docs/schemas/common.schema.json +0 -125
  353. package/docs/schemas/compaction.schema.json +0 -121
  354. package/docs/schemas/concept-compaction.schema.json +0 -73
  355. package/docs/schemas/concept.schema.json +0 -97
  356. package/docs/schemas/epistemic.schema.json +0 -178
  357. package/docs/schemas/federation.schema.json +0 -66
  358. package/docs/schemas/frame.schema.json +0 -86
  359. package/docs/schemas/ledger-snapshot.schema.json +0 -30
  360. package/docs/schemas/manifest.schema.json +0 -169
  361. package/docs/schemas/message.schema.json +0 -41
  362. package/docs/schemas/plan.schema.json +0 -189
  363. package/docs/schemas/receipt.schema.json +0 -45
  364. package/docs/schemas/social.schema.json +0 -153
  365. package/docs/schemas/team.schema.json +0 -56
  366. package/docs/schemas/verb.schema.json +0 -120
  367. package/eslint.config.mts +0 -96
  368. package/index.html +0 -13
  369. package/lib/context/workspaceProtocol.d.ts +0 -30
  370. package/lib/context/workspaceProtocol.js +0 -49
  371. package/prettier.config.mts +0 -37
  372. package/public/favicon.svg +0 -1
  373. package/public/icons.svg +0 -24
  374. package/src/attention/attention.ts +0 -347
  375. package/src/attention/budget.ts +0 -72
  376. package/src/attention/cognition.ts +0 -519
  377. package/src/attention/index.ts +0 -8
  378. package/src/attention/interrupts.ts +0 -228
  379. package/src/attention/resource.ts +0 -316
  380. package/src/attention/social.ts +0 -217
  381. package/src/attention/socialTypes.ts +0 -115
  382. package/src/authz/authz.ts +0 -305
  383. package/src/authz/index.ts +0 -1
  384. package/src/bayes/calibration.ts +0 -204
  385. package/src/bayes/calibrationRegistry.ts +0 -111
  386. package/src/bayes/calibrationSelection.ts +0 -154
  387. package/src/bayes/engine.ts +0 -532
  388. package/src/bayes/index.ts +0 -6
  389. package/src/bayes/reliability.ts +0 -62
  390. package/src/bayes/scopedPriors.ts +0 -88
  391. package/src/boundary/frame.ts +0 -322
  392. package/src/boundary/ids.ts +0 -14
  393. package/src/boundary/index.ts +0 -12
  394. package/src/boundary/refTable.ts +0 -125
  395. package/src/boundary/toolCall.ts +0 -212
  396. package/src/concept/concept.ts +0 -75
  397. package/src/concept/conceptBudget.ts +0 -346
  398. package/src/concept/conceptBudgetTypes.ts +0 -79
  399. package/src/concept/index.ts +0 -3
  400. package/src/concurrency/idempotency.ts +0 -41
  401. package/src/concurrency/index.ts +0 -5
  402. package/src/concurrency/mutex.ts +0 -26
  403. package/src/concurrency/namedLocks.ts +0 -35
  404. package/src/concurrency/retry.ts +0 -50
  405. package/src/concurrency/worker.ts +0 -128
  406. package/src/context/agentContext.ts +0 -308
  407. package/src/context/ast.ts +0 -499
  408. package/src/context/contentStore.ts +0 -75
  409. package/src/context/context.ts +0 -65
  410. package/src/context/contextIndex.ts +0 -85
  411. package/src/context/document.ts +0 -137
  412. package/src/context/fileStorePath.ts +0 -14
  413. package/src/context/index.ts +0 -43
  414. package/src/context/lifecycle.ts +0 -221
  415. package/src/context/projections.ts +0 -151
  416. package/src/context/workspace.ts +0 -422
  417. package/src/context/workspaceProtocol.ts +0 -112
  418. package/src/context/workspaceService.ts +0 -423
  419. package/src/cooperation/board.ts +0 -104
  420. package/src/cooperation/collaboration.ts +0 -60
  421. package/src/cooperation/index.ts +0 -6
  422. package/src/cooperation/merge.ts +0 -190
  423. package/src/cooperation/metrics.ts +0 -57
  424. package/src/cooperation/session.ts +0 -131
  425. package/src/cooperation/unified.ts +0 -56
  426. package/src/counter.ts +0 -9
  427. package/src/database/adapter/adapters.ts +0 -35
  428. package/src/database/adapter/index.ts +0 -4
  429. package/src/database/adapter/runtimeSink.ts +0 -74
  430. package/src/database/adapter/sessionStore.ts +0 -32
  431. package/src/database/adapter/turnStore.ts +0 -23
  432. package/src/database/entity/agent.ts +0 -199
  433. package/src/database/entity/entity.ts +0 -1046
  434. package/src/database/entity/handle.ts +0 -79
  435. package/src/database/entity/index.ts +0 -32
  436. package/src/database/entity/message.ts +0 -74
  437. package/src/database/entity/meta.ts +0 -11
  438. package/src/database/entity/object.ts +0 -42
  439. package/src/database/entity/outbox.ts +0 -15
  440. package/src/database/entity/process.ts +0 -65
  441. package/src/database/entity/registry.ts +0 -155
  442. package/src/database/entity/runtimeApproval.ts +0 -30
  443. package/src/database/entity/runtimeSession.ts +0 -13
  444. package/src/database/entity/runtimeTurn.ts +0 -15
  445. package/src/database/entity/thread.ts +0 -85
  446. package/src/database/entity/window.ts +0 -85
  447. package/src/database/entity/worker.ts +0 -90
  448. package/src/database/index.ts +0 -6
  449. package/src/database/repository/addressSpaceRepository.ts +0 -72
  450. package/src/database/repository/agentRepository.ts +0 -460
  451. package/src/database/repository/base.ts +0 -41
  452. package/src/database/repository/executionRepository.ts +0 -477
  453. package/src/database/repository/handleRepository.ts +0 -64
  454. package/src/database/repository/index.ts +0 -20
  455. package/src/database/repository/messageRepository.ts +0 -52
  456. package/src/database/repository/objectRepository.ts +0 -69
  457. package/src/database/repository/outboxRepository.ts +0 -63
  458. package/src/database/repository/processRepository.ts +0 -58
  459. package/src/database/repository/registryRepository.ts +0 -119
  460. package/src/database/repository/repository.ts +0 -491
  461. package/src/database/repository/runtimeApprovalRepository.ts +0 -44
  462. package/src/database/repository/runtimeSessionRepository.ts +0 -27
  463. package/src/database/repository/runtimeTurnRepository.ts +0 -39
  464. package/src/database/repository/socialRepository.ts +0 -185
  465. package/src/database/repository/threadRepository.ts +0 -57
  466. package/src/database/repository/windowRepository.ts +0 -88
  467. package/src/database/repository/workerRepository.ts +0 -69
  468. package/src/database/service/database.ts +0 -90
  469. package/src/database/service/handleService.ts +0 -48
  470. package/src/database/service/index.ts +0 -16
  471. package/src/database/service/messageService.ts +0 -61
  472. package/src/database/service/objectService.ts +0 -59
  473. package/src/database/service/processService.ts +0 -61
  474. package/src/database/service/registryServices.ts +0 -88
  475. package/src/database/service/service.ts +0 -183
  476. package/src/database/service/stepService.ts +0 -53
  477. package/src/database/service/threadService.ts +0 -65
  478. package/src/database/service/windowService.ts +0 -30
  479. package/src/database/service/workerService.ts +0 -57
  480. package/src/database/sql/index.ts +0 -3
  481. package/src/database/sql/schema.ts +0 -418
  482. package/src/database/sql/sql.ts +0 -543
  483. package/src/database/store/database.ts +0 -210
  484. package/src/database/store/index.ts +0 -3
  485. package/src/database/store/store.ts +0 -266
  486. package/src/delegate/index.ts +0 -2
  487. package/src/delegate/team.ts +0 -268
  488. package/src/delegate/teamTypes.ts +0 -47
  489. package/src/diagnosis/checklist.ts +0 -74
  490. package/src/diagnosis/context.ts +0 -93
  491. package/src/diagnosis/frame.ts +0 -256
  492. package/src/diagnosis/index.ts +0 -5
  493. package/src/diagnosis/router.ts +0 -72
  494. package/src/diagnosis/types.ts +0 -89
  495. package/src/error/agentError.ts +0 -16
  496. package/src/error/ampError.ts +0 -20
  497. package/src/error/authzError.ts +0 -14
  498. package/src/error/boundaryRejection.ts +0 -20
  499. package/src/error/index.ts +0 -7
  500. package/src/error/kernelError.ts +0 -27
  501. package/src/error/mailboxError.ts +0 -9
  502. package/src/error/windowError.ts +0 -12
  503. package/src/evaluation/agreement.ts +0 -96
  504. package/src/evaluation/index.ts +0 -2
  505. package/src/evaluation/labels.ts +0 -241
  506. package/src/evidence/evidenceGraph.ts +0 -205
  507. package/src/evidence/index.ts +0 -4
  508. package/src/evidence/revalidation.ts +0 -191
  509. package/src/execution/exec.ts +0 -408
  510. package/src/execution/handles.ts +0 -65
  511. package/src/execution/index.ts +0 -3
  512. package/src/execution/pin.ts +0 -154
  513. package/src/frame/frame.ts +0 -70
  514. package/src/frame/index.ts +0 -1
  515. package/src/goal/goal.ts +0 -43
  516. package/src/goal/index.ts +0 -2
  517. package/src/goal/plan.ts +0 -196
  518. package/src/governance/attentionManifest.ts +0 -802
  519. package/src/governance/audit.ts +0 -362
  520. package/src/governance/budget.ts +0 -628
  521. package/src/governance/context.ts +0 -356
  522. package/src/governance/control.ts +0 -159
  523. package/src/governance/cooperation.ts +0 -479
  524. package/src/governance/credential.ts +0 -359
  525. package/src/governance/httpServer.ts +0 -99
  526. package/src/governance/index.ts +0 -32
  527. package/src/governance/legal.ts +0 -574
  528. package/src/governance/matrix.ts +0 -460
  529. package/src/governance/perspectives.ts +0 -406
  530. package/src/governance/signing.ts +0 -407
  531. package/src/governance/store.ts +0 -125
  532. package/src/governance/switching.ts +0 -459
  533. package/src/governance/toolPolicy.ts +0 -80
  534. package/src/governance/transport.ts +0 -204
  535. package/src/governance/types.ts +0 -380
  536. package/src/governance/workflow.ts +0 -743
  537. package/src/index.ts +0 -11
  538. package/src/interaction/index.ts +0 -1
  539. package/src/interaction/interaction.ts +0 -391
  540. package/src/ipc/index.ts +0 -4
  541. package/src/ipc/mailbox.ts +0 -132
  542. package/src/ipc/memorySocialStore.ts +0 -93
  543. package/src/ipc/social.ts +0 -93
  544. package/src/kernel/index.ts +0 -14
  545. package/src/kernel/scheduler.ts +0 -498
  546. package/src/ledger/checkpoint.ts +0 -304
  547. package/src/ledger/index.ts +0 -4
  548. package/src/ledger/ledger.ts +0 -31
  549. package/src/ledger/persistence.ts +0 -159
  550. package/src/ledger/share.ts +0 -80
  551. package/src/lock/index.ts +0 -2
  552. package/src/lock/lease.ts +0 -74
  553. package/src/main.ts +0 -199
  554. package/src/manifest/agentCard.ts +0 -72
  555. package/src/manifest/cardSign.ts +0 -87
  556. package/src/manifest/federation.ts +0 -237
  557. package/src/manifest/federationTypes.ts +0 -55
  558. package/src/manifest/index.ts +0 -8
  559. package/src/manifest/keys.ts +0 -643
  560. package/src/manifest/manifest.ts +0 -114
  561. package/src/manifest/sign.ts +0 -134
  562. package/src/manifest/trust.ts +0 -146
  563. package/src/mission/index.ts +0 -2
  564. package/src/mission/missionQueue.ts +0 -75
  565. package/src/module/index.ts +0 -9
  566. package/src/module/llm/deepseek.ts +0 -214
  567. package/src/module/llm/llm.ts +0 -84
  568. package/src/module/mcp/client.ts +0 -110
  569. package/src/module/mcp/http.ts +0 -102
  570. package/src/module/mcp/protocol.ts +0 -77
  571. package/src/module/mcp/server.ts +0 -148
  572. package/src/module/mcp/transport.ts +0 -30
  573. package/src/module/mcp/workspaceServer.ts +0 -111
  574. package/src/module/sandbox/sandbox.ts +0 -29
  575. package/src/module/tool/mcpExecutor.ts +0 -54
  576. package/src/module/tool/policy.ts +0 -52
  577. package/src/module/tool/redact.ts +0 -29
  578. package/src/module/tool/tool.ts +0 -53
  579. package/src/module/tool/workspaceExecutor.ts +0 -35
  580. package/src/policy/guard.ts +0 -240
  581. package/src/policy/index.ts +0 -2
  582. package/src/policy/policy.ts +0 -561
  583. package/src/process/agent.ts +0 -341
  584. package/src/process/agentService.ts +0 -236
  585. package/src/process/agentStore.ts +0 -378
  586. package/src/process/executionStore.ts +0 -151
  587. package/src/process/index.ts +0 -5
  588. package/src/process/process.ts +0 -240
  589. package/src/process/transitions.ts +0 -19
  590. package/src/progress/index.ts +0 -2
  591. package/src/progress/progress.ts +0 -85
  592. package/src/protocol/a2a.ts +0 -120
  593. package/src/protocol/channel.ts +0 -44
  594. package/src/protocol/envelope.ts +0 -655
  595. package/src/protocol/httpChannel.ts +0 -87
  596. package/src/protocol/index.ts +0 -6
  597. package/src/protocol/protocol.ts +0 -138
  598. package/src/protocol/wire.ts +0 -60
  599. package/src/receipt/index.ts +0 -1
  600. package/src/receipt/receipt.ts +0 -47
  601. package/src/registry/content.ts +0 -69
  602. package/src/registry/index.ts +0 -23
  603. package/src/registry/registry.ts +0 -599
  604. package/src/runtime/action.ts +0 -87
  605. package/src/runtime/behavior.ts +0 -330
  606. package/src/runtime/behaviorTuning.ts +0 -163
  607. package/src/runtime/cognitiveRuntime.ts +0 -703
  608. package/src/runtime/escalation.ts +0 -35
  609. package/src/runtime/governedTask.ts +0 -233
  610. package/src/runtime/index.ts +0 -15
  611. package/src/runtime/kernelRuntime.ts +0 -144
  612. package/src/runtime/operation.ts +0 -35
  613. package/src/runtime/replay.ts +0 -71
  614. package/src/runtime/routing.ts +0 -58
  615. package/src/runtime/safety.ts +0 -48
  616. package/src/runtime/session.ts +0 -341
  617. package/src/runtime/sessionStore.ts +0 -43
  618. package/src/runtime/telemetry.ts +0 -55
  619. package/src/runtime/turnStore.ts +0 -30
  620. package/src/scenario/index.ts +0 -1
  621. package/src/scenario/scenario.ts +0 -299
  622. package/src/semantic/canonical.ts +0 -24
  623. package/src/semantic/index.ts +0 -2
  624. package/src/semantic/sha256.ts +0 -82
  625. package/src/storage/appendLog.ts +0 -372
  626. package/src/storage/fileAppendSink.ts +0 -40
  627. package/src/storage/index.ts +0 -2
  628. package/src/store/bindCache.ts +0 -22
  629. package/src/store/cache.ts +0 -191
  630. package/src/store/context.ts +0 -31
  631. package/src/store/handleStore.ts +0 -31
  632. package/src/store/index.ts +0 -9
  633. package/src/store/messageStore.ts +0 -31
  634. package/src/store/objectStore.ts +0 -31
  635. package/src/store/processStore.ts +0 -31
  636. package/src/store/threadStore.ts +0 -31
  637. package/src/store/windowStore.ts +0 -31
  638. package/src/structure/abstractSyntaxTree.ts +0 -493
  639. package/src/structure/avlTree.ts +0 -408
  640. package/src/structure/binarySearchTree.ts +0 -352
  641. package/src/structure/binaryTree.ts +0 -525
  642. package/src/structure/bloomFilter.ts +0 -231
  643. package/src/structure/compare.ts +0 -110
  644. package/src/structure/cuckooFilter.ts +0 -337
  645. package/src/structure/deque.ts +0 -223
  646. package/src/structure/filterKey.ts +0 -36
  647. package/src/structure/hash.ts +0 -279
  648. package/src/structure/heap.ts +0 -268
  649. package/src/structure/index.ts +0 -20
  650. package/src/structure/iterator.ts +0 -22
  651. package/src/structure/lru.ts +0 -619
  652. package/src/structure/ordered.ts +0 -399
  653. package/src/structure/priority.ts +0 -915
  654. package/src/structure/queue.ts +0 -168
  655. package/src/structure/redBlackTree.ts +0 -680
  656. package/src/structure/segmentTree.ts +0 -248
  657. package/src/structure/stack.ts +0 -118
  658. package/src/structure/unordered.ts +0 -407
  659. package/src/style.css +0 -278
  660. package/src/thread/index.ts +0 -2
  661. package/src/thread/thread.ts +0 -176
  662. package/src/thread/transitions.ts +0 -17
  663. package/src/types/common.ts +0 -119
  664. package/src/types/index.ts +0 -23
  665. package/src/warrant/epistemic.ts +0 -154
  666. package/src/warrant/index.ts +0 -3
  667. package/src/warrant/inquiry.ts +0 -166
  668. package/src/warrant/warrant.ts +0 -179
  669. package/src/window/compaction.ts +0 -403
  670. package/src/window/compactionTypes.ts +0 -119
  671. package/src/window/index.ts +0 -7
  672. package/src/window/page.ts +0 -18
  673. package/src/window/window.ts +0 -244
  674. package/tools/_helpers.ts +0 -34
  675. package/tools/a2a-check.ts +0 -110
  676. package/tools/a2a-flow-check.ts +0 -78
  677. package/tools/agent-card-check.ts +0 -76
  678. package/tools/agent-check.ts +0 -236
  679. package/tools/agent-store-check.ts +0 -153
  680. package/tools/agreement-check.ts +0 -27
  681. package/tools/annotation-server-check.ts +0 -48
  682. package/tools/annotation-server.ts +0 -230
  683. package/tools/annotation-status.ts +0 -65
  684. package/tools/approval-access-check.ts +0 -66
  685. package/tools/arch-check.ts +0 -72
  686. package/tools/attention-core-check.ts +0 -48
  687. package/tools/attention-resource-check.ts +0 -169
  688. package/tools/audit-check.ts +0 -207
  689. package/tools/authority-check.ts +0 -124
  690. package/tools/authz-check.ts +0 -138
  691. package/tools/bayes-check.ts +0 -162
  692. package/tools/bayes-probe.ts +0 -154
  693. package/tools/behavior-tuning-check.ts +0 -53
  694. package/tools/board-check.ts +0 -87
  695. package/tools/boundary-check.ts +0 -199
  696. package/tools/bundle-trust-check.ts +0 -96
  697. package/tools/calibrate-imported.ts +0 -108
  698. package/tools/calibration-check.ts +0 -44
  699. package/tools/calibration-registry-check.ts +0 -40
  700. package/tools/calibration-sim.ts +0 -40
  701. package/tools/card-sign-check.ts +0 -71
  702. package/tools/card-trust-check.ts +0 -78
  703. package/tools/channel-check.ts +0 -86
  704. package/tools/checkpoint-check.ts +0 -620
  705. package/tools/cleanup-style-configs.ts +0 -29
  706. package/tools/cognition-check.ts +0 -648
  707. package/tools/cognitive-runtime-check.ts +0 -609
  708. package/tools/collaboration-demo.ts +0 -78
  709. package/tools/compaction-check.ts +0 -266
  710. package/tools/concept-check.ts +0 -260
  711. package/tools/concurrency-check.ts +0 -242
  712. package/tools/conformance-runtime.ts +0 -389
  713. package/tools/conformance.ts +0 -651
  714. package/tools/context-ast-check.ts +0 -90
  715. package/tools/context-ast-deepseek.ts +0 -242
  716. package/tools/context-graph-bench.ts +0 -50
  717. package/tools/context-graph-check.ts +0 -229
  718. package/tools/context-index-check.ts +0 -48
  719. package/tools/context-lifecycle-check.ts +0 -69
  720. package/tools/context-projection-check.ts +0 -73
  721. package/tools/context-scope-check.ts +0 -134
  722. package/tools/context-storage-check.ts +0 -113
  723. package/tools/cooperation-check.ts +0 -517
  724. package/tools/cooperation-http-check.ts +0 -125
  725. package/tools/deepseek-llm-check.ts +0 -147
  726. package/tools/deepseek-probe.ts +0 -494
  727. package/tools/diagnosis-check.ts +0 -109
  728. package/tools/diagnosis-router-check.ts +0 -50
  729. package/tools/document-context-check.ts +0 -237
  730. package/tools/envelope-check.ts +0 -659
  731. package/tools/eslint-rules/index.ts +0 -26
  732. package/tools/eslint-rules/rules/jsdoc-comment-format.ts +0 -105
  733. package/tools/eslint-rules/rules/no-function-keyword.ts +0 -60
  734. package/tools/eslint-rules/rules/no-multiline-parameters.ts +0 -71
  735. package/tools/eslint-rules/rules/no-multiline-type-parameters.ts +0 -57
  736. package/tools/eslint-rules/rules/no-undefined-comparison.ts +0 -115
  737. package/tools/eslint-rules/rules/prefer-template-for-string-conversion.ts +0 -97
  738. package/tools/eslint-rules/rules/require-typeof-parentheses.ts +0 -49
  739. package/tools/eslint-rules/rules/require-variable-type.ts +0 -36
  740. package/tools/eslint-rules/rules/variable-naming.ts +0 -175
  741. package/tools/eval-recorded-replay.ts +0 -215
  742. package/tools/eval-replay.ts +0 -139
  743. package/tools/eval-summary.ts +0 -136
  744. package/tools/evidence-revalidation-check.ts +0 -235
  745. package/tools/exec-check.ts +0 -217
  746. package/tools/federation-check.ts +0 -166
  747. package/tools/federation-demo.ts +0 -45
  748. package/tools/filter-check.ts +0 -243
  749. package/tools/governance-check.ts +0 -680
  750. package/tools/governance-store-check.ts +0 -131
  751. package/tools/governed-task-check.ts +0 -227
  752. package/tools/guard-check.ts +0 -122
  753. package/tools/http-channel-check.ts +0 -125
  754. package/tools/import-campaign.ts +0 -28
  755. package/tools/inquiry-check.ts +0 -126
  756. package/tools/interaction-check.ts +0 -171
  757. package/tools/interrupts-check.ts +0 -173
  758. package/tools/ipc-check.ts +0 -170
  759. package/tools/kernel-async-check.ts +0 -136
  760. package/tools/kernel-check.ts +0 -369
  761. package/tools/kernel-runtime-check.ts +0 -276
  762. package/tools/keys-check.ts +0 -559
  763. package/tools/label-audit.ts +0 -70
  764. package/tools/labels-workflow-check.ts +0 -49
  765. package/tools/linear-structure-check.ts +0 -524
  766. package/tools/live-exec.ts +0 -62
  767. package/tools/live-governance-eval.ts +0 -813
  768. package/tools/live-legacy.ts +0 -50
  769. package/tools/lru-check.ts +0 -182
  770. package/tools/make-annotation-campaign.ts +0 -57
  771. package/tools/mcp-executor-check.ts +0 -33
  772. package/tools/merge-annotations.ts +0 -32
  773. package/tools/merge-check.ts +0 -98
  774. package/tools/metrics-check.ts +0 -84
  775. package/tools/net-probe.ts +0 -107
  776. package/tools/oauth-probe.ts +0 -117
  777. package/tools/outbox-check.ts +0 -50
  778. package/tools/persistence-check.ts +0 -144
  779. package/tools/pin-check.ts +0 -116
  780. package/tools/platform-check.ts +0 -281
  781. package/tools/policy-check.ts +0 -514
  782. package/tools/prettier-plugin/index.mts +0 -84
  783. package/tools/priority-check.ts +0 -627
  784. package/tools/probe-context7.ts +0 -63
  785. package/tools/registry-check.ts +0 -252
  786. package/tools/reliability-check.ts +0 -25
  787. package/tools/retry-check.ts +0 -85
  788. package/tools/rewrite-declaration-extensions.ts +0 -98
  789. package/tools/runtime-check.ts +0 -165
  790. package/tools/runtime-deepseek-check.ts +0 -85
  791. package/tools/runtime-eval.ts +0 -358
  792. package/tools/scenario-check.ts +0 -161
  793. package/tools/scenario-incident.ts +0 -162
  794. package/tools/scoped-priors-check.ts +0 -35
  795. package/tools/scoped-priors-pooling.ts +0 -34
  796. package/tools/segment-ast-check.ts +0 -649
  797. package/tools/session-check.ts +0 -196
  798. package/tools/session-store-check.ts +0 -73
  799. package/tools/share-check.ts +0 -88
  800. package/tools/sign-check.ts +0 -95
  801. package/tools/signing-check.ts +0 -275
  802. package/tools/snapshot-demo.ts +0 -65
  803. package/tools/social-check.ts +0 -146
  804. package/tools/structure-compare-check.ts +0 -95
  805. package/tools/support/documentScanner.ts +0 -577
  806. package/tools/support/texScanner.ts +0 -631
  807. package/tools/support/workspaceScanner.ts +0 -476
  808. package/tools/team-check.ts +0 -163
  809. package/tools/team-demo.ts +0 -36
  810. package/tools/telemetry-check.ts +0 -44
  811. package/tools/tex-context-check.ts +0 -263
  812. package/tools/tool-gate-check.ts +0 -98
  813. package/tools/tree-structure-check.ts +0 -786
  814. package/tools/trust-flow-check.ts +0 -158
  815. package/tools/turn-store-check.ts +0 -57
  816. package/tools/unified-check.ts +0 -83
  817. package/tools/validate.ts +0 -309
  818. package/tools/wire-check.ts +0 -96
  819. package/tools/workspace-agent-check.ts +0 -486
  820. package/tools/workspace-agent-demo.ts +0 -62
  821. package/tools/workspace-agent-server.ts +0 -101
  822. package/tools/workspace-context-check.ts +0 -331
  823. package/tools/workspace-context.ts +0 -131
  824. package/tsconfig.build.json +0 -14
  825. package/tsconfig.json +0 -24
  826. package/tsconfig.tools.json +0 -31
  827. package/viz/context-ast.json +0 -1577
package/docs/probes/runtime-eval.json DELETED
@@ -1,4230 +0,0 @@
1
- {
2
- "generatedAt": "2026-10-28T19:55:29.963Z",
3
- "kind": "runtime-eval",
4
- "model": "ScriptedLlm",
5
- "cases": [
6
- {
7
- "id": "arith-0",
8
- "category": "arithmetic",
9
- "actual": false,
10
- "humanLabel": null,
11
- "answer": "3",
12
- "predictedCorrectness": 0.51278928136419,
13
- "answerOverall": 0.5065686319907432,
14
- "tokens": 11,
15
- "rest": false,
16
- "scenario": [
17
- "chat",
18
- "writing",
19
- "office",
20
- "collaboration",
21
- "engineering"
22
- ],
23
- "interaction": "think"
24
- },
25
- {
26
- "id": "arith-1",
27
- "category": "arithmetic",
28
- "actual": true,
29
- "humanLabel": null,
30
- "answer": "10",
31
- "predictedCorrectness": 0.51278928136419,
32
- "answerOverall": 0.5065686319907432,
33
- "tokens": 11,
34
- "rest": false,
35
- "scenario": [
36
- "chat",
37
- "writing",
38
- "office",
39
- "collaboration",
40
- "engineering"
41
- ],
42
- "interaction": "think"
43
- },
44
- {
45
- "id": "arith-2",
46
- "category": "arithmetic",
47
- "actual": true,
48
- "humanLabel": null,
49
- "answer": "5",
50
- "predictedCorrectness": 0.51278928136419,
51
- "answerOverall": 0.5065686319907432,
52
- "tokens": 11,
53
- "rest": false,
54
- "scenario": [
55
- "chat",
56
- "writing",
57
- "office",
58
- "collaboration",
59
- "engineering"
60
- ],
61
- "interaction": "think"
62
- },
63
- {
64
- "id": "arith-3",
65
- "category": "arithmetic",
66
- "actual": true,
67
- "humanLabel": null,
68
- "answer": "13",
69
- "predictedCorrectness": 0.51278928136419,
70
- "answerOverall": 0.5065686319907432,
71
- "tokens": 11,
72
- "rest": false,
73
- "scenario": [
74
- "chat",
75
- "writing",
76
- "office",
77
- "collaboration",
78
- "engineering"
79
- ],
80
- "interaction": "think"
81
- },
82
- {
83
- "id": "arith-4",
84
- "category": "arithmetic",
85
- "actual": true,
86
- "humanLabel": null,
87
- "answer": "8",
88
- "predictedCorrectness": 0.51278928136419,
89
- "answerOverall": 0.5065686319907432,
90
- "tokens": 11,
91
- "rest": false,
92
- "scenario": [
93
- "chat",
94
- "writing",
95
- "office",
96
- "collaboration",
97
- "engineering"
98
- ],
99
- "interaction": "think"
100
- },
101
- {
102
- "id": "arith-5",
103
- "category": "arithmetic",
104
- "actual": true,
105
- "humanLabel": null,
106
- "answer": "16",
107
- "predictedCorrectness": 0.51278928136419,
108
- "answerOverall": 0.5065686319907432,
109
- "tokens": 11,
110
- "rest": false,
111
- "scenario": [
112
- "chat",
113
- "writing",
114
- "office",
115
- "collaboration",
116
- "engineering"
117
- ],
118
- "interaction": "think"
119
- },
120
- {
121
- "id": "arith-6",
122
- "category": "arithmetic",
123
- "actual": true,
124
- "humanLabel": null,
125
- "answer": "11",
126
- "predictedCorrectness": 0.51278928136419,
127
- "answerOverall": 0.5065686319907432,
128
- "tokens": 11,
129
- "rest": false,
130
- "scenario": [
131
- "chat",
132
- "writing",
133
- "office",
134
- "collaboration",
135
- "engineering"
136
- ],
137
- "interaction": "think"
138
- },
139
- {
140
- "id": "arith-7",
141
- "category": "arithmetic",
142
- "actual": true,
143
- "humanLabel": null,
144
- "answer": "19",
145
- "predictedCorrectness": 0.51278928136419,
146
- "answerOverall": 0.5065686319907432,
147
- "tokens": 11,
148
- "rest": false,
149
- "scenario": [
150
- "chat",
151
- "writing",
152
- "office",
153
- "collaboration",
154
- "engineering"
155
- ],
156
- "interaction": "think"
157
- },
158
- {
159
- "id": "arith-8",
160
- "category": "arithmetic",
161
- "actual": true,
162
- "humanLabel": null,
163
- "answer": "14",
164
- "predictedCorrectness": 0.51278928136419,
165
- "answerOverall": 0.5065686319907432,
166
- "tokens": 11,
167
- "rest": false,
168
- "scenario": [
169
- "chat",
170
- "writing",
171
- "office",
172
- "collaboration",
173
- "engineering"
174
- ],
175
- "interaction": "think"
176
- },
177
- {
178
- "id": "arith-9",
179
- "category": "arithmetic",
180
- "actual": true,
181
- "humanLabel": null,
182
- "answer": "22",
183
- "predictedCorrectness": 0.51278928136419,
184
- "answerOverall": 0.5065686319907432,
185
- "tokens": 11,
186
- "rest": false,
187
- "scenario": [
188
- "chat",
189
- "writing",
190
- "office",
191
- "collaboration",
192
- "engineering"
193
- ],
194
- "interaction": "think"
195
- },
196
- {
197
- "id": "arith-10",
198
- "category": "arithmetic",
199
- "actual": false,
200
- "humanLabel": null,
201
- "answer": "18",
202
- "predictedCorrectness": 0.51278928136419,
203
- "answerOverall": 0.5065686319907432,
204
- "tokens": 11,
205
- "rest": false,
206
- "scenario": [
207
- "chat",
208
- "writing",
209
- "office",
210
- "collaboration",
211
- "engineering"
212
- ],
213
- "interaction": "think"
214
- },
215
- {
216
- "id": "arith-11",
217
- "category": "arithmetic",
218
- "actual": true,
219
- "humanLabel": null,
220
- "answer": "25",
221
- "predictedCorrectness": 0.51278928136419,
222
- "answerOverall": 0.5065686319907432,
223
- "tokens": 11,
224
- "rest": false,
225
- "scenario": [
226
- "chat",
227
- "writing",
228
- "office",
229
- "collaboration",
230
- "engineering"
231
- ],
232
- "interaction": "think"
233
- },
234
- {
235
- "id": "arith-12",
236
- "category": "arithmetic",
237
- "actual": true,
238
- "humanLabel": null,
239
- "answer": "20",
240
- "predictedCorrectness": 0.51278928136419,
241
- "answerOverall": 0.5065686319907432,
242
- "tokens": 11,
243
- "rest": false,
244
- "scenario": [
245
- "chat",
246
- "writing",
247
- "office",
248
- "collaboration",
249
- "engineering"
250
- ],
251
- "interaction": "think"
252
- },
253
- {
254
- "id": "arith-13",
255
- "category": "arithmetic",
256
- "actual": true,
257
- "humanLabel": null,
258
- "answer": "15",
259
- "predictedCorrectness": 0.51278928136419,
260
- "answerOverall": 0.5065686319907432,
261
- "tokens": 11,
262
- "rest": false,
263
- "scenario": [
264
- "chat",
265
- "writing",
266
- "office",
267
- "collaboration",
268
- "engineering"
269
- ],
270
- "interaction": "think"
271
- },
272
- {
273
- "id": "arith-14",
274
- "category": "arithmetic",
275
- "actual": true,
276
- "humanLabel": null,
277
- "answer": "23",
278
- "predictedCorrectness": 0.51278928136419,
279
- "answerOverall": 0.5065686319907432,
280
- "tokens": 11,
281
- "rest": false,
282
- "scenario": [
283
- "chat",
284
- "writing",
285
- "office",
286
- "collaboration",
287
- "engineering"
288
- ],
289
- "interaction": "think"
290
- },
291
- {
292
- "id": "arith-15",
293
- "category": "arithmetic",
294
- "actual": true,
295
- "humanLabel": null,
296
- "answer": "18",
297
- "predictedCorrectness": 0.51278928136419,
298
- "answerOverall": 0.5065686319907432,
299
- "tokens": 11,
300
- "rest": false,
301
- "scenario": [
302
- "chat",
303
- "writing",
304
- "office",
305
- "collaboration",
306
- "engineering"
307
- ],
308
- "interaction": "think"
309
- },
310
- {
311
- "id": "arith-16",
312
- "category": "arithmetic",
313
- "actual": true,
314
- "humanLabel": null,
315
- "answer": "26",
316
- "predictedCorrectness": 0.51278928136419,
317
- "answerOverall": 0.5065686319907432,
318
- "tokens": 11,
319
- "rest": false,
320
- "scenario": [
321
- "chat",
322
- "writing",
323
- "office",
324
- "collaboration",
325
- "engineering"
326
- ],
327
- "interaction": "think"
328
- },
329
- {
330
- "id": "arith-17",
331
- "category": "arithmetic",
332
- "actual": true,
333
- "humanLabel": null,
334
- "answer": "21",
335
- "predictedCorrectness": 0.51278928136419,
336
- "answerOverall": 0.5065686319907432,
337
- "tokens": 11,
338
- "rest": false,
339
- "scenario": [
340
- "chat",
341
- "writing",
342
- "office",
343
- "collaboration",
344
- "engineering"
345
- ],
346
- "interaction": "think"
347
- },
348
- {
349
- "id": "arith-18",
350
- "category": "arithmetic",
351
- "actual": true,
352
- "humanLabel": null,
353
- "answer": "29",
354
- "predictedCorrectness": 0.51278928136419,
355
- "answerOverall": 0.5065686319907432,
356
- "tokens": 11,
357
- "rest": false,
358
- "scenario": [
359
- "chat",
360
- "writing",
361
- "office",
362
- "collaboration",
363
- "engineering"
364
- ],
365
- "interaction": "think"
366
- },
367
- {
368
- "id": "arith-19",
369
- "category": "arithmetic",
370
- "actual": true,
371
- "humanLabel": null,
372
- "answer": "24",
373
- "predictedCorrectness": 0.51278928136419,
374
- "answerOverall": 0.5065686319907432,
375
- "tokens": 11,
376
- "rest": false,
377
- "scenario": [
378
- "chat",
379
- "writing",
380
- "office",
381
- "collaboration",
382
- "engineering"
383
- ],
384
- "interaction": "think"
385
- },
386
- {
387
- "id": "arith-20",
388
- "category": "arithmetic",
389
- "actual": false,
390
- "humanLabel": null,
391
- "answer": "33",
392
- "predictedCorrectness": 0.51278928136419,
393
- "answerOverall": 0.5065686319907432,
394
- "tokens": 11,
395
- "rest": false,
396
- "scenario": [
397
- "chat",
398
- "writing",
399
- "office",
400
- "collaboration",
401
- "engineering"
402
- ],
403
- "interaction": "think"
404
- },
405
- {
406
- "id": "arith-21",
407
- "category": "arithmetic",
408
- "actual": true,
409
- "humanLabel": null,
410
- "answer": "27",
411
- "predictedCorrectness": 0.51278928136419,
412
- "answerOverall": 0.5065686319907432,
413
- "tokens": 11,
414
- "rest": false,
415
- "scenario": [
416
- "chat",
417
- "writing",
418
- "office",
419
- "collaboration",
420
- "engineering"
421
- ],
422
- "interaction": "think"
423
- },
424
- {
425
- "id": "arith-22",
426
- "category": "arithmetic",
427
- "actual": true,
428
- "humanLabel": null,
429
- "answer": "35",
430
- "predictedCorrectness": 0.51278928136419,
431
- "answerOverall": 0.5065686319907432,
432
- "tokens": 11,
433
- "rest": false,
434
- "scenario": [
435
- "chat",
436
- "writing",
437
- "office",
438
- "collaboration",
439
- "engineering"
440
- ],
441
- "interaction": "think"
442
- },
443
- {
444
- "id": "arith-23",
445
- "category": "arithmetic",
446
- "actual": true,
447
- "humanLabel": null,
448
- "answer": "30",
449
- "predictedCorrectness": 0.51278928136419,
450
- "answerOverall": 0.5065686319907432,
451
- "tokens": 11,
452
- "rest": false,
453
- "scenario": [
454
- "chat",
455
- "writing",
456
- "office",
457
- "collaboration",
458
- "engineering"
459
- ],
460
- "interaction": "think"
461
- },
462
- {
463
- "id": "arith-24",
464
- "category": "arithmetic",
465
- "actual": true,
466
- "humanLabel": null,
467
- "answer": "38",
468
- "predictedCorrectness": 0.51278928136419,
469
- "answerOverall": 0.5065686319907432,
470
- "tokens": 11,
471
- "rest": false,
472
- "scenario": [
473
- "chat",
474
- "writing",
475
- "office",
476
- "collaboration",
477
- "engineering"
478
- ],
479
- "interaction": "think"
480
- },
481
- {
482
- "id": "arith-25",
483
- "category": "arithmetic",
484
- "actual": true,
485
- "humanLabel": null,
486
- "answer": "33",
487
- "predictedCorrectness": 0.51278928136419,
488
- "answerOverall": 0.5065686319907432,
489
- "tokens": 11,
490
- "rest": false,
491
- "scenario": [
492
- "chat",
493
- "writing",
494
- "office",
495
- "collaboration",
496
- "engineering"
497
- ],
498
- "interaction": "think"
499
- },
500
- {
501
- "id": "arith-26",
502
- "category": "arithmetic",
503
- "actual": true,
504
- "humanLabel": null,
505
- "answer": "28",
506
- "predictedCorrectness": 0.51278928136419,
507
- "answerOverall": 0.5065686319907432,
508
- "tokens": 11,
509
- "rest": false,
510
- "scenario": [
511
- "chat",
512
- "writing",
513
- "office",
514
- "collaboration",
515
- "engineering"
516
- ],
517
- "interaction": "think"
518
- },
519
- {
520
- "id": "arith-27",
521
- "category": "arithmetic",
522
- "actual": true,
523
- "humanLabel": null,
524
- "answer": "36",
525
- "predictedCorrectness": 0.51278928136419,
526
- "answerOverall": 0.5065686319907432,
527
- "tokens": 11,
528
- "rest": false,
529
- "scenario": [
530
- "chat",
531
- "writing",
532
- "office",
533
- "collaboration",
534
- "engineering"
535
- ],
536
- "interaction": "think"
537
- },
538
- {
539
- "id": "arith-28",
540
- "category": "arithmetic",
541
- "actual": true,
542
- "humanLabel": null,
543
- "answer": "31",
544
- "predictedCorrectness": 0.51278928136419,
545
- "answerOverall": 0.5065686319907432,
546
- "tokens": 11,
547
- "rest": false,
548
- "scenario": [
549
- "chat",
550
- "writing",
551
- "office",
552
- "collaboration",
553
- "engineering"
554
- ],
555
- "interaction": "think"
556
- },
557
- {
558
- "id": "arith-29",
559
- "category": "arithmetic",
560
- "actual": true,
561
- "humanLabel": null,
562
- "answer": "39",
563
- "predictedCorrectness": 0.51278928136419,
564
- "answerOverall": 0.5065686319907432,
565
- "tokens": 11,
566
- "rest": false,
567
- "scenario": [
568
- "chat",
569
- "writing",
570
- "office",
571
- "collaboration",
572
- "engineering"
573
- ],
574
- "interaction": "think"
575
- },
576
- {
577
- "id": "arith-30",
578
- "category": "arithmetic",
579
- "actual": false,
580
- "humanLabel": null,
581
- "answer": "35",
582
- "predictedCorrectness": 0.51278928136419,
583
- "answerOverall": 0.5065686319907432,
584
- "tokens": 11,
585
- "rest": false,
586
- "scenario": [
587
- "chat",
588
- "writing",
589
- "office",
590
- "collaboration",
591
- "engineering"
592
- ],
593
- "interaction": "think"
594
- },
595
- {
596
- "id": "arith-31",
597
- "category": "arithmetic",
598
- "actual": true,
599
- "humanLabel": null,
600
- "answer": "42",
601
- "predictedCorrectness": 0.51278928136419,
602
- "answerOverall": 0.5065686319907432,
603
- "tokens": 11,
604
- "rest": false,
605
- "scenario": [
606
- "chat",
607
- "writing",
608
- "office",
609
- "collaboration",
610
- "engineering"
611
- ],
612
- "interaction": "think"
613
- },
614
- {
615
- "id": "arith-32",
616
- "category": "arithmetic",
617
- "actual": true,
618
- "humanLabel": null,
619
- "answer": "37",
620
- "predictedCorrectness": 0.51278928136419,
621
- "answerOverall": 0.5065686319907432,
622
- "tokens": 11,
623
- "rest": false,
624
- "scenario": [
625
- "chat",
626
- "writing",
627
- "office",
628
- "collaboration",
629
- "engineering"
630
- ],
631
- "interaction": "think"
632
- },
633
- {
634
- "id": "arith-33",
635
- "category": "arithmetic",
636
- "actual": true,
637
- "humanLabel": null,
638
- "answer": "45",
639
- "predictedCorrectness": 0.51278928136419,
640
- "answerOverall": 0.5065686319907432,
641
- "tokens": 11,
642
- "rest": false,
643
- "scenario": [
644
- "chat",
645
- "writing",
646
- "office",
647
- "collaboration",
648
- "engineering"
649
- ],
650
- "interaction": "think"
651
- },
652
- {
653
- "id": "arith-34",
654
- "category": "arithmetic",
655
- "actual": true,
656
- "humanLabel": null,
657
- "answer": "40",
658
- "predictedCorrectness": 0.51278928136419,
659
- "answerOverall": 0.5065686319907432,
660
- "tokens": 11,
661
- "rest": false,
662
- "scenario": [
663
- "chat",
664
- "writing",
665
- "office",
666
- "collaboration",
667
- "engineering"
668
- ],
669
- "interaction": "think"
670
- },
671
- {
672
- "id": "arith-35",
673
- "category": "arithmetic",
674
- "actual": true,
675
- "humanLabel": null,
676
- "answer": "48",
677
- "predictedCorrectness": 0.51278928136419,
678
- "answerOverall": 0.5065686319907432,
679
- "tokens": 11,
680
- "rest": false,
681
- "scenario": [
682
- "chat",
683
- "writing",
684
- "office",
685
- "collaboration",
686
- "engineering"
687
- ],
688
- "interaction": "think"
689
- },
690
- {
691
- "id": "arith-36",
692
- "category": "arithmetic",
693
- "actual": true,
694
- "humanLabel": null,
695
- "answer": "43",
696
- "predictedCorrectness": 0.51278928136419,
697
- "answerOverall": 0.5065686319907432,
698
- "tokens": 11,
699
- "rest": false,
700
- "scenario": [
701
- "chat",
702
- "writing",
703
- "office",
704
- "collaboration",
705
- "engineering"
706
- ],
707
- "interaction": "think"
708
- },
709
- {
710
- "id": "arith-37",
711
- "category": "arithmetic",
712
- "actual": true,
713
- "humanLabel": null,
714
- "answer": "51",
715
- "predictedCorrectness": 0.51278928136419,
716
- "answerOverall": 0.5065686319907432,
717
- "tokens": 11,
718
- "rest": false,
719
- "scenario": [
720
- "chat",
721
- "writing",
722
- "office",
723
- "collaboration",
724
- "engineering"
725
- ],
726
- "interaction": "think"
727
- },
728
- {
729
- "id": "arith-38",
730
- "category": "arithmetic",
731
- "actual": true,
732
- "humanLabel": null,
733
- "answer": "46",
734
- "predictedCorrectness": 0.51278928136419,
735
- "answerOverall": 0.5065686319907432,
736
- "tokens": 11,
737
- "rest": false,
738
- "scenario": [
739
- "chat",
740
- "writing",
741
- "office",
742
- "collaboration",
743
- "engineering"
744
- ],
745
- "interaction": "think"
746
- },
747
- {
748
- "id": "arith-39",
749
- "category": "arithmetic",
750
- "actual": true,
751
- "humanLabel": null,
752
- "answer": "41",
753
- "predictedCorrectness": 0.51278928136419,
754
- "answerOverall": 0.5065686319907432,
755
- "tokens": 11,
756
- "rest": false,
757
- "scenario": [
758
- "chat",
759
- "writing",
760
- "office",
761
- "collaboration",
762
- "engineering"
763
- ],
764
- "interaction": "think"
765
- },
766
- {
767
- "id": "arith-40",
768
- "category": "arithmetic",
769
- "actual": false,
770
- "humanLabel": null,
771
- "answer": "50",
772
- "predictedCorrectness": 0.51278928136419,
773
- "answerOverall": 0.5065686319907432,
774
- "tokens": 11,
775
- "rest": false,
776
- "scenario": [
777
- "chat",
778
- "writing",
779
- "office",
780
- "collaboration",
781
- "engineering"
782
- ],
783
- "interaction": "think"
784
- },
785
- {
786
- "id": "arith-41",
787
- "category": "arithmetic",
788
- "actual": true,
789
- "humanLabel": null,
790
- "answer": "44",
791
- "predictedCorrectness": 0.51278928136419,
792
- "answerOverall": 0.5065686319907432,
793
- "tokens": 11,
794
- "rest": false,
795
- "scenario": [
796
- "chat",
797
- "writing",
798
- "office",
799
- "collaboration",
800
- "engineering"
801
- ],
802
- "interaction": "think"
803
- },
804
- {
805
- "id": "arith-42",
806
- "category": "arithmetic",
807
- "actual": true,
808
- "humanLabel": null,
809
- "answer": "52",
810
- "predictedCorrectness": 0.51278928136419,
811
- "answerOverall": 0.5065686319907432,
812
- "tokens": 11,
813
- "rest": false,
814
- "scenario": [
815
- "chat",
816
- "writing",
817
- "office",
818
- "collaboration",
819
- "engineering"
820
- ],
821
- "interaction": "think"
822
- },
823
- {
824
- "id": "arith-43",
825
- "category": "arithmetic",
826
- "actual": true,
827
- "humanLabel": null,
828
- "answer": "47",
829
- "predictedCorrectness": 0.51278928136419,
830
- "answerOverall": 0.5065686319907432,
831
- "tokens": 11,
832
- "rest": false,
833
- "scenario": [
834
- "chat",
835
- "writing",
836
- "office",
837
- "collaboration",
838
- "engineering"
839
- ],
840
- "interaction": "think"
841
- },
842
- {
843
- "id": "arith-44",
844
- "category": "arithmetic",
845
- "actual": true,
846
- "humanLabel": null,
847
- "answer": "55",
848
- "predictedCorrectness": 0.51278928136419,
849
- "answerOverall": 0.5065686319907432,
850
- "tokens": 11,
851
- "rest": false,
852
- "scenario": [
853
- "chat",
854
- "writing",
855
- "office",
856
- "collaboration",
857
- "engineering"
858
- ],
859
- "interaction": "think"
860
- },
861
- {
862
- "id": "arith-45",
863
- "category": "arithmetic",
864
- "actual": true,
865
- "humanLabel": null,
866
- "answer": "50",
867
- "predictedCorrectness": 0.51278928136419,
868
- "answerOverall": 0.5065686319907432,
869
- "tokens": 11,
870
- "rest": false,
871
- "scenario": [
872
- "chat",
873
- "writing",
874
- "office",
875
- "collaboration",
876
- "engineering"
877
- ],
878
- "interaction": "think"
879
- },
880
- {
881
- "id": "arith-46",
882
- "category": "arithmetic",
883
- "actual": true,
884
- "humanLabel": null,
885
- "answer": "58",
886
- "predictedCorrectness": 0.51278928136419,
887
- "answerOverall": 0.5065686319907432,
888
- "tokens": 11,
889
- "rest": false,
890
- "scenario": [
891
- "chat",
892
- "writing",
893
- "office",
894
- "collaboration",
895
- "engineering"
896
- ],
897
- "interaction": "think"
898
- },
899
- {
900
- "id": "arith-47",
901
- "category": "arithmetic",
902
- "actual": true,
903
- "humanLabel": null,
904
- "answer": "53",
905
- "predictedCorrectness": 0.51278928136419,
906
- "answerOverall": 0.5065686319907432,
907
- "tokens": 11,
908
- "rest": false,
909
- "scenario": [
910
- "chat",
911
- "writing",
912
- "office",
913
- "collaboration",
914
- "engineering"
915
- ],
916
- "interaction": "think"
917
- },
918
- {
919
- "id": "arith-48",
920
- "category": "arithmetic",
921
- "actual": true,
922
- "humanLabel": null,
923
- "answer": "61",
924
- "predictedCorrectness": 0.51278928136419,
925
- "answerOverall": 0.5065686319907432,
926
- "tokens": 11,
927
- "rest": false,
928
- "scenario": [
929
- "chat",
930
- "writing",
931
- "office",
932
- "collaboration",
933
- "engineering"
934
- ],
935
- "interaction": "think"
936
- },
937
- {
938
- "id": "arith-49",
939
- "category": "arithmetic",
940
- "actual": true,
941
- "humanLabel": null,
942
- "answer": "56",
943
- "predictedCorrectness": 0.51278928136419,
944
- "answerOverall": 0.5065686319907432,
945
- "tokens": 11,
946
- "rest": false,
947
- "scenario": [
948
- "chat",
949
- "writing",
950
- "office",
951
- "collaboration",
952
- "engineering"
953
- ],
954
- "interaction": "think"
955
- },
956
- {
957
- "id": "unique-0",
958
- "category": "unique-rows",
959
- "actual": false,
960
- "humanLabel": null,
961
- "answer": "20",
962
- "predictedCorrectness": 0.51278928136419,
963
- "answerOverall": 0.5065686319907432,
964
- "tokens": 11,
965
- "rest": false,
966
- "scenario": [
967
- "chat",
968
- "writing",
969
- "office",
970
- "collaboration",
971
- "engineering"
972
- ],
973
- "interaction": "think"
974
- },
975
- {
976
- "id": "unique-1",
977
- "category": "unique-rows",
978
- "actual": true,
979
- "humanLabel": null,
980
- "answer": "19",
981
- "predictedCorrectness": 0.51278928136419,
982
- "answerOverall": 0.5065686319907432,
983
- "tokens": 11,
984
- "rest": false,
985
- "scenario": [
986
- "chat",
987
- "writing",
988
- "office",
989
- "collaboration",
990
- "engineering"
991
- ],
992
- "interaction": "think"
993
- },
994
- {
995
- "id": "unique-2",
996
- "category": "unique-rows",
997
- "actual": true,
998
- "humanLabel": null,
999
- "answer": "19",
1000
- "predictedCorrectness": 0.51278928136419,
1001
- "answerOverall": 0.5065686319907432,
1002
- "tokens": 11,
1003
- "rest": false,
1004
- "scenario": [
1005
- "chat",
1006
- "writing",
1007
- "office",
1008
- "collaboration",
1009
- "engineering"
1010
- ],
1011
- "interaction": "think"
1012
- },
1013
- {
1014
- "id": "unique-3",
1015
- "category": "unique-rows",
1016
- "actual": true,
1017
- "humanLabel": null,
1018
- "answer": "19",
1019
- "predictedCorrectness": 0.51278928136419,
1020
- "answerOverall": 0.5065686319907432,
1021
- "tokens": 11,
1022
- "rest": false,
1023
- "scenario": [
1024
- "chat",
1025
- "writing",
1026
- "office",
1027
- "collaboration",
1028
- "engineering"
1029
- ],
1030
- "interaction": "think"
1031
- },
1032
- {
1033
- "id": "unique-4",
1034
- "category": "unique-rows",
1035
- "actual": true,
1036
- "humanLabel": null,
1037
- "answer": "19",
1038
- "predictedCorrectness": 0.51278928136419,
1039
- "answerOverall": 0.5065686319907432,
1040
- "tokens": 11,
1041
- "rest": false,
1042
- "scenario": [
1043
- "chat",
1044
- "writing",
1045
- "office",
1046
- "collaboration",
1047
- "engineering"
1048
- ],
1049
- "interaction": "think"
1050
- },
1051
- {
1052
- "id": "unique-5",
1053
- "category": "unique-rows",
1054
- "actual": true,
1055
- "humanLabel": null,
1056
- "answer": "24",
1057
- "predictedCorrectness": 0.51278928136419,
1058
- "answerOverall": 0.5065686319907432,
1059
- "tokens": 11,
1060
- "rest": false,
1061
- "scenario": [
1062
- "chat",
1063
- "writing",
1064
- "office",
1065
- "collaboration",
1066
- "engineering"
1067
- ],
1068
- "interaction": "think"
1069
- },
1070
- {
1071
- "id": "unique-6",
1072
- "category": "unique-rows",
1073
- "actual": true,
1074
- "humanLabel": null,
1075
- "answer": "24",
1076
- "predictedCorrectness": 0.51278928136419,
1077
- "answerOverall": 0.5065686319907432,
1078
- "tokens": 11,
1079
- "rest": false,
1080
- "scenario": [
1081
- "chat",
1082
- "writing",
1083
- "office",
1084
- "collaboration",
1085
- "engineering"
1086
- ],
1087
- "interaction": "think"
1088
- },
1089
- {
1090
- "id": "unique-7",
1091
- "category": "unique-rows",
1092
- "actual": true,
1093
- "humanLabel": null,
1094
- "answer": "24",
1095
- "predictedCorrectness": 0.51278928136419,
1096
- "answerOverall": 0.5065686319907432,
1097
- "tokens": 11,
1098
- "rest": false,
1099
- "scenario": [
1100
- "chat",
1101
- "writing",
1102
- "office",
1103
- "collaboration",
1104
- "engineering"
1105
- ],
1106
- "interaction": "think"
1107
- },
1108
- {
1109
- "id": "unique-8",
1110
- "category": "unique-rows",
1111
- "actual": true,
1112
- "humanLabel": null,
1113
- "answer": "24",
1114
- "predictedCorrectness": 0.51278928136419,
1115
- "answerOverall": 0.5065686319907432,
1116
- "tokens": 11,
1117
- "rest": false,
1118
- "scenario": [
1119
- "chat",
1120
- "writing",
1121
- "office",
1122
- "collaboration",
1123
- "engineering"
1124
- ],
1125
- "interaction": "think"
1126
- },
1127
- {
1128
- "id": "unique-9",
1129
- "category": "unique-rows",
1130
- "actual": true,
1131
- "humanLabel": null,
1132
- "answer": "24",
1133
- "predictedCorrectness": 0.51278928136419,
1134
- "answerOverall": 0.5065686319907432,
1135
- "tokens": 11,
1136
- "rest": false,
1137
- "scenario": [
1138
- "chat",
1139
- "writing",
1140
- "office",
1141
- "collaboration",
1142
- "engineering"
1143
- ],
1144
- "interaction": "think"
1145
- },
1146
- {
1147
- "id": "unique-10",
1148
- "category": "unique-rows",
1149
- "actual": false,
1150
- "humanLabel": null,
1151
- "answer": "30",
1152
- "predictedCorrectness": 0.51278928136419,
1153
- "answerOverall": 0.5065686319907432,
1154
- "tokens": 11,
1155
- "rest": false,
1156
- "scenario": [
1157
- "chat",
1158
- "writing",
1159
- "office",
1160
- "collaboration",
1161
- "engineering"
1162
- ],
1163
- "interaction": "think"
1164
- },
1165
- {
1166
- "id": "unique-11",
1167
- "category": "unique-rows",
1168
- "actual": true,
1169
- "humanLabel": null,
1170
- "answer": "29",
1171
- "predictedCorrectness": 0.51278928136419,
1172
- "answerOverall": 0.5065686319907432,
1173
- "tokens": 11,
1174
- "rest": false,
1175
- "scenario": [
1176
- "chat",
1177
- "writing",
1178
- "office",
1179
- "collaboration",
1180
- "engineering"
1181
- ],
1182
- "interaction": "think"
1183
- },
1184
- {
1185
- "id": "unique-12",
1186
- "category": "unique-rows",
1187
- "actual": true,
1188
- "humanLabel": null,
1189
- "answer": "29",
1190
- "predictedCorrectness": 0.51278928136419,
1191
- "answerOverall": 0.5065686319907432,
1192
- "tokens": 11,
1193
- "rest": false,
1194
- "scenario": [
1195
- "chat",
1196
- "writing",
1197
- "office",
1198
- "collaboration",
1199
- "engineering"
1200
- ],
1201
- "interaction": "think"
1202
- },
1203
- {
1204
- "id": "unique-13",
1205
- "category": "unique-rows",
1206
- "actual": true,
1207
- "humanLabel": null,
1208
- "answer": "29",
1209
- "predictedCorrectness": 0.51278928136419,
1210
- "answerOverall": 0.5065686319907432,
1211
- "tokens": 11,
1212
- "rest": false,
1213
- "scenario": [
1214
- "chat",
1215
- "writing",
1216
- "office",
1217
- "collaboration",
1218
- "engineering"
1219
- ],
1220
- "interaction": "think"
1221
- },
1222
- {
1223
- "id": "unique-14",
1224
- "category": "unique-rows",
1225
- "actual": true,
1226
- "humanLabel": null,
1227
- "answer": "29",
1228
- "predictedCorrectness": 0.51278928136419,
1229
- "answerOverall": 0.5065686319907432,
1230
- "tokens": 11,
1231
- "rest": false,
1232
- "scenario": [
1233
- "chat",
1234
- "writing",
1235
- "office",
1236
- "collaboration",
1237
- "engineering"
1238
- ],
1239
- "interaction": "think"
1240
- },
1241
- {
1242
- "id": "unique-15",
1243
- "category": "unique-rows",
1244
- "actual": true,
1245
- "humanLabel": null,
1246
- "answer": "34",
1247
- "predictedCorrectness": 0.51278928136419,
1248
- "answerOverall": 0.5065686319907432,
1249
- "tokens": 11,
1250
- "rest": false,
1251
- "scenario": [
1252
- "chat",
1253
- "writing",
1254
- "office",
1255
- "collaboration",
1256
- "engineering"
1257
- ],
1258
- "interaction": "think"
1259
- },
1260
- {
1261
- "id": "unique-16",
1262
- "category": "unique-rows",
1263
- "actual": true,
1264
- "humanLabel": null,
1265
- "answer": "34",
1266
- "predictedCorrectness": 0.51278928136419,
1267
- "answerOverall": 0.5065686319907432,
1268
- "tokens": 11,
1269
- "rest": false,
1270
- "scenario": [
1271
- "chat",
1272
- "writing",
1273
- "office",
1274
- "collaboration",
1275
- "engineering"
1276
- ],
1277
- "interaction": "think"
1278
- },
1279
- {
1280
- "id": "unique-17",
1281
- "category": "unique-rows",
1282
- "actual": true,
1283
- "humanLabel": null,
1284
- "answer": "34",
1285
- "predictedCorrectness": 0.51278928136419,
1286
- "answerOverall": 0.5065686319907432,
1287
- "tokens": 11,
1288
- "rest": false,
1289
- "scenario": [
1290
- "chat",
1291
- "writing",
1292
- "office",
1293
- "collaboration",
1294
- "engineering"
1295
- ],
1296
- "interaction": "think"
1297
- },
1298
- {
1299
- "id": "unique-18",
1300
- "category": "unique-rows",
1301
- "actual": true,
1302
- "humanLabel": null,
1303
- "answer": "34",
1304
- "predictedCorrectness": 0.51278928136419,
1305
- "answerOverall": 0.5065686319907432,
1306
- "tokens": 11,
1307
- "rest": false,
1308
- "scenario": [
1309
- "chat",
1310
- "writing",
1311
- "office",
1312
- "collaboration",
1313
- "engineering"
1314
- ],
1315
- "interaction": "think"
1316
- },
1317
- {
1318
- "id": "unique-19",
1319
- "category": "unique-rows",
1320
- "actual": true,
1321
- "humanLabel": null,
1322
- "answer": "34",
1323
- "predictedCorrectness": 0.51278928136419,
1324
- "answerOverall": 0.5065686319907432,
1325
- "tokens": 11,
1326
- "rest": false,
1327
- "scenario": [
1328
- "chat",
1329
- "writing",
1330
- "office",
1331
- "collaboration",
1332
- "engineering"
1333
- ],
1334
- "interaction": "think"
1335
- },
1336
- {
1337
- "id": "unique-20",
1338
- "category": "unique-rows",
1339
- "actual": false,
1340
- "humanLabel": null,
1341
- "answer": "40",
1342
- "predictedCorrectness": 0.51278928136419,
1343
- "answerOverall": 0.5065686319907432,
1344
- "tokens": 11,
1345
- "rest": false,
1346
- "scenario": [
1347
- "chat",
1348
- "writing",
1349
- "office",
1350
- "collaboration",
1351
- "engineering"
1352
- ],
1353
- "interaction": "think"
1354
- },
1355
- {
1356
- "id": "unique-21",
1357
- "category": "unique-rows",
1358
- "actual": true,
1359
- "humanLabel": null,
1360
- "answer": "39",
1361
- "predictedCorrectness": 0.51278928136419,
1362
- "answerOverall": 0.5065686319907432,
1363
- "tokens": 11,
1364
- "rest": false,
1365
- "scenario": [
1366
- "chat",
1367
- "writing",
1368
- "office",
1369
- "collaboration",
1370
- "engineering"
1371
- ],
1372
- "interaction": "think"
1373
- },
1374
- {
1375
- "id": "unique-22",
1376
- "category": "unique-rows",
1377
- "actual": true,
1378
- "humanLabel": null,
1379
- "answer": "39",
1380
- "predictedCorrectness": 0.51278928136419,
1381
- "answerOverall": 0.5065686319907432,
1382
- "tokens": 11,
1383
- "rest": false,
1384
- "scenario": [
1385
- "chat",
1386
- "writing",
1387
- "office",
1388
- "collaboration",
1389
- "engineering"
1390
- ],
1391
- "interaction": "think"
1392
- },
1393
- {
1394
- "id": "unique-23",
1395
- "category": "unique-rows",
1396
- "actual": true,
1397
- "humanLabel": null,
1398
- "answer": "39",
1399
- "predictedCorrectness": 0.51278928136419,
1400
- "answerOverall": 0.5065686319907432,
1401
- "tokens": 11,
1402
- "rest": false,
1403
- "scenario": [
1404
- "chat",
1405
- "writing",
1406
- "office",
1407
- "collaboration",
1408
- "engineering"
1409
- ],
1410
- "interaction": "think"
1411
- },
1412
- {
1413
- "id": "unique-24",
1414
- "category": "unique-rows",
1415
- "actual": true,
1416
- "humanLabel": null,
1417
- "answer": "39",
1418
- "predictedCorrectness": 0.51278928136419,
1419
- "answerOverall": 0.5065686319907432,
1420
- "tokens": 11,
1421
- "rest": false,
1422
- "scenario": [
1423
- "chat",
1424
- "writing",
1425
- "office",
1426
- "collaboration",
1427
- "engineering"
1428
- ],
1429
- "interaction": "think"
1430
- },
1431
- {
1432
- "id": "unique-25",
1433
- "category": "unique-rows",
1434
- "actual": true,
1435
- "humanLabel": null,
1436
- "answer": "44",
1437
- "predictedCorrectness": 0.51278928136419,
1438
- "answerOverall": 0.5065686319907432,
1439
- "tokens": 11,
1440
- "rest": false,
1441
- "scenario": [
1442
- "chat",
1443
- "writing",
1444
- "office",
1445
- "collaboration",
1446
- "engineering"
1447
- ],
1448
- "interaction": "think"
1449
- },
1450
- {
1451
- "id": "unique-26",
1452
- "category": "unique-rows",
1453
- "actual": true,
1454
- "humanLabel": null,
1455
- "answer": "44",
1456
- "predictedCorrectness": 0.51278928136419,
1457
- "answerOverall": 0.5065686319907432,
1458
- "tokens": 11,
1459
- "rest": false,
1460
- "scenario": [
1461
- "chat",
1462
- "writing",
1463
- "office",
1464
- "collaboration",
1465
- "engineering"
1466
- ],
1467
- "interaction": "think"
1468
- },
1469
- {
1470
- "id": "unique-27",
1471
- "category": "unique-rows",
1472
- "actual": true,
1473
- "humanLabel": null,
1474
- "answer": "44",
1475
- "predictedCorrectness": 0.51278928136419,
1476
- "answerOverall": 0.5065686319907432,
1477
- "tokens": 11,
1478
- "rest": false,
1479
- "scenario": [
1480
- "chat",
1481
- "writing",
1482
- "office",
1483
- "collaboration",
1484
- "engineering"
1485
- ],
1486
- "interaction": "think"
1487
- },
1488
- {
1489
- "id": "unique-28",
1490
- "category": "unique-rows",
1491
- "actual": true,
1492
- "humanLabel": null,
1493
- "answer": "44",
1494
- "predictedCorrectness": 0.51278928136419,
1495
- "answerOverall": 0.5065686319907432,
1496
- "tokens": 11,
1497
- "rest": false,
1498
- "scenario": [
1499
- "chat",
1500
- "writing",
1501
- "office",
1502
- "collaboration",
1503
- "engineering"
1504
- ],
1505
- "interaction": "think"
1506
- },
1507
- {
1508
- "id": "unique-29",
1509
- "category": "unique-rows",
1510
- "actual": true,
1511
- "humanLabel": null,
1512
- "answer": "44",
1513
- "predictedCorrectness": 0.51278928136419,
1514
- "answerOverall": 0.5065686319907432,
1515
- "tokens": 11,
1516
- "rest": false,
1517
- "scenario": [
1518
- "chat",
1519
- "writing",
1520
- "office",
1521
- "collaboration",
1522
- "engineering"
1523
- ],
1524
- "interaction": "think"
1525
- },
1526
- {
1527
- "id": "unique-30",
1528
- "category": "unique-rows",
1529
- "actual": false,
1530
- "humanLabel": null,
1531
- "answer": "50",
1532
- "predictedCorrectness": 0.51278928136419,
1533
- "answerOverall": 0.5065686319907432,
1534
- "tokens": 11,
1535
- "rest": false,
1536
- "scenario": [
1537
- "chat",
1538
- "writing",
1539
- "office",
1540
- "collaboration",
1541
- "engineering"
1542
- ],
1543
- "interaction": "think"
1544
- },
1545
- {
1546
- "id": "unique-31",
1547
- "category": "unique-rows",
1548
- "actual": true,
1549
- "humanLabel": null,
1550
- "answer": "49",
1551
- "predictedCorrectness": 0.51278928136419,
1552
- "answerOverall": 0.5065686319907432,
1553
- "tokens": 11,
1554
- "rest": false,
1555
- "scenario": [
1556
- "chat",
1557
- "writing",
1558
- "office",
1559
- "collaboration",
1560
- "engineering"
1561
- ],
1562
- "interaction": "think"
1563
- },
1564
- {
1565
- "id": "unique-32",
1566
- "category": "unique-rows",
1567
- "actual": true,
1568
- "humanLabel": null,
1569
- "answer": "49",
1570
- "predictedCorrectness": 0.51278928136419,
1571
- "answerOverall": 0.5065686319907432,
1572
- "tokens": 11,
1573
- "rest": false,
1574
- "scenario": [
1575
- "chat",
1576
- "writing",
1577
- "office",
1578
- "collaboration",
1579
- "engineering"
1580
- ],
1581
- "interaction": "think"
1582
- },
1583
- {
1584
- "id": "unique-33",
1585
- "category": "unique-rows",
1586
- "actual": true,
1587
- "humanLabel": null,
1588
- "answer": "49",
1589
- "predictedCorrectness": 0.51278928136419,
1590
- "answerOverall": 0.5065686319907432,
1591
- "tokens": 11,
1592
- "rest": false,
1593
- "scenario": [
1594
- "chat",
1595
- "writing",
1596
- "office",
1597
- "collaboration",
1598
- "engineering"
1599
- ],
1600
- "interaction": "think"
1601
- },
1602
- {
1603
- "id": "unique-34",
1604
- "category": "unique-rows",
1605
- "actual": true,
1606
- "humanLabel": null,
1607
- "answer": "49",
1608
- "predictedCorrectness": 0.51278928136419,
1609
- "answerOverall": 0.5065686319907432,
1610
- "tokens": 11,
1611
- "rest": false,
1612
- "scenario": [
1613
- "chat",
1614
- "writing",
1615
- "office",
1616
- "collaboration",
1617
- "engineering"
1618
- ],
1619
- "interaction": "think"
1620
- },
1621
- {
1622
- "id": "unique-35",
1623
- "category": "unique-rows",
1624
- "actual": true,
1625
- "humanLabel": null,
1626
- "answer": "54",
1627
- "predictedCorrectness": 0.51278928136419,
1628
- "answerOverall": 0.5065686319907432,
1629
- "tokens": 11,
1630
- "rest": false,
1631
- "scenario": [
1632
- "chat",
1633
- "writing",
1634
- "office",
1635
- "collaboration",
1636
- "engineering"
1637
- ],
1638
- "interaction": "think"
1639
- },
1640
- {
1641
- "id": "unique-36",
1642
- "category": "unique-rows",
1643
- "actual": true,
1644
- "humanLabel": null,
1645
- "answer": "54",
1646
- "predictedCorrectness": 0.51278928136419,
1647
- "answerOverall": 0.5065686319907432,
1648
- "tokens": 11,
1649
- "rest": false,
1650
- "scenario": [
1651
- "chat",
1652
- "writing",
1653
- "office",
1654
- "collaboration",
1655
- "engineering"
1656
- ],
1657
- "interaction": "think"
1658
- },
1659
- {
1660
- "id": "unique-37",
1661
- "category": "unique-rows",
1662
- "actual": true,
1663
- "humanLabel": null,
1664
- "answer": "54",
1665
- "predictedCorrectness": 0.51278928136419,
1666
- "answerOverall": 0.5065686319907432,
1667
- "tokens": 11,
1668
- "rest": false,
1669
- "scenario": [
1670
- "chat",
1671
- "writing",
1672
- "office",
1673
- "collaboration",
1674
- "engineering"
1675
- ],
1676
- "interaction": "think"
1677
- },
1678
- {
1679
- "id": "unique-38",
1680
- "category": "unique-rows",
1681
- "actual": true,
1682
- "humanLabel": null,
1683
- "answer": "54",
1684
- "predictedCorrectness": 0.51278928136419,
1685
- "answerOverall": 0.5065686319907432,
1686
- "tokens": 11,
1687
- "rest": false,
1688
- "scenario": [
1689
- "chat",
1690
- "writing",
1691
- "office",
1692
- "collaboration",
1693
- "engineering"
1694
- ],
1695
- "interaction": "think"
1696
- },
1697
- {
1698
- "id": "unique-39",
1699
- "category": "unique-rows",
1700
- "actual": true,
1701
- "humanLabel": null,
1702
- "answer": "54",
1703
- "predictedCorrectness": 0.51278928136419,
1704
- "answerOverall": 0.5065686319907432,
1705
- "tokens": 11,
1706
- "rest": false,
1707
- "scenario": [
1708
- "chat",
1709
- "writing",
1710
- "office",
1711
- "collaboration",
1712
- "engineering"
1713
- ],
1714
- "interaction": "think"
1715
- },
1716
- {
1717
- "id": "syllogism-0",
1718
- "category": "syllogism",
1719
- "actual": false,
1720
- "humanLabel": null,
1721
- "answer": "yes",
1722
- "predictedCorrectness": 0.5085995085995085,
1723
- "answerOverall": 0.5035382723871261,
1724
- "tokens": 11,
1725
- "rest": false,
1726
- "scenario": [
1727
- "chat",
1728
- "writing",
1729
- "office",
1730
- "collaboration",
1731
- "engineering"
1732
- ],
1733
- "interaction": "think"
1734
- },
1735
- {
1736
- "id": "syllogism-1",
1737
- "category": "syllogism",
1738
- "actual": true,
1739
- "humanLabel": null,
1740
- "answer": "no",
1741
- "predictedCorrectness": 0.5085995085995085,
1742
- "answerOverall": 0.5035382723871261,
1743
- "tokens": 11,
1744
- "rest": false,
1745
- "scenario": [
1746
- "chat",
1747
- "writing",
1748
- "office",
1749
- "collaboration",
1750
- "engineering"
1751
- ],
1752
- "interaction": "think"
1753
- },
1754
- {
1755
- "id": "syllogism-2",
1756
- "category": "syllogism",
1757
- "actual": true,
1758
- "humanLabel": null,
1759
- "answer": "no",
1760
- "predictedCorrectness": 0.5085995085995085,
1761
- "answerOverall": 0.5035382723871261,
1762
- "tokens": 11,
1763
- "rest": false,
1764
- "scenario": [
1765
- "chat",
1766
- "writing",
1767
- "office",
1768
- "collaboration",
1769
- "engineering"
1770
- ],
1771
- "interaction": "think"
1772
- },
1773
- {
1774
- "id": "syllogism-3",
1775
- "category": "syllogism",
1776
- "actual": true,
1777
- "humanLabel": null,
1778
- "answer": "no",
1779
- "predictedCorrectness": 0.5085995085995085,
1780
- "answerOverall": 0.5035382723871261,
1781
- "tokens": 11,
1782
- "rest": false,
1783
- "scenario": [
1784
- "chat",
1785
- "writing",
1786
- "office",
1787
- "collaboration",
1788
- "engineering"
1789
- ],
1790
- "interaction": "think"
1791
- },
1792
- {
1793
- "id": "syllogism-4",
1794
- "category": "syllogism",
1795
- "actual": true,
1796
- "humanLabel": null,
1797
- "answer": "no",
1798
- "predictedCorrectness": 0.5085995085995085,
1799
- "answerOverall": 0.5035382723871261,
1800
- "tokens": 11,
1801
- "rest": false,
1802
- "scenario": [
1803
- "chat",
1804
- "writing",
1805
- "office",
1806
- "collaboration",
1807
- "engineering"
1808
- ],
1809
- "interaction": "think"
1810
- },
1811
- {
1812
- "id": "syllogism-5",
1813
- "category": "syllogism",
1814
- "actual": true,
1815
- "humanLabel": null,
1816
- "answer": "no",
1817
- "predictedCorrectness": 0.5085995085995085,
1818
- "answerOverall": 0.5035382723871261,
1819
- "tokens": 11,
1820
- "rest": false,
1821
- "scenario": [
1822
- "chat",
1823
- "writing",
1824
- "office",
1825
- "collaboration",
1826
- "engineering"
1827
- ],
1828
- "interaction": "think"
1829
- },
1830
- {
1831
- "id": "syllogism-6",
1832
- "category": "syllogism",
1833
- "actual": true,
1834
- "humanLabel": null,
1835
- "answer": "no",
1836
- "predictedCorrectness": 0.5085995085995085,
1837
- "answerOverall": 0.5035382723871261,
1838
- "tokens": 11,
1839
- "rest": false,
1840
- "scenario": [
1841
- "chat",
1842
- "writing",
1843
- "office",
1844
- "collaboration",
1845
- "engineering"
1846
- ],
1847
- "interaction": "think"
1848
- },
1849
- {
1850
- "id": "syllogism-7",
1851
- "category": "syllogism",
1852
- "actual": true,
1853
- "humanLabel": null,
1854
- "answer": "no",
1855
- "predictedCorrectness": 0.5085995085995085,
1856
- "answerOverall": 0.5035382723871261,
1857
- "tokens": 11,
1858
- "rest": false,
1859
- "scenario": [
1860
- "chat",
1861
- "writing",
1862
- "office",
1863
- "collaboration",
1864
- "engineering"
1865
- ],
1866
- "interaction": "think"
1867
- },
1868
- {
1869
- "id": "syllogism-8",
1870
- "category": "syllogism",
1871
- "actual": true,
1872
- "humanLabel": null,
1873
- "answer": "no",
1874
- "predictedCorrectness": 0.5085995085995085,
1875
- "answerOverall": 0.5035382723871261,
1876
- "tokens": 11,
1877
- "rest": false,
1878
- "scenario": [
1879
- "chat",
1880
- "writing",
1881
- "office",
1882
- "collaboration",
1883
- "engineering"
1884
- ],
1885
- "interaction": "think"
1886
- },
1887
- {
1888
- "id": "syllogism-9",
1889
- "category": "syllogism",
1890
- "actual": true,
1891
- "humanLabel": null,
1892
- "answer": "no",
1893
- "predictedCorrectness": 0.5085995085995085,
1894
- "answerOverall": 0.5035382723871261,
1895
- "tokens": 11,
1896
- "rest": false,
1897
- "scenario": [
1898
- "chat",
1899
- "writing",
1900
- "office",
1901
- "collaboration",
1902
- "engineering"
1903
- ],
1904
- "interaction": "think"
1905
- },
1906
- {
1907
- "id": "syllogism-10",
1908
- "category": "syllogism",
1909
- "actual": true,
1910
- "humanLabel": null,
1911
- "answer": "no",
1912
- "predictedCorrectness": 0.5085995085995085,
1913
- "answerOverall": 0.5035382723871261,
1914
- "tokens": 11,
1915
- "rest": false,
1916
- "scenario": [
1917
- "chat",
1918
- "writing",
1919
- "office",
1920
- "collaboration",
1921
- "engineering"
1922
- ],
1923
- "interaction": "think"
1924
- },
1925
- {
1926
- "id": "syllogism-11",
1927
- "category": "syllogism",
1928
- "actual": true,
1929
- "humanLabel": null,
1930
- "answer": "no",
1931
- "predictedCorrectness": 0.5085995085995085,
1932
- "answerOverall": 0.5035382723871261,
1933
- "tokens": 11,
1934
- "rest": false,
1935
- "scenario": [
1936
- "chat",
1937
- "writing",
1938
- "office",
1939
- "collaboration",
1940
- "engineering"
1941
- ],
1942
- "interaction": "think"
1943
- },
1944
- {
1945
- "id": "syllogism-12",
1946
- "category": "syllogism",
1947
- "actual": true,
1948
- "humanLabel": null,
1949
- "answer": "no",
1950
- "predictedCorrectness": 0.5085995085995085,
1951
- "answerOverall": 0.5035382723871261,
1952
- "tokens": 11,
1953
- "rest": false,
1954
- "scenario": [
1955
- "chat",
1956
- "writing",
1957
- "office",
1958
- "collaboration",
1959
- "engineering"
1960
- ],
1961
- "interaction": "think"
1962
- },
1963
- {
1964
- "id": "syllogism-13",
1965
- "category": "syllogism",
1966
- "actual": true,
1967
- "humanLabel": null,
1968
- "answer": "no",
1969
- "predictedCorrectness": 0.5085995085995085,
1970
- "answerOverall": 0.5035382723871261,
1971
- "tokens": 11,
1972
- "rest": false,
1973
- "scenario": [
1974
- "chat",
1975
- "writing",
1976
- "office",
1977
- "collaboration",
1978
- "engineering"
1979
- ],
1980
- "interaction": "think"
1981
- },
1982
- {
1983
- "id": "syllogism-14",
1984
- "category": "syllogism",
1985
- "actual": true,
1986
- "humanLabel": null,
1987
- "answer": "no",
1988
- "predictedCorrectness": 0.5085995085995085,
1989
- "answerOverall": 0.5035382723871261,
1990
- "tokens": 11,
1991
- "rest": false,
1992
- "scenario": [
1993
- "chat",
1994
- "writing",
1995
- "office",
1996
- "collaboration",
1997
- "engineering"
1998
- ],
1999
- "interaction": "think"
2000
- },
2001
- {
2002
- "id": "syllogism-15",
2003
- "category": "syllogism",
2004
- "actual": false,
2005
- "humanLabel": null,
2006
- "answer": "yes",
2007
- "predictedCorrectness": 0.5085995085995085,
2008
- "answerOverall": 0.5035382723871261,
2009
- "tokens": 11,
2010
- "rest": false,
2011
- "scenario": [
2012
- "chat",
2013
- "writing",
2014
- "office",
2015
- "collaboration",
2016
- "engineering"
2017
- ],
2018
- "interaction": "think"
2019
- },
2020
- {
2021
- "id": "syllogism-16",
2022
- "category": "syllogism",
2023
- "actual": true,
2024
- "humanLabel": null,
2025
- "answer": "no",
2026
- "predictedCorrectness": 0.5085995085995085,
2027
- "answerOverall": 0.5035382723871261,
2028
- "tokens": 11,
2029
- "rest": false,
2030
- "scenario": [
2031
- "chat",
2032
- "writing",
2033
- "office",
2034
- "collaboration",
2035
- "engineering"
2036
- ],
2037
- "interaction": "think"
2038
- },
2039
- {
2040
- "id": "syllogism-17",
2041
- "category": "syllogism",
2042
- "actual": true,
2043
- "humanLabel": null,
2044
- "answer": "no",
2045
- "predictedCorrectness": 0.5085995085995085,
2046
- "answerOverall": 0.5035382723871261,
2047
- "tokens": 11,
2048
- "rest": false,
2049
- "scenario": [
2050
- "chat",
2051
- "writing",
2052
- "office",
2053
- "collaboration",
2054
- "engineering"
2055
- ],
2056
- "interaction": "think"
2057
- },
2058
- {
2059
- "id": "syllogism-18",
2060
- "category": "syllogism",
2061
- "actual": true,
2062
- "humanLabel": null,
2063
- "answer": "no",
2064
- "predictedCorrectness": 0.5085995085995085,
2065
- "answerOverall": 0.5035382723871261,
2066
- "tokens": 11,
2067
- "rest": false,
2068
- "scenario": [
2069
- "chat",
2070
- "writing",
2071
- "office",
2072
- "collaboration",
2073
- "engineering"
2074
- ],
2075
- "interaction": "think"
2076
- },
2077
- {
2078
- "id": "syllogism-19",
2079
- "category": "syllogism",
2080
- "actual": true,
2081
- "humanLabel": null,
2082
- "answer": "no",
2083
- "predictedCorrectness": 0.5085995085995085,
2084
- "answerOverall": 0.5035382723871261,
2085
- "tokens": 11,
2086
- "rest": false,
2087
- "scenario": [
2088
- "chat",
2089
- "writing",
2090
- "office",
2091
- "collaboration",
2092
- "engineering"
2093
- ],
2094
- "interaction": "think"
2095
- },
2096
- {
2097
- "id": "syllogism-20",
2098
- "category": "syllogism",
2099
- "actual": true,
2100
- "humanLabel": null,
2101
- "answer": "no",
2102
- "predictedCorrectness": 0.5085995085995085,
2103
- "answerOverall": 0.5035382723871261,
2104
- "tokens": 11,
2105
- "rest": false,
2106
- "scenario": [
2107
- "chat",
2108
- "writing",
2109
- "office",
2110
- "collaboration",
2111
- "engineering"
2112
- ],
2113
- "interaction": "think"
2114
- },
2115
- {
2116
- "id": "syllogism-21",
2117
- "category": "syllogism",
2118
- "actual": true,
2119
- "humanLabel": null,
2120
- "answer": "no",
2121
- "predictedCorrectness": 0.5085995085995085,
2122
- "answerOverall": 0.5035382723871261,
2123
- "tokens": 11,
2124
- "rest": false,
2125
- "scenario": [
2126
- "chat",
2127
- "writing",
2128
- "office",
2129
- "collaboration",
2130
- "engineering"
2131
- ],
2132
- "interaction": "think"
2133
- },
2134
- {
2135
- "id": "syllogism-22",
2136
- "category": "syllogism",
2137
- "actual": true,
2138
- "humanLabel": null,
2139
- "answer": "no",
2140
- "predictedCorrectness": 0.5085995085995085,
2141
- "answerOverall": 0.5035382723871261,
2142
- "tokens": 11,
2143
- "rest": false,
2144
- "scenario": [
2145
- "chat",
2146
- "writing",
2147
- "office",
2148
- "collaboration",
2149
- "engineering"
2150
- ],
2151
- "interaction": "think"
2152
- },
2153
- {
2154
- "id": "syllogism-23",
2155
- "category": "syllogism",
2156
- "actual": true,
2157
- "humanLabel": null,
2158
- "answer": "no",
2159
- "predictedCorrectness": 0.5085995085995085,
2160
- "answerOverall": 0.5035382723871261,
2161
- "tokens": 11,
2162
- "rest": false,
2163
- "scenario": [
2164
- "chat",
2165
- "writing",
2166
- "office",
2167
- "collaboration",
2168
- "engineering"
2169
- ],
2170
- "interaction": "think"
2171
- },
2172
- {
2173
- "id": "syllogism-24",
2174
- "category": "syllogism",
2175
- "actual": true,
2176
- "humanLabel": null,
2177
- "answer": "no",
2178
- "predictedCorrectness": 0.5085995085995085,
2179
- "answerOverall": 0.5035382723871261,
2180
- "tokens": 11,
2181
- "rest": false,
2182
- "scenario": [
2183
- "chat",
2184
- "writing",
2185
- "office",
2186
- "collaboration",
2187
- "engineering"
2188
- ],
2189
- "interaction": "think"
2190
- },
2191
- {
2192
- "id": "syllogism-25",
2193
- "category": "syllogism",
2194
- "actual": true,
2195
- "humanLabel": null,
2196
- "answer": "no",
2197
- "predictedCorrectness": 0.5085995085995085,
2198
- "answerOverall": 0.5035382723871261,
2199
- "tokens": 11,
2200
- "rest": false,
2201
- "scenario": [
2202
- "chat",
2203
- "writing",
2204
- "office",
2205
- "collaboration",
2206
- "engineering"
2207
- ],
2208
- "interaction": "think"
2209
- },
2210
- {
2211
- "id": "syllogism-26",
2212
- "category": "syllogism",
2213
- "actual": true,
2214
- "humanLabel": null,
2215
- "answer": "no",
2216
- "predictedCorrectness": 0.5085995085995085,
2217
- "answerOverall": 0.5035382723871261,
2218
- "tokens": 11,
2219
- "rest": false,
2220
- "scenario": [
2221
- "chat",
2222
- "writing",
2223
- "office",
2224
- "collaboration",
2225
- "engineering"
2226
- ],
2227
- "interaction": "think"
2228
- },
2229
- {
2230
- "id": "syllogism-27",
2231
- "category": "syllogism",
2232
- "actual": true,
2233
- "humanLabel": null,
2234
- "answer": "no",
2235
- "predictedCorrectness": 0.5085995085995085,
2236
- "answerOverall": 0.5035382723871261,
2237
- "tokens": 11,
2238
- "rest": false,
2239
- "scenario": [
2240
- "chat",
2241
- "writing",
2242
- "office",
2243
- "collaboration",
2244
- "engineering"
2245
- ],
2246
- "interaction": "think"
2247
- },
2248
- {
2249
- "id": "syllogism-28",
2250
- "category": "syllogism",
2251
- "actual": true,
2252
- "humanLabel": null,
2253
- "answer": "no",
2254
- "predictedCorrectness": 0.5085995085995085,
2255
- "answerOverall": 0.5035382723871261,
2256
- "tokens": 11,
2257
- "rest": false,
2258
- "scenario": [
2259
- "chat",
2260
- "writing",
2261
- "office",
2262
- "collaboration",
2263
- "engineering"
2264
- ],
2265
- "interaction": "think"
2266
- },
2267
- {
2268
- "id": "syllogism-29",
2269
- "category": "syllogism",
2270
- "actual": true,
2271
- "humanLabel": null,
2272
- "answer": "no",
2273
- "predictedCorrectness": 0.5085995085995085,
2274
- "answerOverall": 0.5035382723871261,
2275
- "tokens": 11,
2276
- "rest": false,
2277
- "scenario": [
2278
- "chat",
2279
- "writing",
2280
- "office",
2281
- "collaboration",
2282
- "engineering"
2283
- ],
2284
- "interaction": "think"
2285
- },
2286
- {
2287
- "id": "memory-0",
2288
- "category": "memory",
2289
- "actual": false,
2290
- "humanLabel": null,
2291
- "answer": "code-1",
2292
- "predictedCorrectness": 0.51278928136419,
2293
- "answerOverall": 0.5065686319907432,
2294
- "tokens": 12,
2295
- "rest": false,
2296
- "scenario": [
2297
- "engineering"
2298
- ],
2299
- "interaction": "think"
2300
- },
2301
- {
2302
- "id": "memory-1",
2303
- "category": "memory",
2304
- "actual": true,
2305
- "humanLabel": null,
2306
- "answer": "code-1",
2307
- "predictedCorrectness": 0.51278928136419,
2308
- "answerOverall": 0.5065686319907432,
2309
- "tokens": 12,
2310
- "rest": false,
2311
- "scenario": [
2312
- "engineering"
2313
- ],
2314
- "interaction": "think"
2315
- },
2316
- {
2317
- "id": "memory-2",
2318
- "category": "memory",
2319
- "actual": true,
2320
- "humanLabel": null,
2321
- "answer": "code-2",
2322
- "predictedCorrectness": 0.51278928136419,
2323
- "answerOverall": 0.5065686319907432,
2324
- "tokens": 12,
2325
- "rest": false,
2326
- "scenario": [
2327
- "engineering"
2328
- ],
2329
- "interaction": "think"
2330
- },
2331
- {
2332
- "id": "memory-3",
2333
- "category": "memory",
2334
- "actual": true,
2335
- "humanLabel": null,
2336
- "answer": "code-3",
2337
- "predictedCorrectness": 0.51278928136419,
2338
- "answerOverall": 0.5065686319907432,
2339
- "tokens": 12,
2340
- "rest": false,
2341
- "scenario": [
2342
- "engineering"
2343
- ],
2344
- "interaction": "think"
2345
- },
2346
- {
2347
- "id": "memory-4",
2348
- "category": "memory",
2349
- "actual": true,
2350
- "humanLabel": null,
2351
- "answer": "code-4",
2352
- "predictedCorrectness": 0.51278928136419,
2353
- "answerOverall": 0.5065686319907432,
2354
- "tokens": 12,
2355
- "rest": false,
2356
- "scenario": [
2357
- "engineering"
2358
- ],
2359
- "interaction": "think"
2360
- },
2361
- {
2362
- "id": "memory-5",
2363
- "category": "memory",
2364
- "actual": true,
2365
- "humanLabel": null,
2366
- "answer": "code-5",
2367
- "predictedCorrectness": 0.51278928136419,
2368
- "answerOverall": 0.5065686319907432,
2369
- "tokens": 12,
2370
- "rest": false,
2371
- "scenario": [
2372
- "engineering"
2373
- ],
2374
- "interaction": "think"
2375
- },
2376
- {
2377
- "id": "memory-6",
2378
- "category": "memory",
2379
- "actual": true,
2380
- "humanLabel": null,
2381
- "answer": "code-6",
2382
- "predictedCorrectness": 0.51278928136419,
2383
- "answerOverall": 0.5065686319907432,
2384
- "tokens": 12,
2385
- "rest": false,
2386
- "scenario": [
2387
- "engineering"
2388
- ],
2389
- "interaction": "think"
2390
- },
2391
- {
2392
- "id": "memory-7",
2393
- "category": "memory",
2394
- "actual": true,
2395
- "humanLabel": null,
2396
- "answer": "code-7",
2397
- "predictedCorrectness": 0.51278928136419,
2398
- "answerOverall": 0.5065686319907432,
2399
- "tokens": 12,
2400
- "rest": false,
2401
- "scenario": [
2402
- "engineering"
2403
- ],
2404
- "interaction": "think"
2405
- },
2406
- {
2407
- "id": "memory-8",
2408
- "category": "memory",
2409
- "actual": true,
2410
- "humanLabel": null,
2411
- "answer": "code-8",
2412
- "predictedCorrectness": 0.51278928136419,
2413
- "answerOverall": 0.5065686319907432,
2414
- "tokens": 12,
2415
- "rest": false,
2416
- "scenario": [
2417
- "engineering"
2418
- ],
2419
- "interaction": "think"
2420
- },
2421
- {
2422
- "id": "memory-9",
2423
- "category": "memory",
2424
- "actual": true,
2425
- "humanLabel": null,
2426
- "answer": "code-9",
2427
- "predictedCorrectness": 0.51278928136419,
2428
- "answerOverall": 0.5065686319907432,
2429
- "tokens": 12,
2430
- "rest": false,
2431
- "scenario": [
2432
- "engineering"
2433
- ],
2434
- "interaction": "think"
2435
- },
2436
- {
2437
- "id": "memory-10",
2438
- "category": "memory",
2439
- "actual": false,
2440
- "humanLabel": null,
2441
- "answer": "code-11",
2442
- "predictedCorrectness": 0.51278928136419,
2443
- "answerOverall": 0.5065686319907432,
2444
- "tokens": 12,
2445
- "rest": false,
2446
- "scenario": [
2447
- "engineering"
2448
- ],
2449
- "interaction": "think"
2450
- },
2451
- {
2452
- "id": "memory-11",
2453
- "category": "memory",
2454
- "actual": true,
2455
- "humanLabel": null,
2456
- "answer": "code-11",
2457
- "predictedCorrectness": 0.51278928136419,
2458
- "answerOverall": 0.5065686319907432,
2459
- "tokens": 12,
2460
- "rest": false,
2461
- "scenario": [
2462
- "engineering"
2463
- ],
2464
- "interaction": "think"
2465
- },
2466
- {
2467
- "id": "memory-12",
2468
- "category": "memory",
2469
- "actual": true,
2470
- "humanLabel": null,
2471
- "answer": "code-12",
2472
- "predictedCorrectness": 0.51278928136419,
2473
- "answerOverall": 0.5065686319907432,
2474
- "tokens": 12,
2475
- "rest": false,
2476
- "scenario": [
2477
- "engineering"
2478
- ],
2479
- "interaction": "think"
2480
- },
2481
- {
2482
- "id": "memory-13",
2483
- "category": "memory",
2484
- "actual": true,
2485
- "humanLabel": null,
2486
- "answer": "code-13",
2487
- "predictedCorrectness": 0.51278928136419,
2488
- "answerOverall": 0.5065686319907432,
2489
- "tokens": 12,
2490
- "rest": false,
2491
- "scenario": [
2492
- "engineering"
2493
- ],
2494
- "interaction": "think"
2495
- },
2496
- {
2497
- "id": "memory-14",
2498
- "category": "memory",
2499
- "actual": true,
2500
- "humanLabel": null,
2501
- "answer": "code-14",
2502
- "predictedCorrectness": 0.51278928136419,
2503
- "answerOverall": 0.5065686319907432,
2504
- "tokens": 12,
2505
- "rest": false,
2506
- "scenario": [
2507
- "engineering"
2508
- ],
2509
- "interaction": "think"
2510
- },
2511
- {
2512
- "id": "memory-15",
2513
- "category": "memory",
2514
- "actual": true,
2515
- "humanLabel": null,
2516
- "answer": "code-15",
2517
- "predictedCorrectness": 0.51278928136419,
2518
- "answerOverall": 0.5065686319907432,
2519
- "tokens": 12,
2520
- "rest": false,
2521
- "scenario": [
2522
- "engineering"
2523
- ],
2524
- "interaction": "think"
2525
- },
2526
- {
2527
- "id": "memory-16",
2528
- "category": "memory",
2529
- "actual": true,
2530
- "humanLabel": null,
2531
- "answer": "code-16",
2532
- "predictedCorrectness": 0.51278928136419,
2533
- "answerOverall": 0.5065686319907432,
2534
- "tokens": 12,
2535
- "rest": false,
2536
- "scenario": [
2537
- "engineering"
2538
- ],
2539
- "interaction": "think"
2540
- },
2541
- {
2542
- "id": "memory-17",
2543
- "category": "memory",
2544
- "actual": true,
2545
- "humanLabel": null,
2546
- "answer": "code-17",
2547
- "predictedCorrectness": 0.51278928136419,
2548
- "answerOverall": 0.5065686319907432,
2549
- "tokens": 12,
2550
- "rest": false,
2551
- "scenario": [
2552
- "engineering"
2553
- ],
2554
- "interaction": "think"
2555
- },
2556
- {
2557
- "id": "memory-18",
2558
- "category": "memory",
2559
- "actual": true,
2560
- "humanLabel": null,
2561
- "answer": "code-18",
2562
- "predictedCorrectness": 0.51278928136419,
2563
- "answerOverall": 0.5065686319907432,
2564
- "tokens": 12,
2565
- "rest": false,
2566
- "scenario": [
2567
- "engineering"
2568
- ],
2569
- "interaction": "think"
2570
- },
2571
- {
2572
- "id": "memory-19",
2573
- "category": "memory",
2574
- "actual": true,
2575
- "humanLabel": null,
2576
- "answer": "code-19",
2577
- "predictedCorrectness": 0.51278928136419,
2578
- "answerOverall": 0.5065686319907432,
2579
- "tokens": 12,
2580
- "rest": false,
2581
- "scenario": [
2582
- "engineering"
2583
- ],
2584
- "interaction": "think"
2585
- },
2586
- {
2587
- "id": "memory-20",
2588
- "category": "memory",
2589
- "actual": false,
2590
- "humanLabel": null,
2591
- "answer": "code-21",
2592
- "predictedCorrectness": 0.51278928136419,
2593
- "answerOverall": 0.5065686319907432,
2594
- "tokens": 12,
2595
- "rest": false,
2596
- "scenario": [
2597
- "engineering"
2598
- ],
2599
- "interaction": "think"
2600
- },
2601
- {
2602
- "id": "memory-21",
2603
- "category": "memory",
2604
- "actual": true,
2605
- "humanLabel": null,
2606
- "answer": "code-21",
2607
- "predictedCorrectness": 0.51278928136419,
2608
- "answerOverall": 0.5065686319907432,
2609
- "tokens": 12,
2610
- "rest": false,
2611
- "scenario": [
2612
- "engineering"
2613
- ],
2614
- "interaction": "think"
2615
- },
2616
- {
2617
- "id": "memory-22",
2618
- "category": "memory",
2619
- "actual": true,
2620
- "humanLabel": null,
2621
- "answer": "code-22",
2622
- "predictedCorrectness": 0.51278928136419,
2623
- "answerOverall": 0.5065686319907432,
2624
- "tokens": 12,
2625
- "rest": false,
2626
- "scenario": [
2627
- "engineering"
2628
- ],
2629
- "interaction": "think"
2630
- },
2631
- {
2632
- "id": "memory-23",
2633
- "category": "memory",
2634
- "actual": true,
2635
- "humanLabel": null,
2636
- "answer": "code-23",
2637
- "predictedCorrectness": 0.51278928136419,
2638
- "answerOverall": 0.5065686319907432,
2639
- "tokens": 12,
2640
- "rest": false,
2641
- "scenario": [
2642
- "engineering"
2643
- ],
2644
- "interaction": "think"
2645
- },
2646
- {
2647
- "id": "memory-24",
2648
- "category": "memory",
2649
- "actual": true,
2650
- "humanLabel": null,
2651
- "answer": "code-24",
2652
- "predictedCorrectness": 0.51278928136419,
2653
- "answerOverall": 0.5065686319907432,
2654
- "tokens": 12,
2655
- "rest": false,
2656
- "scenario": [
2657
- "engineering"
2658
- ],
2659
- "interaction": "think"
2660
- },
2661
- {
2662
- "id": "memory-25",
2663
- "category": "memory",
2664
- "actual": true,
2665
- "humanLabel": null,
2666
- "answer": "code-25",
2667
- "predictedCorrectness": 0.51278928136419,
2668
- "answerOverall": 0.5065686319907432,
2669
- "tokens": 12,
2670
- "rest": false,
2671
- "scenario": [
2672
- "engineering"
2673
- ],
2674
- "interaction": "think"
2675
- },
2676
- {
2677
- "id": "memory-26",
2678
- "category": "memory",
2679
- "actual": true,
2680
- "humanLabel": null,
2681
- "answer": "code-26",
2682
- "predictedCorrectness": 0.51278928136419,
2683
- "answerOverall": 0.5065686319907432,
2684
- "tokens": 12,
2685
- "rest": false,
2686
- "scenario": [
2687
- "engineering"
2688
- ],
2689
- "interaction": "think"
2690
- },
2691
- {
2692
- "id": "memory-27",
2693
- "category": "memory",
2694
- "actual": true,
2695
- "humanLabel": null,
2696
- "answer": "code-27",
2697
- "predictedCorrectness": 0.51278928136419,
2698
- "answerOverall": 0.5065686319907432,
2699
- "tokens": 12,
2700
- "rest": false,
2701
- "scenario": [
2702
- "engineering"
2703
- ],
2704
- "interaction": "think"
2705
- },
2706
- {
2707
- "id": "memory-28",
2708
- "category": "memory",
2709
- "actual": true,
2710
- "humanLabel": null,
2711
- "answer": "code-28",
2712
- "predictedCorrectness": 0.51278928136419,
2713
- "answerOverall": 0.5065686319907432,
2714
- "tokens": 12,
2715
- "rest": false,
2716
- "scenario": [
2717
- "engineering"
2718
- ],
2719
- "interaction": "think"
2720
- },
2721
- {
2722
- "id": "memory-29",
2723
- "category": "memory",
2724
- "actual": true,
2725
- "humanLabel": null,
2726
- "answer": "code-29",
2727
- "predictedCorrectness": 0.51278928136419,
2728
- "answerOverall": 0.5065686319907432,
2729
- "tokens": 12,
2730
- "rest": false,
2731
- "scenario": [
2732
- "engineering"
2733
- ],
2734
- "interaction": "think"
2735
- },
2736
- {
2737
- "id": "role-0",
2738
- "category": "role",
2739
- "actual": true,
2740
- "humanLabel": null,
2741
- "answer": "Analyst; ground claims in evidence",
2742
- "predictedCorrectness": 0.5085995085995085,
2743
- "answerOverall": 0.5072594599164431,
2744
- "tokens": 19,
2745
- "rest": false,
2746
- "scenario": [
2747
- "chat",
2748
- "writing",
2749
- "office",
2750
- "collaboration",
2751
- "engineering"
2752
- ],
2753
- "interaction": "think"
2754
- },
2755
- {
2756
- "id": "role-1",
2757
- "category": "role",
2758
- "actual": true,
2759
- "humanLabel": null,
2760
- "answer": "Analyst; ground claims in evidence",
2761
- "predictedCorrectness": 0.5085995085995085,
2762
- "answerOverall": 0.5072594599164431,
2763
- "tokens": 19,
2764
- "rest": false,
2765
- "scenario": [
2766
- "chat",
2767
- "writing",
2768
- "office",
2769
- "collaboration",
2770
- "engineering"
2771
- ],
2772
- "interaction": "think"
2773
- },
2774
- {
2775
- "id": "role-2",
2776
- "category": "role",
2777
- "actual": true,
2778
- "humanLabel": null,
2779
- "answer": "Analyst; ground claims in evidence",
2780
- "predictedCorrectness": 0.5085995085995085,
2781
- "answerOverall": 0.5072594599164431,
2782
- "tokens": 19,
2783
- "rest": false,
2784
- "scenario": [
2785
- "chat",
2786
- "writing",
2787
- "office",
2788
- "collaboration",
2789
- "engineering"
2790
- ],
2791
- "interaction": "think"
2792
- },
2793
- {
2794
- "id": "role-3",
2795
- "category": "role",
2796
- "actual": true,
2797
- "humanLabel": null,
2798
- "answer": "Analyst; ground claims in evidence",
2799
- "predictedCorrectness": 0.5085995085995085,
2800
- "answerOverall": 0.5072594599164431,
2801
- "tokens": 19,
2802
- "rest": false,
2803
- "scenario": [
2804
- "chat",
2805
- "writing",
2806
- "office",
2807
- "collaboration",
2808
- "engineering"
2809
- ],
2810
- "interaction": "think"
2811
- },
2812
- {
2813
- "id": "role-4",
2814
- "category": "role",
2815
- "actual": true,
2816
- "humanLabel": null,
2817
- "answer": "Analyst; ground claims in evidence",
2818
- "predictedCorrectness": 0.5085995085995085,
2819
- "answerOverall": 0.5072594599164431,
2820
- "tokens": 19,
2821
- "rest": false,
2822
- "scenario": [
2823
- "chat",
2824
- "writing",
2825
- "office",
2826
- "collaboration",
2827
- "engineering"
2828
- ],
2829
- "interaction": "think"
2830
- },
2831
- {
2832
- "id": "role-5",
2833
- "category": "role",
2834
- "actual": true,
2835
- "humanLabel": null,
2836
- "answer": "Analyst; ground claims in evidence",
2837
- "predictedCorrectness": 0.5085995085995085,
2838
- "answerOverall": 0.5072594599164431,
2839
- "tokens": 19,
2840
- "rest": false,
2841
- "scenario": [
2842
- "chat",
2843
- "writing",
2844
- "office",
2845
- "collaboration",
2846
- "engineering"
2847
- ],
2848
- "interaction": "think"
2849
- },
2850
- {
2851
- "id": "role-6",
2852
- "category": "role",
2853
- "actual": true,
2854
- "humanLabel": null,
2855
- "answer": "Analyst; ground claims in evidence",
2856
- "predictedCorrectness": 0.5085995085995085,
2857
- "answerOverall": 0.5072594599164431,
2858
- "tokens": 19,
2859
- "rest": false,
2860
- "scenario": [
2861
- "chat",
2862
- "writing",
2863
- "office",
2864
- "collaboration",
2865
- "engineering"
2866
- ],
2867
- "interaction": "think"
2868
- },
2869
- {
2870
- "id": "role-7",
2871
- "category": "role",
2872
- "actual": true,
2873
- "humanLabel": null,
2874
- "answer": "Analyst; ground claims in evidence",
2875
- "predictedCorrectness": 0.5085995085995085,
2876
- "answerOverall": 0.5072594599164431,
2877
- "tokens": 19,
2878
- "rest": false,
2879
- "scenario": [
2880
- "chat",
2881
- "writing",
2882
- "office",
2883
- "collaboration",
2884
- "engineering"
2885
- ],
2886
- "interaction": "think"
2887
- },
2888
- {
2889
- "id": "role-8",
2890
- "category": "role",
2891
- "actual": true,
2892
- "humanLabel": null,
2893
- "answer": "Analyst; ground claims in evidence",
2894
- "predictedCorrectness": 0.5085995085995085,
2895
- "answerOverall": 0.5072594599164431,
2896
- "tokens": 19,
2897
- "rest": false,
2898
- "scenario": [
2899
- "chat",
2900
- "writing",
2901
- "office",
2902
- "collaboration",
2903
- "engineering"
2904
- ],
2905
- "interaction": "think"
2906
- },
2907
- {
2908
- "id": "role-9",
2909
- "category": "role",
2910
- "actual": true,
2911
- "humanLabel": null,
2912
- "answer": "Analyst; ground claims in evidence",
2913
- "predictedCorrectness": 0.5085995085995085,
2914
- "answerOverall": 0.5072594599164431,
2915
- "tokens": 19,
2916
- "rest": false,
2917
- "scenario": [
2918
- "chat",
2919
- "writing",
2920
- "office",
2921
- "collaboration",
2922
- "engineering"
2923
- ],
2924
- "interaction": "think"
2925
- },
2926
- {
2927
- "id": "role-10",
2928
- "category": "role",
2929
- "actual": true,
2930
- "humanLabel": null,
2931
- "answer": "Analyst; ground claims in evidence",
2932
- "predictedCorrectness": 0.5085995085995085,
2933
- "answerOverall": 0.5072594599164431,
2934
- "tokens": 19,
2935
- "rest": false,
2936
- "scenario": [
2937
- "chat",
2938
- "writing",
2939
- "office",
2940
- "collaboration",
2941
- "engineering"
2942
- ],
2943
- "interaction": "think"
2944
- },
2945
- {
2946
- "id": "role-11",
2947
- "category": "role",
2948
- "actual": true,
2949
- "humanLabel": null,
2950
- "answer": "Analyst; ground claims in evidence",
2951
- "predictedCorrectness": 0.5085995085995085,
2952
- "answerOverall": 0.5072594599164431,
2953
- "tokens": 19,
2954
- "rest": false,
2955
- "scenario": [
2956
- "chat",
2957
- "writing",
2958
- "office",
2959
- "collaboration",
2960
- "engineering"
2961
- ],
2962
- "interaction": "think"
2963
- },
2964
- {
2965
- "id": "role-12",
2966
- "category": "role",
2967
- "actual": true,
2968
- "humanLabel": null,
2969
- "answer": "Analyst; ground claims in evidence",
2970
- "predictedCorrectness": 0.5085995085995085,
2971
- "answerOverall": 0.5072594599164431,
2972
- "tokens": 19,
2973
- "rest": false,
2974
- "scenario": [
2975
- "chat",
2976
- "writing",
2977
- "office",
2978
- "collaboration",
2979
- "engineering"
2980
- ],
2981
- "interaction": "think"
2982
- },
2983
- {
2984
- "id": "role-13",
2985
- "category": "role",
2986
- "actual": true,
2987
- "humanLabel": null,
2988
- "answer": "Analyst; ground claims in evidence",
2989
- "predictedCorrectness": 0.5085995085995085,
2990
- "answerOverall": 0.5072594599164431,
2991
- "tokens": 19,
2992
- "rest": false,
2993
- "scenario": [
2994
- "chat",
2995
- "writing",
2996
- "office",
2997
- "collaboration",
2998
- "engineering"
2999
- ],
3000
- "interaction": "think"
3001
- },
3002
- {
3003
- "id": "role-14",
3004
- "category": "role",
3005
- "actual": true,
3006
- "humanLabel": null,
3007
- "answer": "Analyst; ground claims in evidence",
3008
- "predictedCorrectness": 0.5085995085995085,
3009
- "answerOverall": 0.5072594599164431,
3010
- "tokens": 19,
3011
- "rest": false,
3012
- "scenario": [
3013
- "chat",
3014
- "writing",
3015
- "office",
3016
- "collaboration",
3017
- "engineering"
3018
- ],
3019
- "interaction": "think"
3020
- },
3021
- {
3022
- "id": "role-15",
3023
- "category": "role",
3024
- "actual": true,
3025
- "humanLabel": null,
3026
- "answer": "Analyst; ground claims in evidence",
3027
- "predictedCorrectness": 0.5085995085995085,
3028
- "answerOverall": 0.5072594599164431,
3029
- "tokens": 19,
3030
- "rest": false,
3031
- "scenario": [
3032
- "chat",
3033
- "writing",
3034
- "office",
3035
- "collaboration",
3036
- "engineering"
3037
- ],
3038
- "interaction": "think"
3039
- },
3040
- {
3041
- "id": "role-16",
3042
- "category": "role",
3043
- "actual": true,
3044
- "humanLabel": null,
3045
- "answer": "Analyst; ground claims in evidence",
3046
- "predictedCorrectness": 0.5085995085995085,
3047
- "answerOverall": 0.5072594599164431,
3048
- "tokens": 19,
3049
- "rest": false,
3050
- "scenario": [
3051
- "chat",
3052
- "writing",
3053
- "office",
3054
- "collaboration",
3055
- "engineering"
3056
- ],
3057
- "interaction": "think"
3058
- },
3059
- {
3060
- "id": "role-17",
3061
- "category": "role",
3062
- "actual": true,
3063
- "humanLabel": null,
3064
- "answer": "Analyst; ground claims in evidence",
3065
- "predictedCorrectness": 0.5085995085995085,
3066
- "answerOverall": 0.5072594599164431,
3067
- "tokens": 19,
3068
- "rest": false,
3069
- "scenario": [
3070
- "chat",
3071
- "writing",
3072
- "office",
3073
- "collaboration",
3074
- "engineering"
3075
- ],
3076
- "interaction": "think"
3077
- },
3078
- {
3079
- "id": "role-18",
3080
- "category": "role",
3081
- "actual": true,
3082
- "humanLabel": null,
3083
- "answer": "Analyst; ground claims in evidence",
3084
- "predictedCorrectness": 0.5085995085995085,
3085
- "answerOverall": 0.5072594599164431,
3086
- "tokens": 19,
3087
- "rest": false,
3088
- "scenario": [
3089
- "chat",
3090
- "writing",
3091
- "office",
3092
- "collaboration",
3093
- "engineering"
3094
- ],
3095
- "interaction": "think"
3096
- },
3097
- {
3098
- "id": "role-19",
3099
- "category": "role",
3100
- "actual": true,
3101
- "humanLabel": null,
3102
- "answer": "Analyst; ground claims in evidence",
3103
- "predictedCorrectness": 0.5085995085995085,
3104
- "answerOverall": 0.5072594599164431,
3105
- "tokens": 19,
3106
- "rest": false,
3107
- "scenario": [
3108
- "chat",
3109
- "writing",
3110
- "office",
3111
- "collaboration",
3112
- "engineering"
3113
- ],
3114
- "interaction": "think"
3115
- },
3116
- {
3117
- "id": "role-20",
3118
- "category": "role",
3119
- "actual": true,
3120
- "humanLabel": null,
3121
- "answer": "Analyst; ground claims in evidence",
3122
- "predictedCorrectness": 0.5085995085995085,
3123
- "answerOverall": 0.5072594599164431,
3124
- "tokens": 19,
3125
- "rest": false,
3126
- "scenario": [
3127
- "chat",
3128
- "writing",
3129
- "office",
3130
- "collaboration",
3131
- "engineering"
3132
- ],
3133
- "interaction": "think"
3134
- },
3135
- {
3136
- "id": "role-21",
3137
- "category": "role",
3138
- "actual": true,
3139
- "humanLabel": null,
3140
- "answer": "Analyst; ground claims in evidence",
3141
- "predictedCorrectness": 0.5085995085995085,
3142
- "answerOverall": 0.5072594599164431,
3143
- "tokens": 19,
3144
- "rest": false,
3145
- "scenario": [
3146
- "chat",
3147
- "writing",
3148
- "office",
3149
- "collaboration",
3150
- "engineering"
3151
- ],
3152
- "interaction": "think"
3153
- },
3154
- {
3155
- "id": "role-22",
3156
- "category": "role",
3157
- "actual": true,
3158
- "humanLabel": null,
3159
- "answer": "Analyst; ground claims in evidence",
3160
- "predictedCorrectness": 0.5085995085995085,
3161
- "answerOverall": 0.5072594599164431,
3162
- "tokens": 19,
3163
- "rest": false,
3164
- "scenario": [
3165
- "chat",
3166
- "writing",
3167
- "office",
3168
- "collaboration",
3169
- "engineering"
3170
- ],
3171
- "interaction": "think"
3172
- },
3173
- {
3174
- "id": "role-23",
3175
- "category": "role",
3176
- "actual": true,
3177
- "humanLabel": null,
3178
- "answer": "Analyst; ground claims in evidence",
3179
- "predictedCorrectness": 0.5085995085995085,
3180
- "answerOverall": 0.5072594599164431,
3181
- "tokens": 19,
3182
- "rest": false,
3183
- "scenario": [
3184
- "chat",
3185
- "writing",
3186
- "office",
3187
- "collaboration",
3188
- "engineering"
3189
- ],
3190
- "interaction": "think"
3191
- },
3192
- {
3193
- "id": "role-24",
3194
- "category": "role",
3195
- "actual": true,
3196
- "humanLabel": null,
3197
- "answer": "Analyst; ground claims in evidence",
3198
- "predictedCorrectness": 0.5085995085995085,
3199
- "answerOverall": 0.5072594599164431,
3200
- "tokens": 19,
3201
- "rest": false,
3202
- "scenario": [
3203
- "chat",
3204
- "writing",
3205
- "office",
3206
- "collaboration",
3207
- "engineering"
3208
- ],
3209
- "interaction": "think"
3210
- },
3211
- {
3212
- "id": "role-25",
3213
- "category": "role",
3214
- "actual": true,
3215
- "humanLabel": null,
3216
- "answer": "Analyst; ground claims in evidence",
3217
- "predictedCorrectness": 0.5085995085995085,
3218
- "answerOverall": 0.5072594599164431,
3219
- "tokens": 19,
3220
- "rest": false,
3221
- "scenario": [
3222
- "chat",
3223
- "writing",
3224
- "office",
3225
- "collaboration",
3226
- "engineering"
3227
- ],
3228
- "interaction": "think"
3229
- },
3230
- {
3231
- "id": "role-26",
3232
- "category": "role",
3233
- "actual": true,
3234
- "humanLabel": null,
3235
- "answer": "Analyst; ground claims in evidence",
3236
- "predictedCorrectness": 0.5085995085995085,
3237
- "answerOverall": 0.5072594599164431,
3238
- "tokens": 19,
3239
- "rest": false,
3240
- "scenario": [
3241
- "chat",
3242
- "writing",
3243
- "office",
3244
- "collaboration",
3245
- "engineering"
3246
- ],
3247
- "interaction": "think"
3248
- },
3249
- {
3250
- "id": "role-27",
3251
- "category": "role",
3252
- "actual": true,
3253
- "humanLabel": null,
3254
- "answer": "Analyst; ground claims in evidence",
3255
- "predictedCorrectness": 0.5085995085995085,
3256
- "answerOverall": 0.5072594599164431,
3257
- "tokens": 19,
3258
- "rest": false,
3259
- "scenario": [
3260
- "chat",
3261
- "writing",
3262
- "office",
3263
- "collaboration",
3264
- "engineering"
3265
- ],
3266
- "interaction": "think"
3267
- },
3268
- {
3269
- "id": "role-28",
3270
- "category": "role",
3271
- "actual": true,
3272
- "humanLabel": null,
3273
- "answer": "Analyst; ground claims in evidence",
3274
- "predictedCorrectness": 0.5085995085995085,
3275
- "answerOverall": 0.5072594599164431,
3276
- "tokens": 19,
3277
- "rest": false,
3278
- "scenario": [
3279
- "chat",
3280
- "writing",
3281
- "office",
3282
- "collaboration",
3283
- "engineering"
3284
- ],
3285
- "interaction": "think"
3286
- },
3287
- {
3288
- "id": "role-29",
3289
- "category": "role",
3290
- "actual": true,
3291
- "humanLabel": null,
3292
- "answer": "Analyst; ground claims in evidence",
3293
- "predictedCorrectness": 0.5085995085995085,
3294
- "answerOverall": 0.5072594599164431,
3295
- "tokens": 19,
3296
- "rest": false,
3297
- "scenario": [
3298
- "chat",
3299
- "writing",
3300
- "office",
3301
- "collaboration",
3302
- "engineering"
3303
- ],
3304
- "interaction": "think"
3305
- },
3306
- {
3307
- "id": "attention-0",
3308
- "category": "attention",
3309
- "actual": true,
3310
- "humanLabel": null,
3311
- "answer": "2 foreground slots; safety floor 1",
3312
- "predictedCorrectness": 0.51278928136419,
3313
- "answerOverall": 0.512225694417802,
3314
- "tokens": 19,
3315
- "rest": false,
3316
- "scenario": [
3317
- "chat",
3318
- "writing",
3319
- "office",
3320
- "collaboration",
3321
- "engineering"
3322
- ],
3323
- "interaction": "think"
3324
- },
3325
- {
3326
- "id": "attention-1",
3327
- "category": "attention",
3328
- "actual": true,
3329
- "humanLabel": null,
3330
- "answer": "2 foreground slots; safety floor 1",
3331
- "predictedCorrectness": 0.51278928136419,
3332
- "answerOverall": 0.512225694417802,
3333
- "tokens": 19,
3334
- "rest": false,
3335
- "scenario": [
3336
- "chat",
3337
- "writing",
3338
- "office",
3339
- "collaboration",
3340
- "engineering"
3341
- ],
3342
- "interaction": "think"
3343
- },
3344
- {
3345
- "id": "attention-2",
3346
- "category": "attention",
3347
- "actual": true,
3348
- "humanLabel": null,
3349
- "answer": "2 foreground slots; safety floor 1",
3350
- "predictedCorrectness": 0.51278928136419,
3351
- "answerOverall": 0.512225694417802,
3352
- "tokens": 19,
3353
- "rest": false,
3354
- "scenario": [
3355
- "chat",
3356
- "writing",
3357
- "office",
3358
- "collaboration",
3359
- "engineering"
3360
- ],
3361
- "interaction": "think"
3362
- },
3363
- {
3364
- "id": "attention-3",
3365
- "category": "attention",
3366
- "actual": true,
3367
- "humanLabel": null,
3368
- "answer": "2 foreground slots; safety floor 1",
3369
- "predictedCorrectness": 0.51278928136419,
3370
- "answerOverall": 0.512225694417802,
3371
- "tokens": 19,
3372
- "rest": false,
3373
- "scenario": [
3374
- "chat",
3375
- "writing",
3376
- "office",
3377
- "collaboration",
3378
- "engineering"
3379
- ],
3380
- "interaction": "think"
3381
- },
3382
- {
3383
- "id": "attention-4",
3384
- "category": "attention",
3385
- "actual": true,
3386
- "humanLabel": null,
3387
- "answer": "2 foreground slots; safety floor 1",
3388
- "predictedCorrectness": 0.51278928136419,
3389
- "answerOverall": 0.512225694417802,
3390
- "tokens": 19,
3391
- "rest": false,
3392
- "scenario": [
3393
- "chat",
3394
- "writing",
3395
- "office",
3396
- "collaboration",
3397
- "engineering"
3398
- ],
3399
- "interaction": "think"
3400
- },
3401
- {
3402
- "id": "attention-5",
3403
- "category": "attention",
3404
- "actual": true,
3405
- "humanLabel": null,
3406
- "answer": "2 foreground slots; safety floor 1",
3407
- "predictedCorrectness": 0.51278928136419,
3408
- "answerOverall": 0.512225694417802,
3409
- "tokens": 19,
3410
- "rest": false,
3411
- "scenario": [
3412
- "chat",
3413
- "writing",
3414
- "office",
3415
- "collaboration",
3416
- "engineering"
3417
- ],
3418
- "interaction": "think"
3419
- },
3420
- {
3421
- "id": "attention-6",
3422
- "category": "attention",
3423
- "actual": true,
3424
- "humanLabel": null,
3425
- "answer": "2 foreground slots; safety floor 1",
3426
- "predictedCorrectness": 0.51278928136419,
3427
- "answerOverall": 0.512225694417802,
3428
- "tokens": 19,
3429
- "rest": false,
3430
- "scenario": [
3431
- "chat",
3432
- "writing",
3433
- "office",
3434
- "collaboration",
3435
- "engineering"
3436
- ],
3437
- "interaction": "think"
3438
- },
3439
- {
3440
- "id": "attention-7",
3441
- "category": "attention",
3442
- "actual": true,
3443
- "humanLabel": null,
3444
- "answer": "2 foreground slots; safety floor 1",
3445
- "predictedCorrectness": 0.51278928136419,
3446
- "answerOverall": 0.512225694417802,
3447
- "tokens": 19,
3448
- "rest": false,
3449
- "scenario": [
3450
- "chat",
3451
- "writing",
3452
- "office",
3453
- "collaboration",
3454
- "engineering"
3455
- ],
3456
- "interaction": "think"
3457
- },
3458
- {
3459
- "id": "attention-8",
3460
- "category": "attention",
3461
- "actual": true,
3462
- "humanLabel": null,
3463
- "answer": "2 foreground slots; safety floor 1",
3464
- "predictedCorrectness": 0.51278928136419,
3465
- "answerOverall": 0.512225694417802,
3466
- "tokens": 19,
3467
- "rest": false,
3468
- "scenario": [
3469
- "chat",
3470
- "writing",
3471
- "office",
3472
- "collaboration",
3473
- "engineering"
3474
- ],
3475
- "interaction": "think"
3476
- },
3477
- {
3478
- "id": "attention-9",
3479
- "category": "attention",
3480
- "actual": true,
3481
- "humanLabel": null,
3482
- "answer": "2 foreground slots; safety floor 1",
3483
- "predictedCorrectness": 0.51278928136419,
3484
- "answerOverall": 0.512225694417802,
3485
- "tokens": 19,
3486
- "rest": false,
3487
- "scenario": [
3488
- "chat",
3489
- "writing",
3490
- "office",
3491
- "collaboration",
3492
- "engineering"
3493
- ],
3494
- "interaction": "think"
3495
- },
3496
- {
3497
- "id": "attention-10",
3498
- "category": "attention",
3499
- "actual": true,
3500
- "humanLabel": null,
3501
- "answer": "2 foreground slots; safety floor 1",
3502
- "predictedCorrectness": 0.51278928136419,
3503
- "answerOverall": 0.512225694417802,
3504
- "tokens": 19,
3505
- "rest": false,
3506
- "scenario": [
3507
- "chat",
3508
- "writing",
3509
- "office",
3510
- "collaboration",
3511
- "engineering"
3512
- ],
3513
- "interaction": "think"
3514
- },
3515
- {
3516
- "id": "attention-11",
3517
- "category": "attention",
3518
- "actual": true,
3519
- "humanLabel": null,
3520
- "answer": "2 foreground slots; safety floor 1",
3521
- "predictedCorrectness": 0.51278928136419,
3522
- "answerOverall": 0.512225694417802,
3523
- "tokens": 19,
3524
- "rest": false,
3525
- "scenario": [
3526
- "chat",
3527
- "writing",
3528
- "office",
3529
- "collaboration",
3530
- "engineering"
3531
- ],
3532
- "interaction": "think"
3533
- },
3534
- {
3535
- "id": "attention-12",
3536
- "category": "attention",
3537
- "actual": true,
3538
- "humanLabel": null,
3539
- "answer": "2 foreground slots; safety floor 1",
3540
- "predictedCorrectness": 0.51278928136419,
3541
- "answerOverall": 0.512225694417802,
3542
- "tokens": 19,
3543
- "rest": false,
3544
- "scenario": [
3545
- "chat",
3546
- "writing",
3547
- "office",
3548
- "collaboration",
3549
- "engineering"
3550
- ],
3551
- "interaction": "think"
3552
- },
3553
- {
3554
- "id": "attention-13",
3555
- "category": "attention",
3556
- "actual": true,
3557
- "humanLabel": null,
3558
- "answer": "2 foreground slots; safety floor 1",
3559
- "predictedCorrectness": 0.51278928136419,
3560
- "answerOverall": 0.512225694417802,
3561
- "tokens": 19,
3562
- "rest": false,
3563
- "scenario": [
3564
- "chat",
3565
- "writing",
3566
- "office",
3567
- "collaboration",
3568
- "engineering"
3569
- ],
3570
- "interaction": "think"
3571
- },
3572
- {
3573
- "id": "attention-14",
3574
- "category": "attention",
3575
- "actual": true,
3576
- "humanLabel": null,
3577
- "answer": "2 foreground slots; safety floor 1",
3578
- "predictedCorrectness": 0.51278928136419,
3579
- "answerOverall": 0.512225694417802,
3580
- "tokens": 19,
3581
- "rest": false,
3582
- "scenario": [
3583
- "chat",
3584
- "writing",
3585
- "office",
3586
- "collaboration",
3587
- "engineering"
3588
- ],
3589
- "interaction": "think"
3590
- },
3591
- {
3592
- "id": "attention-15",
3593
- "category": "attention",
3594
- "actual": true,
3595
- "humanLabel": null,
3596
- "answer": "2 foreground slots; safety floor 1",
3597
- "predictedCorrectness": 0.51278928136419,
3598
- "answerOverall": 0.512225694417802,
3599
- "tokens": 19,
3600
- "rest": false,
3601
- "scenario": [
3602
- "chat",
3603
- "writing",
3604
- "office",
3605
- "collaboration",
3606
- "engineering"
3607
- ],
3608
- "interaction": "think"
3609
- },
3610
- {
3611
- "id": "attention-16",
3612
- "category": "attention",
3613
- "actual": true,
3614
- "humanLabel": null,
3615
- "answer": "2 foreground slots; safety floor 1",
3616
- "predictedCorrectness": 0.51278928136419,
3617
- "answerOverall": 0.512225694417802,
3618
- "tokens": 19,
3619
- "rest": false,
3620
- "scenario": [
3621
- "chat",
3622
- "writing",
3623
- "office",
3624
- "collaboration",
3625
- "engineering"
3626
- ],
3627
- "interaction": "think"
3628
- },
3629
- {
3630
- "id": "attention-17",
3631
- "category": "attention",
3632
- "actual": true,
3633
- "humanLabel": null,
3634
- "answer": "2 foreground slots; safety floor 1",
3635
- "predictedCorrectness": 0.51278928136419,
3636
- "answerOverall": 0.512225694417802,
3637
- "tokens": 19,
3638
- "rest": false,
3639
- "scenario": [
3640
- "chat",
3641
- "writing",
3642
- "office",
3643
- "collaboration",
3644
- "engineering"
3645
- ],
3646
- "interaction": "think"
3647
- },
3648
- {
3649
- "id": "attention-18",
3650
- "category": "attention",
3651
- "actual": true,
3652
- "humanLabel": null,
3653
- "answer": "2 foreground slots; safety floor 1",
3654
- "predictedCorrectness": 0.51278928136419,
3655
- "answerOverall": 0.512225694417802,
3656
- "tokens": 19,
3657
- "rest": false,
3658
- "scenario": [
3659
- "chat",
3660
- "writing",
3661
- "office",
3662
- "collaboration",
3663
- "engineering"
3664
- ],
3665
- "interaction": "think"
3666
- },
3667
- {
3668
- "id": "attention-19",
3669
- "category": "attention",
3670
- "actual": true,
3671
- "humanLabel": null,
3672
- "answer": "2 foreground slots; safety floor 1",
3673
- "predictedCorrectness": 0.51278928136419,
3674
- "answerOverall": 0.512225694417802,
3675
- "tokens": 19,
3676
- "rest": false,
3677
- "scenario": [
3678
- "chat",
3679
- "writing",
3680
- "office",
3681
- "collaboration",
3682
- "engineering"
3683
- ],
3684
- "interaction": "think"
3685
- },
3686
- {
3687
- "id": "bayes-0",
3688
- "category": "bayes",
3689
- "actual": false,
3690
- "humanLabel": null,
3691
- "answer": "8.3%",
3692
- "predictedCorrectness": 0.51278928136419,
3693
- "answerOverall": 0.5065686319907432,
3694
- "tokens": 11,
3695
- "rest": false,
3696
- "scenario": [
3697
- "chat",
3698
- "writing",
3699
- "office",
3700
- "collaboration",
3701
- "engineering"
3702
- ],
3703
- "interaction": "think"
3704
- },
3705
- {
3706
- "id": "bayes-1",
3707
- "category": "bayes",
3708
- "actual": true,
3709
- "humanLabel": null,
3710
- "answer": "7.5%",
3711
- "predictedCorrectness": 0.51278928136419,
3712
- "answerOverall": 0.5065686319907432,
3713
- "tokens": 11,
3714
- "rest": false,
3715
- "scenario": [
3716
- "chat",
3717
- "writing",
3718
- "office",
3719
- "collaboration",
3720
- "engineering"
3721
- ],
3722
- "interaction": "think"
3723
- },
3724
- {
3725
- "id": "bayes-2",
3726
- "category": "bayes",
3727
- "actual": true,
3728
- "humanLabel": null,
3729
- "answer": "7.5%",
3730
- "predictedCorrectness": 0.51278928136419,
3731
- "answerOverall": 0.5065686319907432,
3732
- "tokens": 11,
3733
- "rest": false,
3734
- "scenario": [
3735
- "chat",
3736
- "writing",
3737
- "office",
3738
- "collaboration",
3739
- "engineering"
3740
- ],
3741
- "interaction": "think"
3742
- },
3743
- {
3744
- "id": "bayes-3",
3745
- "category": "bayes",
3746
- "actual": true,
3747
- "humanLabel": null,
3748
- "answer": "7.5%",
3749
- "predictedCorrectness": 0.51278928136419,
3750
- "answerOverall": 0.5065686319907432,
3751
- "tokens": 11,
3752
- "rest": false,
3753
- "scenario": [
3754
- "chat",
3755
- "writing",
3756
- "office",
3757
- "collaboration",
3758
- "engineering"
3759
- ],
3760
- "interaction": "think"
3761
- },
3762
- {
3763
- "id": "bayes-4",
3764
- "category": "bayes",
3765
- "actual": true,
3766
- "humanLabel": null,
3767
- "answer": "7.5%",
3768
- "predictedCorrectness": 0.51278928136419,
3769
- "answerOverall": 0.5065686319907432,
3770
- "tokens": 11,
3771
- "rest": false,
3772
- "scenario": [
3773
- "chat",
3774
- "writing",
3775
- "office",
3776
- "collaboration",
3777
- "engineering"
3778
- ],
3779
- "interaction": "think"
3780
- },
3781
- {
3782
- "id": "bayes-5",
3783
- "category": "bayes",
3784
- "actual": false,
3785
- "humanLabel": null,
3786
- "answer": "8.3%",
3787
- "predictedCorrectness": 0.51278928136419,
3788
- "answerOverall": 0.5065686319907432,
3789
- "tokens": 11,
3790
- "rest": false,
3791
- "scenario": [
3792
- "chat",
3793
- "writing",
3794
- "office",
3795
- "collaboration",
3796
- "engineering"
3797
- ],
3798
- "interaction": "think"
3799
- },
3800
- {
3801
- "id": "bayes-6",
3802
- "category": "bayes",
3803
- "actual": true,
3804
- "humanLabel": null,
3805
- "answer": "7.5%",
3806
- "predictedCorrectness": 0.51278928136419,
3807
- "answerOverall": 0.5065686319907432,
3808
- "tokens": 11,
3809
- "rest": false,
3810
- "scenario": [
3811
- "chat",
3812
- "writing",
3813
- "office",
3814
- "collaboration",
3815
- "engineering"
3816
- ],
3817
- "interaction": "think"
3818
- },
3819
- {
3820
- "id": "bayes-7",
3821
- "category": "bayes",
3822
- "actual": true,
3823
- "humanLabel": null,
3824
- "answer": "7.5%",
3825
- "predictedCorrectness": 0.51278928136419,
3826
- "answerOverall": 0.5065686319907432,
3827
- "tokens": 11,
3828
- "rest": false,
3829
- "scenario": [
3830
- "chat",
3831
- "writing",
3832
- "office",
3833
- "collaboration",
3834
- "engineering"
3835
- ],
3836
- "interaction": "think"
3837
- },
3838
- {
3839
- "id": "bayes-8",
3840
- "category": "bayes",
3841
- "actual": true,
3842
- "humanLabel": null,
3843
- "answer": "7.5%",
3844
- "predictedCorrectness": 0.51278928136419,
3845
- "answerOverall": 0.5065686319907432,
3846
- "tokens": 11,
3847
- "rest": false,
3848
- "scenario": [
3849
- "chat",
3850
- "writing",
3851
- "office",
3852
- "collaboration",
3853
- "engineering"
3854
- ],
3855
- "interaction": "think"
3856
- },
3857
- {
3858
- "id": "bayes-9",
3859
- "category": "bayes",
3860
- "actual": true,
3861
- "humanLabel": null,
3862
- "answer": "7.5%",
3863
- "predictedCorrectness": 0.51278928136419,
3864
- "answerOverall": 0.5065686319907432,
3865
- "tokens": 11,
3866
- "rest": false,
3867
- "scenario": [
3868
- "chat",
3869
- "writing",
3870
- "office",
3871
- "collaboration",
3872
- "engineering"
3873
- ],
3874
- "interaction": "think"
3875
- },
3876
- {
3877
- "id": "bayes-10",
3878
- "category": "bayes",
3879
- "actual": false,
3880
- "humanLabel": null,
3881
- "answer": "8.3%",
3882
- "predictedCorrectness": 0.51278928136419,
3883
- "answerOverall": 0.5065686319907432,
3884
- "tokens": 11,
3885
- "rest": false,
3886
- "scenario": [
3887
- "chat",
3888
- "writing",
3889
- "office",
3890
- "collaboration",
3891
- "engineering"
3892
- ],
3893
- "interaction": "think"
3894
- },
3895
- {
3896
- "id": "bayes-11",
3897
- "category": "bayes",
3898
- "actual": true,
3899
- "humanLabel": null,
3900
- "answer": "7.5%",
3901
- "predictedCorrectness": 0.51278928136419,
3902
- "answerOverall": 0.5065686319907432,
3903
- "tokens": 11,
3904
- "rest": false,
3905
- "scenario": [
3906
- "chat",
3907
- "writing",
3908
- "office",
3909
- "collaboration",
3910
- "engineering"
3911
- ],
3912
- "interaction": "think"
3913
- },
3914
- {
3915
- "id": "bayes-12",
3916
- "category": "bayes",
3917
- "actual": true,
3918
- "humanLabel": null,
3919
- "answer": "7.5%",
3920
- "predictedCorrectness": 0.51278928136419,
3921
- "answerOverall": 0.5065686319907432,
3922
- "tokens": 11,
3923
- "rest": false,
3924
- "scenario": [
3925
- "chat",
3926
- "writing",
3927
- "office",
3928
- "collaboration",
3929
- "engineering"
3930
- ],
3931
- "interaction": "think"
3932
- },
3933
- {
3934
- "id": "bayes-13",
3935
- "category": "bayes",
3936
- "actual": true,
3937
- "humanLabel": null,
3938
- "answer": "7.5%",
3939
- "predictedCorrectness": 0.51278928136419,
3940
- "answerOverall": 0.5065686319907432,
3941
- "tokens": 11,
3942
- "rest": false,
3943
- "scenario": [
3944
- "chat",
3945
- "writing",
3946
- "office",
3947
- "collaboration",
3948
- "engineering"
3949
- ],
3950
- "interaction": "think"
3951
- },
3952
- {
3953
- "id": "bayes-14",
3954
- "category": "bayes",
3955
- "actual": true,
3956
- "humanLabel": null,
3957
- "answer": "7.5%",
3958
- "predictedCorrectness": 0.51278928136419,
3959
- "answerOverall": 0.5065686319907432,
3960
- "tokens": 11,
3961
- "rest": false,
3962
- "scenario": [
3963
- "chat",
3964
- "writing",
3965
- "office",
3966
- "collaboration",
3967
- "engineering"
3968
- ],
3969
- "interaction": "think"
3970
- },
3971
- {
3972
- "id": "bayes-15",
3973
- "category": "bayes",
3974
- "actual": false,
3975
- "humanLabel": null,
3976
- "answer": "8.3%",
3977
- "predictedCorrectness": 0.51278928136419,
3978
- "answerOverall": 0.5065686319907432,
3979
- "tokens": 11,
3980
- "rest": false,
3981
- "scenario": [
3982
- "chat",
3983
- "writing",
3984
- "office",
3985
- "collaboration",
3986
- "engineering"
3987
- ],
3988
- "interaction": "think"
3989
- },
3990
- {
3991
- "id": "bayes-16",
3992
- "category": "bayes",
3993
- "actual": true,
3994
- "humanLabel": null,
3995
- "answer": "7.5%",
3996
- "predictedCorrectness": 0.51278928136419,
3997
- "answerOverall": 0.5065686319907432,
3998
- "tokens": 11,
3999
- "rest": false,
4000
- "scenario": [
4001
- "chat",
4002
- "writing",
4003
- "office",
4004
- "collaboration",
4005
- "engineering"
4006
- ],
4007
- "interaction": "think"
4008
- },
4009
- {
4010
- "id": "bayes-17",
4011
- "category": "bayes",
4012
- "actual": true,
4013
- "humanLabel": null,
4014
- "answer": "7.5%",
4015
- "predictedCorrectness": 0.51278928136419,
4016
- "answerOverall": 0.5065686319907432,
4017
- "tokens": 11,
4018
- "rest": false,
4019
- "scenario": [
4020
- "chat",
4021
- "writing",
4022
- "office",
4023
- "collaboration",
4024
- "engineering"
4025
- ],
4026
- "interaction": "think"
4027
- },
4028
- {
4029
- "id": "bayes-18",
4030
- "category": "bayes",
4031
- "actual": true,
4032
- "humanLabel": null,
4033
- "answer": "7.5%",
4034
- "predictedCorrectness": 0.51278928136419,
4035
- "answerOverall": 0.5065686319907432,
4036
- "tokens": 11,
4037
- "rest": false,
4038
- "scenario": [
4039
- "chat",
4040
- "writing",
4041
- "office",
4042
- "collaboration",
4043
- "engineering"
4044
- ],
4045
- "interaction": "think"
4046
- },
4047
- {
4048
- "id": "bayes-19",
4049
- "category": "bayes",
4050
- "actual": true,
4051
- "humanLabel": null,
4052
- "answer": "7.5%",
4053
- "predictedCorrectness": 0.51278928136419,
4054
- "answerOverall": 0.5065686319907432,
4055
- "tokens": 11,
4056
- "rest": false,
4057
- "scenario": [
4058
- "chat",
4059
- "writing",
4060
- "office",
4061
- "collaboration",
4062
- "engineering"
4063
- ],
4064
- "interaction": "think"
4065
- }
4066
- ],
4067
- "metrics": {
4068
- "totalCases": 220,
4069
- "accuracy": 0.9181818181818182,
4070
- "ece": 0.4065352021170867,
4071
- "brier": 0.24050912323431942,
4072
- "oracleEce": 0.22222222222222549,
4073
- "oracleBrier": 0.049382716049382894,
4074
- "totalTokens": 2850,
4075
- "averageTokens": 12.954545454545455,
4076
- "approximateCostUsd": 0.00042749999999999993,
4077
- "restRate": 0,
4078
- "averageAnswerOverall": 0.5067638833462151,
4079
- "humanLabeled": 0,
4080
- "humanLabelAccuracy": null
4081
- },
4082
- "categories": [
4083
- {
4084
- "category": "arithmetic",
4085
- "total": 50,
4086
- "accuracy": 0.9,
4087
- "ece": 0.38721071863581036
4088
- },
4089
- {
4090
- "category": "unique-rows",
4091
- "total": 40,
4092
- "accuracy": 0.9,
4093
- "ece": 0.38721071863581014
4094
- },
4095
- {
4096
- "category": "syllogism",
4097
- "total": 30,
4098
- "accuracy": 0.9333333333333333,
4099
- "ece": 0.42473382473382437
4100
- },
4101
- {
4102
- "category": "memory",
4103
- "total": 30,
4104
- "accuracy": 0.9,
4105
- "ece": 0.3872107186358098
4106
- },
4107
- {
4108
- "category": "role",
4109
- "total": 30,
4110
- "accuracy": 1,
4111
- "ece": 0.491400491400491
4112
- },
4113
- {
4114
- "category": "attention",
4115
- "total": 20,
4116
- "accuracy": 1,
4117
- "ece": 0.4872107186358099
4118
- },
4119
- {
4120
- "category": "bayes",
4121
- "total": 20,
4122
- "accuracy": 0.8,
4123
- "ece": 0.28721071863580994
4124
- }
4125
- ],
4126
- "textBins": [
4127
- {
4128
- "n": 0,
4129
- "confidence": 0,
4130
- "accuracy": 0
4131
- },
4132
- {
4133
- "n": 0,
4134
- "confidence": 0,
4135
- "accuracy": 0
4136
- },
4137
- {
4138
- "n": 0,
4139
- "confidence": 0,
4140
- "accuracy": 0
4141
- },
4142
- {
4143
- "n": 0,
4144
- "confidence": 0,
4145
- "accuracy": 0
4146
- },
4147
- {
4148
- "n": 0,
4149
- "confidence": 0,
4150
- "accuracy": 0
4151
- },
4152
- {
4153
- "n": 220,
4154
- "confidence": 112.56225553424092,
4155
- "accuracy": 202
4156
- },
4157
- {
4158
- "n": 0,
4159
- "confidence": 0,
4160
- "accuracy": 0
4161
- },
4162
- {
4163
- "n": 0,
4164
- "confidence": 0,
4165
- "accuracy": 0
4166
- },
4167
- {
4168
- "n": 0,
4169
- "confidence": 0,
4170
- "accuracy": 0
4171
- },
4172
- {
4173
- "n": 0,
4174
- "confidence": 0,
4175
- "accuracy": 0
4176
- }
4177
- ],
4178
- "oracleBins": [
4179
- {
4180
- "n": 0,
4181
- "confidence": 0,
4182
- "accuracy": 0
4183
- },
4184
- {
4185
- "n": 0,
4186
- "confidence": 0,
4187
- "accuracy": 0
4188
- },
4189
- {
4190
- "n": 18,
4191
- "confidence": 4.000000000000001,
4192
- "accuracy": 0
4193
- },
4194
- {
4195
- "n": 0,
4196
- "confidence": 0,
4197
- "accuracy": 0
4198
- },
4199
- {
4200
- "n": 0,
4201
- "confidence": 0,
4202
- "accuracy": 0
4203
- },
4204
- {
4205
- "n": 0,
4206
- "confidence": 0,
4207
- "accuracy": 0
4208
- },
4209
- {
4210
- "n": 0,
4211
- "confidence": 0,
4212
- "accuracy": 0
4213
- },
4214
- {
4215
- "n": 202,
4216
- "confidence": 157.1111111111104,
4217
- "accuracy": 202
4218
- },
4219
- {
4220
- "n": 0,
4221
- "confidence": 0,
4222
- "accuracy": 0
4223
- },
4224
- {
4225
- "n": 0,
4226
- "confidence": 0,
4227
- "accuracy": 0
4228
- }
4229
- ]
4230
- }