cortico 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (311) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +47 -0
  3. package/package.json +43 -0
  4. package/src/boot.ts +46 -0
  5. package/src/bot.ts +1342 -0
  6. package/src/config-file.ts +41 -0
  7. package/src/core/README.md +92 -0
  8. package/src/core/billing.ts +26 -0
  9. package/src/core/blobs.ts +106 -0
  10. package/src/core/bus.ts +311 -0
  11. package/src/core/config-schema.ts +201 -0
  12. package/src/core/config.ts +262 -0
  13. package/src/core/core.ts +699 -0
  14. package/src/core/cost.ts +244 -0
  15. package/src/core/event-store.ts +304 -0
  16. package/src/core/fork.ts +113 -0
  17. package/src/core/generation.ts +118 -0
  18. package/src/core/instance-lock.ts +215 -0
  19. package/src/core/ipc-logger.ts +105 -0
  20. package/src/core/language.ts +62 -0
  21. package/src/core/log-context.ts +38 -0
  22. package/src/core/loop.ts +1858 -0
  23. package/src/core/markers.ts +48 -0
  24. package/src/core/prefix.ts +121 -0
  25. package/src/core/run.ts +95 -0
  26. package/src/core/secrets.ts +18 -0
  27. package/src/core/session.ts +56 -0
  28. package/src/core/sessions.ts +261 -0
  29. package/src/core/state.ts +104 -0
  30. package/src/core/template.ts +72 -0
  31. package/src/core/timers.ts +129 -0
  32. package/src/core/tool-log.ts +135 -0
  33. package/src/core/transcript.ts +92 -0
  34. package/src/core/truncate.ts +121 -0
  35. package/src/core/types.ts +1140 -0
  36. package/src/core/usage-log.ts +79 -0
  37. package/src/core/util.ts +386 -0
  38. package/src/deploy-listing.ts +160 -0
  39. package/src/deploy.ts +162 -0
  40. package/src/extensions/README.md +72 -0
  41. package/src/extensions/dry-mount.ts +390 -0
  42. package/src/extensions/manifest.ts +153 -0
  43. package/src/extensions/runtime.ts +42 -0
  44. package/src/extensions.ts +519 -0
  45. package/src/launcher.ts +285 -0
  46. package/src/paths.ts +155 -0
  47. package/src/protocol/open-responses/LICENSE +201 -0
  48. package/src/protocol/open-responses/README.md +14 -0
  49. package/src/protocol/open-responses/context-helpers.ts +27 -0
  50. package/src/protocol/open-responses/context-log.ts +59 -0
  51. package/src/protocol/open-responses/context.ts +89 -0
  52. package/src/protocol/open-responses/generated.ts +115 -0
  53. package/src/protocol/open-responses/index.ts +33 -0
  54. package/src/protocol/open-responses/openapi.json +4230 -0
  55. package/src/protocol/open-responses/stream.ts +146 -0
  56. package/src/protocol/open-responses/tokens.ts +31 -0
  57. package/src/providers/README.md +86 -0
  58. package/src/providers/base.ts +111 -0
  59. package/src/providers/configuration.ts +97 -0
  60. package/src/providers/console/config.ts +21 -0
  61. package/src/providers/console/settings.ts +443 -0
  62. package/src/providers/console/strings.ts +74 -0
  63. package/src/providers/console/types.ts +11 -0
  64. package/src/providers/llamacpp/archive.ts +148 -0
  65. package/src/providers/llamacpp/catalog.ts +247 -0
  66. package/src/providers/llamacpp/config.ts +36 -0
  67. package/src/providers/llamacpp/console/client.ts +6 -0
  68. package/src/providers/llamacpp/console/models-panel.ts +120 -0
  69. package/src/providers/llamacpp/console/runtime-panel.ts +173 -0
  70. package/src/providers/llamacpp/console/server.ts +153 -0
  71. package/src/providers/llamacpp/index.ts +95 -0
  72. package/src/providers/llamacpp/native.ts +51 -0
  73. package/src/providers/llamacpp/options.ts +117 -0
  74. package/src/providers/llamacpp/runtime-store.ts +139 -0
  75. package/src/providers/llamacpp/runtime.ts +214 -0
  76. package/src/providers/llamacpp/server.ts +286 -0
  77. package/src/providers/llamacpp/strings.ts +230 -0
  78. package/src/providers/openai-responses-compat/config.ts +20 -0
  79. package/src/providers/openai-responses-compat/index.ts +89 -0
  80. package/src/providers/openai-responses-compat/native.ts +127 -0
  81. package/src/providers/openai-responses-compat/strings.ts +19 -0
  82. package/src/providers/pricebook.ts +75 -0
  83. package/src/providers/registry.ts +137 -0
  84. package/src/providers/strings.ts +71 -0
  85. package/src/providers/transport/chat.ts +156 -0
  86. package/src/providers/transport/errors.ts +48 -0
  87. package/src/providers/transport/history.ts +68 -0
  88. package/src/providers/transport/native-input.ts +83 -0
  89. package/src/providers/transport/native-types.ts +13 -0
  90. package/src/providers/transport/response-assembly.ts +191 -0
  91. package/src/providers/transport/response-http.ts +196 -0
  92. package/src/providers/transport/response-meters.ts +27 -0
  93. package/src/providers/transport/responses-input.ts +56 -0
  94. package/src/web/README.md +84 -0
  95. package/src/web/client/console-pages/builtins/llm-settings/panel.ts +625 -0
  96. package/src/web/client/console-pages/builtins/llm-settings/pricing-panel.ts +168 -0
  97. package/src/web/client/console-pages/builtins/llm-settings/strings.ts +213 -0
  98. package/src/web/client/console-pages/builtins.ts +10 -0
  99. package/src/web/client/console-pages/context.ts +135 -0
  100. package/src/web/client/console-pages/host.ts +633 -0
  101. package/src/web/client/console-pages/loader.ts +132 -0
  102. package/src/web/client/console-pages/strings.ts +90 -0
  103. package/src/web/client/console-pages/tools/strings.ts +49 -0
  104. package/src/web/client/console-pages/tools/view.ts +215 -0
  105. package/src/web/client/core/api.ts +200 -0
  106. package/src/web/client/core/language.ts +56 -0
  107. package/src/web/client/core/lifecycle.ts +132 -0
  108. package/src/web/client/core/router.ts +199 -0
  109. package/src/web/client/core/stream.ts +157 -0
  110. package/src/web/client/core/websocket.ts +82 -0
  111. package/src/web/client/features/appearance/index.ts +333 -0
  112. package/src/web/client/features/appearance/strings.ts +87 -0
  113. package/src/web/client/features/config/index.ts +11 -0
  114. package/src/web/client/features/config/strings.ts +41 -0
  115. package/src/web/client/features/config/view.ts +428 -0
  116. package/src/web/client/features/core/events.ts +197 -0
  117. package/src/web/client/features/core/index.ts +254 -0
  118. package/src/web/client/features/core/run.ts +56 -0
  119. package/src/web/client/features/core/runlog.ts +214 -0
  120. package/src/web/client/features/core/sessions.ts +53 -0
  121. package/src/web/client/features/core/strings.ts +157 -0
  122. package/src/web/client/features/extensions/index.ts +422 -0
  123. package/src/web/client/features/extensions/strings.ts +164 -0
  124. package/src/web/client/features/feature.ts +93 -0
  125. package/src/web/client/features/live/context.ts +280 -0
  126. package/src/web/client/features/live/fork.ts +156 -0
  127. package/src/web/client/features/live/index.ts +392 -0
  128. package/src/web/client/features/live/onboarding.ts +113 -0
  129. package/src/web/client/features/live/protocol.ts +122 -0
  130. package/src/web/client/features/live/sessions.ts +71 -0
  131. package/src/web/client/features/live/status.ts +68 -0
  132. package/src/web/client/features/live/strings.ts +252 -0
  133. package/src/web/client/features/live/timeline.ts +590 -0
  134. package/src/web/client/features/prompts/editor.ts +292 -0
  135. package/src/web/client/features/prompts/index.ts +346 -0
  136. package/src/web/client/features/prompts/strings.ts +136 -0
  137. package/src/web/client/features/prompts/view.ts +256 -0
  138. package/src/web/client/features/providers/index.ts +100 -0
  139. package/src/web/client/features/providers/strings.ts +23 -0
  140. package/src/web/client/features/settings/general.ts +28 -0
  141. package/src/web/client/features/settings/index.ts +82 -0
  142. package/src/web/client/features/settings/strings.ts +33 -0
  143. package/src/web/client/features/storage/index.ts +1 -0
  144. package/src/web/client/features/storage/strings.ts +45 -0
  145. package/src/web/client/features/storage/view.ts +154 -0
  146. package/src/web/client/features/usage/chart.ts +486 -0
  147. package/src/web/client/features/usage/index.ts +407 -0
  148. package/src/web/client/features/usage/labels.ts +46 -0
  149. package/src/web/client/features/usage/range.ts +36 -0
  150. package/src/web/client/features/usage/state.ts +37 -0
  151. package/src/web/client/features/usage/strings.ts +191 -0
  152. package/src/web/client/features/usage/tooltip.ts +86 -0
  153. package/src/web/client/features/usage/types.ts +96 -0
  154. package/src/web/client/features/worlds/index.ts +424 -0
  155. package/src/web/client/features/worlds/strings.ts +121 -0
  156. package/src/web/client/main.ts +273 -0
  157. package/src/web/client/shell/avatar.ts +233 -0
  158. package/src/web/client/shell/index.ts +549 -0
  159. package/src/web/client/shell/strings.ts +89 -0
  160. package/src/web/client/theme/handoff.ts +43 -0
  161. package/src/web/client/theme/palette.ts +109 -0
  162. package/src/web/client/theme/registry.ts +130 -0
  163. package/src/web/client/theme/storage.ts +72 -0
  164. package/src/web/client/theme/strings.ts +123 -0
  165. package/src/web/client/theme/studio.ts +366 -0
  166. package/src/web/client/ui/actions.ts +109 -0
  167. package/src/web/client/ui/data.ts +185 -0
  168. package/src/web/client/ui/dom.ts +23 -0
  169. package/src/web/client/ui/fields.ts +191 -0
  170. package/src/web/client/ui/format.ts +75 -0
  171. package/src/web/client/ui/icons.ts +183 -0
  172. package/src/web/client/ui/images.ts +91 -0
  173. package/src/web/client/ui/index.ts +89 -0
  174. package/src/web/client/ui/lamp.ts +125 -0
  175. package/src/web/client/ui/log.ts +98 -0
  176. package/src/web/client/ui/overlay.ts +184 -0
  177. package/src/web/client/ui/page.ts +8 -0
  178. package/src/web/client/ui/prompt-input.tsx +311 -0
  179. package/src/web/client/ui/sheet.ts +131 -0
  180. package/src/web/client/ui/strings.ts +71 -0
  181. package/src/web/console-pages.ts +367 -0
  182. package/src/web/files.ts +107 -0
  183. package/src/web/path-picker.ts +296 -0
  184. package/src/web/public/index.html +17 -0
  185. package/src/web/public/styles.css +1715 -0
  186. package/src/web/server.ts +1883 -0
  187. package/src/web/shared/client-panel.ts +603 -0
  188. package/src/web/shared/console-protocol.ts +560 -0
  189. package/src/web/shared/css.d.ts +1 -0
  190. package/src/web/shared/path-picker.ts +23 -0
  191. package/src/web/shared/theme.ts +256 -0
  192. package/src/web/theme-store.ts +37 -0
  193. package/src/world.ts +449 -0
  194. package/src/worlds/bilibili/ENV_PROMPT.md +5 -0
  195. package/src/worlds/bilibili/README.md +176 -0
  196. package/src/worlds/bilibili/audience-admission.ts +995 -0
  197. package/src/worlds/bilibili/client.ts +528 -0
  198. package/src/worlds/bilibili/coalescing-buffer.ts +125 -0
  199. package/src/worlds/bilibili/config.ts +185 -0
  200. package/src/worlds/bilibili/console/client.ts +202 -0
  201. package/src/worlds/bilibili/definition.ts +35 -0
  202. package/src/worlds/bilibili/gift-frame.ts +131 -0
  203. package/src/worlds/bilibili/normalize.ts +452 -0
  204. package/src/worlds/bilibili/overlay/announcement.ts +76 -0
  205. package/src/worlds/bilibili/overlay/assets.ts +82 -0
  206. package/src/worlds/bilibili/overlay/model.ts +494 -0
  207. package/src/worlds/bilibili/overlay/project.ts +202 -0
  208. package/src/worlds/bilibili/overlay/server.ts +286 -0
  209. package/src/worlds/bilibili/overlay/types.ts +217 -0
  210. package/src/worlds/bilibili/overlay/web/app.js +690 -0
  211. package/src/worlds/bilibili/overlay/web/editor.css +297 -0
  212. package/src/worlds/bilibili/overlay/web/editor.html +102 -0
  213. package/src/worlds/bilibili/overlay/web/editor.js +2116 -0
  214. package/src/worlds/bilibili/overlay/web/overlay.html +13 -0
  215. package/src/worlds/bilibili/overlay/web/styles.css +258 -0
  216. package/src/worlds/bilibili/protobuf.ts +123 -0
  217. package/src/worlds/bilibili/wire.ts +94 -0
  218. package/src/worlds/bilibili/world.ts +1389 -0
  219. package/src/worlds/console-fixture/ENV_PROMPT.md +1 -0
  220. package/src/worlds/console-fixture/console/client.ts +72 -0
  221. package/src/worlds/console-fixture/world.ts +73 -0
  222. package/src/worlds/index.ts +15 -0
  223. package/src/worlds/minecraft/ENV_PROMPT.md +32 -0
  224. package/src/worlds/minecraft/ENV_PROMPT_CAMERA.md +2 -0
  225. package/src/worlds/minecraft/LICENSE-mineflayer-pathfinder.txt +21 -0
  226. package/src/worlds/minecraft/README.md +917 -0
  227. package/src/worlds/minecraft/blueprint-plan.ts +915 -0
  228. package/src/worlds/minecraft/blueprint-registry.ts +566 -0
  229. package/src/worlds/minecraft/blueprint-repair.ts +236 -0
  230. package/src/worlds/minecraft/blueprint-resource.ts +272 -0
  231. package/src/worlds/minecraft/blueprint.ts +492 -0
  232. package/src/worlds/minecraft/body-lease.ts +340 -0
  233. package/src/worlds/minecraft/bridge.ts +749 -0
  234. package/src/worlds/minecraft/check.ts +651 -0
  235. package/src/worlds/minecraft/chests.ts +292 -0
  236. package/src/worlds/minecraft/client-launch.ts +272 -0
  237. package/src/worlds/minecraft/client-options.ts +85 -0
  238. package/src/worlds/minecraft/client-skins.ts +164 -0
  239. package/src/worlds/minecraft/client-window.ps1 +91 -0
  240. package/src/worlds/minecraft/client.ts +423 -0
  241. package/src/worlds/minecraft/combat-context.ts +109 -0
  242. package/src/worlds/minecraft/combat.ts +1552 -0
  243. package/src/worlds/minecraft/config.ts +499 -0
  244. package/src/worlds/minecraft/console/access.ts +218 -0
  245. package/src/worlds/minecraft/console/client.ts +244 -0
  246. package/src/worlds/minecraft/console/log.ts +110 -0
  247. package/src/worlds/minecraft/console/mount.ts +420 -0
  248. package/src/worlds/minecraft/console/skin.ts +257 -0
  249. package/src/worlds/minecraft/console/style.css +154 -0
  250. package/src/worlds/minecraft/console/world.ts +449 -0
  251. package/src/worlds/minecraft/deaths.ts +135 -0
  252. package/src/worlds/minecraft/definition.ts +18 -0
  253. package/src/worlds/minecraft/engine-child.ts +274 -0
  254. package/src/worlds/minecraft/engine-ipc.ts +98 -0
  255. package/src/worlds/minecraft/entity-facts.ts +132 -0
  256. package/src/worlds/minecraft/escape.ts +407 -0
  257. package/src/worlds/minecraft/executor.ts +14494 -0
  258. package/src/worlds/minecraft/explored.ts +156 -0
  259. package/src/worlds/minecraft/geometry.ts +205 -0
  260. package/src/worlds/minecraft/goal-plan.ts +647 -0
  261. package/src/worlds/minecraft/item-break.ts +126 -0
  262. package/src/worlds/minecraft/item-facts.ts +146 -0
  263. package/src/worlds/minecraft/item-pick.ts +67 -0
  264. package/src/worlds/minecraft/level-dat.ts +176 -0
  265. package/src/worlds/minecraft/log.ts +132 -0
  266. package/src/worlds/minecraft/mineflayer-fixes.ts +1180 -0
  267. package/src/worlds/minecraft/names.ts +377 -0
  268. package/src/worlds/minecraft/pathfinder-lib.d.ts +13 -0
  269. package/src/worlds/minecraft/pathfinder-perf.ts +796 -0
  270. package/src/worlds/minecraft/piglin.ts +49 -0
  271. package/src/worlds/minecraft/policy.ts +547 -0
  272. package/src/worlds/minecraft/precheck.ts +655 -0
  273. package/src/worlds/minecraft/proxy.ts +445 -0
  274. package/src/worlds/minecraft/ranged.ts +635 -0
  275. package/src/worlds/minecraft/readouts.ts +102 -0
  276. package/src/worlds/minecraft/round.ts +26 -0
  277. package/src/worlds/minecraft/search-observation.ts +92 -0
  278. package/src/worlds/minecraft/server-config.ts +467 -0
  279. package/src/worlds/minecraft/server.ts +658 -0
  280. package/src/worlds/minecraft/show.ts +84 -0
  281. package/src/worlds/minecraft/skills.ts +1763 -0
  282. package/src/worlds/minecraft/terrain.ts +1735 -0
  283. package/src/worlds/minecraft/window.ts +96 -0
  284. package/src/worlds/minecraft/works.ts +179 -0
  285. package/src/worlds/minecraft/world.ts +6438 -0
  286. package/src/worlds/qq/ENV_PROMPT.md +5 -0
  287. package/src/worlds/qq/config.ts +108 -0
  288. package/src/worlds/qq/console/client.ts +126 -0
  289. package/src/worlds/qq/console/events.ts +155 -0
  290. package/src/worlds/qq/console/gate.ts +232 -0
  291. package/src/worlds/qq/console/roster.ts +203 -0
  292. package/src/worlds/qq/console/style.css +65 -0
  293. package/src/worlds/qq/conversation.ts +28 -0
  294. package/src/worlds/qq/definition.ts +62 -0
  295. package/src/worlds/qq/driver.ts +375 -0
  296. package/src/worlds/qq/history-tools.ts +209 -0
  297. package/src/worlds/qq/normalize.ts +323 -0
  298. package/src/worlds/qq/qface-map.ts +282 -0
  299. package/src/worlds/qq/vision-prompt.ts +27 -0
  300. package/src/worlds/qq/vision.ts +642 -0
  301. package/src/worlds/qq/vlm.ts +142 -0
  302. package/src/worlds/qq/world.ts +1503 -0
  303. package/src/worlds/terminal/ENV_PROMPT.md +11 -0
  304. package/src/worlds/terminal/config.ts +9 -0
  305. package/src/worlds/terminal/definition.ts +14 -0
  306. package/src/worlds/terminal/world.ts +700 -0
  307. package/src/worlds/websearch/ENV_PROMPT.md +1 -0
  308. package/src/worlds/websearch/brave-client.ts +261 -0
  309. package/src/worlds/websearch/config.ts +75 -0
  310. package/src/worlds/websearch/definition.ts +20 -0
  311. package/src/worlds/websearch/world.ts +171 -0
@@ -0,0 +1,642 @@
1
+ /**
2
+ * QQ 图片视觉服务。
3
+ * 每个图片 ID 保存被动描述缓存与追问会话;主动追问沿用被动识图上下文。
4
+ * 被动失败以事件报告,主动失败由 `ask` 抛出并由模块包装。
5
+ *
6
+ * registry 与图片字节持久化到 `{dataDir}/vision/`。重启时恢复计数器、描述与
7
+ * 图片字节;追问轮次不持久化,首次追问从已保存描述重建首轮上下文。
8
+ */
9
+ import { existsSync, mkdirSync, readFileSync, writeFileSync, appendFileSync, rmSync } from 'node:fs';
10
+ import { createHash } from 'node:crypto';
11
+ import { join } from 'node:path';
12
+ import type { Logger, LLMUsage } from '../../core/types.ts';
13
+ import type { VLMClient, VLMMessage } from './vlm.ts';
14
+ import { shortTime, nullLogger } from '../../core/util.ts';
15
+ import { VISION_DEFAULT_PROMPT } from './vision-prompt.ts';
16
+
17
+ /** 一张图的持久化记录(registry.jsonl一行) */
18
+ interface ImageRecord {
19
+ id: string;
20
+ url: string;
21
+ /** 图片所在QQ消息的平台message_id(register时未知,attachMessage后填) */
22
+ messageId?: string;
23
+ /** 保存的字节文件名(相对 {dataDir}/vision/) */
24
+ file?: string;
25
+ mime?: string;
26
+ /** GIF动图:VLM只看得到第一帧,事件里据此提醒主agent(完整GIF支持以后再做) */
27
+ gif?: boolean;
28
+ /** 被动识图描述(成功后填) */
29
+ desc?: string;
30
+ /** 失败简短原因(下载/识别失败后填) */
31
+ failedReason?: string;
32
+ /** 图片字节sha256(下载成功后填,用于内容去重) */
33
+ hash?: string;
34
+ /** 与本图内容相同(sha256一致)的更早图片ID——命中时desc是从它那沿用的,不重复跑VLM */
35
+ dupOf?: string;
36
+ ts: string;
37
+ }
38
+
39
+ /** 一张图的内存运行态 */
40
+ interface ImageState {
41
+ record: ImageRecord;
42
+ /** clear() 后置 false,阻止已经在途的下载/VLM回写已清空的缓存。 */
43
+ active: boolean;
44
+ /** 图字节的data URL(下载后内存缓存;重启后惰性从文件重建) */
45
+ dataUrl?: string;
46
+ /** VLM session:[user([图,defaultPrompt]), assistant(desc), user(追问), assistant(答), ...] */
47
+ session: VLMMessage[];
48
+ /** 被动识图的promise(在跑/已跑);ask需等它闭合再继承上下文 */
49
+ passivePromise?: Promise<void>;
50
+ /** ask串行化链(同图并发追问排队,保护session顺序) */
51
+ chain: Promise<void>;
52
+ /** 图片字节下载的in-flight promise(去重并发调用;失败后清空允许重试) */
53
+ bytesPromise?: Promise<{ buffer: Uint8Array; mime: string }>;
54
+ /** "下载+去重判定"这一步的in-flight/已完成promise(precheckDup与runPassive共用,只算一次) */
55
+ resolvePromise?: Promise<ResolveResult>;
56
+ }
57
+
58
+ /** resolveContent 结果:done=true 表示重复内容或下载失败,不进入 VLM。 */
59
+ type ResolveResult = { done: true; ok: boolean } | { done: false };
60
+
61
+ /** 被动识图完成的回调:text=完整事件正文,meta进事件信封 */
62
+ type VisionEmit = (text: string, meta: Record<string, unknown>) => void;
63
+
64
+ /**
65
+ * 自带模型的用量上报出口。 World 自带的 LLM 不经 core 的 usageLog,
66
+ * 归账是**自愿**的——不接这个出口,这些 token 就不会出现在成本页里。
67
+ */
68
+ type VisionUsageSink = (usage: LLMUsage, model: string) => void;
69
+
70
+ /** World 自带辅助 VLM 的配置;模型选择与缓存均由 World 拥有。 */
71
+ export interface VisionConfig {
72
+ /** 总开关;true且OPENROUTER_API_KEY存在 → capabilities.auxVLM=true */
73
+ enabled: boolean;
74
+ /** OpenAI兼容API根地址(OpenRouter) */
75
+ baseUrl: string;
76
+ model: string;
77
+ /** 被动识图的固定prompt */
78
+ defaultPrompt: string;
79
+ maxTokens: number;
80
+ timeoutMs: number;
81
+ /** 每张图主动会话的消息数上限(含首轮被动识图问答;超出丢最早的追问对) */
82
+ sessionMaxMessages: number;
83
+ /** 被动识图VLM请求并发上限 */
84
+ concurrency: number;
85
+ /** 图片下载字节上限 */
86
+ maxImageBytes: number;
87
+ /**
88
+ * 消息渲染前,内容去重预判(下载+算hash+查是否与已识别过的图相同)愿意等待的上限。
89
+ * 命中或下载在此时限内失败 → 结果直接内联进消息,不再走占位+异步qq.vision事件。
90
+ * 超时(下载慢,或是需要真正跑VLM的新内容)→ 照常显示占位符,转交现有的被动识图
91
+ * 异步路径(不受此值影响,仍用timeoutMs)。
92
+ */
93
+ dedupPrecheckMs: number;
94
+ }
95
+ export const VISION_DEFAULTS: VisionConfig = {
96
+ enabled: false,
97
+ baseUrl: 'https://openrouter.ai/api/v1',
98
+ model: 'bytedance-seed/seed-2.0-mini',
99
+ defaultPrompt: VISION_DEFAULT_PROMPT,
100
+ maxTokens: 1024,
101
+ timeoutMs: 60000,
102
+ sessionMaxMessages: 24,
103
+ concurrency: 3,
104
+ maxImageBytes: 10485760,
105
+ dedupPrecheckMs: 2500,
106
+ };
107
+
108
+ interface VisionServiceDeps {
109
+ vlm: VLMClient;
110
+ cfg: VisionConfig;
111
+ dataDir: string;
112
+ timezone: string;
113
+ log?: Logger;
114
+ fetchImpl?: typeof fetch;
115
+ }
116
+
117
+ const MIME_EXT: Record<string, string> = {
118
+ 'image/png': 'png',
119
+ 'image/jpeg': 'jpg',
120
+ 'image/jpg': 'jpg',
121
+ 'image/gif': 'gif',
122
+ 'image/webp': 'webp',
123
+ 'image/bmp': 'bmp',
124
+ };
125
+
126
+ export class VisionService {
127
+ private readonly vlm: VLMClient;
128
+ private readonly cfg: VisionConfig;
129
+ private readonly timezone: string;
130
+ private readonly log: Logger;
131
+ private readonly fetchImpl: typeof fetch;
132
+ private readonly visionDir: string;
133
+ private readonly registryPath: string;
134
+
135
+ private readonly states = new Map<string, ImageState>();
136
+ private counter = 0;
137
+ /** 内容hash → 该内容首个成功识别的图片ID(内容去重索引;重启后从registry重建) */
138
+ private readonly hashIndex = new Map<string, string>();
139
+
140
+ /** 自带模型的用量上报出口(World 挂载时由 world.ts 接到 host.reportUsage) */
141
+ private usageSink?: VisionUsageSink;
142
+
143
+ /** 被动识图并发信号量 */
144
+ private running = 0;
145
+ private readonly waiters: Array<() => void> = [];
146
+
147
+ constructor(deps: VisionServiceDeps) {
148
+ this.vlm = deps.vlm;
149
+ this.cfg = deps.cfg;
150
+ this.timezone = deps.timezone;
151
+ this.log = deps.log ?? nullLogger();
152
+ this.fetchImpl = deps.fetchImpl ?? fetch;
153
+ this.visionDir = join(deps.dataDir, 'vision');
154
+ this.registryPath = join(this.visionDir, 'registry.jsonl');
155
+ if (!existsSync(this.visionDir)) mkdirSync(this.visionDir, { recursive: true });
156
+ this.load();
157
+ }
158
+
159
+
160
+ /**
161
+ * 分配 IMG-N 并登记;消息游标稍后回填。
162
+ * `opts.gif` 是基于 URL 后缀的初值,下载后以实际 MIME 类型为准。
163
+ */
164
+ registerImage(url: string, opts?: { gif?: boolean }): string {
165
+ const id = `IMG-${++this.counter}`;
166
+ const record: ImageRecord = { id, url, ts: shortTime(this.timezone) };
167
+ if (opts?.gif) record.gif = true;
168
+ this.states.set(id, { record, active: true, session: [], chain: Promise.resolve() });
169
+ // 未落消息号前只在内存;attachMessage后才持久化(避免崩溃留孤儿行)
170
+ return id;
171
+ }
172
+
173
+ /** 图片所在消息落库后回填它的平台message_id */
174
+ attachMessage(id: string, messageId: number | string | undefined): void {
175
+ const st = this.states.get(id);
176
+ if (!st) return;
177
+ if (messageId !== undefined) st.record.messageId = String(messageId);
178
+ this.persist(st.record);
179
+ }
180
+
181
+ /** 接上用量上报出口(World 挂载时调一次;不接就不归账) */
182
+ setUsageSink(sink: VisionUsageSink): void {
183
+ this.usageSink = sink;
184
+ }
185
+
186
+ private reportUsage(usage: LLMUsage): void {
187
+ try {
188
+ this.usageSink?.(usage, this.cfg.model);
189
+ } catch {
190
+ /* 归账失败不影响识图本身 */
191
+ }
192
+ }
193
+
194
+ /** 是否认识这个图片ID(qq_view_image用) */
195
+ hasImage(id: string): boolean {
196
+ return this.states.has(id);
197
+ }
198
+
199
+ /**
200
+ * 取某图的原始字节+mime+扩展名(已下载则读本地文件,否则现下载并落盘)。
201
+ * 找不到该id返回null;下载失败抛错。
202
+ */
203
+ async getImageBytes(
204
+ id: string,
205
+ ): Promise<{ buffer: Uint8Array; mime: string; ext: string } | null> {
206
+ const st = this.states.get(id);
207
+ if (!st) return null;
208
+ const { buffer, mime } = await this.ensureBytes(st);
209
+ return { buffer, mime, ext: MIME_EXT[mime] ?? 'img' };
210
+ }
211
+
212
+ /** 按字节内容找已识别过的描述:同一份字节无论从哪来(收藏、重发)都对得上 */
213
+ descriptionByHash(hash: string): string | undefined {
214
+ const id = this.hashIndex.get(hash);
215
+ return id ? this.states.get(id)?.record.desc : undefined;
216
+ }
217
+
218
+ /** 已登记图片数(web存储面板用) */
219
+ imageCount(): number {
220
+ return this.states.size;
221
+ }
222
+
223
+ /**
224
+ * 清空视觉缓存(web运维动作):内存注册表+session+磁盘(registry与图片字节)
225
+ * 删除全部记录并将 ID 计数器归零。历史消息中的旧 IMG-N 此后无法解析。
226
+ * 返回清掉的图片数。
227
+ */
228
+ clear(): number {
229
+ const n = this.states.size;
230
+ for (const state of this.states.values()) state.active = false;
231
+ this.states.clear();
232
+ this.hashIndex.clear();
233
+ this.counter = 0;
234
+ try {
235
+ rmSync(this.visionDir, { recursive: true, force: true });
236
+ } catch {
237
+ /* 删除失败不致命,下次写入覆盖 */
238
+ }
239
+ if (!existsSync(this.visionDir)) mkdirSync(this.visionDir, { recursive: true });
240
+ this.log.warn('视觉缓存已清空', { cleared: n });
241
+ return n;
242
+ }
243
+
244
+ /**
245
+ * 启动被动识图(fire-and-forget,并发受cfg.concurrency限制)。
246
+ * 幂等:同一id重复调用不会重复跑。完成时emit成功或失败事件——
247
+ * 无论成败都必须emit,不让"载入中"悬空。
248
+ */
249
+ startPassive(id: string, emit: VisionEmit): void {
250
+ const st = this.states.get(id);
251
+ if (!st || st.passivePromise) return;
252
+ st.passivePromise = this.runPassive(st, emit).catch((e) => {
253
+ // 顶层消费异常,维持 fire-and-forget 契约。
254
+ this.log.error('被动识图未捕获异常', { id, err: String(e) });
255
+ });
256
+ }
257
+
258
+ /**
259
+ * 消息渲染前的有时限"内容去重预判":只做下载+算hash+查重复,不碰VLM、
260
+ * 不占VLM并发槽(下载不是要限流的资源)。deadlineMs 内查完 → 命中去重或下载
261
+ * 失败均为终态,结果回填进record,供调用方直接内联进消息文字,不再走占位+
262
+ * 异步qq.vision事件。deadlineMs内没查完(下载慢,或是需要真正跑VLM的新内容)
263
+ * → 返回null,调用方照常展示占位符,之后startPassive复用这里已开始的下载,
264
+ * 不会重新发起。
265
+ */
266
+ async precheckDup(
267
+ id: string,
268
+ ): Promise<{ text: string; meta: Record<string, unknown> } | null> {
269
+ const st = this.states.get(id);
270
+ if (!st) return null;
271
+ let timer: ReturnType<typeof setTimeout> | undefined;
272
+ const timeout = new Promise<'timeout'>((resolve) => {
273
+ timer = setTimeout(() => resolve('timeout'), this.cfg.dedupPrecheckMs);
274
+ });
275
+ const raced = await Promise.race([this.resolveContent(st), timeout]);
276
+ if (timer) clearTimeout(timer);
277
+ if (!st.active) return null;
278
+ if (raced === 'timeout' || !raced.done) return null;
279
+ return raced.ok
280
+ ? { text: this.dupText(st.record, st.record.dupOf!), meta: this.metaOf(st.record, true) }
281
+ : { text: this.failText(st.record), meta: this.metaOf(st.record, false) };
282
+ }
283
+
284
+ /**
285
+ * 主动追问:在该图的VLM session里继续对话。
286
+ * - 被动尚未闭合 → 先等被动完成(继承其上下文)
287
+ * - session 已有内容 → 追加一条 user 消息
288
+ * - session为空(被动失败/重启后首问) → 基于持久化desc重建首轮,
289
+ * 无desc则以[图,prompt]起新session
290
+ * 同图并发调用经chain串行化。找不到图不由此处判断(module先查hasImage)。
291
+ */
292
+ ask(id: string, prompt: string): Promise<string> {
293
+ const st = this.states.get(id);
294
+ if (!st) return Promise.reject(new Error(`未知图片 ${id}`));
295
+ const p = st.chain.then(() => this.doAsk(st, prompt));
296
+ // chain 只维持串行顺序;结果与异常由 p 返回给调用方。
297
+ st.chain = p.then(
298
+ () => undefined,
299
+ () => undefined,
300
+ );
301
+ return p;
302
+ }
303
+
304
+
305
+ private async runPassive(st: ImageState, emit: VisionEmit): Promise<void> {
306
+ await this.acquire();
307
+ try {
308
+ if (!st.active) return;
309
+ const resolved = await this.resolveContent(st);
310
+ if (!st.active) return;
311
+ if (resolved.done) {
312
+ emit(
313
+ resolved.ok
314
+ ? this.dupText(st.record, st.record.dupOf!)
315
+ : this.failText(st.record),
316
+ this.metaOf(st.record, resolved.ok),
317
+ );
318
+ return;
319
+ }
320
+
321
+ st.session = [
322
+ {
323
+ role: 'user',
324
+ content: [
325
+ { type: 'image_url', image_url: { url: st.dataUrl! } },
326
+ { type: 'text', text: this.cfg.defaultPrompt },
327
+ ],
328
+ },
329
+ ];
330
+ try {
331
+ const { text, usage } = await this.vlm.chat(st.session);
332
+ this.reportUsage(usage);
333
+ if (!st.active) return;
334
+ st.session.push({ role: 'assistant', content: text });
335
+ st.record.desc = text;
336
+ this.persist(st.record);
337
+ this.hashIndex.set(st.record.hash!, st.record.id);
338
+ emit(this.okText(st.record, text), this.metaOf(st.record, true));
339
+ } catch (e) {
340
+ if (!st.active) return;
341
+ // VLM失败:图字节已在(dataUrl/文件),但session无有效被动轮 → 清空,
342
+ // 允许日后ask基于图片重起新session
343
+ st.session = [];
344
+ st.record.failedReason = this.shortReason(e, '识别失败');
345
+ this.persist(st.record);
346
+ emit(this.failText(st.record), this.metaOf(st.record, false));
347
+ }
348
+ } finally {
349
+ this.release();
350
+ }
351
+ }
352
+
353
+ /**
354
+ * 下载、哈希和内容去重共享同一 Promise,不占用 VLM 并发槽。
355
+ * done=true 表示重复或下载失败终态;done=false 表示 dataUrl 已就绪的新内容。
356
+ */
357
+ private resolveContent(st: ImageState): Promise<ResolveResult> {
358
+ if (!st.resolvePromise) st.resolvePromise = this.doResolveContent(st);
359
+ return st.resolvePromise;
360
+ }
361
+
362
+ private async doResolveContent(st: ImageState): Promise<ResolveResult> {
363
+ let buffer: Uint8Array, mime: string;
364
+ try {
365
+ ({ buffer, mime } = await this.ensureBytes(st));
366
+ } catch (e) {
367
+ if (!st.active) return { done: true, ok: false };
368
+ st.record.failedReason = this.shortReason(e, '下载失败');
369
+ this.persist(st.record);
370
+ return { done: true, ok: false };
371
+ }
372
+ // 下载响应的 MIME 覆盖渲染阶段基于后缀推断的 GIF 标记。
373
+ st.record.gif = mime === 'image/gif';
374
+ st.dataUrl = this.toDataUrl(buffer, mime);
375
+ const hash = (st.record.hash = this.hashOf(buffer));
376
+
377
+ // 内容去重:字节与之前某张成功识别过的图相同 → 沿用其描述,不再跑VLM、
378
+ // 也不把描述正文重复一遍(事件/内联文字只指向那张图的ID)
379
+ const dupId = this.hashIndex.get(hash);
380
+ const dupRec = dupId ? this.states.get(dupId)?.record : undefined;
381
+ if (dupId && dupId !== st.record.id && dupRec?.desc) {
382
+ st.record.desc = dupRec.desc;
383
+ st.record.dupOf = dupId;
384
+ this.persist(st.record);
385
+ return { done: true, ok: true };
386
+ }
387
+ return { done: false };
388
+ }
389
+
390
+
391
+ private async doAsk(st: ImageState, prompt: string): Promise<string> {
392
+ if (st.passivePromise) {
393
+ try {
394
+ await st.passivePromise;
395
+ } catch {
396
+ /* 被动失败不阻断主动;下面按session状态重建 */
397
+ }
398
+ }
399
+ if (!st.active) throw new Error(`图片 ${st.record.id} 已从缓存清除`);
400
+ const dataUrl = await this.ensureDataUrl(st);
401
+ if (!st.active) throw new Error(`图片 ${st.record.id} 已从缓存清除`);
402
+
403
+ if (st.session.length === 0) {
404
+ if (st.record.desc) {
405
+ // 重启后首问:用持久化desc重建被动首轮,再接追问
406
+ st.session.push({
407
+ role: 'user',
408
+ content: [
409
+ { type: 'image_url', image_url: { url: dataUrl } },
410
+ { type: 'text', text: this.cfg.defaultPrompt },
411
+ ],
412
+ });
413
+ st.session.push({ role: 'assistant', content: st.record.desc });
414
+ st.session.push({ role: 'user', content: prompt });
415
+ } else {
416
+ // 无被动描述(被动失败/从未跑) → 以[图,prompt]起新session
417
+ st.session.push({
418
+ role: 'user',
419
+ content: [
420
+ { type: 'image_url', image_url: { url: dataUrl } },
421
+ { type: 'text', text: prompt },
422
+ ],
423
+ });
424
+ }
425
+ } else {
426
+ st.session.push({ role: 'user', content: prompt });
427
+ }
428
+
429
+ // 发送前修剪:保证每次发给VLM的消息都不超上限(丢最早追问对,留带图首轮)
430
+ this.trimSession(st);
431
+ const { text, usage } = await this.vlm.chat(st.session);
432
+ this.reportUsage(usage);
433
+ if (!st.active) throw new Error(`图片 ${st.record.id} 已从缓存清除`);
434
+ st.session.push({ role: 'assistant', content: text });
435
+ return text;
436
+ }
437
+
438
+ /**
439
+ * session超容量时丢最早的追问对,永远保留首轮(带图的user + 首个assistant)。
440
+ * 首轮的user消息是唯一携带图片字节的消息,丢了模型就看不见图了。
441
+ */
442
+ private trimSession(st: ImageState): void {
443
+ const max = this.cfg.sessionMaxMessages;
444
+ if (st.session.length <= max) return;
445
+ const head = st.session.slice(0, 2);
446
+ let tail = st.session.slice(2);
447
+ while (head.length + tail.length > max && tail.length >= 2) {
448
+ tail = tail.slice(2);
449
+ }
450
+ st.session = [...head, ...tail];
451
+ }
452
+
453
+
454
+ /**
455
+ * 解析图像的原始字节与 MIME;本地文件缺失时下载并落盘。失败抛错。
456
+ * in-flight去重:并发调用(比如precheckDup超时后runPassive接手同一张图)
457
+ * 共享同一次下载,不会各发一次网络请求;失败后清空,允许下次重试。
458
+ */
459
+ private ensureBytes(st: ImageState): Promise<{ buffer: Uint8Array; mime: string }> {
460
+ if (!st.bytesPromise) {
461
+ st.bytesPromise = this.doEnsureBytes(st).catch((e) => {
462
+ st.bytesPromise = undefined;
463
+ throw e;
464
+ });
465
+ }
466
+ return st.bytesPromise;
467
+ }
468
+
469
+ private async doEnsureBytes(st: ImageState): Promise<{ buffer: Uint8Array; mime: string }> {
470
+ if (!st.active) throw new Error(`图片 ${st.record.id} 已从缓存清除`);
471
+ const rec = st.record;
472
+ if (rec.file) {
473
+ const path = join(this.visionDir, rec.file);
474
+ if (existsSync(path)) {
475
+ return { buffer: readFileSync(path), mime: rec.mime ?? 'image/jpeg' };
476
+ }
477
+ }
478
+ // 没有本地文件(下载曾失败/被清理)→ 现从url取
479
+ const { buffer, mime } = await this.download(rec.url);
480
+ if (!st.active) throw new Error(`图片 ${st.record.id} 已从缓存清除`);
481
+ const file = this.saveImage(rec.id, buffer, mime);
482
+ rec.file = file;
483
+ rec.mime = mime;
484
+ this.persist(rec);
485
+ return { buffer, mime };
486
+ }
487
+
488
+ /** 返回 VLM 请求使用的 data URL;缓存未命中时通过 ensureBytes 解析。 */
489
+ private async ensureDataUrl(st: ImageState): Promise<string> {
490
+ if (st.dataUrl) return st.dataUrl;
491
+ const { buffer, mime } = await this.ensureBytes(st);
492
+ st.dataUrl = this.toDataUrl(buffer, mime);
493
+ return st.dataUrl;
494
+ }
495
+
496
+ private async download(url: string): Promise<{ buffer: Uint8Array; mime: string }> {
497
+ if (!url) throw new Error('图片地址为空');
498
+ const ac = new AbortController();
499
+ const timer = setTimeout(() => ac.abort(), this.cfg.timeoutMs);
500
+ try {
501
+ const res = await this.fetchImpl(url, { signal: ac.signal });
502
+ if (!res.ok) throw new Error(`HTTP ${res.status}`);
503
+ const ct = res.headers.get('content-type');
504
+ const ab = await res.arrayBuffer();
505
+ if (ab.byteLength > this.cfg.maxImageBytes) {
506
+ throw new Error(`图片过大(${ab.byteLength}字节 > 上限${this.cfg.maxImageBytes})`);
507
+ }
508
+ return { buffer: new Uint8Array(ab), mime: this.pickMime(ct, url) };
509
+ } catch (e) {
510
+ if (e instanceof Error && e.name === 'AbortError') throw new Error('下载超时');
511
+ throw e;
512
+ } finally {
513
+ clearTimeout(timer);
514
+ }
515
+ }
516
+
517
+ private pickMime(contentType: string | null, url: string): string {
518
+ if (contentType) {
519
+ const mime = contentType.split(';')[0].trim().toLowerCase();
520
+ if (mime.startsWith('image/')) return mime;
521
+ }
522
+ const ext = (url.split('?')[0].match(/\.([a-zA-Z0-9]+)$/)?.[1] ?? '').toLowerCase();
523
+ for (const [mime, e] of Object.entries(MIME_EXT)) {
524
+ if (e === ext) return mime;
525
+ }
526
+ return 'image/jpeg';
527
+ }
528
+
529
+ private saveImage(id: string, buffer: Uint8Array, mime: string): string {
530
+ const ext = MIME_EXT[mime] ?? 'img';
531
+ const file = `${id}.${ext}`;
532
+ writeFileSync(join(this.visionDir, file), buffer);
533
+ return file;
534
+ }
535
+
536
+ private toDataUrl(buffer: Uint8Array, mime: string): string {
537
+ return `data:${mime};base64,${Buffer.from(buffer).toString('base64')}`;
538
+ }
539
+
540
+
541
+ /** GIF提醒:VLM只看得到第一帧,附在描述前(完整GIF支持以后再做) */
542
+ private gifNote(rec: ImageRecord): string {
543
+ return rec.gif ? '(GIF; the vision model sees only the first frame)' : '';
544
+ }
545
+
546
+ private okText(rec: ImageRecord, desc: string): string {
547
+ return `[${shortTime(this.timezone)}] [vision] ${rec.id}(re #${rec.messageId ?? '?'}): ${this.gifNote(rec)}${desc}`;
548
+ }
549
+
550
+ private failText(rec: ImageRecord): string {
551
+ return `[${shortTime(this.timezone)}] [vision] ${rec.id}(re #${rec.messageId ?? '?'}): ${this.gifNote(rec)}recognition failed — ${rec.failedReason ?? 'unknown reason'}`;
552
+ }
553
+
554
+ /** 内容去重命中:不复述描述正文,只指向沿用的那张图,让主agent知道是同一张 */
555
+ private dupText(rec: ImageRecord, dupOf: string): string {
556
+ return `[${shortTime(this.timezone)}] [vision] ${rec.id}(re #${rec.messageId ?? '?'}): ${this.gifNote(rec)}same image as ${dupOf} (identical content, already recognized there — not re-run, see ${dupOf}'s description)`;
557
+ }
558
+
559
+ private metaOf(rec: ImageRecord, ok: boolean): Record<string, unknown> {
560
+ return {
561
+ image_id: rec.id,
562
+ of_message_id: rec.messageId,
563
+ ok,
564
+ ...(rec.dupOf ? { dup_of: rec.dupOf } : {}),
565
+ };
566
+ }
567
+
568
+ /** 图片字节的sha256(内容去重键) */
569
+ private hashOf(buffer: Uint8Array): string {
570
+ return createHash('sha256').update(buffer).digest('hex');
571
+ }
572
+
573
+ private shortReason(e: unknown, fallback: string): string {
574
+ const msg = e instanceof Error ? e.message : String(e);
575
+ return msg && msg.length <= 60 ? msg : fallback;
576
+ }
577
+
578
+
579
+ private async acquire(): Promise<void> {
580
+ if (this.running < this.cfg.concurrency) {
581
+ this.running++;
582
+ return;
583
+ }
584
+ await new Promise<void>((r) => this.waiters.push(r));
585
+ this.running++;
586
+ }
587
+
588
+ private release(): void {
589
+ this.running--;
590
+ const w = this.waiters.shift();
591
+ if (w) w();
592
+ }
593
+
594
+
595
+ private persist(rec: ImageRecord): void {
596
+ try {
597
+ appendFileSync(this.registryPath, JSON.stringify(rec) + '\n', 'utf8');
598
+ } catch (e) {
599
+ this.log.warn('vision registry写入失败', { err: String(e) });
600
+ }
601
+ }
602
+
603
+ /** 启动时从registry.jsonl重建:同id最新覆盖旧的,计数器接续 */
604
+ private load(): void {
605
+ if (!existsSync(this.registryPath)) return;
606
+ let content: string;
607
+ try {
608
+ content = readFileSync(this.registryPath, 'utf8');
609
+ } catch {
610
+ return;
611
+ }
612
+ let maxN = 0;
613
+ for (const line of content.split('\n')) {
614
+ const s = line.trim();
615
+ if (!s) continue;
616
+ let rec: ImageRecord;
617
+ try {
618
+ rec = JSON.parse(s) as ImageRecord;
619
+ } catch {
620
+ continue;
621
+ }
622
+ if (!rec.id) continue;
623
+ this.states.set(rec.id, {
624
+ record: rec,
625
+ active: true,
626
+ session: [],
627
+ chain: Promise.resolve(),
628
+ });
629
+ const n = Number(rec.id.replace(/^IMG-/, ''));
630
+ if (Number.isFinite(n) && n > maxN) maxN = n;
631
+ }
632
+ this.counter = maxN;
633
+ // 内容去重索引重建:只收canonical记录(有desc且不是dup),按最终状态,与写入顺序无关
634
+ for (const st of this.states.values()) {
635
+ const rec = st.record;
636
+ if (rec.hash && rec.desc && !rec.dupOf && !this.hashIndex.has(rec.hash)) {
637
+ this.hashIndex.set(rec.hash, rec.id);
638
+ }
639
+ }
640
+ this.log.debug('vision registry重建完成', { images: this.states.size, counter: this.counter });
641
+ }
642
+ }