abelien-harness 0.6.2 → 0.6.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "abelien-harness",
3
- "version": "0.6.2",
3
+ "version": "0.6.3",
4
4
  "description": "npx-installable multi-agent skill harness: generic kernel + pluggable domain packs",
5
5
  "type": "module",
6
6
  "engines": {
@@ -127,10 +127,20 @@ export function createPublicReader({ codes, userAgent, accept, decodeCharset = f
127
127
  try { addresses = await (lookup ? lookup(url.hostname) : dnsTransport === "doh" ? dohLookup(url.hostname, signal, dnsRequest) : dns.lookup(url.hostname, { all: true, verbatim: true })); }
128
128
  catch (err) { if (publicReadFailure(err) || networkError(err)) throw readFailure("dns"); throw err; }
129
129
  if (!Array.isArray(addresses) || !addresses.length || addresses.some(item => !publicAddress(item?.address) || isIP(item.address) !== item.family)) throw readFailure("access");
130
- connections.push({ url: url.href, tls_hostname: url.hostname, addresses, address: addresses[0].address, family: addresses[0].family });
130
+ // IPv4 first: a host that publishes AAAA records first (cdn.shopify.com) otherwise hangs until the read timeout on a
131
+ // network without a working IPv6 route, where curl would have fallen back. A refused or unreachable address moves
132
+ // on to the next validated one; every address tried was checked above, so the connection stays pinned.
133
+ const ordered = [...addresses.filter(item => item.family === 4), ...addresses.filter(item => item.family !== 4)];
131
134
  let response;
132
- try { response = await request(url.href, { address: addresses[0].address, family: addresses[0].family, signal, ...(maxBytes === undefined ? {} : { maxBytes }) }); }
133
- catch (error) { if (publicReadFailure(error)) throw error; if (networkError(error)) throw readFailure("transport"); throw error; }
135
+ for (const [index, pinned] of ordered.entries()) {
136
+ connections.push({ url: url.href, tls_hostname: url.hostname, addresses, address: pinned.address, family: pinned.family });
137
+ try { response = await request(url.href, { address: pinned.address, family: pinned.family, signal, ...(maxBytes === undefined ? {} : { maxBytes }) }); break; }
138
+ catch (error) {
139
+ if (publicReadFailure(error)) throw error;
140
+ if (!networkError(error)) throw error;
141
+ if (index === ordered.length - 1 || signal?.aborted) throw readFailure("transport");
142
+ }
143
+ }
134
144
  const bytes = Buffer.from(response.bytes ?? []), headers = response.headers ?? {};
135
145
  if ([301, 302, 303, 307, 308].includes(response.status)) {
136
146
  if (n === maxRedirects || !headers.location) throw readFailure("redirect");
@@ -21,7 +21,7 @@ compatibility:
21
21
  ## 流程
22
22
 
23
23
  1. **制定检索计划**:依据用户原话和 clarify 的意图判定,写出最多 6 条英文或中文检索,每条注明用途:`identify` 识别、`teardown` 逆向资料、`similar` 相似产品。用户给了链接时,链接优先读取。
24
- 2. **收集**:调用 `reference.research`。它执行 Volc 检索(每条最多 5 个结果)、对公开 HTTPS 页面做安全直读、抓取页面主图、测量用户原图(PNG)的轮廓高宽比,然后把证据保存为 `reference-evidence`(status=collected)。同一请求重复调用直接返回已有结果,不会重复产生检索费用。
24
+ 2. **收集**:调用 `reference.research`。它执行 Volc 检索(每条最多 5 个结果)、对公开 HTTPS 页面做安全直读、为每个读到的页面找一张图、测量用户原图(PNG)的轮廓高宽比,然后把证据保存为 `reference-evidence`(status=collected)。图片格式按字节判断,宿主有转换器时统一存成 PNG;保存字节的 sha256 就是图片 id,画布只展示有图片 id 的条目,所以一般不要调小 `max_images`。同一请求重复调用直接返回已有结果,不会重复产生检索费用。
25
25
  3. **阅读证据并判断**:用 `ab resource get` 读取收集结果,用页面摘录(`excerpt`)和正文判断:哪个候选最可能是用户要的产品、它的尺寸是多少;哪些相似产品可以借用、借用什么、为什么、哪些不照搬;哪些条目应排除。
26
26
  4. **核验**:按 `schemas/reference-judgement.v1.json` 的格式调用 `reference.finalize` 提交判断,生成 status=final 的结果。还原与改现成必须显式给出 `identity.candidates` 和 `chosen_item_id`;定不下型号就把 `chosen_item_id` 设为 `null`,并在 `unidentified_reason` 写明原因。不要用空的 `identity` 试格式。核验规则:
27
27
  - 引用必须逐字出现在已保存的页面正文中;
@@ -48,11 +48,11 @@ node "$AB" --workdir <作品目录> resource get reference-evidence:<id>
48
48
  node "$AB" --workdir <作品目录> capability run reference.finalize --input-file judgement.json
49
49
  ```
50
50
 
51
- 参考图(base64 PNG)会超过命令行长度限制,所以输入用 `--input-file` 传入,文件必须放在作品目录内。宿主负责把 JPEG、WebP 等格式先转成 PNG,本包只解码 PNG。字段、上限、错误码和判定规则见[参考研究契约](references/research-contract.md);数据格式见 `schemas/reference-evidence.v1.json`,样例见 `fixtures/`。
51
+ 参考图(base64 PNG)会超过命令行长度限制,所以输入用 `--input-file` 传入,文件必须放在作品目录内。用户原图由宿主先转成 PNG;页面图片由宿主提供的转换器(`ABELIEN_REFERENCE_IMAGE_CONVERTER`)统一转成 PNG,本包只解码 PNG。字段、上限、错误码和判定规则见[参考研究契约](references/research-contract.md);数据格式见 `schemas/reference-evidence.v1.json`,样例见 `fixtures/`。
52
52
 
53
53
  ## 边界
54
54
 
55
- - **不编造**:读不到的页面、打不开的链接、测量不了的图片,一律记为不可用或未知,并写入 `unknowns`;其他检索照常继续。
55
+ - **不编造**:读不到的页面、打不开的链接记为不可用(原因在 `pages[].reason`),测量不了的图片记为未知;`unknowns` 只写真正待核实的缺口,不逐条罗列打不开的页面。其他检索照常继续。
56
56
  - **第三方图片只作参考**:保存时附带来源和哈希,不进入图鉴、不进入公开分享,也不当作作品资产。
57
57
  - **找不到完整的相似产品时**,按声学、供电、交互等子系统分别参考,不硬凑数量。
58
58
  - **来源产品的功能不等于本次要实现的功能**:还原时,功能按目标深度由 clarify 写入候选需求。
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "abelien-reference",
3
- "version": "0.3.0",
3
+ "version": "0.4.0",
4
4
  "description": "参考研究域包:按 Agent 的检索计划收集产品识别、逆向资料与相似产品证据(检索、公开页面、主图、用户原图测量),再核验 Agent 的判断,产出 reference-evidence v1;判定码来自确定性检查",
5
5
  "requires": [
6
6
  "abelien-harness"
@@ -50,7 +50,6 @@
50
50
  "excerpt": "床头故事机 A 腔体容积 300 mL,40 mm 全频喇叭,电池 2000 mAh。",
51
51
  "final_url": "https://shop.example.com/story-a",
52
52
  "id": "p1",
53
- "image_url": null,
54
53
  "purpose": "similar",
55
54
  "raw_sha256": "db701b787482d7ae3c2967323df3013d93df7c7e628a801ab091218f55a1b87d",
56
55
  "reason": null,
@@ -59,7 +58,8 @@
59
58
  "text_blob_ref": "sha256:a871926ca4bf9606ff9abc75f7b5da5a99a41c7535f03e9572095b1a272b32bc",
60
59
  "text_sha256": "58a51339bbda6d0a69d726e2768faf47752f8529630a4705e5fe81552e187dff",
61
60
  "title": "床头故事机 A",
62
- "url": "https://shop.example.com/story-a"
61
+ "url": "https://shop.example.com/story-a",
62
+ "image_candidates": []
63
63
  }
64
64
  ],
65
65
  "request_digest": "c26edd6284aed5b521396a1093bc0aeb9ff60d24f2c68d6bcf98a6529bd9c325",
@@ -78,7 +78,9 @@
78
78
  "text_blob_ref": "sha256:a9d6a6d8668de3346efe1d94c1afd76e71faf37bee5a6e55d3a82f9bc60b9bf0",
79
79
  "text_sha256": "bae09c2d8ac69290d90e7cf69d7809858b1149a391400d40030037db41896415",
80
80
  "excerpt": "小爱音箱 Pro 参数 产品尺寸:118 × 118 × 215 mm,重量 1187 g。",
81
- "image_url": "https://img.example.com/pro.png"
81
+ "image_candidates": [
82
+ "https://img.example.com/pro.png"
83
+ ]
82
84
  },
83
85
  {
84
86
  "id": "p2",
@@ -93,7 +95,7 @@
93
95
  "text_blob_ref": "sha256:3dd43ad1e906a2e95af747cc8f8386c00b43df981af8b8bafe45bb2d173e7e3c",
94
96
  "text_sha256": "2de9e3c9406bd62a2e9eef887fd42008382616377ac49fb751cd5a109f1e9a00",
95
97
  "excerpt": "小米AI音箱 规格 尺寸 88 mm × 88 mm × 211.6 mm",
96
- "image_url": null
98
+ "image_candidates": []
97
99
  }
98
100
  ],
99
101
  "images": [
@@ -128,7 +130,8 @@
128
130
  255
129
131
  ]
130
132
  }
131
- }
133
+ },
134
+ "source_media_type": "image/png"
132
135
  }
133
136
  ],
134
137
  "items": [
@@ -41,7 +41,8 @@
41
41
  "reason": null,
42
42
  "sha256": "da0fadd6fd3ab5ccfbac94ca4e3f2b48e440987eee6834f1c30f04c02abd5d11",
43
43
  "status": "read",
44
- "url": "https://img.example.com/pro.png"
44
+ "url": "https://img.example.com/pro.png",
45
+ "source_media_type": "image/png"
45
46
  }
46
47
  ],
47
48
  "intent": "restore",
@@ -122,7 +123,6 @@
122
123
  "excerpt": "小爱音箱 Pro 参数 产品尺寸:118 × 118 × 215 mm,重量 1187 g。",
123
124
  "final_url": "https://www.mi.example.com/pro/specs",
124
125
  "id": "p1",
125
- "image_url": "https://img.example.com/pro.png",
126
126
  "purpose": "identify",
127
127
  "raw_sha256": "109cbac90dd51ef3dddfd21c4a97168a25f097b4211b2146d21669281d06e3b8",
128
128
  "reason": null,
@@ -131,13 +131,15 @@
131
131
  "text_blob_ref": "sha256:a9d6a6d8668de3346efe1d94c1afd76e71faf37bee5a6e55d3a82f9bc60b9bf0",
132
132
  "text_sha256": "bae09c2d8ac69290d90e7cf69d7809858b1149a391400d40030037db41896415",
133
133
  "title": "小爱音箱 Pro 参数",
134
- "url": "https://www.mi.example.com/pro/specs"
134
+ "url": "https://www.mi.example.com/pro/specs",
135
+ "image_candidates": [
136
+ "https://img.example.com/pro.png"
137
+ ]
135
138
  },
136
139
  {
137
140
  "excerpt": "小米AI音箱 规格 尺寸 88 mm × 88 mm × 211.6 mm",
138
141
  "final_url": "https://www.mi.example.com/gen1/specs",
139
142
  "id": "p2",
140
- "image_url": null,
141
143
  "purpose": "identify",
142
144
  "raw_sha256": "c139f165c41dc34c2cd30a816a80d7cbef9ecbda49e7496d46163c8692da33c7",
143
145
  "reason": null,
@@ -146,7 +148,8 @@
146
148
  "text_blob_ref": "sha256:3dd43ad1e906a2e95af747cc8f8386c00b43df981af8b8bafe45bb2d173e7e3c",
147
149
  "text_sha256": "2de9e3c9406bd62a2e9eef887fd42008382616377ac49fb751cd5a109f1e9a00",
148
150
  "title": "小米AI音箱 规格",
149
- "url": "https://www.mi.example.com/gen1/specs"
151
+ "url": "https://www.mi.example.com/gen1/specs",
152
+ "image_candidates": []
150
153
  }
151
154
  ],
152
155
  "request_digest": "40ece5312c4ecbf6f5548d36589de25734706b8a333d22e1a3c32adeb75e09f5",
@@ -17,23 +17,29 @@
17
17
  | `links` | 最多 5 个公开 HTTPS 链接,不得带凭据或指向内网 |
18
18
  | `user_image` | 可选,`{media_type: "image/png", base64}`,不超过 16MB;其他格式由宿主先转成 PNG |
19
19
  | `plan.queries` | 最多 6 条,每条为 `{purpose: identify / teardown / similar, query, count?: 1–5}` |
20
- | `plan.max_pages` / `plan.max_images` | 页面最多读几页(0–12,默认 8)、主图最多抓几张(0–8,默认 6) |
20
+ | `plan.max_pages` / `plan.max_images` | 页面最多读几页(0–12,默认 8)、最多为几个页面找图(0–12,默认等于 `max_pages`,即每个读到的页面都找图)。画布只展示有图的条目,一般不要调小 `max_images` |
21
21
 
22
22
  ### 执行
23
23
 
24
24
  - **检索**:只用宿主环境变量 `WEB_SEARCH_API_KEY` 调 Volc。原始回执先脱敏,再计算摘要。单条检索失败时,该条记为 `failed`,并写明错误码,其余照常进行;缺少 key 时,在发出任何请求之前就以 `REFERENCE_WEB_CONFIG` 失败。
25
25
  - **页面**:先读链接,再从各条检索结果中轮流取页面,避免某一条检索占满读取预算。读取只走内核的安全直读:只读公网 HTTPS,每一跳都重新解析并钉住地址,限制重定向次数。每页最多读 8MB,超出时记为 `too_large`、不可用。正文存为内容 blob,资源里只保留前 1200 字的摘录。
26
- - **主图**:取页面的 `og:image`、`twitter:image` 或 `image_src`;路径像头像、logo、图标、二维码的(如 `avatar`、`logo`、`icon`、`qrcode`)跳过,改取该页下一个候选。图片同样走钉住地址的直读,每张不超过 4MB,只接受 PNG、JPEG、WebP,原始字节存为内容 blob。其中的 PNG 会额外测量轮廓。
26
+ - **页面图片**:每个读到的页面最多保存一张图。候选按顺序取:分享图(`og:image`、`twitter:image`、`itemprop=image`)、`image_src`、JSON-LD 的 `image`、正文中没有标成小于 200 像素的 `<img>`(优先 `data-src`),最多 4 个,记在页面的 `image_candidates`。路径或域名像头像、logo、图标、二维码的(如 `avatar`、`logo`、`icon`、`qrcode`)和 `data:` 地址不算候选;`http` 地址改走 `https`。逐个尝试,第一张读得到、能识别为图片、短边不小于 200 像素的被保存,后面的候选不再读取。
27
+ - 下载同样走钉住地址的直读,每张最多 8MB、15 秒。响应头只用来粗筛(`image/*` 或二进制流),格式**按字节判断**:PNG、JPEG、GIF、WebP、AVIF、HEIC、BMP、TIFF、ICO、SVG 都能识别;响应头写的格式不算数(CDN 常把 JPEG 标成 `image/png`)。
28
+ - 宿主提供了转换器(环境变量 `ABELIEN_REFERENCE_IMAGE_CONVERTER`,见「宿主责任」)时,每张图都交给它转成 PNG:长边不超过 1024 像素,不超过 2MB。转换器能解码的格式都能用,PNG 还会测量轮廓。没有转换器时,只保存按字节确认是 PNG、JPEG、WebP 且不超过 2MB 的原图,其他格式记为 `needs_converter`。
29
+ - 保存的字节存为内容 blob,它的 sha256 就是这张图的 **id**:`images[].sha256` 与引用它的 `items[].image.sha256` 相同,宿主按这个 id 转存和展示。同一张图出现在多个页面时 id 相同。`images[].media_type` 是保存字节的真实格式,`source_media_type` 是下载到的原图格式。
30
+ - 都没成功时,该页的图片记为 `unavailable`,`reason` 是最后一个候选的失败原因(如 `timeout`、`http_status`、`not_image`、`needs_converter`、`too_small`、`convert_failed`),条目的 `image` 为 `null`。
27
31
  - **用户原图**:存为内容 blob 并测量轮廓(`silhouette-v1`),得到包围盒和高宽比;背景必须是单色或透明。背景杂乱、没有主体、隔行 PNG、超大图都记为 `unknown`,并写明原因。
28
32
  - **幂等**:结果 id 由完整请求摘要(包括原图哈希)派生。已有同 id 的收集结果时直接返回,不再检索、不再读取。因此单条检索或页面的临时失败不会因为原样重试而自动恢复;要重试,就改写检索计划(比如换个检索词、改结果数),形成新的请求。
29
33
 
30
34
  ### 输出
31
35
 
32
- 输出 `reference-evidence`,status=collected。`items` 中第一项是 `user_input`,其后每个页面对应一格:识别用途为 `candidate`,逆向用途为 `teardown`,相似用途为 `similar`。读不到的页面状态为 `unavailable`。此时还没有任何 `verdict`。
36
+ 输出 `reference-evidence`,status=collected。`items` 中第一项是 `user_input`,其后每个页面对应一格:识别用途为 `candidate`,逆向用途为 `teardown`,相似用途为 `similar`。读不到的页面状态为 `unavailable`,原因记在 `pages[].reason`,不再重复写进 `unknowns`;`unknowns` 只放真正待核实的缺口(检索失败或无结果、没读到可识别产品的页面、原图测不了等)。此时还没有任何 `verdict`。
37
+
38
+ 条目的 `image` 只能指向本份证据保存过的图(`images` 中 status=read 的一张,或用户原图),格式只能是 PNG、JPEG、WebP;校验不通过即 `VALIDATION`。
33
39
 
34
40
  `budget` 记录本次用量和截断:`queries`、`pages`、`images` 是已检索条数、已读页面数、已抓主图数;`skipped_pages` 是检索到、但因 `max_pages` 没有读取的去重公开页面数,`skipped_images` 是有主图、但因 `max_images` 没有抓取的页面数。宿主据此说明「已达本阶段检索上限」,不能把没读到当作没有。final 结果原样带上收集时的 `budget`。
35
41
 
36
- 请求摘要包含证据格式版本(`EVIDENCE_SHAPE`,当前为 2)。旧版本 harness 收集的证据保留原 id,不会被复用,也不会被覆盖;同一请求按新格式重新收集,得到新 id。
42
+ 请求摘要包含证据格式版本(`EVIDENCE_SHAPE`,当前为 3:图片格式按字节判断、可经宿主转换器统一为 PNG)。旧版本 harness 收集的证据保留原 id,不会被复用,也不会被覆盖;同一请求按新格式重新收集,得到新 id。
37
43
 
38
44
  ## reference.finalize
39
45
 
@@ -69,14 +75,16 @@
69
75
  | 码 | 含义 |
70
76
  | --- | --- |
71
77
  | `VALIDATION` | 输入不合规、引用不在原文、尺寸不在引用中、条目种类不符 |
72
- | `REFERENCE_WEB_CONFIG` | 缺少 `WEB_SEARCH_API_KEY`,或 DNS 配置非法 |
78
+ | `REFERENCE_WEB_CONFIG` | 缺少 `WEB_SEARCH_API_KEY`、DNS 配置非法,或配置了转换器但路径不是已存在的绝对路径;都在发出任何请求之前失败 |
73
79
  | `REFERENCE_WEB_*` | 单条检索、读取的失败码,只记在对应的检索、页面或图片里,不使整次运行失败 |
74
80
  | `REFERENCE_HOST` | 宿主没有提供可信的资源读取、内容读写端口 |
75
81
 
76
82
  ## 宿主责任
77
83
 
78
84
  - 把上传的图片转成 PNG,写入作品目录中的输入文件,再用 `--input-file` 传入。
85
+ - 提供图片转换器,并把它的绝对路径放进环境变量 `ABELIEN_REFERENCE_IMAGE_CONVERTER`(可执行文件,或用当前 Node 运行的 `.mjs`/`.cjs`/`.js` 脚本)。协议:标准输入是下载到的原图字节;参数是 `--max-side <像素> --max-bytes <字节>`;标准输出是一张 PNG,长边和大小都不超过参数,按 EXIF 摆正、动图取第一帧、不带元数据;无法转换时以非零码退出。超过 20 秒、输出不是 PNG 或超出限制都视为失败,该候选跳过。转换器只拿到环境变量 `PATH`。
86
+ - 只展示有图片 id 的条目:按 `items[].image.sha256` 转存和读取;没有图的条目、打不开的页面不上画布网格。
79
87
  - 定稿工具的参数定义直接用 `schemas/reference-judgement.v1.json`,`evidence_id` 由宿主填入,不让模型猜格式。
80
88
  - 定稿结果登记为本次需求修订的参考来源后,不再被同一修订的后续定稿替换;更正走需求修订(用户换选或修改需求),见 SKILL 的 K6。
81
- - 只展示 `items` 和 `unknowns`;`verdict` 原样展示,不重新解释,也不改写。
89
+ - `verdict` 原样展示,不重新解释,也不改写;`unknowns` 如实展示。
82
90
  - 第三方图片从内容 blob 转存到私有存储,展示时附来源,不进入图鉴或公开分享。
@@ -99,16 +99,21 @@
99
99
  "status": { "enum": ["read", "unavailable"] },
100
100
  "text_blob_ref": { "oneOf": [{ "type": "null" }, { "$ref": "#/$defs/blobRef" }] },
101
101
  "excerpt": { "type": "string" },
102
- "image_url": { "type": ["string", "null"] }
102
+ "image_candidates": { "description": "页面上按尝试顺序排列的候选图片地址(分享图、image_src、JSON-LD、正文大图),最多 4 个,已排除头像、logo 等", "type": "array", "items": { "type": "string" } }
103
103
  }
104
104
  },
105
105
  "image": {
106
+ "description": "每个页面最多一张图:按候选顺序尝试,第一张读到、格式可识别、短边不小于 200 像素的被保存。有宿主转换器时统一保存为 PNG;保存字节的 sha256 就是这张图的 id",
106
107
  "type": "object",
107
108
  "required": ["id", "url", "page_id", "status"],
108
109
  "properties": {
110
+ "url": { "type": "string", "description": "保存的那张图的地址;都失败时为最后尝试的地址" },
109
111
  "status": { "enum": ["read", "unavailable"] },
112
+ "reason": { "type": ["string", "null"], "description": "unavailable 时最后一个候选的失败原因,如 timeout、not_image、needs_converter、too_small" },
110
113
  "blob_ref": { "oneOf": [{ "type": "null" }, { "$ref": "#/$defs/blobRef" }] },
111
- "media_type": { "type": ["string", "null"] }
114
+ "sha256": { "oneOf": [{ "type": "null" }, { "$ref": "#/$defs/sha256" }] },
115
+ "media_type": { "description": "保存字节的真实格式,按字节判断,不取响应头", "enum": ["image/png", "image/jpeg", "image/webp", null] },
116
+ "source_media_type": { "description": "下载到的原图格式(按字节判断);转换前的格式,未识别为图片时为 null", "type": ["string", "null"] }
112
117
  }
113
118
  },
114
119
  "verdict": {
@@ -129,7 +134,7 @@
129
134
  "title": { "type": "string" },
130
135
  "role": { "type": "string" },
131
136
  "source": { "oneOf": [{ "type": "null" }, { "type": "object", "required": ["url", "site"], "properties": { "url": { "type": "string" }, "site": { "type": "string" } } }] },
132
- "image": { "oneOf": [{ "type": "null" }, { "type": "object", "required": ["blob_ref", "sha256", "media_type"], "properties": { "blob_ref": { "$ref": "#/$defs/blobRef" }, "sha256": { "$ref": "#/$defs/sha256" }, "media_type": { "type": "string" } } }] },
137
+ "image": { "description": "这一格的图:sha256 是图片 id,宿主按它转存与读取;没有图时为 null", "oneOf": [{ "type": "null" }, { "type": "object", "required": ["blob_ref", "sha256", "media_type"], "properties": { "blob_ref": { "$ref": "#/$defs/blobRef" }, "sha256": { "$ref": "#/$defs/sha256" }, "media_type": { "enum": ["image/png", "image/jpeg", "image/webp"] } } }] },
133
138
  "verdict": { "oneOf": [{ "type": "null" }, { "$ref": "#/$defs/verdict" }] },
134
139
  "status": { "enum": ["pending", "ready", "excluded", "unavailable"] },
135
140
  "query": { "type": ["string", "null"], "description": "pending 时为真实检索词" }
@@ -4,6 +4,7 @@ import { createPublicReader } from "../../abelien-harness/scripts/web/public-rea
4
4
  import { createVolcSearch } from "../../abelien-harness/scripts/web/volc.mjs";
5
5
  import { redact } from "../../abelien-harness/scripts/web/redact.mjs";
6
6
  import { failure, text, record } from "../../abelien-harness/scripts/web/common.mjs";
7
+ import { PASS_THROUGH } from "./image-format.mjs";
7
8
 
8
9
  export const SCHEMA = "abelien.reference-evidence.v1";
9
10
  export const INTENTS = Object.freeze(["restore", "create", "modify"]);
@@ -17,18 +18,27 @@ export const KIND_BY_PURPOSE = Object.freeze({ identify: "candidate", teardown:
17
18
 
18
19
  /**
19
20
  * Version of the collected shape, part of every request digest: evidence collected by an older harness (e.g. without
20
- * budget.skipped_*) keeps its own id and is never reused or overwritten; the same request collects afresh under a new id.
21
+ * budget.skipped_*, or with an image whose media_type came from the response header) keeps its own id and is never
22
+ * reused or overwritten; the same request collects afresh under a new id.
21
23
  */
22
- export const EVIDENCE_SHAPE = 2;
23
- export const LIMITS = Object.freeze({ queries: 6, count: 5, pages: 12, images: 8, links: 5, textChars: 4000, queryChars: 200,
24
- pageBytes: 8 * 1024 * 1024, imageBytes: 4 * 1024 * 1024, userImageBytes: 16 * 1024 * 1024, excerptChars: 1200, pageTimeoutMs: 30000, searchTimeoutMs: 30000, concurrency: 4 });
24
+ export const EVIDENCE_SHAPE = 3;
25
+ /**
26
+ * imageBytes caps a download; savedImageBytes and imageSide cap what is saved, so the host can keep every saved image as
27
+ * it is. imageCandidates is how many pictures of one page are tried before the page is left without one.
28
+ */
29
+ export const LIMITS = Object.freeze({ queries: 6, count: 5, pages: 12, images: 12, links: 5, textChars: 4000, queryChars: 200,
30
+ pageBytes: 8 * 1024 * 1024, imageBytes: 8 * 1024 * 1024, savedImageBytes: 2 * 1024 * 1024, imageSide: 1024, imageCandidates: 4,
31
+ userImageBytes: 16 * 1024 * 1024, excerptChars: 1200, pageTimeoutMs: 30000, imageTimeoutMs: 15000, convertTimeoutMs: 20000, searchTimeoutMs: 30000, concurrency: 4 });
25
32
 
26
33
  export const CODES = Object.freeze({ transport: "REFERENCE_WEB_TRANSPORT", dns: "REFERENCE_WEB_DNS", access: "REFERENCE_WEB_URL", redirect: "REFERENCE_WEB_PAGE",
27
34
  timeout: "REFERENCE_WEB_TIMEOUT", url: "REFERENCE_WEB_URL", config: "REFERENCE_WEB_CONFIG", readTimeout: "REFERENCE_PUBLIC_READ_TIMEOUT",
28
35
  capability: "REFERENCE_WEB_CAPABILITY", provider: "REFERENCE_WEB_PROVIDER", auth: "REFERENCE_WEB_AUTH", rateLimit: "REFERENCE_WEB_RATE_LIMIT", response: "REFERENCE_WEB_RESPONSE" });
29
36
 
30
37
  export const reader = createPublicReader({ codes: CODES, userAgent: "Abelien-Reference-Evidence/1", accept: "text/html,text/plain,application/xhtml+xml", decodeCharset: true });
31
- export const imageReader = createPublicReader({ codes: CODES, userAgent: "Abelien-Reference-Evidence/1", accept: "image/png,image/jpeg,image/webp" });
38
+ // Prefer formats a host can keep as they are; anything else is still read and the bytes decide what it is.
39
+ export const imageReader = createPublicReader({ codes: CODES, userAgent: "Abelien-Reference-Evidence/1", accept: "image/png,image/jpeg,image/webp,image/*;q=0.8" });
40
+ /** Response types worth reading as a picture; the format itself is then decided by the bytes (image-format.mjs). */
41
+ export const IMAGE_CONTENT_TYPES = Object.freeze(["image/", "application/octet-stream", "binary/octet-stream"]);
32
42
  export const volc = createVolcSearch({ codes: CODES, ownsError: error => Boolean(error?.code?.startsWith("REFERENCE_WEB_")),
33
43
  redact: (raw, env = {}) => redact(raw, [env.WEB_SEARCH_API_KEY].filter(text)), timeoutMessage: "reference network request timed out" });
34
44
 
@@ -43,7 +53,7 @@ function checkItem(item, index) {
43
53
  const at = `items[${index}]`;
44
54
  if (!record(item) || !text(item.id) || !ITEM_KINDS.includes(item.kind) || !ITEM_STATUSES.includes(item.status) || typeof item.title !== "string" || typeof item.role !== "string") invalid(`${at} has invalid id, kind, status, title or role`);
45
55
  if (item.source !== null && !(record(item.source) && text(item.source.url) && typeof item.source.site === "string")) invalid(`${at}.source must be null or {url, site}`);
46
- if (item.image !== null && !(record(item.image) && /^sha256:[0-9a-f]{64}$/.test(item.image.blob_ref ?? "") && /^[0-9a-f]{64}$/.test(item.image.sha256 ?? "") && text(item.image.media_type))) invalid(`${at}.image must be null or {blob_ref, sha256, media_type}`);
56
+ if (item.image !== null && !(record(item.image) && /^sha256:[0-9a-f]{64}$/.test(item.image.blob_ref ?? "") && /^[0-9a-f]{64}$/.test(item.image.sha256 ?? "") && PASS_THROUGH.includes(item.image.media_type))) invalid(`${at}.image must be null or {blob_ref, sha256, media_type: ${PASS_THROUGH.join(" / ")}}`);
47
57
  if (item.verdict !== null && !(record(item.verdict) && VERDICT_CODES.includes(item.verdict.code) && text(item.verdict.basis))) invalid(`${at}.verdict must be null or {code, basis}`);
48
58
  if (item.query !== null && typeof item.query !== "string") invalid(`${at}.query must be null or a string`);
49
59
  }
@@ -59,6 +69,14 @@ export function validateEvidence(body) {
59
69
  || counts.some(key => !Number.isSafeInteger(budget[key]) || budget[key] < 0)) invalid(`budget must count ${counts.join(", ")}`);
60
70
  body.items.forEach(checkItem);
61
71
  const ids = body.items.map(item => item.id); if (new Set(ids).size !== ids.length) invalid("item ids must be unique");
72
+ // An item's picture is one this evidence saved: its sha256 is the picture's id, so it must name a saved image.
73
+ const saved = new Map(body.images.filter(image => record(image) && image.status === "read").map(image => [image.sha256, image]));
74
+ if ([...saved.values()].some(image => !PASS_THROUGH.includes(image.media_type) || !/^sha256:[0-9a-f]{64}$/.test(image.blob_ref ?? ""))) invalid("a saved image must have a blob_ref and a stored media_type");
75
+ if (body.user_image) saved.set(body.user_image.sha256, body.user_image);
76
+ body.items.forEach((item, index) => {
77
+ const image = item.image && saved.get(item.image.sha256);
78
+ if (item.image && (!image || image.blob_ref !== item.image.blob_ref || image.media_type !== item.image.media_type)) invalid(`items[${index}].image must be a saved image of this evidence`);
79
+ });
62
80
  if (body.status === "final") {
63
81
  if (!record(body.identity) && body.identity !== null) invalid("identity must be null or an object");
64
82
  if (!Array.isArray(body.similar)) invalid("similar must be an array");
@@ -0,0 +1,87 @@
1
+ // 参考图片的格式与统一转换(P2-13.06):格式只按字节识别,不信响应头(E 新创验收:CSDN 的 JPEG 被标成 image/png,
2
+ // 宿主私有存储按格式门禁拒收)。宿主可提供转换器,把任何能解码的图片统一转成 PNG;保存的字节即图片 id(sha256)的来源。
3
+ import { spawn } from "node:child_process";
4
+ import { existsSync } from "node:fs";
5
+ import { isAbsolute } from "node:path";
6
+ import { Buffer } from "node:buffer";
7
+ import { failure } from "../../abelien-harness/scripts/web/common.mjs";
8
+ import { isPng, imageSize } from "./png.mjs";
9
+
10
+ /** Formats a host can store and a browser can show as they are; any other format needs the host's converter. */
11
+ export const PASS_THROUGH = Object.freeze(["image/png", "image/jpeg", "image/webp"]);
12
+ /** Absolute path of the host's converter: an executable, or a Node script (.mjs/.cjs/.js) run with this Node. */
13
+ export const CONVERTER_ENV = "ABELIEN_REFERENCE_IMAGE_CONVERTER";
14
+
15
+ const AVIF_BRANDS = new Set(["avif", "avis"]);
16
+ const HEIF_BRANDS = new Set(["heic", "heix", "hevc", "hevx", "heim", "heis", "hevm", "hevs", "mif1", "msf1"]);
17
+ const SVG_HEAD = /^(?:<\?xml[^>]*>\s*)?(?:<!--[\s\S]*?-->\s*)*(?:<!DOCTYPE[^>]*>\s*)?(?:<!--[\s\S]*?-->\s*)*<svg[\s>]/i;
18
+
19
+ /** The image format the bytes actually are, or null when they are not an image this module knows. */
20
+ export function sniffImage(bytes) {
21
+ if (!Buffer.isBuffer(bytes) || bytes.length < 12) return null;
22
+ if (isPng(bytes)) return "image/png";
23
+ if (bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff) return "image/jpeg";
24
+ const head = bytes.toString("latin1", 0, 12);
25
+ if (head.startsWith("GIF87a") || head.startsWith("GIF89a")) return "image/gif";
26
+ if (head.startsWith("RIFF") && head.slice(8, 12) === "WEBP") return "image/webp";
27
+ if (head.slice(4, 8) === "ftyp") {
28
+ const size = Math.min(bytes.readUInt32BE(0), bytes.length), brands = [head.slice(8, 12)];
29
+ for (let at = 16; at + 4 <= size; at += 4) brands.push(bytes.toString("latin1", at, at + 4));
30
+ if (brands.some(brand => AVIF_BRANDS.has(brand))) return "image/avif";
31
+ if (brands.some(brand => HEIF_BRANDS.has(brand))) return "image/heic";
32
+ return null;
33
+ }
34
+ if (head.startsWith("BM") && bytes.readUInt32LE(2) <= bytes.length + 1024 && bytes.length >= 26) return "image/bmp";
35
+ if (head.startsWith("II*\0") || head.startsWith("MM\0*")) return "image/tiff";
36
+ if (bytes.readUInt32BE(0) === 0x00000100 && bytes.readUInt16LE(4) > 0) return "image/x-icon";
37
+ const textHead = bytes.toString("utf8", 0, Math.min(bytes.length, 4096)).replace(/^/, "").trimStart();
38
+ return SVG_HEAD.test(textHead) ? "image/svg+xml" : null;
39
+ }
40
+
41
+ /**
42
+ * The host's converter as a function, or null when the host provides none. Protocol: the source bytes on stdin;
43
+ * arguments `--max-side <px> --max-bytes <n>`; a PNG on stdout no larger than either limit (auto-rotated, first frame);
44
+ * a non-zero exit means the image cannot be converted. A configured path that is not absolute or does not exist fails
45
+ * the run before any request, like a missing search key.
46
+ */
47
+ export function hostConverter(env = {}) {
48
+ const path = env[CONVERTER_ENV];
49
+ if (path === undefined || path === "") return null;
50
+ if (typeof path !== "string" || !isAbsolute(path) || !existsSync(path)) throw failure("REFERENCE_WEB_CONFIG", `${CONVERTER_ENV} must be the absolute path of an existing converter`);
51
+ const script = /\.(?:mjs|cjs|js)$/.test(path);
52
+ return (bytes, { maxSide, maxBytes, timeoutMs }) => new Promise(resolve => {
53
+ const child = spawn(script ? process.execPath : path, [...(script ? [path] : []), "--max-side", String(maxSide), "--max-bytes", String(maxBytes)],
54
+ { stdio: ["pipe", "pipe", "ignore"], env: { PATH: env.PATH ?? process.env.PATH ?? "" } });
55
+ const chunks = []; let size = 0, settled = false;
56
+ const finish = result => { if (settled) return; settled = true; clearTimeout(timer); child.kill("SIGKILL"); resolve(result); };
57
+ const timer = setTimeout(() => finish({ ok: false, reason: "convert_timeout" }), timeoutMs);
58
+ child.stdout.on("data", chunk => {
59
+ size += chunk.length;
60
+ if (size > maxBytes) finish({ ok: false, reason: "convert_too_large" }); else chunks.push(chunk);
61
+ });
62
+ child.on("error", () => finish({ ok: false, reason: "convert_failed" }));
63
+ child.on("close", code => finish(code === 0 ? { ok: true, bytes: Buffer.concat(chunks) } : { ok: false, reason: "convert_failed" }));
64
+ child.stdin.on("error", () => { /* the converter may exit before reading everything; its exit code decides */ });
65
+ child.stdin.end(bytes);
66
+ });
67
+ }
68
+
69
+ /**
70
+ * One fetched image made ready to save. With a converter every format becomes a PNG within the limits; without one,
71
+ * only PNG, JPEG and WebP pass through unchanged. Returns {bytes, media_type, size} or {reason}.
72
+ */
73
+ export async function prepareImage(raw, { convert, maxSide, maxBytes, timeoutMs }) {
74
+ const source = sniffImage(raw);
75
+ if (!source) return { reason: "not_image" };
76
+ let bytes = raw, mediaType = source;
77
+ if (convert) {
78
+ const converted = await convert(raw, { maxSide, maxBytes, timeoutMs });
79
+ if (!converted.ok) return { reason: converted.reason, source };
80
+ if (!isPng(converted.bytes)) return { reason: "convert_invalid", source };
81
+ bytes = converted.bytes; mediaType = "image/png";
82
+ } else if (!PASS_THROUGH.includes(source)) return { reason: "needs_converter", source };
83
+ const size = imageSize(bytes);
84
+ if (!size || !size.width || !size.height) return { reason: "unreadable", source };
85
+ if (bytes.length > maxBytes || (convert && Math.max(size.width, size.height) > maxSide)) return { reason: convert ? "convert_invalid" : "too_large", source };
86
+ return { bytes, media_type: mediaType, source, size };
87
+ }
@@ -1,12 +1,14 @@
1
1
  // reference.research(P2-13.04):按 Agent 给出的检索计划执行确定性的收集——Volc 检索、公开页面安全直读、
2
2
  // 页面主图抓取、用户原图测量——全部落为证据,不做身份判断。判断由 Agent 读证据后经 reference.finalize 提交并核验。
3
- // 同一请求重复运行直接返回已有结果,不重复产生检索费用。
3
+ // 同一请求重复运行直接返回已有结果,不重复产生检索费用。页面图片的格式按字节判断,有宿主转换器时统一存成 PNG,
4
+ // 存下的字节的 sha256 就是这张图的 id;宿主只展示有 id 的图。
4
5
  import { Buffer } from "node:buffer";
5
6
  import { text, record, digest, sha256 } from "../../abelien-harness/scripts/web/common.mjs";
6
7
  import { decodeBody, decodeHtmlEntities } from "../../abelien-harness/scripts/web/public-read.mjs";
7
- import { isPng, imageSize } from "./png.mjs";
8
+ import { isPng } from "./png.mjs";
8
9
  import { measurePng } from "./measure.mjs";
9
- import { SCHEMA, INTENTS, DEPTHS, PURPOSES, KIND_BY_PURPOSE, LIMITS, EVIDENCE_SHAPE, reader, imageReader, volc, invalid, stringList, site, validateEvidence } from "./evidence.mjs";
10
+ import { hostConverter, prepareImage } from "./image-format.mjs";
11
+ import { SCHEMA, INTENTS, DEPTHS, PURPOSES, KIND_BY_PURPOSE, LIMITS, EVIDENCE_SHAPE, IMAGE_CONTENT_TYPES, reader, imageReader, volc, invalid, stringList, site, validateEvidence } from "./evidence.mjs";
10
12
 
11
13
  function normalizeLink(value) {
12
14
  try { return reader.normalizePublicSourceUrl(value).href; } catch { invalid("links must be public HTTPS URLs without credentials"); }
@@ -39,10 +41,12 @@ export function validateResearchInput(input) {
39
41
  return { purpose: q.purpose, query: q.query.trim(), count: q.count ?? LIMITS.count };
40
42
  });
41
43
  const bounded = (value, max, fallback, label) => { if (value === undefined) return fallback; if (!Number.isSafeInteger(value) || value < 0 || value > max) invalid(`plan.${label} must be 0–${max}`); return value; };
44
+ // Only pictures are shown on the canvas, so by default every page read gets its picture looked for.
45
+ const maxPages = bounded(plan.max_pages, LIMITS.pages, 8, "max_pages");
42
46
  return {
43
47
  intent: input.intent, keep: stringList(input.keep ?? [], "keep"), changeable: stringList(input.changeable ?? [], "changeable"),
44
48
  depth: input.depth ?? null, text: input.text.trim(), links: links.map(normalizeLink), userImage,
45
- plan: { queries: normalizedQueries, max_pages: bounded(plan.max_pages, LIMITS.pages, 8, "max_pages"), max_images: bounded(plan.max_images, LIMITS.images, 6, "max_images") },
49
+ plan: { queries: normalizedQueries, max_pages: maxPages, max_images: bounded(plan.max_images, LIMITS.images, maxPages, "max_images") },
46
50
  };
47
51
  }
48
52
 
@@ -53,18 +57,56 @@ const titleOf = html => {
53
57
  };
54
58
  // Page furniture is not the product: a CSDN author avatar once became a candidate's picture (P2-13 acceptance, E1).
55
59
  export const NOT_PRODUCT_IMAGE = /(?:^|[/_.-])(?:avatar|logo|icon|favicon|sprite|badge|profile|userhead|headimg|portrait|qrcode)(?=$|[/_.\d-])/i;
60
+ /** Lazy-load stand-ins (Wix `blur_2`, LQIP, spacer GIFs): a blurred or blank copy of the real picture, never the picture. */
61
+ export const PLACEHOLDER_IMAGE = /(?:^|[/_.,=&?-])(?:blur|lqip|placeholder|spacer|blank|lazy|loading)(?=$|[/_.,=&\d-])/i;
56
62
  /** A main image whose shorter side is below this many pixels is an icon or thumbnail, not a product picture. */
57
63
  export const MIN_IMAGE_SIDE = 200;
58
- function mainImageUrl(html, base) {
59
- const patterns = [/<meta[^>]+(?:property|name)=["'](?:og:image|og:image:url|twitter:image)["'][^>]*content=["']([^"']+)["']/gi,
60
- /<meta[^>]+content=["']([^"']+)["'][^>]*(?:property|name)=["'](?:og:image|og:image:url|twitter:image)["']/gi,
61
- /<link[^>]+rel=["']image_src["'][^>]*href=["']([^"']+)["']/gi];
62
- for (const pattern of patterns) for (const match of html.matchAll(pattern)) {
64
+ const META_IMAGE = /^(?:og:image|og:image:url|og:image:secure_url|twitter:image|twitter:image:src|image)$/i;
65
+ const attr = (tag, name) => {
66
+ const match = new RegExp(`\\s${name}\\s*=\\s*(?:"([^"]*)"|'([^']*)'|([^\\s>]+))`, "i").exec(tag);
67
+ return match ? match[1] ?? match[2] ?? match[3] : null;
68
+ };
69
+ /** JSON-LD `image`: a URL, a list, an ImageObject, or nested in @graph and other objects. */
70
+ function ldImages(node, out, depth = 0) {
71
+ if (depth > 6 || !node || typeof node !== "object") return;
72
+ if (Array.isArray(node)) { for (const entry of node) ldImages(entry, out, depth + 1); return; }
73
+ const take = value => typeof value === "string" ? out.push(value) : Array.isArray(value) ? value.forEach(take)
74
+ : record(value) && typeof (value.contentUrl ?? value.url) === "string" ? out.push(value.contentUrl ?? value.url) : undefined;
75
+ take(node.image); take(node.thumbnailUrl);
76
+ for (const [key, value] of Object.entries(node)) if (key !== "image" && value && typeof value === "object") ldImages(value, out, depth + 1);
77
+ }
78
+ /**
79
+ * The page's pictures in the order they are tried: share images (og/twitter/itemprop), image_src, JSON-LD, then body
80
+ * images not declared smaller than the floor. HTTP links are read over HTTPS; page furniture is never tried.
81
+ */
82
+ export function imageCandidates(html, base) {
83
+ const raw = [];
84
+ for (const tag of html.match(/<meta\b[^>]*>/gi) ?? []) {
85
+ const key = attr(tag, "property") ?? attr(tag, "name") ?? attr(tag, "itemprop");
86
+ if (key && META_IMAGE.test(key.trim())) raw.push(attr(tag, "content"));
87
+ }
88
+ for (const tag of html.match(/<link\b[^>]*>/gi) ?? []) if (/^\s*image_src\s*$/i.test(attr(tag, "rel") ?? "")) raw.push(attr(tag, "href"));
89
+ for (const block of html.matchAll(/<script[^>]+type=["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi)) {
90
+ try { ldImages(JSON.parse(block[1]), raw); } catch { /* malformed JSON-LD is skipped */ }
91
+ }
92
+ for (const tag of html.match(/<img\b[^>]*>/gi) ?? []) {
93
+ const width = Number(attr(tag, "width")), height = Number(attr(tag, "height"));
94
+ if ((width && width < MIN_IMAGE_SIDE) || (height && height < MIN_IMAGE_SIDE)) continue;
95
+ raw.push(attr(tag, "data-src") ?? attr(tag, "data-original") ?? attr(tag, "src"));
96
+ }
97
+ const out = [];
98
+ for (const value of raw) {
99
+ if (!value || /^\s*data:/i.test(value)) continue;
63
100
  let url;
64
- try { url = new URL(decodeHtmlEntities(match[1]), base); } catch { continue; }
65
- if (!NOT_PRODUCT_IMAGE.test(`${url.hostname}${url.pathname}`)) return url.href;
101
+ try { url = new URL(decodeHtmlEntities(value.trim()), base); } catch { continue; }
102
+ if (url.protocol === "http:") url.protocol = "https:";
103
+ url.hash = "";
104
+ if (url.protocol !== "https:" || NOT_PRODUCT_IMAGE.test(`${url.hostname}${url.pathname}`) || PLACEHOLDER_IMAGE.test(`${url.pathname}${url.search}`)
105
+ || out.includes(url.href)) continue;
106
+ out.push(url.href);
107
+ if (out.length >= LIMITS.imageCandidates) break;
66
108
  }
67
- return null;
109
+ return out;
68
110
  }
69
111
 
70
112
  /** Run tasks with bounded concurrency; results keep input order. */
@@ -105,6 +147,8 @@ function pageQueue(links, intent, searches, max) {
105
147
  export async function researchReference({ input, env = {}, fetchImpl, readResource, writeBlob, ports = {} }) {
106
148
  const request = validateResearchInput(input);
107
149
  if (typeof writeBlob !== "function" || typeof readResource !== "function") throw Object.assign(new Error("trusted content writer and resource reader are required"), { code: "REFERENCE_HOST" });
150
+ // A misconfigured converter stops the run before any request, like a missing search key.
151
+ const convert = ports.convertImage ?? hostConverter(env);
108
152
  const requestDigest = digest({ ...request, userImage: request.userImage?.sha256 ?? null, shape: EVIDENCE_SHAPE });
109
153
  const id = `ref-${requestDigest.slice(0, 16)}`;
110
154
  const existing = await readResource(`reference-evidence:${id}`);
@@ -145,14 +189,14 @@ export async function researchReference({ input, env = {}, fetchImpl, readResour
145
189
  // Phase 1: read pages concurrently; results keep queue order.
146
190
  const pages = await pool(queue, async (entry, index) => {
147
191
  const page = { id: `p${index + 1}`, url: entry.url, purpose: entry.purpose, search_id: entry.search_id, status: "unavailable", reason: null, final_url: null, title: entry.search_title ?? "",
148
- raw_sha256: null, text_blob_ref: null, text_sha256: null, excerpt: "", image_url: null };
192
+ raw_sha256: null, text_blob_ref: null, text_sha256: null, excerpt: "", image_candidates: [] };
149
193
  try {
150
194
  const read = await reader.fetchPublicPage(entry.url, { ...readOptions, maxBytes: LIMITS.pageBytes });
151
195
  const html = decodeBody(Buffer.from(read.raw_base64, "base64"), read.content_type);
152
196
  Object.assign(page, { status: read.status, reason: read.reason, final_url: read.final_url, raw_sha256: read.raw_sha256, title: titleOf(html) || page.title });
153
197
  if (read.status === "read") {
154
198
  page.text_blob_ref = writeBlob({ text: read.text }); page.text_sha256 = read.text_sha256; page.excerpt = read.text.slice(0, LIMITS.excerptChars);
155
- page.image_url = read.content_type.includes("html") ? mainImageUrl(html, read.final_url) : null;
199
+ page.image_candidates = read.content_type.includes("html") ? imageCandidates(html, read.final_url) : [];
156
200
  }
157
201
  } catch (error) {
158
202
  page.reason = reader.publicReadFailure(error)?.stage ?? error?.code ?? "read_failed";
@@ -160,24 +204,28 @@ export async function researchReference({ input, env = {}, fetchImpl, readResour
160
204
  }
161
205
  return page;
162
206
  });
163
- for (const page of pages) if (page.status !== "read") unknowns.push(`页面不可用:${page.url}(${page.reason})`);
164
- // Phase 2: main images for the first pages that have one, up to the image budget, fetched concurrently.
165
- const withImage = pages.filter(page => page.image_url), wanted = withImage.slice(0, request.plan.max_images);
207
+ // A page that could not be read is already an unavailable page and item; it is not repeated in unknowns.
208
+ // Phase 2: one picture per page, up to the image budget, pages fetched concurrently. A page's candidates are tried in
209
+ // order until one reads as a real picture of at least the floor size; that saved picture's sha256 is its id.
210
+ const withImage = pages.filter(page => page.image_candidates.length), wanted = withImage.slice(0, request.plan.max_images);
166
211
  const images = await pool(wanted, async (page, index) => {
167
- const entryImage = { id: `i${index + 1}`, url: page.image_url, page_id: page.id, status: "unavailable", reason: null, blob_ref: null, sha256: null, media_type: null, measurement: null };
168
- try {
169
- const bytes = await imageReader.fetchPublicBytes(page.image_url, { ...readOptions, maxBytes: LIMITS.imageBytes, contentTypes: ["image/png", "image/jpeg", "image/webp"] });
170
- Object.assign(entryImage, { status: bytes.status, reason: bytes.reason });
171
- const size = bytes.status === "read" ? imageSize(Buffer.from(bytes.raw_base64, "base64")) : null;
172
- if (size && Math.min(size.width, size.height) < MIN_IMAGE_SIDE) Object.assign(entryImage, { status: "unavailable", reason: "too_small" });
173
- if (entryImage.status === "read") {
174
- const media = bytes.content_type.split(";")[0].trim();
175
- entryImage.blob_ref = writeBlob({ media_type: media, base64: bytes.raw_base64 }); entryImage.sha256 = bytes.raw_sha256; entryImage.media_type = media;
176
- if (media === "image/png") entryImage.measurement = measurePng(Buffer.from(bytes.raw_base64, "base64"));
212
+ const entryImage = { id: `i${index + 1}`, url: page.image_candidates[0], page_id: page.id, status: "unavailable", reason: null, blob_ref: null, sha256: null,
213
+ media_type: null, source_media_type: null, measurement: null };
214
+ for (const url of page.image_candidates) {
215
+ entryImage.url = url;
216
+ try {
217
+ const read = await imageReader.fetchPublicBytes(url, { ...readOptions, timeoutMs: LIMITS.imageTimeoutMs, maxBytes: LIMITS.imageBytes, contentTypes: IMAGE_CONTENT_TYPES });
218
+ if (read.status !== "read") { entryImage.reason = read.reason; continue; }
219
+ const ready = await prepareImage(Buffer.from(read.raw_base64, "base64"), { convert, maxSide: LIMITS.imageSide, maxBytes: LIMITS.savedImageBytes, timeoutMs: LIMITS.convertTimeoutMs });
220
+ entryImage.source_media_type = ready.source ?? null;
221
+ if (!ready.bytes) { entryImage.reason = ready.reason; continue; }
222
+ if (Math.min(ready.size.width, ready.size.height) < MIN_IMAGE_SIDE) { entryImage.reason = "too_small"; continue; }
223
+ return Object.assign(entryImage, { status: "read", reason: null, blob_ref: writeBlob({ media_type: ready.media_type, base64: ready.bytes.toString("base64") }),
224
+ sha256: sha256(ready.bytes), media_type: ready.media_type, measurement: ready.media_type === "image/png" ? measurePng(ready.bytes) : null });
225
+ } catch (error) {
226
+ entryImage.reason = imageReader.publicReadFailure(error)?.stage ?? error?.code ?? "read_failed";
227
+ if (!imageReader.publicReadFailure(error) && !String(error?.code ?? "").startsWith("REFERENCE_WEB_")) throw error;
177
228
  }
178
- } catch (error) {
179
- entryImage.reason = imageReader.publicReadFailure(error)?.stage ?? error?.code ?? "read_failed";
180
- if (!imageReader.publicReadFailure(error) && !String(error?.code ?? "").startsWith("REFERENCE_WEB_")) throw error;
181
229
  }
182
230
  return entryImage;
183
231
  });