argo-search 2.9.7 → 2.9.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.en.md +73 -150
- package/README.es.md +73 -150
- package/README.ja.md +79 -156
- package/README.ko.md +78 -155
- package/README.md +80 -159
- package/SKILL.md +3 -3
- package/assets/readme/hero.svg +2 -2
- package/backends/domain_profiles.json +48 -0
- package/backends/engine_registry.yaml +118 -4
- package/backends/quota_profiles.json +92 -12
- package/backends/source_types_cn.json +27 -1
- package/bin/argo +78 -5
- package/bin/argo.js +33 -2
- package/config.yaml +269 -63
- package/docs/ARGO_INTRO.md +5 -3
- package/docs/ENGINE_CATALOG.md +50 -39
- package/docs/MCP_SETUP.md +3 -1
- package/docs/RELEASE_NOTES_v2.9.8.md +73 -0
- package/docs/RELEASE_NOTES_v2.9.9.md +78 -0
- package/engines/specs/cfets.yaml +41 -0
- package/engines/specs/defillama.yaml +38 -0
- package/engines/specs/firecrawl.yaml +1 -1
- package/engines/specs/fxtwitter.yaml +5 -0
- package/engines/specs/gleif.yaml +55 -0
- package/engines/specs/hkex.yaml +40 -0
- package/engines/specs/keenable_free.yaml +36 -0
- package/engines/specs/shfe.yaml +39 -0
- package/engines/specs/stackexchange.yaml +2 -2
- package/engines/specs/tavily_free.yaml +56 -0
- package/engines/specs/yuanbao.yaml +93 -0
- package/mcp/clients.yaml +32 -0
- package/package.json +7 -2
- package/packages/dsh-plugin/README.md +25 -6
- package/packages/dsh-plugin/dsh/client.js +402 -47
- package/packages/dsh-plugin/dsh/diagnostics.js +97 -0
- package/packages/dsh-plugin/dsh/index.js +135 -1876
- package/packages/dsh-plugin/dsh/native-tools.mjs +7 -1
- package/packages/dsh-plugin/dsh/runner.js +666 -0
- package/packages/dsh-plugin/dsh/wide-research.js +878 -0
- package/packages/dsh-plugin/package.json +3 -3
- package/packages/dsh-plugin/scripts/sync-local.sh +70 -0
- package/packages/dsh-plugin/test/client.test.js +177 -2
- package/packages/dsh-plugin/test/diagnostics.test.js +28 -0
- package/packages/dsh-plugin/test/index.test.js +44 -0
- package/scripts/adaptive.py +126 -43
- package/scripts/article_meta.py +131 -0
- package/scripts/bench_tail_ab.py +281 -0
- package/scripts/bounded_run.py +91 -49
- package/scripts/budget.py +247 -0
- package/scripts/budget_cli.py +93 -0
- package/scripts/build_release_assets.sh +65 -0
- package/scripts/cache.py +60 -72
- package/scripts/ci_drift_gate.py +90 -0
- package/scripts/clock_bridge.py +22 -0
- package/scripts/content_boundary.py +46 -0
- package/scripts/content_security.py +119 -4
- package/scripts/content_signals.py +26 -5
- package/scripts/cross_lingual_trace.py +293 -0
- package/scripts/dispatch_result.py +105 -0
- package/scripts/engine_add.py +602 -0
- package/scripts/engine_dispatch.py +58 -86
- package/scripts/engine_env.py +5 -0
- package/scripts/engine_families.py +6 -0
- package/scripts/engine_policy.py +7 -7
- package/scripts/engines.py +21 -0
- package/scripts/engines_base.py +3 -3
- package/scripts/engines_builders.py +12 -0
- package/scripts/engines_builders_batch10.py +48 -3
- package/scripts/engines_builders_batch14.py +656 -0
- package/scripts/engines_builders_batch15.py +249 -0
- package/scripts/engines_builders_keyless.py +111 -0
- package/scripts/engines_builders_tech.py +0 -1
- package/scripts/evidence.py +31 -3
- package/scripts/evidence_loop.py +58 -22
- package/scripts/fetch_quality.py +13 -0
- package/scripts/fetch_v3.py +28 -3
- package/scripts/gen_routing_snapshot.py +11 -0
- package/scripts/http_client.py +24 -38
- package/scripts/install.ps1 +18 -0
- package/scripts/install.sh +16 -1
- package/scripts/job.py +3 -3
- package/scripts/local_seek.py +41 -5
- package/scripts/macro_countries.py +15 -0
- package/scripts/matrix_search_eval.py +192 -0
- package/scripts/mcp_compact.py +234 -0
- package/scripts/mcp_handlers.py +60 -642
- package/scripts/mcp_launch.sh +10 -8
- package/scripts/mcp_local.py +241 -0
- package/scripts/mcp_server.py +19 -14
- package/scripts/mcp_setup.py +33 -9
- package/scripts/mcp_tools.py +1 -0
- package/scripts/mcp_transport.py +1 -1
- package/scripts/mcp_warm.py +127 -0
- package/scripts/net_proxy.py +27 -0
- package/scripts/output_transparency.py +200 -0
- package/scripts/paid_policy.py +98 -0
- package/scripts/quota.py +158 -13
- package/scripts/recovery.py +7 -3
- package/scripts/recovery_runner.py +150 -0
- package/scripts/replay_eval.py +18 -0
- package/scripts/research.py +37 -21
- package/scripts/research_expand.py +24 -1
- package/scripts/retry_policy.py +76 -0
- package/scripts/route.py +6 -6
- package/scripts/route_cache.py +18 -0
- package/scripts/route_combo.py +26 -5
- package/scripts/route_domains.py +170 -31
- package/scripts/safe_conn.py +52 -0
- package/scripts/search.py +15 -27
- package/scripts/search_attribution.py +100 -3
- package/scripts/search_cli.py +24 -11
- package/scripts/search_entry.py +8 -1
- package/scripts/search_output.py +99 -15
- package/scripts/search_pipeline.py +62 -195
- package/scripts/search_rank.py +6 -0
- package/scripts/search_types.py +107 -9
- package/scripts/semantic_evidence.py +179 -3
- package/scripts/serp_guard.py +138 -7
- package/scripts/serp_kill_log.py +25 -2
- package/scripts/stats_cli.py +13 -1
- package/scripts/tail_completion.py +212 -0
- package/scripts/url_safety.py +126 -3
- package/scripts/usage_log.py +45 -0
- package/scripts/wide_research.py +45 -0
- package/scripts/zhihu_quota.py +139 -0
- package/sub-skills/local-search/bm25.py +27 -3
- package/sub-skills/local-search/cli_runner.py +331 -0
- package/sub-skills/local-search/parsers.py +328 -0
- package/sub-skills/local-search/search_v3.py +31 -674
- package/sub-skills/local-seek/scripts/seek_rank.py +7 -3
- package/assets/readme/settings-keys.png +0 -0
- package/assets/readme/settings-tools.png +0 -0
- package/sub-skills/ego-search/references/browser-runtime.md +0 -43
- package/sub-skills/ego-search/references/install.md +0 -67
- package/sub-skills/ego-search/references/original-ego-upgrade.md +0 -51
- package/sub-skills/ego-search/tests/test_query_normalize.py +0 -68
- package/sub-skills/js-run/tests/fixtures/challenge_like.js +0 -27
- package/sub-skills/js-run/tests/test_jsrun.py +0 -165
- package/sub-skills/local-seek/references/auto-behaviors.md +0 -24
- package/sub-skills/local-seek/references/fd-mdfind-recipes.md +0 -62
- package/sub-skills/local-seek/references/git-integration.md +0 -54
- package/sub-skills/local-seek/references/remote-handoff.md +0 -46
- package/sub-skills/local-seek/references/rg-recipes.md +0 -96
- package/sub-skills/local-seek/references/strategies.md +0 -52
- package/sub-skills/local-seek/references/structural-search.md +0 -59
- package/sub-skills/local-seek/tests/test_content_rank.py +0 -222
- package/sub-skills/local-seek/tests/test_determinism_20261006.py +0 -145
- package/sub-skills/local-seek/tests/test_noise_tier.py +0 -101
- package/sub-skills/local-seek/tests/test_pcre2_probe.py +0 -136
- package/sub-skills/local-seek/tests/test_rank_pinyin.py +0 -77
- package/sub-skills/local-seek/tests/test_regression_20261003.py +0 -169
- package/sub-skills/local-seek/tests/test_regression_20261006.py +0 -146
- package/sub-skills/local-seek/tests/test_seek_degradation.py +0 -111
- package/sub-skills/local-seek/tests/test_seek_fts.py +0 -313
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
"""tail_completion.py — 早停后台补全:慢源交付从「丢弃」变「补齐」。
|
|
2
|
+
|
|
3
|
+
## 为什么(2026-10-25)
|
|
4
|
+
|
|
5
|
+
七天窗口 1185 条真实查询里 74% 发生 early-stop。被放弃的交付不是低质货:
|
|
6
|
+
离线回放(bench_tail_ab --quality)显示被放弃交付 81% 来自 anysearch,相关度
|
|
7
|
+
代理比全库均线高 0.125;五条对照查询里四条「唯一有结果的就是慢源」(证据与
|
|
8
|
+
口径见 docs/QUALITY_OF_SLOW_SOURCES.md)。早停省的是**用户等待**,不该连
|
|
9
|
+
**交付**一起丢——那是在拿准确性换速度。缩尾宽已被实证否决,本模块是拍板的
|
|
10
|
+
正确方向:慢源不拖住快源(早交付),但慢源的交付进缓存(后台补全)。
|
|
11
|
+
|
|
12
|
+
## 做什么
|
|
13
|
+
|
|
14
|
+
engine_dispatch 在早停弃置引擎时把 (engine, holder, now_fn) 交给本模块——
|
|
15
|
+
主响应与既有记账逐位不变(cancelled outcome / abandoned_ms 照记)。本模块
|
|
16
|
+
起一个 daemon 收割线程,在主流程返回后继续等被弃置的引擎,宽限内完成的:
|
|
17
|
+
|
|
18
|
+
1. 学习器记真实成绩(success / latency / quality / 场景分桶标签)——此前
|
|
19
|
+
cancelled 被 cache_guard.should_record_adaptive 跳过,慢源的真实表现
|
|
20
|
+
从没进过学习器,场景分桶(engine×domain×lang)也少一笔真数据;
|
|
21
|
+
2. 把交付合并写回 L2 combo 缓存(URL 去重、新条目追加、缺省 score 用词面
|
|
22
|
+
相关度)——同查询第二次免费拿到「快源+慢源」的全量覆盖。
|
|
23
|
+
|
|
24
|
+
timeout 弃置不收:那是引擎确实超窗(甚至卡死),补全等于鼓励慢源;
|
|
25
|
+
cancelled 是「编排收工决定,引擎没失败」——两者语义不同(见
|
|
26
|
+
engine_dispatch._settle_pending 的注释)。
|
|
27
|
+
|
|
28
|
+
## 进程语义
|
|
29
|
+
|
|
30
|
+
收割线程是 daemon;CLI 进程退出前由 atexit 宽限等待(hand_off 首次调用时
|
|
31
|
+
注册,无 stranded 的进程零成本、不注册)。MCP/daemon 常驻进程不退出,
|
|
32
|
+
自然收割全部。宽限从**入队时刻**起算整批(job 级 deadline),不做逐引擎
|
|
33
|
+
累加——实测被放弃交付从早停时刻起的剩余等待 p50 2306ms / p90 3883ms。
|
|
34
|
+
|
|
35
|
+
## 开关
|
|
36
|
+
|
|
37
|
+
ARGO_TAIL_COMPLETE_S:宽限秒数(默认 3.0;0=禁用整个机制,行为回到
|
|
38
|
+
纯丢弃)。改动记得同步 references/usage.md 的开关总表(usage_doc_gates)。
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
from __future__ import annotations
|
|
42
|
+
|
|
43
|
+
import atexit
|
|
44
|
+
import os
|
|
45
|
+
import threading
|
|
46
|
+
|
|
47
|
+
_LOCK = threading.Lock()
|
|
48
|
+
_JOBS: list[dict] = []
|
|
49
|
+
_WORKER: threading.Thread | None = None
|
|
50
|
+
_ATEXIT_REGISTERED = False
|
|
51
|
+
_STATS = {"salvaged": 0, "expired": 0, "recorded": 0}
|
|
52
|
+
_POLL_INTERVAL_S = 0.05
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def grace_seconds() -> float:
|
|
56
|
+
"""收割宽限(ARGO_TAIL_COMPLETE_S,默认 3.0;<=0 禁用)。"""
|
|
57
|
+
try:
|
|
58
|
+
return float(os.environ.get("ARGO_TAIL_COMPLETE_S", "3.0") or 3.0)
|
|
59
|
+
except ValueError:
|
|
60
|
+
return 3.0
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def hand_off(stranded, *, query, cache_engine_key, max_results,
|
|
64
|
+
domain, mode, depth, decision, skip_cache=False,
|
|
65
|
+
cache_instance=None) -> None:
|
|
66
|
+
"""把早停弃置的引擎交给后台收割。
|
|
67
|
+
|
|
68
|
+
``stranded`` 是 engine_dispatch 产出的 ``[(engine, holder, now_fn), ...]``:
|
|
69
|
+
holder["r"] 出现即完成(与 _ingest_holder 同判据),now_fn 用于算真实
|
|
70
|
+
latency(dispatch 可能跑在单调钟上,收割线程不能用墙钟去减)。
|
|
71
|
+
``cache_instance`` 是本次请求的缓存实例(req.cache):补全读写的必须是
|
|
72
|
+
同一实例(L1 内存态 + L2 落盘一致),模块级没有全局缓存可拿。
|
|
73
|
+
其余参数与 finalize 的 cache.set / learner.record 同源同口径。
|
|
74
|
+
任何异常 fail-open:补全失败只丢「锦上添花」,不影响主响应。
|
|
75
|
+
"""
|
|
76
|
+
grace = grace_seconds()
|
|
77
|
+
if grace <= 0 or not stranded:
|
|
78
|
+
return
|
|
79
|
+
global _ATEXIT_REGISTERED
|
|
80
|
+
with _LOCK:
|
|
81
|
+
if not _ATEXIT_REGISTERED:
|
|
82
|
+
atexit.register(wait_for_completion)
|
|
83
|
+
_ATEXIT_REGISTERED = True
|
|
84
|
+
_JOBS.append({
|
|
85
|
+
"stranded": list(stranded),
|
|
86
|
+
"deadline": _now_wall() + grace,
|
|
87
|
+
"query": query, "cache_engine_key": cache_engine_key,
|
|
88
|
+
"max_results": max_results, "domain": domain,
|
|
89
|
+
"mode": mode, "depth": depth, "decision": decision or {},
|
|
90
|
+
"skip_cache": bool(skip_cache) or cache_instance is None,
|
|
91
|
+
"cache": cache_instance,
|
|
92
|
+
})
|
|
93
|
+
_start_worker_locked()
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def wait_for_completion(timeout_s: float | None = None) -> None:
|
|
97
|
+
"""等收割线程收尾(atexit 出口 / 测试用)。宽限耗尽即返回,不无限等。"""
|
|
98
|
+
t = _WORKER
|
|
99
|
+
if t is not None and t.is_alive():
|
|
100
|
+
budget = timeout_s if timeout_s is not None else grace_seconds() + 0.5
|
|
101
|
+
t.join(timeout=budget)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def stats() -> dict:
|
|
105
|
+
"""收割计数(可观测口,供诊断;不改任何行为)。"""
|
|
106
|
+
with _LOCK:
|
|
107
|
+
return dict(_STATS)
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def _now_wall() -> float:
|
|
111
|
+
import time
|
|
112
|
+
return time.time()
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def _start_worker_locked() -> None:
|
|
116
|
+
global _WORKER
|
|
117
|
+
if _WORKER is not None and _WORKER.is_alive():
|
|
118
|
+
return
|
|
119
|
+
_WORKER = threading.Thread(target=_worker, daemon=True,
|
|
120
|
+
name="argo-tail-completion")
|
|
121
|
+
_WORKER.start()
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def _worker() -> None:
|
|
125
|
+
"""串行收割队列:宽限内完成的记学习器+合并缓存,超宽限标 expired。"""
|
|
126
|
+
import time
|
|
127
|
+
while True:
|
|
128
|
+
with _LOCK:
|
|
129
|
+
if not _JOBS:
|
|
130
|
+
return
|
|
131
|
+
job = _JOBS.pop(0)
|
|
132
|
+
for eng, holder, now_fn in job["stranded"]:
|
|
133
|
+
remaining = job["deadline"] - _now_wall()
|
|
134
|
+
while holder.get("r") is None and remaining > 0:
|
|
135
|
+
time.sleep(min(_POLL_INTERVAL_S, remaining))
|
|
136
|
+
remaining = job["deadline"] - _now_wall()
|
|
137
|
+
res = (holder.get("r") or (None,))[1] if holder.get("r") else None
|
|
138
|
+
if res is None:
|
|
139
|
+
with _LOCK:
|
|
140
|
+
_STATS["expired"] += 1
|
|
141
|
+
continue
|
|
142
|
+
goods = [r for r in (res or [])
|
|
143
|
+
if isinstance(r, dict) and "error" not in r]
|
|
144
|
+
with _LOCK:
|
|
145
|
+
_STATS["salvaged"] += 1
|
|
146
|
+
try:
|
|
147
|
+
_record_adaptive(eng, goods, holder, now_fn, job)
|
|
148
|
+
except Exception:
|
|
149
|
+
pass
|
|
150
|
+
if not job["skip_cache"]:
|
|
151
|
+
try:
|
|
152
|
+
_merge_into_cache(eng, goods, job)
|
|
153
|
+
except Exception:
|
|
154
|
+
pass
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def _record_adaptive(eng, goods, holder, now_fn, job) -> None:
|
|
158
|
+
"""学习器记真实成绩(口径与 search_pipeline.finalize 对齐)。"""
|
|
159
|
+
from adaptive import get_learner, scenario_of
|
|
160
|
+
from query_signals import result_relevance
|
|
161
|
+
success = bool(goods)
|
|
162
|
+
try:
|
|
163
|
+
lat_ms = float((now_fn() - holder.get("t0", now_fn())) * 1000.0)
|
|
164
|
+
except Exception:
|
|
165
|
+
lat_ms = 0.0
|
|
166
|
+
quality = None
|
|
167
|
+
if success:
|
|
168
|
+
try:
|
|
169
|
+
quality = result_relevance(job["query"], goods)
|
|
170
|
+
except Exception:
|
|
171
|
+
quality = None
|
|
172
|
+
with _LOCK:
|
|
173
|
+
_STATS["recorded"] += 1
|
|
174
|
+
get_learner().record(eng, success=success, latency_ms=max(0.0, lat_ms),
|
|
175
|
+
cost=0.001, empty=not success, quality=quality,
|
|
176
|
+
**scenario_of(job["decision"]))
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def _merge_into_cache(eng, goods, job) -> None:
|
|
180
|
+
"""交付合并写回 L2 combo 缓存:读现条目 → URL 去重追加 → 同 key 覆盖。
|
|
181
|
+
|
|
182
|
+
只在已有缓存条目时合并(本机制补的是「快源已写的那份」);主缓存被守卫
|
|
183
|
+
拒写时这里不硬造。score 缺省用词面相关度,保证追加条目有可排序的分数。
|
|
184
|
+
"""
|
|
185
|
+
from query_signals import result_relevance
|
|
186
|
+
cache = job["cache"]
|
|
187
|
+
cur = cache.get(job["query"], job["cache_engine_key"],
|
|
188
|
+
job["max_results"], domain=job["domain"],
|
|
189
|
+
mode=job["mode"], depth=job["depth"])
|
|
190
|
+
if not cur or not isinstance(cur.get("results"), list) or not goods:
|
|
191
|
+
return
|
|
192
|
+
existing = {r.get("url") for r in cur["results"]
|
|
193
|
+
if isinstance(r, dict)}
|
|
194
|
+
new: list[dict] = []
|
|
195
|
+
for r in goods:
|
|
196
|
+
if not isinstance(r, dict) or r.get("url") in existing:
|
|
197
|
+
continue
|
|
198
|
+
item = dict(r)
|
|
199
|
+
if "score" not in item:
|
|
200
|
+
try:
|
|
201
|
+
item["score"] = round(float(result_relevance(job["query"], [r])), 4)
|
|
202
|
+
except Exception:
|
|
203
|
+
item["score"] = 0.3
|
|
204
|
+
new.append(item)
|
|
205
|
+
if not new:
|
|
206
|
+
return
|
|
207
|
+
payload = dict(cur)
|
|
208
|
+
payload["results"] = list(cur["results"]) + new
|
|
209
|
+
payload["engines_used"] = sorted({*(cur.get("engines_used") or []), eng})
|
|
210
|
+
cache.set(job["query"], job["cache_engine_key"], job["max_results"],
|
|
211
|
+
payload, domain=job["domain"], mode=job["mode"],
|
|
212
|
+
depth=job["depth"])
|
package/scripts/url_safety.py
CHANGED
|
@@ -23,6 +23,8 @@ import ipaddress
|
|
|
23
23
|
import os
|
|
24
24
|
import re
|
|
25
25
|
import socket
|
|
26
|
+
import time
|
|
27
|
+
import urllib.parse
|
|
26
28
|
from urllib.parse import urlparse
|
|
27
29
|
|
|
28
30
|
|
|
@@ -184,9 +186,10 @@ def host_is_private(host: str) -> bool:
|
|
|
184
186
|
if host_is_private_name(host):
|
|
185
187
|
return True
|
|
186
188
|
norm = _normalize_numeric_host(host)
|
|
187
|
-
if norm is not None
|
|
188
|
-
#
|
|
189
|
-
#
|
|
189
|
+
if norm is not None:
|
|
190
|
+
# 数字字面量:直接按规范化结果查表,不依赖 getaddrinfo。
|
|
191
|
+
# 标准点分形式(norm == host)此前会落到 getaddrinfo——既多一次系统
|
|
192
|
+
# 调用,又让「被 mock/劫持的解析器」能影响 SSRF 判定。
|
|
190
193
|
return is_private_ip(norm)
|
|
191
194
|
try:
|
|
192
195
|
infos = socket.getaddrinfo(host, None, proto=socket.IPPROTO_TCP)
|
|
@@ -234,3 +237,123 @@ def is_safe_fetch_url(url: str) -> bool:
|
|
|
234
237
|
"""便捷布尔封装:仅 http/https 且目标非私有地址。"""
|
|
235
238
|
ok, _ = check_url(url)
|
|
236
239
|
return ok
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def max_redirects() -> int:
|
|
243
|
+
"""最大重定向跳数(默认 5,合法 1-20,越界回落默认)。"""
|
|
244
|
+
try:
|
|
245
|
+
from engine_env import get_env
|
|
246
|
+
val = int(str(get_env(["ARGO_MAX_REDIRECTS"])).strip())
|
|
247
|
+
except (TypeError, ValueError):
|
|
248
|
+
return 5
|
|
249
|
+
return val if 0 < val <= 20 else 5
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def check_redirect_target(url: str, start: float) -> dict | None:
|
|
253
|
+
"""校验重定向目标是否可安全请求;被拦截返回统一错误 dict,放行返回 None。
|
|
254
|
+
|
|
255
|
+
与 resolve_redirect_chain 的逐跳校验对齐——此前 _do_get/_do_post 的
|
|
256
|
+
内联重定向循环只校验首跳,重定向到内网即可绕过 SSRF。
|
|
257
|
+
"""
|
|
258
|
+
ok, reason = check_url(url)
|
|
259
|
+
if ok:
|
|
260
|
+
return None
|
|
261
|
+
return {
|
|
262
|
+
"status": 0, "headers": {}, "text": "",
|
|
263
|
+
"url": url,
|
|
264
|
+
"elapsed_ms": int((time.time() - start) * 1000),
|
|
265
|
+
"error": f"重定向目标被 SSRF 防护拦截: {reason}",
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
def resolve_redirect_chain(url: str, headers: dict, timeout: float,
|
|
270
|
+
limit: int | None = None) -> str | None:
|
|
271
|
+
"""逐跳安全跟随重定向:每跳目标都过 SSRF 校验(curl 通道用)。
|
|
272
|
+
|
|
273
|
+
返回最终 URL;任一跳被拦截或超过跳数返回 None。
|
|
274
|
+
"""
|
|
275
|
+
if limit is None:
|
|
276
|
+
limit = max_redirects()
|
|
277
|
+
current = url
|
|
278
|
+
for _ in range(limit):
|
|
279
|
+
ok, _reason = check_url(current)
|
|
280
|
+
if not ok:
|
|
281
|
+
return None
|
|
282
|
+
parsed = urlparse(current)
|
|
283
|
+
if parsed.scheme not in ("http", "https"):
|
|
284
|
+
return None
|
|
285
|
+
try:
|
|
286
|
+
from net_proxy import open_connection, request_selector, resolve_proxy
|
|
287
|
+
proxy_url = resolve_proxy(current)
|
|
288
|
+
conn, via_proxy = open_connection(parsed, timeout, proxy_url)
|
|
289
|
+
path = parsed.path or "/"
|
|
290
|
+
if parsed.query:
|
|
291
|
+
path += "?" + parsed.query
|
|
292
|
+
conn.request("HEAD", request_selector(parsed, path, via_proxy),
|
|
293
|
+
headers=headers)
|
|
294
|
+
resp = conn.getresponse()
|
|
295
|
+
status = resp.status
|
|
296
|
+
location = resp.getheader("Location")
|
|
297
|
+
conn.close()
|
|
298
|
+
except Exception:
|
|
299
|
+
return None
|
|
300
|
+
if status not in (301, 302, 303, 307, 308) or not location:
|
|
301
|
+
return current
|
|
302
|
+
current = urllib.parse.urljoin(current, location)
|
|
303
|
+
return None
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
# ── 解析即锁定(DNS rebinding / TOCTOU 防护)─────────────────────────────────
|
|
307
|
+
#
|
|
308
|
+
# check_url() 在请求前解析一次 DNS,http.client 在 connect() 时再解析一次——
|
|
309
|
+
# 两次之间域名可改指内网(DNS rebinding)。下面两个连接类在 connect() 里
|
|
310
|
+
# 解析一次、过滤私网、把结果钉给本次 TCP 连接,窗口关闭;SNI/Host 仍用原域名。
|
|
311
|
+
|
|
312
|
+
|
|
313
|
+
def _is_ip_literal(host: str) -> bool:
|
|
314
|
+
"""主机名是否为 IP 字面量(v4 / v6,含 [v6] 与 IPv4-mapped)。"""
|
|
315
|
+
h = (host or "").strip().strip("[]")
|
|
316
|
+
if not h:
|
|
317
|
+
return False
|
|
318
|
+
if ":" in h: # IPv6(含 ::ffff:a.b.c.d)
|
|
319
|
+
return True
|
|
320
|
+
parts = h.split(".")
|
|
321
|
+
return len(parts) == 4 and all(p.isdigit() for p in parts)
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
def resolve_public_addresses(host: str) -> list[str]:
|
|
325
|
+
"""解析主机名 → 公网地址名单(过滤私网/保留段;fake-ip 占位段放行)。
|
|
326
|
+
|
|
327
|
+
返回空列表表示「无可连的公网地址」——调用方应拒绝连接(fail-closed)。
|
|
328
|
+
"""
|
|
329
|
+
try:
|
|
330
|
+
infos = socket.getaddrinfo(host, None, proto=socket.IPPROTO_TCP)
|
|
331
|
+
except OSError:
|
|
332
|
+
return []
|
|
333
|
+
out: list[str] = []
|
|
334
|
+
seen: set[str] = set()
|
|
335
|
+
for info in infos:
|
|
336
|
+
ip = info[4][0]
|
|
337
|
+
if ip in seen:
|
|
338
|
+
continue
|
|
339
|
+
seen.add(ip)
|
|
340
|
+
if is_fake_ip_address(ip): # 代理接管域名(占位段),不可直达内网
|
|
341
|
+
out.append(ip)
|
|
342
|
+
continue
|
|
343
|
+
if is_private_ip(ip):
|
|
344
|
+
continue
|
|
345
|
+
out.append(ip)
|
|
346
|
+
return out
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
def pin_host(host: str) -> str | None:
|
|
350
|
+
"""返回要钉扎的公网 IP;IP 字面量或显式放行私网时返回 None。
|
|
351
|
+
|
|
352
|
+
无可连的公网地址则抛错(fail-closed)。供 safe_conn 的连接类使用。
|
|
353
|
+
"""
|
|
354
|
+
if allow_private() or _is_ip_literal(host):
|
|
355
|
+
return None
|
|
356
|
+
addrs = resolve_public_addresses(host)
|
|
357
|
+
if not addrs:
|
|
358
|
+
raise OSError(f"SSRF 防护:{host} 无可连的公网地址")
|
|
359
|
+
return addrs[0]
|
package/scripts/usage_log.py
CHANGED
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
- 失败静默:任何异常都吞掉返回 False,绝不拖累搜索主路径
|
|
7
7
|
- 目录可注入:ARGO_USAGE_LOG_DIR 覆盖(测试隔离),默认 <状态目录>/usage_log/
|
|
8
8
|
- 总开关:ARGO_USAGE_LOG=0 / false 完全关闭
|
|
9
|
+
- 夹具守卫:query 流 engines_used 出现注册表外引擎名即判测试夹具,整条拒写
|
|
9
10
|
- 脱敏:query 等敏感字段由调用方截断,本模块不放大
|
|
10
11
|
|
|
11
12
|
Schema(每条一行 JSON,UTF-8):
|
|
@@ -68,6 +69,40 @@ def _enabled() -> bool:
|
|
|
68
69
|
return env_flag("ARGO_USAGE_LOG")
|
|
69
70
|
|
|
70
71
|
|
|
72
|
+
# 引擎改名前的历史别名(表外但真实):与 usage_log_clean._ALIAS_KEEP 同一账。
|
|
73
|
+
_RENAMED_ALIAS_KEEP = frozenset({"openlibrary", "bing"})
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _canonical_engines() -> frozenset[str]:
|
|
77
|
+
"""注册表引擎名全集(含历史别名豁免)。注册表加载失败返回空集——
|
|
78
|
+
守卫随之失效放行(fail-open):守卫是防脏数据的,不能反过来丢真日志。
|
|
79
|
+
|
|
80
|
+
不做外层缓存:available_engines 走 get_registry 的 mtime 指纹自愈,每次
|
|
81
|
+
现取才能看见「daemon 常驻期间新增的本地引擎」;外层冻结只会把新引擎的
|
|
82
|
+
真日志误杀成夹具。每次构建 frozenset 是微秒级,query 流每搜索只写一条。
|
|
83
|
+
"""
|
|
84
|
+
try:
|
|
85
|
+
from engines import available_engines
|
|
86
|
+
return frozenset(available_engines()) | _RENAMED_ALIAS_KEEP
|
|
87
|
+
except Exception:
|
|
88
|
+
return frozenset()
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _fixture_engines(record: dict[str, Any]) -> list[str]:
|
|
92
|
+
"""query 记录里的注册表外引擎名(别名豁免后)。
|
|
93
|
+
|
|
94
|
+
engines_used 缺失/非列表/为空一律视为无夹具信号:真实搜索的该字段
|
|
95
|
+
永远是引擎 id 列表,守卫只对「有列表且含表外名」的形态做判定。
|
|
96
|
+
"""
|
|
97
|
+
used = record.get("engines_used")
|
|
98
|
+
if not isinstance(used, list) or not used:
|
|
99
|
+
return []
|
|
100
|
+
canon = _canonical_engines()
|
|
101
|
+
if not canon:
|
|
102
|
+
return []
|
|
103
|
+
return [e for e in used if not isinstance(e, str) or e not in canon]
|
|
104
|
+
|
|
105
|
+
|
|
71
106
|
def emit(stream: str, record: dict[str, Any]) -> bool:
|
|
72
107
|
"""追加一条日志记录到 <usage_log_dir>/<stream>.jsonl。
|
|
73
108
|
|
|
@@ -75,6 +110,16 @@ def emit(stream: str, record: dict[str, Any]) -> bool:
|
|
|
75
110
|
"""
|
|
76
111
|
if not _enabled():
|
|
77
112
|
return False
|
|
113
|
+
if stream == "query" and _fixture_engines(record):
|
|
114
|
+
# 夹具守卫(2026-10-25):测试夹具引擎名(fast / slow / quick_primary
|
|
115
|
+
# 这类注册表外名)屡次写进生产 query.jsonl(conftest 的
|
|
116
|
+
# ARGO_USAGE_LOG=0 与 is_synthetic_run 的黑名单各漏过一批,最近一次
|
|
117
|
+
# 2026-10-24),stats 的均值与最慢/高频引擎榜随之失真。黑名单永远
|
|
118
|
+
# 追不全新夹具名,这里在唯一落盘口用注册表白名单反向判定:表外名
|
|
119
|
+
# 即夹具,整条拒写。存量清理走 usage_log_clean.py(隔离不删除)。
|
|
120
|
+
# 只拦 query 流:route/recovery 的字段形态与测试惯例不同(route 的
|
|
121
|
+
# 格式测试本身用 e0..e14 假名),且实测污染只发生在 query 流。
|
|
122
|
+
return False
|
|
78
123
|
try:
|
|
79
124
|
line = json.dumps(
|
|
80
125
|
{
|
package/scripts/wide_research.py
CHANGED
|
@@ -113,6 +113,40 @@ def normalize_tracks(raw: Any, maximum: int = 9) -> list[dict[str, Any]]:
|
|
|
113
113
|
return out
|
|
114
114
|
|
|
115
115
|
|
|
116
|
+
def build_cross_lingual_tracks(question: str, maximum: int = 4) -> list[dict[str, Any]]:
|
|
117
|
+
"""为溯源与多语言任务自动构建分阶段跨语言长程轨道。"""
|
|
118
|
+
q = (question or "").strip()
|
|
119
|
+
has_tracing = bool(re.search(
|
|
120
|
+
r"(?:溯源|起源|出处|最初|源头|提出者|发明|草案|历史沿革|origin|provenance|root cause|first proposed|specification|rfc|cve)",
|
|
121
|
+
q, re.I
|
|
122
|
+
))
|
|
123
|
+
tracks = []
|
|
124
|
+
tracks.append({
|
|
125
|
+
"id": "track-primary-lang",
|
|
126
|
+
"title": "原语言背景与核心定义",
|
|
127
|
+
"question": f"{q} 背景 定义 现状",
|
|
128
|
+
"rationale": "Primary language overview and conceptual framing.",
|
|
129
|
+
"depends_on": [],
|
|
130
|
+
})
|
|
131
|
+
if has_tracing or len(tracks) < maximum:
|
|
132
|
+
tracks.append({
|
|
133
|
+
"id": "track-origin-trace",
|
|
134
|
+
"title": "源头出处与权威一手规范长程溯源",
|
|
135
|
+
"question": f"{q} original paper inception specification first proposed RFC official repository",
|
|
136
|
+
"rationale": "Cross-lingual long-range tracing for primary sources, papers, or official specifications.",
|
|
137
|
+
"depends_on": ["track-primary-lang"],
|
|
138
|
+
})
|
|
139
|
+
tracks.append({
|
|
140
|
+
"id": "track-cross-reconciliation",
|
|
141
|
+
"title": "多语言证据交叉对账与互补验证",
|
|
142
|
+
"question": f"{q} milestone benchmark evaluation comparison timeline",
|
|
143
|
+
"rationale": "Cross-lingual reconciliation of evidence, timeline differences, and complementary facts.",
|
|
144
|
+
"depends_on": ["track-origin-trace"],
|
|
145
|
+
})
|
|
146
|
+
return tracks[:maximum]
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
|
|
116
150
|
def stage_tracks(tracks: list[dict[str, Any]]) -> tuple[list[list[dict[str, Any]]], list[str]]:
|
|
117
151
|
"""轨道分层。work package 的分层逻辑与轨道同构:id/question/depends_on。"""
|
|
118
152
|
packages = [
|
|
@@ -255,6 +289,17 @@ def evaluate_gates(
|
|
|
255
289
|
if len(caveats) + len(unanswered) >= max(3, (len(sources) + 1) // 2):
|
|
256
290
|
warnings.append({"id": "high_uncertainty",
|
|
257
291
|
"detail": f"{len(caveats)} caveats and {len(unanswered)} unanswered questions exceed the evidence threshold."})
|
|
292
|
+
|
|
293
|
+
# 内容农场与镜像站占优检查
|
|
294
|
+
farm_markers = ("ctolib.com", "coder.work", "voidcn.com", "it1352.com", "oommake.com", "py4u.net",
|
|
295
|
+
"wenku.zone", "chinadmd.com", "jb51.net", "pc6.com", "onlinedown.net")
|
|
296
|
+
farm_hits = [s for s in sources if any(m in str(s.get("url", "")).lower() for m in farm_markers)]
|
|
297
|
+
if farm_hits and len(farm_hits) >= (len(sources) + 1) // 2:
|
|
298
|
+
warnings.append({
|
|
299
|
+
"id": "content_farm_dominant",
|
|
300
|
+
"detail": f"{len(farm_hits)} sources originate from content farms or SEO scraper mirrors; evidence quality severely degraded.",
|
|
301
|
+
})
|
|
302
|
+
|
|
258
303
|
cap = "low" if failures else ("medium" if warnings else "high")
|
|
259
304
|
return {"passed": not failures, "conclusion_cap": cap, "failures": failures, "warnings": warnings}
|
|
260
305
|
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""zhihu_quota.py — 知乎开放平台额度查询(官方 `/api/v1/quota`)。
|
|
3
|
+
|
|
4
|
+
为什么需要(2026-10-21):知乎各能力的每日免费额度是**账号级**的,同一账号的
|
|
5
|
+
多个 Access Secret 共用;本地按引擎计数会与真实额度漂移。官方额度查询不消耗
|
|
6
|
+
业务额度,是唯一权威来源。本模块只做读取与缓存,不参与路由决策(路由仍用
|
|
7
|
+
quota.py 的本地计数),避免每次搜索多一次外呼。
|
|
8
|
+
|
|
9
|
+
用法:
|
|
10
|
+
argo quota # 人读表
|
|
11
|
+
argo quota --json # 机器读
|
|
12
|
+
argo quota --refresh # 跳过 10 分钟缓存
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import argparse
|
|
18
|
+
import json
|
|
19
|
+
import time
|
|
20
|
+
import urllib.request
|
|
21
|
+
|
|
22
|
+
from cli_io import dumps
|
|
23
|
+
from engine_env import get_env
|
|
24
|
+
from engines_base import http_open
|
|
25
|
+
|
|
26
|
+
API_URL = "https://developer.zhihu.com/api/v1/quota"
|
|
27
|
+
CACHE_TTL_S = 600
|
|
28
|
+
|
|
29
|
+
# APIID → argo 引擎名(与 config.yaml / quota_profiles.json 对齐)
|
|
30
|
+
API_TO_ENGINE = {
|
|
31
|
+
"global_search": "zhihu_global",
|
|
32
|
+
"zhihu_search": "zhihu",
|
|
33
|
+
"hot_list": "zhihu_hot",
|
|
34
|
+
"question_discovery": "zhihu_questions",
|
|
35
|
+
"zhida_openai": "zhida",
|
|
36
|
+
"tools": "zhihu_tools",
|
|
37
|
+
"knowledge": "zhihu_kb",
|
|
38
|
+
"user_data": "zhihu_user",
|
|
39
|
+
"creator": "zhihu_creator",
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _cache_path():
|
|
44
|
+
import argo_paths
|
|
45
|
+
return argo_paths.state_path("zhihu_quota.json")
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _secret() -> str:
|
|
49
|
+
return get_env(["ARGO_ZHIHU_ACCESS_SECRET", "ZHIHU_ACCESS_SECRET"])
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _load_cache() -> dict | None:
|
|
53
|
+
try:
|
|
54
|
+
raw = json.loads(_cache_path().read_text(encoding="utf-8"))
|
|
55
|
+
except (OSError, ValueError):
|
|
56
|
+
return None
|
|
57
|
+
if not isinstance(raw, dict):
|
|
58
|
+
return None
|
|
59
|
+
if time.time() - float(raw.get("ts") or 0) > CACHE_TTL_S:
|
|
60
|
+
return None
|
|
61
|
+
return raw
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _save_cache(items: list[dict]) -> None:
|
|
65
|
+
try:
|
|
66
|
+
import argo_paths
|
|
67
|
+
argo_paths.atomic_write_json(_cache_path(),
|
|
68
|
+
{"ts": time.time(), "items": items})
|
|
69
|
+
except Exception:
|
|
70
|
+
pass
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def fetch_quota(*, refresh: bool = False, timeout: float = 8.0) -> dict:
|
|
74
|
+
"""返回 {ok, items, error, cached};永不抛异常(观测层不阻断主路径)。"""
|
|
75
|
+
if not refresh:
|
|
76
|
+
cached = _load_cache()
|
|
77
|
+
if cached is not None:
|
|
78
|
+
return {"ok": True, "items": cached.get("items") or [], "cached": True}
|
|
79
|
+
secret = _secret()
|
|
80
|
+
if not secret:
|
|
81
|
+
return {"ok": False, "items": [], "cached": False,
|
|
82
|
+
"error": "缺少 ZHIHU_ACCESS_SECRET"}
|
|
83
|
+
req = urllib.request.Request(API_URL, headers={
|
|
84
|
+
"Authorization": f"Bearer {secret}",
|
|
85
|
+
"X-Request-Timestamp": str(int(time.time())),
|
|
86
|
+
"Content-Type": "application/json",
|
|
87
|
+
"User-Agent": "argo-search/2.6 (unified-search@local)",
|
|
88
|
+
})
|
|
89
|
+
try:
|
|
90
|
+
with http_open(req, timeout=timeout, engine="zhihu_quota") as resp:
|
|
91
|
+
data = json.loads(resp.read().decode("utf-8", "replace"))
|
|
92
|
+
except Exception as exc:
|
|
93
|
+
return {"ok": False, "items": [], "cached": False,
|
|
94
|
+
"error": f"{type(exc).__name__}: {exc}"}
|
|
95
|
+
if data.get("Code") not in (0, None):
|
|
96
|
+
return {"ok": False, "items": [], "cached": False,
|
|
97
|
+
"error": f"Code={data.get('Code')} {data.get('Message') or ''}"}
|
|
98
|
+
items = []
|
|
99
|
+
for row in data.get("Data") or []:
|
|
100
|
+
if not isinstance(row, dict):
|
|
101
|
+
continue
|
|
102
|
+
api_id = str(row.get("APIID") or "")
|
|
103
|
+
items.append({
|
|
104
|
+
"api_id": api_id,
|
|
105
|
+
"engine": API_TO_ENGINE.get(api_id, api_id),
|
|
106
|
+
"name": row.get("APIName") or api_id,
|
|
107
|
+
"total": row.get("TotalQuota"),
|
|
108
|
+
"used": row.get("TotalUsed"),
|
|
109
|
+
"remaining": row.get("RemainingQuota"),
|
|
110
|
+
})
|
|
111
|
+
_save_cache(items)
|
|
112
|
+
return {"ok": True, "items": items, "cached": False}
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def main(argv: list[str] | None = None) -> int:
|
|
116
|
+
ap = argparse.ArgumentParser(prog="argo quota",
|
|
117
|
+
description="知乎开放平台当日额度(官方 API)")
|
|
118
|
+
ap.add_argument("--json", action="store_true")
|
|
119
|
+
ap.add_argument("--refresh", action="store_true")
|
|
120
|
+
args = ap.parse_args(argv)
|
|
121
|
+
out = fetch_quota(refresh=args.refresh)
|
|
122
|
+
if args.json:
|
|
123
|
+
print(dumps(out))
|
|
124
|
+
return 0 if out.get("ok") else 1
|
|
125
|
+
if not out.get("ok"):
|
|
126
|
+
print(f"额度查询失败:{out.get('error')}")
|
|
127
|
+
return 1
|
|
128
|
+
print("知乎开放平台当日额度(北京时间 00:00 重置)")
|
|
129
|
+
print(f"{'引擎':<18}{'额度项':<20}{'总额':>8}{'已用':>8}{'剩余':>8}")
|
|
130
|
+
for row in out["items"]:
|
|
131
|
+
print(f"{row['engine']:<18}{row['name']:<20}"
|
|
132
|
+
f"{str(row['total']):>8}{str(row['used']):>8}{str(row['remaining']):>8}")
|
|
133
|
+
if out.get("cached"):
|
|
134
|
+
print("(10 分钟内缓存;--refresh 强制刷新)")
|
|
135
|
+
return 0
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
if __name__ == "__main__":
|
|
139
|
+
raise SystemExit(main())
|
|
@@ -43,6 +43,7 @@ _CJK_RE = re.compile(r"[\u4e00-\u9fff]+")
|
|
|
43
43
|
|
|
44
44
|
_K1 = 1.5
|
|
45
45
|
_B = 0.75
|
|
46
|
+
_DELTA = 0.5 # BM25+ 下界 (Lv & Zhai 2011):长文档命中词保底,消除过惩罚
|
|
46
47
|
# 混合权重:BM25 侧。位次先验占 1 - _W_BM25。
|
|
47
48
|
_W_BM25 = 0.6
|
|
48
49
|
# 旧位次衰减口径(max(0.7 - idx*0.05, 0.1))——先验的归一化基准。
|
|
@@ -79,9 +80,10 @@ class BM25:
|
|
|
79
80
|
引擎的 5 条结果上稳定得多——「含查询词的文档有多罕见」是跨源问题。
|
|
80
81
|
"""
|
|
81
82
|
|
|
82
|
-
def __init__(self, docs: list[list[str]], k1: float = _K1, b: float = _B):
|
|
83
|
+
def __init__(self, docs: list[list[str]], k1: float = _K1, b: float = _B, delta: float = _DELTA):
|
|
83
84
|
self.k1 = k1
|
|
84
85
|
self.b = b
|
|
86
|
+
self.delta = delta
|
|
85
87
|
self.n_docs = len(docs)
|
|
86
88
|
self.doc_len = [len(d) for d in docs]
|
|
87
89
|
self.avgdl = (sum(self.doc_len) / self.n_docs) if self.n_docs else 0.0
|
|
@@ -113,10 +115,30 @@ class BM25:
|
|
|
113
115
|
f = self.tf.get(t, {}).get(doc_idx, 0)
|
|
114
116
|
if not f:
|
|
115
117
|
continue
|
|
116
|
-
|
|
118
|
+
sat = (f * (self.k1 + 1.0)) / (f + denom_norm)
|
|
119
|
+
# BM25+ 饱和项下界增益(Lv & Zhai 2011)
|
|
120
|
+
total += self.idf(t) * (sat + self.delta)
|
|
117
121
|
return total
|
|
118
122
|
|
|
119
123
|
|
|
124
|
+
def _phrase_proximity_bonus(query: str, title: str, snippet: str) -> float:
|
|
125
|
+
"""词邻近度与短语连续匹配加成(归一化为 0.0~1.0)。
|
|
126
|
+
|
|
127
|
+
当完整查询短语在标题或摘要逐字紧密出现时给予增益,提升复合词精准命中率。
|
|
128
|
+
"""
|
|
129
|
+
q_low = (query or "").lower().strip()
|
|
130
|
+
if not q_low or len(q_low) < 2:
|
|
131
|
+
return 0.0
|
|
132
|
+
t_low = (title or "").lower()
|
|
133
|
+
s_low = (snippet or "").lower()
|
|
134
|
+
if q_low in t_low:
|
|
135
|
+
return 1.0 # 标题完整覆盖查询短语
|
|
136
|
+
if q_low in s_low:
|
|
137
|
+
return 0.5 # 摘要完整覆盖查询短语
|
|
138
|
+
return 0.0
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
|
|
120
142
|
def _doc_text(r: dict[str, Any]) -> str:
|
|
121
143
|
"""参与相关度计算的文档文本:title 权重高于 snippet——做法是标题
|
|
122
144
|
拼两遍(TF 加成),不引入第二套字段权重机制。"""
|
|
@@ -153,7 +175,9 @@ def rerank(by_engine: dict[str, list[dict[str, Any]]], query: str,
|
|
|
153
175
|
for (eng, idx, r), bm in zip(flat, raw):
|
|
154
176
|
bm25_norm = (bm / max_raw) if max_raw > 0 else 0.0
|
|
155
177
|
prior_norm = _position_prior(idx) / _PRIOR_BASE
|
|
156
|
-
|
|
178
|
+
prox_val = _phrase_proximity_bonus(query, r.get("title", ""), r.get("snippet", ""))
|
|
179
|
+
rel_score = min(bm25_norm * 0.80 + prox_val * 0.20, 1.0) if bm25_norm > 0 else 0.0
|
|
180
|
+
r["score"] = round(w_bm25 * rel_score + (1.0 - w_bm25) * prior_norm, 3)
|
|
157
181
|
|
|
158
182
|
for eng in by_engine:
|
|
159
183
|
by_engine[eng].sort(
|