argo-search 2.9.7 → 2.9.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (140) hide show
  1. package/README.en.md +58 -145
  2. package/README.es.md +58 -145
  3. package/README.ja.md +64 -151
  4. package/README.ko.md +63 -150
  5. package/README.md +65 -154
  6. package/SKILL.md +3 -3
  7. package/assets/readme/hero.svg +2 -2
  8. package/backends/domain_profiles.json +48 -0
  9. package/backends/engine_registry.yaml +118 -4
  10. package/backends/quota_profiles.json +92 -12
  11. package/backends/source_types_cn.json +27 -1
  12. package/bin/argo +74 -5
  13. package/bin/argo.js +33 -2
  14. package/config.yaml +219 -60
  15. package/docs/ENGINE_CATALOG.md +50 -39
  16. package/docs/MCP_SETUP.md +3 -1
  17. package/docs/RELEASE_NOTES_v2.9.8.md +73 -0
  18. package/engines/specs/cfets.yaml +41 -0
  19. package/engines/specs/defillama.yaml +38 -0
  20. package/engines/specs/firecrawl.yaml +1 -1
  21. package/engines/specs/fxtwitter.yaml +5 -0
  22. package/engines/specs/gleif.yaml +55 -0
  23. package/engines/specs/hkex.yaml +40 -0
  24. package/engines/specs/keenable_free.yaml +36 -0
  25. package/engines/specs/shfe.yaml +39 -0
  26. package/engines/specs/stackexchange.yaml +2 -2
  27. package/engines/specs/tavily_free.yaml +56 -0
  28. package/engines/specs/yuanbao.yaml +93 -0
  29. package/mcp/clients.yaml +32 -0
  30. package/package.json +6 -2
  31. package/packages/dsh-plugin/README.md +23 -5
  32. package/packages/dsh-plugin/dsh/client.js +304 -15
  33. package/packages/dsh-plugin/dsh/diagnostics.js +54 -0
  34. package/packages/dsh-plugin/dsh/index.js +112 -1875
  35. package/packages/dsh-plugin/dsh/native-tools.mjs +7 -1
  36. package/packages/dsh-plugin/dsh/runner.js +666 -0
  37. package/packages/dsh-plugin/dsh/wide-research.js +878 -0
  38. package/packages/dsh-plugin/package.json +2 -2
  39. package/packages/dsh-plugin/scripts/sync-local.sh +70 -0
  40. package/packages/dsh-plugin/test/client.test.js +175 -1
  41. package/packages/dsh-plugin/test/diagnostics.test.js +28 -0
  42. package/packages/dsh-plugin/test/index.test.js +44 -0
  43. package/scripts/adaptive.py +24 -1
  44. package/scripts/article_meta.py +131 -0
  45. package/scripts/budget.py +247 -0
  46. package/scripts/budget_cli.py +93 -0
  47. package/scripts/build_release_assets.sh +65 -0
  48. package/scripts/cache.py +60 -72
  49. package/scripts/ci_drift_gate.py +90 -0
  50. package/scripts/content_boundary.py +46 -0
  51. package/scripts/content_signals.py +26 -5
  52. package/scripts/cross_lingual_trace.py +293 -0
  53. package/scripts/dispatch_result.py +98 -0
  54. package/scripts/engine_dispatch.py +49 -86
  55. package/scripts/engine_env.py +5 -0
  56. package/scripts/engine_families.py +6 -0
  57. package/scripts/engine_policy.py +7 -7
  58. package/scripts/engines.py +21 -0
  59. package/scripts/engines_base.py +3 -3
  60. package/scripts/engines_builders.py +12 -0
  61. package/scripts/engines_builders_batch10.py +48 -3
  62. package/scripts/engines_builders_batch14.py +656 -0
  63. package/scripts/engines_builders_batch15.py +249 -0
  64. package/scripts/engines_builders_keyless.py +111 -0
  65. package/scripts/engines_builders_tech.py +0 -1
  66. package/scripts/evidence.py +31 -3
  67. package/scripts/evidence_loop.py +54 -22
  68. package/scripts/fetch_v3.py +25 -0
  69. package/scripts/gen_routing_snapshot.py +11 -0
  70. package/scripts/install.ps1 +18 -0
  71. package/scripts/install.sh +16 -1
  72. package/scripts/job.py +3 -3
  73. package/scripts/local_seek.py +41 -5
  74. package/scripts/macro_countries.py +15 -0
  75. package/scripts/matrix_search_eval.py +82 -0
  76. package/scripts/mcp_compact.py +234 -0
  77. package/scripts/mcp_handlers.py +60 -642
  78. package/scripts/mcp_launch.sh +10 -8
  79. package/scripts/mcp_local.py +241 -0
  80. package/scripts/mcp_server.py +19 -14
  81. package/scripts/mcp_setup.py +33 -9
  82. package/scripts/mcp_tools.py +1 -0
  83. package/scripts/mcp_transport.py +1 -1
  84. package/scripts/mcp_warm.py +127 -0
  85. package/scripts/output_transparency.py +200 -0
  86. package/scripts/paid_policy.py +98 -0
  87. package/scripts/quota.py +158 -13
  88. package/scripts/recovery_runner.py +150 -0
  89. package/scripts/replay_eval.py +18 -0
  90. package/scripts/research.py +37 -21
  91. package/scripts/research_expand.py +24 -1
  92. package/scripts/retry_policy.py +76 -0
  93. package/scripts/route.py +6 -6
  94. package/scripts/route_cache.py +18 -0
  95. package/scripts/route_combo.py +7 -0
  96. package/scripts/route_domains.py +32 -19
  97. package/scripts/search.py +15 -27
  98. package/scripts/search_attribution.py +100 -3
  99. package/scripts/search_cli.py +24 -11
  100. package/scripts/search_entry.py +7 -1
  101. package/scripts/search_output.py +99 -15
  102. package/scripts/search_pipeline.py +42 -193
  103. package/scripts/search_types.py +107 -9
  104. package/scripts/semantic_evidence.py +179 -3
  105. package/scripts/serp_guard.py +115 -7
  106. package/scripts/serp_kill_log.py +25 -2
  107. package/scripts/stats_cli.py +13 -1
  108. package/scripts/wide_research.py +45 -0
  109. package/scripts/zhihu_quota.py +139 -0
  110. package/sub-skills/local-search/bm25.py +27 -3
  111. package/sub-skills/local-search/cli_runner.py +331 -0
  112. package/sub-skills/local-search/parsers.py +328 -0
  113. package/sub-skills/local-search/search_v3.py +31 -674
  114. package/sub-skills/local-seek/scripts/seek_rank.py +7 -3
  115. package/assets/readme/settings-keys.png +0 -0
  116. package/assets/readme/settings-tools.png +0 -0
  117. package/scripts/image_vision/vision_probe +0 -0
  118. package/scripts/local_engines.py +0 -321
  119. package/sub-skills/ego-search/references/browser-runtime.md +0 -43
  120. package/sub-skills/ego-search/references/install.md +0 -67
  121. package/sub-skills/ego-search/references/original-ego-upgrade.md +0 -51
  122. package/sub-skills/ego-search/tests/test_query_normalize.py +0 -68
  123. package/sub-skills/js-run/tests/fixtures/challenge_like.js +0 -27
  124. package/sub-skills/js-run/tests/test_jsrun.py +0 -165
  125. package/sub-skills/local-seek/references/auto-behaviors.md +0 -24
  126. package/sub-skills/local-seek/references/fd-mdfind-recipes.md +0 -62
  127. package/sub-skills/local-seek/references/git-integration.md +0 -54
  128. package/sub-skills/local-seek/references/remote-handoff.md +0 -46
  129. package/sub-skills/local-seek/references/rg-recipes.md +0 -96
  130. package/sub-skills/local-seek/references/strategies.md +0 -52
  131. package/sub-skills/local-seek/references/structural-search.md +0 -59
  132. package/sub-skills/local-seek/tests/test_content_rank.py +0 -222
  133. package/sub-skills/local-seek/tests/test_determinism_20261006.py +0 -145
  134. package/sub-skills/local-seek/tests/test_noise_tier.py +0 -101
  135. package/sub-skills/local-seek/tests/test_pcre2_probe.py +0 -136
  136. package/sub-skills/local-seek/tests/test_rank_pinyin.py +0 -77
  137. package/sub-skills/local-seek/tests/test_regression_20261003.py +0 -169
  138. package/sub-skills/local-seek/tests/test_regression_20261006.py +0 -146
  139. package/sub-skills/local-seek/tests/test_seek_degradation.py +0 -111
  140. package/sub-skills/local-seek/tests/test_seek_fts.py +0 -313
@@ -30,7 +30,7 @@ http_client / engines_base / engine_dispatch。
30
30
 
31
31
  from __future__ import annotations
32
32
 
33
- from dataclasses import dataclass, field, replace
33
+ from search_types import slot_record as dataclass, slot_field as field, slot_replace as replace
34
34
  from typing import Any
35
35
 
36
36
  # 模块级曾有一行 `import logging`,而本模块 7 处 `logging.getLogger(...)`
@@ -56,8 +56,9 @@ from variant_recall import augment_with_variants
56
56
  from phrase_filter import apply_phrase_filter, extract_phrases
57
57
  from except_sets import OPT_IMPORT, SHAPE_BENIGN
58
58
  from postprocess_signals import apply_tfidf_polish, detect_number_conflict
59
- from search_attribution import usage_log_attribution
59
+ from search_attribution import usage_log_attribution, is_synthetic_run
60
60
  from search_output import _collect_errors, _slow_query_ttl, build_funnel
61
+ from serp_kill_log import clean_engine_lists
61
62
  from search_rank import (
62
63
  _LANG_SCRIPT,
63
64
  _RERANK_DEGRADED_STATUSES,
@@ -153,6 +154,7 @@ class _SearchRun:
153
154
  engine_latency: dict[str, float] = field(default_factory=dict)
154
155
  wasted_ms: int = 0
155
156
  useful_ms: int = 0
157
+ dispatch_tail: dict[str, int] = field(default_factory=dict)
156
158
  early_stopped: bool = False
157
159
  budget_used_ms: int | None = None
158
160
  budget_total_ms: int | None = None
@@ -236,16 +238,8 @@ def postprocess(req: _SearchRequest, run: _SearchRun, hooks: Any) -> _SearchRun:
236
238
  merged = run.merged
237
239
 
238
240
  # 融合输入派生(原先在 execute_search 里,属于加工的第一步)
239
- valid_lists = [
240
- res for res in raw_results.values()
241
- if res and any(isinstance(r, dict) and "error" not in r for r in res)
242
- ]
243
- # 去掉 error-only 列表中的 error 条目
244
- clean_lists = []
245
- for res in valid_lists:
246
- clean = [r for r in res if isinstance(r, dict) and "error" not in r]
247
- if clean:
248
- clean_lists.append(clean)
241
+ # 去 error 与 SERP 守卫同源:缓存命中的结果也要过同一道门(issue #18)。
242
+ clean_lists = clean_engine_lists(raw_results, query)
249
243
 
250
244
  # 查询主语言:噪声门与语言能力加权共用同一个判定(唯一来源,
251
245
  # 避免两处各自算导致行为漂移)。
@@ -387,8 +381,8 @@ def postprocess(req: _SearchRequest, run: _SearchRun, hooks: Any) -> _SearchRun:
387
381
  _funnel_deduped = len(merged)
388
382
  if merged:
389
383
  try:
390
- from serp_guard import is_serp_or_jump_url as _is_serp # 轻依赖,锁见 test_hotpath_import_lint.py
391
- merged = [r for r in merged if not _is_serp(r.get("url", ""))]
384
+ from serp_guard import is_unusable_url as _is_unusable # SERP + 内容农场,标定见 test_content_farm_guard.py
385
+ merged = [r for r in merged if not _is_unusable(r.get("url", ""), r.get("title", ""), r.get("snippet", ""))]
392
386
  except ImportError:
393
387
  pass # serp_guard 不可用时跳过(本地五维 rerank 已对 SERP 降权)
394
388
 
@@ -515,176 +509,33 @@ def postprocess(req: _SearchRequest, run: _SearchRun, hooks: Any) -> _SearchRun:
515
509
  except OPT_IMPORT + SHAPE_BENIGN: # 守卫不可用不设卡(fail-open)
516
510
  _echo_triggered = False
517
511
  if not merged or _echo_triggered:
518
- # ── 恢复链 L0:early-stop 误判时补跑被跳过的同域引擎 ──
519
- # (同域优先于 L3 换跨域引擎;实现在 recovery_l0)。死线 = 用户
520
- # 声明的整体超时扣掉 dispatch 已耗墙钟——补跑不得把总时长推出
521
- # 用户预算(语义见 recovery_l0 模块 docstring 的死线约束)。
522
- try:
523
- import time as _time
524
- from recovery_l0 import maybe_retry_skipped
525
- try:
526
- _l0_deadline = _time.monotonic() + max(
527
- 0.0, float(req.timeout or 0)
528
- - float(run.budget_used_ms or 0) / 1000.0)
529
- except (TypeError, ValueError):
530
- _l0_deadline = None
531
- _l0_merged, _l0_info = maybe_retry_skipped(
532
- run.early_stopped, engines_combo, raw_results,
533
- available_engines, query, _run_one, _ingest,
534
- max_results, _emit_usage, deadline=_l0_deadline)
535
- except SHAPE_BENIGN:
536
- _l0_merged, _l0_info = [], None
537
- if _l0_merged and _l0_info:
538
- recovery_info = _l0_info
539
- merged = _l0_merged
540
- if not merged or _echo_triggered:
541
- try:
542
- from recovery import run_recovery
543
- tried = list(raw_results.keys()) or list(engines)
544
- fallback_engines = decision.get("engines_fallback") or []
545
- # 域路由零结果:恢复链放行 L3 换引擎。复杂度门此前把简单查询
546
- # 压到 L2——L3 被禁 + 全域零结果 = 域命中查询无解(实测
547
- # macro_data「中国GDP」零结果、恢复链空转)。engines_fallback
548
- # 里是路由的定向保底声明(域未试成员优先),代价可控。
549
- rec_level = _max_rec_level
550
- if _echo_triggered:
551
- # 退化是「引擎选错了」而非「查询写得不好」:L1 放宽 / L2 换词
552
- # 都救不了「上游把整句当单词查」,必须换引擎(L3)。
553
- rec_level = "L3"
554
- elif fallback_engines and decision.get("domain") not in (
555
- None, "", "general", "general_search") \
556
- and (rec_level is None or rec_level < "L3"):
557
- rec_level = "L3"
558
- try:
559
- enabled_set = set(available_engines())
560
- except Exception:
561
- enabled_set = None
562
-
563
- def _recovery_executor(rq: str, rengines: list[str]) -> list[dict[str, Any]]:
564
- """恢复执行器:对冲竞速跑候选引擎,取首个非空。跳过缓存避免污染。
565
-
566
- 并行形态(2026-09-29):原为严格串行——域查询零结果时 recovery
567
- 段占 process_ms 的 79–86%(实测中位 906ms;「黄金价格」2019ms、
568
- 「比特币行情」1989ms),是体感「有时特别慢」的直接原因。改为
569
- 「起步间隔 = grace 的有界并发」(bounded_run.run_bounded 的
570
- stagger 语义,与 dispatch 串行路径同一套对冲哲学,见
571
- engine_dispatch._run_engines_bounded):首引擎先跑,grace 内
572
- 交付仍只付一次调用,超时才补发下一匹;任一引擎交付非空即收工。
573
- grace 复用 hooks.primary_grace_s(标定依据见 search._PRIMARY_GRACE_S),
574
- 不新增开关面。总等待上限 = grace×(n-1) + timeout + 1s ≤ 串行版
575
- n×timeout,最坏情况也不慢于改造前。消融:把 primary_grace 调到
576
- 大于引擎超时上限即完全退回严格串行(与 ARGO_SERIAL_STAGGER_S
577
- 同思路)。daemon 线程不 join,卡住的 HTTP 读拖不垮止损。
578
- """
579
- def _fire(eng: str) -> list[dict[str, Any]]:
580
- _dk = {k: v for k in ("domain", "sub_domain") if (v := getattr(req, "engine_" + k, None))}
581
- # 记下真正发出的调用:恢复段走的不是 dispatch,若不单独记账,
582
- # 漏斗会算出「routed 2 → called 2 → returned 5」这种自相矛盾的
583
- # 账(实测「python 怎么读csv」),而这正是最需要被看见的路径。
584
- _recovery_engines.add(eng)
585
- try:
586
- # 恢复路径同样携带时间窗,避免恢复时丢弃用户约束;
587
- # --no-cache 也要透传:本函数的契约是「跳过缓存避免污染」,
588
- # 不透传会让恢复段既读到用户明确拒绝的旧缓存,又把自己的
589
- # 临时查询写进缓存(dispatch 那条路径是透传的,两处不一致)。
590
- res = engine_search(rq, eng, n=max_results,
591
- timeout=timeout, depth=depth, mode=mode,
592
- since=since_iso, until=until_iso,
593
- skip_cache=skip_cache, **_dk)
594
- except Exception as e:
595
- import sys
596
- sys.stderr.write(
597
- f"[recovery] 恢复链异常({type(e).__name__}): {e}\n")
598
- res = []
599
- goods = [r for r in (res or [])
600
- if isinstance(r, dict) and "error" not in r]
601
- # 恢复段同样进配额账:它真的打了引擎(engine_search),
602
- # 而 quota_batch.add 的全仓唯一常规入口在 dispatch 的
603
- # _run_one 里——恢复链不走那条路,不在这里补一行,
604
- # 这批消耗就落在本地账本之外(见下方 flush 处注释)。
605
- # add 是 list.append(GIL 下原子),对冲线程并发调用安全。
606
- quota_batch.add(eng, bool(goods))
607
- if goods:
608
- for r in goods:
609
- r.setdefault("_engine", eng)
610
- r.setdefault("_recovered", True)
611
- # 回声不算「恢复成功」(判据与理由见
612
- # postprocess_signals.drop_echo_items):不拦则 L1 放宽
613
- # 查询后收工,L3 换引擎永不执行。
614
- try:
615
- from postprocess_signals import drop_echo_items as _drop_echo
616
- goods = _drop_echo(goods, rq)
617
- except OPT_IMPORT + SHAPE_BENIGN:
618
- pass
619
- return goods
620
-
621
- if len(rengines) <= 1:
622
- return _fire(rengines[0]) if rengines else []
623
- grace = max(0.0, float(hooks.primary_grace_s or 0.0))
624
- wait_s = grace * (len(rengines) - 1) \
625
- + max(1.0, float(timeout)) + 1.0
626
-
627
- def _enough(finished: list[tuple[str, Any]]) -> bool:
628
- # 任一引擎交付非空即收工:不再起排队中的引擎(省调用),
629
- # 已在跑的归 unfinished(daemon,结果丢弃但配额已如实记)。
630
- return any(isinstance(v, list) and v for _i, v in finished)
631
-
632
- finished, _unfinished = run_bounded(
633
- rengines, _fire, wait_s, max_workers=3,
634
- enough=_enough, stagger_s=grace)
635
- # finished 按完成先后排列:首个非空即「最快非空」。与串行版
636
- # 「首个非空」的差别只在来源引擎——恢复候选都过 run_recovery
637
- # 的 query-signal 门,质量由它把关。
638
- for _item, value in finished:
639
- if isinstance(value, list) and value:
640
- return value
641
- return []
642
-
643
- rec_results, rec_result = run_recovery(
644
- query, tried, _recovery_executor,
645
- engines_fallback=fallback_engines, enabled=enabled_set, mode=mode,
646
- max_level=rec_level)
647
- recovery_info = rec_result.to_dict()
648
- # P2-6:恢复使用日志——query 截断脱敏,只记概览不记明细
649
- if _emit_usage is not None:
650
- try:
651
- _emit_usage("recovery", {
652
- "query": (query[:60] if query else query),
653
- "triggered": recovery_info.get("triggered"),
654
- "recovered": recovery_info.get("recovered"),
655
- "level_used": recovery_info.get("level_used"),
656
- "strategy_used": recovery_info.get("strategy_used"),
657
- "steps_tried": len(recovery_info.get("steps_tried") or []),
658
- "final_query": (recovery_info.get("final_query") or "")[:60],
659
- "note": recovery_info.get("note", ""),
660
- })
661
- except SHAPE_BENIGN: # 使用日志记录失败不影响恢复结果本身
662
- pass
663
- if rec_results:
664
- # 恢复结果复检壳口径:主链的壳过滤跑在恢复链之前,恢复产出
665
- # 是唯一可能把壳带回最终输出的入口(唯一出口在
666
- # serp_guard.drop_shell_urls,L0/L3/充分性三处共用)。
667
- try:
668
- from recovery_l0 import finalize_l3_results
669
- merged = finalize_l3_results(rec_results, max_results)
670
- except ImportError:
671
- merged = deduplicate_by_url(rec_results)[:max_results]
672
- # 恢复引擎按引擎分组记回 raw_results:engines_used 此前不含
673
- # 救援引擎(provenance 断链,实测恢复成功后 engines_used 仍
674
- # 只列原 combo),自适应学习也看不到恢复成功信号。
675
- _rec_by_eng: dict[str, list] = {}
676
- for r in merged:
677
- eng = r.get("_engine") or r.get("source") or ""
678
- if eng:
679
- r.setdefault("consensus_engines", [eng])
680
- _rec_by_eng.setdefault(eng, []).append(r)
681
- for _eng, _lst in _rec_by_eng.items():
682
- raw_results.setdefault(_eng, _lst)
683
- except ImportError:
684
- pass # recovery 模块不可用
685
- except Exception as e:
686
- _log(
687
- f"错误恢复跳过: {type(e).__name__}")
512
+ from recovery_runner import run_pipeline_recovery
513
+ _dk = {k: v for k in ("domain", "sub_domain") if (v := getattr(req, "engine_" + k, None))}
514
+ merged, _rec_info = run_pipeline_recovery(
515
+ query=query,
516
+ merged=merged,
517
+ raw_results=raw_results,
518
+ decision=decision,
519
+ engines=engines,
520
+ available_engines_fn=available_engines,
521
+ max_results=max_results,
522
+ max_rec_level=_max_rec_level,
523
+ echo_triggered=_echo_triggered,
524
+ primary_grace_s=float(hooks.primary_grace_s or 0.0),
525
+ timeout=timeout,
526
+ depth=depth,
527
+ mode=mode,
528
+ since_iso=since_iso,
529
+ until_iso=until_iso,
530
+ skip_cache=skip_cache,
531
+ quota_batch=quota_batch,
532
+ engine_search_fn=engine_search,
533
+ emit_usage_fn=_emit_usage,
534
+ recovery_engines_set=_recovery_engines,
535
+ engine_domain_extra=_dk,
536
+ )
537
+ if _rec_info:
538
+ recovery_info = _rec_info
688
539
 
689
540
  # 配额记账落盘:整批一次写入文件(同一次搜索的 N 个引擎合并为一次写)。
690
541
  # 位置是契约:必须在**最后一个 quota add 点**之后。D6 补搜走后,
@@ -886,6 +737,7 @@ def finalize(req: _SearchRequest, run: _SearchRun, hooks: Any) -> dict[str, Any]
886
737
  funnel = run.funnel
887
738
  result_payload = run.result_payload
888
739
  wasted_ms, useful_ms = run.wasted_ms, run.useful_ms
740
+ dispatch_tail = run.dispatch_tail
889
741
  early_stopped = run.early_stopped
890
742
  budget_used_ms, budget_total_ms = run.budget_used_ms, run.budget_total_ms
891
743
  elapsed = run.elapsed
@@ -979,15 +831,15 @@ def finalize(req: _SearchRequest, run: _SearchRun, hooks: Any) -> dict[str, Any]
979
831
  # 字符(与 recovery 流同一脱敏纪律);记录失败静默,绝不拖累主路径。
980
832
  # 取自 req 而非局部别名:_emit_usage 是 postprocess 的局部变量,
981
833
  # 本函数(finalize)不在其作用域内,直接引用会 NameError。
982
- if req.emit_usage_log is not None:
834
+ if req.emit_usage_log is not None and not is_synthetic_run(raw_results):
983
835
  try:
984
- # 引擎级耗时归因(2026-10-15 起):写在 search_attribution,本处只接线。
836
+ # 归因走 search_attribution;合成引擎(基准/混沌)整条不入日志。
985
837
  req.emit_usage_log("query", {
986
838
  "query": (query[:60] if query else query),
987
839
  "count": len(merged),
988
840
  "elapsed_ms": round(elapsed),
989
841
  "engines_used": list(raw_results.keys())[:8],
990
- **usage_log_attribution(engine_latency),
842
+ **usage_log_attribution(engine_latency, engine_outcomes=engine_outcomes),
991
843
  "errors": len(_collect_errors(raw_results, engine_outcomes)),
992
844
  "recovered": bool(recovery_info.get("recovered")) if recovery_info else False,
993
845
  "early_stopped": bool(early_stopped),
@@ -1054,19 +906,16 @@ def finalize(req: _SearchRequest, run: _SearchRun, hooks: Any) -> dict[str, Any]
1054
906
  out["timing"] = timing.summary()
1055
907
  out["timing"]["elapsed_ms"] = elapsed
1056
908
  out["timing"]["dispatch"] = {
1057
- # 用 dispatch 自己那口单调钟量出来的墙钟(budget_used_ms),不用外层
1058
- # 这笔 time.time() 差值:useful/wasted 都是单调钟算的,混用两种钟会
1059
- # 对不上账。注意 2026-10-10 记账改版起 useful + wasted 与 wall 没有
1060
- # 恒等式关系,别再按恒等式读仪表(口径真源:engine_dispatch 记账注释)。
909
+ # 墙钟/尾等待/弃置的口径真源见 dispatch_result.py;useful + wasted 与
910
+ # wall 没有恒等式(2026-10-10 起),不要再按恒等式读仪表。
1061
911
  "wall_ms": budget_used_ms,
1062
912
  "engines_run": len(engine_latency),
1063
913
  "engine_sum_ms": eng_sum,
1064
914
  "parallel_efficiency": (round(eng_sum / budget_used_ms, 2)
1065
915
  if budget_used_ms else None),
1066
- # useful = 最后一个贡献结果引擎的完成时刻;wasted = 被编排主动弃置
1067
- # 的引擎已花墙钟之和。与 wall 无恒等式(真源:engine_dispatch 记账注释)。
1068
916
  "useful_ms": useful_ms,
1069
917
  "wasted_ms": wasted_ms,
918
+ **dispatch_tail,
1070
919
  "early_stopped": early_stopped,
1071
920
  }
1072
921
  if budget_total_ms is not None:
@@ -1,20 +1,118 @@
1
1
  #!/usr/bin/env python3
2
- """search_types.py — 统一类型系统(已废弃,勿接入)
2
+ """search_types.py — 统一类型系统(轻量记录 + 零依赖 dataclass 替代)。
3
3
 
4
- 早期设计声明的统一 schema 层,从未被任何模块 import(grep 全仓库无引用):
5
- - 引擎输出直接透传 dict,运行时防御在 engines_base/safe_search 与
6
- search.py 的 execute_search 里完成
7
- - 保留本文件仅避免旧版外部脚本 import 断裂;新代码禁止使用
4
+ `slot_record` / `slot_field` / `slot_replace` 是为热路径准备的
5
+ `@dataclass` / `field` / `replace` 零依赖替代:`dataclasses` 的导入链是
6
+ `dataclasses → inspect → tokenize → dis`(实测 ~8ms),而 `SearchResult`
7
+ 这类记录在每次搜索都要构造,导入成本全落在固定开销上。
8
+
9
+ 兼容面:挂 `__dataclass_fields__`,标准库的 `dataclasses.replace` /
10
+ `is_dataclass` / `fields` 对实例照常工作(`_field_type` 惰性解析)。
11
+
12
+ 仍不建议新代码接入本模块的 `SearchResult`(引擎输出直接透传 dict,运行时
13
+ 防御在 engines_base/safe_search 里);这里的价值在**替代实现本身**。
8
14
  """
9
15
  from __future__ import annotations
10
16
 
17
+ import sys
11
18
  from enum import Enum
12
-
13
- from dataclasses import dataclass, field, asdict
14
19
  from typing import Any
15
20
 
16
21
 
17
- @dataclass
22
+ class _FieldMeta:
23
+ """`__dataclass_fields__` 的兼容描述子。
24
+
25
+ `dataclasses.replace` 只读 `.init`;`dataclasses.fields` 会按
26
+ `_field_type is dataclasses._FIELD` 过滤,所以 `_field_type` 做成**惰性
27
+ 属性**:dataclasses 已被 import 时(也就是有人真去调 `fields()` 时)
28
+ 返回真哨兵,没 import 时返回 None。热路径因此仍然零导入——`_FIELD`
29
+ 只在 dataclasses 已经进场后才去取它。
30
+ """
31
+
32
+ __slots__ = ("name", "init")
33
+
34
+ def __init__(self, name: str) -> None:
35
+ self.name = name
36
+ self.init = True
37
+
38
+ @property
39
+ def _field_type(self) -> Any:
40
+ dc = sys.modules.get("dataclasses")
41
+ return getattr(dc, "_FIELD", None) if dc is not None else None
42
+
43
+
44
+ class _SlotField:
45
+ __slots__ = ("factory",)
46
+
47
+ def __init__(self, default_factory: Any) -> None:
48
+ self.factory = default_factory
49
+
50
+
51
+ def slot_field(*, default_factory: Any) -> Any:
52
+ """零依赖 field(default_factory=...) 替代。"""
53
+ return _SlotField(default_factory)
54
+
55
+
56
+ def _class_annotations(cls: Any) -> dict:
57
+ """读类的注解,兼容 PEP 649 惰性注解。
58
+
59
+ `cls.__dict__["__annotations__"]` 在 3.14 的惰性注解下**可能根本不存在**
60
+ (注解由 `__annotate__` 描述子按需计算,不进类字典)——此时取到的是
61
+ `{}`,类会静默变成「零字段」,实例化时报一个与真因毫不相干的
62
+ TypeError。必须走 `getattr` 触发描述子。
63
+ """
64
+ return getattr(cls, "__annotations__", None) or {}
65
+
66
+
67
+ def slot_record(_cls: Any = None, *, frozen: bool = False) -> Any:
68
+ """零依赖 @dataclass 替代(省去 import dataclasses→inspect→tokenize→dis ~8ms)。
69
+
70
+ 挂上 `__dataclass_fields__` 兼容描述子,标准库 `dataclasses.replace` /
71
+ `dataclasses.is_dataclass` / `dataclasses.fields` 对实例照常工作,但热
72
+ 路径自身零导入(`_field_type` 惰性解析,只在 dataclasses 已进场时取哨兵)。
73
+ """
74
+ def _wrap(cls: Any) -> Any:
75
+ ann = list(_class_annotations(cls).keys())
76
+ defaults = {k: cls.__dict__[k] for k in ann if k in cls.__dict__}
77
+
78
+ def __init__(self: Any, *args: Any, **kwargs: Any) -> None:
79
+ if len(args) > len(ann):
80
+ raise TypeError(f"{cls.__name__}.__init__() takes {len(ann)} positional arguments but {len(args)} were given")
81
+ for i, val in enumerate(args):
82
+ object.__setattr__(self, ann[i], val)
83
+ for k in ann[len(args):]:
84
+ if k in kwargs:
85
+ val = kwargs.pop(k)
86
+ elif k in defaults:
87
+ d = defaults[k]
88
+ val = d.factory() if isinstance(d, _SlotField) else d
89
+ else:
90
+ raise TypeError(f"{cls.__name__}.__init__() missing required argument: {k!r}")
91
+ object.__setattr__(self, k, val)
92
+ if kwargs:
93
+ bad = next(iter(kwargs))
94
+ raise TypeError(f"{cls.__name__}.__init__() got an unexpected keyword argument {bad!r}")
95
+
96
+ cls._slot_fields = tuple(ann)
97
+ cls.__dataclass_fields__ = {k: _FieldMeta(k) for k in ann}
98
+ cls.__init__ = __init__
99
+ if frozen:
100
+ def __setattr__(self: Any, name: str, value: Any) -> None:
101
+ raise AttributeError(f"cannot assign to field {name!r}")
102
+ cls.__setattr__ = __setattr__
103
+ return cls
104
+
105
+ return _wrap if _cls is None else _wrap(_cls)
106
+
107
+
108
+ def slot_replace(obj: Any, **changes: Any) -> Any:
109
+ """零依赖 replace(obj, **changes) 替代。"""
110
+ kw = {k: getattr(obj, k) for k in obj._slot_fields}
111
+ kw.update(changes)
112
+ return type(obj)(**kw)
113
+
114
+
115
+ @slot_record
18
116
  class SearchResult:
19
117
  """统一的搜索结果数据类。"""
20
118
  title: str
@@ -22,7 +120,7 @@ class SearchResult:
22
120
  snippet: str = ""
23
121
  score: float = 0.5
24
122
  source: str = ""
25
- metadata: dict = field(default_factory=dict)
123
+ metadata: dict = slot_field(default_factory=dict)
26
124
  social_meta: dict | None = None
27
125
 
28
126
  def to_dict(self) -> dict[str, Any]:
@@ -43,6 +43,31 @@ logger = logging.getLogger("unified_search.semantic_evidence")
43
43
 
44
44
  DEFAULT_ENDPOINT = "https://classifier.dev"
45
45
  DEFAULT_TIMEOUT_S = 5.0
46
+ # 未声明 transport 时的默认通道。jev = TypeSafe System One(当前唯一可用);
47
+ # classifier.dev 的匿名路径 2026-10-22 起回 403 proxy_requires_payment。
48
+ DEFAULT_TRANSPORT = "jev"
49
+
50
+ # ── JEV / System One 通道(TypeSafe)──────────────────────────────────────
51
+ # 官方 API:POST https://api.typesafe.ai/v1/systemone,Bearer 鉴权
52
+ # (docs.typesafe.ai/api/system-one)。与 classifier.dev 的 /v1/classify 是
53
+ # **两种形状**:那边一次发 N 条文本 + 一组标签;这边一次发一个 state + 若干
54
+ # 具名问题,每题独立作答(文档原话:questions are evaluated in parallel and
55
+ # in isolation)。所以接过来要做的是形状转换,不是换个 URL。
56
+ DEFAULT_JEV_ENDPOINT = "https://api.typesafe.ai/v1/systemone"
57
+ DEFAULT_JEV_MODEL = "jev-latest"
58
+ # key 候选链:argo 推荐名在前;TYPESAFE_API_KEY 是官方 SDK 与 JEV 插件
59
+ # (dsh-verbatim-compaction)用的名字,照文档配的人直接可用。
60
+ JEV_KEY_ENV = ("ARGO_JEV_API_KEY", "TYPESAFE_API_KEY")
61
+ # state 超过这个长度就带长上下文头。实测(2026-10-22 本机):
62
+ # 不带 TypeSafe-Danger-Zone-Long-Context:32K tokens 通过(32350)、
63
+ # 40K 中文字(≈40K tokens)回 `max_tokens_exceeded`(400)
64
+ # 带上:40K 字(40358 tokens)与 80K 字(80358 tokens)均通过
65
+ # 24000 字是保守拐点(正常路径不带头也稳),不是实测上限。
66
+ JEV_LONG_CONTEXT_CHARS = 24000
67
+ # 并发上限。官方 cookbook(classifying_rag_passages)取 4,理由是端点限流;
68
+ # 429/529 要求指数退避,所以不铺大。
69
+ JEV_MAX_WORKERS = 4
70
+
46
71
  # conf ≥ 0.7 才给结论:低于此值的判定不可信(B 实测 4 个严格错误全部 ≤0.67)
47
72
  SUPPORT_THRESHOLD = 0.7
48
73
  # 单条文本上限:核验链路取回的正文 ≤8000 字,再高无收益(A 实测 400→7798 字
@@ -100,20 +125,167 @@ def _timeout() -> float:
100
125
  return DEFAULT_TIMEOUT_S
101
126
 
102
127
 
128
+ def jev_api_key() -> str:
129
+ """JEV(TypeSafe)key:os.environ 优先,~/.config/argo/env 热读保底。"""
130
+ try:
131
+ from engine_env import get_env
132
+ return (get_env(list(JEV_KEY_ENV)) or "").strip()
133
+ except Exception: # pragma: no cover - 环境层不可用时不冒险
134
+ import os
135
+ for _n in JEV_KEY_ENV:
136
+ _v = (os.environ.get(_n) or "").strip()
137
+ if _v:
138
+ return _v
139
+ return ""
140
+
141
+
142
+ def _transport() -> str:
143
+ """通道选择:`jev`(TypeSafe System One)或 `classify`(classifier.dev)。
144
+
145
+ **纯配置驱动,不做「按键就绪自动选」**。初版按 jev_api_key() 是否就绪自动
146
+ 选,结果是通道取决于本机 ~/.config/argo/env 里有没有那把 key——同一份代码
147
+ 在开发机与 CI 上走不同分支,既有测试在开发机上会静默改道(实测:写进 key
148
+ 后 test_semantic_evidence 的 5 条 HttpClient 桩里 1 条改走 JEV 路径而挂)。
149
+ 机器状态不该改变代码路径。
150
+
151
+ 未声明时的默认是 jev:classifier.dev 的匿名路径 2026-10-22 起回 403
152
+ proxy_requires_payment,JEV 是当前可用的那条。
153
+ """
154
+ declared = str(_config().get("transport") or "").strip().lower()
155
+ return declared if declared in ("jev", "classify") else DEFAULT_TRANSPORT
156
+
157
+
158
+ def _jev_endpoint() -> str:
159
+ """JEV 端点:**独立于 classify 通道的 endpoint 字段**。
160
+
161
+ 不共用是刻意的坑修:config.yaml 里 `endpoint` 长期写的是
162
+ https://classifier.dev,JEV 通道若读同一个字段,就会把 System One 的
163
+ 请求体发到 classifier.dev,拿回「Invalid API key / Create a workspace
164
+ key at /app/keys」这种指向完全错误方向的 401(2026-10-22 实测踩中)。
165
+ 每个通道自带端点,杜绝「换 transport 忘了换 endpoint」。
166
+ """
167
+ return str(_config().get("jev_endpoint") or DEFAULT_JEV_ENDPOINT).rstrip("/")
168
+
169
+
170
+ def _jev_model() -> str:
171
+ return str(_config().get("jev_model") or _config().get("model")
172
+ or DEFAULT_JEV_MODEL)
173
+
174
+
175
+ def _jev_post(path_state: Any, questions: dict[str, Any], *,
176
+ timeout: float) -> Optional[dict[str, Any]]:
177
+ """一次 System One 请求;失败返回 None(fail-open 的全部出口都在这里)。"""
178
+ key = jev_api_key()
179
+ if not key:
180
+ logger.debug("JEV key 未配置(%s),语义层跳过", " / ".join(JEV_KEY_ENV))
181
+ return None
182
+ body = {
183
+ "model": _jev_model(),
184
+ "state": path_state,
185
+ "questions": questions,
186
+ }
187
+ # Content-Type 必须显式给:HttpClient.post 会把 dict 做 json.dumps,
188
+ # 但**从不设 Content-Type**(头只来自浏览器 UA 模板 + extra_headers)。
189
+ # 缺它时 FastAPI 侧把 body 当字符串,回
190
+ # 422 {"loc":["body"],"msg":"Input should be a valid dictionary…"}
191
+ # ——报错指向「请求体不是对象」,而真因是少了一个头(2026-10-22 实测踩中)。
192
+ headers = {"Authorization": f"Bearer {key}",
193
+ "Content-Type": "application/json"}
194
+ # 长上下文要显式带头,否则 state 一过 32K tokens 就 400 max_tokens_exceeded
195
+ if len(json.dumps(path_state, ensure_ascii=False)) > JEV_LONG_CONTEXT_CHARS:
196
+ headers["TypeSafe-Danger-Zone-Long-Context"] = "true"
197
+ try:
198
+ from http_client import HttpClient
199
+ resp = HttpClient(timeout=timeout, max_retries=1, jitter=False).post(
200
+ _jev_endpoint(), body, extra_headers=headers)
201
+ if resp.get("status") != 200:
202
+ logger.debug("JEV 非 200: %s %s", resp.get("status"),
203
+ str(resp.get("text") or resp.get("error") or "")[:200])
204
+ return None
205
+ return json.loads(resp.get("text") or "{}")
206
+ except Exception as e: # fail-open:语义层任何问题都不阻断核验
207
+ logger.debug("JEV 请求失败: %s", type(e).__name__)
208
+ return None
209
+
210
+
211
+ def _jev_choice(text: str, labels: tuple[str, ...] | list[str],
212
+ instructions: str | None, timeout: float
213
+ ) -> Optional[dict[str, Any]]:
214
+ """单条文本 → 一个 Choice 问题。
215
+
216
+ **每条一个请求**,不是「多段一个 state、每段一问」。理由在官方文档里:
217
+ jaggedness 那篇明写「state 里塞与决策无关的细节会降低准确率(context
218
+ rot)」,而判第 i 段时其余各段就是无关细节;cookbook 的
219
+ classifying_rag_passages 也是「One request per passage」。(我们试过 3 段
220
+ 合一,结果全对,但样本太小,不拿它赌准确率。)
221
+ """
222
+ out = _jev_post(text, {
223
+ "cls": {
224
+ "type": "choice",
225
+ "instructions": instructions or "Classify the state into one of the criteria.",
226
+ # criteria 的**值**是描述:jaggedness 说 instructions 与 criteria
227
+ # 不一致时模型会困惑,所以两边用同一句话,不给它两个版本。
228
+ "criteria": {str(lab): str(lab) for lab in labels},
229
+ },
230
+ }, timeout=timeout)
231
+ if not isinstance(out, dict):
232
+ return None
233
+ ans = (out.get("answers") or {}).get("cls")
234
+ if not isinstance(ans, dict) or not ans.get("choice"):
235
+ if isinstance(ans, dict):
236
+ logger.debug("JEV 答案缺 choice: %s", str(ans)[:120])
237
+ return None
238
+ return {
239
+ "label": str(ans.get("choice")),
240
+ "confidence": ans.get("confidence"),
241
+ "scores": ans.get("probabilities"),
242
+ }
243
+
244
+
245
+ def _classify_jev(texts: list[str], labels: tuple[str, ...] | list[str], *,
246
+ instructions: str | None, keep: list[tuple[int, str]],
247
+ timeout: float) -> Optional[list[dict[str, Any]]]:
248
+ """JEV 通道:每条文本一个请求,并发 4。
249
+
250
+ 任一条失败就在该位留 None(与 classifier.dev 通道「整批失败即 None」
251
+ 不同):一次核验有几条正文,坏一条不该把其余判定的价值也丢掉。全部失败
252
+ 时仍返回 None,调用方行为与未开启时一致。
253
+ """
254
+ from concurrent.futures import ThreadPoolExecutor
255
+ texts_kept = [t[:MAX_TEXT_CHARS] for _i, t in keep]
256
+ with ThreadPoolExecutor(max_workers=min(JEV_MAX_WORKERS, len(texts_kept))) as pool:
257
+ results = list(pool.map(
258
+ lambda t: _jev_choice(t, labels, instructions, timeout), texts_kept))
259
+ if not any(r for r in results):
260
+ return None
261
+ out: list[Optional[dict[str, Any]]] = [None] * len(texts)
262
+ for (idx, _t), r in zip(keep, results):
263
+ if isinstance(r, dict):
264
+ out[idx] = r
265
+ return out # type: ignore[return-value]
266
+
267
+
103
268
  def classify(texts: list[str], labels: tuple[str, ...] | list[str], *,
104
269
  instructions: str | None = None,
105
270
  timeout: float | None = None,
106
271
  tier: str | None = None) -> Optional[list[dict[str, Any]]]:
107
- """一次批量分类(≤1000 条)。任何异常/非 200/条数不匹配 → None。
272
+ """一次批量分类。任何异常/非 200/条数不匹配 → None。
108
273
 
109
274
  返回按输入顺序排列的 [{label, confidence, scores}];调用方负责判定阈值。
110
275
  空文本会被过滤后再发(空串会让整批 400),返回列表与输入等长、空缺位为 None。
276
+
277
+ 两个通道的实现分居 _classify_jev(TypeSafe,每条一请求)与下方内联的
278
+ classifier.dev 路径(一次发 N 条);开关判定见 _transport。
111
279
  """
112
280
  if not texts or len(labels) < 2:
113
281
  return None
114
282
  keep = [(i, t) for i, t in enumerate(texts) if (t or "").strip()]
115
283
  if not keep:
116
284
  return None
285
+ to = timeout or _timeout()
286
+ if _transport() == "jev":
287
+ return _classify_jev(texts, labels, instructions=instructions,
288
+ keep=keep, timeout=to)
117
289
  body: dict[str, Any] = {
118
290
  "labels": [str(x) for x in labels],
119
291
  "inputs": [t[:MAX_TEXT_CHARS] for _i, t in keep],
@@ -123,8 +295,12 @@ def classify(texts: list[str], labels: tuple[str, ...] | list[str], *,
123
295
  body["instructions"] = instructions
124
296
  try:
125
297
  from http_client import HttpClient
126
- resp = HttpClient(timeout=timeout or _timeout(), max_retries=1,
127
- jitter=False).post(_endpoint(), body)
298
+ # Content-Type 同 JEV 通道:HttpClient.post 不设它,缺了会被服务端
299
+ # 当字符串解析。这条路径此前只在 403(匿名要付费 key)处止步,没走到
300
+ # body 校验,所以同一个坑没暴露——配了有效 workspace key 就会现形。
301
+ resp = HttpClient(timeout=to, max_retries=1, jitter=False).post(
302
+ _endpoint(), body,
303
+ extra_headers={"Content-Type": "application/json"})
128
304
  if resp.get("status") != 200:
129
305
  logger.debug(f"语义分类非 200: {resp.get('status')} {resp.get('error')}")
130
306
  return None