@furongjun1999/dsh-memory 0.9.6 → 0.9.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (216) hide show
  1. package/data/policy.json +38 -38
  2. package/docs/discipline/templates/spec.txt.tmpl +1 -1
  3. package/dsh/cordis.yml.example +139 -139
  4. package/lib/hooks.d.ts +19 -0
  5. package/lib/hooks.js +29 -6
  6. package/lib/index.js +11 -2
  7. package/lib/lib/hook_audit.js +38 -2
  8. package/lib/lib/prompt_safety.js +14 -3
  9. package/lib/tools.d.ts +7 -0
  10. package/lib/tools.js +22 -1
  11. package/md_cg/_guard_kit.py +121 -121
  12. package/md_cg/anchor.py +114 -114
  13. package/md_cg/audit.py +15 -2
  14. package/md_cg/chain.py +3 -2
  15. package/md_cg/config_registry_guard.py +487 -487
  16. package/md_cg/conformance.py +747 -747
  17. package/md_cg/consolidate.py +1793 -1793
  18. package/md_cg/crypto.py +794 -794
  19. package/md_cg/hyperedge.py +262 -262
  20. package/md_cg/interface_schema.py +146 -146
  21. package/md_cg/lifecycle.py +426 -426
  22. package/md_cg/mdcos.py +12 -8
  23. package/md_cg/predict.py +1103 -1103
  24. package/md_cg/protect.py +654 -654
  25. package/md_cg/provenance.py +606 -606
  26. package/md_cg/queue_health.py +130 -130
  27. package/md_cg/refine.py +613 -613
  28. package/md_cg/retrieve.py +128 -128
  29. package/md_cg/revise.py +140 -140
  30. package/md_cg/rollback.py +657 -657
  31. package/md_cg/rollback_cli.py +124 -124
  32. package/md_cg/rotate.py +346 -346
  33. package/md_cg/routing.py +490 -490
  34. package/md_cg/self_surface.py +50 -50
  35. package/md_cg/semantic/canonical.py +144 -144
  36. package/md_cg/semantic/zh_en_atoms.py +141 -141
  37. package/md_cg/sources.py +1091 -1091
  38. package/md_cg/stg.py +3 -2
  39. package/md_cg/test_audit_rotate_parity.py +279 -279
  40. package/md_cg/test_autonomy_modes.py +2013 -2013
  41. package/md_cg/test_cn_recall_grams.py +206 -206
  42. package/md_cg/test_code_kind_forbidden.py +101 -0
  43. package/md_cg/test_condaccept.py +156 -156
  44. package/md_cg/test_condgate.py +141 -141
  45. package/md_cg/test_heartbeat_ledger.py +557 -557
  46. package/md_cg/test_i50e_readside_protection.py +658 -658
  47. package/md_cg/test_issue111_gate_order.py +76 -76
  48. package/md_cg/test_issue111_scoring_contract.py +86 -86
  49. package/md_cg/test_issue121_calibration_bounds.py +293 -293
  50. package/md_cg/test_issue145_sleep_no_silent_rollback.py +167 -167
  51. package/md_cg/test_issue147_pack_id_escape.py +378 -378
  52. package/md_cg/test_issue148_merge_downgrade_leak.py +192 -192
  53. package/md_cg/test_issue149_accel_default_face.py +248 -248
  54. package/md_cg/test_issue150_empty_query_terms.py +328 -328
  55. package/md_cg/test_issue151_153_157_write_cache_index.py +151 -151
  56. package/md_cg/test_issue152_sealed_postings.py +309 -309
  57. package/md_cg/test_issue153b_index_generation_invalidation.py +142 -142
  58. package/md_cg/test_issue157b_index_remainder.py +344 -344
  59. package/md_cg/test_issue189_conclusion_bidirectional.py +223 -223
  60. package/md_cg/test_issue191_import_trust.py +382 -382
  61. package/md_cg/test_issue193_write_scope.py +378 -378
  62. package/md_cg/test_issue212_writeface_gaps.py +349 -349
  63. package/md_cg/test_issue214_reject_sensitivity.py +266 -266
  64. package/md_cg/test_issue215_condition_source_gap.py +272 -272
  65. package/md_cg/test_issue216_clash_routes.py +214 -214
  66. package/md_cg/test_issue217_slot_routes.py +248 -248
  67. package/md_cg/test_issue218_postcondition_value_blind.py +222 -222
  68. package/md_cg/test_issue258_pack_sensitivity.py +468 -468
  69. package/md_cg/test_issue265_coldverify_multiproc.py +438 -438
  70. package/md_cg/test_issue274_read_node_id_none.py +341 -341
  71. package/md_cg/test_issue275_layer_param.py +250 -250
  72. package/md_cg/test_issue277_node_id_traversal.py +384 -384
  73. package/md_cg/test_issue278_neg_separated_tension.py +271 -271
  74. package/md_cg/test_issue278_neg_window.py +294 -294
  75. package/md_cg/test_issue278_negative_layering.py +83 -83
  76. package/md_cg/test_issue284_skip_not_pass.py +291 -291
  77. package/md_cg/test_issue294_flywheel_sensitivity.py +573 -573
  78. package/md_cg/test_issue301_baseline_corpus_gate.py +245 -245
  79. package/md_cg/test_issue303_design_boundaries.py +262 -262
  80. package/md_cg/test_issue315_merge_branch_sensitivity.py +383 -383
  81. package/md_cg/test_issue319_tenant_unregistered_bypass.py +256 -256
  82. package/md_cg/test_issue321_mcp_deep_json.py +304 -304
  83. package/md_cg/test_issue322_node_write_lock.py +727 -727
  84. package/md_cg/test_issue324_write_size_fields.py +214 -214
  85. package/md_cg/test_issue325_audit_poison.py +229 -229
  86. package/md_cg/test_issue326_ingest_auto_whitelist.py +331 -331
  87. package/md_cg/test_issue326_key_perms.py +355 -355
  88. package/md_cg/test_issue326_twin_parser_parity.py +323 -323
  89. package/md_cg/test_issue326_vm_call_frame.py +569 -569
  90. package/md_cg/test_issue345_writelimit_effective_sens.py +233 -233
  91. package/md_cg/test_issue346_overwrite_sensitivity_gate.py +424 -424
  92. package/md_cg/test_issue348_insight_verify_op.py +199 -199
  93. package/md_cg/test_issue348_merge_rmw_lock.py +345 -345
  94. package/md_cg/test_issue350_casefold_collision.py +235 -235
  95. package/md_cg/test_issue351_ingest_root_fallback.py +201 -201
  96. package/md_cg/test_issue352_imgaudit.py +418 -418
  97. package/md_cg/test_issue352_pathsjson.py +537 -537
  98. package/md_cg/test_issue352_refindex.py +412 -412
  99. package/md_cg/test_issue352_rollback.py +604 -604
  100. package/md_cg/test_issue354_p2_writelimit_toctou.py +327 -327
  101. package/md_cg/test_issue354_rust_index_containment.py +571 -571
  102. package/md_cg/test_issue354_rust_wal_containment.py +606 -606
  103. package/md_cg/test_issue358_truncation_observable.py +247 -247
  104. package/md_cg/test_issue360_tidy_private_excerpt.py +195 -195
  105. package/md_cg/test_issue361_blindspot_not_silent_drop.py +494 -494
  106. package/md_cg/test_issue362_neg_condition.py +468 -468
  107. package/md_cg/test_issue363_lexical_idf.py +275 -275
  108. package/md_cg/test_issue370_readcache_assembly.py +488 -488
  109. package/md_cg/test_issue376_heartbeat_failed.py +293 -293
  110. package/md_cg/test_issue377_layer_protect_stamp.py +324 -324
  111. package/md_cg/test_issue378_protected_floor.py +310 -310
  112. package/md_cg/test_issue383_state_event_write_gate.py +462 -462
  113. package/md_cg/test_issue397_sens_eff_wiring.py +435 -435
  114. package/md_cg/test_issue398_toolface_op_parity.py +511 -511
  115. package/md_cg/test_issue399_hive_json_shape.py +317 -317
  116. package/md_cg/test_issue401_wisdom_codegen.py +295 -295
  117. package/md_cg/test_issue404_heal_receipt.py +389 -389
  118. package/md_cg/test_issue409_compact_index_completeness.py +509 -509
  119. package/md_cg/test_issue410_verify_demote_write.py +366 -366
  120. package/md_cg/test_issue411_scrub_idempotence.py +575 -575
  121. package/md_cg/test_issue412_discipline_producer_parity.py +311 -311
  122. package/md_cg/test_issue413_preimage_failclosed.py +239 -239
  123. package/md_cg/test_issue413_preimage_twin.py +339 -339
  124. package/md_cg/test_issue415_deps_gate_probe.py +357 -357
  125. package/md_cg/test_issue416_entry_key_subset.py +525 -525
  126. package/md_cg/test_issue417_twophase_merge_landed.py +423 -423
  127. package/md_cg/test_issue425_ingest_library_path.py +331 -331
  128. package/md_cg/test_issue426c_enqueue_not_silent.py +346 -346
  129. package/md_cg/test_issue426e_restore_overwrite_live.py +310 -310
  130. package/md_cg/test_issue426f_paths_gbk_failclosed.py +301 -301
  131. package/md_cg/test_issue427_serve_status_shape.py +191 -191
  132. package/md_cg/test_issue431_432_468_protect_single_point.py +322 -322
  133. package/md_cg/test_issue433_outlook_threshold_single_point.py +278 -278
  134. package/md_cg/test_issue434_435_float_gate.py +90 -90
  135. package/md_cg/test_issue439_sibling_sensitivity_inherit.py +448 -448
  136. package/md_cg/test_issue43_default_policy.py +874 -874
  137. package/md_cg/test_issue444_scrub_evidence_plaintext.py +281 -281
  138. package/md_cg/test_issue445_metacognition_ledger_gate.py +454 -454
  139. package/md_cg/test_issue446_453_403_verified_evidence.py +382 -382
  140. package/md_cg/test_issue450_sensitivity_fallback_reason.py +51 -51
  141. package/md_cg/test_issue450_writegate_unknown_sens.py +203 -203
  142. package/md_cg/test_issue451_audit_actor.py +313 -313
  143. package/md_cg/test_issue452_ingest_root_sensitivity.py +299 -299
  144. package/md_cg/test_issue463_ghost_prune_guard.py +259 -259
  145. package/md_cg/test_issue466_basis_registry.py +469 -469
  146. package/md_cg/test_issue467_identity_sensitivity_parity.py +360 -360
  147. package/md_cg/test_issue469_scrub_override_passthrough.py +126 -126
  148. package/md_cg/test_issue470_wal_event_durable.py +427 -427
  149. package/md_cg/test_issue471_identity_readable.py +136 -136
  150. package/md_cg/test_issue472_audit_chain.py +90 -90
  151. package/md_cg/test_issue472_prune_ledger.py +105 -105
  152. package/md_cg/test_issue472_rollback_degraded.py +81 -81
  153. package/md_cg/test_issue491_compose_sandbox.py +460 -460
  154. package/md_cg/test_issue498_merge_trace_stale_index.py +280 -280
  155. package/md_cg/test_issue511_scoreboard_bad_files.py +256 -256
  156. package/md_cg/test_issue521_rotate_lock_suffix.py +110 -110
  157. package/md_cg/test_issue524_orphan_noise_no_demote.py +88 -88
  158. package/md_cg/test_issue525_read_null_semantics.py +52 -52
  159. package/md_cg/test_issue547_append_jsonl_whitelist.py +114 -114
  160. package/md_cg/test_issue547_audit_durable.py +96 -96
  161. package/md_cg/test_issue557_r2_write_entries_size_gate.py +83 -83
  162. package/md_cg/test_issue559_refine_prefix.py +69 -69
  163. package/md_cg/test_issue562_retired_faces.py +150 -0
  164. package/md_cg/test_issue71_queue_backlog.py +376 -376
  165. package/md_cg/test_issue72_coldverify_controlled_drain.py +420 -420
  166. package/md_cg/test_issue82_credential_rules.py +271 -271
  167. package/md_cg/test_issue91_verifier_sandbox.py +314 -314
  168. package/md_cg/test_issue93_remember_redaction.py +282 -282
  169. package/md_cg/test_logref.py +1141 -1141
  170. package/md_cg/test_md_access_parity.py +330 -330
  171. package/md_cg/test_md_cg_no_hive_dep.py +139 -139
  172. package/md_cg/test_mutation_residue.py +119 -119
  173. package/md_cg/test_mutation_rollback.py +835 -835
  174. package/md_cg/test_n123_derive_expiry_chain.py +210 -210
  175. package/md_cg/test_n209_verify_write_face_gates.py +466 -466
  176. package/md_cg/test_n62_tenant_bind_failclosed.py +200 -200
  177. package/md_cg/test_neg_tail_honesty.py +733 -733
  178. package/md_cg/test_p21_tokens.py +294 -294
  179. package/md_cg/test_p27_docindex.py +792 -792
  180. package/md_cg/test_p28_refcheck.py +1053 -1053
  181. package/md_cg/test_p2_six_elements.py +476 -476
  182. package/md_cg/test_p33_ccg_wiring.py +300 -300
  183. package/md_cg/test_p3_legacy_closure.py +438 -438
  184. package/md_cg/test_p40_refine_worklist.py +244 -244
  185. package/md_cg/test_p4_freshness.py +781 -781
  186. package/md_cg/test_p9c_dedup_hints.py +288 -288
  187. package/md_cg/test_policy_required_ccg.py +444 -444
  188. package/md_cg/test_retr_s5.py +312 -312
  189. package/md_cg/test_s1_ast_judge.py +109 -109
  190. package/md_cg/test_state_event_op.py +521 -521
  191. package/md_cg/test_stgidx_index_parity.py +1024 -1024
  192. package/md_cg/test_sustain_bounded.py +652 -652
  193. package/md_cg/test_trust_edge_reachability.py +405 -405
  194. package/md_cg/test_w7_redlines.py +659 -659
  195. package/md_cg/test_write_return_shape.py +919 -919
  196. package/md_cg/test_writepipe.py +228 -228
  197. package/md_cg/trust.py +1114 -1114
  198. package/md_cg/twophase.py +303 -303
  199. package/md_cg/whitebox.py +536 -536
  200. package/md_cg/whitebox_kb/wisdom/compose_engine.py +1634 -1634
  201. package/md_cg/whitebox_kb/wisdom/core/condition_vm.py +318 -318
  202. package/md_cg/whitebox_kb/wisdom/core/parser.py +1082 -1082
  203. package/md_cg/whitebox_kb/wisdom/md_access.py +490 -490
  204. package/md_cg/whitebox_kb/wisdom/migrate_wisdom.py +497 -497
  205. package/md_cg/whitebox_kb/wisdom/mini_python.py +995 -995
  206. package/md_cg/whitebox_kb/wisdom/net_units.py +2900 -2900
  207. package/md_cg/whitebox_kb/wisdom/semantic_translate.py +4060 -4060
  208. package/md_cg/writepipe.py +1337 -1337
  209. package/package.json +3 -2
  210. package/scripts/workspace_index.py +328 -0
  211. package/src/hooks.ts +30 -6
  212. package/src/index.ts +14 -5
  213. package/src/lib/hook_audit.ts +282 -256
  214. package/src/lib/mdcg_client.ts +469 -469
  215. package/src/lib/prompt_safety.ts +17 -3
  216. package/src/tools.ts +23 -1
package/md_cg/refine.py CHANGED
@@ -1,614 +1,614 @@
1
- # -*- coding: utf-8 -*-
2
- """G6:node_ 结构提炼 · 小样本抽检工单与扩批闸门(只读预演,纯标准库)。
3
-
4
- 缺口(G6):`node_*`(P37 双子管线派生记忆)自带 `ccg_exempt=true`,正文只有
5
- 「功能名 / 生效条件」两行,`子功能 / 执行 / 不适用条件` 均未声明(实测 1324 条)。
6
- 若直接对全量跑归纳提炼(`consolidate.induce`),一旦**提炼口径**失准,就会把
7
- 模板化词面当成「共性条件」固化成概念节点,污染单一真相源。
8
-
9
- 裁定单条件 A:**先抽 20 条做提炼预演,人工核对提炼口径,绝不盲跑全量**。
10
-
11
- 本模块只做三件事(零 LLM、零第三方依赖,且**不改写任何节点**):
12
- 1. `plan` —— 确定性分层抽样出 20 条工单(原文摘录 / 字段缺口 / 口径声明);
13
- 2. `preview` —— 在抽检样本上跑与 `consolidate.induce` **同源的**聚类口径,
14
- 输出候选概念 + 每条共性条件的可追溯来源 + **样本内文档频率**
15
- (df 高 = 模板/通用水位线,不得当作区分性共性);
16
- 3. `apply`/`gate` —— 把抽检批次与人工裁决落 `_refine.jsonl` 留痕,并据此
17
- 计算「是否允许扩大批次」。
18
-
19
- 纪律与诚实边界:
20
- · 提炼执行仍由 `consolidate.induce` 承担(产出恒 `inferred`);本模块不实现
21
- 第二套提炼器,避免口径分叉——`preview` 复用同一套 `_jaccard/_common_terms`。
22
- · 抽检若取「20 条节点」,在库内实测**可能产不出候选**(`min_cluster=3` 时
23
- 20 条随机样本 0 簇);此时如实报 `candidates=0` 并**拒绝放行扩批**,
24
- 由 `sample_adequacy` 提示「按候选为抽检单位」或上调样本量。
25
- · 人工裁决必须由库外人类给出;本模块只做算术与留痕,**不代替裁决**。
26
- · 共性条件的去模板判据只用**词面统计**,不做语义猜测(宁可漏判,不可误判)。
27
- """
28
- from __future__ import annotations
29
-
30
- import hashlib
31
- import math
32
- import os
33
- import time
34
-
35
- from . import consolidate, nodefile
36
- from .fsutil import append_jsonl, read_jsonl
37
- from .mdcos import MdCGOS
38
-
39
- REFINE_LOG = "_refine.jsonl"
40
-
41
- #: 抽检对象 id 前缀(P37 派生记忆)
42
- # issue #559:默认**不过滤**(空前缀 = 全库抽样)。
43
- # 此前是 "node_",而本仓默认铸出的 id 是 mem_* ⇒ 默认参数下抽样池恒为 0 且不报错(静默空池);
44
- # 不把默认绑死在任何一代前缀上,对混有 node_/mem_ 两代 id 的历史库同样安全。
45
- PREFIX_DEFAULT = ""
46
- #: 裁定单条件 A 规定的抽检条数
47
- SAMPLE_N = 20
48
- #: 抽样种子:同 seed ⇒ 同样本(可复算、可复核)
49
- SAMPLE_SEED = "g6-sample-v1"
50
-
51
- MIN_JACCARD = consolidate.INDUCE_MIN_JACCARD
52
- MIN_CLUSTER = consolidate.INDUCE_MIN_CLUSTER
53
- MAX_TERMS = consolidate.INDUCE_MAX_TERMS
54
-
55
- #: 共性条件在**抽检样本**内的文档频率 ≥ 该值 → 判为模板/通用水位线,不具区分性
56
- GENERIC_DF = 0.80
57
- #: 工单正文摘录上限(超出即标 truncated;完整原文请 `cg read`)
58
- EXCERPT_MAX = 4000
59
- #: 扩批通过率闸门:人工核对「忠实」比例下限
60
- GATE_MIN_PASS_RATE = 0.90
61
-
62
- #: 样本量校准:目标产出候选数 / 有界爬坡上限(样本量非唯一杠杆,故设上限)
63
- CALIBRATE_TARGET = 20
64
- CALIBRATE_CAP = 200
65
-
66
- #: 人工核对裁决键(与工单 review_items 一一对应)
67
- VERDICT_KEYS = ("faithful", "added_info")
68
-
69
- #: 提炼口径声明——人工核对的就是这份口径,不是某一次的输出
70
- SPEC = {
71
- "method": "consolidate.induce 同源口径:bigram-jaccard 贪心聚类(确定性、零 LLM)",
72
- "min_cluster": MIN_CLUSTER,
73
- "min_jaccard": MIN_JACCARD,
74
- "fields": ["功能名", "生效条件", "子功能", "执行", "不适用条件"],
75
- "evidence": "inferred(未经验证,不得直接当事实使用)",
76
- "common_condition_source": "成员 positive_body 的词面统计(不做语义推断)",
77
- "review_items": [
78
- {"key": "faithful",
79
- "ask": "每条共性条件是否能在**全部**成员原文中原样找到"
80
- "(grounding_gap 为空即无缺口)"},
81
- {"key": "added_info",
82
- "ask": "概念是否引入了成员原文之外的词(外部知识/编造)——命中即整批否决"},
83
- {"key": "discriminating",
84
- "ask": "共性条件是否为区分性特征;df≥%.2f 的模板词不得充当共性"
85
- % GENERIC_DF},
86
- {"key": "member_consistency",
87
- "ask": "成员是否真属同一范畴(无强行拼团)"},
88
- ],
89
- "gate": {"min_pass_rate": GATE_MIN_PASS_RATE,
90
- "require_review_all": True,
91
- "fail_on_added_info": True},
92
- "honest_limits": [
93
- "本模块不执行提炼写入;扩批需 `consolidate.induce` 另开批次执行并留痕",
94
- "人工裁决由库外人类给出,本模块不代替裁决",
95
- ],
96
- }
97
-
98
- # ---- 通用工具 -------------------------------------------------------------
99
-
100
- # 生效条件:x 为 str 时返回 MdCGOS(x),否则原样返回 x;
101
- def _as_cg(x):
102
- return MdCGOS(x) if isinstance(x, str) else x
103
-
104
-
105
- # 生效条件:cg 具 "root" 属性时以该值为根、否则以 str(cg) 为根,与模块常量 REFINE_LOG 拼接返回;
106
- def _log_path(cg) -> str:
107
- root = cg.root if hasattr(cg, "root") else str(cg)
108
- return os.path.join(root, REFINE_LOG)
109
-
110
-
111
- # 生效条件:read_jsonl(_log_path(cg)) 返回假值时按空列表处理,仅保留 action 字段等于 "refine" 的记录;
112
- def _read_log(cg) -> list:
113
- return [r for r in (read_jsonl(_log_path(cg)) or [])
114
- if r.get("action") == "refine"]
115
-
116
-
117
- # 生效条件:cg 无 index 或 index["nodes"] 为假值时以空字典查找;nodes.get(nid) 为假值时返回空字典 {};
118
- def _entry(cg, nid):
119
- nodes = (getattr(cg, "index", None) or {}).get("nodes") or {}
120
- return nodes.get(nid) or {}
121
-
122
-
123
- # 生效条件:取 _entry(cg, nid) 的 path(假值回落空串)的父目录 basename;该 basename 为假值时返回 "(root)";
124
- def _family(cg, nid):
125
- """家族 = 节点所在 `cond_*` 目录名(无则 `(root)`)。"""
126
- p = str(_entry(cg, nid).get("path") or "").replace("\\", "/")
127
- d = os.path.basename(os.path.dirname(p))
128
- return d or "(root)"
129
-
130
-
131
- # 生效条件:对传入的 parts 逐项 str 后以 "|" 连接并 UTF-8 编码,返回 sha1 的 hexdigest;
132
- def _sha(*parts) -> str:
133
- h = hashlib.sha1("|".join(str(p) for p in parts).encode("utf-8"))
134
- return h.hexdigest()
135
-
136
-
137
- # 生效条件:遍历 cg.index 的 nodes,仅收 str(nid) 以 prefix 开头且条目 protected 为假值的 nid 进 ids(排序后返回),protected 为真值的计入 protected 计数;
138
- def _pool(cg, prefix) -> tuple:
139
- """抽检池:id 以 prefix 开头、非受保护节点(与 induce 的池口径一致)。"""
140
- nodes = (getattr(cg, "index", None) or {}).get("nodes") or {}
141
- ids, protected = [], 0
142
- for nid, e in list(nodes.items()):
143
- if not str(nid).startswith(prefix):
144
- continue
145
- if (e or {}).get("protected"):
146
- protected += 1
147
- continue
148
- ids.append(nid)
149
- ids.sort()
150
- return ids, protected
151
-
152
-
153
- # 生效条件:cg 的 id 池经 prefix(假值回落 PREFIX_DEFAULT)过滤后,n 为 None 用 SAMPLE_N、否则 int(n),seed 假值回落 SAMPLE_SEED;pool 为空或 n<=0 时返回 ([], meta),否则按家族分层最大余数分配并在层内按 _sha(seed, id) 升序取前 k 个返回 (picked, meta);
154
- def sample_ids(cg, n=None, seed=None, prefix=None) -> tuple:
155
- """确定性分层抽样(家族=层,家族内按 sha1(seed|id) 排序)。
156
-
157
- 分层 + 最大余数分配:主家族按规模拿到多数名额,稀有家族亦保留席位(n 足够时)。
158
- 返回 `(ids, meta)`;同 `seed` 必得同样本——抽检可复核、可复算。
159
- """
160
- n = SAMPLE_N if n is None else int(n)
161
- seed = seed or SAMPLE_SEED
162
- explicit = bool(prefix)
163
- prefix = prefix or PREFIX_DEFAULT
164
- pool, protected = _pool(cg, prefix)
165
- fallback = False
166
- if not pool and not explicit:
167
- # 调用方没显式给前缀、而默认前缀又抽不到东西:wgid 全量回落并显式标注(#559:不静默给空池)
168
- pool, protected = _pool(cg, "")
169
- fallback = True
170
- fam = {}
171
- for nid in pool:
172
- fam.setdefault(_family(cg, nid), []).append(nid)
173
- meta = {"prefix": prefix, "seed": seed, "requested": int(n), "prefix_fallback": fallback,
174
- "pool": len(pool), "skipped_protected": protected,
175
- "families": len(fam), "sampled": 0, "strata": {}}
176
- if not pool or n <= 0:
177
- return [], meta
178
- n = min(int(n), len(pool))
179
- names = sorted(fam)
180
- raw = {f: n * len(fam[f]) / float(len(pool)) for f in names}
181
- alloc = {f: int(math.floor(raw[f])) for f in names}
182
- rest = n - sum(alloc.values())
183
- for f in sorted(names, key=lambda x: (-(raw[x] - alloc[x]), x))[:rest]:
184
- alloc[f] += 1
185
- picked, strata = [], {}
186
- for f in names:
187
- k = int(alloc[f])
188
- if k <= 0:
189
- continue
190
- rank = sorted(fam[f], key=lambda x: _sha(seed, x))
191
- take = rank[:k]
192
- picked.extend(take)
193
- strata[f] = {"pool": len(fam[f]), "picked": len(take)}
194
- picked.sort()
195
- meta["sampled"] = len(picked)
196
- meta["strata"] = strata
197
- return picked, meta
198
-
199
-
200
- # 生效条件:对 cg 中 nid 对应节点(cg.get(nid) 为假值时用空字典)生成字段;content 长度大于 EXCERPT_MAX 时 body 截断且 body_truncated 为 True,否则 body 为全 content;
201
- def _item(cg, nid) -> dict:
202
- node = cg.get(nid) or {}
203
- fm = node.get("frontmatter") or {}
204
- content = node.get("content") or ""
205
- comp = nodefile.ccg_completeness(content)
206
- e = _entry(cg, nid)
207
- truncated = len(content) > EXCERPT_MAX
208
- return {
209
- "id": nid, "family": _family(cg, nid), "layer": e.get("layer"),
210
- "tags": list(e.get("tags") or []), "importance": e.get("importance"),
211
- "edges": len(fm.get("edges") or []),
212
- "ccg_exempt": bool(fm.get("ccg_exempt")),
213
- "ccg_present": comp["present"],
214
- "ccg_missing": [m for m in nodefile.CCG_MARKS if m not in comp["present"]],
215
- "comment": fm.get("comment"),
216
- "body": content[:EXCERPT_MAX], "body_len": len(content),
217
- "body_truncated": truncated,
218
- "source_sha": _sha(content)[:16],
219
- }
220
-
221
-
222
- # ---- 预演:与 induce 同源的聚类口径 ---------------------------------------
223
-
224
- # 生效条件:pool 中能取到 grams 的 nid 进入 cache;按 keys 顺序贪心,与当前 a 的 _jaccard >= float(min_jaccard) 且未分配的后续 b 并入组,组大小达到 int(min_cluster) 才成组,返回 (cache, keys, groups);
225
- def _cluster(cg, pool, min_jaccard, min_cluster) -> tuple:
226
- """贪心聚类(与 `consolidate.induce_memories` 同源)。返回 (cache, keys, groups)。"""
227
- from . import subgraph
228
- cache = {}
229
- for nid in pool:
230
- got = subgraph._node_terms_and_grams(cg, nid)
231
- if got and got.get("grams"):
232
- cache[nid] = got
233
- keys = sorted(cache)
234
- assigned, groups = set(), []
235
- for i, a in enumerate(keys):
236
- if a in assigned:
237
- continue
238
- ga = cache[a]["grams"]
239
- grp = [b for b in keys[i + 1:]
240
- if b not in assigned
241
- and consolidate._jaccard(ga, cache[b]["grams"]) >= float(min_jaccard)]
242
- if len(grp) + 1 < int(min_cluster):
243
- continue
244
- members = [a] + grp
245
- assigned.update(members)
246
- groups.append(members)
247
- return cache, keys, groups
248
-
249
-
250
- # 生效条件:terms 为空时对每个 t 返回 df[t]=0;否则对 keys 中每个 k 的 cache[k]["pos"] 统计各 term 出现次数,返回 df;
251
- def _term_df(cache, keys, terms) -> dict:
252
- """词面在抽检样本内的文档频率(不含语义推断)。"""
253
- df = {t: 0 for t in terms}
254
- if not terms:
255
- return df
256
- for k in keys:
257
- pos = set(cache[k]["pos"])
258
- for t in terms:
259
- if t in pos:
260
- df[t] += 1
261
- return df
262
-
263
-
264
- # 生效条件:x 转为 cg;min_jaccard 为 None 取 MIN_JACCARD、否则 float(min_jaccard),min_cluster 为 None 取 MIN_CLUSTER、否则 int(min_cluster);ids 为真值时 pool 为显式 ids 去重排序且 meta 标记 explicit_ids=True,否则 pool/meta 来自 sample_ids(cg, n=n, seed=seed, prefix=prefix);对 pool 聚类后,require_conditions 为真且某组 common 为空时跳过该组并累计 skipped_no_cond;返回含 candidates 与 sample_adequacy 的只读预演字典;
265
- def preview(x, ids=None, n=None, seed=None, prefix=None,
266
- min_jaccard=None, min_cluster=None, require_conditions=True) -> dict:
267
- """提炼预演(只读):抽样 → 同源聚类 → 共性条件 + 来源 + 泛化度标记。"""
268
- cg = _as_cg(x)
269
- min_j = MIN_JACCARD if min_jaccard is None else float(min_jaccard)
270
- min_c = MIN_CLUSTER if min_cluster is None else int(min_cluster)
271
- if ids:
272
- pool = sorted({str(i) for i in ids})
273
- meta = {"prefix": prefix or PREFIX_DEFAULT, "seed": seed or SAMPLE_SEED,
274
- "requested": len(pool), "pool": len(pool),
275
- "skipped_protected": 0, "families": 0, "sampled": len(pool),
276
- "strata": {}, "explicit_ids": True}
277
- else:
278
- pool, meta = sample_ids(cg, n=n, seed=seed, prefix=prefix)
279
-
280
- cache, keys, groups = _cluster(cg, pool, min_j, min_c)
281
- base = max(1, len(keys))
282
- cands, skipped_no_cond, all_terms = [], 0, set()
283
- for members in groups:
284
- common = consolidate._common_terms([cache[m]["pos"] for m in members])
285
- if require_conditions and not common:
286
- skipped_no_cond += 1
287
- continue
288
- neg = consolidate._union_terms([cache[m]["neg"] for m in members])
289
- all_terms.update(common)
290
- cands.append({"concept_id": consolidate._concept_id(members),
291
- "members": members, "size": len(members),
292
- "common_conditions": common,
293
- "non_applicable": neg})
294
-
295
- df = _term_df(cache, keys, sorted(all_terms))
296
- for c in cands:
297
- members = c["members"]
298
- src = {t: [m for m in members if t in cache[m]["pos"]]
299
- for t in c["common_conditions"]}
300
- gap = [t for t, ms in src.items() if len(ms) < len(members)]
301
- generic = [{"term": t, "df": df.get(t, 0),
302
- "ratio": round(df.get(t, 0) / float(base), 4)}
303
- for t in c["common_conditions"]
304
- if df.get(t, 0) / float(base) >= GENERIC_DF]
305
- c.update({
306
- "condition_sources": src,
307
- "grounding_gap": gap,
308
- "generic_conditions": generic,
309
- "discriminating": [t for t in c["common_conditions"]
310
- if df.get(t, 0) / float(base) < GENERIC_DF],
311
- "reason": ("%d 条记忆内容相近且共享条件「%s」→ 归纳为概念"
312
- % (len(members),
313
- "、".join(c["common_conditions"][:MAX_TERMS]) or "无")),
314
- })
315
- generic_only = sum(1 for c in cands if not c["discriminating"])
316
- adequacy = "ok" if cands else "insufficient"
317
- return {
318
- "root": cg.root, "dry_run": True, "readonly": True,
319
- "action": "refine_preview", "op": "maintain",
320
- "prefix": meta["prefix"], "seed": meta["seed"],
321
- "requested": meta["requested"], "pool": meta["pool"],
322
- "skipped_protected": meta["skipped_protected"],
323
- "families": meta["families"], "strata": meta["strata"],
324
- "sampled": meta["sampled"], "sample": pool,
325
- "indexed": len(keys), "min_cluster": min_c, "min_jaccard": min_j,
326
- "require_conditions": bool(require_conditions),
327
- "clusters": len(cands), "candidates": cands,
328
- "skipped_no_condition": skipped_no_cond,
329
- "candidates_generic_only": generic_only,
330
- "df_basis": "sampled", "generic_df_threshold": GENERIC_DF,
331
- "sample_adequacy": adequacy,
332
- "note": ("预演:未写盘、未改任何节点;候选供人工核对提炼口径"
333
- if cands else
334
- "抽检样本不足以产出候选(样本量或抽检单位需校准),不得据此扩批"),
335
- }
336
-
337
-
338
- # ---- 工单 ---------------------------------------------------------------
339
-
340
- # 生效条件:x 转为 cg;prefix 假值回落 PREFIX_DEFAULT,seed 假值回落 SAMPLE_SEED;ids 为真值时 sample 为显式 ids 去重排序且 real_pool=len(sample)、skipped=0、families=0,否则 sample/meta 来自 sample_ids(cg, n=n, seed=seed, prefix=prefix) 并取 meta["pool"]/meta["skipped_protected"]/meta["families"];items 为 sample 的 _item,preview 以 sample 为显式 ids 调用,返回只读工单;
341
- def plan(x, ids=None, n=None, seed=None, prefix=None,
342
- min_jaccard=None, min_cluster=None) -> dict:
343
- """抽检工单(只读):确定性样本 + 原文摘录 + 字段缺口 + 口径声明 + 预演。"""
344
- cg = _as_cg(x)
345
- prefix = prefix or PREFIX_DEFAULT
346
- seed = seed or SAMPLE_SEED
347
- if ids:
348
- sample = sorted({str(i) for i in ids})
349
- real_pool, skipped, families = len(sample), 0, 0
350
- else:
351
- sample, meta = sample_ids(cg, n=n, seed=seed, prefix=prefix)
352
- real_pool, skipped, families = (meta["pool"], meta["skipped_protected"],
353
- meta["families"])
354
- items = [_item(cg, nid) for nid in sample]
355
- pv = preview(cg, ids=sample, prefix=prefix,
356
- min_jaccard=min_jaccard, min_cluster=min_cluster)
357
- strata = {}
358
- for it in items:
359
- k = it["family"]
360
- s = strata.setdefault(k, {"picked": 0, "missing_fields": {},
361
- "ccg_exempt": 0})
362
- s["picked"] += 1
363
- if it["ccg_exempt"]:
364
- s["ccg_exempt"] += 1
365
- for m in it["ccg_missing"]:
366
- s["missing_fields"][m] = s["missing_fields"].get(m, 0) + 1
367
- return {
368
- "root": cg.root, "dry_run": True, "readonly": True,
369
- "action": "refine", "op": "maintain",
370
- "prefix": prefix, "seed": seed,
371
- "requested": n if n is not None else (len(sample) if ids else SAMPLE_N),
372
- "pool": real_pool, "skipped_protected": skipped,
373
- "sampled": len(items), "families": families,
374
- "sample": sample, "sample_sha": _sha(*sample)[:16],
375
- "strata": strata, "items": items,
376
- "worklist": items, "spec": SPEC,
377
- "preview": {"clusters": pv["clusters"], "candidates": pv["candidates"],
378
- "min_cluster": pv["min_cluster"],
379
- "min_jaccard": pv["min_jaccard"],
380
- "skipped_no_condition": pv["skipped_no_condition"],
381
- "candidates_generic_only": pv["candidates_generic_only"],
382
- "sample_adequacy": pv["sample_adequacy"],
383
- "df_basis": pv["df_basis"],
384
- "generic_df_threshold": pv["generic_df_threshold"]},
385
- "gate_rule": SPEC["gate"],
386
- "note": ("抽检工单(只读):供人工核对提炼口径;核对通过前不得扩批。"
387
- "行 `apply=true` 只落抽检留痕,不改任何节点。"),
388
- }
389
-
390
-
391
- # 生效条件:x 转为 cg;target 为 None 取 CALIBRATE_TARGET、否则 int(target),cap 为 None 取 CALIBRATE_CAP、否则 int(cap),初始 n=min(SAMPLE_N if n0 is None else int(n0), cap);循环以 n 调 preview(cg, n=n, seed=seed, prefix=prefix, min_jaccard=min_jaccard, min_cluster=min_cluster) 并记录 step,直到 pv["clusters"] >= target 则 chosen=step 并 break,或 n >= min(cap, pool or cap) 则 break;chosen 为 None 时 blocked=True 并按 steps 汇总 recommend,否则 recommend 基于 chosen;返回只读校准报告;
392
- def calibrate(x, target=None, cap=None, n0=None, seed=None, prefix=None,
393
- min_jaccard=None, min_cluster=None) -> dict:
394
- """样本量校准(只读):20 条起步有界爬坡,直到产出 `target` 个候选或触顶。
395
-
396
- 目的是回答「20 条工单为什么交不出候选」,并给出**有界**的替代样本量;
397
- 若爬坡到顶仍无候选、或候选共性条件全为高 df 模板词,则如实报
398
- `blocked` 并指出真正的瓶颈是**提炼口径**而非样本量——不得据此扩批。
399
- """
400
- cg = _as_cg(x)
401
- target = CALIBRATE_TARGET if target is None else int(target)
402
- cap = CALIBRATE_CAP if cap is None else int(cap)
403
- n = min(SAMPLE_N if n0 is None else int(n0), cap)
404
- steps, chosen, pool = [], None, None
405
- while True:
406
- pv = preview(cg, n=n, seed=seed, prefix=prefix,
407
- min_jaccard=min_jaccard, min_cluster=min_cluster)
408
- pool = pv["pool"]
409
- step = {"n": n, "sampled": pv["sampled"], "clusters": pv["clusters"],
410
- "candidates_generic_only": pv["candidates_generic_only"],
411
- "discriminating_terms": sorted(
412
- {t for c in pv["candidates"] for t in c["discriminating"]})[:12]}
413
- steps.append(step)
414
- if pv["clusters"] >= target:
415
- chosen = step
416
- break
417
- top = min(cap, pool or cap)
418
- if n >= top:
419
- break
420
- n = min(n * 2, top)
421
- blocked = chosen is None
422
- saw_clusters = any(s["clusters"] for s in steps)
423
- total_c = sum(s["clusters"] for s in steps)
424
- gen_only = sum(s["candidates_generic_only"] for s in steps)
425
- if not blocked:
426
- rec = ("抽检样本量校准为 n=%d(产出 %d 个候选);仍须人工核对提炼口径,"
427
- "核对通过前不得扩批" % (chosen["n"], chosen["clusters"]))
428
- else:
429
- parts = ["爬坡至 cap=%d 仍只产出 %d 个候选(< target=%d),样本量不足以"
430
- "直接支撑抽检" % (cap, steps[-1]["clusters"], target)]
431
- if saw_clusters:
432
- if gen_only >= total_c:
433
- parts.append("且已产出候选的共性条件**全部**为高 df 模板词(无区分性)")
434
- else:
435
- parts.append("且候选中 %d/%d 的共性条件全为高 df 模板词,"
436
- "其余『区分性』项形如整行条件文本(含时间戳),"
437
- "属归一化缺失导致的伪区分性" % (gen_only, total_c))
438
- parts.append("须先修正提炼口径(剔除模板词 + 归一化/裁剪整行文本)"
439
- "或以「候选」为抽检单位重新校准;当前状态不得扩批")
440
- rec = ";".join(parts)
441
- return {"ok": True, "action": "refine_calibrate", "op": "maintain",
442
- "root": cg.root, "dry_run": True, "readonly": True,
443
- "target": target, "cap": cap, "pool": pool,
444
- "steps": steps, "chosen": chosen, "blocked": blocked,
445
- "sample_adequacy": "ok" if not blocked else "insufficient",
446
- "recommend": rec, "note": "只读校准:未写盘、未改任何节点。"}
447
-
448
-
449
- # ---- 留痕与扩批闸门 ------------------------------------------------------
450
-
451
- # 生效条件:candidates 中取 concept_id 为真值的 id 列表;verdicts 中为 dict 且 concept_id 非空且首次出现的条目计入 reviewed 并累加其中 faithful/added_info 的真值计数;ids 非空时按 reviewed < len(ids) → awaiting_human_review、added 非零 → added_info_detected、faithful_rate < GATE_MIN_PASS_RATE → faithful_rate_below_gate、否则 ok 且 expand_allowed=True;ids 为空时 reason="no_candidates" 且 expand_allowed=False;
452
- def _verdict_stats(verdicts, candidates) -> dict:
453
- ids = [c.get("concept_id") for c in candidates if c.get("concept_id")]
454
- seen, faithful, added = set(), 0, 0
455
- for v in verdicts or []:
456
- if not isinstance(v, dict):
457
- continue
458
- cid = v.get("concept_id")
459
- if not cid or cid in seen:
460
- continue
461
- seen.add(cid)
462
- faithful += 1 if v.get("faithful") else 0
463
- added += 1 if v.get("added_info") else 0
464
- reviewed = len(seen)
465
- rate = (faithful / float(reviewed)) if reviewed else 0.0
466
- expand, reason = False, "no_candidates"
467
- if ids:
468
- if reviewed < len(ids):
469
- reason = "awaiting_human_review"
470
- elif added:
471
- reason = "added_info_detected"
472
- elif rate < GATE_MIN_PASS_RATE:
473
- reason = "faithful_rate_below_gate"
474
- else:
475
- reason = "ok"
476
- expand = True
477
- return {"candidates": len(ids), "reviewed": reviewed,
478
- "faithful": faithful, "added_info": added,
479
- "faithful_rate": round(rate, 4),
480
- "min_pass_rate": GATE_MIN_PASS_RATE,
481
- "expand_allowed": expand, "reason": reason}
482
-
483
-
484
- # 生效条件:x 转为 cg;以 ids/n/seed/prefix/min_jaccard/min_cluster 调 plan 得到 p,batch 假值(含 None/空串)回落当前时间字符串;从 p["preview"]["candidates"] 与 verdicts 经 _verdict_stats 得 stats;把含 batch/actor/prefix/seed/sample/verdicts/gate/note 的 record 追加写入 _log_path(cg),返回 rep;
485
- def apply(x, batch=None, actor=None, verdicts=None, note=None,
486
- ids=None, n=None, seed=None, prefix=None,
487
- min_jaccard=None, min_cluster=None) -> dict:
488
- """落抽检批次 + 人工裁决到 `_refine.jsonl`。**不改写任何节点**。"""
489
- cg = _as_cg(x)
490
- p = plan(cg, ids=ids, n=n, seed=seed, prefix=prefix,
491
- min_jaccard=min_jaccard, min_cluster=min_cluster)
492
- batch = batch or time.strftime("%Y%m%d-%H%M%S")
493
- cands = p["preview"]["candidates"]
494
- stats = _verdict_stats(verdicts, cands)
495
- record = {
496
- "t": time.time(), "action": "refine", "batch": batch, "actor": actor,
497
- "prefix": p["prefix"], "seed": p["seed"], "requested": p["requested"],
498
- "pool": p["pool"], "sampled": p["sampled"],
499
- "sample": p["sample"], "sample_sha": p["sample_sha"],
500
- "sample_adequacy": p["preview"]["sample_adequacy"],
501
- "min_cluster": p["preview"].get("min_cluster"),
502
- "candidates": [{"concept_id": c["concept_id"], "members": c["members"],
503
- "common_conditions": c["common_conditions"],
504
- "grounding_gap": c["grounding_gap"],
505
- "discriminating": c["discriminating"]}
506
- for c in cands],
507
- "verdicts": list(verdicts or []), "gate": stats, "note": note,
508
- }
509
- append_jsonl(_log_path(cg), record)
510
- rep = {"ok": True, "action": "refine", "op": "maintain", "batch": batch,
511
- "root": cg.root, "sampled": record["sampled"],
512
- "candidates": stats["candidates"], "reviewed": stats["reviewed"],
513
- "gate": stats, "log": REFINE_LOG,
514
- "note": ("抽检留痕已落盘(未改任何节点);"
515
- + ("闸门放行扩批(仍需另开 induce 批次并留痕)"
516
- if stats["expand_allowed"] else
517
- "闸门未放行:" + stats["reason"]))}
518
- return rep
519
-
520
-
521
- # 生效条件:x 转为 cg;batch 为真值时只保留 _read_log(cg) 中 batch 字段相等的记录,过滤后为空时返回 batches=0、expand_allowed=False、reason="no_batch";否则取最后一条记录,以其 verdicts 与 candidates(假值转空列表)经 _verdict_stats 复算并返回 expand_allowed/reason 等;
522
- def gate(x, batch=None) -> dict:
523
- """扩批闸门:读留痕复算通过率(不写盘)。"""
524
- cg = _as_cg(x)
525
- recs = _read_log(cg)
526
- if batch:
527
- recs = [r for r in recs if r.get("batch") == batch]
528
- if not recs:
529
- return {"ok": True, "action": "refine_gate", "op": "maintain",
530
- "root": cg.root, "batch": batch, "batches": 0,
531
- "expand_allowed": False, "reason": "no_batch"}
532
- rec = recs[-1]
533
- stats = _verdict_stats(rec.get("verdicts"), rec.get("candidates") or [])
534
- return {"ok": True, "action": "refine_gate", "op": "maintain",
535
- "root": cg.root, "batch": rec.get("batch"),
536
- "batches": len({r.get("batch") for r in recs}),
537
- "sampled": rec.get("sampled"),
538
- "sample_adequacy": rec.get("sample_adequacy"),
539
- "gate": stats, "expand_allowed": stats["expand_allowed"],
540
- "reason": stats["reason"],
541
- "note": ("闸门放行:可另开 induce 批次扩大提炼规模并逐批留痕"
542
- if stats["expand_allowed"] else
543
- "闸门未放行,禁止扩大批次;不得盲跑全量。")}
544
-
545
-
546
- # 生效条件:batch 为真值时只保留该批次记录,total 先记为过滤后条数;limit 非 None 且 >=0 时以 recs[-int(limit):] 截尾(limit=0 因 [-0:] 等价 [0:] 仍返回全部记录),limit 为 None 或负数时不截断;
547
- def history(x, limit=100, batch=None) -> dict:
548
- cg = _as_cg(x)
549
- recs = _read_log(cg)
550
- if batch:
551
- recs = [r for r in recs if r.get("batch") == batch]
552
- total = len(recs)
553
- if limit is not None and limit >= 0:
554
- recs = recs[-int(limit):]
555
- return {"root": cg.root, "total": total, "returned": len(recs),
556
- "action": "refine", "records": recs}
557
-
558
-
559
- # ---- CLI(真实库抽检/预演用;MCP 侧走 maintain action) -------------------
560
-
561
- # 生效条件:解析 argv(None 时由 argparse 取 sys.argv)得到 root/n/seed/prefix/min-jaccard/min-cluster/report 等参数;a.calibrate 为真时调 calibrate 并打印 target/cap/pool/blocked/sample_adequacy/recommend 及每步摘要;否则 a.preview 为真选 preview、为假选 plan,调用后打印 thin 摘要与候选并返回 0;
562
- def _main(argv=None) -> int:
563
- import argparse
564
- import json
565
- ap = argparse.ArgumentParser(description="G6 提炼抽检(默认只预演,不写盘)")
566
- ap.add_argument("root", help="认知图库根")
567
- ap.add_argument("--n", type=int, default=SAMPLE_N, help="抽检条数(默认 20)")
568
- ap.add_argument("--seed", default=SAMPLE_SEED, help="抽样种子(同 seed 同样本)")
569
- ap.add_argument("--prefix", default=PREFIX_DEFAULT, help="id 前缀(默认空 = 不过滤,全库抽样;给出则只抽以该前缀开头的 id)")
570
- ap.add_argument("--min-jaccard", type=float, default=None)
571
- ap.add_argument("--min-cluster", type=int, default=None)
572
- ap.add_argument("--preview", action="store_true", help="只出预演(不出工单正文)")
573
- ap.add_argument("--calibrate", action="store_true",
574
- help="只做样本量校准(20 条起步有界爬坡)")
575
- ap.add_argument("--report", default=None, help="把汇总 JSON 另存一份")
576
- a = ap.parse_args(argv)
577
- if a.calibrate:
578
- rep = calibrate(a.root, seed=a.seed, prefix=a.prefix,
579
- min_jaccard=a.min_jaccard, min_cluster=a.min_cluster)
580
- if a.report:
581
- with open(a.report, "w", encoding="utf-8") as f:
582
- json.dump(rep, f, ensure_ascii=False, indent=2)
583
- print(json.dumps({k: rep[k] for k in
584
- ("target", "cap", "pool", "blocked",
585
- "sample_adequacy", "recommend")},
586
- ensure_ascii=False, indent=2))
587
- for s in rep["steps"]:
588
- print(" - n=%4d clusters=%2d generic_only=%2d discriminating=%s"
589
- % (s["n"], s["clusters"], s["candidates_generic_only"],
590
- s["discriminating_terms"][:6]))
591
- return 0
592
- fn = preview if a.preview else plan
593
- rep = fn(a.root, n=a.n, seed=a.seed, prefix=a.prefix,
594
- min_jaccard=a.min_jaccard, min_cluster=a.min_cluster)
595
- if a.report:
596
- with open(a.report, "w", encoding="utf-8") as f:
597
- json.dump(rep, f, ensure_ascii=False, indent=2)
598
- pv = rep.get("preview") if "preview" in rep else rep
599
- thin = {k: rep.get(k) for k in ("action", "prefix", "seed", "requested",
600
- "pool", "sampled", "families")}
601
- thin.update({k: pv.get(k) for k in ("clusters", "sample_adequacy",
602
- "candidates_generic_only")})
603
- print(json.dumps(thin, ensure_ascii=False, indent=2))
604
- for c in (pv.get("candidates") or []):
605
- print(" - %s size=%d common=%s generic=%s gap=%s"
606
- % (c["concept_id"], c["size"], c["common_conditions"][:4],
607
- [g["term"] for g in c.get("generic_conditions", [])],
608
- c.get("grounding_gap")))
609
- return 0
610
-
611
-
612
- if __name__ == "__main__":
613
- import sys
1
+ # -*- coding: utf-8 -*-
2
+ """G6:node_ 结构提炼 · 小样本抽检工单与扩批闸门(只读预演,纯标准库)。
3
+
4
+ 缺口(G6):`node_*`(P37 双子管线派生记忆)自带 `ccg_exempt=true`,正文只有
5
+ 「功能名 / 生效条件」两行,`子功能 / 执行 / 不适用条件` 均未声明(实测 1324 条)。
6
+ 若直接对全量跑归纳提炼(`consolidate.induce`),一旦**提炼口径**失准,就会把
7
+ 模板化词面当成「共性条件」固化成概念节点,污染单一真相源。
8
+
9
+ 裁定单条件 A:**先抽 20 条做提炼预演,人工核对提炼口径,绝不盲跑全量**。
10
+
11
+ 本模块只做三件事(零 LLM、零第三方依赖,且**不改写任何节点**):
12
+ 1. `plan` —— 确定性分层抽样出 20 条工单(原文摘录 / 字段缺口 / 口径声明);
13
+ 2. `preview` —— 在抽检样本上跑与 `consolidate.induce` **同源的**聚类口径,
14
+ 输出候选概念 + 每条共性条件的可追溯来源 + **样本内文档频率**
15
+ (df 高 = 模板/通用水位线,不得当作区分性共性);
16
+ 3. `apply`/`gate` —— 把抽检批次与人工裁决落 `_refine.jsonl` 留痕,并据此
17
+ 计算「是否允许扩大批次」。
18
+
19
+ 纪律与诚实边界:
20
+ · 提炼执行仍由 `consolidate.induce` 承担(产出恒 `inferred`);本模块不实现
21
+ 第二套提炼器,避免口径分叉——`preview` 复用同一套 `_jaccard/_common_terms`。
22
+ · 抽检若取「20 条节点」,在库内实测**可能产不出候选**(`min_cluster=3` 时
23
+ 20 条随机样本 0 簇);此时如实报 `candidates=0` 并**拒绝放行扩批**,
24
+ 由 `sample_adequacy` 提示「按候选为抽检单位」或上调样本量。
25
+ · 人工裁决必须由库外人类给出;本模块只做算术与留痕,**不代替裁决**。
26
+ · 共性条件的去模板判据只用**词面统计**,不做语义猜测(宁可漏判,不可误判)。
27
+ """
28
+ from __future__ import annotations
29
+
30
+ import hashlib
31
+ import math
32
+ import os
33
+ import time
34
+
35
+ from . import consolidate, nodefile
36
+ from .fsutil import append_jsonl, read_jsonl
37
+ from .mdcos import MdCGOS
38
+
39
+ REFINE_LOG = "_refine.jsonl"
40
+
41
+ #: 抽检对象 id 前缀(P37 派生记忆)
42
+ # issue #559:默认**不过滤**(空前缀 = 全库抽样)。
43
+ # 此前是 "node_",而本仓默认铸出的 id 是 mem_* ⇒ 默认参数下抽样池恒为 0 且不报错(静默空池);
44
+ # 不把默认绑死在任何一代前缀上,对混有 node_/mem_ 两代 id 的历史库同样安全。
45
+ PREFIX_DEFAULT = ""
46
+ #: 裁定单条件 A 规定的抽检条数
47
+ SAMPLE_N = 20
48
+ #: 抽样种子:同 seed ⇒ 同样本(可复算、可复核)
49
+ SAMPLE_SEED = "g6-sample-v1"
50
+
51
+ MIN_JACCARD = consolidate.INDUCE_MIN_JACCARD
52
+ MIN_CLUSTER = consolidate.INDUCE_MIN_CLUSTER
53
+ MAX_TERMS = consolidate.INDUCE_MAX_TERMS
54
+
55
+ #: 共性条件在**抽检样本**内的文档频率 ≥ 该值 → 判为模板/通用水位线,不具区分性
56
+ GENERIC_DF = 0.80
57
+ #: 工单正文摘录上限(超出即标 truncated;完整原文请 `cg read`)
58
+ EXCERPT_MAX = 4000
59
+ #: 扩批通过率闸门:人工核对「忠实」比例下限
60
+ GATE_MIN_PASS_RATE = 0.90
61
+
62
+ #: 样本量校准:目标产出候选数 / 有界爬坡上限(样本量非唯一杠杆,故设上限)
63
+ CALIBRATE_TARGET = 20
64
+ CALIBRATE_CAP = 200
65
+
66
+ #: 人工核对裁决键(与工单 review_items 一一对应)
67
+ VERDICT_KEYS = ("faithful", "added_info")
68
+
69
+ #: 提炼口径声明——人工核对的就是这份口径,不是某一次的输出
70
+ SPEC = {
71
+ "method": "consolidate.induce 同源口径:bigram-jaccard 贪心聚类(确定性、零 LLM)",
72
+ "min_cluster": MIN_CLUSTER,
73
+ "min_jaccard": MIN_JACCARD,
74
+ "fields": ["功能名", "生效条件", "子功能", "执行", "不适用条件"],
75
+ "evidence": "inferred(未经验证,不得直接当事实使用)",
76
+ "common_condition_source": "成员 positive_body 的词面统计(不做语义推断)",
77
+ "review_items": [
78
+ {"key": "faithful",
79
+ "ask": "每条共性条件是否能在**全部**成员原文中原样找到"
80
+ "(grounding_gap 为空即无缺口)"},
81
+ {"key": "added_info",
82
+ "ask": "概念是否引入了成员原文之外的词(外部知识/编造)——命中即整批否决"},
83
+ {"key": "discriminating",
84
+ "ask": "共性条件是否为区分性特征;df≥%.2f 的模板词不得充当共性"
85
+ % GENERIC_DF},
86
+ {"key": "member_consistency",
87
+ "ask": "成员是否真属同一范畴(无强行拼团)"},
88
+ ],
89
+ "gate": {"min_pass_rate": GATE_MIN_PASS_RATE,
90
+ "require_review_all": True,
91
+ "fail_on_added_info": True},
92
+ "honest_limits": [
93
+ "本模块不执行提炼写入;扩批需 `consolidate.induce` 另开批次执行并留痕",
94
+ "人工裁决由库外人类给出,本模块不代替裁决",
95
+ ],
96
+ }
97
+
98
+ # ---- 通用工具 -------------------------------------------------------------
99
+
100
+ # 生效条件:x 为 str 时返回 MdCGOS(x),否则原样返回 x;
101
+ def _as_cg(x):
102
+ return MdCGOS(x) if isinstance(x, str) else x
103
+
104
+
105
+ # 生效条件:cg 具 "root" 属性时以该值为根、否则以 str(cg) 为根,与模块常量 REFINE_LOG 拼接返回;
106
+ def _log_path(cg) -> str:
107
+ root = cg.root if hasattr(cg, "root") else str(cg)
108
+ return os.path.join(root, REFINE_LOG)
109
+
110
+
111
+ # 生效条件:read_jsonl(_log_path(cg)) 返回假值时按空列表处理,仅保留 action 字段等于 "refine" 的记录;
112
+ def _read_log(cg) -> list:
113
+ return [r for r in (read_jsonl(_log_path(cg)) or [])
114
+ if r.get("action") == "refine"]
115
+
116
+
117
+ # 生效条件:cg 无 index 或 index["nodes"] 为假值时以空字典查找;nodes.get(nid) 为假值时返回空字典 {};
118
+ def _entry(cg, nid):
119
+ nodes = (getattr(cg, "index", None) or {}).get("nodes") or {}
120
+ return nodes.get(nid) or {}
121
+
122
+
123
+ # 生效条件:取 _entry(cg, nid) 的 path(假值回落空串)的父目录 basename;该 basename 为假值时返回 "(root)";
124
+ def _family(cg, nid):
125
+ """家族 = 节点所在 `cond_*` 目录名(无则 `(root)`)。"""
126
+ p = str(_entry(cg, nid).get("path") or "").replace("\\", "/")
127
+ d = os.path.basename(os.path.dirname(p))
128
+ return d or "(root)"
129
+
130
+
131
+ # 生效条件:对传入的 parts 逐项 str 后以 "|" 连接并 UTF-8 编码,返回 sha1 的 hexdigest;
132
+ def _sha(*parts) -> str:
133
+ h = hashlib.sha1("|".join(str(p) for p in parts).encode("utf-8"))
134
+ return h.hexdigest()
135
+
136
+
137
+ # 生效条件:遍历 cg.index 的 nodes,仅收 str(nid) 以 prefix 开头且条目 protected 为假值的 nid 进 ids(排序后返回),protected 为真值的计入 protected 计数;
138
+ def _pool(cg, prefix) -> tuple:
139
+ """抽检池:id 以 prefix 开头、非受保护节点(与 induce 的池口径一致)。"""
140
+ nodes = (getattr(cg, "index", None) or {}).get("nodes") or {}
141
+ ids, protected = [], 0
142
+ for nid, e in list(nodes.items()):
143
+ if not str(nid).startswith(prefix):
144
+ continue
145
+ if (e or {}).get("protected"):
146
+ protected += 1
147
+ continue
148
+ ids.append(nid)
149
+ ids.sort()
150
+ return ids, protected
151
+
152
+
153
+ # 生效条件:cg 的 id 池经 prefix(假值回落 PREFIX_DEFAULT)过滤后,n 为 None 用 SAMPLE_N、否则 int(n),seed 假值回落 SAMPLE_SEED;pool 为空或 n<=0 时返回 ([], meta),否则按家族分层最大余数分配并在层内按 _sha(seed, id) 升序取前 k 个返回 (picked, meta);
154
+ def sample_ids(cg, n=None, seed=None, prefix=None) -> tuple:
155
+ """确定性分层抽样(家族=层,家族内按 sha1(seed|id) 排序)。
156
+
157
+ 分层 + 最大余数分配:主家族按规模拿到多数名额,稀有家族亦保留席位(n 足够时)。
158
+ 返回 `(ids, meta)`;同 `seed` 必得同样本——抽检可复核、可复算。
159
+ """
160
+ n = SAMPLE_N if n is None else int(n)
161
+ seed = seed or SAMPLE_SEED
162
+ explicit = bool(prefix)
163
+ prefix = prefix or PREFIX_DEFAULT
164
+ pool, protected = _pool(cg, prefix)
165
+ fallback = False
166
+ if not pool and not explicit:
167
+ # 调用方没显式给前缀、而默认前缀又抽不到东西:wgid 全量回落并显式标注(#559:不静默给空池)
168
+ pool, protected = _pool(cg, "")
169
+ fallback = True
170
+ fam = {}
171
+ for nid in pool:
172
+ fam.setdefault(_family(cg, nid), []).append(nid)
173
+ meta = {"prefix": prefix, "seed": seed, "requested": int(n), "prefix_fallback": fallback,
174
+ "pool": len(pool), "skipped_protected": protected,
175
+ "families": len(fam), "sampled": 0, "strata": {}}
176
+ if not pool or n <= 0:
177
+ return [], meta
178
+ n = min(int(n), len(pool))
179
+ names = sorted(fam)
180
+ raw = {f: n * len(fam[f]) / float(len(pool)) for f in names}
181
+ alloc = {f: int(math.floor(raw[f])) for f in names}
182
+ rest = n - sum(alloc.values())
183
+ for f in sorted(names, key=lambda x: (-(raw[x] - alloc[x]), x))[:rest]:
184
+ alloc[f] += 1
185
+ picked, strata = [], {}
186
+ for f in names:
187
+ k = int(alloc[f])
188
+ if k <= 0:
189
+ continue
190
+ rank = sorted(fam[f], key=lambda x: _sha(seed, x))
191
+ take = rank[:k]
192
+ picked.extend(take)
193
+ strata[f] = {"pool": len(fam[f]), "picked": len(take)}
194
+ picked.sort()
195
+ meta["sampled"] = len(picked)
196
+ meta["strata"] = strata
197
+ return picked, meta
198
+
199
+
200
+ # 生效条件:对 cg 中 nid 对应节点(cg.get(nid) 为假值时用空字典)生成字段;content 长度大于 EXCERPT_MAX 时 body 截断且 body_truncated 为 True,否则 body 为全 content;
201
+ def _item(cg, nid) -> dict:
202
+ node = cg.get(nid) or {}
203
+ fm = node.get("frontmatter") or {}
204
+ content = node.get("content") or ""
205
+ comp = nodefile.ccg_completeness(content)
206
+ e = _entry(cg, nid)
207
+ truncated = len(content) > EXCERPT_MAX
208
+ return {
209
+ "id": nid, "family": _family(cg, nid), "layer": e.get("layer"),
210
+ "tags": list(e.get("tags") or []), "importance": e.get("importance"),
211
+ "edges": len(fm.get("edges") or []),
212
+ "ccg_exempt": bool(fm.get("ccg_exempt")),
213
+ "ccg_present": comp["present"],
214
+ "ccg_missing": [m for m in nodefile.CCG_MARKS if m not in comp["present"]],
215
+ "comment": fm.get("comment"),
216
+ "body": content[:EXCERPT_MAX], "body_len": len(content),
217
+ "body_truncated": truncated,
218
+ "source_sha": _sha(content)[:16],
219
+ }
220
+
221
+
222
+ # ---- 预演:与 induce 同源的聚类口径 ---------------------------------------
223
+
224
+ # 生效条件:pool 中能取到 grams 的 nid 进入 cache;按 keys 顺序贪心,与当前 a 的 _jaccard >= float(min_jaccard) 且未分配的后续 b 并入组,组大小达到 int(min_cluster) 才成组,返回 (cache, keys, groups);
225
+ def _cluster(cg, pool, min_jaccard, min_cluster) -> tuple:
226
+ """贪心聚类(与 `consolidate.induce_memories` 同源)。返回 (cache, keys, groups)。"""
227
+ from . import subgraph
228
+ cache = {}
229
+ for nid in pool:
230
+ got = subgraph._node_terms_and_grams(cg, nid)
231
+ if got and got.get("grams"):
232
+ cache[nid] = got
233
+ keys = sorted(cache)
234
+ assigned, groups = set(), []
235
+ for i, a in enumerate(keys):
236
+ if a in assigned:
237
+ continue
238
+ ga = cache[a]["grams"]
239
+ grp = [b for b in keys[i + 1:]
240
+ if b not in assigned
241
+ and consolidate._jaccard(ga, cache[b]["grams"]) >= float(min_jaccard)]
242
+ if len(grp) + 1 < int(min_cluster):
243
+ continue
244
+ members = [a] + grp
245
+ assigned.update(members)
246
+ groups.append(members)
247
+ return cache, keys, groups
248
+
249
+
250
+ # 生效条件:terms 为空时对每个 t 返回 df[t]=0;否则对 keys 中每个 k 的 cache[k]["pos"] 统计各 term 出现次数,返回 df;
251
+ def _term_df(cache, keys, terms) -> dict:
252
+ """词面在抽检样本内的文档频率(不含语义推断)。"""
253
+ df = {t: 0 for t in terms}
254
+ if not terms:
255
+ return df
256
+ for k in keys:
257
+ pos = set(cache[k]["pos"])
258
+ for t in terms:
259
+ if t in pos:
260
+ df[t] += 1
261
+ return df
262
+
263
+
264
+ # 生效条件:x 转为 cg;min_jaccard 为 None 取 MIN_JACCARD、否则 float(min_jaccard),min_cluster 为 None 取 MIN_CLUSTER、否则 int(min_cluster);ids 为真值时 pool 为显式 ids 去重排序且 meta 标记 explicit_ids=True,否则 pool/meta 来自 sample_ids(cg, n=n, seed=seed, prefix=prefix);对 pool 聚类后,require_conditions 为真且某组 common 为空时跳过该组并累计 skipped_no_cond;返回含 candidates 与 sample_adequacy 的只读预演字典;
265
+ def preview(x, ids=None, n=None, seed=None, prefix=None,
266
+ min_jaccard=None, min_cluster=None, require_conditions=True) -> dict:
267
+ """提炼预演(只读):抽样 → 同源聚类 → 共性条件 + 来源 + 泛化度标记。"""
268
+ cg = _as_cg(x)
269
+ min_j = MIN_JACCARD if min_jaccard is None else float(min_jaccard)
270
+ min_c = MIN_CLUSTER if min_cluster is None else int(min_cluster)
271
+ if ids:
272
+ pool = sorted({str(i) for i in ids})
273
+ meta = {"prefix": prefix or PREFIX_DEFAULT, "seed": seed or SAMPLE_SEED,
274
+ "requested": len(pool), "pool": len(pool),
275
+ "skipped_protected": 0, "families": 0, "sampled": len(pool),
276
+ "strata": {}, "explicit_ids": True}
277
+ else:
278
+ pool, meta = sample_ids(cg, n=n, seed=seed, prefix=prefix)
279
+
280
+ cache, keys, groups = _cluster(cg, pool, min_j, min_c)
281
+ base = max(1, len(keys))
282
+ cands, skipped_no_cond, all_terms = [], 0, set()
283
+ for members in groups:
284
+ common = consolidate._common_terms([cache[m]["pos"] for m in members])
285
+ if require_conditions and not common:
286
+ skipped_no_cond += 1
287
+ continue
288
+ neg = consolidate._union_terms([cache[m]["neg"] for m in members])
289
+ all_terms.update(common)
290
+ cands.append({"concept_id": consolidate._concept_id(members),
291
+ "members": members, "size": len(members),
292
+ "common_conditions": common,
293
+ "non_applicable": neg})
294
+
295
+ df = _term_df(cache, keys, sorted(all_terms))
296
+ for c in cands:
297
+ members = c["members"]
298
+ src = {t: [m for m in members if t in cache[m]["pos"]]
299
+ for t in c["common_conditions"]}
300
+ gap = [t for t, ms in src.items() if len(ms) < len(members)]
301
+ generic = [{"term": t, "df": df.get(t, 0),
302
+ "ratio": round(df.get(t, 0) / float(base), 4)}
303
+ for t in c["common_conditions"]
304
+ if df.get(t, 0) / float(base) >= GENERIC_DF]
305
+ c.update({
306
+ "condition_sources": src,
307
+ "grounding_gap": gap,
308
+ "generic_conditions": generic,
309
+ "discriminating": [t for t in c["common_conditions"]
310
+ if df.get(t, 0) / float(base) < GENERIC_DF],
311
+ "reason": ("%d 条记忆内容相近且共享条件「%s」→ 归纳为概念"
312
+ % (len(members),
313
+ "、".join(c["common_conditions"][:MAX_TERMS]) or "无")),
314
+ })
315
+ generic_only = sum(1 for c in cands if not c["discriminating"])
316
+ adequacy = "ok" if cands else "insufficient"
317
+ return {
318
+ "root": cg.root, "dry_run": True, "readonly": True,
319
+ "action": "refine_preview", "op": "maintain",
320
+ "prefix": meta["prefix"], "seed": meta["seed"],
321
+ "requested": meta["requested"], "pool": meta["pool"],
322
+ "skipped_protected": meta["skipped_protected"],
323
+ "families": meta["families"], "strata": meta["strata"],
324
+ "sampled": meta["sampled"], "sample": pool,
325
+ "indexed": len(keys), "min_cluster": min_c, "min_jaccard": min_j,
326
+ "require_conditions": bool(require_conditions),
327
+ "clusters": len(cands), "candidates": cands,
328
+ "skipped_no_condition": skipped_no_cond,
329
+ "candidates_generic_only": generic_only,
330
+ "df_basis": "sampled", "generic_df_threshold": GENERIC_DF,
331
+ "sample_adequacy": adequacy,
332
+ "note": ("预演:未写盘、未改任何节点;候选供人工核对提炼口径"
333
+ if cands else
334
+ "抽检样本不足以产出候选(样本量或抽检单位需校准),不得据此扩批"),
335
+ }
336
+
337
+
338
+ # ---- 工单 ---------------------------------------------------------------
339
+
340
+ # 生效条件:x 转为 cg;prefix 假值回落 PREFIX_DEFAULT,seed 假值回落 SAMPLE_SEED;ids 为真值时 sample 为显式 ids 去重排序且 real_pool=len(sample)、skipped=0、families=0,否则 sample/meta 来自 sample_ids(cg, n=n, seed=seed, prefix=prefix) 并取 meta["pool"]/meta["skipped_protected"]/meta["families"];items 为 sample 的 _item,preview 以 sample 为显式 ids 调用,返回只读工单;
341
+ def plan(x, ids=None, n=None, seed=None, prefix=None,
342
+ min_jaccard=None, min_cluster=None) -> dict:
343
+ """抽检工单(只读):确定性样本 + 原文摘录 + 字段缺口 + 口径声明 + 预演。"""
344
+ cg = _as_cg(x)
345
+ prefix = prefix or PREFIX_DEFAULT
346
+ seed = seed or SAMPLE_SEED
347
+ if ids:
348
+ sample = sorted({str(i) for i in ids})
349
+ real_pool, skipped, families = len(sample), 0, 0
350
+ else:
351
+ sample, meta = sample_ids(cg, n=n, seed=seed, prefix=prefix)
352
+ real_pool, skipped, families = (meta["pool"], meta["skipped_protected"],
353
+ meta["families"])
354
+ items = [_item(cg, nid) for nid in sample]
355
+ pv = preview(cg, ids=sample, prefix=prefix,
356
+ min_jaccard=min_jaccard, min_cluster=min_cluster)
357
+ strata = {}
358
+ for it in items:
359
+ k = it["family"]
360
+ s = strata.setdefault(k, {"picked": 0, "missing_fields": {},
361
+ "ccg_exempt": 0})
362
+ s["picked"] += 1
363
+ if it["ccg_exempt"]:
364
+ s["ccg_exempt"] += 1
365
+ for m in it["ccg_missing"]:
366
+ s["missing_fields"][m] = s["missing_fields"].get(m, 0) + 1
367
+ return {
368
+ "root": cg.root, "dry_run": True, "readonly": True,
369
+ "action": "refine", "op": "maintain",
370
+ "prefix": prefix, "seed": seed,
371
+ "requested": n if n is not None else (len(sample) if ids else SAMPLE_N),
372
+ "pool": real_pool, "skipped_protected": skipped,
373
+ "sampled": len(items), "families": families,
374
+ "sample": sample, "sample_sha": _sha(*sample)[:16],
375
+ "strata": strata, "items": items,
376
+ "worklist": items, "spec": SPEC,
377
+ "preview": {"clusters": pv["clusters"], "candidates": pv["candidates"],
378
+ "min_cluster": pv["min_cluster"],
379
+ "min_jaccard": pv["min_jaccard"],
380
+ "skipped_no_condition": pv["skipped_no_condition"],
381
+ "candidates_generic_only": pv["candidates_generic_only"],
382
+ "sample_adequacy": pv["sample_adequacy"],
383
+ "df_basis": pv["df_basis"],
384
+ "generic_df_threshold": pv["generic_df_threshold"]},
385
+ "gate_rule": SPEC["gate"],
386
+ "note": ("抽检工单(只读):供人工核对提炼口径;核对通过前不得扩批。"
387
+ "行 `apply=true` 只落抽检留痕,不改任何节点。"),
388
+ }
389
+
390
+
391
+ # 生效条件:x 转为 cg;target 为 None 取 CALIBRATE_TARGET、否则 int(target),cap 为 None 取 CALIBRATE_CAP、否则 int(cap),初始 n=min(SAMPLE_N if n0 is None else int(n0), cap);循环以 n 调 preview(cg, n=n, seed=seed, prefix=prefix, min_jaccard=min_jaccard, min_cluster=min_cluster) 并记录 step,直到 pv["clusters"] >= target 则 chosen=step 并 break,或 n >= min(cap, pool or cap) 则 break;chosen 为 None 时 blocked=True 并按 steps 汇总 recommend,否则 recommend 基于 chosen;返回只读校准报告;
392
+ def calibrate(x, target=None, cap=None, n0=None, seed=None, prefix=None,
393
+ min_jaccard=None, min_cluster=None) -> dict:
394
+ """样本量校准(只读):20 条起步有界爬坡,直到产出 `target` 个候选或触顶。
395
+
396
+ 目的是回答「20 条工单为什么交不出候选」,并给出**有界**的替代样本量;
397
+ 若爬坡到顶仍无候选、或候选共性条件全为高 df 模板词,则如实报
398
+ `blocked` 并指出真正的瓶颈是**提炼口径**而非样本量——不得据此扩批。
399
+ """
400
+ cg = _as_cg(x)
401
+ target = CALIBRATE_TARGET if target is None else int(target)
402
+ cap = CALIBRATE_CAP if cap is None else int(cap)
403
+ n = min(SAMPLE_N if n0 is None else int(n0), cap)
404
+ steps, chosen, pool = [], None, None
405
+ while True:
406
+ pv = preview(cg, n=n, seed=seed, prefix=prefix,
407
+ min_jaccard=min_jaccard, min_cluster=min_cluster)
408
+ pool = pv["pool"]
409
+ step = {"n": n, "sampled": pv["sampled"], "clusters": pv["clusters"],
410
+ "candidates_generic_only": pv["candidates_generic_only"],
411
+ "discriminating_terms": sorted(
412
+ {t for c in pv["candidates"] for t in c["discriminating"]})[:12]}
413
+ steps.append(step)
414
+ if pv["clusters"] >= target:
415
+ chosen = step
416
+ break
417
+ top = min(cap, pool or cap)
418
+ if n >= top:
419
+ break
420
+ n = min(n * 2, top)
421
+ blocked = chosen is None
422
+ saw_clusters = any(s["clusters"] for s in steps)
423
+ total_c = sum(s["clusters"] for s in steps)
424
+ gen_only = sum(s["candidates_generic_only"] for s in steps)
425
+ if not blocked:
426
+ rec = ("抽检样本量校准为 n=%d(产出 %d 个候选);仍须人工核对提炼口径,"
427
+ "核对通过前不得扩批" % (chosen["n"], chosen["clusters"]))
428
+ else:
429
+ parts = ["爬坡至 cap=%d 仍只产出 %d 个候选(< target=%d),样本量不足以"
430
+ "直接支撑抽检" % (cap, steps[-1]["clusters"], target)]
431
+ if saw_clusters:
432
+ if gen_only >= total_c:
433
+ parts.append("且已产出候选的共性条件**全部**为高 df 模板词(无区分性)")
434
+ else:
435
+ parts.append("且候选中 %d/%d 的共性条件全为高 df 模板词,"
436
+ "其余『区分性』项形如整行条件文本(含时间戳),"
437
+ "属归一化缺失导致的伪区分性" % (gen_only, total_c))
438
+ parts.append("须先修正提炼口径(剔除模板词 + 归一化/裁剪整行文本)"
439
+ "或以「候选」为抽检单位重新校准;当前状态不得扩批")
440
+ rec = ";".join(parts)
441
+ return {"ok": True, "action": "refine_calibrate", "op": "maintain",
442
+ "root": cg.root, "dry_run": True, "readonly": True,
443
+ "target": target, "cap": cap, "pool": pool,
444
+ "steps": steps, "chosen": chosen, "blocked": blocked,
445
+ "sample_adequacy": "ok" if not blocked else "insufficient",
446
+ "recommend": rec, "note": "只读校准:未写盘、未改任何节点。"}
447
+
448
+
449
+ # ---- 留痕与扩批闸门 ------------------------------------------------------
450
+
451
+ # 生效条件:candidates 中取 concept_id 为真值的 id 列表;verdicts 中为 dict 且 concept_id 非空且首次出现的条目计入 reviewed 并累加其中 faithful/added_info 的真值计数;ids 非空时按 reviewed < len(ids) → awaiting_human_review、added 非零 → added_info_detected、faithful_rate < GATE_MIN_PASS_RATE → faithful_rate_below_gate、否则 ok 且 expand_allowed=True;ids 为空时 reason="no_candidates" 且 expand_allowed=False;
452
+ def _verdict_stats(verdicts, candidates) -> dict:
453
+ ids = [c.get("concept_id") for c in candidates if c.get("concept_id")]
454
+ seen, faithful, added = set(), 0, 0
455
+ for v in verdicts or []:
456
+ if not isinstance(v, dict):
457
+ continue
458
+ cid = v.get("concept_id")
459
+ if not cid or cid in seen:
460
+ continue
461
+ seen.add(cid)
462
+ faithful += 1 if v.get("faithful") else 0
463
+ added += 1 if v.get("added_info") else 0
464
+ reviewed = len(seen)
465
+ rate = (faithful / float(reviewed)) if reviewed else 0.0
466
+ expand, reason = False, "no_candidates"
467
+ if ids:
468
+ if reviewed < len(ids):
469
+ reason = "awaiting_human_review"
470
+ elif added:
471
+ reason = "added_info_detected"
472
+ elif rate < GATE_MIN_PASS_RATE:
473
+ reason = "faithful_rate_below_gate"
474
+ else:
475
+ reason = "ok"
476
+ expand = True
477
+ return {"candidates": len(ids), "reviewed": reviewed,
478
+ "faithful": faithful, "added_info": added,
479
+ "faithful_rate": round(rate, 4),
480
+ "min_pass_rate": GATE_MIN_PASS_RATE,
481
+ "expand_allowed": expand, "reason": reason}
482
+
483
+
484
+ # 生效条件:x 转为 cg;以 ids/n/seed/prefix/min_jaccard/min_cluster 调 plan 得到 p,batch 假值(含 None/空串)回落当前时间字符串;从 p["preview"]["candidates"] 与 verdicts 经 _verdict_stats 得 stats;把含 batch/actor/prefix/seed/sample/verdicts/gate/note 的 record 追加写入 _log_path(cg),返回 rep;
485
+ def apply(x, batch=None, actor=None, verdicts=None, note=None,
486
+ ids=None, n=None, seed=None, prefix=None,
487
+ min_jaccard=None, min_cluster=None) -> dict:
488
+ """落抽检批次 + 人工裁决到 `_refine.jsonl`。**不改写任何节点**。"""
489
+ cg = _as_cg(x)
490
+ p = plan(cg, ids=ids, n=n, seed=seed, prefix=prefix,
491
+ min_jaccard=min_jaccard, min_cluster=min_cluster)
492
+ batch = batch or time.strftime("%Y%m%d-%H%M%S")
493
+ cands = p["preview"]["candidates"]
494
+ stats = _verdict_stats(verdicts, cands)
495
+ record = {
496
+ "t": time.time(), "action": "refine", "batch": batch, "actor": actor,
497
+ "prefix": p["prefix"], "seed": p["seed"], "requested": p["requested"],
498
+ "pool": p["pool"], "sampled": p["sampled"],
499
+ "sample": p["sample"], "sample_sha": p["sample_sha"],
500
+ "sample_adequacy": p["preview"]["sample_adequacy"],
501
+ "min_cluster": p["preview"].get("min_cluster"),
502
+ "candidates": [{"concept_id": c["concept_id"], "members": c["members"],
503
+ "common_conditions": c["common_conditions"],
504
+ "grounding_gap": c["grounding_gap"],
505
+ "discriminating": c["discriminating"]}
506
+ for c in cands],
507
+ "verdicts": list(verdicts or []), "gate": stats, "note": note,
508
+ }
509
+ append_jsonl(_log_path(cg), record)
510
+ rep = {"ok": True, "action": "refine", "op": "maintain", "batch": batch,
511
+ "root": cg.root, "sampled": record["sampled"],
512
+ "candidates": stats["candidates"], "reviewed": stats["reviewed"],
513
+ "gate": stats, "log": REFINE_LOG,
514
+ "note": ("抽检留痕已落盘(未改任何节点);"
515
+ + ("闸门放行扩批(仍需另开 induce 批次并留痕)"
516
+ if stats["expand_allowed"] else
517
+ "闸门未放行:" + stats["reason"]))}
518
+ return rep
519
+
520
+
521
+ # 生效条件:x 转为 cg;batch 为真值时只保留 _read_log(cg) 中 batch 字段相等的记录,过滤后为空时返回 batches=0、expand_allowed=False、reason="no_batch";否则取最后一条记录,以其 verdicts 与 candidates(假值转空列表)经 _verdict_stats 复算并返回 expand_allowed/reason 等;
522
+ def gate(x, batch=None) -> dict:
523
+ """扩批闸门:读留痕复算通过率(不写盘)。"""
524
+ cg = _as_cg(x)
525
+ recs = _read_log(cg)
526
+ if batch:
527
+ recs = [r for r in recs if r.get("batch") == batch]
528
+ if not recs:
529
+ return {"ok": True, "action": "refine_gate", "op": "maintain",
530
+ "root": cg.root, "batch": batch, "batches": 0,
531
+ "expand_allowed": False, "reason": "no_batch"}
532
+ rec = recs[-1]
533
+ stats = _verdict_stats(rec.get("verdicts"), rec.get("candidates") or [])
534
+ return {"ok": True, "action": "refine_gate", "op": "maintain",
535
+ "root": cg.root, "batch": rec.get("batch"),
536
+ "batches": len({r.get("batch") for r in recs}),
537
+ "sampled": rec.get("sampled"),
538
+ "sample_adequacy": rec.get("sample_adequacy"),
539
+ "gate": stats, "expand_allowed": stats["expand_allowed"],
540
+ "reason": stats["reason"],
541
+ "note": ("闸门放行:可另开 induce 批次扩大提炼规模并逐批留痕"
542
+ if stats["expand_allowed"] else
543
+ "闸门未放行,禁止扩大批次;不得盲跑全量。")}
544
+
545
+
546
+ # 生效条件:batch 为真值时只保留该批次记录,total 先记为过滤后条数;limit 非 None 且 >=0 时以 recs[-int(limit):] 截尾(limit=0 因 [-0:] 等价 [0:] 仍返回全部记录),limit 为 None 或负数时不截断;
547
+ def history(x, limit=100, batch=None) -> dict:
548
+ cg = _as_cg(x)
549
+ recs = _read_log(cg)
550
+ if batch:
551
+ recs = [r for r in recs if r.get("batch") == batch]
552
+ total = len(recs)
553
+ if limit is not None and limit >= 0:
554
+ recs = recs[-int(limit):]
555
+ return {"root": cg.root, "total": total, "returned": len(recs),
556
+ "action": "refine", "records": recs}
557
+
558
+
559
+ # ---- CLI(真实库抽检/预演用;MCP 侧走 maintain action) -------------------
560
+
561
+ # 生效条件:解析 argv(None 时由 argparse 取 sys.argv)得到 root/n/seed/prefix/min-jaccard/min-cluster/report 等参数;a.calibrate 为真时调 calibrate 并打印 target/cap/pool/blocked/sample_adequacy/recommend 及每步摘要;否则 a.preview 为真选 preview、为假选 plan,调用后打印 thin 摘要与候选并返回 0;
562
+ def _main(argv=None) -> int:
563
+ import argparse
564
+ import json
565
+ ap = argparse.ArgumentParser(description="G6 提炼抽检(默认只预演,不写盘)")
566
+ ap.add_argument("root", help="认知图库根")
567
+ ap.add_argument("--n", type=int, default=SAMPLE_N, help="抽检条数(默认 20)")
568
+ ap.add_argument("--seed", default=SAMPLE_SEED, help="抽样种子(同 seed 同样本)")
569
+ ap.add_argument("--prefix", default=PREFIX_DEFAULT, help="id 前缀(默认空 = 不过滤,全库抽样;给出则只抽以该前缀开头的 id)")
570
+ ap.add_argument("--min-jaccard", type=float, default=None)
571
+ ap.add_argument("--min-cluster", type=int, default=None)
572
+ ap.add_argument("--preview", action="store_true", help="只出预演(不出工单正文)")
573
+ ap.add_argument("--calibrate", action="store_true",
574
+ help="只做样本量校准(20 条起步有界爬坡)")
575
+ ap.add_argument("--report", default=None, help="把汇总 JSON 另存一份")
576
+ a = ap.parse_args(argv)
577
+ if a.calibrate:
578
+ rep = calibrate(a.root, seed=a.seed, prefix=a.prefix,
579
+ min_jaccard=a.min_jaccard, min_cluster=a.min_cluster)
580
+ if a.report:
581
+ with open(a.report, "w", encoding="utf-8") as f:
582
+ json.dump(rep, f, ensure_ascii=False, indent=2)
583
+ print(json.dumps({k: rep[k] for k in
584
+ ("target", "cap", "pool", "blocked",
585
+ "sample_adequacy", "recommend")},
586
+ ensure_ascii=False, indent=2))
587
+ for s in rep["steps"]:
588
+ print(" - n=%4d clusters=%2d generic_only=%2d discriminating=%s"
589
+ % (s["n"], s["clusters"], s["candidates_generic_only"],
590
+ s["discriminating_terms"][:6]))
591
+ return 0
592
+ fn = preview if a.preview else plan
593
+ rep = fn(a.root, n=a.n, seed=a.seed, prefix=a.prefix,
594
+ min_jaccard=a.min_jaccard, min_cluster=a.min_cluster)
595
+ if a.report:
596
+ with open(a.report, "w", encoding="utf-8") as f:
597
+ json.dump(rep, f, ensure_ascii=False, indent=2)
598
+ pv = rep.get("preview") if "preview" in rep else rep
599
+ thin = {k: rep.get(k) for k in ("action", "prefix", "seed", "requested",
600
+ "pool", "sampled", "families")}
601
+ thin.update({k: pv.get(k) for k in ("clusters", "sample_adequacy",
602
+ "candidates_generic_only")})
603
+ print(json.dumps(thin, ensure_ascii=False, indent=2))
604
+ for c in (pv.get("candidates") or []):
605
+ print(" - %s size=%d common=%s generic=%s gap=%s"
606
+ % (c["concept_id"], c["size"], c["common_conditions"][:4],
607
+ [g["term"] for g in c.get("generic_conditions", [])],
608
+ c.get("grounding_gap")))
609
+ return 0
610
+
611
+
612
+ if __name__ == "__main__":
613
+ import sys
614
614
  sys.exit(_main())