@a9i5k4/dsh-auto-memory 2.1.5 → 2.1.7

package/python/m7_embedding_v1.py

@@ -0,0 +1,381 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""M7-3 production embedding core (frozen by docs/M7-ALGORITHM-DECISION.md).
+
+Pure re-implementation of the frozen policy for the production sidecar path
+(the benchmark rig lives separately under python/bench/ and stays untouched):
+
+  provider      bge-m3 pinned revision, CLS pooling, L2-normalized float32
+  chunk policy  m7_chunk_v1 = para-512-noov (tokenizer id space,
+                greedy paragraph packing, oversized paragraph hard-split,
+                no overlap; special tokens added by the model tokenizer)
+  identity      provider/model/revision/dimension/normalization/policyVersion
+                -> configHash; mismatch = stale = full rebuild
+
+Providers:
+  hash-pre-v1   deterministic stdlib-only embedding (sha256-seeded bag of
+                token-trigram dims). Zero dependencies, zero network. Used
+                by CI and offline protocol tests. NOT a quality provider.
+  bge-m3-pre-v1 real model via transformers (lazy import; requires the
+                pinned local snapshot dir passed in the embedding config).
+
+No DSH file reads; no writes except what the worker explicitly passes in.
+"""
+import hashlib
+import json
+import re
+
+PROVIDER_REAL = 'bge-m3-pre-v1'
+PROVIDER_REAL_INT8 = 'bge-m3-onnx-int8-v1'
+PROVIDER_HASH = 'hash-pre-v1'
+CHUNK_POLICY_VERSION = 'm7_chunk_v1'
+CHUNK_MAX_TOKENS = 512
+QUERY_MAX_TOKENS = 256
+DIMENSION = 1024
+
+RE_CHUNK = re.compile(r'^chk_[0-9a-f]{16,}$')
+
+
+def sha_hex(data):
+    return hashlib.sha256(data).hexdigest()
+
+
+def canonical(v):
+    if v is None:
+        return 'null'
+    if isinstance(v, bool):
+        return 'true' if v else 'false'
+    if isinstance(v, (int, float, str)):
+        return json.dumps(v, ensure_ascii=False, separators=(',', ':'))
+    if isinstance(v, list):
+        return '[' + ','.join(canonical(x) for x in v) + ']'
+    if isinstance(v, dict):
+        return '{' + ','.join(json.dumps(str(k), ensure_ascii=False,
+                                         separators=(',', ':')) + ':' + canonical(v[k])
+                              for k in sorted(v.keys())) + '}'
+    return 'null'
+
+
+def config_hash(provider, model_revision, dimension):
+    payload = {
+        'provider': provider,
+        'model': 'bge-m3' if provider == PROVIDER_REAL else provider,
+        'modelRevision': model_revision,
+        'dimension': dimension,
+        'normalization': 'l2_normalize',
+        'dtype': 'float32',
+        'chunkPolicyVersion': CHUNK_POLICY_VERSION,
+        'chunkParams': {'maxTokens': CHUNK_MAX_TOKENS, 'paraAligned': True,
+                        'overlap': 0},
+        'queryMaxTokens': QUERY_MAX_TOKENS,
+    }
+    return 'cfgh_' + sha_hex(canonical(payload).encode('utf-8'))
+
+
+def chunk_id_for(memory_id, record_digest, ordinal):
+    return 'chk_' + sha_hex(
+        ('m7-chunk-pre-v1\u0000' + memory_id + '\u0000' + record_digest +
+         '\u0000' + str(ordinal)).encode('utf-8'))[:32]
+
+
+def chunk_record_token_ids(tokenizer, text):
+    """para-512-noov over tokenizer ids; returns list of id-lists."""
+    paras = []
+    for p in text.split('\n'):
+        ids = tokenizer(p, add_special_tokens=False)['input_ids']
+        if ids:
+            paras.append(ids)
+    chunks, cur = [], []
+    for ids in paras:
+        if cur and len(cur) + len(ids) <= CHUNK_MAX_TOKENS:
+            cur.extend(ids)
+            continue
+        if cur:
+            chunks.append(cur)
+            cur = []
+        if len(ids) <= CHUNK_MAX_TOKENS:
+            cur = list(ids)
+        else:
+            for i in range(0, len(ids), CHUNK_MAX_TOKENS):
+                chunks.append(ids[i:i + CHUNK_MAX_TOKENS])
+    if cur:
+        chunks.append(cur)
+    return chunks or [[]]
+
+
+class HashEmbedder:
+    """Deterministic dependency-free embedder for tests/offline paths.
+
+    Unnormalized bag dims come from sha256 of character trigrams; the final
+    vector is L2-normalized float32. Same text -> same vector, always.
+    """
+
+    provider = PROVIDER_HASH
+
+    def __init__(self, config):
+        self.dimension = int(config.get('dimension') or DIMENSION)
+
+    def encode_texts(self, texts):
+        import math
+        out = []
+        for t in texts:
+            vec = [0.0] * self.dimension
+            s = t if isinstance(t, str) else ''
+            for i in range(max(0, len(s) - 2)):
+                h = int.from_bytes(hashlib.sha256(s[i:i + 3].encode('utf-8',
+                                    'ignore')).digest()[:8], 'big')
+                vec[h % self.dimension] += 1.0
+            norm = math.sqrt(sum(x * x for x in vec)) or 1.0
+            out.append([round(x / norm, 8) for x in vec])
+        return out
+
+    def close(self):
+        pass
+
+
+class BgeM3Embedder:
+    """Real frozen provider: transformers AutoModel, CLS pooling, L2 norm."""
+
+    provider = PROVIDER_REAL
+
+    def __init__(self, config):
+        import os
+        os.environ.setdefault('TOKENIZERS_PARALLELISM', 'false')
+        import torch
+        from transformers import AutoModel, AutoTokenizer
+        torch.set_num_threads(int(config.get('torchThreads') or 16))
+        self._torch = torch
+        path = config['modelDir']
+        revision = str(config.get('modelRevision') or '')
+        self.tokenizer = AutoTokenizer.from_pretrained(path, revision=revision)
+        self.model = AutoModel.from_pretrained(path, revision=revision,
+                                               dtype=torch.float32)
+        self.model.eval()
+        self.dimension = int(config.get('dimension') or DIMENSION)
+
+    def _specials(self):
+        probe = self.tokenizer('x', add_special_tokens=True)['input_ids']
+        core = self.tokenizer('x', add_special_tokens=False)['input_ids']
+        for i in range(len(probe) - len(core) + 1):
+            if probe[i:i + len(core)] == core:
+                return probe[:i], probe[i + len(core):]
+        raise RuntimeError('content not found in tokenizer probe')
+
+    def build_doc_ids(self, chunk_ids, max_total=512):
+        """Wrap chunk token ids with the model's special tokens exactly once,
+        capping total length at the XLM-R position limit (audit P0/P1)."""
+        prefix, suffix = self._specials()
+        budget = max_total - len(prefix) - len(suffix)
+        body = list(chunk_ids)[:max(0, budget)]
+        return list(prefix) + body + list(suffix)
+
+    def encode_ids(self, ids_list, batch_size=8):
+        torch = self._torch
+        prefix, suffix = self._specials()
+        pad = (self.tokenizer.pad_token_id if self.tokenizer.pad_token_id
+               is not None else self.tokenizer.eos_token_id)
+        vecs = []
+        for i in range(0, len(ids_list), batch_size):
+            part = [list(prefix) + list(ids) + list(suffix)
+                    for ids in ids_list[i:i + batch_size]]
+            maxlen = max(len(x) for x in part)
+            inp = torch.full((len(part), maxlen), pad, dtype=torch.long)
+            att = torch.zeros((len(part), maxlen), dtype=torch.long)
+            for r, ids in enumerate(part):
+                inp[r, :len(ids)] = torch.tensor(ids, dtype=torch.long)
+                att[r, :len(ids)] = 1
+            with torch.no_grad():
+                hidden = self.model(input_ids=inp,
+                                    attention_mask=att).last_hidden_state
+            pooled = hidden[:, 0]  # CLS per repo 1_Pooling/config.json
+            pooled = torch.nn.functional.normalize(pooled, p=2, dim=1)
+            vecs.extend(pooled.to(torch.float32).numpy().tolist())
+        return vecs
+
+    def _encode_texts_via_ids(self, texts, max_tokens):
+        """Single source of truth for text->vector: tokenize WITHOUT specials,
+        then let encode_ids wrap exactly once. Guarantees query/corpus share
+        one template, and the truncation budget RESERVES room for the special
+        tokens so wrapped length never exceeds max_tokens (audit round 2:
+        same latent overlimit class as the e5 512 crash)."""
+        prefix, suffix = self._specials()
+        budget = max(1, max_tokens - len(prefix) - len(suffix))
+        ids_list = []
+        for t in texts:
+            enc = self.tokenizer(t or '', add_special_tokens=False,
+                                 truncation=True, max_length=budget)
+            ids_list.append(enc['input_ids'])
+        return self.encode_ids(ids_list)
+
+    def chunk_and_encode(self, text):
+        id_chunks = chunk_record_token_ids(self.tokenizer, text)
+        return id_chunks, self.encode_ids(id_chunks)
+
+    def encode_texts(self, texts, batch_size=8):
+        """Record-level text -> vector (chunker bypasses: caller chunks).
+        Used by the worker's hash-style flows and as the parity entry."""
+        return self._encode_texts_via_ids(texts, 512)
+
+    def encode_query(self, text):
+        # audit fix P1: previously tokenized with add_special_tokens=True and
+        # then wrapped again by encode_ids -> double specials, template drift.
+        # query_cap already reserves the specials budget (see below).
+        enc = self.tokenizer(text, add_special_tokens=False, truncation=True,
+                             max_length=self.query_cap())
+        return self.encode_ids([enc['input_ids']])[0]
+
+    def query_cap(self):
+        # 256 content + specials headroom: wrapped total never exceeds cap
+        return 256 - 8
+
+    def close(self):
+        try:
+            del self.model
+            import gc
+            gc.collect()
+        except Exception:
+            pass
+
+
+class BgeM3OnnxInt8Embedder:
+    """Quantized slim tier: Xenova/bge-m3 onnx/model_int8.onnx (dynamic int8)
+    via onnxruntime. Same XLM-R tokenizer + CLS pooling + L2 norm contract as
+    BgeM3Embedder; text->vector template methods are byte-twin copies of the
+    parent so query/corpus share one wrapping path. L2 head-to-head 2026-08-25:
+    R@5 identical to fp32 (0.925), encode ~6x faster, vec cos mean 0.975.
+    Identity dtype differs from fp32 -> switching providers invalidates the
+    vector store by design (stale -> rebuild)."""
+
+    provider = PROVIDER_REAL_INT8
+
+    def __init__(self, config):
+        import os
+        os.environ.setdefault('TOKENIZERS_PARALLELISM', 'false')
+        import numpy as np
+        import onnxruntime as ort
+        from transformers import AutoTokenizer
+        self._np = np
+        base = config['modelDir']
+        onnx_rel = str(config.get('onnxFile') or 'onnx/model_int8.onnx')
+        self.session = ort.InferenceSession(
+            os.path.join(base, *onnx_rel.split('/')),
+            providers=['CPUExecutionProvider'])
+        self.tokenizer = AutoTokenizer.from_pretrained(base)
+        self._inp = self.session.get_inputs()[0].name
+        self._att = self.session.get_inputs()[1].name
+        self.dimension = int(config.get('dimension') or DIMENSION)
+
+    def _specials(self):
+        probe = self.tokenizer('x', add_special_tokens=True)['input_ids']
+        core = self.tokenizer('x', add_special_tokens=False)['input_ids']
+        for i in range(len(probe) - len(core) + 1):
+            if probe[i:i + len(core)] == core:
+                return probe[:i], probe[i + len(core):]
+        raise RuntimeError('content not found in tokenizer probe')
+
+    def build_doc_ids(self, chunk_ids, max_total=512):
+        prefix, suffix = self._specials()
+        budget = max_total - len(prefix) - len(suffix)
+        body = list(chunk_ids)[:max(0, budget)]
+        return list(prefix) + body + list(suffix)
+
+    def encode_ids(self, ids_list, batch_size=16):
+        np = self._np
+        prefix, suffix = self._specials()
+        pad = (self.tokenizer.pad_token_id if self.tokenizer.pad_token_id
+               is not None else self.tokenizer.eos_token_id)
+        vecs = []
+        for i in range(0, len(ids_list), batch_size):
+            part = [list(prefix) + list(ids) + list(suffix)
+                    for ids in ids_list[i:i + batch_size]]
+            maxlen = max(len(x) for x in part)
+            inp = np.full((len(part), maxlen), pad, dtype=np.int64)
+            att = np.zeros((len(part), maxlen), dtype=np.int64)
+            for r, ids in enumerate(part):
+                inp[r, :len(ids)] = np.asarray(ids, dtype=np.int64)
+                att[r, :len(ids)] = 1
+            hidden = self.session.run(None, {self._inp: inp,
+                                             self._att: att})[0]
+            pooled = hidden[:, 0].astype(np.float32)  # CLS per 1_Pooling
+            norms = np.linalg.norm(pooled, axis=1, keepdims=True)
+            pooled = pooled / np.maximum(norms, 1e-12)
+            vecs.extend([row.tolist() for row in pooled])
+        return vecs
+
+    def _encode_texts_via_ids(self, texts, max_tokens):
+        prefix, suffix = self._specials()
+        budget = max(1, max_tokens - len(prefix) - len(suffix))
+        ids_list = []
+        for t in texts:
+            enc = self.tokenizer(t or '', add_special_tokens=False,
+                                 truncation=True, max_length=budget)
+            ids_list.append(enc['input_ids'])
+        return self.encode_ids(ids_list)
+
+    def chunk_and_encode(self, text):
+        id_chunks = chunk_record_token_ids(self.tokenizer, text)
+        return id_chunks, self.encode_ids(id_chunks)
+
+    def encode_texts(self, texts, batch_size=16):
+        return self._encode_texts_via_ids(texts, 512)
+
+    def encode_query(self, text):
+        enc = self.tokenizer(text, add_special_tokens=False, truncation=True,
+                             max_length=self.query_cap())
+        return self.encode_ids([enc['input_ids']])[0]
+
+    def query_cap(self):
+        return 256 - 8
+
+    def close(self):
+        try:
+            del self.session
+            import gc
+            gc.collect()
+        except Exception:
+            pass
+
+
+def load_embedder(config):
+    provider = str(config.get('provider') or '')
+    if provider == PROVIDER_HASH:
+        return HashEmbedder(config)
+    if provider == PROVIDER_REAL:
+        return BgeM3Embedder(config)
+    if provider == PROVIDER_REAL_INT8:
+        return BgeM3OnnxInt8Embedder(config)
+    raise ValueError('unknown embedding provider: %r' % (provider,))
+
+
+def identity_block(provider, config):
+    model_name = provider
+    dtype = 'float32'
+    if provider == PROVIDER_REAL:
+        model_name = 'bge-m3'
+    elif provider == PROVIDER_REAL_INT8:
+        model_name = 'bge-m3-int8'
+        # different dtype -> fp32-built vector stores are stale under the
+        # int8 tier and must rebuild (provider switch is never silent)
+        dtype = 'int8-dynamic-onnx'
+    return {
+        'schemaVersion': 1,
+        'namespace': 'dsh-auto-memory',
+        'policyVersion': 'semantic_vectors_v1',
+        'provider': provider,
+        'model': model_name,
+        'modelRevision': str(config.get('modelRevision') or 'hash'),
+        'dimension': int(config.get('dimension') or DIMENSION),
+        'normalization': 'l2_normalize',
+        'dtype': dtype,
+        'chunkPolicyVersion': CHUNK_POLICY_VERSION,
+        'configHash': config_hash(provider,
+                                  str(config.get('modelRevision') or 'hash'),
+                                  int(config.get('dimension') or DIMENSION)),
+    }
+
+
+def identity_matches(block, provider, config):
+    want = identity_block(provider, config)
+    keys = ('provider', 'model', 'modelRevision', 'dimension',
+            'normalization', 'dtype', 'chunkPolicyVersion', 'configHash')
+    return all(block.get(k) == want.get(k) for k in keys)

package/python/policies/activation_policy_v2.json

@@ -0,0 +1,88 @@
+{
+ "schemaVersion": 1,
+ "policyVersion": "activation_policy_v2",
+ "parentPolicyVersion": "m7_semantic_threshold_v1",
+ "createdAt": "2026-08-25",
+ "runId": "label-review-cal20260824-1954",
+ "goldDigest": "e74ccdcda998ae51596adf8119c818ac84fc52a4fa657fd687daeda7d14a072d",
+ "mode": "shadow-candidate",
+ "decisionRecordId": "activation-v2-delta-exp-override-20260824",
+ "deltaDeviationFromBrief": {
+  "briefFrozen": 0.02,
+  "effective": 0.03,
+  "reason": "production-completeness refit: delta=0.02 fails gates (cal-0008 emitOnP=1); nearest passing = delta_exp=0.03; requires retro-ratification"
+ },
+ "decisionOrder": [
+  "js_hard_gates",
+  "lane_decision",
+  "explicit_lane",
+  "proactive_lane",
+  "completeness_margin",
+  "decision"
+ ],
+ "thresholds": {
+  "tauLane": 0.45,
+  "tauHi": 0.45,
+  "tauLo": 0.35,
+  "deltaExp": 0.03,
+  "deltaPro": 0.05
+ },
+ "echoVeto": {
+  "scope": "proactive-lane-only",
+  "containmentArm": 0.3,
+  "denseTopArm": 0.7,
+  "requiresMarkZero": true,
+  "requiresIntentBelow": 0.5
+ },
+ "completenessGate": {
+  "phase": 1,
+  "lexicon": [
+   "对比",
+   "分别",
+   "两个",
+   "一起",
+   "都调",
+   "各自"
+  ],
+  "rule": "status!=complete -> max prefetch",
+  "outputs": [
+   "requiredTargetCount",
+   "resolvedTargetCount",
+   "status"
+  ]
+ },
+ "repetition": {
+  "round": "logging-only",
+  "suppressToPrefetchAllowed": true,
+  "activateOnCountsAlone": false
+ },
+ "hardGates": [
+  "harmful",
+  "correction",
+  "ignored",
+  "stale",
+  "wrong_scope",
+  "pii_class_never_proactive"
+ ],
+ "reasonCodes": [
+  "hard_gate_harmful",
+  "echo_veto_proactive",
+  "explicit_lane",
+  "explicit_lane_weak",
+  "completeness_complete",
+  "completeness_partial",
+  "completeness_unknown",
+  "proactive_margin",
+  "margin_below_delta",
+  "suppress_low_signal"
+ ],
+ "offlineMetrics86": {
+  "actPrecision": 1.0,
+  "actRecall": 0.289,
+  "emitOnP": 0,
+  "sViolations": 0,
+  "emits": 11,
+  "emitCorrectA": 11
+ },
+ "configHash": "cfgh_31e6d977a2c40d5e8bf4edf900557b8b"
+}

package/python/policies/decision-record-activation-v2-delta-exp-override-20260824.json

@@ -0,0 +1,21 @@
+{
+ "schemaVersion": 1,
+ "decisionId": "activation-v2-delta-exp-override-20260824",
+ "policyVersion": "activation_policy_v2",
+ "previousDeltaExp": 0.02,
+ "approvedDeltaExp": 0.03,
+ "reason": "production completeness semantics exposed cal-0008 prefetch-to-emit violation at 0.02",
+ "rollback": "0.02 historical only",
+ "evidence": {
+  "sampleId": "cal-0008",
+  "action": "P",
+  "denseTop": 0.654468,
+  "margin": 0.029178
+ },
+ "runId": "label-review-cal20260824-1954",
+ "goldDigest": "e74ccdcda998ae51596adf8119c818ac84fc52a4fa657fd687daeda7d14a072d",
+ "approvedBy": [
+  "user",
+  "mainAgent"
+ ]
+}

package/python/policies/decision-record-reasoning-kind-admission-20260826.json

@@ -0,0 +1,20 @@
+{
+  "id": "decision-record-reasoning-kind-admission-20260826",
+  "date": "2026-08-26",
+  "author": "calibration-agent (ox-alpha)",
+  "approvedBy": "user (产品目标正式裁定: 监听模型思维链与可见输出是本系统核心, 而非仅用户输入)",
+  "type": "contract-revision (M5 segment kind whitelist)",
+  "summary": "'reasoning' admitted as a legal Segment kind (weight 0.5 in lexical query plan), superseding the M4/M5-era rule that reasoning never enters context.",
+  "trigger": "controlled live shadow round 2: envelope drops 'trigger:invalid:kind' + 'window:invalid:kind' — the new CoT segments (kind='reasoning') were rejected by the M5 whitelist ['user','tool_call','tool_result','assistant']",
+  "change": [
+    "lib/context-bridge.js: segment kind whitelist + 'reasoning'",
+    "lib/shadow-retrieval.js: trigger.kind whitelist + 'reasoning'; query-plan origin map adds 'reasoning' at weight 0.5 (between recent-user 0.8 and tool-result 0.6... actually below tool-result, above assistant 0.2)",
+    "smoke-test-m51-pre.mjs: old rejection assertion replaced with admission assertion"
+  ],
+  "safety": [
+    "reasoning remains a FEATURE only; no hard gate may read it",
+    "CoT aggregation buffer bounded 4096 chars; flush >=512 chars or >=1500ms",
+    "purge on assoc-disable clears CoT buffers (zero-retention contract preserved)"
+  ],
+  "references": ["docs/COT-WATCH-RFC.md", "docs/M5-CONTRACT.md"]
+}

package/python/policies/decision-record-stale-gate-per-candidate-20260825.json

@@ -0,0 +1,33 @@
+{
+  "id": "decision-record-stale-gate-per-candidate-20260825",
+  "date": "2026-08-25",
+  "author": "calibration-agent (ox-alpha)",
+  "approvedBy": "user (直接授权实施, 主 Agent 沟通成本考量)",
+  "type": "implementation-defect-fix (non-policy-parameter change)",
+  "summary": "fv2 stale/correction hard gates narrowed from session-wide any() to per-candidate-topk.",
+  "trigger": {
+    "source": "M7-8 controlled live shadow (20 scripted requests, user-restarted host PID 5540)",
+    "observation": "94/94 observations suppressed with reasonCodes=[hard_gate_stale], features=null",
+    "rootCause": "worker constructed hardGates.stale as any(e.freshness=='stale' for e in payload.evidence); on real traffic every context_push carries >=1 stale evidence entry (anchored files updated today -> all M5 historical evidence judged stale), so fv2 was permanently muted"
+  },
+  "change": {
+    "file": "python/worker_semantic_v1.py (_fv2_shadow_decide gate construction only)",
+    "semantics": "stale/correction gates fire ONLY when the affected evidence memoryId is among the CURRENT top-K candidates; unrelated evidence never gates",
+    "untouched": [
+      "m7_activation_features_v2.py (decision core unchanged)",
+      "python/policies/*.json (versions, configHashes unchanged: activation cfgh_31e6d977..., intent cfgh_69465e69...)",
+      "golden-parity-fixtures-v1.jsonl (fixtures carry explicit hardGates inputs; all False -> zero output drift)"
+    ]
+  },
+  "verification": [
+    "scenario matrix 7/7 PASS (related-stale fires / unrelated-stale open / empty open / correction symmetric / end-to-end emit restored / related-stale still suppresses)",
+    "smoke m73 59/59, m79 20/20",
+    "py_compile OK"
+  ],
+  "residualRequirements": [
+    "user restarts 3080 again to load the fix",
+    "re-run the same 20-request controlled shadow and compare against archived rows (artifacts/m7-live-pre/controlled-shadow-rows-20260825.json): expected = stale-gate rows disappear, explicit recall requests produce emit/prefetch decisions consistent with offline replay",
+    "parity fixtures should gain stale-scenario cases in the next fixture regeneration window"
+  ],
+  "references": ["docs/M7-ACTIVATION-V2-CONTROLLED-SHADOW.md", "docs/M7-ACTIVATION-V2-HOLDEDOUT-EVAL.md"]
+}