dsh-fast-mode 1.5.22 → 1.5.23

package/CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## 1.5.23（打包：把量测脚本一起发出去）
+
+- 1.5.22 的文档让读者用 `bench/mechanism.py` / `textshare.py` / `reminders.py` / `patchshare.py` 复算结论，但 npm 包里只带了 `block_score.py`，从 npm 装的人复算不了。现在这四个只读分析脚本一起进包（都是 stdlib、只读 `~/.dsh/sessions`）。无行为变更。
+
 ## 1.5.22（只加量测与文档：机制拆解、纪律份额、同配置重复性；默认行为不变）
 
 - **机制拆解落地成工具**：新增 `bench/mechanism.py`，把 168 对「同 tag、同任务、同 pair、只差 arm」的会话转录逐项数出来（步数、调用数、并行率、只叙述步、prose/参数/推理字符）。结论写进 BENCHMARK：并行的两条口径都是 1.00（即并行**不是**杠杆）；步数 0.83 / 调用 0.79 / 参数 0.84 / prose 0.80 / 推理 0.75，全都落在 0.75–0.84——插件是「每样都少一点」，不是砍某一种输出。转录里推理字符占输出字符的 99.8 %（每会话中位 47.5 k vs 92），所以只盯可见文字优化、天花板是 0.2 %。

package/bench/mechanism.py

@@ -0,0 +1,129 @@
+#!/usr/bin/env python3
+"""What actually changes between two arms: steps, tool calls, batching, prose.
+
+The wall-clock win of fast mode is a step-count win (see BENCHMARK.md). This
+script opens the retained transcripts behind matched runs and reports *which*
+count moved, so the mechanism can be named instead of inferred:
+
+  messages      assistant messages = steps as the runner counts them
+  calls         tool calls emitted
+  calls/msg     batching width
+  multiShare    share of messages carrying >= 2 tool calls
+  textOnlyShare share of messages carrying no tool call (narration-only steps)
+  proseShare    visible prose chars / (prose + tool-argument chars)
+
+Usage:
+  python3 bench/mechanism.py --arm off --arm def15            # every matched pair
+  python3 bench/mechanism.py --arm off --arm def15 --tag DEF_MIG
+  python3 bench/mechanism.py --arm off --arm def15 --json
+  python3 bench/mechanism.py --list-arms                     # what is on disk
+
+Runs are the session directories named `suite-<tag>-<task>-p<pair>-<arm>`;
+only (tag, task, pair) groups that contain **both** arms are used, so the task
+mix cancels exactly like the wall-clock block score does.
+"""
+import argparse
+import collections
+import glob
+import importlib.util
+import json
+import os
+import statistics as st
+import sys
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+SESS = os.path.expanduser('~/.dsh/sessions')
+
+spec = importlib.util.spec_from_file_location('textshare', os.path.join(HERE, 'textshare.py'))
+textshare = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(textshare)
+
+
+def collect_all():
+    out = []
+    for d in glob.glob(os.path.join(SESS, '--*dsh-runs-suite-*')):
+        name = os.path.basename(d).strip('-')
+        m = textshare.RUN_RE.search(name)
+        if not m:
+            continue
+        out.append((m.group('tag'), m.group('task'), int(m.group('pair')), m.group('arm'), d, name))
+    out.sort(key=lambda x: x[5])
+    return out
+
+
+def stats_for(run):
+    text = textshare.read_transcript(run[4])
+    prose, args, msgs, calls, multi, textonly = textshare.tally(text)
+    if not msgs:
+        return None
+    return {'prose': prose, 'args': args, 'messages': msgs, 'calls': calls,
+            'multi': multi, 'textonly': textonly,
+            'callsPerMsg': calls / msgs,
+            'multiShare': multi / msgs,
+            'textOnlyShare': textonly / msgs,
+            'proseShare': prose / (prose + args) if prose + args else 0.0}
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument('--arm', action='append', default=[])
+    ap.add_argument('--tag', default='')
+    ap.add_argument('--limit', type=int, default=0, help='max matched groups; 0 = all')
+    ap.add_argument('--json', action='store_true')
+    ap.add_argument('--list-arms', action='store_true')
+    a = ap.parse_args()
+
+    runs = collect_all()
+    if a.list_arms:
+        c = collections.Counter(r[3] for r in runs)
+        for arm, n in c.most_common():
+            print(f'{arm:12} {n:5} runs')
+        return 0
+    arms = a.arm or ['off', 'def15']
+    if len(arms) != 2:
+        print('give exactly two --arm values', file=sys.stderr)
+        return 2
+
+    groups = collections.defaultdict(dict)
+    for tag, task, pair, arm, d, name in runs:
+        if a.tag and tag != a.tag:
+            continue
+        if arm not in arms:
+            continue
+        groups[(tag, task, pair)][arm] = (tag, task, pair, arm, d, name)
+
+    cells = []
+    for key in sorted(groups):
+        g = groups[key]
+        if not all(x in g for x in arms):
+            continue
+        row = {x: stats_for(g[x]) for x in arms}
+        if any(v is None for v in row.values()):
+            continue
+        cells.append(row)
+    if a.limit:
+        cells = cells[:a.limit]
+    if not cells:
+        print('no matched groups with transcripts; nothing to compute', file=sys.stderr)
+        return 1
+
+    keys = ['messages', 'calls', 'callsPerMsg', 'multiShare', 'textOnlyShare', 'proseShare']
+    med = {x: {k: st.median([c[x][k] for c in cells]) for k in keys} for x in arms}
+    summary = {'arms': arms, 'tag': a.tag or '*', 'groups': len(cells)}
+    for k in keys:
+        summary[k] = {'median': {x: med[x][k] for x in arms}}
+        ratios = [c[arms[1]][k] / c[arms[0]][k] for c in cells if c[arms[0]][k]]
+        summary[k]['medianRatio'] = st.median(ratios) if ratios else None
+    if a.json:
+        print(json.dumps(summary, indent=2))
+        return 0
+    print(f"matched groups: {len(cells)}  (arm A = {arms[0]}, arm B = {arms[1]})")
+    print(f"{'metric':14} {arms[0]:>10} {arms[1]:>10} {'B/A':>7}")
+    for k in keys:
+        b, r = med[arms[1]][k], summary[k]['medianRatio']
+        print(f"{k:14} {med[arms[0]][k]:10.3f} {b:10.3f} {(r if r is not None else float('nan')):7.3f}")
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())

package/bench/patchshare.py

@@ -0,0 +1,95 @@
+#!/usr/bin/env python3
+"""How much of the reuse lever was switched off by patch-shaped edits.
+
+1.5.19 marked a mutation whose target it could not locate as "uncertain" and then
+stopped offering check-reuse suggestions for the rest of that session. Patch calls
+(`apply_patch` / `patch`) name their targets inside the diff body, so before 1.5.20
+any session that edited once with a patch paid that cost for every later check.
+This counts, from retained transcripts, how common that shape is: sessions that
+call a patch tool, and the tool calls that follow it in those sessions.
+
+Usage:
+  python3 bench/patchshare.py              # all local sessions
+  python3 bench/patchshare.py --limit 0    # explicit: all
+"""
+
+from __future__ import annotations
+
+import argparse
+import glob
+import json
+import os
+import subprocess
+
+SESS = os.path.expanduser('~/.dsh/sessions')
+PATCH_CALLS = {'apply_patch', 'patch', 'apply_diff', 'edit_patch'}
+WRITE_CALLS = {'write', 'write_file', 'create_file', 'edit', 'multi_edit'} | PATCH_CALLS
+
+
+def sessions(limit):
+    out = []
+    for d in sorted(glob.glob(os.path.join(SESS, '--*'))):
+        files = glob.glob(os.path.join(d, 'session-*', 'session.v4.jsonl.zstd'))
+        if files:
+            out.append((d, files[0]))
+    return out[:limit] if limit else out
+
+
+def tally(path):
+    p = subprocess.run(['zstd', '-dc', path], capture_output=True)
+    if p.returncode != 0:
+        return None
+    calls = []
+    for line in p.stdout.decode('utf-8', 'replace').splitlines():
+        try:
+            o = json.loads(line)
+        except Exception:  # noqa: BLE001 - a torn line is not fatal
+            continue
+        if o.get('type') != 'assistant/message':
+            continue
+        for part in (o.get('data') or {}).get('message', {}).get('content') or []:
+            if isinstance(part, dict) and part.get('type') == 'tool-call':
+                calls.append(part.get('toolName') or part.get('name') or '')
+    if not calls:
+        return None
+    first_patch = next((i for i, n in enumerate(calls) if n in PATCH_CALLS), None)
+    return {
+        'calls': len(calls),
+        'patch_calls': sum(1 for n in calls if n in PATCH_CALLS),
+        'write_calls': sum(1 for n in calls if n in WRITE_CALLS),
+        'first_patch': first_patch,
+        'after_patch': 0 if first_patch is None else len(calls) - first_patch - 1,
+    }
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument('--limit', type=int, default=60, help='sessions, most recent first; 0 = all')
+    a = ap.parse_args()
+    rows = []
+    for d, f in reversed(sessions(a.limit)):
+        r = tally(f)
+        if r:
+            r['session'] = os.path.basename(d)[:24]
+            rows.append(r)
+    if not rows:
+        print('no sessions')
+        return
+    with_patch = [r for r in rows if r['patch_calls'] > 0]
+    calls = sum(r['calls'] for r in rows)
+    pcalls = sum(r['patch_calls'] for r in rows)
+    after = sum(r['after_patch'] for r in with_patch)
+    print(f"sessions {len(rows)}  calls {calls}")
+    print(f"sessions that patched {len(with_patch)} ({100 * len(with_patch) / len(rows):.0f} %)")
+    print(f"patch calls {pcalls} ({100 * pcalls / calls:.1f} % of calls)")
+    print(f"calls after the first patch, in patching sessions {after} "
+          f"({100 * after / calls:.1f} % of all calls)")
+    closed = [r for r in with_patch if r['after_patch'] > 0]
+    print(f"patching sessions with work still to do after the patch {len(closed)}")
+    med = sorted(r['after_patch'] for r in closed)
+    if med:
+        print(f"  median calls still to do after the first patch: {med[len(med) // 2]}")
+
+
+if __name__ == '__main__':
+    main()

package/bench/reminders.py

@@ -0,0 +1,112 @@
+#!/usr/bin/env python3
+"""Which scheduler reminders actually reached the model, and what followed.
+
+A reminder is only a lever if it is delivered. This counts, over retained
+transcripts, every directive text the plugin injected (`Auto schedule: ...`),
+grouped by kind, plus how often the quoted text shows up inside the model's own
+prose (the model echoing the instruction back is not delivery).
+
+Usage:
+  python3 bench/reminders.py                     # suite sessions, default cap
+  python3 bench/reminders.py --limit 0           # every retained session
+  python3 bench/reminders.py --dir '--*dsh-runs-suite-*'
+"""
+
+from __future__ import annotations
+
+import argparse
+import collections
+import glob
+import json
+import os
+import subprocess
+
+SESS = os.path.expanduser('~/.dsh/sessions')
+KINDS = (
+    ('that exact check already passed', 'reuse-check'),
+    ('this turn is long enough that prose', 'output-discipline'),
+    ('batch now. Put every independent tool call', 'batch-now'),
+    ('files changed since the last run', 'write-and-verify'),
+    ('reached the step budget', 'step-budget'),
+    ('consecutive steps produced nothing new', 'change-approach'),
+    ('consecutive steps produced nothing', 'change-approach'),
+)
+
+
+def transcripts(pattern, limit):
+    out = []
+    for d in sorted(glob.glob(os.path.join(SESS, pattern))):
+        for f in glob.glob(os.path.join(d, 'session-*', 'session.v4.jsonl.zstd')):
+            out.append((d, f))
+    return out[:limit] if limit else out
+
+
+def scan(path):
+    p = subprocess.run(['zstd', '-dc', path], capture_output=True)
+    if p.returncode != 0:
+        return None
+    text = p.stdout.decode('utf-8', 'replace')
+    counts = collections.Counter()
+    echoed = collections.Counter()
+    delivered = 0
+    for needle, kind in KINDS:
+        n = text.count(needle)
+        if n == 0:
+            continue
+        counts[kind] += n
+    # A directive is delivered as its own message part; the model repeating the
+    # phrase inside an assistant text block is an echo, so count both and report
+    # the delivery count only when it is separable.
+    for line in text.splitlines():
+        if 'Auto schedule:' not in line:
+            continue
+        try:
+            o = json.loads(line)
+        except Exception:  # noqa: BLE001 - a torn line is not fatal
+            continue
+        role = o.get('type')
+        data = o.get('data') or {}
+        msg = data.get('message') or data
+        body = ' '.join(part.get('text') or '' for part in msg.get('content') or []
+                        if isinstance(part, dict) and part.get('type') == 'text')
+        if 'Auto schedule:' not in body:
+            continue
+        for needle, kind in KINDS:
+            if needle in body:
+                if role == 'assistant/message':
+                    echoed[kind] += 1
+                else:
+                    delivered += 1
+                break
+    return counts, echoed, delivered
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument('--limit', type=int, default=0, help='transcripts; 0 = all')
+    ap.add_argument('--dir', default='--*dsh-runs-suite-*')
+    a = ap.parse_args()
+    files = transcripts(a.dir, a.limit)
+    total = collections.Counter()
+    echo = collections.Counter()
+    delivered = 0
+    with_any = 0
+    scanned = 0
+    for d, f in files:
+        r = scan(f)
+        if not r:
+            continue
+        scanned += 1
+        counts, echoed, n = r
+        delivered += n
+        total.update(counts)
+        echo.update(echoed)
+        with_any += 1 if sum(counts.values()) > 0 else 0
+    print(f'transcripts {scanned}  with any directive {with_any} '
+          f'({100 * with_any / max(scanned, 1):.0f} %)  delivered {delivered}')
+    for kind, n in total.most_common():
+        print(f'  {kind:18} raw {n:4}  echoed-in-prose {echo[kind]:4}')
+
+
+if __name__ == '__main__':
+    main()

package/bench/textshare.py

@@ -0,0 +1,150 @@
+#!/usr/bin/env python3
+"""Visible assistant prose vs tool-call argument characters.
+
+Recomputes the claim "visible assistant prose is only ~1.2 % of (prose + tool
+arguments)" from retained session transcripts, so the number in the docs has a
+command behind it.
+
+Counting unit is one `assistant/message` record of a session transcript:
+  * numerator   = sum of len(text) over `content` parts of type `text`;
+  * denominator = numerator + sum of len(arguments) over parts of type `tool-call`.
+Reasoning parts, tool *results*, and user/system messages are excluded; a tool-call
+part's `arguments` is the raw JSON string the model emitted, so the ratio compares
+what the model wrote out of band with what it said in prose.
+
+Usage:
+  python3 bench/textshare.py                      # 40 def15 sessions, fixed rule
+  python3 bench/textshare.py --all                # every def15 session on disk
+  python3 bench/textshare.py --arm disc-only --limit 0 --tag D2B_MIG
+  python3 bench/textshare.py --json               # machine-readable summary
+
+Sampling rule (deterministic, no RNG): all session directories whose run-directory
+name matches `suite-<tag>-<task>-p<pair>-<arm>` are collected, sorted by that
+directory name, and the first --limit are used (--limit 0 = all). The same corpus
+therefore always yields the same number.
+"""
+import argparse
+import glob
+import json
+import os
+import re
+import statistics as st
+import subprocess
+import sys
+
+SESS = os.path.expanduser('~/.dsh/sessions')
+RUN_RE = re.compile(r'suite-(?P<tag>[A-Za-z0-9_]+)-(?P<task>\w+)-p(?P<pair>\d+)'
+                    r'-(?P<arm>[A-Za-z0-9_]+)$')
+
+
+def collect(arm, tag, limit):
+    """[(session_dir, run_name)] for one arm, in a fixed sorted order."""
+    out = []
+    for d in glob.glob(os.path.join(SESS, '--*dsh-runs-suite-*')):
+        name = os.path.basename(d).strip('-')
+        m = RUN_RE.search(name)
+        if not m or m.group('arm') != arm:
+            continue
+        if tag and m.group('tag') != tag:
+            continue
+        out.append((d, name))
+    out.sort(key=lambda x: x[1])
+    return out[:limit] if limit else out
+
+
+def read_transcript(d):
+    files = glob.glob(os.path.join(d, 'session-*', 'session.v4.jsonl.zstd'))
+    for f in files:
+        p = subprocess.run(['zstd', '-dc', f], capture_output=True)
+        if p.returncode == 0:
+            return p.stdout.decode('utf-8', 'replace')
+    return ''
+
+
+def tally(text):
+    """(prose chars, tool-arg chars, messages, tool calls, multi-call messages)."""
+    prose = args = msgs = calls = multi = textonly = 0
+    for line in text.splitlines():
+        try:
+            o = json.loads(line)
+        except Exception:  # noqa: BLE001 - a torn transcript line is not fatal
+            continue
+        if o.get('type') != 'assistant/message':
+            continue
+        msgs += 1
+        this = 0
+        for part in (o.get('data') or {}).get('message', {}).get('content') or []:
+            if not isinstance(part, dict):
+                continue
+            if part.get('type') == 'text':
+                prose += len(part.get('text') or '')
+            elif part.get('type') == 'tool-call':
+                calls += 1
+                this += 1
+                args += len(part.get('arguments') or '')
+        if this >= 2:
+            multi += 1
+        elif this == 0:
+            textonly += 1
+    return prose, args, msgs, calls, multi, textonly
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument('--arm', default='def15', help='arm name to sample (default def15)')
+    ap.add_argument('--tag', default='', help='restrict to one block tag, e.g. D2B_MIG')
+    ap.add_argument('--limit', type=int, default=40, help='sessions, first in sorted order; 0 = all')
+    ap.add_argument('--json', action='store_true')
+    a = ap.parse_args()
+
+    rows = collect(a.arm, a.tag, a.limit)
+    per = []
+    for d, name in rows:
+        prose, args, msgs, calls, multi, textonly = tally(read_transcript(d))
+        if msgs:
+            per.append({'run': name, 'prose': prose, 'args': args, 'messages': msgs,
+                        'calls': calls, 'multi': multi, 'textonly': textonly,
+                        'share': prose / (prose + args) if prose + args else 0})
+    if not per:
+        print(f'no transcripts for arm={a.arm} tag={a.tag or "*"}; nothing to compute')
+        return 1
+    prose = sum(r['prose'] for r in per)
+    args = sum(r['args'] for r in per)
+    total = prose + args
+    shares = sorted(r['share'] for r in per)
+    summary = {
+        'arm': a.arm, 'tag': a.tag or '*', 'limit': a.limit,
+        'sessions': len(per), 'messages': sum(r['messages'] for r in per),
+        'calls': sum(r['calls'] for r in per),
+        'multiCallMessages': sum(r['multi'] for r in per),
+        'multiCallShare': (sum(r['multi'] for r in per) /
+                           max(sum(r['messages'] for r in per), 1)),
+        'textOnlyMessages': sum(r['textonly'] for r in per),
+        'textOnlyShare': (sum(r['textonly'] for r in per) /
+                          max(sum(r['messages'] for r in per), 1)),
+        'proseChars': prose, 'toolArgChars': args, 'totalChars': total,
+        'share': prose / total if total else None,
+        'medianShare': st.median(shares), 'p10': shares[int(0.1 * (len(shares) - 1))],
+        'p90': shares[int(0.9 * (len(shares) - 1))],
+        'rule': f'sorted by run name, first {a.limit or "all"} of arm={a.arm}' +
+                (f' tag={a.tag}' if a.tag else ''),
+        'command': 'python3 bench/textshare.py' + ('' if a.arm == 'def15' else f' --arm {a.arm}') +
+                   (f' --tag {a.tag}' if a.tag else '') +
+                   ('' if a.limit == 40 else f' --limit {a.limit}'),
+    }
+    if a.json:
+        print(json.dumps(summary, indent=2, ensure_ascii=False))
+        return 0
+    print(f"arm {a.arm} tag {a.tag or '*'}  {summary['rule']}")
+    print(f"  sessions {summary['sessions']}  messages {summary['messages']}  "
+          f"tool calls {summary['calls']}")
+    print(f"  prose {prose:,} chars  tool args {args:,} chars  total {total:,} chars")
+    print(f"  prose share {100 * summary['share']:.2f} %  "
+          f"(median {100 * summary['medianShare']:.2f} %, "
+          f"p10 {100 * summary['p10']:.2f} %, p90 {100 * summary['p90']:.2f} %)")
+    print(f"  command: {summary['command']}")
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())

package/index.js

@@ -97,7 +97,7 @@ const TAG = '[fast-mode]'
  * Shipped identity of this build. Traced into the debug log so a stale install is
  * visible at a glance, and pinned by a test against `package.json`.
  */
-export const VERSION = '1.5.22'
+export const VERSION = '1.5.23'
 const GLOBAL_AGENT = '__global__'
 
 /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dsh-fast-mode",
-  "version": "1.5.22",
+  "version": "1.5.23",
   "description": "DeepSeek Harness fast mode: same thinking effort, fewer round trips. An adaptive parallel tool fan-out, output discipline that never trims the answer, and a cancellable stall guard. Never changes reasoning effort, the model, or the request body. Provider-agnostic, one dependency (@deepseek-ai/schemastery).",
   "repository": {
     "type": "git",
@@ -39,6 +39,10 @@
     "CHANGELOG.md",
     "BENCHMARK.md",
     "bench/block_score.py",
+    "bench/mechanism.py",
+    "bench/textshare.py",
+    "bench/reminders.py",
+    "bench/patchshare.py",
     "docs/",
     "LICENSE"
   ],