@sma1lboy/rove 0.9.281-next.31 → 0.9.282

package/dist/voice/learn.py

@@ -0,0 +1,184 @@
+"""Learning from the user's own fixes: what the recognizer wrote, against what was sent.
+
+The phone bar remembers the draft as a take left it and which spans were dictated. When the message
+goes out, the two are diffed word by word and each fix inside a dictated span that lands on a term
+becomes a `spoken` form of that term in `vocab.json`, so the next take is rewritten before it shows.
+
+Most edits are not lessons, so every gate here errs on learning nothing:
+  - typography: punctuation, spaces and full-width forms never count ("中 GitHub，" = "中GitHub,");
+  - casing: a case-only fix is learned only when it is a term's casing (github -> GitHub, ios -> iOS),
+    never a capital at the start of a word (the -> The);
+  - the fix must land on term-like words (vocab.term_like): "pipe line" -> "pipeline" is the user's
+    style, not a mishearing;
+  - a rewrite teaches nothing: past REWRITE_SHARE of the dictated words changed, or more than
+    MAX_TERMS fixes, and the whole send is ignored.
+"""
+
+from __future__ import annotations
+
+import difflib
+import functools
+import re
+import threading
+import time
+import unicodedata
+from dataclasses import dataclass
+from typing import Any, Callable, TypeVar
+
+import vocab
+
+# A Latin word with its joiners (vocab.WORD, plus full-width forms), or one CJK character.
+TOKEN = re.compile(r"[A-Za-z0-9\uff10-\uff19\uff21-\uff3a\uff41-\uff5a]+(?:[-_.][A-Za-z0-9]+)*|[\u3400-\u9fff]")
+MAX_TERMS = 4
+REWRITE_SHARE = 0.5
+REWRITE_FLOOR = 2.0
+MAX_HEARD_WORDS = 4
+MAX_HEARD_CJK = 6
+MAX_TERM_WORDS = 3
+
+
+@dataclass(frozen=True)
+class Token:
+    text: str
+    start: int
+    end: int
+
+    @property
+    def cjk(self) -> bool:
+        return not self.text.isascii()
+
+
+def tokens(text: str) -> list[Token]:
+    return [Token(unicodedata.normalize("NFKC", m.group()), m.start(), m.end()) for m in TOKEN.finditer(text)]
+
+
+def _inside(token: Token, spans: list[tuple[int, int]]) -> bool:
+    return any(start <= token.start and token.end <= end for start, end in spans)
+
+
+def _term_casing(heard: str, sent: str) -> bool:
+    """`sent` differs from `heard` only in case, and the difference is a term's casing: a capital past
+    the first letter (GitHub, iOS) or an acronym (API). A word merely capitalized (The) is not."""
+    if heard == sent or heard.lower() != sent.lower():
+        return False
+    return any(c.isupper() for c in sent[1:])
+
+
+def _heard_fits(run: list[Token]) -> bool:
+    cjk = sum(t.cjk for t in run)
+    return 0 < len(run) and cjk <= MAX_HEARD_CJK and len(run) - cjk <= MAX_HEARD_WORDS
+
+
+def _term_fits(run: list[Token]) -> bool:
+    if not run or len(run) > MAX_TERM_WORDS or any(t.cjk for t in run):
+        return False
+    return any(vocab.term_like(t.text) for t in run)
+
+
+def _words(run: list[Token]) -> float:
+    """Length in words: a CJK character is about half of one."""
+    return sum(0.5 if t.cjk else 1.0 for t in run)
+
+
+def fixes(base: str, spans: list[tuple[int, int]], sent: str) -> list[dict[str, str]]:
+    """The `{term, spoken}` pairs one send teaches: empty when nothing qualifies or it was a rewrite."""
+    a, b = tokens(base), tokens(sent)
+    dictated = [t for t in a if _inside(t, spans)]
+    if not dictated:
+        return []
+    matcher = difflib.SequenceMatcher(None, [t.text.lower() for t in a], [t.text.lower() for t in b], autojunk=False)
+    found: list[dict[str, str]] = []
+    changed = 0.0
+    for op, i1, i2, j1, j2 in matcher.get_opcodes():
+        heard, said = a[i1:i2], b[j1:j2]
+        if op == "equal":
+            for x, y in zip(heard, said):
+                if _inside(x, spans) and _term_casing(x.text, y.text):
+                    changed += 1
+                    found.append({"term": y.text, "spoken": x.text})
+            continue
+        if op == "insert":
+            # Words typed into the middle of a take change it; words typed before or after it do not.
+            if 0 < i1 < len(a) and _inside(a[i1 - 1], spans) and _inside(a[i1], spans):
+                changed += _words(said)
+            continue
+        inside = [t for t in heard if _inside(t, spans)]
+        if not inside:
+            continue
+        changed += max(_words(inside), _words(said))
+        if op == "replace" and len(inside) == len(heard) and _heard_fits(heard) and _term_fits(said):
+            found.append({"term": sent[said[0].start : said[-1].end], "spoken": base[heard[0].start : heard[-1].end]})
+    # One or two words fixed in a short take is still a fix, not a rewrite.
+    if changed > max(REWRITE_FLOOR, REWRITE_SHARE * _words(dictated)) or len(found) > MAX_TERMS:
+        return []
+    return found
+
+
+_store = threading.Lock()
+F = TypeVar("F", bound=Callable[..., Any])
+
+
+def _locked(fn: F) -> F:
+    """The sidecar serves requests on threads; two sends must not each write over the other's lesson."""
+
+    @functools.wraps(fn)
+    def run(*args: Any, **kwargs: Any) -> Any:
+        with _store:
+            return fn(*args, **kwargs)
+
+    return run  # type: ignore[return-value]
+
+
+@_locked
+def remember(found: list[dict[str, str]]) -> list[dict[str, Any]]:
+    """Adds each fix to `vocab.json`; returns what was added, in the shape `forget` takes back."""
+    if not found:
+        return []
+    data = vocab.load()
+    now = int(time.time() * 1000)
+    added: list[dict[str, Any]] = []
+    for fix in found:
+        term, spoken = fix["term"], fix["spoken"]
+        entry = next((e for e in data["terms"] if e["term"].lower() == term.lower()), None)
+        created = entry is None
+        if entry is None:
+            entry = {"term": term, "spoken": [], "source": "edit", "at": now}
+            data["terms"].append(entry)
+        renamed = entry["term"] if entry["term"] != term else None
+        forms = entry.setdefault("spoken", [])
+        new_form = spoken not in forms
+        if not new_form and renamed is None:
+            continue
+        # The newest spelling the user sent wins: Github stored, GitHub sent.
+        entry["term"] = term
+        if new_form:
+            forms.append(spoken)
+        entry["at"] = now
+        item: dict[str, Any] = {"term": term, "spoken": spoken, "created": created, "newForm": new_form}
+        if renamed:
+            item["renamed"] = renamed
+        added.append(item)
+    vocab.save(data)
+    return added
+
+
+@_locked
+def forget(added: list[dict[str, Any]]) -> int:
+    """Undoes `remember`: a term it created goes; otherwise the spoken form it added, and its old spelling."""
+    data = vocab.load()
+    undone = 0
+    for item in added:
+        term, spoken = str(item.get("term") or ""), str(item.get("spoken") or "")
+        entry = next((e for e in data["terms"] if e["term"] == term), None)
+        if entry is None:
+            continue
+        undone += 1
+        if item.get("created"):
+            data["terms"].remove(entry)
+            continue
+        if item.get("newForm") and spoken in entry.get("spoken", []):
+            entry["spoken"].remove(spoken)
+        if item.get("renamed"):
+            entry["term"] = str(item["renamed"])
+    vocab.save(data)
+    return undone

package/dist/voice/live.py

@@ -0,0 +1,93 @@
+"""Live dictation: a take streamed in chunks while the button is held, re-transcribed as it grows.
+
+The page sends the audio recorded since its last chunk about once a second. Each chunk is appended
+to the take's buffer; when there is enough new audio and the GPU is free, the whole buffer is decoded
+again and the text so far goes back as a partial. The final chunk decodes the buffer once more,
+through the same pipeline as a one-shot take, so a streamed take ends with the same text it would
+have had without streaming.
+
+Partials are a local recognizer's only: a cloud one is billed per request, so its takes are buffered
+and decoded once, at the end. A partial never waits for the GPU; the final one does.
+"""
+
+from __future__ import annotations
+
+import threading
+import time
+from dataclasses import dataclass, field
+from typing import Any
+
+import numpy as np
+
+import engine
+
+# Decoding again for less new audio than this changes little on screen and keeps the GPU busy.
+PARTIAL_EVERY_SEC = 0.8
+PARTIAL_MIN_SEC = 1.0
+# A take whose page went away (a dropped socket, a closed tab) is forgotten after this.
+IDLE_SEC = 120
+MAX_SESSIONS = 8
+MAX_SEC = 240
+
+
+class Gap(Exception):
+    """A chunk arrived out of order: the page should send the whole take instead."""
+
+
+@dataclass
+class Take:
+    chunks: list[np.ndarray] = field(default_factory=list)
+    samples: int = 0
+    seq: int = 0
+    decoded: int = 0
+    conversation: str | None = None
+    draft: str | None = None
+    language: str | None = None
+    touched: float = field(default_factory=time.monotonic)
+
+    def pcm(self) -> np.ndarray:
+        return np.concatenate(self.chunks) if self.chunks else np.zeros(0, dtype=np.float32)
+
+
+_takes: dict[str, Take] = {}
+_lock = threading.Lock()
+
+
+def append(session: str, seq: int, pcm: np.ndarray, first: dict[str, Any]) -> Take:
+    """Adds one chunk; the first chunk (`seq` 0) opens the take and brings its conversation and draft."""
+    now = time.monotonic()
+    with _lock:
+        for key in [k for k, t in _takes.items() if now - t.touched > IDLE_SEC]:
+            del _takes[key]
+        take = _takes.get(session)
+        if seq == 0:
+            if take is None and len(_takes) >= MAX_SESSIONS:
+                del _takes[min(_takes, key=lambda k: _takes[k].touched)]
+            take = _takes[session] = Take(
+                conversation=first.get("conversation"), draft=first.get("context"), language=first.get("language")
+            )
+        if take is None or seq != take.seq:
+            _takes.pop(session, None)
+            raise Gap(f"chunk {seq} of {session} out of order")
+        if take.samples + len(pcm) > MAX_SEC * engine.SAMPLE_RATE:
+            _takes.pop(session, None)
+            raise Gap("take is longer than a dictation")
+        if len(pcm):
+            take.chunks.append(pcm)
+            take.samples += len(pcm)
+        take.seq += 1
+        take.touched = now
+        return take
+
+
+def finish(session: str) -> Take | None:
+    with _lock:
+        return _takes.pop(session, None)
+
+
+def due(take: Take, backend: engine.Backend) -> bool:
+    """Whether this chunk earns a partial decode."""
+    if not isinstance(backend, engine.LocalWhisper):
+        return False
+    new = (take.samples - take.decoded) / engine.SAMPLE_RATE
+    return take.samples / engine.SAMPLE_RATE >= PARTIAL_MIN_SEC and new >= PARTIAL_EVERY_SEC

package/dist/voice/phonetic.py

@@ -0,0 +1,87 @@
+"""Sound-alike correction: a run the recognizer wrote that *sounds* like a vocabulary term becomes the term.
+
+`vocab.correct` fixes spelling (\"Graph, Q, L\", \"Decorfile\"); this fixes hearing: \"Cuban Eats\" for
+Kubernetes, \"JitHub\" for GitHub, \"reserver\" for resolver. Both sides are reduced to a Metaphone key and
+compared; a Chinese run is read as toneless pinyin first, so a transliteration (库伯内特斯) keys the same way.
+
+Guards, each from a false hit in the bench: a run made only of dictionary words is never touched (\"Pot\"
+stays a pot); the keys must start alike and be long enough to mean something; a Chinese run must match
+more closely than English, because any two syllables key to *something*.
+"""
+
+from __future__ import annotations
+
+import re
+from functools import cache
+
+import jellyfish
+from pypinyin import lazy_pinyin
+
+import vocab
+
+ASCII_RATIO = 0.85
+CJK_RATIO = 0.92
+MIN_KEY = 3
+CJK_RUN = re.compile(r"[\u4e00-\u9fff]{2,6}")
+
+
+@cache
+def key(text: str) -> str:
+    if CJK_RUN.fullmatch(text):
+        text = "".join(lazy_pinyin(text))
+    return jellyfish.metaphone(re.sub(r"[^A-Za-z]", "", text))
+
+
+def _best(heard: str, pool: list[tuple[str, str]], ratio: float) -> str | None:
+    k = key(heard)
+    if len(k) < MIN_KEY:
+        return None
+    best, score = None, ratio
+    for term_key, term in pool:
+        # A key much longer than the term's is a term plus a word that was said ("Redis TTL"): leave it.
+        if term_key[0] != k[0] or not 0.75 <= len(term_key) / len(k) <= 1.34:
+            continue
+        s = jellyfish.jaro_winkler_similarity(k, term_key)
+        if s >= score:
+            best, score = term, s
+    return best
+
+
+def correct(text: str, terms: list[str]) -> str:
+    """Replace sound-alike runs (up to two ASCII words, or one Chinese run) with the term they key to."""
+    pool = [(key(t), t) for t in terms if t.isascii() and len(key(t)) >= MIN_KEY]
+    if not pool:
+        return text
+    spelled = {t.lower() for t in terms}
+
+    def ascii_runs(s: str) -> str:
+        tokens = list(vocab.WORD.finditer(s))
+        out, cursor, i = [], 0, 0
+        while i < len(tokens):
+            hit = None
+            for n in (2, 1):
+                run = tokens[i : i + n]
+                if len(run) < n or any(s[a.end() : b.start()] != " " for a, b in zip(run, run[1:])):
+                    continue
+                words = [m.group() for m in run]
+                if not all(w.isalpha() for w in words) or all(vocab.common(w) for w in words):
+                    continue
+                heard = s[run[0].start() : run[-1].end()]
+                if heard.lower() in spelled:
+                    break
+                term = _best(heard, pool, ASCII_RATIO)
+                if term:
+                    hit = (run[0].start(), run[-1].end(), term, n)
+                    break
+            if hit:
+                out.append(s[cursor : hit[0]] + hit[2])
+                cursor = hit[1]
+                i += hit[3]
+            else:
+                i += 1
+        return "".join(out) + s[cursor:]
+
+    def cjk_run(m: re.Match[str]) -> str:
+        return _best(m.group(), pool, CJK_RATIO) or m.group()
+
+    return CJK_RUN.sub(cjk_run, ascii_runs(text))

package/dist/voice/polish.py

@@ -0,0 +1,84 @@
+"""Optional cleanup of a finished take by a chat model: fillers out, self-corrections applied, numbers,
+spacing and lists as written text. Off unless `config.json` has a `polish` block:
+
+  {"polish": {"model": "gpt-6-luna"}}                                      # OpenAI, key in OPENAI_API_KEY
+  {"polish": {"baseURL": "https://openrouter.ai/api/v1", "model": "deepseek/deepseek-v4.1-flash",
+              "apiKeyEnv": "OPENROUTER_API_KEY",
+              "body": {"reasoning": {"effort": "none"}, "provider": {"sort": "throughput"}}}}
+
+Any OpenAI-compatible `/chat/completions` works; `body` is merged into the request for the fields one
+provider needs (gpt-6-luna is fastest with `"reasoning_effort": "none"`, which other models may reject).
+
+`polish_prompt.md` is yap's VoiceInk/Resources/RecommendedPrompt.md (00bbde00) as written. What makes it fit
+messages to a coding agent is its input: <CUSTOM_VOCABULARY> is the take's ranked vocabulary, the
+conversation's identifiers first. Rewording it for agents scored lower on polish_bench.py (gpt-4.1-mini 157-160
+of 192 against 168; even on gpt-6-luna), so the text stays yap's. A polish that fails or runs late returns
+the take as the recognizer left it: cleanup is never worth losing what was said.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import urllib.error
+import urllib.request
+from functools import cache
+from pathlib import Path
+from typing import Any
+
+HERE = Path(__file__).resolve().parent
+TIMEOUT_S = 8
+MAX_VOCAB = 60
+
+
+@cache
+def prompt() -> str:
+    return (HERE / "polish_prompt.md").read_text()
+
+
+def enabled(config: dict[str, Any]) -> bool:
+    block = config.get("polish")
+    return isinstance(block, dict) and bool(block.get("model")) and block.get("enabled", True) is not False
+
+
+def messages(text: str, terms: list[str], system: str | None = None) -> list[dict[str, str]]:
+    vocabulary = f"\n<CUSTOM_VOCABULARY>\n{', '.join(terms[:MAX_VOCAB])}\n</CUSTOM_VOCABULARY>" if terms else ""
+    return [
+        {"role": "system", "content": system or prompt()},
+        {"role": "user", "content": f"{vocabulary}\n<TRANSCRIPT>\n{text}\n</TRANSCRIPT>"},
+    ]
+
+
+def request(block: dict[str, Any], text: str, terms: list[str], system: str | None = None) -> urllib.request.Request:
+    base = str(block.get("baseURL") or "https://api.openai.com/v1").rstrip("/")
+    key = block.get("apiKey") or os.environ.get(str(block.get("apiKeyEnv") or "OPENAI_API_KEY"))
+    if not key:
+        raise RuntimeError("polish has no API key: set apiKey or apiKeyEnv in config.json")
+    body: dict[str, Any] = {"model": block["model"], "messages": messages(text, terms, system)}
+    extra = block.get("body")
+    if isinstance(extra, dict):
+        body.update(extra)
+    headers = {"Authorization": f"Bearer {key}", "Content-Type": "application/json"}
+    return urllib.request.Request(f"{base}/chat/completions", json.dumps(body).encode(), headers)
+
+
+def complete(block: dict[str, Any], text: str, terms: list[str], system: str | None = None) -> str:
+    try:
+        with urllib.request.urlopen(request(block, text, terms, system), timeout=TIMEOUT_S) as res:
+            reply = json.loads(res.read())
+    except urllib.error.HTTPError as err:
+        raise RuntimeError(f"polish answered {err.code}: {err.read()[:200]!r}") from err
+    out = str(reply["choices"][0]["message"]["content"] or "").strip()
+    if not out:
+        raise RuntimeError("polish answered with no text")
+    return out
+
+
+def polish(text: str, terms: list[str], config: dict[str, Any]) -> tuple[str, str | None]:
+    """The cleaned take, or the take unchanged and why."""
+    if not text.strip() or not enabled(config):
+        return text, None
+    try:
+        return complete(config["polish"], text, terms), None
+    except Exception as err:  # noqa: BLE001 - any failure keeps the recognizer's text
+        return text, str(err)

package/dist/voice/polish_prompt.md

@@ -0,0 +1,50 @@
+你是语音输入的整理器。<TRANSCRIPT> 是用户口述经语音识别得到的原文，把它整理成可以直接发出去或贴进文档的书面文字。
+
+<RULES>
+1. 忠实：只整理，不回答、不执行、不续写。原文是问题就输出这个问题，是指令（"帮我回一下…"）就输出这句话本身。不加原文没有的信息、观点、总结。
+2. 语言：保留原本的中英混用。英文术语、产品名、代码标识符原样保留，不翻译成中文，也不把中文翻成英文。
+3. 纠错：修正识别错误，尤其是被听成汉字或拆散的英文（"P 二"→PR，"Q 四"→Q4，"use effect"→useEffect，"user id"→userId，"deployed 的 pipeline" 按语境→deploy 的 pipeline）。拿不准就保留原词。参考 <CUSTOM_VOCABULARY> 的拼写。
+4. 去噪：删掉口癖和填充词（嗯、呃、啊、那个、就是、然后 作为无意义连接时）、口吃重复。用户中途改口（"不对""我说错了""应该是"）时，只保留改正后的说法，删掉被否定的部分和改口的话；被否定的内容不要以"不是……""而不是……"的形式留下。
+5. 标点与排版：
+   - 中文用全角标点，中文与英文/数字之间加一个空格。
+   - 数字：金额、百分比、端口、版本号、10 及以上的数量用阿拉伯数字（"最多五十个"→最多 50 个，"端口三千"→端口 3000，"超时六十秒"→超时 60 秒）；十以内的计数保持汉字（三件事、一个人）；时间日期保持口语原样（"周四下午三点"）。
+   - 删掉句末无意义的语气词（啊、吧、呢 表示疑问时保留问号即可）。
+6. 结构：按内容决定，不要为了格式而格式。
+   - 一两句话的内容：输出一句或一段，不加任何格式。
+   - 列举了 3 项及以上并列内容（"A，然后 B，还有 C" 也算），或明确说了"第一/第二/首先/其次/最后"：用列表，即使每项很短、整段只有一句话也要拆成列表。
+   - 描述了 3 步及以上的操作流程（"先…然后…再…最后…"）：用编号列表，每步一行。
+   - 列表的写法：引导句保留为列表前的一行；有先后顺序用 1. 2. 3.，无顺序用 - ；每项简短，不以句号结尾；只有可以直接粘到终端运行的命令才放进反引号。
+   - 讲了几件不相干的事：分段，段落之间空一行。
+   - 不用标题、不加粗、不用表格、不用 emoji。
+7. 只输出整理后的文字，不要任何前言、解释、引号或代码块。
+</RULES>
+
+<EXAMPLES>
+原文：嗯周末露营要带的东西有帐篷，然后睡袋，还有那个 power bank 记得充满，哦对还有驱蚊的
+输出：
+周末露营要带的东西：
+- 帐篷
+- 睡袋
+- power bank（记得充满）
+- 驱蚊用品
+
+原文：部署的话你先 ssh 到那台机器，然后 git pull 一下，再跑一下 migrate，最后重启那个 service 就行
+输出：
+部署步骤：
+1. ssh 到那台机器
+2. `git pull`
+3. 跑一下 migrate
+4. 重启 service
+
+原文：这个季度的 marketing 预算批了三十万，不对，是四十五万，下周一前报给 finance
+输出：这个季度的 marketing 预算批了 45 万，下周一前报给 finance。
+
+原文：首先把 unit test 补齐，其次把 README 更新一下，最后再 merge 到 main
+输出：
+1. 把 unit test 补齐
+2. 更新 README
+3. merge 到 main
+
+原文：呃你看一下这个 bug 能不能今天修掉
+输出：你看一下这个 bug 能不能今天修掉？
+</EXAMPLES>

package/dist/voice/pyproject.toml

@@ -0,0 +1,28 @@
+# The dictation sidecar and its bench. Not a package: run the scripts through uv, which builds the
+# environment from this file and uv.lock (`uv run --project scripts/voice --extra local scripts/voice/asr_server.py`).
+[project]
+name = "rove-voice"
+version = "0"
+requires-python = ">=3.11"
+dependencies = [
+  "jellyfish",
+  "numpy",
+  "opencc-python-reimplemented",
+  "pypinyin",
+]
+
+[project.optional-dependencies]
+# The local recognizer: Apple Silicon only. A cloud-only setup syncs without it.
+local = ["mlx-whisper; sys_platform == 'darwin' and platform_machine == 'arm64'"]
+
+[tool.uv]
+package = false
+
+[tool.ruff]
+line-length = 120
+
+[tool.ruff.lint]
+select = ["E", "F", "W", "I"]
+
+[tool.ruff.lint.isort]
+known-first-party = ["asr_server", "engine", "fetch_model", "learn", "live", "phonetic", "polish", "vocab"]