@sma1lboy/rove 0.9.281-next.31 → 0.9.281

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. package/dist/cli/chunk-01tghgf3.js +8 -0
  2. package/dist/cli/{chunk-jk4f0xsh.js → chunk-03fv92ya.js} +1 -1
  3. package/dist/cli/{chunk-7aaw17vr.js → chunk-08mb3qss.js} +1 -1
  4. package/dist/cli/{chunk-y7mjhjby.js → chunk-0gbg7bg8.js} +1 -1
  5. package/dist/cli/{chunk-zv0rrhck.js → chunk-0nkk13z9.js} +1 -1
  6. package/dist/cli/{chunk-zp0q931y.js → chunk-14tevt5v.js} +1 -1
  7. package/dist/cli/{chunk-1g6vhgfe.js → chunk-16e6jqq6.js} +1 -1
  8. package/dist/cli/{chunk-3hxm56h6.js → chunk-1a57ejnh.js} +1 -1
  9. package/dist/cli/{chunk-4z1a65bz.js → chunk-32y7f5dr.js} +1 -1
  10. package/dist/cli/{chunk-j54c5a0a.js → chunk-3m0ydze9.js} +1 -1
  11. package/dist/cli/{chunk-12wn9jtf.js → chunk-3pgp6z3k.js} +1 -1
  12. package/dist/cli/{chunk-amq030gb.js → chunk-3qj9yw5c.js} +2 -2
  13. package/dist/cli/chunk-453qpfae.js +2 -0
  14. package/dist/cli/{chunk-gk8efsx0.js → chunk-46qs744w.js} +1 -1
  15. package/dist/cli/{chunk-am5cpb9n.js → chunk-4vkrv4eb.js} +1 -1
  16. package/dist/cli/{chunk-hfvzvj91.js → chunk-4yr1534j.js} +1 -1
  17. package/dist/cli/{chunk-byh32qnx.js → chunk-5swmvt20.js} +1 -1
  18. package/dist/cli/{chunk-j8d9sk5j.js → chunk-6wt08a28.js} +1 -1
  19. package/dist/cli/{chunk-f262ywvg.js → chunk-7911jkq6.js} +1 -1
  20. package/dist/cli/chunk-8ng1756g.js +3 -0
  21. package/dist/cli/{chunk-ebdp0z1d.js → chunk-95kk75wh.js} +1 -1
  22. package/dist/cli/{chunk-6mk6ceb6.js → chunk-9qpea48c.js} +2 -2
  23. package/dist/cli/chunk-9vwb9f7q.js +7 -0
  24. package/dist/cli/{chunk-nqwza8b0.js → chunk-a5xbsvx2.js} +1 -1
  25. package/dist/cli/{chunk-t7dyf50m.js → chunk-ap3fgbbt.js} +1 -1
  26. package/dist/cli/{chunk-5kgq379z.js → chunk-b5fdqynx.js} +1 -1
  27. package/dist/cli/{chunk-x85rcep5.js → chunk-bbq6wc47.js} +1 -1
  28. package/dist/cli/{chunk-np398f2a.js → chunk-bbt217yd.js} +1 -1
  29. package/dist/cli/{chunk-w2tcb3rn.js → chunk-bm6qhcck.js} +1 -1
  30. package/dist/cli/{chunk-txvjg1g9.js → chunk-btxfcc77.js} +3 -3
  31. package/dist/cli/chunk-c9ptxthj.js +2 -0
  32. package/dist/cli/{chunk-zkby63ej.js → chunk-cs410j8b.js} +1 -1
  33. package/dist/cli/{chunk-rvjnafyt.js → chunk-cx78tyg1.js} +1 -1
  34. package/dist/cli/{chunk-4drm07mp.js → chunk-cx8wnz54.js} +1 -1
  35. package/dist/cli/{chunk-t2p22ww7.js → chunk-ejfs22aq.js} +1 -1
  36. package/dist/cli/chunk-f23manyf.js +8 -0
  37. package/dist/cli/{chunk-7nnpp89t.js → chunk-ga82ras9.js} +1 -1
  38. package/dist/cli/{chunk-f7egph8g.js → chunk-gzkes5bw.js} +1 -1
  39. package/dist/cli/{chunk-f0sahrfw.js → chunk-hmy01yg2.js} +16 -14
  40. package/dist/cli/{chunk-3a7x7jg7.js → chunk-hnxs5shy.js} +16 -17
  41. package/dist/cli/{chunk-bw2y7670.js → chunk-j2e7jgh8.js} +1 -1
  42. package/dist/cli/{chunk-qc9sj5wh.js → chunk-j5y7hhc2.js} +1 -1
  43. package/dist/cli/{chunk-qzhfs668.js → chunk-jar50zcp.js} +1 -1
  44. package/dist/cli/{chunk-r98h6b42.js → chunk-k2rmj0v3.js} +2 -2
  45. package/dist/cli/{chunk-bv050ssb.js → chunk-k3h4cpdy.js} +1 -1
  46. package/dist/cli/{chunk-wz307hf6.js → chunk-k9xabf0t.js} +1 -1
  47. package/dist/cli/{chunk-02x0ahxk.js → chunk-kp43j420.js} +1 -1
  48. package/dist/cli/{chunk-6cfs0nqe.js → chunk-kt5ngemc.js} +1 -1
  49. package/dist/cli/chunk-nkgn0bqq.js +2 -0
  50. package/dist/cli/{chunk-fkgemeww.js → chunk-q2nhmaq9.js} +2 -2
  51. package/dist/cli/{chunk-hd1szhba.js → chunk-q6cwy6v5.js} +1 -1
  52. package/dist/cli/{chunk-jkgp816v.js → chunk-q72w3dsx.js} +1 -1
  53. package/dist/cli/{chunk-hmsvt5nj.js → chunk-rqm2kn0c.js} +1 -1
  54. package/dist/cli/{chunk-q6tsx6zc.js → chunk-sm96b61g.js} +1 -1
  55. package/dist/cli/chunk-sqdmngys.js +2 -0
  56. package/dist/cli/{chunk-k2j0tg1h.js → chunk-tgv90744.js} +7 -3
  57. package/dist/cli/{chunk-vbvbwjad.js → chunk-twy9bfps.js} +1 -1
  58. package/dist/cli/{chunk-5g6v6tq5.js → chunk-v3ecphrz.js} +1 -1
  59. package/dist/cli/{chunk-mjzjah9d.js → chunk-vtay379p.js} +1 -1
  60. package/dist/cli/chunk-vx44yxra.js +3 -0
  61. package/dist/cli/{chunk-w003rpr2.js → chunk-w3kvkax7.js} +1 -1
  62. package/dist/cli/{chunk-s1xn6wav.js → chunk-w6ft4djp.js} +1 -1
  63. package/dist/cli/{chunk-8hernv9y.js → chunk-w8dmmnys.js} +1 -1
  64. package/dist/cli/{chunk-9f34p80j.js → chunk-ww0zctb7.js} +1 -1
  65. package/dist/cli/{chunk-qwwt4ch2.js → chunk-x119xqa6.js} +1 -1
  66. package/dist/cli/{chunk-nrx8pf9p.js → chunk-x35dfb1x.js} +1 -1
  67. package/dist/cli/{chunk-ggjmcqpt.js → chunk-x94e0rw4.js} +3 -3
  68. package/dist/cli/chunk-xaam0d8r.js +4 -0
  69. package/dist/cli/{chunk-azt0c2b5.js → chunk-xekk4k54.js} +1 -1
  70. package/dist/cli/{chunk-mqvkd2g1.js → chunk-xqvyzv62.js} +1 -1
  71. package/dist/cli/chunk-xxz3np4w.js +2 -0
  72. package/dist/cli/{chunk-p11emdb2.js → chunk-y07cp9mb.js} +1 -1
  73. package/dist/cli/{chunk-5dq0n3h7.js → chunk-y2v1x23z.js} +1 -1
  74. package/dist/cli/chunk-yn01fet6.js +3 -0
  75. package/dist/cli/{chunk-cakyme4s.js → chunk-z8rvxvx5.js} +1 -1
  76. package/dist/cli/chunk-zr058kbe.js +3 -0
  77. package/dist/cli/index.js +3 -3
  78. package/dist/cli/rove.js +3 -3
  79. package/dist/skills/rove/SKILL.md +4 -2
  80. package/dist/skills/rove/references/api-flags.md +7 -4
  81. package/dist/voice/asr_server.py +318 -0
  82. package/dist/voice/engine.py +174 -0
  83. package/dist/voice/fetch_model.py +30 -0
  84. package/dist/voice/learn.py +184 -0
  85. package/dist/voice/live.py +93 -0
  86. package/dist/voice/phonetic.py +87 -0
  87. package/dist/voice/polish.py +84 -0
  88. package/dist/voice/polish_prompt.md +50 -0
  89. package/dist/voice/pyproject.toml +28 -0
  90. package/dist/voice/uv.lock +815 -0
  91. package/dist/voice/vocab.py +193 -0
  92. package/dist/web/assets/index-Bo92_NP5.css +1 -0
  93. package/dist/web/assets/index-Ct8dDnCc.js +154 -0
  94. package/dist/web/index.html +3 -3
  95. package/dist/web/relay/index.html +5 -5
  96. package/package.json +4 -3
  97. package/dist/cli/chunk-0g1h35mr.js +0 -2
  98. package/dist/cli/chunk-1wwqf6kx.js +0 -3
  99. package/dist/cli/chunk-26ydgwym.js +0 -2
  100. package/dist/cli/chunk-2pfh3zm0.js +0 -33
  101. package/dist/cli/chunk-2wrt80ft.js +0 -3
  102. package/dist/cli/chunk-7fya3y0n.js +0 -2
  103. package/dist/cli/chunk-d2wndk4q.js +0 -2
  104. package/dist/cli/chunk-mey3vksj.js +0 -2
  105. package/dist/cli/chunk-nv8eyjep.js +0 -3
  106. package/dist/cli/chunk-ve76p2mx.js +0 -3
  107. package/dist/cli/chunk-wfgrn8p2.js +0 -4
  108. package/dist/cli/chunk-xmjknfy2.js +0 -7
  109. package/dist/web/assets/index-Dc0Khk0q.js +0 -149
  110. package/dist/web/assets/index-T8cCSdnk.css +0 -1
@@ -0,0 +1,174 @@
1
+ """The recognizer behind the sidecar: local Whisper by default, or any OpenAI-compatible
2
+ `/audio/transcriptions` endpoint when `~/.rove/voice/config.json` names one.
3
+
4
+ config.json, all keys optional:
5
+ {"backend": "local", "model": "mlx-community/whisper-large-v3-turbo"} # faster, see LOCAL_MODEL
6
+ {"backend": "cloud", "baseURL": "https://openrouter.ai/api/v1", "model": "microsoft/mai-transcribe-2",
7
+ "apiKeyEnv": "OPENROUTER_API_KEY"} # or "apiKey": "..."; the key never leaves this Mac
8
+ {"simplified": false} # keep Traditional characters as recognized
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ import base64
14
+ import io
15
+ import json
16
+ import os
17
+ import threading
18
+ import urllib.error
19
+ import urllib.request
20
+ import uuid
21
+ import wave
22
+ from functools import cache
23
+ from pathlib import Path
24
+ from typing import Any, Protocol
25
+
26
+ import numpy as np
27
+
28
+ HOME = Path(os.environ.get("ROVE_VOICE_HOME") or Path.home() / ".rove" / "voice")
29
+ SAMPLE_RATE = 16_000
30
+ # Code-switch bench (bench.py, 82 terms): large-v3 68, 74 with the conversation's terms, p50 1.8 s;
31
+ # turbo 63, 69 at best, p50 1.0 s. Turbo is the faster pick in config.json; it loses most when context is thin.
32
+ LOCAL_MODEL = "mlx-community/whisper-large-v3-mlx"
33
+ # Without this hint large-v3 often answers Mandarin in Traditional script; English is unaffected.
34
+ ZH_GUARD = "以下是普通话的句子。"
35
+ CLOUD_TIMEOUT_S = 30
36
+ # One decode at a time on the GPU: the server answers on threads, and a live take's partials run
37
+ # beside other requests. Reentrant so a caller can hold it across a whole dictation (live.py).
38
+ GPU = threading.RLock()
39
+
40
+
41
+ class Backend(Protocol):
42
+ name: str
43
+ max_terms: int
44
+
45
+ def transcribe(self, pcm: np.ndarray, terms: list[str], language: str | None) -> tuple[str, str | None]: ...
46
+
47
+
48
+ def load_config() -> dict[str, Any]:
49
+ path = HOME / "config.json"
50
+ if not path.exists():
51
+ return {}
52
+ return json.loads(path.read_text())
53
+
54
+
55
+ class LocalWhisper:
56
+ """mlx-whisper on the Mac's GPU. The terms ride in the prompt, which Whisper reads as preceding
57
+ text: spelled-right terms there are the cheapest bias a decoder without a boost list accepts."""
58
+
59
+ max_terms = 60
60
+
61
+ def __init__(self, model: str) -> None:
62
+ self.model = model
63
+ self.name = f"local:{model}"
64
+
65
+ def transcribe(self, pcm: np.ndarray, terms: list[str], language: str | None) -> tuple[str, str | None]:
66
+ import mlx_whisper
67
+
68
+ prompt = "\n".join([ZH_GUARD, ", ".join(terms)]) if terms else ZH_GUARD
69
+ with GPU:
70
+ result = mlx_whisper.transcribe(
71
+ pcm,
72
+ path_or_hf_repo=self.model,
73
+ initial_prompt=prompt,
74
+ language=language or None,
75
+ word_timestamps=False,
76
+ condition_on_previous_text=False,
77
+ verbose=None,
78
+ )
79
+ return str(result.get("text") or "").strip(), result.get("language")
80
+
81
+
82
+ def provider_options(model: str, terms: list[str]) -> dict[str, Any] | None:
83
+ """OpenRouter routes one body to many providers, and each reads the vocabulary from its own field:
84
+ mai-transcribe-2 ignores `prompt` and goes 59 → 68 of 82 terms with `azure.phraseList`."""
85
+ if model.startswith("microsoft/mai-transcribe"):
86
+ return {"azure": {"phraseList": {"phrases": terms}}}
87
+ if model.startswith("openai/gpt-4o"):
88
+ return {"openai": {"prompt": ", ".join(terms)}}
89
+ if model == "openai/whisper-large-v3":
90
+ return {slug: {"prompt": ", ".join(terms)} for slug in ("groq", "deepinfra/us", "together")}
91
+ return None # gemini rejects a prompt outright
92
+
93
+
94
+ def wav_bytes(pcm: np.ndarray) -> bytes:
95
+ out = io.BytesIO()
96
+ with wave.open(out, "wb") as wav:
97
+ wav.setnchannels(1)
98
+ wav.setsampwidth(2)
99
+ wav.setframerate(SAMPLE_RATE)
100
+ wav.writeframes((np.clip(pcm, -1, 1) * 32767).astype("<i2").tobytes())
101
+ return out.getvalue()
102
+
103
+
104
+ class CloudASR:
105
+ """One OpenAI-compatible transcription endpoint. OpenRouter gets its JSON body so the
106
+ vocabulary reaches the provider's own field; everything else gets the standard multipart
107
+ form with the vocabulary as `prompt`."""
108
+
109
+ max_terms = 100
110
+
111
+ def __init__(self, config: dict[str, Any]) -> None:
112
+ self.base = str(config.get("baseURL") or "https://openrouter.ai/api/v1").rstrip("/")
113
+ self.model = str(config.get("model") or "microsoft/mai-transcribe-2")
114
+ key = config.get("apiKey") or os.environ.get(str(config.get("apiKeyEnv") or "OPENROUTER_API_KEY"))
115
+ if not key:
116
+ raise RuntimeError("cloud backend has no API key: set apiKey or apiKeyEnv in config.json")
117
+ self.key = str(key)
118
+ self.name = f"cloud:{self.model}"
119
+
120
+ def request(self, audio: bytes, terms: list[str], language: str | None) -> urllib.request.Request:
121
+ url = f"{self.base}/audio/transcriptions"
122
+ auth = {"Authorization": f"Bearer {self.key}"}
123
+ if "openrouter.ai" in self.base:
124
+ body: dict[str, Any] = {
125
+ "model": self.model,
126
+ "input_audio": {"data": base64.b64encode(audio).decode(), "format": "wav"},
127
+ }
128
+ if language:
129
+ body["language"] = language
130
+ options = provider_options(self.model, terms) if terms else None
131
+ if options:
132
+ body["provider"] = {"options": options}
133
+ return urllib.request.Request(url, json.dumps(body).encode(), {**auth, "Content-Type": "application/json"})
134
+ fields = {"model": self.model, **({"language": language} if language else {})}
135
+ if terms:
136
+ fields["prompt"] = ", ".join(terms)
137
+ boundary = uuid.uuid4().hex
138
+ parts = [
139
+ f'--{boundary}\r\nContent-Disposition: form-data; name="file"; filename="audio.wav"\r\n'
140
+ "Content-Type: audio/wav\r\n\r\n".encode()
141
+ + audio
142
+ + b"\r\n"
143
+ ]
144
+ for name, value in fields.items():
145
+ parts.append(f'--{boundary}\r\nContent-Disposition: form-data; name="{name}"\r\n\r\n{value}\r\n'.encode())
146
+ parts.append(f"--{boundary}--\r\n".encode())
147
+ headers = {**auth, "Content-Type": f"multipart/form-data; boundary={boundary}"}
148
+ return urllib.request.Request(url, b"".join(parts), headers)
149
+
150
+ def transcribe(self, pcm: np.ndarray, terms: list[str], language: str | None) -> tuple[str, str | None]:
151
+ try:
152
+ with urllib.request.urlopen(self.request(wav_bytes(pcm), terms, language), timeout=CLOUD_TIMEOUT_S) as res:
153
+ body = json.loads(res.read())
154
+ except urllib.error.HTTPError as err:
155
+ raise RuntimeError(f"{self.name} answered {err.code}: {err.read()[:200]!r}") from err
156
+ return str(body.get("text") or "").strip(), body.get("language")
157
+
158
+
159
+ def backend(config: dict[str, Any]) -> Backend:
160
+ if config.get("backend") == "cloud":
161
+ return CloudASR(config)
162
+ return LocalWhisper(os.environ.get("ROVE_VOICE_MODEL") or str(config.get("model") or LOCAL_MODEL))
163
+
164
+
165
+ @cache
166
+ def _t2s() -> Any:
167
+ from opencc import OpenCC
168
+
169
+ return OpenCC("t2s")
170
+
171
+
172
+ def simplified(text: str) -> str:
173
+ """Whisper drifts into Traditional characters on some Mandarin clips; English passes through."""
174
+ return _t2s().convert(text) if any("一" <= c <= "鿿" for c in text) else text
@@ -0,0 +1,30 @@
1
+ """Download the local recognizer once, so the sidecar's first start does not wait on the network.
2
+
3
+ Run by the bridge's install job (`rove-bridge/src/ops/voice-setup.ts`) after `uv sync --extra local`.
4
+ Prints one progress line per file; writes `~/.rove/voice/installed.json` when the model is complete.
5
+ A rerun resumes: the Hugging Face cache keeps what already arrived.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import json
11
+ import sys
12
+ import time
13
+
14
+ import engine
15
+
16
+
17
+ def main() -> None:
18
+ from huggingface_hub import snapshot_download
19
+
20
+ config = engine.load_config()
21
+ model = str(config.get("model") or engine.LOCAL_MODEL) if config.get("backend") != "cloud" else engine.LOCAL_MODEL
22
+ print(f"downloading {model}", flush=True)
23
+ path = snapshot_download(model)
24
+ engine.HOME.mkdir(parents=True, exist_ok=True)
25
+ (engine.HOME / "installed.json").write_text(json.dumps({"model": model, "path": path, "at": int(time.time())}))
26
+ print(f"installed {model}", flush=True)
27
+
28
+
29
+ if __name__ == "__main__":
30
+ sys.exit(main())
@@ -0,0 +1,184 @@
1
+ """Learning from the user's own fixes: what the recognizer wrote, against what was sent.
2
+
3
+ The phone bar remembers the draft as a take left it and which spans were dictated. When the message
4
+ goes out, the two are diffed word by word and each fix inside a dictated span that lands on a term
5
+ becomes a `spoken` form of that term in `vocab.json`, so the next take is rewritten before it shows.
6
+
7
+ Most edits are not lessons, so every gate here errs on learning nothing:
8
+ - typography: punctuation, spaces and full-width forms never count ("中 GitHub," = "中GitHub,");
9
+ - casing: a case-only fix is learned only when it is a term's casing (github -> GitHub, ios -> iOS),
10
+ never a capital at the start of a word (the -> The);
11
+ - the fix must land on term-like words (vocab.term_like): "pipe line" -> "pipeline" is the user's
12
+ style, not a mishearing;
13
+ - a rewrite teaches nothing: past REWRITE_SHARE of the dictated words changed, or more than
14
+ MAX_TERMS fixes, and the whole send is ignored.
15
+ """
16
+
17
+ from __future__ import annotations
18
+
19
+ import difflib
20
+ import functools
21
+ import re
22
+ import threading
23
+ import time
24
+ import unicodedata
25
+ from dataclasses import dataclass
26
+ from typing import Any, Callable, TypeVar
27
+
28
+ import vocab
29
+
30
+ # A Latin word with its joiners (vocab.WORD, plus full-width forms), or one CJK character.
31
+ TOKEN = re.compile(r"[A-Za-z0-9\uff10-\uff19\uff21-\uff3a\uff41-\uff5a]+(?:[-_.][A-Za-z0-9]+)*|[\u3400-\u9fff]")
32
+ MAX_TERMS = 4
33
+ REWRITE_SHARE = 0.5
34
+ REWRITE_FLOOR = 2.0
35
+ MAX_HEARD_WORDS = 4
36
+ MAX_HEARD_CJK = 6
37
+ MAX_TERM_WORDS = 3
38
+
39
+
40
+ @dataclass(frozen=True)
41
+ class Token:
42
+ text: str
43
+ start: int
44
+ end: int
45
+
46
+ @property
47
+ def cjk(self) -> bool:
48
+ return not self.text.isascii()
49
+
50
+
51
+ def tokens(text: str) -> list[Token]:
52
+ return [Token(unicodedata.normalize("NFKC", m.group()), m.start(), m.end()) for m in TOKEN.finditer(text)]
53
+
54
+
55
+ def _inside(token: Token, spans: list[tuple[int, int]]) -> bool:
56
+ return any(start <= token.start and token.end <= end for start, end in spans)
57
+
58
+
59
+ def _term_casing(heard: str, sent: str) -> bool:
60
+ """`sent` differs from `heard` only in case, and the difference is a term's casing: a capital past
61
+ the first letter (GitHub, iOS) or an acronym (API). A word merely capitalized (The) is not."""
62
+ if heard == sent or heard.lower() != sent.lower():
63
+ return False
64
+ return any(c.isupper() for c in sent[1:])
65
+
66
+
67
+ def _heard_fits(run: list[Token]) -> bool:
68
+ cjk = sum(t.cjk for t in run)
69
+ return 0 < len(run) and cjk <= MAX_HEARD_CJK and len(run) - cjk <= MAX_HEARD_WORDS
70
+
71
+
72
+ def _term_fits(run: list[Token]) -> bool:
73
+ if not run or len(run) > MAX_TERM_WORDS or any(t.cjk for t in run):
74
+ return False
75
+ return any(vocab.term_like(t.text) for t in run)
76
+
77
+
78
+ def _words(run: list[Token]) -> float:
79
+ """Length in words: a CJK character is about half of one."""
80
+ return sum(0.5 if t.cjk else 1.0 for t in run)
81
+
82
+
83
+ def fixes(base: str, spans: list[tuple[int, int]], sent: str) -> list[dict[str, str]]:
84
+ """The `{term, spoken}` pairs one send teaches: empty when nothing qualifies or it was a rewrite."""
85
+ a, b = tokens(base), tokens(sent)
86
+ dictated = [t for t in a if _inside(t, spans)]
87
+ if not dictated:
88
+ return []
89
+ matcher = difflib.SequenceMatcher(None, [t.text.lower() for t in a], [t.text.lower() for t in b], autojunk=False)
90
+ found: list[dict[str, str]] = []
91
+ changed = 0.0
92
+ for op, i1, i2, j1, j2 in matcher.get_opcodes():
93
+ heard, said = a[i1:i2], b[j1:j2]
94
+ if op == "equal":
95
+ for x, y in zip(heard, said):
96
+ if _inside(x, spans) and _term_casing(x.text, y.text):
97
+ changed += 1
98
+ found.append({"term": y.text, "spoken": x.text})
99
+ continue
100
+ if op == "insert":
101
+ # Words typed into the middle of a take change it; words typed before or after it do not.
102
+ if 0 < i1 < len(a) and _inside(a[i1 - 1], spans) and _inside(a[i1], spans):
103
+ changed += _words(said)
104
+ continue
105
+ inside = [t for t in heard if _inside(t, spans)]
106
+ if not inside:
107
+ continue
108
+ changed += max(_words(inside), _words(said))
109
+ if op == "replace" and len(inside) == len(heard) and _heard_fits(heard) and _term_fits(said):
110
+ found.append({"term": sent[said[0].start : said[-1].end], "spoken": base[heard[0].start : heard[-1].end]})
111
+ # One or two words fixed in a short take is still a fix, not a rewrite.
112
+ if changed > max(REWRITE_FLOOR, REWRITE_SHARE * _words(dictated)) or len(found) > MAX_TERMS:
113
+ return []
114
+ return found
115
+
116
+
117
+ _store = threading.Lock()
118
+ F = TypeVar("F", bound=Callable[..., Any])
119
+
120
+
121
+ def _locked(fn: F) -> F:
122
+ """The sidecar serves requests on threads; two sends must not each write over the other's lesson."""
123
+
124
+ @functools.wraps(fn)
125
+ def run(*args: Any, **kwargs: Any) -> Any:
126
+ with _store:
127
+ return fn(*args, **kwargs)
128
+
129
+ return run # type: ignore[return-value]
130
+
131
+
132
+ @_locked
133
+ def remember(found: list[dict[str, str]]) -> list[dict[str, Any]]:
134
+ """Adds each fix to `vocab.json`; returns what was added, in the shape `forget` takes back."""
135
+ if not found:
136
+ return []
137
+ data = vocab.load()
138
+ now = int(time.time() * 1000)
139
+ added: list[dict[str, Any]] = []
140
+ for fix in found:
141
+ term, spoken = fix["term"], fix["spoken"]
142
+ entry = next((e for e in data["terms"] if e["term"].lower() == term.lower()), None)
143
+ created = entry is None
144
+ if entry is None:
145
+ entry = {"term": term, "spoken": [], "source": "edit", "at": now}
146
+ data["terms"].append(entry)
147
+ renamed = entry["term"] if entry["term"] != term else None
148
+ forms = entry.setdefault("spoken", [])
149
+ new_form = spoken not in forms
150
+ if not new_form and renamed is None:
151
+ continue
152
+ # The newest spelling the user sent wins: Github stored, GitHub sent.
153
+ entry["term"] = term
154
+ if new_form:
155
+ forms.append(spoken)
156
+ entry["at"] = now
157
+ item: dict[str, Any] = {"term": term, "spoken": spoken, "created": created, "newForm": new_form}
158
+ if renamed:
159
+ item["renamed"] = renamed
160
+ added.append(item)
161
+ vocab.save(data)
162
+ return added
163
+
164
+
165
+ @_locked
166
+ def forget(added: list[dict[str, Any]]) -> int:
167
+ """Undoes `remember`: a term it created goes; otherwise the spoken form it added, and its old spelling."""
168
+ data = vocab.load()
169
+ undone = 0
170
+ for item in added:
171
+ term, spoken = str(item.get("term") or ""), str(item.get("spoken") or "")
172
+ entry = next((e for e in data["terms"] if e["term"] == term), None)
173
+ if entry is None:
174
+ continue
175
+ undone += 1
176
+ if item.get("created"):
177
+ data["terms"].remove(entry)
178
+ continue
179
+ if item.get("newForm") and spoken in entry.get("spoken", []):
180
+ entry["spoken"].remove(spoken)
181
+ if item.get("renamed"):
182
+ entry["term"] = str(item["renamed"])
183
+ vocab.save(data)
184
+ return undone
@@ -0,0 +1,93 @@
1
+ """Live dictation: a take streamed in chunks while the button is held, re-transcribed as it grows.
2
+
3
+ The page sends the audio recorded since its last chunk about once a second. Each chunk is appended
4
+ to the take's buffer; when there is enough new audio and the GPU is free, the whole buffer is decoded
5
+ again and the text so far goes back as a partial. The final chunk decodes the buffer once more,
6
+ through the same pipeline as a one-shot take, so a streamed take ends with the same text it would
7
+ have had without streaming.
8
+
9
+ Partials are a local recognizer's only: a cloud one is billed per request, so its takes are buffered
10
+ and decoded once, at the end. A partial never waits for the GPU; the final one does.
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ import threading
16
+ import time
17
+ from dataclasses import dataclass, field
18
+ from typing import Any
19
+
20
+ import numpy as np
21
+
22
+ import engine
23
+
24
+ # Decoding again for less new audio than this changes little on screen and keeps the GPU busy.
25
+ PARTIAL_EVERY_SEC = 0.8
26
+ PARTIAL_MIN_SEC = 1.0
27
+ # A take whose page went away (a dropped socket, a closed tab) is forgotten after this.
28
+ IDLE_SEC = 120
29
+ MAX_SESSIONS = 8
30
+ MAX_SEC = 240
31
+
32
+
33
+ class Gap(Exception):
34
+ """A chunk arrived out of order: the page should send the whole take instead."""
35
+
36
+
37
+ @dataclass
38
+ class Take:
39
+ chunks: list[np.ndarray] = field(default_factory=list)
40
+ samples: int = 0
41
+ seq: int = 0
42
+ decoded: int = 0
43
+ conversation: str | None = None
44
+ draft: str | None = None
45
+ language: str | None = None
46
+ touched: float = field(default_factory=time.monotonic)
47
+
48
+ def pcm(self) -> np.ndarray:
49
+ return np.concatenate(self.chunks) if self.chunks else np.zeros(0, dtype=np.float32)
50
+
51
+
52
+ _takes: dict[str, Take] = {}
53
+ _lock = threading.Lock()
54
+
55
+
56
+ def append(session: str, seq: int, pcm: np.ndarray, first: dict[str, Any]) -> Take:
57
+ """Adds one chunk; the first chunk (`seq` 0) opens the take and brings its conversation and draft."""
58
+ now = time.monotonic()
59
+ with _lock:
60
+ for key in [k for k, t in _takes.items() if now - t.touched > IDLE_SEC]:
61
+ del _takes[key]
62
+ take = _takes.get(session)
63
+ if seq == 0:
64
+ if take is None and len(_takes) >= MAX_SESSIONS:
65
+ del _takes[min(_takes, key=lambda k: _takes[k].touched)]
66
+ take = _takes[session] = Take(
67
+ conversation=first.get("conversation"), draft=first.get("context"), language=first.get("language")
68
+ )
69
+ if take is None or seq != take.seq:
70
+ _takes.pop(session, None)
71
+ raise Gap(f"chunk {seq} of {session} out of order")
72
+ if take.samples + len(pcm) > MAX_SEC * engine.SAMPLE_RATE:
73
+ _takes.pop(session, None)
74
+ raise Gap("take is longer than a dictation")
75
+ if len(pcm):
76
+ take.chunks.append(pcm)
77
+ take.samples += len(pcm)
78
+ take.seq += 1
79
+ take.touched = now
80
+ return take
81
+
82
+
83
+ def finish(session: str) -> Take | None:
84
+ with _lock:
85
+ return _takes.pop(session, None)
86
+
87
+
88
+ def due(take: Take, backend: engine.Backend) -> bool:
89
+ """Whether this chunk earns a partial decode."""
90
+ if not isinstance(backend, engine.LocalWhisper):
91
+ return False
92
+ new = (take.samples - take.decoded) / engine.SAMPLE_RATE
93
+ return take.samples / engine.SAMPLE_RATE >= PARTIAL_MIN_SEC and new >= PARTIAL_EVERY_SEC
@@ -0,0 +1,87 @@
1
+ """Sound-alike correction: a run the recognizer wrote that *sounds* like a vocabulary term becomes the term.
2
+
3
+ `vocab.correct` fixes spelling (\"Graph, Q, L\", \"Decorfile\"); this fixes hearing: \"Cuban Eats\" for
4
+ Kubernetes, \"JitHub\" for GitHub, \"reserver\" for resolver. Both sides are reduced to a Metaphone key and
5
+ compared; a Chinese run is read as toneless pinyin first, so a transliteration (库伯内特斯) keys the same way.
6
+
7
+ Guards, each from a false hit in the bench: a run made only of dictionary words is never touched (\"Pot\"
8
+ stays a pot); the keys must start alike and be long enough to mean something; a Chinese run must match
9
+ more closely than English, because any two syllables key to *something*.
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ import re
15
+ from functools import cache
16
+
17
+ import jellyfish
18
+ from pypinyin import lazy_pinyin
19
+
20
+ import vocab
21
+
22
+ ASCII_RATIO = 0.85
23
+ CJK_RATIO = 0.92
24
+ MIN_KEY = 3
25
+ CJK_RUN = re.compile(r"[\u4e00-\u9fff]{2,6}")
26
+
27
+
28
+ @cache
29
+ def key(text: str) -> str:
30
+ if CJK_RUN.fullmatch(text):
31
+ text = "".join(lazy_pinyin(text))
32
+ return jellyfish.metaphone(re.sub(r"[^A-Za-z]", "", text))
33
+
34
+
35
+ def _best(heard: str, pool: list[tuple[str, str]], ratio: float) -> str | None:
36
+ k = key(heard)
37
+ if len(k) < MIN_KEY:
38
+ return None
39
+ best, score = None, ratio
40
+ for term_key, term in pool:
41
+ # A key much longer than the term's is a term plus a word that was said ("Redis TTL"): leave it.
42
+ if term_key[0] != k[0] or not 0.75 <= len(term_key) / len(k) <= 1.34:
43
+ continue
44
+ s = jellyfish.jaro_winkler_similarity(k, term_key)
45
+ if s >= score:
46
+ best, score = term, s
47
+ return best
48
+
49
+
50
+ def correct(text: str, terms: list[str]) -> str:
51
+ """Replace sound-alike runs (up to two ASCII words, or one Chinese run) with the term they key to."""
52
+ pool = [(key(t), t) for t in terms if t.isascii() and len(key(t)) >= MIN_KEY]
53
+ if not pool:
54
+ return text
55
+ spelled = {t.lower() for t in terms}
56
+
57
+ def ascii_runs(s: str) -> str:
58
+ tokens = list(vocab.WORD.finditer(s))
59
+ out, cursor, i = [], 0, 0
60
+ while i < len(tokens):
61
+ hit = None
62
+ for n in (2, 1):
63
+ run = tokens[i : i + n]
64
+ if len(run) < n or any(s[a.end() : b.start()] != " " for a, b in zip(run, run[1:])):
65
+ continue
66
+ words = [m.group() for m in run]
67
+ if not all(w.isalpha() for w in words) or all(vocab.common(w) for w in words):
68
+ continue
69
+ heard = s[run[0].start() : run[-1].end()]
70
+ if heard.lower() in spelled:
71
+ break
72
+ term = _best(heard, pool, ASCII_RATIO)
73
+ if term:
74
+ hit = (run[0].start(), run[-1].end(), term, n)
75
+ break
76
+ if hit:
77
+ out.append(s[cursor : hit[0]] + hit[2])
78
+ cursor = hit[1]
79
+ i += hit[3]
80
+ else:
81
+ i += 1
82
+ return "".join(out) + s[cursor:]
83
+
84
+ def cjk_run(m: re.Match[str]) -> str:
85
+ return _best(m.group(), pool, CJK_RATIO) or m.group()
86
+
87
+ return CJK_RUN.sub(cjk_run, ascii_runs(text))
@@ -0,0 +1,84 @@
1
+ """Optional cleanup of a finished take by a chat model: fillers out, self-corrections applied, numbers,
2
+ spacing and lists as written text. Off unless `config.json` has a `polish` block:
3
+
4
+ {"polish": {"model": "gpt-6-luna"}} # OpenAI, key in OPENAI_API_KEY
5
+ {"polish": {"baseURL": "https://openrouter.ai/api/v1", "model": "deepseek/deepseek-v4.1-flash",
6
+ "apiKeyEnv": "OPENROUTER_API_KEY",
7
+ "body": {"reasoning": {"effort": "none"}, "provider": {"sort": "throughput"}}}}
8
+
9
+ Any OpenAI-compatible `/chat/completions` works; `body` is merged into the request for the fields one
10
+ provider needs (gpt-6-luna is fastest with `"reasoning_effort": "none"`, which other models may reject).
11
+
12
+ `polish_prompt.md` is yap's VoiceInk/Resources/RecommendedPrompt.md (00bbde00) as written. What makes it fit
13
+ messages to a coding agent is its input: <CUSTOM_VOCABULARY> is the take's ranked vocabulary, the
14
+ conversation's identifiers first. Rewording it for agents scored lower on polish_bench.py (gpt-4.1-mini 157-160
15
+ of 192 against 168; even on gpt-6-luna), so the text stays yap's. A polish that fails or runs late returns
16
+ the take as the recognizer left it: cleanup is never worth losing what was said.
17
+ """
18
+
19
+ from __future__ import annotations
20
+
21
+ import json
22
+ import os
23
+ import urllib.error
24
+ import urllib.request
25
+ from functools import cache
26
+ from pathlib import Path
27
+ from typing import Any
28
+
29
+ HERE = Path(__file__).resolve().parent
30
+ TIMEOUT_S = 8
31
+ MAX_VOCAB = 60
32
+
33
+
34
+ @cache
35
+ def prompt() -> str:
36
+ return (HERE / "polish_prompt.md").read_text()
37
+
38
+
39
+ def enabled(config: dict[str, Any]) -> bool:
40
+ block = config.get("polish")
41
+ return isinstance(block, dict) and bool(block.get("model")) and block.get("enabled", True) is not False
42
+
43
+
44
+ def messages(text: str, terms: list[str], system: str | None = None) -> list[dict[str, str]]:
45
+ vocabulary = f"\n<CUSTOM_VOCABULARY>\n{', '.join(terms[:MAX_VOCAB])}\n</CUSTOM_VOCABULARY>" if terms else ""
46
+ return [
47
+ {"role": "system", "content": system or prompt()},
48
+ {"role": "user", "content": f"{vocabulary}\n<TRANSCRIPT>\n{text}\n</TRANSCRIPT>"},
49
+ ]
50
+
51
+
52
+ def request(block: dict[str, Any], text: str, terms: list[str], system: str | None = None) -> urllib.request.Request:
53
+ base = str(block.get("baseURL") or "https://api.openai.com/v1").rstrip("/")
54
+ key = block.get("apiKey") or os.environ.get(str(block.get("apiKeyEnv") or "OPENAI_API_KEY"))
55
+ if not key:
56
+ raise RuntimeError("polish has no API key: set apiKey or apiKeyEnv in config.json")
57
+ body: dict[str, Any] = {"model": block["model"], "messages": messages(text, terms, system)}
58
+ extra = block.get("body")
59
+ if isinstance(extra, dict):
60
+ body.update(extra)
61
+ headers = {"Authorization": f"Bearer {key}", "Content-Type": "application/json"}
62
+ return urllib.request.Request(f"{base}/chat/completions", json.dumps(body).encode(), headers)
63
+
64
+
65
+ def complete(block: dict[str, Any], text: str, terms: list[str], system: str | None = None) -> str:
66
+ try:
67
+ with urllib.request.urlopen(request(block, text, terms, system), timeout=TIMEOUT_S) as res:
68
+ reply = json.loads(res.read())
69
+ except urllib.error.HTTPError as err:
70
+ raise RuntimeError(f"polish answered {err.code}: {err.read()[:200]!r}") from err
71
+ out = str(reply["choices"][0]["message"]["content"] or "").strip()
72
+ if not out:
73
+ raise RuntimeError("polish answered with no text")
74
+ return out
75
+
76
+
77
+ def polish(text: str, terms: list[str], config: dict[str, Any]) -> tuple[str, str | None]:
78
+ """The cleaned take, or the take unchanged and why."""
79
+ if not text.strip() or not enabled(config):
80
+ return text, None
81
+ try:
82
+ return complete(config["polish"], text, terms), None
83
+ except Exception as err: # noqa: BLE001 - any failure keeps the recognizer's text
84
+ return text, str(err)