@urun-sh/sense 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/README.md +38 -2
- package/dist/context.d.ts +44 -0
- package/dist/context.js +110 -0
- package/dist/errors.d.ts +6 -0
- package/dist/errors.js +9 -0
- package/dist/index.d.ts +15 -0
- package/dist/index.js +9 -0
- package/dist/predicate.d.ts +81 -0
- package/dist/predicate.js +132 -0
- package/dist/question.d.ts +43 -0
- package/dist/question.js +75 -0
- package/dist/sense.d.ts +54 -0
- package/dist/sense.js +109 -0
- package/dist/session.d.ts +151 -0
- package/dist/session.js +472 -0
- package/dist/transport.d.ts +106 -0
- package/dist/transport.js +467 -0
- package/dist/wave.d.ts +58 -0
- package/dist/wave.js +209 -0
- package/examples/assets/mic-question-48k-mono.wav +0 -0
- package/examples/cheese_demo.html +290 -0
- package/examples/cheese_demo.test.ts +191 -0
- package/examples/cheese_demo.ts +250 -0
- package/examples/voice_agent.ts +166 -0
- package/package.json +55 -4
package/dist/wave.js
ADDED
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
import { ProtocolError } from "./errors.js";
|
|
2
|
+
/** One question's answer in a wave. */
|
|
3
|
+
export class Result {
|
|
4
|
+
id;
|
|
5
|
+
kind;
|
|
6
|
+
waveId;
|
|
7
|
+
/** noul: boolean; choice: top option name; score: expected ordinal Σi·p. */
|
|
8
|
+
value;
|
|
9
|
+
/** noul: P(true); choice: P(top); score: P(top level). */
|
|
10
|
+
p;
|
|
11
|
+
confidence;
|
|
12
|
+
/** Label-keyed probabilities (score: level names). */
|
|
13
|
+
distribution;
|
|
14
|
+
/** argmax label ("true"/"false" for noul, option name, level name). */
|
|
15
|
+
top;
|
|
16
|
+
constructor(init) {
|
|
17
|
+
this.id = init.id;
|
|
18
|
+
this.kind = init.kind;
|
|
19
|
+
this.waveId = init.waveId;
|
|
20
|
+
this.value = init.value;
|
|
21
|
+
this.p = init.p;
|
|
22
|
+
this.confidence = init.confidence;
|
|
23
|
+
this.distribution = init.distribution;
|
|
24
|
+
this.top = init.top;
|
|
25
|
+
}
|
|
26
|
+
pOption(name) {
|
|
27
|
+
if (this.kind === "score") {
|
|
28
|
+
throw new ProtocolError("protocol", `pOption(${name}) on score question ${this.id}`);
|
|
29
|
+
}
|
|
30
|
+
return this.distribution[name] ?? 0;
|
|
31
|
+
}
|
|
32
|
+
/** Only valid on score results: Σ p over levels >= the given level. */
|
|
33
|
+
pAtLeast(level) {
|
|
34
|
+
throw new ProtocolError("protocol", `pAtLeast(${level}) on non-score question ${this.id}`);
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
/** Score result: ordinal distribution over ordered levels. */
|
|
38
|
+
export class Score extends Result {
|
|
39
|
+
levels;
|
|
40
|
+
/** argmax level name. */
|
|
41
|
+
level;
|
|
42
|
+
constructor(init) {
|
|
43
|
+
const byName = {};
|
|
44
|
+
let expected = 0;
|
|
45
|
+
let bestIdx = 0;
|
|
46
|
+
init.levels.forEach((name, i) => {
|
|
47
|
+
const p = init.probs[String(i)] ?? 0;
|
|
48
|
+
byName[name] = p;
|
|
49
|
+
expected += i * p;
|
|
50
|
+
if (p > (init.probs[String(bestIdx)] ?? 0))
|
|
51
|
+
bestIdx = i;
|
|
52
|
+
});
|
|
53
|
+
const level = init.levels[bestIdx] ?? "";
|
|
54
|
+
super({
|
|
55
|
+
id: init.id,
|
|
56
|
+
kind: "score",
|
|
57
|
+
waveId: init.waveId,
|
|
58
|
+
value: expected,
|
|
59
|
+
p: byName[level] ?? 0,
|
|
60
|
+
confidence: init.confidence,
|
|
61
|
+
distribution: byName,
|
|
62
|
+
top: level,
|
|
63
|
+
});
|
|
64
|
+
this.levels = init.levels;
|
|
65
|
+
this.level = level;
|
|
66
|
+
}
|
|
67
|
+
pAtLeast(level) {
|
|
68
|
+
const idx = this.levels.indexOf(level);
|
|
69
|
+
if (idx < 0) {
|
|
70
|
+
throw new ProtocolError("protocol", `unknown level "${level}" for ${this.id}`);
|
|
71
|
+
}
|
|
72
|
+
let sum = 0;
|
|
73
|
+
for (let i = idx; i < this.levels.length; i++) {
|
|
74
|
+
sum += this.distribution[this.levels[i]] ?? 0;
|
|
75
|
+
}
|
|
76
|
+
return sum;
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
const noulTop = (choice) => (choice ? "true" : "false");
|
|
80
|
+
/** One wave: exactly one answer per registered question (§6). */
|
|
81
|
+
export class Wave {
|
|
82
|
+
waveId;
|
|
83
|
+
tMs;
|
|
84
|
+
results;
|
|
85
|
+
constructor(waveId, tMs, results) {
|
|
86
|
+
this.waveId = waveId;
|
|
87
|
+
this.tMs = tMs;
|
|
88
|
+
this.results = results;
|
|
89
|
+
}
|
|
90
|
+
result(qid) {
|
|
91
|
+
const r = this.results[qid];
|
|
92
|
+
if (!r) {
|
|
93
|
+
throw new ProtocolError("unknown_question", `no result for question "${qid}"`);
|
|
94
|
+
}
|
|
95
|
+
return r;
|
|
96
|
+
}
|
|
97
|
+
has(qid) {
|
|
98
|
+
return qid in this.results;
|
|
99
|
+
}
|
|
100
|
+
questionIds() {
|
|
101
|
+
return Object.keys(this.results);
|
|
102
|
+
}
|
|
103
|
+
/** Decode one wire payload ({wave_id, t_ms, results}) against registered questions. */
|
|
104
|
+
static decode(questions, payload) {
|
|
105
|
+
if (typeof payload !== "object" || payload === null) {
|
|
106
|
+
throw new ProtocolError("protocol", "wave payload must be an object");
|
|
107
|
+
}
|
|
108
|
+
const raw = payload;
|
|
109
|
+
if (typeof raw.results !== "object" || raw.results === null) {
|
|
110
|
+
throw new ProtocolError("protocol", "wave payload is missing results");
|
|
111
|
+
}
|
|
112
|
+
const incoming = raw.results;
|
|
113
|
+
const results = {};
|
|
114
|
+
for (const q of questions) {
|
|
115
|
+
const ans = incoming[q.id];
|
|
116
|
+
if (ans === undefined) {
|
|
117
|
+
throw new ProtocolError("missing_answer", `wave has no answer for registered question "${q.id}"`);
|
|
118
|
+
}
|
|
119
|
+
results[q.id] = decodeResult(q, ans, numOr(raw.wave_id, 0));
|
|
120
|
+
}
|
|
121
|
+
for (const qid of Object.keys(incoming)) {
|
|
122
|
+
if (!questions.some((q) => q.id === qid)) {
|
|
123
|
+
throw new ProtocolError("unknown_question", `wave carries unknown question "${qid}"`);
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
return new Wave(numOr(raw.wave_id, 0), numOr(raw.t_ms, 0), results);
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
function numOr(v, fallback) {
|
|
130
|
+
return typeof v === "number" ? v : fallback;
|
|
131
|
+
}
|
|
132
|
+
function decodeAnswerEnvelope(q, ans) {
|
|
133
|
+
if (typeof ans !== "object" || ans === null) {
|
|
134
|
+
throw new ProtocolError("protocol", `answer for "${q.id}" must be an object`);
|
|
135
|
+
}
|
|
136
|
+
const a = ans;
|
|
137
|
+
const probs = (typeof a.probabilities === "object" && a.probabilities !== null
|
|
138
|
+
? a.probabilities
|
|
139
|
+
: {});
|
|
140
|
+
const confidence = typeof a.confidence === "number" ? a.confidence : 0;
|
|
141
|
+
return { a, probs, confidence };
|
|
142
|
+
}
|
|
143
|
+
function decodeNoulResult(q, a, probs, confidence, waveId) {
|
|
144
|
+
let choice;
|
|
145
|
+
if (typeof a.choice === "boolean") {
|
|
146
|
+
choice = a.choice;
|
|
147
|
+
}
|
|
148
|
+
else if (a.choice === "true" || a.choice === "false") {
|
|
149
|
+
choice = a.choice === "true";
|
|
150
|
+
}
|
|
151
|
+
else {
|
|
152
|
+
throw new ProtocolError("protocol", `noul answer for "${q.id}" has bad choice ${JSON.stringify(a.choice)}`);
|
|
153
|
+
}
|
|
154
|
+
const pTrue = probs["true"] ?? (choice ? 1 : 0);
|
|
155
|
+
return new Result({
|
|
156
|
+
id: q.id,
|
|
157
|
+
kind: "noul",
|
|
158
|
+
waveId,
|
|
159
|
+
value: choice,
|
|
160
|
+
p: pTrue,
|
|
161
|
+
confidence,
|
|
162
|
+
distribution: probs,
|
|
163
|
+
top: noulTop(choice),
|
|
164
|
+
});
|
|
165
|
+
}
|
|
166
|
+
function decodeChoiceResult(q, a, probs, confidence, waveId) {
|
|
167
|
+
const top = typeof a.choice === "string" ? a.choice : argmax(probs);
|
|
168
|
+
if (top === undefined) {
|
|
169
|
+
throw new ProtocolError("protocol", `choice answer for "${q.id}" has no choice`);
|
|
170
|
+
}
|
|
171
|
+
return new Result({
|
|
172
|
+
id: q.id,
|
|
173
|
+
kind: "choice",
|
|
174
|
+
waveId,
|
|
175
|
+
value: top,
|
|
176
|
+
p: probs[top] ?? 0,
|
|
177
|
+
confidence,
|
|
178
|
+
distribution: probs,
|
|
179
|
+
top,
|
|
180
|
+
});
|
|
181
|
+
}
|
|
182
|
+
function decodeResult(q, ans, waveId) {
|
|
183
|
+
const { a, probs, confidence } = decodeAnswerEnvelope(q, ans);
|
|
184
|
+
if (q.kind === "noul") {
|
|
185
|
+
return decodeNoulResult(q, a, probs, confidence, waveId);
|
|
186
|
+
}
|
|
187
|
+
if (q.kind === "choice") {
|
|
188
|
+
return decodeChoiceResult(q, a, probs, confidence, waveId);
|
|
189
|
+
}
|
|
190
|
+
// score
|
|
191
|
+
if (!q.levels?.length) {
|
|
192
|
+
throw new ProtocolError("bad_question", `score question "${q.id}" has no levels`);
|
|
193
|
+
}
|
|
194
|
+
return new Score({
|
|
195
|
+
id: q.id,
|
|
196
|
+
waveId,
|
|
197
|
+
probs,
|
|
198
|
+
levels: q.levels,
|
|
199
|
+
confidence,
|
|
200
|
+
});
|
|
201
|
+
}
|
|
202
|
+
function argmax(probs) {
|
|
203
|
+
let best;
|
|
204
|
+
for (const [k, v] of Object.entries(probs)) {
|
|
205
|
+
if (best === undefined || v > (probs[best] ?? -Infinity))
|
|
206
|
+
best = k;
|
|
207
|
+
}
|
|
208
|
+
return best;
|
|
209
|
+
}
|
|
Binary file
|
|
@@ -0,0 +1,290 @@
|
|
|
1
|
+
<!doctype html>
|
|
2
|
+
<!-- 🧀 Sense SDK cheese classifier — served by examples/typescript/cheese_demo.ts -->
|
|
3
|
+
<html lang="en">
|
|
4
|
+
<head>
|
|
5
|
+
<meta charset="utf-8">
|
|
6
|
+
<meta name="viewport" content="width=device-width, initial-scale=1, viewport-fit=cover">
|
|
7
|
+
<title>🧀 cheese demo</title>
|
|
8
|
+
<style>
|
|
9
|
+
:root {
|
|
10
|
+
color-scheme: dark;
|
|
11
|
+
--green: #3fb950; --yellow: #d29922; --red: #f85149;
|
|
12
|
+
--grey: #656d76; --magenta: #db61a2; --cyan: #76e3ea; --dim: #8b949e;
|
|
13
|
+
}
|
|
14
|
+
* { box-sizing: border-box; margin: 0; }
|
|
15
|
+
body {
|
|
16
|
+
background: #0d1117; color: #e6edf3;
|
|
17
|
+
font-family: -apple-system, system-ui, "Segoe UI", Roboto, sans-serif;
|
|
18
|
+
min-height: 100dvh; display: flex; flex-direction: column;
|
|
19
|
+
padding: max(16px, env(safe-area-inset-top)) 16px max(16px, env(safe-area-inset-bottom));
|
|
20
|
+
}
|
|
21
|
+
header { display: flex; flex-direction: column; gap: 10px; }
|
|
22
|
+
#pill { font-size: 14px; }
|
|
23
|
+
#pill.online { color: var(--green); }
|
|
24
|
+
#pill.connecting, #pill.waiting, #pill.stalled { color: var(--yellow); }
|
|
25
|
+
#pill.error { color: var(--red); }
|
|
26
|
+
#pill.mic-only, #pill.stopped { color: var(--grey); }
|
|
27
|
+
#meter { display: flex; gap: 2px; align-items: flex-end; height: 28px; }
|
|
28
|
+
#meter i { flex: 1; background: var(--grey); min-height: 2px; border-radius: 1px; }
|
|
29
|
+
#speech { font-size: 14px; color: var(--dim); }
|
|
30
|
+
main { flex: 1; display: flex; align-items: center; justify-content: center; text-align: center; }
|
|
31
|
+
#verdict { font-size: 20px; color: var(--dim); max-width: 28em; }
|
|
32
|
+
#verdict.love, #verdict.like { color: var(--green); }
|
|
33
|
+
#verdict.hate, #verdict.dislike { color: var(--red); }
|
|
34
|
+
#verdict.meh { color: var(--yellow); }
|
|
35
|
+
#verdict.love .big, #verdict.like .big, #verdict.hate .big, #verdict.dislike .big, #verdict.meh .big { color: var(--magenta); }
|
|
36
|
+
#verdict .kind { color: var(--cyan); font-weight: 700; }
|
|
37
|
+
#verdict .big { font-size: 30px; font-weight: 700; }
|
|
38
|
+
#verdict .detail { margin-top: 8px; font-size: 20px; }
|
|
39
|
+
footer { display: flex; flex-direction: column; gap: 10px; }
|
|
40
|
+
#stats { font-size: 12px; color: var(--grey); font-family: ui-monospace, monospace; text-align: center; min-height: 1em; }
|
|
41
|
+
#toggle {
|
|
42
|
+
min-height: 64px; font-size: 20px; font-weight: 600; border: 0; border-radius: 14px;
|
|
43
|
+
background: #21262d; color: #e6edf3; touch-action: manipulation;
|
|
44
|
+
}
|
|
45
|
+
#toggle:disabled { opacity: 0.5; }
|
|
46
|
+
#toggle.streaming { background: #da3633; }
|
|
47
|
+
</style>
|
|
48
|
+
</head>
|
|
49
|
+
<body>
|
|
50
|
+
<header>
|
|
51
|
+
<div id="pill" class="connecting">● connecting</div>
|
|
52
|
+
<div id="meter"></div>
|
|
53
|
+
<div id="speech">speech unknown</div>
|
|
54
|
+
</header>
|
|
55
|
+
<main><div id="verdict">listening…</div></main>
|
|
56
|
+
<footer>
|
|
57
|
+
<div id="stats"></div>
|
|
58
|
+
<button id="toggle">Start</button>
|
|
59
|
+
</footer>
|
|
60
|
+
<script>
|
|
61
|
+
"use strict";
|
|
62
|
+
|
|
63
|
+
const pill = document.getElementById("pill");
|
|
64
|
+
const meterEl = document.getElementById("meter");
|
|
65
|
+
const speechEl = document.getElementById("speech");
|
|
66
|
+
const verdictEl = document.getElementById("verdict");
|
|
67
|
+
const statsEl = document.getElementById("stats");
|
|
68
|
+
const toggle = document.getElementById("toggle");
|
|
69
|
+
|
|
70
|
+
// -- status and meter ---------------------------------------------------------
|
|
71
|
+
|
|
72
|
+
const STATES = {
|
|
73
|
+
connecting: "● connecting",
|
|
74
|
+
waiting: "● connected · waiting for Sense",
|
|
75
|
+
stalled: "● connected, but Sense is not responding",
|
|
76
|
+
online: "● Sense online",
|
|
77
|
+
"mic-only": "○ sense disabled (mic test)",
|
|
78
|
+
stopped: "○ stopped",
|
|
79
|
+
error: "● demo stopped",
|
|
80
|
+
};
|
|
81
|
+
|
|
82
|
+
function setStatus(name) {
|
|
83
|
+
pill.className = name;
|
|
84
|
+
pill.textContent = STATES[name] ?? name;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
// "waiting" becomes "stalled" after a sustained silence; the timer arms below.
|
|
88
|
+
const STALL_MS = 15000;
|
|
89
|
+
let stallTimer = null;
|
|
90
|
+
let micStreaming = false;
|
|
91
|
+
let helloSeen = false;
|
|
92
|
+
function disarmStall() {
|
|
93
|
+
if (stallTimer !== null) {
|
|
94
|
+
clearTimeout(stallTimer);
|
|
95
|
+
stallTimer = null;
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
function armStallIfReady() {
|
|
99
|
+
if (micStreaming && helloSeen) armStall();
|
|
100
|
+
}
|
|
101
|
+
function armStall() {
|
|
102
|
+
disarmStall();
|
|
103
|
+
stallTimer = setTimeout(() => {
|
|
104
|
+
stallTimer = null;
|
|
105
|
+
if (pill.className === "waiting") setStatus("stalled");
|
|
106
|
+
}, STALL_MS);
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
const CELLS = 20;
|
|
110
|
+
for (let i = 0; i < CELLS; i++) meterEl.appendChild(document.createElement("i"));
|
|
111
|
+
let level = 0;
|
|
112
|
+
|
|
113
|
+
function meter(peak) {
|
|
114
|
+
level = Math.max(peak, level * 0.7);
|
|
115
|
+
const db = level > 0 ? 20 * Math.log10(level) : -Infinity;
|
|
116
|
+
const filled = Math.max(0, Math.min(CELLS, ((db + 60) / 60) * CELLS));
|
|
117
|
+
const color = db >= -6 ? "var(--red)" : db >= -12 ? "var(--yellow)" : db > -60 ? "var(--green)" : "var(--grey)";
|
|
118
|
+
const cells = [...meterEl.children];
|
|
119
|
+
for (let i = 0; i < cells.length; i++) {
|
|
120
|
+
const frac = Math.max(0, Math.min(1, filled - i));
|
|
121
|
+
cells[i].style.height = Math.max(2, frac * 28) + "px";
|
|
122
|
+
cells[i].style.background = color;
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
let speaking = null;
|
|
127
|
+
function setSpeaking(v) {
|
|
128
|
+
speaking = v;
|
|
129
|
+
speechEl.textContent = v === null ? "speech unknown" : v ? "🎙️ talking" : "· no speech";
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
// -- verdict card -------------------------------------------------------------
|
|
133
|
+
|
|
134
|
+
// One emoji per feeling level so the verdict card can distinguish like from love.
|
|
135
|
+
const FEELING_EMOJI = {
|
|
136
|
+
hate: "🧀🤮",
|
|
137
|
+
dislike: "🧀🙁",
|
|
138
|
+
meh: "🧀😐",
|
|
139
|
+
like: "🧀🙂",
|
|
140
|
+
love: "🧀❤️",
|
|
141
|
+
};
|
|
142
|
+
|
|
143
|
+
function renderVerdict(v) {
|
|
144
|
+
if (!v.talking) {
|
|
145
|
+
verdictEl.className = "";
|
|
146
|
+
verdictEl.textContent = "— not cheese";
|
|
147
|
+
return;
|
|
148
|
+
}
|
|
149
|
+
const emoji = FEELING_EMOJI[v.feeling] ?? "🧀😐";
|
|
150
|
+
const sentiment = v.feeling === "meh" ? "neutral sentiment" : `you ${v.feeling} it`;
|
|
151
|
+
verdictEl.className = v.feeling;
|
|
152
|
+
verdictEl.innerHTML =
|
|
153
|
+
`<div class="big">🧀 CHEESE</div>` +
|
|
154
|
+
`<div class="detail"><span class="kind">${v.kind}</span> (${v.p.toFixed(2)}) · ${emoji} ${sentiment} (${v.score.toFixed(1)})</div>`;
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
// -- capture: mic → 24 kHz PCM16, 40 ms frames --------------------------------
|
|
158
|
+
|
|
159
|
+
const WORKLET = `
|
|
160
|
+
class Capture extends AudioWorkletProcessor {
|
|
161
|
+
constructor() { super(); this.buf = []; }
|
|
162
|
+
process(inputs) {
|
|
163
|
+
const ch = inputs[0] && inputs[0][0];
|
|
164
|
+
if (ch) {
|
|
165
|
+
for (let i = 0; i < ch.length; i++) this.buf.push(ch[i]);
|
|
166
|
+
while (this.buf.length >= 960) {
|
|
167
|
+
const frame = this.buf.splice(0, 960);
|
|
168
|
+
const pcm = new Int16Array(960);
|
|
169
|
+
let peak = 0;
|
|
170
|
+
for (let i = 0; i < 960; i++) {
|
|
171
|
+
const s = Math.max(-1, Math.min(1, frame[i]));
|
|
172
|
+
pcm[i] = Math.round(s * 32767);
|
|
173
|
+
peak = Math.max(peak, Math.abs(s));
|
|
174
|
+
}
|
|
175
|
+
this.port.postMessage({ pcm: pcm.buffer, peak }, [pcm.buffer]);
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
return true;
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
registerProcessor("capture", Capture);
|
|
182
|
+
`;
|
|
183
|
+
|
|
184
|
+
const state = { ws: null, ctx: null, stream: null, started: false };
|
|
185
|
+
|
|
186
|
+
async function start() {
|
|
187
|
+
toggle.disabled = true;
|
|
188
|
+
setStatus("connecting");
|
|
189
|
+
micStreaming = false;
|
|
190
|
+
helloSeen = false;
|
|
191
|
+
try {
|
|
192
|
+
state.ws = new WebSocket(`wss://${location.host}/ws`);
|
|
193
|
+
state.ws.onmessage = onMessage;
|
|
194
|
+
state.ws.onclose = () => {
|
|
195
|
+
disarmStall();
|
|
196
|
+
if (state.started) {
|
|
197
|
+
setStatus("error");
|
|
198
|
+
verdictEl.textContent = "relay connection lost";
|
|
199
|
+
stop();
|
|
200
|
+
}
|
|
201
|
+
};
|
|
202
|
+
await new Promise((resolve, reject) => {
|
|
203
|
+
state.ws.onopen = resolve;
|
|
204
|
+
state.ws.onerror = () => reject(new Error("relay connection failed"));
|
|
205
|
+
});
|
|
206
|
+
state.stream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
|
207
|
+
state.ctx = new AudioContext({ sampleRate: 24000 });
|
|
208
|
+
if (state.ctx.sampleRate !== 24000) throw new Error("browser ignored the 24 kHz capture rate");
|
|
209
|
+
await state.ctx.audioWorklet.addModule(URL.createObjectURL(new Blob([WORKLET], { type: "text/javascript" })));
|
|
210
|
+
const node = new AudioWorkletNode(state.ctx, "capture");
|
|
211
|
+
node.port.onmessage = (e) => {
|
|
212
|
+
meter(e.data.peak);
|
|
213
|
+
if (state.ws && state.ws.readyState === WebSocket.OPEN) state.ws.send(e.data.pcm);
|
|
214
|
+
};
|
|
215
|
+
state.ctx.createMediaStreamSource(state.stream).connect(node);
|
|
216
|
+
// The relay may have closed during setup; fail so the catch block releases the microphone.
|
|
217
|
+
if (!state.ws || state.ws.readyState !== WebSocket.OPEN) throw new Error("relay connection lost");
|
|
218
|
+
state.started = true;
|
|
219
|
+
// Arm the stall timer once the mic is streaming and Sense has said hello
|
|
220
|
+
// (either may come first).
|
|
221
|
+
micStreaming = true;
|
|
222
|
+
armStallIfReady();
|
|
223
|
+
toggle.textContent = "Stop";
|
|
224
|
+
toggle.classList.add("streaming");
|
|
225
|
+
} catch (err) {
|
|
226
|
+
setStatus("error");
|
|
227
|
+
verdictEl.textContent = err.message;
|
|
228
|
+
await stop();
|
|
229
|
+
} finally {
|
|
230
|
+
toggle.disabled = false;
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
async function stop() {
|
|
235
|
+
state.started = false;
|
|
236
|
+
disarmStall();
|
|
237
|
+
if (pill.className !== "error") setStatus("stopped");
|
|
238
|
+
toggle.textContent = "Start";
|
|
239
|
+
toggle.classList.remove("streaming");
|
|
240
|
+
if (state.ws) {
|
|
241
|
+
state.ws.onmessage = null;
|
|
242
|
+
state.ws.onclose = null;
|
|
243
|
+
state.ws.close();
|
|
244
|
+
state.ws = null;
|
|
245
|
+
}
|
|
246
|
+
if (state.stream) {
|
|
247
|
+
for (const track of state.stream.getTracks()) track.stop();
|
|
248
|
+
state.stream = null;
|
|
249
|
+
}
|
|
250
|
+
if (state.ctx) {
|
|
251
|
+
await state.ctx.close().catch(() => {});
|
|
252
|
+
state.ctx = null;
|
|
253
|
+
}
|
|
254
|
+
level = 0;
|
|
255
|
+
meter(0);
|
|
256
|
+
speechEl.textContent = "speech unknown";
|
|
257
|
+
speaking = null;
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
// -- relay events ---------------------------------------------------------------
|
|
261
|
+
|
|
262
|
+
function onMessage(ev) {
|
|
263
|
+
const msg = JSON.parse(ev.data);
|
|
264
|
+
if (msg.type === "hello") {
|
|
265
|
+
setStatus(msg.mode === "online" ? "waiting" : "mic-only");
|
|
266
|
+
helloSeen = msg.mode === "online";
|
|
267
|
+
armStallIfReady();
|
|
268
|
+
} else if (msg.type === "wave") {
|
|
269
|
+
disarmStall();
|
|
270
|
+
setStatus("online");
|
|
271
|
+
setSpeaking(msg.verdict.speaking);
|
|
272
|
+
renderVerdict(msg.verdict);
|
|
273
|
+
} else if (msg.type === "stats") {
|
|
274
|
+
statsEl.textContent = "stats " + JSON.stringify(msg.stats);
|
|
275
|
+
} else if (msg.type === "error") {
|
|
276
|
+
disarmStall();
|
|
277
|
+
setStatus("error");
|
|
278
|
+
verdictEl.textContent = msg.message;
|
|
279
|
+
stop();
|
|
280
|
+
} else if (msg.type === "ended") {
|
|
281
|
+
disarmStall();
|
|
282
|
+
if (msg.reason === "cap") verdictEl.textContent = "5 min cap reached — stopping";
|
|
283
|
+
stop();
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
toggle.onclick = () => (state.started ? stop() : start());
|
|
288
|
+
</script>
|
|
289
|
+
</body>
|
|
290
|
+
</html>
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Tests for the cheese demo's wave → verdict mapping (mirrors
|
|
3
|
+
* python/tests/test_cheese_demo.py's decision-threshold checks).
|
|
4
|
+
* Pure logic only: no sockets, no mic.
|
|
5
|
+
*/
|
|
6
|
+
import { describe, expect, test } from "bun:test";
|
|
7
|
+
import { Result, Score, Wave, type Session } from "@urun-sh/sense";
|
|
8
|
+
import { closeRelay, handleAudio, pump, toVerdict, type Relay } from "./cheese_demo.ts";
|
|
9
|
+
|
|
10
|
+
function result(
|
|
11
|
+
id: string,
|
|
12
|
+
kind: "noul" | "choice",
|
|
13
|
+
value: boolean | string,
|
|
14
|
+
p: number,
|
|
15
|
+
distribution: Record<string, number>,
|
|
16
|
+
): Result {
|
|
17
|
+
return new Result({ id, kind, waveId: 7, value, p, confidence: 1, distribution, top: String(value) });
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
const FEELING_LEVELS = ["hate", "dislike", "meh", "like", "love"];
|
|
21
|
+
|
|
22
|
+
/** A score result whose name-keyed dist is mapped onto the feeling levels. */
|
|
23
|
+
function scoreResult(dist: Record<string, number>): Score {
|
|
24
|
+
const probs: Record<string, number> = {};
|
|
25
|
+
for (const [name, p] of Object.entries(dist)) {
|
|
26
|
+
probs[String(FEELING_LEVELS.indexOf(name))] = p;
|
|
27
|
+
}
|
|
28
|
+
return new Score({ id: "feeling", waveId: 7, probs, levels: FEELING_LEVELS, confidence: 1 });
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
function cheeseWave(p: number, kindDist: Record<string, number>, feelingDist: Record<string, number>): Wave {
|
|
32
|
+
return new Wave(7, 0, {
|
|
33
|
+
speech: result("speech", "noul", true, 1, { true: 1, false: 0 }),
|
|
34
|
+
cheese: result("cheese", "noul", true, p, { true: p, false: 1 - p }),
|
|
35
|
+
cheese_type: result("cheese_type", "choice", "cheddar", 0.99, kindDist),
|
|
36
|
+
feeling: scoreResult(feelingDist),
|
|
37
|
+
});
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
describe("toVerdict", () => {
|
|
41
|
+
test("talking threshold: not cheese below 0.6", () => {
|
|
42
|
+
const verdict = toVerdict(cheeseWave(0.59, { cheddar: 0.9 }, { love: 0.9 }));
|
|
43
|
+
if (verdict.talking) throw new Error("expected talking=false at p=0.59");
|
|
44
|
+
expect(verdict.speaking).toBe(true);
|
|
45
|
+
expect("kind" in verdict).toBe(false);
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
test("talking threshold: cheese at exactly 0.6", () => {
|
|
49
|
+
const verdict = toVerdict(cheeseWave(0.6, { cheddar: 0.9 }, { love: 0.9, like: 0.1 }));
|
|
50
|
+
if (!verdict.talking) throw new Error("expected talking=true at p=0.6");
|
|
51
|
+
expect(verdict.p).toBe(0.6);
|
|
52
|
+
expect(verdict.kind).toBe("cheddar");
|
|
53
|
+
expect(verdict.feeling).toBe("love");
|
|
54
|
+
expect(verdict.score).toBeCloseTo(3.9);
|
|
55
|
+
expect(verdict.waveId).toBe(7);
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
test("kind falls back to 'unknown type' below 0.6", () => {
|
|
59
|
+
const verdict = toVerdict(cheeseWave(0.9, { cheddar: 0.59, other: 0.41 }, { love: 0.9 }));
|
|
60
|
+
if (!verdict.talking) throw new Error("expected talking=true at p=0.9");
|
|
61
|
+
expect(verdict.kind).toBe("unknown type");
|
|
62
|
+
expect(verdict.feeling).toBe("love");
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
test("feeling falls back to 'meh' below 0.6", () => {
|
|
66
|
+
const verdict = toVerdict(cheeseWave(0.9, { cheddar: 0.9 }, { love: 0.59, meh: 0.41 }));
|
|
67
|
+
if (!verdict.talking) throw new Error("expected talking=true at p=0.9");
|
|
68
|
+
expect(verdict.kind).toBe("cheddar");
|
|
69
|
+
expect(verdict.feeling).toBe("meh");
|
|
70
|
+
});
|
|
71
|
+
|
|
72
|
+
test("speaking follows the speech result, not the cheese detection", () => {
|
|
73
|
+
const silent = new Wave(1, 0, {
|
|
74
|
+
speech: result("speech", "noul", false, 0, { true: 0, false: 1 }),
|
|
75
|
+
cheese: result("cheese", "noul", false, 0.2, { true: 0.2, false: 0.8 }),
|
|
76
|
+
cheese_type: result("cheese_type", "choice", "cheddar", 0.5, { cheddar: 0.5 }),
|
|
77
|
+
feeling: scoreResult({ meh: 0.5 }),
|
|
78
|
+
});
|
|
79
|
+
const verdict = toVerdict(silent);
|
|
80
|
+
expect(verdict.speaking).toBe(false);
|
|
81
|
+
});
|
|
82
|
+
});
|
|
83
|
+
|
|
84
|
+
// -- relay lifecycle (fake session; no sockets) --------------------------------
|
|
85
|
+
|
|
86
|
+
/** Minimal Session double: enough for the relay functions under test. */
|
|
87
|
+
function fakeSession(opts: { connect?: () => Promise<void>; run?: () => AsyncGenerator<Wave> } = {}) {
|
|
88
|
+
const calls = { connect: 0, close: 0, run: 0, pushed: 0 as number };
|
|
89
|
+
const session = {
|
|
90
|
+
connect: opts.connect ?? (async () => {}),
|
|
91
|
+
run: opts.run ?? (async function* () {}),
|
|
92
|
+
audio: { push: () => { calls.pushed++; } },
|
|
93
|
+
close: () => { calls.close++; },
|
|
94
|
+
};
|
|
95
|
+
return { calls, session: session as unknown as Session };
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
function testRelay(session: Session | null, send: (event: unknown) => void = () => {}): Relay {
|
|
99
|
+
return {
|
|
100
|
+
session,
|
|
101
|
+
send,
|
|
102
|
+
capTimer: null,
|
|
103
|
+
done: false,
|
|
104
|
+
closed: false,
|
|
105
|
+
pending: [],
|
|
106
|
+
streaming: false,
|
|
107
|
+
wake: null,
|
|
108
|
+
end: () => {},
|
|
109
|
+
};
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
describe("relay lifecycle", () => {
|
|
113
|
+
test("frames buffer during the dial and flush in order before run()", async () => {
|
|
114
|
+
const { calls, session } = fakeSession();
|
|
115
|
+
const relay = testRelay(session);
|
|
116
|
+
const runPromise = pump(relay);
|
|
117
|
+
handleAudio(relay, new Uint8Array(4));
|
|
118
|
+
handleAudio(relay, new Uint8Array(8));
|
|
119
|
+
expect(relay.pending.length).toBe(2); // still dialing: buffered, not pushed
|
|
120
|
+
expect(calls.pushed).toBe(0);
|
|
121
|
+
await runPromise;
|
|
122
|
+
expect(calls.pushed).toBe(2); // flushed before the run loop consumed them
|
|
123
|
+
expect(relay.pending.length).toBe(0);
|
|
124
|
+
expect(relay.streaming).toBe(true);
|
|
125
|
+
});
|
|
126
|
+
|
|
127
|
+
test("pump stops when the client leaves during the dial", async () => {
|
|
128
|
+
let runStarted = false;
|
|
129
|
+
const { calls, session } = fakeSession({
|
|
130
|
+
run: async function* () {
|
|
131
|
+
runStarted = true;
|
|
132
|
+
yield* [];
|
|
133
|
+
},
|
|
134
|
+
});
|
|
135
|
+
const relay = testRelay(session);
|
|
136
|
+
const runPromise = pump(relay);
|
|
137
|
+
closeRelay(relay); // the browser socket closes mid-dial
|
|
138
|
+
await runPromise;
|
|
139
|
+
expect(relay.closed).toBe(true);
|
|
140
|
+
expect(runStarted).toBe(false); // the upstream session was shut down, not left open
|
|
141
|
+
expect(calls.close).toBeGreaterThan(0);
|
|
142
|
+
});
|
|
143
|
+
|
|
144
|
+
test("pump stops when the client leaves during the frame wait", async () => {
|
|
145
|
+
let runStarted = false;
|
|
146
|
+
const { calls, session } = fakeSession({
|
|
147
|
+
run: async function* () {
|
|
148
|
+
runStarted = true;
|
|
149
|
+
yield* [];
|
|
150
|
+
},
|
|
151
|
+
});
|
|
152
|
+
const relay = testRelay(session);
|
|
153
|
+
const runPromise = pump(relay);
|
|
154
|
+
await new Promise((r) => setTimeout(r, 5)); // let the pump reach the frame wait
|
|
155
|
+
expect(relay.wake).not.toBeNull();
|
|
156
|
+
closeRelay(relay);
|
|
157
|
+
await runPromise;
|
|
158
|
+
expect(runStarted).toBe(false);
|
|
159
|
+
expect(calls.close).toBeGreaterThan(0);
|
|
160
|
+
});
|
|
161
|
+
|
|
162
|
+
test("handleAudio drops frames after close", () => {
|
|
163
|
+
const { calls, session } = fakeSession();
|
|
164
|
+
const relay = testRelay(session);
|
|
165
|
+
closeRelay(relay);
|
|
166
|
+
handleAudio(relay, new Uint8Array(4));
|
|
167
|
+
expect(relay.pending.length).toBe(0);
|
|
168
|
+
expect(calls.pushed).toBe(0);
|
|
169
|
+
});
|
|
170
|
+
|
|
171
|
+
test("pending buffer overflow ends the relay instead of growing unboundedly", () => {
|
|
172
|
+
const sent: unknown[] = [];
|
|
173
|
+
let ended = false;
|
|
174
|
+
const { calls, session } = fakeSession();
|
|
175
|
+
const relay = testRelay(session, (e) => sent.push(e));
|
|
176
|
+
relay.end = () => { ended = true; };
|
|
177
|
+
const big = new Uint8Array(1_100_000); // > MAX_PENDING_BYTES on its own
|
|
178
|
+
handleAudio(relay, big);
|
|
179
|
+
expect(ended).toBe(true);
|
|
180
|
+
expect(relay.pending.length).toBe(0);
|
|
181
|
+
expect(calls.pushed).toBe(0);
|
|
182
|
+
expect(sent.some((e) => (e as { type: string }).type === "error")).toBe(true);
|
|
183
|
+
});
|
|
184
|
+
|
|
185
|
+
test("closeRelay is idempotent: only the first close releases the slot", () => {
|
|
186
|
+
const { session } = fakeSession();
|
|
187
|
+
const relay = testRelay(session);
|
|
188
|
+
expect(closeRelay(relay)).toBe(true);
|
|
189
|
+
expect(closeRelay(relay)).toBe(false);
|
|
190
|
+
});
|
|
191
|
+
});
|