cotutor 0.2.0 → 0.3.0

package/dist/memory/vault.js

@@ -0,0 +1,169 @@
+import { mkdir, readFile, readdir, rename, stat, writeFile } from 'node:fs/promises';
+import { basename, dirname, join, relative, sep } from 'node:path';
+import { removeDiaryBlock } from "./diary.js";
+import { CHILD_MEMORY_AGENT, CHILD_MEMORY_CAP, CHILD_MEMORY_PATH, applyMemoryOps, childMemoryTemplate, clipNote, frontmatter, memoryPath, memoryTemplate, parseMemoryOp, textHash, undoMemoryLines } from "./vault-notes.js";
+/**
+ * 删话题时撤回它留在 vault 里的:各轮写进记忆的行(memoryLines;孩子的记忆是 childLines)倒着撤、记账写进日记的那段(diaryBlock)摘掉;
+ * 找不到原文的(家长或整理那轮后来改过)不撤,一条一句列进 misses
+ */
+export async function undoThreadVault(v, tutor, date, gone) {
+    const undo = { memory: 0, diary: 0, misses: [] };
+    // activity 话题写的是 activity 自己的记忆(agent: <老师>/<activity>);孩子的记忆大家共用一篇
+    const run = gone.find((m) => m.activity)?.activity;
+    const files = [
+        { agent: run ? `${tutor}/${run.name}` : tutor, edits: gone.flatMap((m) => m.memoryLines ?? []), name: '记忆' },
+        { agent: CHILD_MEMORY_AGENT, edits: gone.flatMap((m) => m.childLines ?? []), name: '孩子的记忆' },
+    ].filter((f) => f.edits.length);
+    const notes = files.length ? (await scanVault(v)).notes : [];
+    for (const f of files) {
+        const found = notes.find((n) => n.props.cotutor === 'memory' && n.props.agent?.trim() === f.agent);
+        if (!found) {
+            undo.misses.push(`${f.name}文件不在了,${f.name}没撤`);
+            continue;
+        }
+        const r = undoMemoryLines(found.text, f.edits);
+        undo.memory += r.undone;
+        undo.misses.push(...r.misses.map((l) => `${f.name}:找不到这行,没撤:${l}`));
+        if (r.undone) {
+            const file = join(v.paths.vault, found.path);
+            await writeFile(`${file}.tmp`, r.text);
+            await rename(`${file}.tmp`, file);
+        }
+    }
+    for (const m of gone) {
+        if (!m.diaryBlock)
+            continue;
+        const existing = await readFile(join(v.paths.diary, `${date}.md`), 'utf8').catch(() => null);
+        const out = existing === null ? null : removeDiaryBlock(existing, m.diaryBlock);
+        if (out === null) {
+            undo.misses.push(`日记:${date} 里找不到这个话题记的那段,没摘`);
+            continue;
+        }
+        await writeDiary(v, date, () => out);
+        undo.diary++;
+    }
+    return undo;
+}
+/** 这几天的日记(<日记目录>/<日期>.md);没有的跳过 */
+export async function readDiaries(v, dates) {
+    const out = [];
+    for (const date of dates) {
+        try {
+            out.push({ date, text: await readFile(join(v.paths.diary, `${date}.md`), 'utf8') });
+        }
+        catch {
+            /* 那天没写 */
+        }
+    }
+    return out;
+}
+/** 当天的日记:update 拿现有内容(没有 → null)回新内容;先 .tmp 再 rename。返回文件名(<日期>.md) */
+export async function writeDiary(v, date, update) {
+    await mkdir(v.paths.diary, { recursive: true });
+    const file = join(v.paths.diary, `${date}.md`);
+    const existing = await readFile(file, 'utf8').catch(() => null);
+    const tmp = `${file}.tmp`;
+    await writeFile(tmp, update(existing));
+    await rename(tmp, file);
+    return basename(file);
+}
+/** 教材目录下的每册文件(<教材目录>/<册>.md),给 textbookHeadings;没有目录 → [] */
+export async function readTextbooks(v) {
+    let names;
+    try {
+        names = (await readdir(v.paths.textbooks)).filter((f) => f.endsWith('.md') && !f.startsWith('.')).sort();
+    }
+    catch {
+        return [];
+    }
+    const out = [];
+    for (const name of names) {
+        try {
+            out.push({ name, text: await readFile(join(v.paths.textbooks, name), 'utf8') });
+        }
+        catch {
+            /* 读不到就跳过 */
+        }
+    }
+    return out;
+}
+/**
+ * 扫 vault:所有文件的相对路径(解析 [[链接]] 用)+ 带 `cotutor:` 属性的 md(按属性定位,《obsidian仓库设计.md》2026-09-17)。
+ * 跳过点开头的目录(.obsidian / .git)与日记、计划目录(机器写的,不带属性,越积越多)。读不到的文件跳过;vault 不在 → 全空。
+ */
+export async function scanVault(v) {
+    const root = v.paths.vault;
+    const skip = new Set([v.paths.diary, v.paths.plans]);
+    const notes = [];
+    const files = [];
+    const walk = async (dir) => {
+        let entries;
+        try {
+            entries = await readdir(dir, { withFileTypes: true });
+        }
+        catch {
+            return;
+        }
+        for (const e of entries) {
+            if (e.name.startsWith('.'))
+                continue;
+            const abs = join(dir, e.name);
+            if (e.isDirectory()) {
+                if (!skip.has(abs))
+                    await walk(abs);
+                continue;
+            }
+            if (!e.isFile())
+                continue;
+            const rel = relative(root, abs).split(sep).join('/');
+            files.push(rel);
+            if (!e.name.endsWith('.md'))
+                continue;
+            try {
+                const text = await readFile(abs, 'utf8');
+                const props = frontmatter(text);
+                if (props.cotutor)
+                    notes.push({ path: rel, props, text });
+            }
+            catch {
+                /* 读不到(iCloud 占位)就跳过 */
+            }
+        }
+    };
+    await walk(root);
+    return { notes, files: files.sort() };
+}
+/**
+ * 「## 记忆」段 → 这位 agent 的记忆文件要怎么改(按 `cotutor: memory` + `agent:` 找;没有就建 记忆/<显示名>.md,孩子的记忆建 记忆/孩子.md),只算不写:
+ * 新增追加到末尾,「改:」「删:」按原话找行改、删(任何一行都能动,2026-09-18);缺省位置已有一篇没属性的同名文件就不碰,进提醒。
+ * seen = 这一轮上下文包里老师看到的那版(截过的那段的 textHash 与截的字数):文件和它对不上 = 这轮里家长改过,
+ * 改、删只认一字不差的原话,找不到的不改——按「包含」找会把家长刚补上的半句一起盖掉(2026-10-30)。
+ * 孩子的记忆(agent: all)最多 CHILD_MEMORY_CAP 条,满了新增的不进,提醒家长删几条。
+ */
+export async function planVaultMemory(v, o) {
+    const found = (await scanVault(v)).notes.find((n) => n.props.cotutor === 'memory' && n.props.agent?.trim() === o.agent);
+    const child = o.agent === CHILD_MEMORY_AGENT;
+    const file = found?.path ?? (child ? CHILD_MEMORY_PATH : memoryPath(o.display));
+    if (!found && (await stat(join(v.paths.vault, file)).catch(() => null)) !== null)
+        return { plan: null, warnings: [`记忆没写:${file} 已经有了但没有 cotutor: memory / agent: ${o.agent} 属性;加上属性,或挪开它`] };
+    const base = found?.text ?? null;
+    const strict = base !== null && o.seen?.path === file && textHash(clipNote(base, o.seen.clip, 'tail').text) !== o.seen.hash;
+    const r = applyMemoryOps(base ?? (child ? childMemoryTemplate() : memoryTemplate(o.agent, o.display)), o.items.map(parseMemoryOp), o.date, { strict, ...(child ? { cap: CHILD_MEMORY_CAP } : {}) });
+    const name = child ? '孩子的记忆' : '记忆';
+    const warnings = !r.misses.length ? [] : strict ? [`${name}:这轮里家长改过记忆文件,找不到原话的没改(不盖掉家长改的):${r.misses.join(';')}`] : [`${name}:找不到原话,没改:${r.misses.join(';')}`];
+    if (r.full.length)
+        warnings.push(`孩子的记忆满了(${CHILD_MEMORY_CAP} 条),这几条没记:${r.full.join(';')};在 Obsidian 里删几条才有地方`);
+    if (!r.changes.length)
+        return { plan: null, warnings };
+    return { plan: { file, base, text: r.text, changes: r.changes, lines: r.lines }, warnings };
+}
+/** 写下 planVaultMemory 算好的:先重读,和算的时候不一样(家长刚存了一版)就不写,回为什么;写成回 null。先 .tmp 再 rename */
+export async function writeVaultMemory(v, plan) {
+    const file = join(v.paths.vault, plan.file);
+    if ((await readFile(file, 'utf8').catch(() => null)) !== plan.base)
+        return `记忆没写:${plan.file} 刚被改过,这轮的记忆没写:${plan.changes.join(';')}`;
+    await mkdir(dirname(file), { recursive: true });
+    await writeFile(`${file}.tmp`, plan.text);
+    await rename(`${file}.tmp`, file);
+    return null;
+}

package/dist/realtime/bailian.d.ts

@@ -0,0 +1,120 @@
+export declare const REALTIME_MODEL = "qwen3.8-omni-flash-realtime";
+export declare const REALTIME_VOICE = "Serena";
+export declare const INPUT_RATE = 16000;
+export declare const OUTPUT_RATE = 24000;
+/** 服务端一帧最多 262144 字节(2026-10-19 真连:超了就 close 1009,整条连接没了);一张图的 base64 连同 JSON 的壳得在这以内 */
+export declare const MAX_IMAGE_B64: number;
+export type TurnMode = 'manual' | 'semantic_vad' | 'server_vad';
+/** VAD 判出说完以后服务端自己做不做:回一次(create_response)、孩子开口时掐掉在出的回答(interrupt_response);缺省都做(口语课聊那段) */
+export interface VadOptions {
+    createResponse?: boolean;
+    interruptResponse?: boolean;
+}
+export interface RealtimeOptions {
+    /** 系统提示(人设 + 规矩;口语课还有口语单) */
+    instructions: string;
+    voice?: string;
+    model?: string;
+    turn?: TurnMode;
+    /** VAD 判「说完了」的静音毫秒(一年级孩子想词慢,缺省比百炼的 800 长) */
+    silenceMs?: number;
+    /** VAD 判出说完以后服务端自己回不回、掐不掐(实时老师的一直听两样都关,由应用定) */
+    vad?: VadOptions;
+    /** 测试用:整个端点(含 ?model=);不给就按 workspaceId 拼 */
+    endpoint?: string;
+    /** 指定 key 与业务空间 id(家长端「试连」还没存的那份);不给按 dashscopeAuth 找 */
+    auth?: {
+        key: string;
+        workspaceId: string;
+    };
+    env?: NodeJS.ProcessEnv;
+    /** 握手后等 session.updated 最多多久 */
+    timeoutMs?: number;
+}
+export interface Usage {
+    input: number;
+    output: number;
+    inputAudio: number;
+    outputAudio: number;
+}
+export interface RealtimeEvents {
+    /** 一块老师的声音(PCM16 24k 单声道) */
+    onAudio?(pcm: Buffer): void;
+    /** 老师的字:delta 一截截来,done 时给整句 */
+    onTutorText?(delta: string, done: string | null): void;
+    /** 孩子那句转写完了 */
+    onKidText?(text: string): void;
+    /** VAD:孩子开口 / 停了 */
+    onSpeech?(state: 'started' | 'stopped'): void;
+    /** semantic_vad 判出这一段不算一句(附和声、噪声):不会有 committed 与转写;不是错 */
+    onRejected?(): void;
+    /** 孩子那句 commit 了,转写失败 */
+    onKidFailed?(): void;
+    /** 老师一次回答开始 / 结束(status 来自服务端:completed / cancelled / incomplete / failed) */
+    onResponse?(state: 'created' | 'done', status: string | null, usage: Usage | null): void;
+    onError?(message: string): void;
+    onClose?(reason: string): void;
+}
+/** 百炼按量的 key 与业务空间 id(Token Plan 没有 omni realtime;key 怎么找见 src/shared/qianwen.ts,业务空间 id 先看家长端配的、再看环境、再看 voxtell 的配置) */
+export declare function dashscopeAuth(env?: NodeJS.ProcessEnv): {
+    key: string;
+    workspaceId: string;
+} | null;
+export declare const endpointOf: (workspaceId: string, model: string) => string;
+/** 轮次模式 → session.turn_detection */
+export declare function turnDetection(turn: TurnMode, silenceMs: number, vad?: VadOptions): Record<string, unknown> | null;
+/** session.update 的正文 */
+export declare function sessionConfig(o: {
+    instructions: string;
+    voice: string;
+    turn: TurnMode;
+    silenceMs: number;
+    vad?: VadOptions;
+}): Record<string, unknown>;
+export declare class RealtimeSession {
+    readonly model: string;
+    readonly voice: string;
+    private ws;
+    private ready;
+    private closed;
+    private turn;
+    private readonly silenceMs;
+    private tutorText;
+    private pendingUpdate;
+    /** 正在出的回答数(response.cancel 只在有回答时发,不然服务端报错) */
+    private active;
+    private readonly opts;
+    private readonly ev;
+    constructor(opts: RealtimeOptions, ev?: RealtimeEvents);
+    get isReady(): boolean;
+    /** 老师说到一半、还没 done 的字(挂断时记进字幕用) */
+    get partialTutorText(): string;
+    get turnMode(): TurnMode;
+    get responding(): boolean;
+    /** 连上、配好会话;回 null 成功,否则是原因(no_auth / connect / timeout / <服务端的话>) */
+    connect(): Promise<string | null>;
+    private handle;
+    private sendRaw;
+    /** 一块孩子的声音(PCM16 16k 单声道) */
+    appendAudio(pcm: Buffer): boolean;
+    /** 一张图(JPEG)跟着这一段声音交:百炼要先有声音再有图(Qwen-Live-Harness 也这样),commit 时一起进对话;太大(见 MAX_IMAGE_B64)不发、回 false,调用方先缩 */
+    appendImage(jpeg: Buffer): boolean;
+    /** 手动轮次:松手 */
+    commit(): boolean;
+    clearAudio(): boolean;
+    /**
+     * 心跳:1 秒全零的声音接着 clear,不 commit、不要回答(照 Qwen-Live-Harness 的静音心跳)。百炼 300 秒没输入就关会话(user_idle_timeout)。
+     * 2026-10-20 真连(scripts/probe-live-idle.mjs,手动轮次,每 30 秒一次):不发的 302 秒被关;只 append 不 clear 的 300 秒报 response_idle_timeout
+     * (缓冲里有输入一直没回答),会话就死了;append 完马上 clear 的过了 400 秒照常说话。
+     */
+    heartbeat(): boolean;
+    /** 让老师说;instructions = 只管这一次回答的指令(response.instructions,不进对话历史、不改系统提示) */
+    createResponse(instructions?: string): boolean;
+    /** 老师正在说就掐掉(没在说不发,服务端会报错) */
+    cancel(): boolean;
+    /** 应用塞一条文字进对话(口语课跟读段「现在读这句」、切段的指令);role 缺省 user */
+    addText(text: string, role?: 'user' | 'system'): boolean;
+    /** 中途在手动与 VAD 之间切(只给 type,silence 喂过音频后改不了);等 session.updated,错或超时回 false */
+    setTurn(turn: TurnMode, timeoutMs?: number): Promise<boolean>;
+    close(reason?: string): void;
+}

package/dist/realtime/bailian.js

@@ -0,0 +1,288 @@
+/**
+ * 百炼 Qwen-Omni-Realtime 的 WebSocket 客户端:口语课和实时老师共用。
+ * 一次通话一条会话。服务端连,key 不出服务端;孩子端的声音由调用方推进来,老师的声音从这里出去。
+ *
+ * 协议(OpenAI realtime 同族,2026-10-10 真连核对):
+ *   连上先收 session.created;发 session.update,收 session.updated 后才能喂音频。
+ *   上行:input_audio_buffer.append {audio: base64 PCM16 16k 单声道} / commit / clear;input_image_buffer.append {image: base64 JPEG}(先有声音再有图,commit 时一起交);response.create / cancel;
+ *         conversation.item.create {item: {type: message, role, content: [{type: input_text, text}]}}。
+ *   下行:response.audio.delta {delta: base64 PCM16 24k};response.audio_transcript.delta / .done(老师的字);
+ *         conversation.item.input_audio_transcription.completed {transcript}(孩子那句,qwen3-asr-flash-realtime 转的);
+ *         input_audio_buffer.speech_started / speech_stopped(VAD);response.done {response: {status, usage}};error。
+ *   端点要业务空间 id:wss://<WorkspaceId>.cn-beijing.maas.aliyuncs.com/api-ws/v1/realtime?model=…(公网域名 dashscope.aliyuncs.com 连上 1 秒就被关)。
+ *
+ * 轮次:manual = 手动(实时老师的按住说话:松手 commit);semantic_vad / server_vad = 服务端判(实时老师的一直听、热身后打电话那段;一直听时 create_response / interrupt_response 关着,回不回、掐不掐由应用定)。
+ *   一直听的真连(2026-10-20,scripts/probe-live-vad.mjs):孩子还在说时就有 conversation.item.created(不算收下)、转写片段;speech_stopped 与 committed 同一刻,
+ *   转写约 0.2 秒后到;semantic_vad 不收的一段回 error「not accepted by semantic turn detection」(走 onRejected,不是错);commit 了 300 秒没回答报 response_idle_timeout,会话死。
+ *   semantic_vad 一两个词的回答(好的、Yes、Red)不报开口,嘴先问了也不报;噪声、「嗯」、笑声也不报。server_vad 短的都收,「嗯」、笑声也报(噪声不报)。
+ *   服务端的规矩(2026-10-10 真连):turn_detection 的 silence_duration_ms 喂过音频就不能再改(切成 null 再切回来也回到缺省 800),
+ *   type 在 null 与 vad 之间随时能切。所以跟读与聊各开一条会话(跟读手动;聊 semantic_vad 带自己的 silence),setTurn 只在不改 silence 时用。
+ * 不抛:连不上、断了、服务端报错都走 onClose / onError;send 在没连上时丢弃并回 false。
+ */
+import { bailianWorkspaceId, qianwenKeys, workspaceHost } from "../shared/qianwen.js";
+export const REALTIME_MODEL = 'qwen3.8-omni-flash-realtime';
+export const REALTIME_VOICE = 'Serena';
+export const INPUT_RATE = 16000;
+export const OUTPUT_RATE = 24000;
+/** 服务端一帧最多 262144 字节(2026-10-19 真连:超了就 close 1009,整条连接没了);一张图的 base64 连同 JSON 的壳得在这以内 */
+export const MAX_IMAGE_B64 = 262144 - 1024;
+const TRANSCRIBE_MODEL = 'qwen3-asr-flash-realtime';
+const HEARTBEAT_PCM = Buffer.alloc(INPUT_RATE * 2);
+/** 百炼按量的 key 与业务空间 id(Token Plan 没有 omni realtime;key 怎么找见 src/shared/qianwen.ts,业务空间 id 先看家长端配的、再看环境、再看 voxtell 的配置) */
+export function dashscopeAuth(env = process.env) {
+    const key = qianwenKeys(env).payg;
+    const workspaceId = bailianWorkspaceId(env);
+    return key && workspaceId ? { key, workspaceId } : null;
+}
+export const endpointOf = (workspaceId, model) => `wss://${workspaceHost(workspaceId)}/api-ws/v1/realtime?model=${model}`;
+/** 轮次模式 → session.turn_detection */
+export function turnDetection(turn, silenceMs, vad = {}) {
+    if (turn === 'manual')
+        return null;
+    return { type: turn, threshold: 0.5, prefix_padding_ms: 300, silence_duration_ms: silenceMs, create_response: vad.createResponse ?? true, interrupt_response: vad.interruptResponse ?? true };
+}
+/** session.update 的正文 */
+export function sessionConfig(o) {
+    return {
+        modalities: ['text', 'audio'],
+        instructions: o.instructions,
+        voice: o.voice,
+        audio: {
+            input: { format: { type: 'pcm', sample_rate: INPUT_RATE, sample_format: 's16le', channels: 1 } },
+            output: { voice: o.voice, format: { type: 'pcm', sample_rate: OUTPUT_RATE } },
+        },
+        turn_detection: turnDetection(o.turn, o.silenceMs, o.vad),
+        input_audio_transcription: { model: TRANSCRIBE_MODEL },
+    };
+}
+const usageOf = (r) => {
+    const u = r?.usage;
+    if (!u)
+        return null;
+    const n = (v) => (typeof v === 'number' ? v : 0);
+    const inD = (u.input_token_details ?? u.input_tokens_details);
+    const outD = (u.output_token_details ?? u.output_tokens_details);
+    return { input: n(u.input_tokens), output: n(u.output_tokens), inputAudio: n(inD?.audio_tokens), outputAudio: n(outD?.audio_tokens) };
+};
+export class RealtimeSession {
+    model;
+    voice;
+    ws = null;
+    ready = false;
+    closed = false;
+    turn;
+    silenceMs;
+    tutorText = '';
+    pendingUpdate = null;
+    /** 正在出的回答数(response.cancel 只在有回答时发,不然服务端报错) */
+    active = 0;
+    opts;
+    ev;
+    constructor(opts, ev = {}) {
+        this.opts = opts;
+        this.ev = ev;
+        this.model = opts.model ?? REALTIME_MODEL;
+        this.voice = opts.voice ?? REALTIME_VOICE;
+        this.turn = opts.turn ?? 'manual';
+        this.silenceMs = opts.silenceMs ?? 1200;
+    }
+    get isReady() { return this.ready && !this.closed; }
+    /** 老师说到一半、还没 done 的字(挂断时记进字幕用) */
+    get partialTutorText() { return this.tutorText; }
+    get turnMode() { return this.turn; }
+    get responding() { return this.active > 0; }
+    /** 连上、配好会话;回 null 成功,否则是原因(no_auth / connect / timeout / <服务端的话>) */
+    async connect() {
+        let url = this.opts.endpoint;
+        let key = '';
+        if (url)
+            key = this.opts.env?.DASHSCOPE_API_KEY ?? 'sk-test';
+        else {
+            const auth = this.opts.auth ?? dashscopeAuth(this.opts.env);
+            if (!auth)
+                return 'no_auth';
+            key = auth.key;
+            url = endpointOf(auth.workspaceId, this.model);
+        }
+        const timeoutMs = this.opts.timeoutMs ?? 10_000;
+        return new Promise((resolve) => {
+            let settled = false;
+            const finish = (why) => { if (!settled) {
+                settled = true;
+                clearTimeout(timer);
+                resolve(why);
+            } };
+            const timer = setTimeout(() => { finish('timeout'); this.close('timeout'); }, timeoutMs);
+            let ws;
+            try {
+                ws = new WebSocket(url, { headers: { authorization: `Bearer ${key}` } });
+            }
+            catch (err) {
+                finish(`connect:${err instanceof Error ? err.message : String(err)}`);
+                return;
+            }
+            this.ws = ws;
+            ws.onerror = () => { finish('connect'); };
+            ws.onclose = (e) => {
+                this.ready = false;
+                finish(`closed:${e.code}`);
+                if (!this.closed) {
+                    this.closed = true;
+                    this.ev.onClose?.(`closed:${e.code}${e.reason ? ` ${e.reason}` : ''}`);
+                }
+            };
+            ws.onmessage = (m) => {
+                let e;
+                try {
+                    e = JSON.parse(String(m.data));
+                }
+                catch {
+                    return;
+                }
+                if (e.type === 'session.created') {
+                    this.sendRaw({ type: 'session.update', session: sessionConfig({ instructions: this.opts.instructions, voice: this.voice, turn: this.turn, silenceMs: this.silenceMs, vad: this.opts.vad }) });
+                    return;
+                }
+                if (e.type === 'session.updated') {
+                    if (!this.ready) {
+                        this.ready = true;
+                        finish(null);
+                    }
+                    if (this.pendingUpdate) {
+                        clearTimeout(this.pendingUpdate.timer);
+                        this.pendingUpdate.resolve(true);
+                        this.pendingUpdate = null;
+                    }
+                    return;
+                }
+                if (e.type === 'error') {
+                    const err = e.error;
+                    const msg = `${err?.code ? `${err.code}: ` : ''}${err?.message ?? JSON.stringify(e).slice(0, 200)}`;
+                    if (!this.ready) {
+                        finish(msg);
+                        this.close('error');
+                        return;
+                    }
+                    if (this.pendingUpdate) {
+                        clearTimeout(this.pendingUpdate.timer);
+                        this.pendingUpdate.resolve(false);
+                        this.pendingUpdate = null;
+                    }
+                    // semantic_vad 不收的一段(2026-10-20 照 Qwen-Live-Harness:不致命,回到在听)
+                    if (/not accepted by semantic turn detection/i.test(err?.message ?? '')) {
+                        this.ev.onRejected?.();
+                        return;
+                    }
+                    this.ev.onError?.(msg);
+                    // user_idle_timeout / response_idle_timeout:服务端说会话关了,连接却还开着,再发什么都没回音(2026-10-19 ray 那次挂了 10 分钟才 close),当场关
+                    if (err?.code?.endsWith('_idle_timeout'))
+                        this.close(`idle:${err.code}`);
+                    return;
+                }
+                this.handle(e);
+            };
+        });
+    }
+    handle(e) {
+        switch (e.type) {
+            case 'response.audio.delta':
+                if (typeof e.delta === 'string')
+                    this.ev.onAudio?.(Buffer.from(e.delta, 'base64'));
+                break;
+            case 'response.audio_transcript.delta':
+            case 'response.text.delta':
+                if (typeof e.delta === 'string') {
+                    this.tutorText += e.delta;
+                    this.ev.onTutorText?.(e.delta, null);
+                }
+                break;
+            case 'response.audio_transcript.done':
+            case 'response.text.done': {
+                const whole = typeof e.transcript === 'string' ? e.transcript : typeof e.text === 'string' ? e.text : this.tutorText;
+                this.tutorText = '';
+                this.ev.onTutorText?.('', whole);
+                break;
+            }
+            case 'conversation.item.input_audio_transcription.completed':
+                if (typeof e.transcript === 'string')
+                    this.ev.onKidText?.(e.transcript.trim());
+                break;
+            case 'conversation.item.input_audio_transcription.failed':
+                this.ev.onKidFailed?.();
+                break;
+            case 'input_audio_buffer.speech_started':
+                this.ev.onSpeech?.('started');
+                break;
+            case 'input_audio_buffer.speech_stopped':
+                this.ev.onSpeech?.('stopped');
+                break;
+            case 'response.created':
+                this.active++;
+                this.ev.onResponse?.('created', null, null);
+                break;
+            case 'response.done': {
+                this.active = Math.max(0, this.active - 1);
+                const r = e.response;
+                this.ev.onResponse?.('done', r?.status ?? null, usageOf(r));
+                break;
+            }
+            default: break;
+        }
+    }
+    sendRaw(msg) {
+        if (!this.ws || this.ws.readyState !== WebSocket.OPEN)
+            return false;
+        this.ws.send(JSON.stringify(msg));
+        return true;
+    }
+    /** 一块孩子的声音(PCM16 16k 单声道) */
+    appendAudio(pcm) { return this.isReady && this.sendRaw({ type: 'input_audio_buffer.append', audio: pcm.toString('base64') }); }
+    /** 一张图(JPEG)跟着这一段声音交:百炼要先有声音再有图(Qwen-Live-Harness 也这样),commit 时一起进对话;太大(见 MAX_IMAGE_B64)不发、回 false,调用方先缩 */
+    appendImage(jpeg) {
+        const image = jpeg.toString('base64');
+        return this.isReady && image.length <= MAX_IMAGE_B64 && this.sendRaw({ type: 'input_image_buffer.append', image });
+    }
+    /** 手动轮次:松手 */
+    commit() { return this.isReady && this.sendRaw({ type: 'input_audio_buffer.commit' }); }
+    clearAudio() { return this.isReady && this.sendRaw({ type: 'input_audio_buffer.clear' }); }
+    /**
+     * 心跳:1 秒全零的声音接着 clear,不 commit、不要回答(照 Qwen-Live-Harness 的静音心跳)。百炼 300 秒没输入就关会话(user_idle_timeout)。
+     * 2026-10-20 真连(scripts/probe-live-idle.mjs,手动轮次,每 30 秒一次):不发的 302 秒被关;只 append 不 clear 的 300 秒报 response_idle_timeout
+     * (缓冲里有输入一直没回答),会话就死了;append 完马上 clear 的过了 400 秒照常说话。
+     */
+    heartbeat() { return this.appendAudio(HEARTBEAT_PCM) && this.clearAudio(); }
+    /** 让老师说;instructions = 只管这一次回答的指令(response.instructions,不进对话历史、不改系统提示) */
+    createResponse(instructions) { return this.isReady && this.sendRaw({ type: 'response.create', ...(instructions ? { response: { instructions } } : {}) }); }
+    /** 老师正在说就掐掉(没在说不发,服务端会报错) */
+    cancel() { return this.isReady && this.active > 0 && this.sendRaw({ type: 'response.cancel' }); }
+    /** 应用塞一条文字进对话(口语课跟读段「现在读这句」、切段的指令);role 缺省 user */
+    addText(text, role = 'user') {
+        return this.isReady && this.sendRaw({ type: 'conversation.item.create', item: { type: 'message', role, content: [{ type: 'input_text', text }] } });
+    }
+    /** 中途在手动与 VAD 之间切(只给 type,silence 喂过音频后改不了);等 session.updated,错或超时回 false */
+    setTurn(turn, timeoutMs = 5000) {
+        if (!this.isReady)
+            return Promise.resolve(false);
+        return new Promise((resolve) => {
+            if (this.pendingUpdate) {
+                clearTimeout(this.pendingUpdate.timer);
+                this.pendingUpdate.resolve(false);
+            }
+            const timer = setTimeout(() => { this.pendingUpdate = null; resolve(false); }, timeoutMs);
+            this.pendingUpdate = { resolve: (ok) => { if (ok)
+                    this.turn = turn; resolve(ok); }, timer };
+            this.sendRaw({ type: 'session.update', session: { turn_detection: turn === 'manual' ? null : { type: turn, create_response: true, interrupt_response: true } } });
+        });
+    }
+    close(reason = 'bye') {
+        if (this.closed)
+            return;
+        this.closed = true;
+        this.ready = false;
+        try {
+            this.ws?.close(1000, reason);
+        }
+        catch { /* 已经断了 */ }
+        this.ev.onClose?.(reason);
+    }
+}

package/dist/realtime/cues.d.ts

@@ -0,0 +1,51 @@
+/**
+ * 实时老师每次让嘴开口的那句指令(response.instructions):纯函数。
+ * 嘴自己不决定说什么,只两件事:几种场合先说一句(拍照、没听清、热身)、把脑写的讲稿原样念。说法是 2026-10-19 P1 实测过的(scripts/probe-live.mjs cue):
+ * 「原样说」和那句一起放进 response.instructions,4/4 一字不差;只放 system 消息、或要说的话当用户消息发,它会改写、会把问题自己答了。
+ */
+/**
+ * 孩子这一轮是怎么来的:按住说的、打字的、拍的照片、交的卡、点了首页的按钮(activity 的开场、接着);
+ * answer = 按住说的、上一句念给他的是问句(看图说话这种一问一答);empty = 按住了但转写是空的;
+ * warm = 口语单拍好、进来上口语课,嘴先说要热身
+ */
+export type CatchKind = 'voice' | 'answer' | 'text' | 'photo' | 'card' | 'start' | 'empty' | 'warm';
+/** 这一轮嘴先说的那句的指令;null = 嘴不出声。en = 口语课打电话那段 */
+export declare function catchCue(kind: CatchKind, lang?: 'zh' | 'en'): string | null;
+/**
+ * 交给嘴之前,把它念不好的写法换成念法(板上、字幕照旧是原文)。2026-10-20 真连(照念 → 声音转写回字):× + − = ÷ > 1/2 都念对;
+ * 空括号念成「空格」「括号」(「每份都是( )个」成了「每份都是空格」),□ 念得含糊(「□+5=12」听成「正五等于十二」)。
+ * 空括号、下划线的空照数学老师的说法念「几」(「二四得几」「每份都是几个」),□ 念「方框」;换了以后「方框+5」的 + 念成「正」,
+ * 所以运算符一律换成字(+ 加、− 减、× 乘、÷ 除以、= 等于、> 大于、< 小于;ASCII 的 - 只在两个数之间算减)。
+ */
+export declare function spoken(text: string): string;
+/** 照念脑写的这一拍讲稿(几句连着念);句子换成念法(spoken)、原样拼起来放进「」 */
+export declare function readCue(texts: readonly string[]): string;
+/**
+ * 边画边聊:画板开着时嘴可以自己说一句(看图说、问一句),也接住孩子画着说的话。
+ * 应用定题:什么时候说、说哪个颜色、问不问、说过什么、孩子答过什么;嘴只看图(那一刻导出的 jpeg 跟着这一段声音进了对话)说一两句具体的。
+ * 10-22 探针(qwen3.8-omni-flash-realtime,10-21 晚那张的四个时刻):清楚的形状认得出,照题目猜会错(乌云说成山、浪说成黄河)、
+ * 会原样重复上一句、会用大人词(生动、翻涌)。所以:不给它照题目猜,只用颜色问、用「像……」;把它说过的原话给它,不许重复。
+ */
+export interface ChatAsk {
+    /** 画板卡的题目(纸没有);只让嘴知道他想画什么,不许照它猜 */
+    prompt?: string;
+    /** 说哪种颜色(「红」);null / 没有 = 整张 */
+    color?: string | null;
+    /** 问不问(上一问没回就只说不问) */
+    ask?: boolean;
+    /** 画了多久(「约 3 分钟」) */
+    minutes?: string;
+    /** 各颜色几笔、在哪(笔多的在前) */
+    colors?: {
+        name: string;
+        strokes: number;
+        where: string;
+    }[];
+    /** 清空重画过几次 */
+    cleared?: number;
+}
+/**
+ * 边画边聊嘴那一次的指令:look = 应用觉得该说一句了(孩子换了颜色、停笔很久),对面没说话,那段声音是静音;
+ * reply = 孩子按着头像画着说了一句(听他的声音)。said = 这张画上之前说过的,一句一行(「孩子:…」「你:…」)
+ */
+export declare function chatCue(kind: 'look' | 'reply', a: ChatAsk, said?: readonly string[]): string;