cotutor 0.2.0 → 0.4.0

package/dist/runtime/talk-run.d.ts

@@ -0,0 +1,21 @@
+import type { Workspace } from '../schema/index.ts';
+import type { ActivityRun } from '../schema/index.ts';
+/** 口语单交给哪个 activity(英语老师的家里那份) */
+export declare const TALK_ACTIVITY = "\u53E3\u8BED\u8BFE";
+interface Item {
+    en: string;
+    zh?: string;
+    emoji?: string;
+    chunks?: string[];
+}
+/** 口语单写成一段字:单词一行一个(emoji、英文、中文、拼读分段),句子一行一句(英文、中文) */
+export declare function talkListText(list: {
+    words: readonly Item[];
+    sentences: readonly Item[];
+}): string;
+/** body.talk = {id}:那份口语单在、有词或句子、这位老师有「口语课」activity → 这个话题的 activity;不然 null */
+export declare function talkRun(ws: Workspace, tutor: string, talk: unknown): Promise<{
+    run: ActivityRun;
+    id: string;
+} | null>;
+export {};

package/dist/runtime/talk-run.js

@@ -0,0 +1,34 @@
+/**
+ * 口语课交给英语老师的实时页:口语课页拍好、改好的那份口语单 → activity「口语课」的参数。
+ * 只读口语课的数据文件(talk/<id>/talk.json),不引用 src/app/talk/(那边只往这边引用);口语课页不在了,这条路自然走不通。
+ */
+import { readFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { activityFile } from "./activities.js";
+import { parseActivity, resolveParams } from "./activity.js";
+/** 口语单交给哪个 activity(英语老师的家里那份) */
+export const TALK_ACTIVITY = '口语课';
+/** 同 src/scenes/talk/store.ts 的 ID_RE */
+const TALK_ID_RE = /^\d{4}-\d{2}-\d{2}-\d{4}(?:-\d{1,2})?$/;
+/** 口语单写成一段字:单词一行一个(emoji、英文、中文、拼读分段),句子一行一句(英文、中文) */
+export function talkListText(list) {
+    const words = list.words.filter((w) => w.en.trim()).map((w) => [w.emoji, w.en.trim(), w.zh, w.chunks && w.chunks.length > 1 ? w.chunks.join('-') : ''].filter(Boolean).join(' '));
+    const sentences = list.sentences.filter((s) => s.en.trim()).map((s) => [s.en.trim(), s.zh].filter(Boolean).join(' '));
+    return [...(words.length ? ['单词:', ...words] : []), ...(sentences.length ? ['句子:', ...sentences] : [])].join('\n');
+}
+/** body.talk = {id}:那份口语单在、有词或句子、这位老师有「口语课」activity → 这个话题的 activity;不然 null */
+export async function talkRun(ws, tutor, talk) {
+    const id = typeof talk === 'object' && talk !== null && typeof talk.id === 'string' ? talk.id : '';
+    if (!TALK_ID_RE.test(id))
+        return null;
+    const meta = await readFile(join(ws.root, 'talk', id, 'talk.json'), 'utf8').then((t) => JSON.parse(t), () => null);
+    const text = meta?.list ? talkListText({ words: meta.list.words ?? [], sentences: meta.list.sentences ?? [] }) : '';
+    if (!text)
+        return null;
+    const file = await readFile(activityFile(ws.root, tutor, TALK_ACTIVITY), 'utf8').catch(() => null);
+    if (file === null)
+        return null;
+    const { talk: cfg } = ws.config;
+    const r = resolveParams(parseActivity(file).activity, { 口语单: text, 打电话几分钟: String(cfg.minutes), ...(cfg.note ? { 家长的话: cfg.note } : {}) });
+    return r.issues.length ? null : { run: { name: TALK_ACTIVITY, params: r.values }, id };
+}

package/dist/scenes/dictation/match.d.ts

@@ -0,0 +1,57 @@
+/**
+ * 听写的比对:孩子在田字格里写的每一笔,对 hanzi-writer-data 的中线(medians),判这个字写没写全、先后对不对、占格偏不偏。
+ * 一笔像不像一笔照搬 hanzi-writer 3.7.2 的 strokeMatches(MIT,Copyright (c) 2014 David Chanin):均距、起止点、方向、Fréchet 形状、长度五项;
+ * 它是描红时一笔一笔现判(知道该写第几笔),听写是写完了一起判,所以这里另做一步配对:每一笔对每一笔都试,按均距从近到远一对一配上。
+ * 坐标同 hanzi-writer:1024 见方、y 向上(页面上 y = 900 − 屏幕 y)。纯函数,离屏可测。
+ */
+export interface Pt {
+    x: number;
+    y: number;
+}
+/** 孩子写的一笔:[x, y, t](t 毫秒,只用来回放,比对不看) */
+export type InkPoint = [number, number, number];
+export interface StrokeFit {
+    isMatch: boolean;
+    avgDist: number;
+    /** 正着不像、倒着像:这一笔写反了方向 */
+    backwards: boolean;
+}
+/** 孩子的一笔像不像标准的这一笔(hanzi-writer 的 getMatchData,门槛见 AVG_DIST_THRESHOLD) */
+export declare function strokeFit(raw: Pt[], median: Pt[], leniency?: number, checkBackwards?: boolean): StrokeFit;
+/** 第二遍放宽多少:点、短撇这种短笔手一抖形状就差得远,第一遍剩下的笔两两再试一次 */
+export declare const SECOND_PASS_LENIENCY = 1.5;
+export interface CharJudge {
+    /** 数据里有这个字、判过了;没有(生僻字)= false,别的字段都是空的 */
+    judged: boolean;
+    /** 一笔没写 */
+    empty: boolean;
+    /** 标准几笔、孩子写了几笔 */
+    std: number;
+    wrote: number;
+    /** 孩子的第 k 笔是标准的第几笔(0 起);对不上 null */
+    matched: (number | null)[];
+    /** 标准里没配上的笔(0 起):少写的,或写得认不出的 */
+    missing: number[];
+    /** 孩子写的、哪一笔都对不上的(0 起) */
+    extra: number[];
+    /** 配上的笔,先后和书上一样 */
+    order: boolean;
+    /** 写反了方向的笔(标准的,0 起) */
+    backwards: number[];
+    /** 占格:孩子的字相对书上的,size = 大小之比,dx / dy = 中心偏多少(1024 见方,y 向上) */
+    place: {
+        size: number;
+        dx: number;
+        dy: number;
+    } | null;
+    /** 孩子端问不问「这个再写一遍?」:空着、少笔、多笔、有一笔对不上 */
+    ask: boolean;
+}
+export declare const EMPTY_JUDGE: Omit<CharJudge, 'judged' | 'ask' | 'std'>;
+/**
+ * 判一个字:先原样比,再把孩子的字挪进标准字的框里比,取配上更多的那次。
+ * medians 是 hanzi-writer-data 的(null = 数据里没有,不判)。
+ */
+export declare function judgeChar(ink: InkPoint[][], medians: number[][][] | null, leniency?: number): CharJudge;
+/** 占格:偏小 / 偏大 / 偏左上……;正常 → null(给家长的一句) */
+export declare function placeNote(place: CharJudge['place']): string | null;

package/dist/scenes/dictation/match.js

@@ -0,0 +1,255 @@
+/**
+ * 听写的比对:孩子在田字格里写的每一笔,对 hanzi-writer-data 的中线(medians),判这个字写没写全、先后对不对、占格偏不偏。
+ * 一笔像不像一笔照搬 hanzi-writer 3.7.2 的 strokeMatches(MIT,Copyright (c) 2014 David Chanin):均距、起止点、方向、Fréchet 形状、长度五项;
+ * 它是描红时一笔一笔现判(知道该写第几笔),听写是写完了一起判,所以这里另做一步配对:每一笔对每一笔都试,按均距从近到远一对一配上。
+ * 坐标同 hanzi-writer:1024 见方、y 向上(页面上 y = 900 − 屏幕 y)。纯函数,离屏可测。
+ */
+// ---- 照搬 hanzi-writer 的几何 ----
+const sub = (a, b) => ({ x: a.x - b.x, y: a.y - b.y });
+const mag = (p) => Math.sqrt(p.x * p.x + p.y * p.y);
+const dist = (a, b) => mag(sub(a, b));
+const mean = (xs) => xs.reduce((a, b) => a + b, 0) / xs.length;
+const last = (xs) => xs[xs.length - 1];
+function polyLength(pts) {
+    let n = 0;
+    for (let i = 1; i < pts.length; i++)
+        n += dist(pts[i], pts[i - 1]);
+    return n;
+}
+const cosine = (a, b) => (a.x * b.x + a.y * b.y) / mag(a) / mag(b);
+/** p1、p2 连线上、过 p2 再走 d 的点 */
+function extend(p1, p2, d) {
+    const v = sub(p2, p1);
+    const k = d / mag(v);
+    return { x: p2.x + k * v.x, y: p2.y + k * v.y };
+}
+function frechet(c1, c2) {
+    const long = c1.length >= c2.length ? c1 : c2;
+    const short = c1.length >= c2.length ? c2 : c1;
+    let prev = [];
+    for (let i = 0; i < long.length; i++) {
+        const cur = [];
+        for (let j = 0; j < short.length; j++) {
+            const d = dist(long[i], short[j]);
+            if (i === 0 && j === 0)
+                cur.push(d);
+            else if (i > 0 && j === 0)
+                cur.push(Math.max(prev[0], d));
+            else if (i === 0)
+                cur.push(Math.max(cur[j - 1], d));
+            else
+                cur.push(Math.max(Math.min(prev[j], prev[j - 1], cur[j - 1]), d));
+        }
+        prev = cur;
+    }
+    return prev[short.length - 1];
+}
+function subdivide(curve, maxLen = 0.05) {
+    const out = curve.slice(0, 1);
+    for (const p of curve.slice(1)) {
+        const q = last(out);
+        const len = dist(p, q);
+        if (len > maxLen) {
+            const n = Math.ceil(len / maxLen);
+            const step = len / n;
+            for (let i = 0; i < n; i++)
+                out.push(extend(p, q, -1 * step * (i + 1)));
+        }
+        else
+            out.push(p);
+    }
+    return out;
+}
+/** 沿曲线等距取 n 个点 */
+function outline(curve, n = 30) {
+    const seg = polyLength(curve) / (n - 1);
+    const out = [curve[0]];
+    const end = last(curve);
+    const rest = curve.slice(1);
+    for (let i = 0; i < n - 2; i++) {
+        let p = last(out);
+        let remain = seg;
+        for (;;) {
+            if (!rest.length)
+                break;
+            const d = dist(p, rest[0]);
+            if (d < remain) {
+                remain -= d;
+                p = rest.shift();
+            }
+            else {
+                out.push(extend(p, rest[0], remain - d));
+                break;
+            }
+        }
+        if (!rest.length)
+            break;
+    }
+    out.push(end);
+    return out;
+}
+function normalize(curve) {
+    const o = outline(curve);
+    const m = { x: mean(o.map((p) => p.x)), y: mean(o.map((p) => p.y)) };
+    const t = o.map((p) => sub(p, m));
+    const scale = Math.sqrt(mean([t[0].x ** 2 + t[0].y ** 2, last(t).x ** 2 + last(t).y ** 2])) || 1;
+    return subdivide(t.map((p) => ({ x: p.x / scale, y: p.y / scale })));
+}
+const rotate = (curve, th) => curve.map((p) => ({ x: Math.cos(th) * p.x - Math.sin(th) * p.y, y: Math.sin(th) * p.x + Math.cos(th) * p.y }));
+const COSINE_SIMILARITY_THRESHOLD = 0;
+const START_AND_END_DIST_THRESHOLD = 250;
+const FRECHET_THRESHOLD = 0.4;
+const MIN_LEN_THRESHOLD = 0.35;
+const SHAPE_FIT_ROTATIONS = [Math.PI / 16, Math.PI / 32, 0, (-1 * Math.PI) / 32, (-1 * Math.PI) / 16];
+/**
+ * 均距的门槛:hanzi-writer 缺省 350,第二笔起(描红时字的位置已经有了)× 0.5。
+ * 听写没有描的字,但比对前字已经对齐过(fitTo),所以每一笔都用 × 0.5 那一档。
+ */
+const AVG_DIST_THRESHOLD = 350 * 0.5;
+function shapeFit(a, b, leniency) {
+    const na = normalize(a);
+    const nb = normalize(b);
+    let min = Infinity;
+    for (const th of SHAPE_FIT_ROTATIONS)
+        min = Math.min(min, frechet(na, rotate(nb, th)));
+    return min <= FRECHET_THRESHOLD * leniency;
+}
+function directionMatches(points, median) {
+    const edges = [];
+    for (let i = 1; i < points.length; i++)
+        edges.push(sub(points[i], points[i - 1]));
+    const vecs = [];
+    for (let i = 1; i < median.length; i++)
+        vecs.push(sub(median[i], median[i - 1]));
+    const sims = edges.map((e) => Math.max(...vecs.map((v) => cosine(v, e))));
+    return mean(sims) > COSINE_SIMILARITY_THRESHOLD;
+}
+const avgDistance = (points, median) => mean(points.map((p) => Math.min(...median.map((m) => dist(m, p)))));
+function dedupe(points) {
+    const out = points.slice(0, 1);
+    for (const p of points.slice(1))
+        if (p.x !== last(out).x || p.y !== last(out).y)
+            out.push(p);
+    return out;
+}
+/** 孩子的一笔像不像标准的这一笔(hanzi-writer 的 getMatchData,门槛见 AVG_DIST_THRESHOLD) */
+export function strokeFit(raw, median, leniency = 1, checkBackwards = true) {
+    const points = dedupe(raw);
+    if (points.length < 2 || median.length < 2)
+        return { isMatch: false, avgDist: Infinity, backwards: false };
+    const avgDist = avgDistance(points, median);
+    if (avgDist > AVG_DIST_THRESHOLD * leniency)
+        return { isMatch: false, avgDist, backwards: false };
+    const isMatch = dist(median[0], points[0]) <= START_AND_END_DIST_THRESHOLD * leniency &&
+        dist(last(median), last(points)) <= START_AND_END_DIST_THRESHOLD * leniency &&
+        directionMatches(points, median) &&
+        shapeFit(points, median, leniency) &&
+        (leniency * (polyLength(points) + 25)) / (polyLength(median) + 25) >= MIN_LEN_THRESHOLD;
+    if (!isMatch && checkBackwards && strokeFit([...points].reverse(), median, leniency, false).isMatch)
+        return { isMatch: false, avgDist, backwards: true };
+    return { isMatch, avgDist, backwards: false };
+}
+function boxOf(strokes) {
+    const all = strokes.flat();
+    if (!all.length)
+        return null;
+    const xs = all.map((p) => p.x);
+    const ys = all.map((p) => p.y);
+    const [x0, x1, y0, y1] = [Math.min(...xs), Math.max(...xs), Math.min(...ys), Math.max(...ys)];
+    return { cx: (x0 + x1) / 2, cy: (y0 + y1) / 2, size: Math.max(x1 - x0, y1 - y0, 1) };
+}
+/** 把孩子的字等比挪到标准字的框里(写偏了、写小了也照样能比笔画) */
+function fitTo(strokes, from, to) {
+    const k = to.size / from.size;
+    return strokes.map((s) => s.map((p) => ({ x: (p.x - from.cx) * k + to.cx, y: (p.y - from.cy) * k + to.cy })));
+}
+/** 第二遍放宽多少:点、短撇这种短笔手一抖形状就差得远,第一遍剩下的笔两两再试一次 */
+export const SECOND_PASS_LENIENCY = 1.5;
+/**
+ * 每一笔对每一笔都试,像的按均距从近到远一对一配上;倒着写像的也算配上,记在 backwards。
+ * 第一遍剩下的(孩子的、标准的都没配上的)再放宽到 SECOND_PASS_LENIENCY 配一遍。
+ */
+function pair(kid, medians, leniency) {
+    const matched = kid.map(() => null);
+    const used = new Set();
+    const backwards = [];
+    let total = 0;
+    for (const len of [leniency, leniency * SECOND_PASS_LENIENCY]) {
+        const cand = [];
+        kid.forEach((pts, k) => {
+            if (matched[k] !== null)
+                return;
+            medians.forEach((m, s) => {
+                if (used.has(s))
+                    return;
+                const f = strokeFit(pts, m, len);
+                if (f.isMatch || f.backwards)
+                    cand.push({ k, s, d: f.avgDist, back: f.backwards });
+            });
+        });
+        cand.sort((a, b) => a.d - b.d);
+        for (const c of cand) {
+            if (matched[c.k] !== null || used.has(c.s))
+                continue;
+            matched[c.k] = c.s;
+            used.add(c.s);
+            total += c.d;
+            if (c.back)
+                backwards.push(c.s);
+        }
+    }
+    return { matched, backwards: backwards.sort((a, b) => a - b), dist: total };
+}
+export const EMPTY_JUDGE = { empty: true, wrote: 0, matched: [], missing: [], extra: [], order: true, backwards: [], place: null };
+/**
+ * 判一个字:先原样比,再把孩子的字挪进标准字的框里比,取配上更多的那次。
+ * medians 是 hanzi-writer-data 的(null = 数据里没有,不判)。
+ */
+export function judgeChar(ink, medians, leniency = 1) {
+    const kid = ink.map((s) => s.map(([x, y]) => ({ x, y }))).filter((s) => s.length > 0);
+    if (!medians)
+        return { judged: false, ask: false, std: 0, ...EMPTY_JUDGE, empty: kid.length === 0, wrote: kid.length };
+    const std = medians.map((m) => m.map(([x, y]) => ({ x, y })));
+    if (!kid.length)
+        return { judged: true, ask: true, std: std.length, ...EMPTY_JUDGE, missing: std.map((_, i) => i) };
+    const kb = boxOf(kid);
+    const sb = boxOf(std);
+    const raw = pair(kid, std, leniency);
+    const fitted = pair(fitTo(kid, kb, sb), std, leniency);
+    const count = (p) => p.matched.filter((x) => x !== null).length;
+    const best = count(fitted) > count(raw) || (count(fitted) === count(raw) && fitted.dist < raw.dist) ? fitted : raw;
+    const hit = new Set(best.matched.filter((x) => x !== null));
+    const missing = std.map((_, i) => i).filter((i) => !hit.has(i));
+    const extra = best.matched.map((m, k) => (m === null ? k : -1)).filter((k) => k >= 0);
+    const seq = best.matched.filter((x) => x !== null);
+    const order = seq.every((s, i) => i === 0 || s > seq[i - 1]);
+    return {
+        judged: true,
+        empty: false,
+        std: std.length,
+        wrote: kid.length,
+        matched: best.matched,
+        missing,
+        extra,
+        order,
+        backwards: best.backwards,
+        place: { size: round2(kb.size / sb.size), dx: Math.round(kb.cx - sb.cx), dy: Math.round(kb.cy - sb.cy) },
+        ask: missing.length > 0 || extra.length > 0,
+    };
+}
+const round2 = (n) => Math.round(n * 100) / 100;
+/** 占格:偏小 / 偏大 / 偏左上……;正常 → null(给家长的一句) */
+export function placeNote(place) {
+    if (!place)
+        return null;
+    const bits = [];
+    if (place.size < 0.7)
+        bits.push('偏小');
+    else if (place.size > 1.3)
+        bits.push('偏大');
+    const v = place.dy > 140 ? '上' : place.dy < -140 ? '下' : '';
+    const h = place.dx < -140 ? '左' : place.dx > 140 ? '右' : '';
+    if (v || h)
+        bits.push(`偏${h}${v}`);
+    return bits.length ? bits.join('、') : null;
+}

package/dist/scenes/dictation/recognize.d.ts

@@ -0,0 +1,86 @@
+import { type DictationLang } from '../../board/cards/index.ts';
+import type { Look } from '../look.ts';
+export declare const VISION_TIMEOUT_MS = 120000;
+/** 一栏:char 写字(田字格里的)/ word 词语表、读读写写 / mark 课文里课本标出来的 / list 听写单上的 / read 识字(会认就行)/ other */
+export type PhotoKind = 'char' | 'word' | 'mark' | 'list' | 'read' | 'other';
+export declare const PHOTO_KINDS: readonly PhotoKind[];
+export interface PhotoItem {
+    text: string;
+    kind: PhotoKind;
+    /** 默认选上(模型的判断;没说就按栏) */
+    pick: boolean;
+    /** 在照片上的框 [左, 上, 右, 下],0–1000(按照片宽高的千分比);没给 null */
+    box: [number, number, number, number] | null;
+    /** 认准了;糊的、反光的、拿不准的 false */
+    sure: boolean;
+    /** 老师怎么念 */
+    say: string | null;
+    /** 英文单词配的 emoji(只在对一对时出);汉字的、抽象的词没有 */
+    emoji?: string;
+}
+/** 认到这时为止的全部:挑出的字词、课名、全文(一行一个字符串,词之间空一格) */
+export interface PhotoProgress {
+    items: PhotoItem[];
+    lesson: string | null;
+    full: string[];
+}
+export type Recognized = ({
+    ok: true;
+    model: string;
+} | {
+    ok: false;
+    error: string;
+}) & PhotoProgress & {
+    ms: number;
+};
+/** 认一张照片;每收到一行调一次 onProgress;lang = en 认英文单词(英语老师卡上的拍照听写) */
+export type Recognize = (image: {
+    data: Buffer;
+    mime: string;
+}, onProgress: (p: PhotoProgress) => void, lang?: DictationLang) => Promise<Recognized>;
+/** 回什么格式:接在家长的要求后面,解析靠它 */
+export declare const FORMAT = "\u56DE\u7B54\u7684\u683C\u5F0F:\u4E00\u884C\u4E00\u6BB5 JSON,\u4E0D\u8981\u522B\u7684\u5B57\u3001\u4E0D\u8981\u4EE3\u7801\u5757\u3002\u6309\u8FD9\u4E2A\u987A\u5E8F\u5199:\n1. \u7B2C\u4E00\u884C\u5199\u8BFE\u540D:{\"lesson\": \"\u5C0F\u874C\u86AA\u627E\u5988\u5988\"},\u6CA1\u6709\u8BFE\u540D\u5199 {\"lesson\": null}\u3002\n2. \u6311\u51FA\u7684\u5B57\u8BCD,\u4E00\u9879\u4E00\u884C,\u9ED8\u8BA4\u9009\u4E0A\u7684\u5148\u5199:\n{\"text\": \"\u8111\u888B\", \"kind\": \"word\", \"pick\": true, \"box\": [\u5DE6, \u4E0A, \u53F3, \u4E0B], \"sure\": true, \"say\": \"\u8111\u888B,\u5C0F\u874C\u86AA\u7684\u5927\u8111\u888B\"}\n   - text:\u53EA\u5199\u6C49\u5B57,\u4E00\u4E2A\u5B57\u6216\u4E00\u4E2A\u8BCD,\u6700\u591A\u56DB\u4E2A\u5B57;\u7167\u7247\u4E0A\u600E\u4E48\u5206\u5C31\u600E\u4E48\u5206\u3002\n   - kind:\u300C\u5199\u5B57\u300D\u680F\u5199 char;\u300C\u8BCD\u8BED\u8868\u300D\u300C\u8BFB\u8BFB\u5199\u5199\u300D\u5199 word;\u8BFE\u6587\u91CC\u8BFE\u672C\u6807\u51FA\u6765\u7684\u5199 mark;\u542C\u5199\u5355\u4E0A\u7684\u5199 list;\u300C\u8BC6\u5B57\u300D\u680F\u5199 read;\u522B\u7684\u5199 other\u3002\n   - pick:\u9ED8\u8BA4\u9009\u4E0D\u9009(true / false)\u3002\n   - box:\u8FD9\u4E00\u9879\u5728\u7167\u7247\u4E0A\u7684\u4F4D\u7F6E,\u6309\u7167\u7247\u5BBD\u3001\u9AD8\u7684\u5343\u5206\u6BD4,0 \u5230 1000 \u7684\u6574\u6570\u3002\n   - sure:\u770B\u6E05\u4E86 true;\u7CCA\u3001\u53CD\u5149\u3001\u88AB\u6321\u4F4F\u3001\u62FF\u4E0D\u51C6\u5199 false,text \u5199\u4F60\u89C9\u5F97\u6700\u53EF\u80FD\u7684\u5B57\u3002\n   - say:\u5FF5\u6CD5\u3002\n3. \u6700\u540E\u662F\u5168\u6587:\u7167\u7247\u4E0A\u7684\u6B63\u6587\u7167\u539F\u6837\u4E00\u884C\u5199\u4E00\u884C {\"full\": \"\u51AC\u5929\u6765\u5566!\u4E00\u573A\u5927\u96EA\u8FC7\u540E\"};\u4E0D\u8981\u62FC\u97F3\u3001\u9875\u7801\u3001\u680F\u76EE\u540D\u3002";
+/** 英文单词的格式:一项一个单词;kind 只分 word(单词表、要会写的)与 read(会认就行)、other */
+export declare const FORMAT_EN = "\u56DE\u7B54\u7684\u683C\u5F0F:\u4E00\u884C\u4E00\u6BB5 JSON,\u4E0D\u8981\u522B\u7684\u5B57\u3001\u4E0D\u8981\u4EE3\u7801\u5757\u3002\u6309\u8FD9\u4E2A\u987A\u5E8F\u5199:\n1. \u7B2C\u4E00\u884C\u5199\u5355\u5143\u540D:{\"lesson\": \"Unit 3 Colours\"},\u6CA1\u6709\u5199 {\"lesson\": null}\u3002\n2. \u6311\u51FA\u7684\u5355\u8BCD,\u4E00\u9879\u4E00\u884C,\u9ED8\u8BA4\u9009\u4E0A\u7684\u5148\u5199:\n{\"text\": \"yellow\", \"kind\": \"word\", \"pick\": true, \"box\": [\u5DE6, \u4E0A, \u53F3, \u4E0B], \"sure\": true, \"say\": \"yellow, a yellow bus\", \"emoji\": \"\uD83D\uDFE1\"}\n   - text:\u4E00\u4E2A\u82F1\u6587\u5355\u8BCD,\u53EA\u5199\u5B57\u6BCD(\u53EF\u4EE5\u5E26 - \u548C '),\u4E0D\u5E26\u7A7A\u683C;\u7167\u8BFE\u672C\u7684\u5927\u5C0F\u5199\u3002\n   - kind:\u8981\u4F1A\u5199\u7684\u5355\u8BCD\u5199 word;\u53EA\u8981\u4F1A\u8BA4\u3001\u4F1A\u8BFB\u7684\u5199 read;\u522B\u7684\u5199 other\u3002\n   - pick:\u9ED8\u8BA4\u9009\u4E0D\u9009(true / false)\u3002\n   - box:\u8FD9\u4E00\u9879\u5728\u7167\u7247\u4E0A\u7684\u4F4D\u7F6E,\u6309\u7167\u7247\u5BBD\u3001\u9AD8\u7684\u5343\u5206\u6BD4,0 \u5230 1000 \u7684\u6574\u6570\u3002\n   - sure:\u770B\u6E05\u4E86 true;\u7CCA\u3001\u53CD\u5149\u3001\u88AB\u6321\u4F4F\u3001\u62FF\u4E0D\u51C6\u5199 false\u3002\n   - say:\u5FF5\u6CD5,\u82F1\u6587\u3002\n   - emoji:\u80FD\u753B\u51FA\u6765\u7684\u4E1C\u897F\u914D\u4E00\u4E2A emoji(\uD83D\uDC75 \uD83D\uDE8C \uD83C\uDF4E);\u62BD\u8C61\u7684\u8BCD(like\u3001the\u3001is)\u4E0D\u5199\u8FD9\u4E00\u680F\u3002\u5B69\u5B50\u5199\u5B8C\u5BF9\u7B54\u6848\u65F6\u624D\u770B\u5230\u3002\n3. \u6700\u540E\u662F\u5168\u6587:\u7167\u7247\u4E0A\u7684\u6B63\u6587\u7167\u539F\u6837\u4E00\u884C\u5199\u4E00\u884C {\"full\": \"What colour is it? It's yellow.\"};\u4E0D\u8981\u9875\u7801\u3001\u680F\u76EE\u540D\u3002";
+/** 交给模型的整段(没有听写 activity 文件时):出厂的要求 + 格式 */
+export declare function photoPrompt(prompt?: readonly string[], lang?: DictationLang): string;
+/** 交给模型的整段:听写 activity 文件「认字收到什么」的模板填好 {{格式}} */
+export declare function photoPromptFrom(template: string, lang?: DictationLang): string;
+/**
+ * 全文的一行分成词(词之间空一格),孩子端一个词一个词点。不靠模型分:2026-10-10 同一张真照片,qwen3.8-max 一次整行连着回、
+ * 一次每个字之间都空一格;所以去掉空格,一律用 Intl.Segmenter 的中文分词(Node 与 Safari 都有),分得准不准都一个样。
+ */
+export declare function splitWords(line: string, lang?: DictationLang): string;
+/** 模型的一行 → 一项;不是一项(形状不对、不是汉字、超过四个字;lang = en:不是一个英文单词)→ null */
+export declare function parseItem(v: unknown, lang?: DictationLang): PhotoItem | null;
+/**
+ * 一行一行地收:吐出来的字攒着,遇到换行就认一行;同一栏同一个字词只留第一个。
+ * 课名、一项、一行全文都算一行;每认出一行调一次 onProgress。
+ */
+export declare class LineReader implements PhotoProgress {
+    items: PhotoItem[];
+    lesson: string | null;
+    full: string[];
+    private buf;
+    private seen;
+    private onProgress;
+    private lang;
+    constructor(onProgress?: (p: PhotoProgress) => void, lang?: DictationLang);
+    push(chunk: string): void;
+    end(): void;
+    snapshot(): PhotoProgress;
+    private line;
+}
+/** 认一张照片(流式;look = 看图的请求,src/media/vision.ts 的 lookAt);出错、超时都不抛,回 {ok: false} 并带上已经认出的 */
+export declare function recognizeWith(look: Look, image: {
+    data: Buffer;
+    mime: string;
+}, onProgress?: (p: PhotoProgress) => void, opts?: {
+    env?: NodeJS.ProcessEnv;
+    endpoint?: string;
+    model?: string;
+    prompt?: readonly string[];
+    text?: string;
+    timeoutMs?: number;
+    lang?: DictationLang;
+}): Promise<Recognized>;

package/dist/scenes/dictation/recognize.js

@@ -0,0 +1,163 @@
+/**
+ * 拍照认词表:孩子端拍课本(课文、写字、词语、识字)或老师发的听写单,交给百炼的视觉模型,一次回齐:
+ * 挑出的字词(是什么、哪一栏、在照片上的哪儿、默认选不选、认没认准、老师怎么念),再是正文全文(分好词,孩子端能点着加)。
+ * 「认哪些、默认选哪些、怎么念」在听写 activity 文件的「认字收到什么」(`{{格式}}` 插槽由这里的 FORMAT 填);
+ * workspace 里没有听写 activity 文件时照出厂的要求(DICTATION_PROMPT_DEFAULT / EN),FORMAT 接在后面。改要求不会把解析改坏。
+ * 流式:模型一行一段地吐,收到一行就交出到这时为止认出的全部,孩子端轮询着一个一个画框,不用干等整张认完。
+ * 2026-10-10 拿一页仿课本试(识字 9、写字 9、读读写写 6):qwen3.8-max 24 项全对、框都套在字上,第一项 2.8 秒、全部 23 秒;
+ * qwen3.5-omni-flash 整张 20 秒但不照一行一段回;qwen-vl-max / qwen3-vl-plus 36–37 秒。
+ * 这里是听写这个场景的规则(要求怎么拼、一行一段怎么解析);看图的请求在 src/media/vision.ts(key 与端点按模型定),由路由那层交进来。出错、超时都不抛。
+ */
+import { DICTATION_PROMPT_DEFAULT, DICTATION_PROMPT_EN_DEFAULT, DICTATION_DEFAULT } from "../../schema/index.js";
+import { fillSlots } from "../../schema/index.js";
+import { DICTATION_SAY_MAX, dictationEmoji, dictationWordLang } from "../../board/cards/index.js";
+export const VISION_TIMEOUT_MS = 120_000;
+export const PHOTO_KINDS = ['char', 'word', 'mark', 'list', 'read', 'other'];
+/** 模型没说默认选不选时,这几栏选上 */
+const PICK_BY_KIND = ['char', 'word', 'mark', 'list'];
+/** 回什么格式:接在家长的要求后面,解析靠它 */
+export const FORMAT = `回答的格式:一行一段 JSON,不要别的字、不要代码块。按这个顺序写:
+1. 第一行写课名:{"lesson": "小蝌蚪找妈妈"},没有课名写 {"lesson": null}。
+2. 挑出的字词,一项一行,默认选上的先写:
+{"text": "脑袋", "kind": "word", "pick": true, "box": [左, 上, 右, 下], "sure": true, "say": "脑袋,小蝌蚪的大脑袋"}
+   - text:只写汉字,一个字或一个词,最多四个字;照片上怎么分就怎么分。
+   - kind:「写字」栏写 char;「词语表」「读读写写」写 word;课文里课本标出来的写 mark;听写单上的写 list;「识字」栏写 read;别的写 other。
+   - pick:默认选不选(true / false)。
+   - box:这一项在照片上的位置,按照片宽、高的千分比,0 到 1000 的整数。
+   - sure:看清了 true;糊、反光、被挡住、拿不准写 false,text 写你觉得最可能的字。
+   - say:念法。
+3. 最后是全文:照片上的正文照原样一行写一行 {"full": "冬天来啦!一场大雪过后"};不要拼音、页码、栏目名。`;
+/** 英文单词的格式:一项一个单词;kind 只分 word(单词表、要会写的)与 read(会认就行)、other */
+export const FORMAT_EN = `回答的格式:一行一段 JSON,不要别的字、不要代码块。按这个顺序写:
+1. 第一行写单元名:{"lesson": "Unit 3 Colours"},没有写 {"lesson": null}。
+2. 挑出的单词,一项一行,默认选上的先写:
+{"text": "yellow", "kind": "word", "pick": true, "box": [左, 上, 右, 下], "sure": true, "say": "yellow, a yellow bus", "emoji": "🟡"}
+   - text:一个英文单词,只写字母(可以带 - 和 '),不带空格;照课本的大小写。
+   - kind:要会写的单词写 word;只要会认、会读的写 read;别的写 other。
+   - pick:默认选不选(true / false)。
+   - box:这一项在照片上的位置,按照片宽、高的千分比,0 到 1000 的整数。
+   - sure:看清了 true;糊、反光、被挡住、拿不准写 false。
+   - say:念法,英文。
+   - emoji:能画出来的东西配一个 emoji(👵 🚌 🍎);抽象的词(like、the、is)不写这一栏。孩子写完对答案时才看到。
+3. 最后是全文:照片上的正文照原样一行写一行 {"full": "What colour is it? It's yellow."};不要页码、栏目名。`;
+/** 交给模型的整段(没有听写 activity 文件时):出厂的要求 + 格式 */
+export function photoPrompt(prompt, lang = 'zh') {
+    const ask = prompt ?? (lang === 'en' ? DICTATION_PROMPT_EN_DEFAULT : DICTATION_PROMPT_DEFAULT);
+    return `${ask.join('\n')}\n\n${lang === 'en' ? FORMAT_EN : FORMAT}`;
+}
+/** 交给模型的整段:听写 activity 文件「认字收到什么」的模板填好 {{格式}} */
+export function photoPromptFrom(template, lang = 'zh') {
+    return fillSlots(template, { 格式: lang === 'en' ? FORMAT_EN : FORMAT });
+}
+const isObj = (v) => typeof v === 'object' && v !== null && !Array.isArray(v);
+const str = (v) => (typeof v === 'string' && v.trim() ? v.trim() : null);
+const ZH_WORDS = typeof Intl !== 'undefined' && 'Segmenter' in Intl ? new Intl.Segmenter('zh', { granularity: 'word' }) : null;
+/**
+ * 全文的一行分成词(词之间空一格),孩子端一个词一个词点。不靠模型分:2026-10-10 同一张真照片,qwen3.8-max 一次整行连着回、
+ * 一次每个字之间都空一格;所以去掉空格,一律用 Intl.Segmenter 的中文分词(Node 与 Safari 都有),分得准不准都一个样。
+ */
+export function splitWords(line, lang = 'zh') {
+    // 英文:只留单词(去掉标点),词之间空一格
+    if (lang === 'en')
+        return (line.match(/[A-Za-z][A-Za-z'-]*/g) ?? []).join(' ');
+    const t = line.replace(/\s+/g, '');
+    if (!ZH_WORDS)
+        return t;
+    return [...ZH_WORDS.segment(t)].map((x) => x.segment.trim()).filter(Boolean).join(' ');
+}
+/** 模型的一行 → 一项;不是一项(形状不对、不是汉字、超过四个字;lang = en:不是一个英文单词)→ null */
+export function parseItem(v, lang = 'zh') {
+    if (!isObj(v))
+        return null;
+    const t = lang === 'en' ? (str(v.text) ?? '') : (str(v.text)?.replace(/\s+/g, '') ?? '');
+    if (dictationWordLang(t) !== lang)
+        return null;
+    const kind = PHOTO_KINDS.includes(v.kind) ? v.kind : 'other';
+    const b = Array.isArray(v.box) && v.box.length === 4 && v.box.every((x) => typeof x === 'number' && Number.isFinite(x)) ? v.box.map((x) => Math.max(0, Math.min(1000, Math.round(x)))) : null;
+    const box = b && b[2] > b[0] && b[3] > b[1] ? b : null;
+    const say = str(v.say);
+    const pick = typeof v.pick === 'boolean' ? v.pick : PICK_BY_KIND.includes(kind);
+    const emoji = lang === 'en' ? dictationEmoji(v.emoji) : undefined;
+    return { text: t, kind, pick, box, sure: v.sure !== false, say: say && Array.from(say).length <= DICTATION_SAY_MAX ? say : null, ...(emoji ? { emoji } : {}) };
+}
+/**
+ * 一行一行地收:吐出来的字攒着,遇到换行就认一行;同一栏同一个字词只留第一个。
+ * 课名、一项、一行全文都算一行;每认出一行调一次 onProgress。
+ */
+export class LineReader {
+    items = [];
+    lesson = null;
+    full = [];
+    buf = '';
+    seen = new Set();
+    onProgress;
+    lang;
+    constructor(onProgress = () => { }, lang = 'zh') {
+        this.onProgress = onProgress;
+        this.lang = lang;
+    }
+    push(chunk) {
+        this.buf += chunk;
+        let i;
+        while ((i = this.buf.indexOf('\n')) >= 0) {
+            this.line(this.buf.slice(0, i));
+            this.buf = this.buf.slice(i + 1);
+        }
+    }
+    end() {
+        this.line(this.buf);
+        this.buf = '';
+    }
+    snapshot() {
+        return { items: [...this.items], lesson: this.lesson, full: [...this.full] };
+    }
+    line(raw) {
+        const s = raw.trim().replace(/,$/, '');
+        if (!s.startsWith('{') || !s.endsWith('}'))
+            return;
+        let v;
+        try {
+            v = JSON.parse(s);
+        }
+        catch {
+            return;
+        }
+        if (!isObj(v))
+            return;
+        if ('lesson' in v && !('text' in v)) {
+            this.lesson = str(v.lesson);
+        }
+        else if ('full' in v && !('text' in v)) {
+            const f = str(v.full);
+            if (!f)
+                return;
+            this.full.push(splitWords(f, this.lang));
+        }
+        else {
+            const it = parseItem(v, this.lang);
+            if (!it)
+                return;
+            const key = `${it.kind}\n${it.text}`;
+            if (this.seen.has(key))
+                return;
+            this.seen.add(key);
+            this.items.push(it);
+        }
+        this.onProgress(this.snapshot());
+    }
+}
+/** 认一张照片(流式;look = 看图的请求,src/media/vision.ts 的 lookAt);出错、超时都不抛,回 {ok: false} 并带上已经认出的 */
+export async function recognizeWith(look, image, onProgress = () => { }, opts = {}) {
+    const t0 = Date.now();
+    const ms = () => Date.now() - t0;
+    const reader = new LineReader(onProgress, opts.lang);
+    const fail = (error) => ({ ok: false, error, ...reader.snapshot(), ms: ms() });
+    const model = opts.model ?? DICTATION_DEFAULT.model;
+    const r = await look({ image, prompt: opts.text ?? photoPrompt(opts.prompt, opts.lang), model, params: { temperature: 0.1, enable_thinking: false }, timeoutMs: opts.timeoutMs ?? VISION_TIMEOUT_MS, env: opts.env, endpoint: opts.endpoint, onDelta: (d) => reader.push(d) });
+    if (!r.ok)
+        return fail(r.error);
+    reader.end();
+    if (!reader.items.length && !reader.full.length)
+        return fail('nothing');
+    return { ok: true, model, ...reader.snapshot(), ms: ms() };
+}