dsh-fish-soul 0.11.0 → 0.11.1

package/lib/client.soul.js

@@ -225,6 +225,17 @@ window.__ModuleLoader__.load({
 								h("span", null, `${s.count} 条`),
 								opts.self[s.key] ? null : Input({ value: opts.qq[s.key] ?? "", placeholder: "QQ 号", inputMode: "numeric", "aria-label": `${s.name} 的 QQ 号`, style: { flex: "0 0 120px", width: "120px" },
 									onChange: (e) => setOpts({ ...opts, qq: { ...opts.qq, [s.key]: e.target.value } }) }))))) : null,
+						pv.coverage && pv.coverage.droppedChunks > 0 ? Row({
+							title: `${pv.coverage.truncatedPersons} 个人的较早聊天这次不会导入`,
+							desc: `每人只提炼最近 ${pv.coverage.chunksPerPerson} 段（每段约 ${pv.coverage.chunkChars} 字）。` +
+								`跳过 ${pv.coverage.droppedChunks} 段、约 ${Math.round(pv.coverage.droppedChars / 1000) / 10} 万字` +
+								(pv.coverage.droppedFromMs ? `，最早到 ${new Date(pv.coverage.droppedFromMs).toLocaleDateString()}` : "") + "。" +
+								(pv.coverage.topTruncated ?? []).map((t) => `${t.name} 跳过 ${t.droppedChunks} 段${t.keptFromMs ? `（从 ${new Date(t.keptFromMs).toLocaleDateString()} 起导入）` : ""}`).join("；")
+						}, Tag("部分导入", "warning")) : null,
+						pv.coverage && Number(opts.maxCalls) < pv.coverage.keptChunks ? Row({
+							title: `调用上限 ${opts.maxCalls} 次，还有 ${pv.coverage.keptChunks - Number(opts.maxCalls)} 段会跳过`,
+							desc: `要全部导入需要 ${pv.coverage.keptChunks} 次（最多 500）；不够时每人从最近的往前轮流分`
+						}, Tag("额度不够", "warning")) : null,
 						Row({ title: "最多调用模型", desc: pv.memoryCount && !pv.messageCount ? undefined : `${mapped} 个人能导入；没填 QQ 号的人跳过` },
 							Num({ min: 0, max: 500, value: opts.maxCalls, "aria-label": "最多调用模型", onChange: (e) => setOpts({ ...opts, maxCalls: e.target.value }) }), Muted("次"),
 							Button({ variant: "ghost", onClick: () => { setPv(null); setOpts(null); } }, "取消"),

package/lib/gateway.js

@@ -31,6 +31,7 @@ import { parseCollect } from './stickers.js';
 import { UNHEARD } from './voice.js';
 import { typingDelayMs, interjectScore, interjectConfig, platformOf, splitChannelId, sessionChannelOf, service } from './gateway-util.js';
 import { mediaMethods } from './gateway-media.js';
+import { Outbox, classifySendError } from './outbox.js';
 import { behaviorMethods } from './gateway-behavior.js';
 import { 该收这条, 提取正文, 提取图块, 拆成消息段, 抠正文图 } from './channels/vendor/qq-bridge.js';
 
@@ -97,6 +98,20 @@ export class ChatGateway {
 		this.douyinBrowser = douyinBrowser;
 		this.canvasConvert = canvasConvert;
 		this.soul = soul;
+		/** v0.11.1：QQ 没发出去的回复排队重发（落在 soul.db，重启也不丢） */
+		this.outbox = new Outbox({
+			db, channel, log: this.log,
+			windowMs: () => (Number(this.qqConfig().replyWindowMinutes) || 10) * 60_000,
+			onSent: (id, item) => {
+				this.rememberOwn(id, item.text);
+				this.note('sent', { channelId: item.channel_id, title: item.title, text: item.text || '[图片]', reason: `重发成功（第 ${item.attempts + 1} 次）` });
+			},
+			onGiveUp: (item, reason) => {
+				this.note('failed', { channelId: item.channel_id, title: item.title, text: item.text || '[图片]', reason });
+				this.log('warn', `[fish-soul] 放弃重发到 ${item.title || item.channel_id}：${reason}`);
+			}
+		});
+		if (this.outbox.size() > 0) this.outbox.schedule(3000);
 		/** v0.7：语音（听 / 说）、收藏表情包、学群里的说法（都可以没有） */
 		this.voice = voice;
 		this.stickers = stickers;
@@ -258,6 +273,7 @@ export class ChatGateway {
 
 	stop() {
 		this.stopped = true;
+		this.outbox?.stop();
 		for (const b of this.batches.values()) clearTimeout(b.timer);
 		this.batches.clear();
 		for (const w of this.waiters.values()) { clearTimeout(w.timer); w.resolve(false); }
@@ -1020,15 +1036,33 @@ export class ChatGateway {
 		// 语音消息不能带引用
 		if (replyTo && messages[0][0]?.type !== 'record') messages[0].unshift({ type: 'reply', data: { id: String(replyTo) } });
 		let allOk = true;
+		// 这个聊天里还有没重发完的：新回复排到后面，顺序不乱
+		let queued = this.outbox.hasQueued(row.channel_id);
+		let queuedCount = 0;
 		await this.sleep(this.firstDelay(profile));
 		for (const [i, segs] of messages.entries()) {
+			const text = voiced[i] ? `[语音] ${paragraphs[i]}` : paragraphs[i] ?? '[图片]';
+			if (queued) {
+				this.outbox.enqueue({ channelId: row.channel_id, title: row.title, segs, text });
+				queuedCount++;
+				continue;
+			}
 			if (i > 0) await this.sleep(Math.min(6000, typingDelayMs(paragraphs[i] ?? '', this.random) * profile.delay));
 			try {
 				const id = await this.channel.send(row.channel_id, segs);
-				this.rememberOwn(id, voiced[i] ? `[语音] ${paragraphs[i]}` : paragraphs[i] ?? '[图片]');
+				this.rememberOwn(id, text);
 			} catch (error) {
-				allOk = false;
-				this.log('error', `[fish-soul] 发到 ${row.title || row.channel_id} 失败：${String(error instanceof Error ? error.message : error)}`);
+				const msg = String(error instanceof Error ? error.message : error);
+				if (classifySendError(error) === 'retry') {
+					// 肯定没发出去：这条和后面的都排队重发
+					this.outbox.enqueue({ channelId: row.channel_id, title: row.title, segs, text, error: msg });
+					queued = true;
+					queuedCount++;
+					this.log('warn', `[fish-soul] 发到 ${row.title || row.channel_id} 没成功（${msg}），排队重发`);
+				} else {
+					allOk = false;
+					this.log('error', `[fish-soul] 发到 ${row.title || row.channel_id} 失败（${msg}），可能已经发出去了，不重发`);
+				}
 			}
 		}
 		if (isGroup) {
@@ -1043,7 +1077,8 @@ export class ChatGateway {
 			this.soul.journalOut?.(row.channel_id, paragraphs.join('\n'));
 			this.slang?.observe(row.channel_id, '', paragraphs.join(' '), { self: true }); // 学说法时当上下文，不从她自己的话里学
 		}
-		this.note(allOk ? 'sent' : 'failed', { channelId: row.channel_id, title: row.title, text: paragraphs.map((p, i) => (voiced[i] ? `🔊${p}` : p)).join(' / ') || '[图片]', reason: allOk ? (interject ? '插了一句' : '') : '有消息没发出去', memories });
+		const reason = !allOk ? '有消息没发出去' : queuedCount > 0 ? `${queuedCount} 条暂时没发出去，排队重发中` : interject ? '插了一句' : '';
+		this.note(allOk && queuedCount === 0 ? 'sent' : 'failed', { channelId: row.channel_id, title: row.title, text: paragraphs.map((p, i) => (voiced[i] ? `🔊${p}` : p)).join(' / ') || '[图片]', reason, memories });
 		return allOk;
 	}
 }

package/lib/history-import.js

@@ -328,7 +328,14 @@ export function buildPersonChunks(parsed, { selfKeys, mapping, personaName = '
 			cur.endMs = l.timeMs ?? cur.endMs;
 		}
 		// 只留最近的几段（越近越要紧），按时间顺序交给模型
-		out.push({ qq: p.qq, name: p.name, messageCount: p.messageCount, firstMs: p.firstMs, chunks: chunks.slice(-CHUNKS_PER_PERSON).map(({ chat, ...rest }) => rest) });
+		const kept = chunks.slice(-CHUNKS_PER_PERSON);
+		const dropped = chunks.slice(0, Math.max(0, chunks.length - CHUNKS_PER_PERSON));
+		out.push({
+			qq: p.qq, name: p.name, messageCount: p.messageCount, firstMs: p.firstMs,
+			chunks: kept.map(({ chat, ...rest }) => rest),
+			// 超出每人段数上限、这次不会交给模型的较早部分（预览里要明说，别让人以为全导进去了）
+			dropped: { chunks: dropped.length, chars: dropped.reduce((n, c) => n + c.text.length, 0), fromMs: dropped[0]?.startMs ?? null, toMs: dropped.at(-1)?.endMs ?? null }
+		});
 	}
 	return out.sort((a, b) => b.messageCount - a.messageCount);
 }
@@ -348,6 +355,24 @@ export function capChunks(persons, maxCalls) {
 	return want.map(({ keep, ...p }) => ({ ...p, chunks: keep === 0 ? [] : p.chunks.slice(-keep) })).filter((p) => p.chunks.length > 0);
 }
 
+/** 预览用：这次会导入多少、有多少较早的聊天不会交给模型。 */
+export function coverageOf(persons) {
+	const truncated = persons.filter((p) => (p.dropped?.chunks ?? 0) > 0);
+	const froms = truncated.map((p) => p.dropped.fromMs).filter((t) => t !== null);
+	return {
+		persons: persons.length,
+		chunksPerPerson: CHUNKS_PER_PERSON,
+		chunkChars: CHUNK_CHARS,
+		keptChunks: persons.reduce((n, p) => n + p.chunks.length, 0),
+		droppedChunks: truncated.reduce((n, p) => n + p.dropped.chunks, 0),
+		droppedChars: truncated.reduce((n, p) => n + p.dropped.chars, 0),
+		droppedFromMs: froms.length > 0 ? Math.min(...froms) : null,
+		truncatedPersons: truncated.length,
+		topTruncated: [...truncated].sort((a, b) => b.dropped.chunks - a.dropped.chunks).slice(0, 5)
+			.map((p) => ({ name: p.name, qq: p.qq, droppedChunks: p.dropped.chunks, keptFromMs: p.chunks[0]?.startMs ?? null }))
+	};
+}
+
 function extractPrompt({ personaName, name, chunk }) {
 	const range = [fmtDay(chunk.startMs), fmtDay(chunk.endMs)].filter(Boolean);
 	const when = range.length === 0 ? '' : range[0] === range[1] ? `，${range[0]}` : `，${range[0]} ~ ${range[1]}`;
@@ -421,6 +446,7 @@ export class HistoryImporter {
 			memoryCount: (parsed.memories ?? []).length,
 			speakers: list,
 			estimate: { persons: persons.length, calls: Math.min(DEFAULT_MAX_CALLS, persons.reduce((s, p) => s + p.chunks.length, 0)) },
+			coverage: coverageOf(persons),
 			warnings: parsed.warnings ?? []
 		};
 	}

package/lib/index.js

@@ -192,7 +192,7 @@ async function startup(ctx, config, { log, gate }) {
 		getConfig: qqCfg,
 		log,
 		onEvent: (event) => { void gateway?.onEvent(event).catch((error) => log('error', `[fish-soul] QQ 消息处理失败：${String(error instanceof Error ? error.message : error)}`)); },
-		onConnected: () => gateway?.backfill()
+		onConnected: () => { void gateway?.outbox.flush({ force: true }); return gateway?.backfill(); }
 	});
 
 	step(STARTUP_STEPS[3]);

package/lib/outbox.js

@@ -0,0 +1,134 @@
+/**
+ * outbox.js —— QQ 发送失败的重发队列（v0.11.1）。
+ *
+ * 模型已经写好的回复，因为 QQ 没连上 / 被接口拒了而没发出去，以前只记一条「发送失败」就丢了。
+ * 现在先落到 soul.db 的 outbox 表，按 2s → 10s → 30s → 60s → 120s 退避重发，连上 QQ 时立刻补发；
+ * 超过回复时限（replyWindowMinutes）或重试用完就放弃并记进「动态」。
+ *
+ * 只重发"肯定没发出去"的：QQ 未连接、接口明确拒绝（retcode）。
+ * 连接中途断开 / 调用超时时消息可能已经到了对方那里，重发会刷屏，所以不重发，只记一笔。
+ * 同一个聊天里有排队的消息时，新的回复也排到后面，保证顺序不乱。
+ */
+const BACKOFF_MS = [2000, 10_000, 30_000, 60_000, 120_000];
+const KEEP_DONE_MS = 24 * 3600_000;
+
+export const OUTBOX_SQL = `
+CREATE TABLE IF NOT EXISTS outbox (
+  id          INTEGER PRIMARY KEY AUTOINCREMENT,
+  channel_id  TEXT NOT NULL,
+  title       TEXT NOT NULL DEFAULT '',
+  segs        TEXT NOT NULL,
+  text        TEXT NOT NULL DEFAULT '',
+  attempts    INTEGER NOT NULL DEFAULT 0,
+  next_at     INTEGER NOT NULL,
+  created_at  INTEGER NOT NULL,
+  status      TEXT NOT NULL DEFAULT 'queued',
+  last_error  TEXT NOT NULL DEFAULT ''
+);
+CREATE INDEX IF NOT EXISTS idx_outbox_queued ON outbox(status, channel_id, id);
+`;
+
+/** 'retry' = 肯定没发出去，可以重发；'unknown' = 可能已经发出去了，不重发。 */
+export function classifySendError(error) {
+	const msg = String(error instanceof Error ? error.message : error);
+	if (/未连接/.test(msg)) return 'retry';
+	if (/retcode/.test(msg)) return 'retry';
+	return 'unknown';
+}
+
+export class Outbox {
+	/**
+	 * @param {object} deps
+	 * @param {import('node:sqlite').DatabaseSync} deps.db
+	 * @param {{ connected?: boolean, send: (channelId: string, segs: object[]) => Promise<string> }} deps.channel
+	 * @param {() => number} deps.windowMs 超过多久就不再重发（取回复时限）
+	 * @param {(id: string, item: object) => void} [deps.onSent]
+	 * @param {(item: object, reason: string) => void} [deps.onGiveUp]
+	 * @param {(level: string, msg: string) => void} [deps.log]
+	 * @param {() => number} [deps.now]
+	 */
+	constructor({ db, channel, windowMs, onSent, onGiveUp, log, now }) {
+		this.db = db;
+		this.channel = channel;
+		this.windowMs = windowMs;
+		this.onSent = onSent ?? (() => {});
+		this.onGiveUp = onGiveUp ?? (() => {});
+		this.log = log ?? (() => {});
+		this.now = now ?? (() => Date.now());
+		this.timer = null;
+		this.flushing = null;
+		this.stopped = false;
+		db.exec(OUTBOX_SQL);
+	}
+
+	hasQueued(channelId) {
+		return this.db.prepare("SELECT 1 FROM outbox WHERE status = 'queued' AND channel_id = ? LIMIT 1").get(channelId) !== undefined;
+	}
+
+	size() {
+		return Number(this.db.prepare("SELECT COUNT(*) AS n FROM outbox WHERE status = 'queued'").get().n);
+	}
+
+	enqueue({ channelId, title = '', segs, text = '', error = '' }) {
+		const now = this.now();
+		this.db.prepare(`INSERT INTO outbox (channel_id, title, segs, text, next_at, created_at, last_error)
+			VALUES (?, ?, ?, ?, ?, ?, ?)`).run(channelId, title, JSON.stringify(segs), text, now + BACKOFF_MS[0], now, String(error));
+		this.schedule(BACKOFF_MS[0]);
+	}
+
+	schedule(delayMs) {
+		if (this.stopped) return;
+		if (this.timer !== null) clearTimeout(this.timer);
+		this.timer = setTimeout(() => { this.timer = null; void this.flush(); }, Math.max(0, delayMs));
+		this.timer.unref?.();
+	}
+
+	/** 把到点的排队消息按聊天、按顺序发出去；某个聊天发失败就停在那里，下一轮再试。 */
+	flush({ force = false } = {}) {
+		this.flushing ??= this.flushOnce({ force }).finally(() => { this.flushing = null; });
+		return this.flushing;
+	}
+
+	async flushOnce({ force }) {
+		if (this.stopped) return;
+		const now = this.now();
+		this.db.prepare("DELETE FROM outbox WHERE status <> 'queued' AND created_at < ?").run(now - KEEP_DONE_MS);
+		const rows = this.db.prepare("SELECT * FROM outbox WHERE status = 'queued' ORDER BY id").all();
+		const blocked = new Set();
+		const mark = this.db.prepare('UPDATE outbox SET status = ?, attempts = ?, next_at = ?, last_error = ? WHERE id = ?');
+		for (const row of rows) {
+			if (blocked.has(row.channel_id)) continue;
+			const item = { ...row, segs: JSON.parse(row.segs) };
+			if (now - row.created_at > this.windowMs()) {
+				mark.run('expired', row.attempts, row.next_at, row.last_error, row.id);
+				this.onGiveUp(item, '超过回复时限，不再重发');
+				continue;
+			}
+			if (!force && row.next_at > now) { blocked.add(row.channel_id); continue; }
+			if (this.channel.connected === false) { blocked.add(row.channel_id); continue; }
+			try {
+				const id = await this.channel.send(row.channel_id, item.segs);
+				mark.run('sent', row.attempts + 1, row.next_at, '', row.id);
+				this.onSent(id, item);
+			} catch (error) {
+				const attempts = row.attempts + 1;
+				const msg = String(error instanceof Error ? error.message : error);
+				if (classifySendError(error) !== 'retry' || attempts >= BACKOFF_MS.length) {
+					mark.run('gave_up', attempts, row.next_at, msg, row.id);
+					this.onGiveUp(item, attempts >= BACKOFF_MS.length ? `重发 ${attempts} 次都没成功：${msg}` : `重发时${msg}，可能已经发出去了，不再重发`);
+					continue;
+				}
+				mark.run('queued', attempts, now + BACKOFF_MS[attempts], msg, row.id);
+				blocked.add(row.channel_id);
+			}
+		}
+		const next = this.db.prepare("SELECT MIN(next_at) AS t FROM outbox WHERE status = 'queued'").get().t;
+		if (next !== null && next !== undefined) this.schedule(Number(next) - this.now());
+	}
+
+	stop() {
+		this.stopped = true;
+		if (this.timer !== null) clearTimeout(this.timer);
+		this.timer = null;
+	}
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dsh-fish-soul",
-  "version": "0.11.0",
+  "version": "0.11.1",
   "description": "鱼 · 人格、按人记忆（可纠正、有印象、能导入旧聊天记录）、日程自走与主动开口，直连 SnowLuma 收发 QQ（群里会自己插话），B站 / 抖音小号私信与自己刷视频；按功能选模型并统计用量花费，能听会说 QQ 语音、收藏表情包、学群里的梗，可选上网查；心情影响说话、去 AI 味、每个群的设定、按聊天选模型、每晚写日记（DSH 插件，单角色 feiyu）",
   "type": "module",
   "main": "./lib/index.js",
@@ -62,7 +62,7 @@
     }
   },
   "scripts": {
-    "test": "node test/smoke.mjs && node test/douyin-browser.mjs && node test/startup-gate.mjs",
+    "test": "node test/smoke.mjs && node test/douyin-browser.mjs && node test/startup-gate.mjs && node test/outbox.mjs && node test/import-coverage.mjs",
     "eval": "node test/eval/persona-eval.mjs"
   }
 }