metame-cli 1.6.0 → 1.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.js +6 -7
- package/package.json +1 -1
- package/scripts/core/chunker.js +100 -0
- package/scripts/core/embedding.js +225 -0
- package/scripts/core/hybrid-search.js +296 -0
- package/scripts/core/wiki-db.js +144 -3
- package/scripts/daemon-bridges.js +9 -6
- package/scripts/daemon-command-router.js +25 -1
- package/scripts/daemon-default.yaml +31 -0
- package/scripts/daemon-embedding.js +162 -0
- package/scripts/daemon-engine-runtime.js +1 -1
- package/scripts/daemon-health-scan.js +185 -0
- package/scripts/daemon-runtime-lifecycle.js +1 -1
- package/scripts/daemon-task-scheduler.js +5 -3
- package/scripts/daemon-wiki.js +126 -4
- package/scripts/daemon.js +4 -2
- package/scripts/feishu-adapter.js +208 -29
- package/scripts/memory-backfill-chunks.js +92 -0
- package/scripts/memory-search.js +43 -15
- package/scripts/memory-wiki-schema.js +161 -2
- package/scripts/memory.js +15 -0
- package/scripts/providers.js +37 -6
- package/scripts/wiki-cluster.js +121 -0
- package/scripts/wiki-extract.js +171 -0
- package/scripts/wiki-facts.js +351 -0
- package/scripts/wiki-import.js +256 -0
- package/scripts/wiki-reflect-build.js +352 -28
- package/scripts/wiki-reflect-export.js +115 -0
- package/scripts/wiki-reflect.js +34 -1
- package/scripts/wiki-synthesis.js +224 -0
|
@@ -660,7 +660,7 @@ function createTaskScheduler(deps) {
|
|
|
660
660
|
return found || null;
|
|
661
661
|
}
|
|
662
662
|
|
|
663
|
-
function startHeartbeat(config, notifyFn, notifyPersonalFn) {
|
|
663
|
+
function startHeartbeat(config, notifyFn, notifyPersonalFn, adminNotifyFn) {
|
|
664
664
|
const { all: tasks } = getAllTasks(config);
|
|
665
665
|
|
|
666
666
|
const enabledTasks = tasks.filter(t => t.enabled !== false);
|
|
@@ -787,10 +787,12 @@ function createTaskScheduler(deps) {
|
|
|
787
787
|
}
|
|
788
788
|
if (task.notify && notifyFn && !result.skipped) {
|
|
789
789
|
const proj = task._project || null;
|
|
790
|
+
// Tasks without a project are system-level — send to admin chat only
|
|
791
|
+
const sendFn = proj ? notifyFn : (adminNotifyFn || notifyFn);
|
|
790
792
|
if (result.success) {
|
|
791
|
-
|
|
793
|
+
sendFn(`✅ *${task.name}* completed\n\n${result.output}`, proj);
|
|
792
794
|
} else {
|
|
793
|
-
|
|
795
|
+
sendFn(`❌ *${task.name}* failed: ${result.error}`, proj);
|
|
794
796
|
}
|
|
795
797
|
}
|
|
796
798
|
})
|
package/scripts/daemon-wiki.js
CHANGED
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
|
|
18
18
|
const os = require('os');
|
|
19
19
|
const path = require('path');
|
|
20
|
-
const {
|
|
20
|
+
const { execFileSync } = require('child_process');
|
|
21
21
|
|
|
22
22
|
const {
|
|
23
23
|
listWikiPages,
|
|
@@ -80,6 +80,11 @@ function createWikiCommandHandler(deps) {
|
|
|
80
80
|
await _handleHelp(bot, chatId);
|
|
81
81
|
return true;
|
|
82
82
|
}
|
|
83
|
+
if (trimmed === '/wiki import' || trimmed.startsWith('/wiki import ')) {
|
|
84
|
+
const args = trimmed.slice(12).trim();
|
|
85
|
+
await _handleImport(bot, chatId, args);
|
|
86
|
+
return true;
|
|
87
|
+
}
|
|
83
88
|
// Unknown /wiki subcommand — show help
|
|
84
89
|
if (trimmed.startsWith('/wiki ')) {
|
|
85
90
|
await _handleHelp(bot, chatId);
|
|
@@ -123,7 +128,14 @@ function createWikiCommandHandler(deps) {
|
|
|
123
128
|
}
|
|
124
129
|
|
|
125
130
|
const db = getDb();
|
|
126
|
-
|
|
131
|
+
let wikiPages, facts;
|
|
132
|
+
try {
|
|
133
|
+
const { hybridSearchWiki } = require('./core/hybrid-search');
|
|
134
|
+
({ wikiPages, facts } = await hybridSearchWiki(db, query, { trackSearch: true }));
|
|
135
|
+
} catch (err) {
|
|
136
|
+
log('WARN', `[wiki-research] hybrid search fallback to FTS: ${err.message}`);
|
|
137
|
+
({ wikiPages, facts } = searchWikiAndFacts(db, query, { trackSearch: true }));
|
|
138
|
+
}
|
|
127
139
|
|
|
128
140
|
if (wikiPages.length === 0 && facts.length === 0) {
|
|
129
141
|
await bot.sendMessage(chatId,
|
|
@@ -206,6 +218,13 @@ function createWikiCommandHandler(deps) {
|
|
|
206
218
|
const lines = ['✅ Wiki 重建完成'];
|
|
207
219
|
if (result.built.length > 0) {
|
|
208
220
|
lines.push(`• 重建: ${result.built.join(', ')}`);
|
|
221
|
+
try {
|
|
222
|
+
const { execFile } = require('child_process');
|
|
223
|
+
const embScript = path.join(os.homedir(), '.metame', 'daemon-embedding.js');
|
|
224
|
+
execFile('node', [embScript], { timeout: 120000, stdio: 'ignore' }, (err) => {
|
|
225
|
+
if (err) log('WARN', `[wiki-sync] embedding trigger failed: ${err.message}`);
|
|
226
|
+
});
|
|
227
|
+
} catch { }
|
|
209
228
|
}
|
|
210
229
|
if (result.failed.length > 0) {
|
|
211
230
|
lines.push(`• 失败: ${result.failed.map(f => f.slug).join(', ')}`);
|
|
@@ -214,6 +233,68 @@ function createWikiCommandHandler(deps) {
|
|
|
214
233
|
lines.push(`• 文件导出失败 (DB 已更新): ${result.exportFailed.join(', ')}`);
|
|
215
234
|
}
|
|
216
235
|
await bot.sendMessage(chatId, lines.join('\n'));
|
|
236
|
+
|
|
237
|
+
// Phased doc + cluster rebuild (wiki-import feature extension)
|
|
238
|
+
const { listStaleDocSources } = require('./core/wiki-db');
|
|
239
|
+
const { buildDocWikiPage } = require('./wiki-reflect-build');
|
|
240
|
+
const { extractText } = require('./wiki-extract');
|
|
241
|
+
const allDocSlugsForSync = db.prepare("SELECT slug FROM doc_sources WHERE status='active'").all().map(r => r.slug);
|
|
242
|
+
const staleDocSources = listStaleDocSources(db);
|
|
243
|
+
const builtDocSlugs = [];
|
|
244
|
+
for (const docSrc of staleDocSources) {
|
|
245
|
+
try {
|
|
246
|
+
const { text } = await extractText(docSrc.file_path);
|
|
247
|
+
const docResult = await buildDocWikiPage(db, docSrc, text, { allowedSlugs: allDocSlugsForSync, providers });
|
|
248
|
+
if (docResult) {
|
|
249
|
+
db.prepare("UPDATE doc_sources SET content_stale=0, built_at=? WHERE id=?")
|
|
250
|
+
.run(new Date().toISOString(), docSrc.id);
|
|
251
|
+
builtDocSlugs.push(docSrc.slug);
|
|
252
|
+
}
|
|
253
|
+
} catch (docErr) {
|
|
254
|
+
db.prepare("UPDATE doc_sources SET error_message=? WHERE id=?").run(docErr.message, docSrc.id);
|
|
255
|
+
log('WARN', `[wiki-sync] doc rebuild failed ${docSrc.slug}: ${docErr.message}`);
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
// Cascade stale cluster pages
|
|
259
|
+
if (builtDocSlugs.length > 0) {
|
|
260
|
+
const ph = builtDocSlugs.map(() => '?').join(',');
|
|
261
|
+
const affected = db.prepare(`SELECT DISTINCT page_slug FROM wiki_page_doc_sources
|
|
262
|
+
WHERE role='cluster_member' AND doc_source_id IN
|
|
263
|
+
(SELECT id FROM doc_sources WHERE slug IN (${ph}))`).all(...builtDocSlugs).map(r => r.page_slug);
|
|
264
|
+
if (affected.length) {
|
|
265
|
+
db.prepare(`UPDATE wiki_pages SET staleness=1 WHERE slug IN (${affected.map(() => '?').join(',')})`).run(...affected);
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
// Rebuild stale cluster pages after embedding drain
|
|
269
|
+
const { waitForEmbeddingDrain } = require('./wiki-import');
|
|
270
|
+
const phase1ChunkIds = builtDocSlugs.flatMap(slug =>
|
|
271
|
+
db.prepare("SELECT id FROM content_chunks WHERE page_slug=?").all(slug).map(c => c.id)
|
|
272
|
+
);
|
|
273
|
+
const drainedOk = await waitForEmbeddingDrain(db, phase1ChunkIds, (msg) => log('INFO', msg));
|
|
274
|
+
if (drainedOk) {
|
|
275
|
+
const { buildTopicClusterPage } = require('./wiki-reflect-build');
|
|
276
|
+
const { getClusterMemberIds } = require('./core/wiki-db');
|
|
277
|
+
const staleClusterPages = db.prepare("SELECT * FROM wiki_pages WHERE source_type='topic_cluster' AND staleness=1").all();
|
|
278
|
+
for (const cp of staleClusterPages) {
|
|
279
|
+
const memberIds = getClusterMemberIds(db, cp.slug);
|
|
280
|
+
const getDocSrcById = db.prepare("SELECT * FROM doc_sources WHERE id=?");
|
|
281
|
+
const docRows = memberIds.map(id => getDocSrcById.get(id)).filter(Boolean);
|
|
282
|
+
try {
|
|
283
|
+
await buildTopicClusterPage(db, docRows, { allowedSlugs: allDocSlugsForSync, providers, existingClusters: [] });
|
|
284
|
+
} catch (clErr) {
|
|
285
|
+
log('WARN', `[wiki-sync] cluster rebuild failed ${cp.slug}: ${clErr.message}`);
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
if (staleDocSources.length > 0) {
|
|
290
|
+
const docMsg = [];
|
|
291
|
+
if (builtDocSlugs.length > 0) docMsg.push(`• 文档页面重建: ${builtDocSlugs.join(', ')}`);
|
|
292
|
+
const docFailed = staleDocSources.length - builtDocSlugs.length;
|
|
293
|
+
if (docFailed > 0) docMsg.push(`• 文档重建失败: ${docFailed} 个(见日志)`);
|
|
294
|
+
if (docMsg.length > 0) {
|
|
295
|
+
await bot.sendMessage(chatId, `📄 文档页面同步\n\n${docMsg.join('\n')}`);
|
|
296
|
+
}
|
|
297
|
+
}
|
|
217
298
|
} catch (err) {
|
|
218
299
|
log('ERROR', `[wiki-sync] ${err.message}`);
|
|
219
300
|
if (err.message.includes('another instance')) {
|
|
@@ -263,10 +344,10 @@ function createWikiCommandHandler(deps) {
|
|
|
263
344
|
// Try Obsidian URI first (opens vault by path if already configured)
|
|
264
345
|
const vaultName = path.basename(outputDir);
|
|
265
346
|
try {
|
|
266
|
-
|
|
347
|
+
execFileSync('open', [`obsidian://open?vault=${encodeURIComponent(vaultName)}`], { timeout: 5000 });
|
|
267
348
|
} catch {
|
|
268
349
|
// Fallback: open folder in Finder — user can then drag into Obsidian
|
|
269
|
-
|
|
350
|
+
execFileSync('open', [outputDir], { timeout: 5000 });
|
|
270
351
|
}
|
|
271
352
|
await bot.sendMessage(chatId,
|
|
272
353
|
`📂 已打开 Obsidian vault: \`${outputDir}\`\n\n` +
|
|
@@ -279,6 +360,46 @@ function createWikiCommandHandler(deps) {
|
|
|
279
360
|
}
|
|
280
361
|
}
|
|
281
362
|
|
|
363
|
+
async function _handleImport(bot, chatId, args) {
|
|
364
|
+
const noCluster = args.includes('--no-cluster');
|
|
365
|
+
const inputPath = args.replace('--no-cluster', '').trim();
|
|
366
|
+
|
|
367
|
+
if (!inputPath) {
|
|
368
|
+
await bot.sendMessage(chatId, '用法: `/wiki import <路径或文件>` [--no-cluster]\n\n示例:\n`/wiki import ~/Documents/notes`\n`/wiki import ~/report.pdf`');
|
|
369
|
+
return;
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
const resolvedPath = inputPath.replace(/^~/, require('node:os').homedir());
|
|
373
|
+
|
|
374
|
+
let stat;
|
|
375
|
+
try { stat = require('node:fs').statSync(resolvedPath); }
|
|
376
|
+
catch { await bot.sendMessage(chatId, `❌ 路径不存在: ${resolvedPath}`); return; }
|
|
377
|
+
|
|
378
|
+
const isDir = stat.isDirectory();
|
|
379
|
+
await bot.sendMessage(chatId, `⏳ 开始导入 ${isDir ? '目录' : '文件'}: \`${resolvedPath}\`\n${noCluster ? '(跳过聚类)' : '(含自动聚类)'}`);
|
|
380
|
+
|
|
381
|
+
const { runWikiImport } = require('./wiki-import');
|
|
382
|
+
const db = getDb();
|
|
383
|
+
const logFn = (msg) => { log('INFO', msg); };
|
|
384
|
+
|
|
385
|
+
try {
|
|
386
|
+
const stats = await runWikiImport(db, resolvedPath, {
|
|
387
|
+
providers, noCluster, log: logFn,
|
|
388
|
+
});
|
|
389
|
+
await bot.sendMessage(chatId,
|
|
390
|
+
`✅ 导入完成\n\n` +
|
|
391
|
+
`- 新建/更新页面: ${stats.imported}\n` +
|
|
392
|
+
`- 跳过 (未变更): ${stats.skipped}\n` +
|
|
393
|
+
`- 失败: ${stats.failed}\n` +
|
|
394
|
+
`- 聚类页面: ${stats.clusters}\n\n` +
|
|
395
|
+
`使用 \`/wiki\` 查看全部页面`
|
|
396
|
+
);
|
|
397
|
+
} catch (err) {
|
|
398
|
+
log('ERROR', `[wiki-import] ${err.message}`);
|
|
399
|
+
await bot.sendMessage(chatId, `❌ 导入失败: ${err.message}`);
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
|
|
282
403
|
async function _handleHelp(bot, chatId) {
|
|
283
404
|
await bot.sendMessage(chatId, [
|
|
284
405
|
'📚 **Wiki 命令**',
|
|
@@ -287,6 +408,7 @@ function createWikiCommandHandler(deps) {
|
|
|
287
408
|
'`/wiki research <关键词>` — 搜索知识',
|
|
288
409
|
'`/wiki page <slug>` — 查看页面全文',
|
|
289
410
|
'`/wiki sync` — 重建陈旧页面',
|
|
411
|
+
'`/wiki import <路径>` — 导入本地文档 (md/txt/PDF)',
|
|
290
412
|
'`/wiki pin <标签> [标题]` — 手工注册主题',
|
|
291
413
|
'`/wiki open` — 在 Obsidian 中打开 vault',
|
|
292
414
|
].join('\n'));
|
package/scripts/daemon.js
CHANGED
|
@@ -2387,7 +2387,7 @@ async function main() {
|
|
|
2387
2387
|
}
|
|
2388
2388
|
|
|
2389
2389
|
// Config validation: warn on unknown/suspect fields
|
|
2390
|
-
const KNOWN_SECTIONS = ['daemon', 'telegram', 'feishu', 'weixin', 'heartbeat', 'budget', 'projects', 'imessage', 'siri_bridge'];
|
|
2390
|
+
const KNOWN_SECTIONS = ['daemon', 'telegram', 'feishu', 'weixin', 'heartbeat', 'budget', 'projects', 'imessage', 'siri_bridge', 'hooks'];
|
|
2391
2391
|
const KNOWN_DAEMON = [
|
|
2392
2392
|
'model', // legacy (still valid as fallback)
|
|
2393
2393
|
'models', // per-engine model map: { claude, codex }
|
|
@@ -2402,6 +2402,8 @@ async function main() {
|
|
|
2402
2402
|
'mac_control_mode',
|
|
2403
2403
|
'enable_nl_mac_control',
|
|
2404
2404
|
'enable_nl_mac_fallback',
|
|
2405
|
+
'wiki_output_dir', // wiki export path (used by daemon-command-router)
|
|
2406
|
+
'skill_evolution_notify', // whether to notify on skill evolution (used by daemon-task-scheduler)
|
|
2405
2407
|
];
|
|
2406
2408
|
for (const key of Object.keys(config)) {
|
|
2407
2409
|
if (!KNOWN_SECTIONS.includes(key)) log('WARN', `Config: unknown section "${key}" (typo?)`);
|
|
@@ -2510,7 +2512,7 @@ async function main() {
|
|
|
2510
2512
|
};
|
|
2511
2513
|
|
|
2512
2514
|
// Start heartbeat scheduler
|
|
2513
|
-
let heartbeatTimer = startHeartbeat(config, notifyFn, notifyPersonalFn);
|
|
2515
|
+
let heartbeatTimer = startHeartbeat(config, notifyFn, notifyPersonalFn, adminNotifyFn);
|
|
2514
2516
|
|
|
2515
2517
|
let shuttingDown = false;
|
|
2516
2518
|
function spawnReplacementDaemon(reason) {
|
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
|
|
9
9
|
const fs = require('fs');
|
|
10
10
|
const path = require('path');
|
|
11
|
+
const dns = require('dns');
|
|
11
12
|
|
|
12
13
|
let Lark;
|
|
13
14
|
function _tryRequireLark() {
|
|
@@ -58,6 +59,40 @@ function withTimeout(promise, ms = 10000) {
|
|
|
58
59
|
return Promise.race([promise, timeout]).finally(() => clearTimeout(timer));
|
|
59
60
|
}
|
|
60
61
|
|
|
62
|
+
// Wait for DNS to resolve a target host with exponential backoff.
|
|
63
|
+
// Used after system wake / before reconnect: the OS may report clock/events
|
|
64
|
+
// restored before WiFi+DNS are actually usable. Retries 1/2/4/8s, total cap 30s.
|
|
65
|
+
async function waitForNetworkReady(hostname, opts = {}) {
|
|
66
|
+
const log = opts.log || (() => {});
|
|
67
|
+
const totalBudget = Number.isFinite(opts.totalBudgetMs) ? opts.totalBudgetMs : 30000;
|
|
68
|
+
const lookup = opts.lookup || dns.promises.lookup;
|
|
69
|
+
const sleep = opts.sleep || ((ms) => new Promise((r) => setTimeout(r, ms)));
|
|
70
|
+
const startedAt = Date.now();
|
|
71
|
+
let attempt = 0;
|
|
72
|
+
let lastError = null;
|
|
73
|
+
// Backoff schedule: 0s, 1s, 2s, 4s, 8s between attempts (before the next attempt)
|
|
74
|
+
const backoff = [0, 1000, 2000, 4000, 8000];
|
|
75
|
+
// Always make at least one attempt; subsequent attempts are budget-gated.
|
|
76
|
+
do {
|
|
77
|
+
const wait = backoff[Math.min(attempt, backoff.length - 1)];
|
|
78
|
+
if (wait > 0) await sleep(wait);
|
|
79
|
+
attempt += 1;
|
|
80
|
+
try {
|
|
81
|
+
await lookup(hostname);
|
|
82
|
+
return { ok: true, attempts: attempt, elapsed: Date.now() - startedAt };
|
|
83
|
+
} catch (err) {
|
|
84
|
+
lastError = err;
|
|
85
|
+
log('DEBUG', `[net-ready] ${hostname} attempt ${attempt} failed: ${err.code || err.message}`);
|
|
86
|
+
}
|
|
87
|
+
} while (Date.now() - startedAt < totalBudget);
|
|
88
|
+
return {
|
|
89
|
+
ok: false,
|
|
90
|
+
attempts: attempt,
|
|
91
|
+
elapsed: Date.now() - startedAt,
|
|
92
|
+
error: lastError && (lastError.message || String(lastError)),
|
|
93
|
+
};
|
|
94
|
+
}
|
|
95
|
+
|
|
61
96
|
// Max chars per lark_md element (Feishu limit ~4000)
|
|
62
97
|
const MAX_CHUNK = 3800;
|
|
63
98
|
|
|
@@ -101,12 +136,25 @@ function createBot(config) {
|
|
|
101
136
|
return { ok: true };
|
|
102
137
|
} catch (err) {
|
|
103
138
|
const msg = err && err.message || String(err);
|
|
104
|
-
|
|
139
|
+
// Only flag as auth error when we have strong evidence: known Feishu
|
|
140
|
+
// auth error codes, HTTP 401/403, or explicit 'invalid app_id/secret'.
|
|
141
|
+
// Previously a loose /token/ regex false-positived on SDK-internal
|
|
142
|
+
// messages like "Cannot destructure 'tenant_access_token' of undefined"
|
|
143
|
+
// (which is really a network/empty-response failure) and caused the
|
|
144
|
+
// bridge to refuse to start across a lid-close/wake cycle.
|
|
145
|
+
const authPatterns = [
|
|
146
|
+
/\b(99991663|99991664|99991665)\b/, // Feishu token invalid codes
|
|
147
|
+
/\b(401|403)\b/, // HTTP 401/403
|
|
148
|
+
/invalid\s+(app_?id|app_?secret|tenant_access_token|access_?token)/i,
|
|
149
|
+
/unauthorized/i,
|
|
150
|
+
/\bforbidden\b/i,
|
|
151
|
+
];
|
|
152
|
+
const isAuthError = authPatterns.some((p) => p.test(msg));
|
|
105
153
|
return {
|
|
106
154
|
ok: false,
|
|
107
155
|
error: isAuthError
|
|
108
156
|
? `Feishu credential validation failed (app_id/app_secret may be incorrect): ${msg}`
|
|
109
|
-
: `Feishu API probe failed (network or
|
|
157
|
+
: `Feishu API probe failed (network or transient issue): ${msg}`,
|
|
110
158
|
isAuthError,
|
|
111
159
|
};
|
|
112
160
|
}
|
|
@@ -393,14 +441,24 @@ function createBot(config) {
|
|
|
393
441
|
let stopped = false;
|
|
394
442
|
let currentWs = null;
|
|
395
443
|
let healthTimer = null;
|
|
444
|
+
let sleepWakeTimer = null;
|
|
396
445
|
let reconnectTimer = null;
|
|
397
|
-
let
|
|
446
|
+
let aliveTimer = null;
|
|
447
|
+
let reconnectScheduled = false; // dedup flag: true while a reconnect is pending
|
|
448
|
+
let wsEpoch = 0; // increments each connect(); underlying-ws hooks capture their own epoch
|
|
449
|
+
const INITIAL_RECONNECT_DELAY = 5000;
|
|
398
450
|
const MAX_RECONNECT_DELAY = 60000;
|
|
399
|
-
|
|
400
|
-
const
|
|
451
|
+
let reconnectDelay = INITIAL_RECONNECT_DELAY;
|
|
452
|
+
const HEALTH_CHECK_INTERVAL = 30000; // tighter bottom-line probe (was 90s)
|
|
453
|
+
const SILENT_THRESHOLD = 90000; // 90s no SDK activity → probe (was 300s)
|
|
454
|
+
const SLEEP_DETECT_INTERVAL = 5000;
|
|
455
|
+
const SLEEP_JUMP_THRESHOLD = 30000; // clock jump >30s = was sleeping
|
|
456
|
+
const ALIVE_CHECK_WINDOW = 15000; // after connect, must see activity within 15s
|
|
457
|
+
const FEISHU_HOST = 'open.feishu.cn';
|
|
401
458
|
|
|
402
459
|
// Track last SDK activity (any event received = alive)
|
|
403
460
|
let _lastActivityAt = Date.now();
|
|
461
|
+
let _connectedAt = 0; // when the current WSClient was (re)started
|
|
404
462
|
function touchActivity() { _lastActivityAt = Date.now(); }
|
|
405
463
|
|
|
406
464
|
// Dedup: track recent message_ids (Feishu may redeliver on slow ack)
|
|
@@ -487,75 +545,196 @@ function createBot(config) {
|
|
|
487
545
|
});
|
|
488
546
|
}
|
|
489
547
|
|
|
548
|
+
// Hook the underlying ws instance for first-class close/error notification.
|
|
549
|
+
// Lark SDK stores the live WebSocket via wsConfig.setWSInstance; we wrap it
|
|
550
|
+
// so we learn about 'close' immediately instead of waiting for silence.
|
|
551
|
+
// Defensive: SDK internals can change between versions — any failure just
|
|
552
|
+
// downgrades to the silent/health/sleep bottom-lines.
|
|
553
|
+
function hookUnderlyingWs(wsClient, epoch) {
|
|
554
|
+
try {
|
|
555
|
+
const cfg = wsClient && wsClient.wsConfig;
|
|
556
|
+
if (!cfg || typeof cfg.setWSInstance !== 'function') return;
|
|
557
|
+
const orig = cfg.setWSInstance.bind(cfg);
|
|
558
|
+
cfg.setWSInstance = (inst) => {
|
|
559
|
+
orig(inst);
|
|
560
|
+
if (!inst || inst._metameHooked) return;
|
|
561
|
+
inst._metameHooked = true;
|
|
562
|
+
try {
|
|
563
|
+
inst.on('close', () => {
|
|
564
|
+
if (stopped) return;
|
|
565
|
+
if (epoch !== wsEpoch) return; // stale: a newer connect() has superseded this one
|
|
566
|
+
_log('INFO', 'Feishu underlying WS closed — scheduling reconnect');
|
|
567
|
+
scheduleReconnect({ immediate: true, reason: 'ws-close' });
|
|
568
|
+
});
|
|
569
|
+
inst.on('error', (e) => {
|
|
570
|
+
if (epoch !== wsEpoch) return;
|
|
571
|
+
_log('WARN', `Feishu underlying WS error: ${e && e.message || e}`);
|
|
572
|
+
});
|
|
573
|
+
} catch (hookErr) {
|
|
574
|
+
_log('WARN', `Feishu ws event hook failed: ${hookErr.message}`);
|
|
575
|
+
}
|
|
576
|
+
};
|
|
577
|
+
} catch (err) {
|
|
578
|
+
_log('WARN', `Feishu SDK hook unavailable (${err.message}) — falling back to silence/sleep detection`);
|
|
579
|
+
}
|
|
580
|
+
}
|
|
581
|
+
|
|
490
582
|
function connect() {
|
|
491
583
|
if (stopped) return;
|
|
584
|
+
clearTimeout(aliveTimer);
|
|
585
|
+
wsEpoch += 1;
|
|
586
|
+
const myEpoch = wsEpoch;
|
|
587
|
+
let ws;
|
|
492
588
|
try {
|
|
493
|
-
|
|
589
|
+
ws = new Lark.WSClient({
|
|
494
590
|
appId: app_id,
|
|
495
591
|
appSecret: app_secret,
|
|
496
592
|
loggerLevel: Lark.LoggerLevel.info,
|
|
593
|
+
autoReconnect: false, // we own the reconnect lifecycle
|
|
497
594
|
});
|
|
595
|
+
currentWs = ws;
|
|
596
|
+
hookUnderlyingWs(ws, myEpoch);
|
|
498
597
|
const eventDispatcher = buildDispatcher();
|
|
499
|
-
|
|
598
|
+
const startResult = ws.start({ eventDispatcher });
|
|
599
|
+
_connectedAt = Date.now();
|
|
500
600
|
touchActivity();
|
|
501
|
-
reconnectDelay = 5000; // reset backoff on successful start
|
|
502
601
|
_log('INFO', 'Feishu WebSocket connecting...');
|
|
602
|
+
startAliveCheck();
|
|
603
|
+
// start() may return a Promise. Surface async failures into the reconnect pipeline
|
|
604
|
+
// so we don't depend solely on the 15s alive-check to recover.
|
|
605
|
+
if (startResult && typeof startResult.then === 'function') {
|
|
606
|
+
startResult.catch((err) => {
|
|
607
|
+
if (stopped) return;
|
|
608
|
+
if (myEpoch !== wsEpoch) return; // superseded
|
|
609
|
+
_log('ERROR', `Feishu WSClient.start rejected: ${err && err.message || err}`);
|
|
610
|
+
scheduleReconnect({ immediate: true, reason: 'start-rejected', failed: true });
|
|
611
|
+
});
|
|
612
|
+
}
|
|
503
613
|
} catch (err) {
|
|
504
614
|
_log('ERROR', `Feishu WSClient.start failed: ${err.message}`);
|
|
505
|
-
scheduleReconnect();
|
|
615
|
+
scheduleReconnect({ immediate: true, reason: 'start-failed', failed: true });
|
|
506
616
|
}
|
|
507
617
|
}
|
|
508
618
|
|
|
509
|
-
|
|
619
|
+
// Single entry point for all reconnect signals. Dedup'd via reconnectScheduled
|
|
620
|
+
// so concurrent ws-close + alive-probe-fail + sleep events collapse into one
|
|
621
|
+
// reconnect. Backoff only grows when the caller marks this as a failure recovery
|
|
622
|
+
// (failed:true) — known-cause resets (manual / system-wake) start from 0s.
|
|
623
|
+
function scheduleReconnect({ immediate = false, reason = '', failed = false } = {}) {
|
|
510
624
|
if (stopped) return;
|
|
625
|
+
if (reconnectScheduled) {
|
|
626
|
+
_log('DEBUG', `Feishu reconnect already scheduled — dropping duplicate (reason: ${reason})`);
|
|
627
|
+
return;
|
|
628
|
+
}
|
|
629
|
+
reconnectScheduled = true;
|
|
511
630
|
clearTimeout(reconnectTimer);
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
631
|
+
clearTimeout(aliveTimer);
|
|
632
|
+
try { currentWs?.stop?.(); } catch { /* ignore */ }
|
|
633
|
+
currentWs = null;
|
|
634
|
+
if (failed) {
|
|
635
|
+
// Only failure paths grow the backoff ceiling for the *next* attempt.
|
|
636
|
+
reconnectDelay = Math.min(reconnectDelay * 2, MAX_RECONNECT_DELAY);
|
|
637
|
+
}
|
|
638
|
+
const delay = immediate ? 0 : reconnectDelay;
|
|
639
|
+
_log('INFO', `Feishu reconnect in ${Math.round(delay / 1000)}s (reason: ${reason || 'unspecified'})`);
|
|
640
|
+
reconnectTimer = setTimeout(async () => {
|
|
641
|
+
reconnectScheduled = false;
|
|
642
|
+
if (stopped) return;
|
|
643
|
+
const net = await waitForNetworkReady(FEISHU_HOST, { log: _log });
|
|
644
|
+
if (stopped) return;
|
|
645
|
+
if (!net.ok) {
|
|
646
|
+
_log('WARN', `Feishu network still down after ${Math.round(net.elapsed / 1000)}s (${net.error || 'unknown'}) — retrying`);
|
|
647
|
+
scheduleReconnect({ immediate: false, reason: 'network-wait-timeout', failed: true });
|
|
648
|
+
return;
|
|
649
|
+
}
|
|
650
|
+
if (net.attempts > 1) {
|
|
651
|
+
_log('INFO', `Feishu network ready after ${net.attempts} attempts (${Math.round(net.elapsed / 1000)}s)`);
|
|
652
|
+
}
|
|
515
653
|
connect();
|
|
516
|
-
},
|
|
517
|
-
|
|
654
|
+
}, delay);
|
|
655
|
+
}
|
|
656
|
+
|
|
657
|
+
// Alive-check: after each connect, require either SDK activity or a
|
|
658
|
+
// successful API probe within ALIVE_CHECK_WINDOW. Otherwise reconnect.
|
|
659
|
+
// This catches the "WSClient.start returned but underlying socket is
|
|
660
|
+
// dead" case that the 120s SDK loop would otherwise sit on.
|
|
661
|
+
function startAliveCheck() {
|
|
662
|
+
clearTimeout(aliveTimer);
|
|
663
|
+
const connectedAt = _connectedAt;
|
|
664
|
+
aliveTimer = setTimeout(async () => {
|
|
665
|
+
if (stopped) return;
|
|
666
|
+
if (_lastActivityAt > connectedAt) {
|
|
667
|
+
// SDK delivered at least one event strictly after connect → healthy.
|
|
668
|
+
// Using `>` (not `>=`) because connect() calls touchActivity(), so
|
|
669
|
+
// _lastActivityAt === _connectedAt at connect time — `>=` would
|
|
670
|
+
// false-positive immediately without any real post-connect activity.
|
|
671
|
+
reconnectDelay = INITIAL_RECONNECT_DELAY;
|
|
672
|
+
return;
|
|
673
|
+
}
|
|
674
|
+
try {
|
|
675
|
+
await withTimeout(client.im.chat.list({ params: { page_size: 1 } }), 8000);
|
|
676
|
+
touchActivity();
|
|
677
|
+
reconnectDelay = INITIAL_RECONNECT_DELAY;
|
|
678
|
+
_log('INFO', 'Feishu alive probe ok');
|
|
679
|
+
} catch (err) {
|
|
680
|
+
_log('WARN', `Feishu alive probe failed: ${err.message} — reconnecting`);
|
|
681
|
+
scheduleReconnect({ immediate: true, reason: 'alive-probe-failed', failed: true });
|
|
682
|
+
}
|
|
683
|
+
}, ALIVE_CHECK_WINDOW);
|
|
518
684
|
}
|
|
519
685
|
|
|
520
|
-
// Health check:
|
|
686
|
+
// Health check: bottom-line probe for silent dead-sockets the hooks missed.
|
|
521
687
|
function startHealthCheck() {
|
|
522
688
|
clearInterval(healthTimer);
|
|
523
689
|
healthTimer = setInterval(async () => {
|
|
524
690
|
if (stopped) return;
|
|
525
691
|
const silentMs = Date.now() - _lastActivityAt;
|
|
526
|
-
if (silentMs < SILENT_THRESHOLD) return;
|
|
527
|
-
// Probe: try a lightweight API call to verify token + connectivity
|
|
692
|
+
if (silentMs < SILENT_THRESHOLD) return;
|
|
528
693
|
try {
|
|
529
|
-
await withTimeout(client.im.chat.list({ params: { page_size: 1 } }),
|
|
530
|
-
// API works — connection might still be alive, just quiet. Reset activity.
|
|
694
|
+
await withTimeout(client.im.chat.list({ params: { page_size: 1 } }), 8000);
|
|
531
695
|
touchActivity();
|
|
532
696
|
} catch (err) {
|
|
533
697
|
_log('WARN', `Feishu health check failed after ${Math.round(silentMs / 1000)}s silence: ${err.message} — reconnecting`);
|
|
534
|
-
|
|
535
|
-
currentWs = null;
|
|
536
|
-
connect();
|
|
698
|
+
scheduleReconnect({ immediate: true, reason: 'health-probe-failed', failed: true });
|
|
537
699
|
}
|
|
538
700
|
}, HEALTH_CHECK_INTERVAL);
|
|
539
701
|
}
|
|
540
702
|
|
|
703
|
+
// Sleep/wake detector: JS clock jump >30s ⇒ system was suspended.
|
|
704
|
+
function startSleepWakeDetector() {
|
|
705
|
+
let _lastTickAt = Date.now();
|
|
706
|
+
sleepWakeTimer = setInterval(() => {
|
|
707
|
+
if (stopped) return;
|
|
708
|
+
const now = Date.now();
|
|
709
|
+
const elapsed = now - _lastTickAt;
|
|
710
|
+
_lastTickAt = now;
|
|
711
|
+
if (elapsed > SLEEP_JUMP_THRESHOLD) {
|
|
712
|
+
_log('INFO', `Feishu system wake detected (${Math.round(elapsed / 1000)}s gap) — reconnecting`);
|
|
713
|
+
reconnectDelay = INITIAL_RECONNECT_DELAY; // wake is a known cause, not a failure
|
|
714
|
+
scheduleReconnect({ immediate: true, reason: 'system-wake' });
|
|
715
|
+
}
|
|
716
|
+
}, SLEEP_DETECT_INTERVAL);
|
|
717
|
+
}
|
|
718
|
+
|
|
541
719
|
// Initial connect
|
|
542
720
|
connect();
|
|
543
721
|
startHealthCheck();
|
|
722
|
+
startSleepWakeDetector();
|
|
544
723
|
|
|
545
724
|
return Promise.resolve({
|
|
546
725
|
stop() {
|
|
547
726
|
stopped = true;
|
|
548
727
|
clearTimeout(reconnectTimer);
|
|
728
|
+
clearTimeout(aliveTimer);
|
|
549
729
|
clearInterval(healthTimer);
|
|
730
|
+
clearInterval(sleepWakeTimer);
|
|
731
|
+
try { currentWs?.stop?.(); } catch { /* ignore */ }
|
|
550
732
|
currentWs = null;
|
|
551
733
|
},
|
|
552
734
|
reconnect() {
|
|
553
735
|
_log('INFO', 'Feishu manual reconnect triggered');
|
|
554
|
-
reconnectDelay =
|
|
555
|
-
|
|
556
|
-
try { currentWs?.stop?.(); } catch { /* ignore */ }
|
|
557
|
-
currentWs = null;
|
|
558
|
-
connect();
|
|
736
|
+
reconnectDelay = INITIAL_RECONNECT_DELAY;
|
|
737
|
+
scheduleReconnect({ immediate: true, reason: 'manual' });
|
|
559
738
|
},
|
|
560
739
|
isAlive() {
|
|
561
740
|
return !stopped && (Date.now() - _lastActivityAt) < SILENT_THRESHOLD;
|
|
@@ -567,4 +746,4 @@ function createBot(config) {
|
|
|
567
746
|
};
|
|
568
747
|
}
|
|
569
748
|
|
|
570
|
-
module.exports = { createBot };
|
|
749
|
+
module.exports = { createBot, _internal: { waitForNetworkReady } };
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
'use strict';
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* memory-backfill-chunks.js — One-time backfill for existing wiki pages
|
|
7
|
+
*
|
|
8
|
+
* For each wiki page that has content but no content_chunks rows:
|
|
9
|
+
* 1. Splits content into chunks via recursive chunker
|
|
10
|
+
* 2. Inserts chunk rows
|
|
11
|
+
* 3. Enqueues each chunk for embedding generation
|
|
12
|
+
*
|
|
13
|
+
* Idempotent: pages with existing chunks are skipped.
|
|
14
|
+
*
|
|
15
|
+
* Usage: node scripts/memory-backfill-chunks.js
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
const path = require('path');
|
|
19
|
+
const os = require('os');
|
|
20
|
+
|
|
21
|
+
const DB_PATH = path.join(os.homedir(), '.metame', 'memory.db');
|
|
22
|
+
|
|
23
|
+
function main() {
|
|
24
|
+
const { DatabaseSync } = require('node:sqlite');
|
|
25
|
+
const db = new DatabaseSync(DB_PATH);
|
|
26
|
+
db.exec('PRAGMA journal_mode = WAL');
|
|
27
|
+
db.exec('PRAGMA busy_timeout = 3000');
|
|
28
|
+
|
|
29
|
+
// Ensure schema is up to date
|
|
30
|
+
try {
|
|
31
|
+
const { applyWikiSchema } = require('./memory-wiki-schema');
|
|
32
|
+
applyWikiSchema(db);
|
|
33
|
+
} catch (err) {
|
|
34
|
+
process.stderr.write(`Schema init failed: ${err.message}\n`);
|
|
35
|
+
db.close();
|
|
36
|
+
process.exit(1);
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
const { chunkText } = require('./core/chunker');
|
|
40
|
+
|
|
41
|
+
// Find pages without chunks
|
|
42
|
+
const pages = db.prepare(`
|
|
43
|
+
SELECT wp.slug, wp.content
|
|
44
|
+
FROM wiki_pages wp
|
|
45
|
+
WHERE wp.content IS NOT NULL
|
|
46
|
+
AND wp.content != ''
|
|
47
|
+
AND NOT EXISTS (
|
|
48
|
+
SELECT 1 FROM content_chunks cc WHERE cc.page_slug = wp.slug
|
|
49
|
+
)
|
|
50
|
+
`).all();
|
|
51
|
+
|
|
52
|
+
if (pages.length === 0) {
|
|
53
|
+
console.log('All wiki pages already have chunks. Nothing to backfill.');
|
|
54
|
+
db.close();
|
|
55
|
+
return;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
console.log(`Backfilling ${pages.length} wiki pages...`);
|
|
59
|
+
|
|
60
|
+
const insertChunk = db.prepare(
|
|
61
|
+
'INSERT INTO content_chunks (id, page_slug, chunk_text, chunk_idx) VALUES (?, ?, ?, ?)',
|
|
62
|
+
);
|
|
63
|
+
const enqueue = db.prepare(
|
|
64
|
+
"INSERT INTO embedding_queue (item_type, item_id) VALUES ('chunk', ?)",
|
|
65
|
+
);
|
|
66
|
+
|
|
67
|
+
let totalChunks = 0;
|
|
68
|
+
|
|
69
|
+
db.prepare('BEGIN').run();
|
|
70
|
+
try {
|
|
71
|
+
for (const page of pages) {
|
|
72
|
+
const chunks = chunkText(page.content, { targetWords: 300 });
|
|
73
|
+
for (let i = 0; i < chunks.length; i++) {
|
|
74
|
+
const chunkId = `ck_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
|
|
75
|
+
insertChunk.run(chunkId, page.slug, chunks[i], i);
|
|
76
|
+
enqueue.run(chunkId);
|
|
77
|
+
totalChunks++;
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
db.prepare('COMMIT').run();
|
|
81
|
+
} catch (err) {
|
|
82
|
+
try { db.prepare('ROLLBACK').run(); } catch { }
|
|
83
|
+
process.stderr.write(`Backfill failed: ${err.message}\n`);
|
|
84
|
+
db.close();
|
|
85
|
+
process.exit(1);
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
console.log(`Done. Created ${totalChunks} chunks for ${pages.length} pages. Run daemon-embedding.js to generate embeddings.`);
|
|
89
|
+
db.close();
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
main();
|