chattercatcher 0.1.13 → 0.1.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +335 -11
- package/dist/cli.js.map +1 -1
- package/dist/index.d.ts +105 -33
- package/dist/index.js +316 -10
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.d.ts
CHANGED
|
@@ -37,6 +37,10 @@ declare const appConfigSchema: z.ZodObject<{
|
|
|
37
37
|
schedules: z.ZodObject<{
|
|
38
38
|
indexing: z.ZodDefault<z.ZodString>;
|
|
39
39
|
}, z.core.$strip>;
|
|
40
|
+
episodes: z.ZodObject<{
|
|
41
|
+
windowMinutes: z.ZodDefault<z.ZodNumber>;
|
|
42
|
+
quietMinutes: z.ZodDefault<z.ZodNumber>;
|
|
43
|
+
}, z.core.$strip>;
|
|
40
44
|
}, z.core.$strip>;
|
|
41
45
|
declare const appSecretsSchema: z.ZodObject<{
|
|
42
46
|
feishu: z.ZodObject<{
|
|
@@ -135,6 +139,49 @@ declare function restoreLocalData(input: {
|
|
|
135
139
|
replace?: boolean;
|
|
136
140
|
}): Promise<DataRestoreResult>;
|
|
137
141
|
|
|
142
|
+
type SourceType = "message" | "episode" | "file" | "image" | "audio" | "link" | "feishu_doc";
|
|
143
|
+
interface EvidenceSource {
|
|
144
|
+
type: SourceType;
|
|
145
|
+
label: string;
|
|
146
|
+
timestamp?: string;
|
|
147
|
+
sender?: string;
|
|
148
|
+
location?: string;
|
|
149
|
+
}
|
|
150
|
+
interface EvidenceBlock {
|
|
151
|
+
id: string;
|
|
152
|
+
text: string;
|
|
153
|
+
score: number;
|
|
154
|
+
source: EvidenceSource;
|
|
155
|
+
}
|
|
156
|
+
interface Citation {
|
|
157
|
+
marker: string;
|
|
158
|
+
evidenceId: string;
|
|
159
|
+
source: EvidenceSource;
|
|
160
|
+
text: string;
|
|
161
|
+
}
|
|
162
|
+
interface GroundedAnswer {
|
|
163
|
+
answer: string;
|
|
164
|
+
citations: Citation[];
|
|
165
|
+
}
|
|
166
|
+
interface ChatMessage {
|
|
167
|
+
role: "system" | "user" | "assistant";
|
|
168
|
+
content: string;
|
|
169
|
+
}
|
|
170
|
+
interface ChatModel {
|
|
171
|
+
complete(messages: ChatMessage[]): Promise<string>;
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
interface ProcessEpisodesResult {
|
|
175
|
+
created: number;
|
|
176
|
+
}
|
|
177
|
+
declare function processEpisodesNow(input: {
|
|
178
|
+
config: AppConfig;
|
|
179
|
+
secrets: AppSecrets;
|
|
180
|
+
database: SqliteDatabase;
|
|
181
|
+
model: ChatModel;
|
|
182
|
+
now?: Date;
|
|
183
|
+
}): Promise<ProcessEpisodesResult>;
|
|
184
|
+
|
|
138
185
|
interface ChatRecord {
|
|
139
186
|
id: string;
|
|
140
187
|
platform: string;
|
|
@@ -192,6 +239,52 @@ interface MessageSearchResult {
|
|
|
192
239
|
sentAt: string;
|
|
193
240
|
}
|
|
194
241
|
|
|
242
|
+
interface EpisodeMessage {
|
|
243
|
+
id: string;
|
|
244
|
+
chatId: string;
|
|
245
|
+
chatName: string;
|
|
246
|
+
senderName: string;
|
|
247
|
+
text: string;
|
|
248
|
+
sentAt: string;
|
|
249
|
+
}
|
|
250
|
+
interface EpisodeWindow {
|
|
251
|
+
chatId: string;
|
|
252
|
+
chatName: string;
|
|
253
|
+
startedAt: string;
|
|
254
|
+
endedAt: string;
|
|
255
|
+
messages: EpisodeMessage[];
|
|
256
|
+
}
|
|
257
|
+
interface EpisodeSummaryRecord {
|
|
258
|
+
id: string;
|
|
259
|
+
chatId: string;
|
|
260
|
+
chatName: string;
|
|
261
|
+
text: string;
|
|
262
|
+
startedAt: string;
|
|
263
|
+
endedAt: string;
|
|
264
|
+
messageIds: string[];
|
|
265
|
+
}
|
|
266
|
+
interface EpisodeSearchResult extends MessageSearchResult {
|
|
267
|
+
sourceMessageIds: string[];
|
|
268
|
+
startedAt: string;
|
|
269
|
+
endedAt: string;
|
|
270
|
+
}
|
|
271
|
+
declare class EpisodeRepository {
|
|
272
|
+
private readonly database;
|
|
273
|
+
constructor(database: SqliteDatabase);
|
|
274
|
+
summarizeReadyWindows(input: {
|
|
275
|
+
now: Date;
|
|
276
|
+
quietMs: number;
|
|
277
|
+
windowMs: number;
|
|
278
|
+
summarize: (window: EpisodeWindow) => Promise<string>;
|
|
279
|
+
}): Promise<EpisodeSummaryRecord[]>;
|
|
280
|
+
private insertEpisode;
|
|
281
|
+
searchEpisodes(query: string, limit?: number): EpisodeSearchResult[];
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
declare function sanitizeEpisodeSummary(summary: string): string;
|
|
285
|
+
|
|
286
|
+
declare function summarizeEpisodeWindow(window: EpisodeWindow, model: ChatModel): Promise<string>;
|
|
287
|
+
|
|
195
288
|
type JsonObject = Record<string, unknown>;
|
|
196
289
|
interface FeishuAttachmentMetadata {
|
|
197
290
|
platform: "feishu";
|
|
@@ -396,38 +489,6 @@ declare class GatewayIngestor {
|
|
|
396
489
|
}): Promise<GatewayIngestAndDownloadResult>;
|
|
397
490
|
}
|
|
398
491
|
|
|
399
|
-
type SourceType = "message" | "file" | "image" | "audio" | "link" | "feishu_doc";
|
|
400
|
-
interface EvidenceSource {
|
|
401
|
-
type: SourceType;
|
|
402
|
-
label: string;
|
|
403
|
-
timestamp?: string;
|
|
404
|
-
sender?: string;
|
|
405
|
-
location?: string;
|
|
406
|
-
}
|
|
407
|
-
interface EvidenceBlock {
|
|
408
|
-
id: string;
|
|
409
|
-
text: string;
|
|
410
|
-
score: number;
|
|
411
|
-
source: EvidenceSource;
|
|
412
|
-
}
|
|
413
|
-
interface Citation {
|
|
414
|
-
marker: string;
|
|
415
|
-
evidenceId: string;
|
|
416
|
-
source: EvidenceSource;
|
|
417
|
-
text: string;
|
|
418
|
-
}
|
|
419
|
-
interface GroundedAnswer {
|
|
420
|
-
answer: string;
|
|
421
|
-
citations: Citation[];
|
|
422
|
-
}
|
|
423
|
-
interface ChatMessage {
|
|
424
|
-
role: "system" | "user" | "assistant";
|
|
425
|
-
content: string;
|
|
426
|
-
}
|
|
427
|
-
interface ChatModel {
|
|
428
|
-
complete(messages: ChatMessage[]): Promise<string>;
|
|
429
|
-
}
|
|
430
|
-
|
|
431
492
|
interface MessageSender {
|
|
432
493
|
sendTextToChat(chatId: string, text: string): Promise<void>;
|
|
433
494
|
replyTextToMessage?(messageId: string, text: string): Promise<void>;
|
|
@@ -551,6 +612,11 @@ interface FeishuGatewayOptions {
|
|
|
551
612
|
chunks: number;
|
|
552
613
|
vectors: number;
|
|
553
614
|
}>;
|
|
615
|
+
episodeProcessor?: {
|
|
616
|
+
database: SqliteDatabase;
|
|
617
|
+
model: ChatModel;
|
|
618
|
+
now?: () => Date;
|
|
619
|
+
};
|
|
554
620
|
wsClientFactory?: (params: {
|
|
555
621
|
appId: string;
|
|
556
622
|
appSecret: string;
|
|
@@ -563,6 +629,7 @@ interface FeishuGatewayOptions {
|
|
|
563
629
|
}
|
|
564
630
|
declare function createFeishuEventDispatcher(options: {
|
|
565
631
|
config: AppConfig;
|
|
632
|
+
secrets: AppSecrets;
|
|
566
633
|
ingestor: GatewayIngestor;
|
|
567
634
|
questionHandler?: FeishuQuestionHandler;
|
|
568
635
|
resourceDownloader?: FeishuResourceDownloader;
|
|
@@ -570,6 +637,11 @@ declare function createFeishuEventDispatcher(options: {
|
|
|
570
637
|
chunks: number;
|
|
571
638
|
vectors: number;
|
|
572
639
|
}>;
|
|
640
|
+
episodeProcessor?: {
|
|
641
|
+
database: SqliteDatabase;
|
|
642
|
+
model: ChatModel;
|
|
643
|
+
now?: () => Date;
|
|
644
|
+
};
|
|
573
645
|
}): lark.EventDispatcher;
|
|
574
646
|
declare function createFeishuGateway(options: FeishuGatewayOptions): FeishuGatewayRuntime;
|
|
575
647
|
|
|
@@ -796,4 +868,4 @@ declare class VectorRetriever implements Retriever {
|
|
|
796
868
|
declare function createWebApp(config: AppConfig): FastifyInstance;
|
|
797
869
|
declare function startWebServer(config: AppConfig): Promise<void>;
|
|
798
870
|
|
|
799
|
-
export { type AppConfig, type AppSecrets, type AskWithRagInput, type BuildEvidencePromptOptions, type ChatMessage, type ChatModel, type ChatRecord, type Citation, type DataExportResult, type DataRestoreResult, type DeleteLocalDataResult, type DeleteTargetType, type DoctorCheck, type DoctorOptions, type DoctorStatus, type EmbeddingModel, type EvidenceBlock, type EvidencePrompt, type EvidenceSource, type FeishuAttachmentMetadata, type FeishuDownloadResourceInput, type FeishuDownloadedResource, type FeishuGatewayOptions, type FeishuGatewayRuntime, FeishuMessageSender, type FeishuQuestionDecision, FeishuQuestionHandler, type FeishuQuestionHandlerOptions, type FeishuReceiveMessageEvent, FeishuResourceDownloader, type FileJobRecord, FileJobRepository, type FileJobStatus, type FileRecord, type GatewayAttachmentIngestResult, type GatewayIngestAndDownloadResult, type GatewayIngestResult, GatewayIngestor, type GatewayPidRecord, type GatewayRuntimeState, type GroundedAnswer, HybridRetriever, type HybridRetrieverOptions, type IngestLocalFileResult, type IngestMessageInput, type LogFileInfo, type LogTailResult, type ManualMessageIndexResult, MemoryVectorStore, MessageFtsRetriever, type MessageRecord, MessageRepository, type MessageSearchResult, type MessageSender, OpenAICompatibleChatModel, type OpenAICompatibleChatOptions, OpenAICompatibleEmbeddingModel, type OpenAICompatibleEmbeddingOptions, type ParsedFile, type SourceType, type SqliteDatabase, type StopGatewayResult, type TextChunk, type VectorIndexStats, type VectorRecord, VectorRetriever, type VectorSearchResult, type VectorStore, appConfigSchema, appSecretsSchema, applySecretInput, askWithRag, buildEvidencePrompt, chunkText, cosineSimilarity, createChatModel, createDefaultConfig, createDefaultSecrets, createEmbeddingModel, createFeishuEventDispatcher, createFeishuGateway, createHybridRetriever, createWebApp, deleteLocalData, describeSupportedParseTypes, ensureConfigFiles, exportLocalData, extractFeishuAttachment, followLogFile, formatCitation, formatCitations, formatDoctorChecks, generateGroundedAnswer, getDatabasePath, getFeishuQuestionDecision, getGatewayLogPath, getGatewayPidPath, getGatewayRuntimeState, getLogsDirectory, hasEmbeddingConfig, indexMessageChunks, ingestLocalFile, isFeishuMessageAddressedToBot, isProcessRunning, isSupportedParseFile, isSupportedTextFile, listLogFiles, loadConfig, loadSecrets, mapDomain, maskSecret, migrateDatabase, normalizeFeishuReceiveMessageEvent, normalizeLineCount, openDatabase, parseFileToText, processMessagesNow, rankEvidenceForPrompt, readGatewayPidRecord, readLatestLogTail, readLogTail, removeGatewayPidRecord, resetConfigFiles, resolveEmbeddingApiKey, resolveLogPath, restoreLocalData, runDoctor, saveConfig, saveSecrets, startWebServer, stopGatewayProcess, writeGatewayPidRecord };
|
|
871
|
+
export { type AppConfig, type AppSecrets, type AskWithRagInput, type BuildEvidencePromptOptions, type ChatMessage, type ChatModel, type ChatRecord, type Citation, type DataExportResult, type DataRestoreResult, type DeleteLocalDataResult, type DeleteTargetType, type DoctorCheck, type DoctorOptions, type DoctorStatus, type EmbeddingModel, type EpisodeMessage, EpisodeRepository, type EpisodeSearchResult, type EpisodeSummaryRecord, type EpisodeWindow, type EvidenceBlock, type EvidencePrompt, type EvidenceSource, type FeishuAttachmentMetadata, type FeishuDownloadResourceInput, type FeishuDownloadedResource, type FeishuGatewayOptions, type FeishuGatewayRuntime, FeishuMessageSender, type FeishuQuestionDecision, FeishuQuestionHandler, type FeishuQuestionHandlerOptions, type FeishuReceiveMessageEvent, FeishuResourceDownloader, type FileJobRecord, FileJobRepository, type FileJobStatus, type FileRecord, type GatewayAttachmentIngestResult, type GatewayIngestAndDownloadResult, type GatewayIngestResult, GatewayIngestor, type GatewayPidRecord, type GatewayRuntimeState, type GroundedAnswer, HybridRetriever, type HybridRetrieverOptions, type IngestLocalFileResult, type IngestMessageInput, type LogFileInfo, type LogTailResult, type ManualMessageIndexResult, MemoryVectorStore, MessageFtsRetriever, type MessageRecord, MessageRepository, type MessageSearchResult, type MessageSender, OpenAICompatibleChatModel, type OpenAICompatibleChatOptions, OpenAICompatibleEmbeddingModel, type OpenAICompatibleEmbeddingOptions, type ParsedFile, type ProcessEpisodesResult, type SourceType, type SqliteDatabase, type StopGatewayResult, type TextChunk, type VectorIndexStats, type VectorRecord, VectorRetriever, type VectorSearchResult, type VectorStore, appConfigSchema, appSecretsSchema, applySecretInput, askWithRag, buildEvidencePrompt, chunkText, cosineSimilarity, createChatModel, createDefaultConfig, createDefaultSecrets, createEmbeddingModel, createFeishuEventDispatcher, createFeishuGateway, createHybridRetriever, createWebApp, deleteLocalData, describeSupportedParseTypes, ensureConfigFiles, exportLocalData, extractFeishuAttachment, followLogFile, formatCitation, formatCitations, formatDoctorChecks, generateGroundedAnswer, getDatabasePath, getFeishuQuestionDecision, getGatewayLogPath, getGatewayPidPath, getGatewayRuntimeState, getLogsDirectory, hasEmbeddingConfig, indexMessageChunks, ingestLocalFile, isFeishuMessageAddressedToBot, isProcessRunning, isSupportedParseFile, isSupportedTextFile, listLogFiles, loadConfig, loadSecrets, mapDomain, maskSecret, migrateDatabase, normalizeFeishuReceiveMessageEvent, normalizeLineCount, openDatabase, parseFileToText, processEpisodesNow, processMessagesNow, rankEvidenceForPrompt, readGatewayPidRecord, readLatestLogTail, readLogTail, removeGatewayPidRecord, resetConfigFiles, resolveEmbeddingApiKey, resolveLogPath, restoreLocalData, runDoctor, sanitizeEpisodeSummary, saveConfig, saveSecrets, startWebServer, stopGatewayProcess, summarizeEpisodeWindow, writeGatewayPidRecord };
|
package/dist/index.js
CHANGED
|
@@ -31,6 +31,10 @@ var appConfigSchema = z.object({
|
|
|
31
31
|
}),
|
|
32
32
|
schedules: z.object({
|
|
33
33
|
indexing: z.string().default("*/10 * * * *")
|
|
34
|
+
}),
|
|
35
|
+
episodes: z.object({
|
|
36
|
+
windowMinutes: z.number().int().positive().default(10),
|
|
37
|
+
quietMinutes: z.number().int().positive().default(2)
|
|
34
38
|
})
|
|
35
39
|
});
|
|
36
40
|
var appSecretsSchema = z.object({
|
|
@@ -51,7 +55,8 @@ function createDefaultConfig() {
|
|
|
51
55
|
embedding: {},
|
|
52
56
|
storage: {},
|
|
53
57
|
web: {},
|
|
54
|
-
schedules: {}
|
|
58
|
+
schedules: {},
|
|
59
|
+
episodes: {}
|
|
55
60
|
});
|
|
56
61
|
}
|
|
57
62
|
function createDefaultSecrets() {
|
|
@@ -337,6 +342,39 @@ function migrateDatabase(database) {
|
|
|
337
342
|
tokenize = 'unicode61'
|
|
338
343
|
);
|
|
339
344
|
|
|
345
|
+
CREATE TABLE IF NOT EXISTS memory_episodes (
|
|
346
|
+
id TEXT PRIMARY KEY,
|
|
347
|
+
chat_id TEXT NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
|
348
|
+
summary TEXT NOT NULL,
|
|
349
|
+
message_count INTEGER NOT NULL,
|
|
350
|
+
started_at TEXT NOT NULL,
|
|
351
|
+
ended_at TEXT NOT NULL,
|
|
352
|
+
created_at TEXT NOT NULL,
|
|
353
|
+
UNIQUE(chat_id, started_at, ended_at)
|
|
354
|
+
);
|
|
355
|
+
|
|
356
|
+
CREATE TABLE IF NOT EXISTS memory_episode_messages (
|
|
357
|
+
episode_id TEXT NOT NULL REFERENCES memory_episodes(id) ON DELETE CASCADE,
|
|
358
|
+
message_id TEXT NOT NULL REFERENCES messages(id) ON DELETE CASCADE,
|
|
359
|
+
position INTEGER NOT NULL,
|
|
360
|
+
PRIMARY KEY (episode_id, message_id)
|
|
361
|
+
);
|
|
362
|
+
|
|
363
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS memory_episodes_fts USING fts5(
|
|
364
|
+
summary,
|
|
365
|
+
episode_id UNINDEXED,
|
|
366
|
+
tokenize = 'unicode61'
|
|
367
|
+
);
|
|
368
|
+
|
|
369
|
+
CREATE TRIGGER IF NOT EXISTS memory_episodes_delete_fts
|
|
370
|
+
AFTER DELETE ON memory_episodes
|
|
371
|
+
BEGIN
|
|
372
|
+
DELETE FROM memory_episodes_fts WHERE episode_id = old.id;
|
|
373
|
+
END;
|
|
374
|
+
|
|
375
|
+
CREATE INDEX IF NOT EXISTS memory_episode_messages_message_idx
|
|
376
|
+
ON memory_episode_messages(message_id);
|
|
377
|
+
|
|
340
378
|
CREATE TABLE IF NOT EXISTS message_chunk_embeddings (
|
|
341
379
|
chunk_id TEXT NOT NULL REFERENCES message_chunks(id) ON DELETE CASCADE,
|
|
342
380
|
model TEXT NOT NULL,
|
|
@@ -1193,6 +1231,211 @@ var MessageRepository = class {
|
|
|
1193
1231
|
}
|
|
1194
1232
|
};
|
|
1195
1233
|
|
|
1234
|
+
// src/episodes/repository.ts
|
|
1235
|
+
import crypto3 from "crypto";
|
|
1236
|
+
|
|
1237
|
+
// src/episodes/sanitizer.ts
|
|
1238
|
+
var SECRET_PATTERNS = [
|
|
1239
|
+
[/-----BEGIN [^-]+ PRIVATE KEY-----[\s\S]*?-----END [^-]+ PRIVATE KEY-----/g, "[REDACTED_SECRET]"],
|
|
1240
|
+
[/(\bAuthorization\s*:\s*Bearer\s+)[A-Za-z0-9._~+/=-]{12,}/gi, "$1[REDACTED_SECRET]"],
|
|
1241
|
+
[/(https?:\/\/)[^\s/@:]+:[^\s/@]+@/gi, "$1[REDACTED_SECRET]@"],
|
|
1242
|
+
[/([?&](?:api[_-]?key|access[_-]?token|refresh[_-]?token|token|secret|password|session(?:id)?|client[_-]?secret)=)[^\s&,。;;]+/gi, "$1[REDACTED_SECRET]"],
|
|
1243
|
+
[/("(?:api[_-]?key|access[_-]?token|refresh[_-]?token|token|secret|password|session(?:id)?|client[_-]?secret|private[_-]?key)"\s*:\s*")[^"]+(")/gi, "$1[REDACTED_SECRET]$2"],
|
|
1244
|
+
[/(\b(?:api[_-]?key|access[_-]?token|refresh[_-]?token|token|secret|password|session(?:id)?|client[_-]?secret)\s*[=:]\s*)[^\s;,。]+/gi, "$1[REDACTED_SECRET]"],
|
|
1245
|
+
[/\b(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9_]{20,}\b/g, "[REDACTED_SECRET]"],
|
|
1246
|
+
[/\bxox[baprs]-[A-Za-z0-9-]{20,}\b/g, "[REDACTED_SECRET]"],
|
|
1247
|
+
[/\bsk-[A-Za-z0-9_-]{6,}\b/g, "[REDACTED_SECRET]"]
|
|
1248
|
+
];
|
|
1249
|
+
function sanitizeEpisodeSummary(summary) {
|
|
1250
|
+
let sanitized = summary;
|
|
1251
|
+
for (const [pattern, replacement] of SECRET_PATTERNS) {
|
|
1252
|
+
sanitized = sanitized.replace(pattern, replacement);
|
|
1253
|
+
}
|
|
1254
|
+
return sanitized;
|
|
1255
|
+
}
|
|
1256
|
+
|
|
1257
|
+
// src/episodes/repository.ts
|
|
1258
|
+
function nowIso3() {
|
|
1259
|
+
return (/* @__PURE__ */ new Date()).toISOString();
|
|
1260
|
+
}
|
|
1261
|
+
function stableId2(parts) {
|
|
1262
|
+
return crypto3.createHash("sha256").update(parts.join("")).digest("hex").slice(0, 32);
|
|
1263
|
+
}
|
|
1264
|
+
function escapeFtsQuery2(query) {
|
|
1265
|
+
const terms = query.trim().split(/\s+/).map((term) => term.replace(/[^\p{L}\p{N}_-]+/gu, " ").trim()).flatMap((term) => term.split(/\s+/)).filter(Boolean);
|
|
1266
|
+
if (terms.length === 0) {
|
|
1267
|
+
return '""';
|
|
1268
|
+
}
|
|
1269
|
+
return terms.map((term) => `"${term.replace(/"/g, '""')}"`).join(" OR ");
|
|
1270
|
+
}
|
|
1271
|
+
function toMillis(value) {
|
|
1272
|
+
const time = Date.parse(value);
|
|
1273
|
+
return Number.isFinite(time) ? time : 0;
|
|
1274
|
+
}
|
|
1275
|
+
var EpisodeRepository = class {
|
|
1276
|
+
constructor(database) {
|
|
1277
|
+
this.database = database;
|
|
1278
|
+
}
|
|
1279
|
+
database;
|
|
1280
|
+
async summarizeReadyWindows(input) {
|
|
1281
|
+
const rows = this.database.prepare(
|
|
1282
|
+
`
|
|
1283
|
+
SELECT
|
|
1284
|
+
m.id,
|
|
1285
|
+
m.chat_id AS chatId,
|
|
1286
|
+
c.name AS chatName,
|
|
1287
|
+
m.sender_name AS senderName,
|
|
1288
|
+
m.text,
|
|
1289
|
+
m.sent_at AS sentAt
|
|
1290
|
+
FROM messages m
|
|
1291
|
+
JOIN chats c ON c.id = m.chat_id
|
|
1292
|
+
WHERE NOT EXISTS (
|
|
1293
|
+
SELECT 1 FROM memory_episode_messages mem WHERE mem.message_id = m.id
|
|
1294
|
+
)
|
|
1295
|
+
ORDER BY m.chat_id ASC, m.sent_at ASC
|
|
1296
|
+
`
|
|
1297
|
+
).all();
|
|
1298
|
+
const byChat = /* @__PURE__ */ new Map();
|
|
1299
|
+
for (const row of rows) {
|
|
1300
|
+
byChat.set(row.chatId, [...byChat.get(row.chatId) ?? [], row]);
|
|
1301
|
+
}
|
|
1302
|
+
const created = [];
|
|
1303
|
+
const nowMs = input.now.getTime();
|
|
1304
|
+
for (const messages of byChat.values()) {
|
|
1305
|
+
const windows = [];
|
|
1306
|
+
let current = [];
|
|
1307
|
+
for (const message of messages) {
|
|
1308
|
+
const first = current[0];
|
|
1309
|
+
if (first && toMillis(message.sentAt) - toMillis(first.sentAt) > input.windowMs) {
|
|
1310
|
+
windows.push(current);
|
|
1311
|
+
current = [];
|
|
1312
|
+
}
|
|
1313
|
+
current.push(message);
|
|
1314
|
+
}
|
|
1315
|
+
if (current.length > 0) {
|
|
1316
|
+
windows.push(current);
|
|
1317
|
+
}
|
|
1318
|
+
for (const windowMessages of windows) {
|
|
1319
|
+
const last = windowMessages.at(-1);
|
|
1320
|
+
if (!last || nowMs - toMillis(last.sentAt) < input.quietMs) {
|
|
1321
|
+
continue;
|
|
1322
|
+
}
|
|
1323
|
+
const first = windowMessages[0];
|
|
1324
|
+
const window = {
|
|
1325
|
+
chatId: first.chatId,
|
|
1326
|
+
chatName: first.chatName,
|
|
1327
|
+
startedAt: first.sentAt,
|
|
1328
|
+
endedAt: last.sentAt,
|
|
1329
|
+
messages: windowMessages
|
|
1330
|
+
};
|
|
1331
|
+
const summary = await input.summarize(window);
|
|
1332
|
+
created.push(this.insertEpisode(window, summary));
|
|
1333
|
+
}
|
|
1334
|
+
}
|
|
1335
|
+
return created;
|
|
1336
|
+
}
|
|
1337
|
+
insertEpisode(window, summary) {
|
|
1338
|
+
const safeSummary = sanitizeEpisodeSummary(summary);
|
|
1339
|
+
const createdAt = nowIso3();
|
|
1340
|
+
const id = stableId2([window.chatId, window.startedAt, window.endedAt]);
|
|
1341
|
+
const transaction = this.database.transaction(() => {
|
|
1342
|
+
this.database.prepare(
|
|
1343
|
+
`
|
|
1344
|
+
INSERT INTO memory_episodes (id, chat_id, summary, message_count, started_at, ended_at, created_at)
|
|
1345
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
1346
|
+
ON CONFLICT(chat_id, started_at, ended_at)
|
|
1347
|
+
DO UPDATE SET summary = excluded.summary, message_count = excluded.message_count
|
|
1348
|
+
`
|
|
1349
|
+
).run(id, window.chatId, safeSummary, window.messages.length, window.startedAt, window.endedAt, createdAt);
|
|
1350
|
+
this.database.prepare("DELETE FROM memory_episode_messages WHERE episode_id = ?").run(id);
|
|
1351
|
+
this.database.prepare("DELETE FROM memory_episodes_fts WHERE episode_id = ?").run(id);
|
|
1352
|
+
const insertMessage = this.database.prepare(
|
|
1353
|
+
"INSERT INTO memory_episode_messages (episode_id, message_id, position) VALUES (?, ?, ?)"
|
|
1354
|
+
);
|
|
1355
|
+
for (const [index, message] of window.messages.entries()) {
|
|
1356
|
+
insertMessage.run(id, message.id, index);
|
|
1357
|
+
}
|
|
1358
|
+
this.database.prepare("INSERT INTO memory_episodes_fts (summary, episode_id) VALUES (?, ?)").run(safeSummary, id);
|
|
1359
|
+
});
|
|
1360
|
+
transaction();
|
|
1361
|
+
return {
|
|
1362
|
+
id,
|
|
1363
|
+
chatId: window.chatId,
|
|
1364
|
+
chatName: window.chatName,
|
|
1365
|
+
text: safeSummary,
|
|
1366
|
+
startedAt: window.startedAt,
|
|
1367
|
+
endedAt: window.endedAt,
|
|
1368
|
+
messageIds: window.messages.map((message) => message.id)
|
|
1369
|
+
};
|
|
1370
|
+
}
|
|
1371
|
+
searchEpisodes(query, limit = 8) {
|
|
1372
|
+
const ftsQuery = escapeFtsQuery2(query);
|
|
1373
|
+
return this.database.prepare(
|
|
1374
|
+
`
|
|
1375
|
+
SELECT
|
|
1376
|
+
e.id AS chunkId,
|
|
1377
|
+
e.id AS messageId,
|
|
1378
|
+
'episode' AS platform,
|
|
1379
|
+
e.summary AS text,
|
|
1380
|
+
1.0 AS score,
|
|
1381
|
+
'episode' AS messageType,
|
|
1382
|
+
c.name AS chatName,
|
|
1383
|
+
'\u4F1A\u8BDD\u8BB0\u5FC6' AS senderName,
|
|
1384
|
+
e.ended_at AS sentAt,
|
|
1385
|
+
e.started_at AS startedAt,
|
|
1386
|
+
e.ended_at AS endedAt,
|
|
1387
|
+
(
|
|
1388
|
+
SELECT json_group_array(message_id)
|
|
1389
|
+
FROM (
|
|
1390
|
+
SELECT message_id
|
|
1391
|
+
FROM memory_episode_messages
|
|
1392
|
+
WHERE episode_id = e.id
|
|
1393
|
+
ORDER BY position ASC
|
|
1394
|
+
)
|
|
1395
|
+
) AS sourceMessageIdsJson
|
|
1396
|
+
FROM memory_episodes_fts fts
|
|
1397
|
+
JOIN memory_episodes e ON e.id = fts.episode_id
|
|
1398
|
+
JOIN chats c ON c.id = e.chat_id
|
|
1399
|
+
WHERE memory_episodes_fts MATCH ?
|
|
1400
|
+
GROUP BY e.id
|
|
1401
|
+
ORDER BY e.ended_at DESC
|
|
1402
|
+
LIMIT ?
|
|
1403
|
+
`
|
|
1404
|
+
).all(ftsQuery, limit).map((row) => {
|
|
1405
|
+
const item = row;
|
|
1406
|
+
return {
|
|
1407
|
+
...item,
|
|
1408
|
+
sourceMessageIds: JSON.parse(item.sourceMessageIdsJson)
|
|
1409
|
+
};
|
|
1410
|
+
});
|
|
1411
|
+
}
|
|
1412
|
+
};
|
|
1413
|
+
|
|
1414
|
+
// src/rag/episode-retriever.ts
|
|
1415
|
+
function toEpisodeEvidence(result) {
|
|
1416
|
+
return {
|
|
1417
|
+
id: result.chunkId,
|
|
1418
|
+
text: result.text,
|
|
1419
|
+
score: result.score,
|
|
1420
|
+
source: {
|
|
1421
|
+
type: "episode",
|
|
1422
|
+
label: result.chatName,
|
|
1423
|
+
sender: result.senderName,
|
|
1424
|
+
timestamp: result.endedAt,
|
|
1425
|
+
location: `${result.startedAt} - ${result.endedAt}`
|
|
1426
|
+
}
|
|
1427
|
+
};
|
|
1428
|
+
}
|
|
1429
|
+
var EpisodeFtsRetriever = class {
|
|
1430
|
+
constructor(episodes) {
|
|
1431
|
+
this.episodes = episodes;
|
|
1432
|
+
}
|
|
1433
|
+
episodes;
|
|
1434
|
+
async retrieve(question) {
|
|
1435
|
+
return this.episodes.searchEpisodes(question, 8).map(toEpisodeEvidence);
|
|
1436
|
+
}
|
|
1437
|
+
};
|
|
1438
|
+
|
|
1196
1439
|
// src/rag/hybrid-retriever.ts
|
|
1197
1440
|
function normalizeScore(score) {
|
|
1198
1441
|
if (!Number.isFinite(score)) {
|
|
@@ -1200,6 +1443,14 @@ function normalizeScore(score) {
|
|
|
1200
1443
|
}
|
|
1201
1444
|
return Math.max(0, Math.min(1, score));
|
|
1202
1445
|
}
|
|
1446
|
+
function evidenceTimestampMs(evidence) {
|
|
1447
|
+
const timestamp = evidence.source.timestamp;
|
|
1448
|
+
if (!timestamp) {
|
|
1449
|
+
return 0;
|
|
1450
|
+
}
|
|
1451
|
+
const parsed = Date.parse(timestamp);
|
|
1452
|
+
return Number.isFinite(parsed) ? parsed : 0;
|
|
1453
|
+
}
|
|
1203
1454
|
var HybridRetriever = class {
|
|
1204
1455
|
constructor(retrievers, options = {}) {
|
|
1205
1456
|
this.retrievers = retrievers;
|
|
@@ -1210,19 +1461,19 @@ var HybridRetriever = class {
|
|
|
1210
1461
|
async retrieve(question) {
|
|
1211
1462
|
const results = await Promise.all(this.retrievers.map((retriever) => retriever.retrieve(question)));
|
|
1212
1463
|
const merged = /* @__PURE__ */ new Map();
|
|
1213
|
-
for (const
|
|
1464
|
+
for (const evidenceList of results) {
|
|
1214
1465
|
for (const evidence of evidenceList) {
|
|
1215
1466
|
const existing = merged.get(evidence.id);
|
|
1216
|
-
const
|
|
1217
|
-
if (!existing ||
|
|
1467
|
+
const score = normalizeScore(evidence.score);
|
|
1468
|
+
if (!existing || score > existing.score) {
|
|
1218
1469
|
merged.set(evidence.id, {
|
|
1219
1470
|
...evidence,
|
|
1220
|
-
score
|
|
1471
|
+
score
|
|
1221
1472
|
});
|
|
1222
1473
|
}
|
|
1223
1474
|
}
|
|
1224
1475
|
}
|
|
1225
|
-
return [...merged.values()].sort((left, right) => right.score - left.score).slice(0, this.options.limit ?? 8);
|
|
1476
|
+
return [...merged.values()].sort((left, right) => right.score - left.score || evidenceTimestampMs(right) - evidenceTimestampMs(left)).slice(0, this.options.limit ?? 8);
|
|
1226
1477
|
}
|
|
1227
1478
|
};
|
|
1228
1479
|
|
|
@@ -1396,7 +1647,10 @@ function hasEmbeddingConfig(config, secrets) {
|
|
|
1396
1647
|
return Boolean((config.embedding.baseUrl || config.llm.baseUrl) && config.embedding.model && (secrets.embedding.apiKey || secrets.llm.apiKey));
|
|
1397
1648
|
}
|
|
1398
1649
|
async function createHybridRetriever(input) {
|
|
1399
|
-
const retrievers = [
|
|
1650
|
+
const retrievers = [
|
|
1651
|
+
new EpisodeFtsRetriever(new EpisodeRepository(input.database)),
|
|
1652
|
+
new MessageFtsRetriever(input.messages, { excludeMessageIds: input.excludeMessageIds })
|
|
1653
|
+
];
|
|
1400
1654
|
const closers = [];
|
|
1401
1655
|
if (hasEmbeddingConfig(input.config, input.secrets)) {
|
|
1402
1656
|
const vectorStore = new SqliteVectorStore(input.database, {
|
|
@@ -1870,6 +2124,40 @@ async function restoreLocalData(input) {
|
|
|
1870
2124
|
};
|
|
1871
2125
|
}
|
|
1872
2126
|
|
|
2127
|
+
// src/episodes/summarizer.ts
|
|
2128
|
+
async function summarizeEpisodeWindow(window, model) {
|
|
2129
|
+
const transcript = window.messages.map((message) => `[${message.sentAt}] ${message.senderName}\uFF1A${message.text}`).join("\n");
|
|
2130
|
+
const summary = await model.complete([
|
|
2131
|
+
{
|
|
2132
|
+
role: "system",
|
|
2133
|
+
content: "\u4F60\u662F ChatterCatcher \u7684\u4F1A\u8BDD\u8BB0\u5FC6\u6574\u7406\u6A21\u5757\u3002\u4F60\u7684\u4EFB\u52A1\u662F\u628A\u788E\u7247\u5316\u95F2\u804A\u6574\u7406\u6210\u53EF\u68C0\u7D22\u4E8B\u5B9E\uFF0C\u8865\u5168\u77ED\u6D88\u606F\u3001\u4EE3\u8BCD\u3001\u7F29\u5199\u4E0E\u4E0A\u4E0B\u6587\u4E4B\u95F4\u7684\u5173\u7CFB\u3002\u53EA\u603B\u7ED3\u660E\u786E\u4E8B\u5B9E\uFF0C\u4E0D\u8981\u7F16\u9020\u3002\u4FDD\u7559\u91CD\u8981\u6570\u5B57\u3001\u65E5\u671F\u3001\u94FE\u63A5\u548C\u4EE3\u7801\uFF1B\u5982\u679C\u5185\u5BB9\u50CF\u5BC6\u7801\u3001API key\u3001token \u6216\u5BC6\u94A5\uFF0C\u53EA\u63CF\u8FF0\u5176\u4E0A\u4E0B\u6587\u5173\u7CFB\uFF0C\u4E0D\u8981\u5728\u6458\u8981\u4E2D\u590D\u5199\u539F\u6587\u3002"
|
|
2134
|
+
},
|
|
2135
|
+
{
|
|
2136
|
+
role: "user",
|
|
2137
|
+
content: `\u7FA4\u804A\uFF1A${window.chatName}
|
|
2138
|
+
\u65F6\u95F4\uFF1A${window.startedAt} - ${window.endedAt}
|
|
2139
|
+
|
|
2140
|
+
\u804A\u5929\u8BB0\u5F55\uFF1A
|
|
2141
|
+
${transcript}
|
|
2142
|
+
|
|
2143
|
+
\u8BF7\u8F93\u51FA\u4E00\u6BB5\u7B80\u6D01\u7684\u4F1A\u8BDD\u8BB0\u5FC6\u6458\u8981\u3002`
|
|
2144
|
+
}
|
|
2145
|
+
]);
|
|
2146
|
+
return sanitizeEpisodeSummary(summary);
|
|
2147
|
+
}
|
|
2148
|
+
|
|
2149
|
+
// src/episodes/manual-process.ts
|
|
2150
|
+
async function processEpisodesNow(input) {
|
|
2151
|
+
const episodes = new EpisodeRepository(input.database);
|
|
2152
|
+
const created = await episodes.summarizeReadyWindows({
|
|
2153
|
+
now: input.now ?? /* @__PURE__ */ new Date(),
|
|
2154
|
+
quietMs: input.config.episodes.quietMinutes * 60 * 1e3,
|
|
2155
|
+
windowMs: input.config.episodes.windowMinutes * 60 * 1e3,
|
|
2156
|
+
summarize: (window) => summarizeEpisodeWindow(window, input.model)
|
|
2157
|
+
});
|
|
2158
|
+
return { created: created.length };
|
|
2159
|
+
}
|
|
2160
|
+
|
|
1873
2161
|
// src/feishu/gateway.ts
|
|
1874
2162
|
import * as lark2 from "@larksuiteoapi/node-sdk";
|
|
1875
2163
|
|
|
@@ -2267,6 +2555,18 @@ function createFeishuEventDispatcher(options) {
|
|
|
2267
2555
|
console.log("\u98DE\u4E66\u6D88\u606F\u91CD\u590D\u6295\u9012\uFF1A\u5DF2\u8DF3\u8FC7\u9644\u4EF6\u5904\u7406\u548C\u56DE\u7B54\u3002");
|
|
2268
2556
|
return;
|
|
2269
2557
|
}
|
|
2558
|
+
if (options.episodeProcessor) {
|
|
2559
|
+
const episodeResult = await processEpisodesNow({
|
|
2560
|
+
config: options.config,
|
|
2561
|
+
secrets: options.secrets,
|
|
2562
|
+
database: options.episodeProcessor.database,
|
|
2563
|
+
model: options.episodeProcessor.model,
|
|
2564
|
+
now: options.episodeProcessor.now?.()
|
|
2565
|
+
});
|
|
2566
|
+
if (episodeResult.created > 0) {
|
|
2567
|
+
console.log(`\u98DE\u4E66\u4F1A\u8BDD\u8BB0\u5FC6\u5DF2\u751F\u6210\uFF1A${episodeResult.created}`);
|
|
2568
|
+
}
|
|
2569
|
+
}
|
|
2270
2570
|
if (result.attachment?.downloaded) {
|
|
2271
2571
|
console.log(`\u98DE\u4E66\u9644\u4EF6\u5DF2\u4E0B\u8F7D\uFF1A${result.attachment.downloaded.storedPath}`);
|
|
2272
2572
|
if (result.attachment.indexedMessageId) {
|
|
@@ -2314,10 +2614,12 @@ function createFeishuGateway(options) {
|
|
|
2314
2614
|
});
|
|
2315
2615
|
const eventDispatcher = createFeishuEventDispatcher({
|
|
2316
2616
|
config: options.config,
|
|
2617
|
+
secrets: options.secrets,
|
|
2317
2618
|
ingestor: options.ingestor,
|
|
2318
2619
|
questionHandler: options.questionHandler,
|
|
2319
2620
|
resourceDownloader: options.resourceDownloader,
|
|
2320
|
-
attachmentVectorIndexer: options.attachmentVectorIndexer
|
|
2621
|
+
attachmentVectorIndexer: options.attachmentVectorIndexer,
|
|
2622
|
+
episodeProcessor: options.episodeProcessor
|
|
2321
2623
|
});
|
|
2322
2624
|
return {
|
|
2323
2625
|
async start() {
|
|
@@ -2545,7 +2847,7 @@ var FeishuResourceDownloader = class _FeishuResourceDownloader {
|
|
|
2545
2847
|
};
|
|
2546
2848
|
|
|
2547
2849
|
// src/files/ingest.ts
|
|
2548
|
-
import
|
|
2850
|
+
import crypto4 from "crypto";
|
|
2549
2851
|
import fs11 from "fs/promises";
|
|
2550
2852
|
import path13 from "path";
|
|
2551
2853
|
|
|
@@ -2609,7 +2911,7 @@ function ensureSupportedTextFile(filePath) {
|
|
|
2609
2911
|
}
|
|
2610
2912
|
}
|
|
2611
2913
|
function stableStoredName(sourcePath, fileName) {
|
|
2612
|
-
const digest =
|
|
2914
|
+
const digest = crypto4.createHash("sha256").update(sourcePath).digest("hex").slice(0, 16);
|
|
2613
2915
|
return `${digest}-${fileName}`;
|
|
2614
2916
|
}
|
|
2615
2917
|
async function ingestLocalFile(input) {
|
|
@@ -3314,6 +3616,7 @@ async function startWebServer(config) {
|
|
|
3314
3616
|
console.log(`ChatterCatcher Web UI: ${url}`);
|
|
3315
3617
|
}
|
|
3316
3618
|
export {
|
|
3619
|
+
EpisodeRepository,
|
|
3317
3620
|
FeishuMessageSender,
|
|
3318
3621
|
FeishuQuestionHandler,
|
|
3319
3622
|
FeishuResourceDownloader,
|
|
@@ -3374,6 +3677,7 @@ export {
|
|
|
3374
3677
|
normalizeLineCount,
|
|
3375
3678
|
openDatabase,
|
|
3376
3679
|
parseFileToText,
|
|
3680
|
+
processEpisodesNow,
|
|
3377
3681
|
processMessagesNow,
|
|
3378
3682
|
rankEvidenceForPrompt,
|
|
3379
3683
|
readGatewayPidRecord,
|
|
@@ -3385,10 +3689,12 @@ export {
|
|
|
3385
3689
|
resolveLogPath,
|
|
3386
3690
|
restoreLocalData,
|
|
3387
3691
|
runDoctor,
|
|
3692
|
+
sanitizeEpisodeSummary,
|
|
3388
3693
|
saveConfig,
|
|
3389
3694
|
saveSecrets,
|
|
3390
3695
|
startWebServer,
|
|
3391
3696
|
stopGatewayProcess,
|
|
3697
|
+
summarizeEpisodeWindow,
|
|
3392
3698
|
writeGatewayPidRecord
|
|
3393
3699
|
};
|
|
3394
3700
|
//# sourceMappingURL=index.js.map
|