clementine-agent 1.18.27 → 1.18.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/dashboard.js +97 -0
- package/dist/gateway/episodic-consolidation.d.ts +75 -0
- package/dist/gateway/episodic-consolidation.js +205 -0
- package/dist/gateway/heartbeat-scheduler.d.ts +10 -0
- package/dist/gateway/heartbeat-scheduler.js +45 -0
- package/dist/memory/store.d.ts +88 -1
- package/dist/memory/store.js +239 -1
- package/package.json +1 -1
package/dist/cli/dashboard.js
CHANGED
|
@@ -7044,6 +7044,35 @@ If the tool returns nothing or errors, return an empty array \`[]\`.`,
|
|
|
7044
7044
|
res.status(500).json({ error: String(err) });
|
|
7045
7045
|
}
|
|
7046
7046
|
});
|
|
7047
|
+
// Recent episodes — durable consolidated session summaries.
|
|
7048
|
+
app.get('/api/memory/episodes', async (req, res) => {
|
|
7049
|
+
try {
|
|
7050
|
+
const gateway = await getGateway();
|
|
7051
|
+
const store = gateway.assistant?.memoryStore;
|
|
7052
|
+
if (!store || typeof store.listRecentEpisodes !== 'function') {
|
|
7053
|
+
res.status(503).json({ error: 'Episodes store not available' });
|
|
7054
|
+
return;
|
|
7055
|
+
}
|
|
7056
|
+
const limit = Math.min(parseInt(String(req.query.limit ?? '30'), 10) || 30, 200);
|
|
7057
|
+
const sessionKey = req.query.session ? String(req.query.session) : undefined;
|
|
7058
|
+
const sinceParam = req.query.since ? String(req.query.since) : '';
|
|
7059
|
+
// since: '24h' | '7d' | '30d' | '' (all) | ISO string
|
|
7060
|
+
let sinceIso;
|
|
7061
|
+
if (sinceParam === '24h')
|
|
7062
|
+
sinceIso = new Date(Date.now() - 24 * 3600_000).toISOString();
|
|
7063
|
+
else if (sinceParam === '7d')
|
|
7064
|
+
sinceIso = new Date(Date.now() - 7 * 24 * 3600_000).toISOString();
|
|
7065
|
+
else if (sinceParam === '30d')
|
|
7066
|
+
sinceIso = new Date(Date.now() - 30 * 24 * 3600_000).toISOString();
|
|
7067
|
+
else if (sinceParam)
|
|
7068
|
+
sinceIso = sinceParam;
|
|
7069
|
+
const episodes = store.listRecentEpisodes({ limit, sessionKey, sinceIso });
|
|
7070
|
+
res.json({ ok: true, episodes });
|
|
7071
|
+
}
|
|
7072
|
+
catch (err) {
|
|
7073
|
+
res.status(500).json({ error: String(err) });
|
|
7074
|
+
}
|
|
7075
|
+
});
|
|
7047
7076
|
// Coverage + recall telemetry for both chunks and transcripts. Powers the
|
|
7048
7077
|
// Memory Coverage card showing whether dense recall is actually earning its
|
|
7049
7078
|
// keep on the current corpus.
|
|
@@ -14998,6 +15027,23 @@ if('serviceWorker' in navigator){navigator.serviceWorker.getRegistrations().then
|
|
|
14998
15027
|
<div class="skel-block" style="padding:14px"><div class="skel-row med"></div><div class="skel-row short"></div></div>
|
|
14999
15028
|
</div>
|
|
15000
15029
|
</div>
|
|
15030
|
+
<div class="card" style="margin-bottom:14px">
|
|
15031
|
+
<div class="card-header" style="display:flex;align-items:center;justify-content:space-between;gap:8px;flex-wrap:wrap">
|
|
15032
|
+
<span>Recent episodes</span>
|
|
15033
|
+
<div style="display:flex;align-items:center;gap:8px">
|
|
15034
|
+
<select id="episodes-filter-since" onchange="refreshRecentEpisodes()" style="font-size:12px;padding:4px 6px;border:1px solid var(--border);border-radius:4px;background:var(--bg-input);color:var(--text)">
|
|
15035
|
+
<option value="24h">Last 24h</option>
|
|
15036
|
+
<option value="7d" selected>Last 7d</option>
|
|
15037
|
+
<option value="30d">Last 30d</option>
|
|
15038
|
+
<option value="">All</option>
|
|
15039
|
+
</select>
|
|
15040
|
+
<span style="font-size:11px;color:var(--text-muted)">Consolidated session summaries</span>
|
|
15041
|
+
</div>
|
|
15042
|
+
</div>
|
|
15043
|
+
<div class="card-body" id="panel-recent-episodes" style="padding:0">
|
|
15044
|
+
<div class="skel-block" style="padding:14px"><div class="skel-row med"></div><div class="skel-row short"></div></div>
|
|
15045
|
+
</div>
|
|
15046
|
+
</div>
|
|
15001
15047
|
<div class="card">
|
|
15002
15048
|
<div class="card-header" style="display:flex;align-items:center;justify-content:space-between">
|
|
15003
15049
|
<span>Self-correction (supersedes)</span>
|
|
@@ -18508,6 +18554,7 @@ function switchTab(group, tab) {
|
|
|
18508
18554
|
// Consolidated Memory tab: search results + stats + MEMORY.md + recent writes + supersedes + coverage strip.
|
|
18509
18555
|
refreshMemory();
|
|
18510
18556
|
if (typeof refreshRecentWrites === 'function') refreshRecentWrites();
|
|
18557
|
+
if (typeof refreshRecentEpisodes === 'function') refreshRecentEpisodes();
|
|
18511
18558
|
if (typeof refreshSupersedes === 'function') refreshSupersedes();
|
|
18512
18559
|
if (typeof refreshCoverageStrip === 'function') refreshCoverageStrip();
|
|
18513
18560
|
}
|
|
@@ -24865,6 +24912,7 @@ async function submitQuickAddMemory() {
|
|
|
24865
24912
|
setTimeout(function() {
|
|
24866
24913
|
closeQuickAddMemory();
|
|
24867
24914
|
if (typeof refreshRecentWrites === 'function') refreshRecentWrites();
|
|
24915
|
+
if (typeof refreshRecentEpisodes === 'function') refreshRecentEpisodes();
|
|
24868
24916
|
if (typeof refreshMemory === 'function') refreshMemory();
|
|
24869
24917
|
}, 600);
|
|
24870
24918
|
} catch (err) {
|
|
@@ -25018,6 +25066,55 @@ async function refreshRecentWrites() {
|
|
|
25018
25066
|
}
|
|
25019
25067
|
}
|
|
25020
25068
|
|
|
25069
|
+
async function refreshRecentEpisodes() {
|
|
25070
|
+
var el = document.getElementById('panel-recent-episodes');
|
|
25071
|
+
if (!el) return;
|
|
25072
|
+
try {
|
|
25073
|
+
var sel = document.getElementById('episodes-filter-since');
|
|
25074
|
+
var since = sel ? sel.value : '7d';
|
|
25075
|
+
var url = '/api/memory/episodes?limit=30' + (since ? '&since=' + encodeURIComponent(since) : '');
|
|
25076
|
+
var r = await apiFetch(url);
|
|
25077
|
+
var d = await r.json();
|
|
25078
|
+
if (!d.ok || !Array.isArray(d.episodes)) {
|
|
25079
|
+
el.innerHTML = '<div class="empty-state" style="padding:14px">' + esc(d.error || 'No data') + '</div>';
|
|
25080
|
+
return;
|
|
25081
|
+
}
|
|
25082
|
+
if (d.episodes.length === 0) {
|
|
25083
|
+
el.innerHTML = '<div class="empty-state" style="padding:14px">No episodes yet. They land automatically when a session has been idle for ~20 min with at least 3 exchanges.</div>';
|
|
25084
|
+
return;
|
|
25085
|
+
}
|
|
25086
|
+
var html = '<table class="data-table" style="width:100%">';
|
|
25087
|
+
html += '<thead><tr>'
|
|
25088
|
+
+ '<th style="width:120px">When</th>'
|
|
25089
|
+
+ '<th style="width:160px">Session</th>'
|
|
25090
|
+
+ '<th>Summary</th>'
|
|
25091
|
+
+ '<th style="width:140px">Topics</th>'
|
|
25092
|
+
+ '<th style="width:120px">Outcome</th>'
|
|
25093
|
+
+ '<th style="width:50px;text-align:right">Open</th>'
|
|
25094
|
+
+ '</tr></thead><tbody>';
|
|
25095
|
+
for (var i = 0; i < d.episodes.length; i++) {
|
|
25096
|
+
var ep = d.episodes[i];
|
|
25097
|
+
var when = '';
|
|
25098
|
+
try { when = new Date(ep.createdAt + 'Z').toLocaleString(); } catch { when = ep.createdAt; }
|
|
25099
|
+
var topics = (ep.topics || []).slice(0, 3).map(esc).join(', ');
|
|
25100
|
+
var openCount = (ep.openLoops || []).length;
|
|
25101
|
+
var openColor = openCount > 0 ? '#f59e0b' : 'var(--text-muted)';
|
|
25102
|
+
html += '<tr>'
|
|
25103
|
+
+ '<td style="font-size:11px;color:var(--text-muted)">' + esc(when) + '</td>'
|
|
25104
|
+
+ '<td style="font-size:11px">' + esc(ep.sessionKey) + '</td>'
|
|
25105
|
+
+ '<td style="font-size:12px">' + esc(ep.summary) + '</td>'
|
|
25106
|
+
+ '<td style="font-size:11px;color:var(--text-muted)">' + (topics || '—') + '</td>'
|
|
25107
|
+
+ '<td style="font-size:11px">' + esc(ep.outcome || '—') + '</td>'
|
|
25108
|
+
+ '<td style="text-align:right;font-weight:600;color:' + openColor + '">' + openCount + '</td>'
|
|
25109
|
+
+ '</tr>';
|
|
25110
|
+
}
|
|
25111
|
+
html += '</tbody></table>';
|
|
25112
|
+
el.innerHTML = html;
|
|
25113
|
+
} catch (err) {
|
|
25114
|
+
el.innerHTML = '<div class="empty-state" style="padding:14px">Failed to load: ' + esc(String(err)) + '</div>';
|
|
25115
|
+
}
|
|
25116
|
+
}
|
|
25117
|
+
|
|
25021
25118
|
async function memoryHealthAction(action, extra) {
|
|
25022
25119
|
var labels = { 'janitor': 'cleanup', 'rebuild-fts': 'FTS rebuild', 'fix-orphans': 'orphan fix', 'install-dense-model': 'local embedding model install/verify', 'reembed-dense': 'dense embedding backfill' };
|
|
25023
25120
|
if (!confirm('Run ' + (labels[action] || action) + ' now?')) return;
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Episodic consolidation — turn raw transcript ranges into durable, indexed
|
|
3
|
+
* episodes that hybrid recall can surface across sessions.
|
|
4
|
+
*
|
|
5
|
+
* Why not just keep transcripts? Transcripts are noisy and minute-grained.
|
|
6
|
+
* "What did we decide about auth?" should match a clean summary of the
|
|
7
|
+
* decision, not the eight messages where we worked toward it. Episodes
|
|
8
|
+
* compress a session range into {summary, topics, entities, outcome,
|
|
9
|
+
* openLoops}, persist that to the episodes table, and also write the
|
|
10
|
+
* summary into chunks so PR-1's hybrid recall picks them up automatically.
|
|
11
|
+
*
|
|
12
|
+
* The pass is driven by the heartbeat: every few minutes we look for
|
|
13
|
+
* sessions that have been idle for ≥20 min with ≥3 new exchanges and
|
|
14
|
+
* consolidate up to a small bounded number per pass to keep LLM cost
|
|
15
|
+
* predictable.
|
|
16
|
+
*/
|
|
17
|
+
import Anthropic from '@anthropic-ai/sdk';
|
|
18
|
+
import type { MemoryStore } from '../memory/store.js';
|
|
19
|
+
export interface EpisodicConsolidationOptions {
|
|
20
|
+
/** Minutes of inactivity before a session becomes consolidation-eligible. */
|
|
21
|
+
idleMinutes?: number;
|
|
22
|
+
/** Minimum turns since last cursor for a session to qualify. */
|
|
23
|
+
minExchanges?: number;
|
|
24
|
+
/** Cap LLM calls per pass to bound cost. */
|
|
25
|
+
maxSessionsPerPass?: number;
|
|
26
|
+
/** How long to back off after a consolidation failure for a session. */
|
|
27
|
+
failBackoffMinutes?: number;
|
|
28
|
+
/** Override Anthropic client (used by tests). */
|
|
29
|
+
anthropicClient?: Pick<Anthropic, 'messages'>;
|
|
30
|
+
/** Override the model id (used by tests). */
|
|
31
|
+
model?: string;
|
|
32
|
+
/** Wallclock now() — used by tests for deterministic timestamps. */
|
|
33
|
+
now?: () => Date;
|
|
34
|
+
}
|
|
35
|
+
export interface EpisodeExtraction {
|
|
36
|
+
summary: string;
|
|
37
|
+
topics: string[];
|
|
38
|
+
entities: string[];
|
|
39
|
+
outcome: string;
|
|
40
|
+
openLoops: string[];
|
|
41
|
+
}
|
|
42
|
+
interface CandidateRow {
|
|
43
|
+
sessionKey: string;
|
|
44
|
+
startTranscriptId: number;
|
|
45
|
+
endTranscriptId: number;
|
|
46
|
+
startedAt: string;
|
|
47
|
+
endedAt: string;
|
|
48
|
+
exchanges: number;
|
|
49
|
+
}
|
|
50
|
+
export interface ConsolidationPassResult {
|
|
51
|
+
consolidated: number;
|
|
52
|
+
failed: number;
|
|
53
|
+
skipped: number;
|
|
54
|
+
candidates: number;
|
|
55
|
+
}
|
|
56
|
+
/** Parse the model's output as JSON, tolerating leading/trailing whitespace and
|
|
57
|
+
* occasional code fences. Returns null on any structural problem. */
|
|
58
|
+
export declare function parseEpisodeJson(raw: string): EpisodeExtraction | null;
|
|
59
|
+
/**
|
|
60
|
+
* Consolidate a single candidate session range. Returns the new episode id
|
|
61
|
+
* + chunk id on success, or null on failure (the caller bumps the failure
|
|
62
|
+
* cursor so we don't retry every tick).
|
|
63
|
+
*/
|
|
64
|
+
export declare function consolidateOneSession(store: MemoryStore, candidate: CandidateRow, opts?: EpisodicConsolidationOptions): Promise<{
|
|
65
|
+
episodeId: number;
|
|
66
|
+
chunkId: number | null;
|
|
67
|
+
} | null>;
|
|
68
|
+
/**
|
|
69
|
+
* Run one bounded consolidation pass. Designed to be called from the
|
|
70
|
+
* heartbeat tick — quick to no-op when nothing's eligible, capped at
|
|
71
|
+
* `maxSessionsPerPass` LLM calls when work exists.
|
|
72
|
+
*/
|
|
73
|
+
export declare function runEpisodicConsolidationPass(store: MemoryStore, opts?: EpisodicConsolidationOptions): Promise<ConsolidationPassResult>;
|
|
74
|
+
export {};
|
|
75
|
+
//# sourceMappingURL=episodic-consolidation.d.ts.map
|
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Episodic consolidation — turn raw transcript ranges into durable, indexed
|
|
3
|
+
* episodes that hybrid recall can surface across sessions.
|
|
4
|
+
*
|
|
5
|
+
* Why not just keep transcripts? Transcripts are noisy and minute-grained.
|
|
6
|
+
* "What did we decide about auth?" should match a clean summary of the
|
|
7
|
+
* decision, not the eight messages where we worked toward it. Episodes
|
|
8
|
+
* compress a session range into {summary, topics, entities, outcome,
|
|
9
|
+
* openLoops}, persist that to the episodes table, and also write the
|
|
10
|
+
* summary into chunks so PR-1's hybrid recall picks them up automatically.
|
|
11
|
+
*
|
|
12
|
+
* The pass is driven by the heartbeat: every few minutes we look for
|
|
13
|
+
* sessions that have been idle for ≥20 min with ≥3 new exchanges and
|
|
14
|
+
* consolidate up to a small bounded number per pass to keep LLM cost
|
|
15
|
+
* predictable.
|
|
16
|
+
*/
|
|
17
|
+
import Anthropic from '@anthropic-ai/sdk';
|
|
18
|
+
import pino from 'pino';
|
|
19
|
+
import { MODELS } from '../config.js';
|
|
20
|
+
const logger = pino({
|
|
21
|
+
name: 'clementine.episodic-consolidation',
|
|
22
|
+
level: process.env.CLEMENTINE_CONSOLIDATION_LOG_LEVEL || 'warn',
|
|
23
|
+
});
|
|
24
|
+
const SYSTEM_PROMPT = [
|
|
25
|
+
'You are a memory consolidator for a personal AI assistant.',
|
|
26
|
+
'You read a transcript range and produce a compact, durable record of what happened.',
|
|
27
|
+
'Output STRICT JSON matching the schema, with no prose, no markdown, no code fences.',
|
|
28
|
+
'Schema:',
|
|
29
|
+
'{',
|
|
30
|
+
' "summary": string (2-4 sentences, neutral, factual),',
|
|
31
|
+
' "topics": string[] (lowercase noun phrases, max 6),',
|
|
32
|
+
' "entities": string[] (named things: files, services, people; max 8),',
|
|
33
|
+
' "outcome": string (one short clause: decided / implemented / discussed / blocked / none),',
|
|
34
|
+
' "openLoops": string[] (unresolved follow-ups; empty array if none, max 5)',
|
|
35
|
+
'}',
|
|
36
|
+
].join('\n');
|
|
37
|
+
function buildUserPrompt(turns) {
|
|
38
|
+
const formatted = turns
|
|
39
|
+
.map(t => `[${t.createdAt}] ${t.role}: ${t.content.replace(/\s+/g, ' ').slice(0, 1200)}`)
|
|
40
|
+
.join('\n');
|
|
41
|
+
return [
|
|
42
|
+
'Consolidate the following conversation range into the JSON schema described.',
|
|
43
|
+
'Only include facts present in the conversation. Use empty arrays for unknown fields.',
|
|
44
|
+
'',
|
|
45
|
+
formatted,
|
|
46
|
+
].join('\n');
|
|
47
|
+
}
|
|
48
|
+
/** Parse the model's output as JSON, tolerating leading/trailing whitespace and
|
|
49
|
+
* occasional code fences. Returns null on any structural problem. */
|
|
50
|
+
export function parseEpisodeJson(raw) {
|
|
51
|
+
if (!raw)
|
|
52
|
+
return null;
|
|
53
|
+
let text = raw.trim();
|
|
54
|
+
if (text.startsWith('```')) {
|
|
55
|
+
// Strip fence; keep everything between first and last triple-backtick.
|
|
56
|
+
const m = text.match(/```(?:json)?\s*([\s\S]*?)\s*```/);
|
|
57
|
+
if (m)
|
|
58
|
+
text = m[1];
|
|
59
|
+
}
|
|
60
|
+
let parsed;
|
|
61
|
+
try {
|
|
62
|
+
parsed = JSON.parse(text);
|
|
63
|
+
}
|
|
64
|
+
catch {
|
|
65
|
+
return null;
|
|
66
|
+
}
|
|
67
|
+
if (!parsed || typeof parsed !== 'object')
|
|
68
|
+
return null;
|
|
69
|
+
const obj = parsed;
|
|
70
|
+
const arr = (v) => Array.isArray(v) ? v.filter(x => typeof x === 'string').map(s => s.trim()).filter(Boolean) : [];
|
|
71
|
+
const summary = typeof obj.summary === 'string' ? obj.summary.trim() : '';
|
|
72
|
+
if (!summary)
|
|
73
|
+
return null;
|
|
74
|
+
return {
|
|
75
|
+
summary,
|
|
76
|
+
topics: arr(obj.topics).slice(0, 6),
|
|
77
|
+
entities: arr(obj.entities).slice(0, 8),
|
|
78
|
+
outcome: typeof obj.outcome === 'string' ? obj.outcome.trim().slice(0, 200) : '',
|
|
79
|
+
openLoops: arr(obj.openLoops).slice(0, 5),
|
|
80
|
+
};
|
|
81
|
+
}
|
|
82
|
+
function getAnthropicClient(opts) {
|
|
83
|
+
if (opts.anthropicClient)
|
|
84
|
+
return opts.anthropicClient;
|
|
85
|
+
const apiKey = process.env.ANTHROPIC_API_KEY;
|
|
86
|
+
if (!apiKey)
|
|
87
|
+
return null;
|
|
88
|
+
return new Anthropic({ apiKey });
|
|
89
|
+
}
|
|
90
|
+
/**
|
|
91
|
+
* Consolidate a single candidate session range. Returns the new episode id
|
|
92
|
+
* + chunk id on success, or null on failure (the caller bumps the failure
|
|
93
|
+
* cursor so we don't retry every tick).
|
|
94
|
+
*/
|
|
95
|
+
export async function consolidateOneSession(store, candidate, opts = {}) {
|
|
96
|
+
const turns = store.getTranscriptsByIdRange(candidate.sessionKey, candidate.startTranscriptId, candidate.endTranscriptId);
|
|
97
|
+
if (turns.length === 0)
|
|
98
|
+
return null;
|
|
99
|
+
const client = getAnthropicClient(opts);
|
|
100
|
+
if (!client) {
|
|
101
|
+
logger.debug({ sessionKey: candidate.sessionKey }, 'No Anthropic client available — skipping consolidation');
|
|
102
|
+
return null;
|
|
103
|
+
}
|
|
104
|
+
let extraction = null;
|
|
105
|
+
try {
|
|
106
|
+
const response = await client.messages.create({
|
|
107
|
+
model: opts.model ?? MODELS.haiku,
|
|
108
|
+
max_tokens: 1024,
|
|
109
|
+
system: SYSTEM_PROMPT,
|
|
110
|
+
messages: [{ role: 'user', content: buildUserPrompt(turns.map(t => ({ role: t.role, content: t.content, createdAt: t.createdAt }))) }],
|
|
111
|
+
});
|
|
112
|
+
const text = (response.content ?? []).map((b) => b.type === 'text' ? (b.text ?? '') : '').join('');
|
|
113
|
+
extraction = parseEpisodeJson(text);
|
|
114
|
+
}
|
|
115
|
+
catch (err) {
|
|
116
|
+
logger.warn({ err, sessionKey: candidate.sessionKey }, 'Episode LLM call failed');
|
|
117
|
+
return null;
|
|
118
|
+
}
|
|
119
|
+
if (!extraction) {
|
|
120
|
+
logger.warn({ sessionKey: candidate.sessionKey }, 'Episode JSON parse failed — skipping');
|
|
121
|
+
return null;
|
|
122
|
+
}
|
|
123
|
+
// Index the summary into chunks so hybrid recall surfaces it. The
|
|
124
|
+
// source_file shape mirrors how internal-derived chunks are stored
|
|
125
|
+
// elsewhere; section is the session key for traceability.
|
|
126
|
+
let chunkId = null;
|
|
127
|
+
try {
|
|
128
|
+
chunkId = store.insertSummaryChunk(`episodes/${candidate.sessionKey}.md`, `Episode ${candidate.startedAt}`, [
|
|
129
|
+
extraction.summary,
|
|
130
|
+
extraction.topics.length ? `Topics: ${extraction.topics.join(', ')}` : '',
|
|
131
|
+
extraction.entities.length ? `Entities: ${extraction.entities.join(', ')}` : '',
|
|
132
|
+
extraction.outcome ? `Outcome: ${extraction.outcome}` : '',
|
|
133
|
+
extraction.openLoops.length ? `Open: ${extraction.openLoops.join('; ')}` : '',
|
|
134
|
+
].filter(Boolean).join('\n'));
|
|
135
|
+
}
|
|
136
|
+
catch (err) {
|
|
137
|
+
logger.debug({ err }, 'insertSummaryChunk failed — episode still persisted');
|
|
138
|
+
}
|
|
139
|
+
const transcriptIds = turns.map(t => t.id ?? 0).filter(n => n > 0);
|
|
140
|
+
const insert = store.insertEpisode({
|
|
141
|
+
sessionKey: candidate.sessionKey,
|
|
142
|
+
startedAt: candidate.startedAt,
|
|
143
|
+
endedAt: candidate.endedAt,
|
|
144
|
+
summary: extraction.summary,
|
|
145
|
+
topics: extraction.topics,
|
|
146
|
+
entities: extraction.entities,
|
|
147
|
+
outcome: extraction.outcome,
|
|
148
|
+
openLoops: extraction.openLoops,
|
|
149
|
+
transcriptIds,
|
|
150
|
+
chunkId,
|
|
151
|
+
});
|
|
152
|
+
store.updateConsolidationCursor(candidate.sessionKey, {
|
|
153
|
+
lastTranscriptId: candidate.endTranscriptId,
|
|
154
|
+
success: true,
|
|
155
|
+
});
|
|
156
|
+
logger.info({
|
|
157
|
+
sessionKey: candidate.sessionKey,
|
|
158
|
+
episodeId: insert.episodeId,
|
|
159
|
+
chunkId,
|
|
160
|
+
turns: turns.length,
|
|
161
|
+
}, 'Consolidated episode');
|
|
162
|
+
return { episodeId: insert.episodeId, chunkId };
|
|
163
|
+
}
|
|
164
|
+
/**
|
|
165
|
+
* Run one bounded consolidation pass. Designed to be called from the
|
|
166
|
+
* heartbeat tick — quick to no-op when nothing's eligible, capped at
|
|
167
|
+
* `maxSessionsPerPass` LLM calls when work exists.
|
|
168
|
+
*/
|
|
169
|
+
export async function runEpisodicConsolidationPass(store, opts = {}) {
|
|
170
|
+
const idleMinutes = opts.idleMinutes ?? 20;
|
|
171
|
+
const minExchanges = opts.minExchanges ?? 3;
|
|
172
|
+
const maxSessions = Math.max(1, opts.maxSessionsPerPass ?? 3);
|
|
173
|
+
const failBackoffMinutes = opts.failBackoffMinutes ?? 60;
|
|
174
|
+
const candidates = store.getIdleSessionsForEpisodicConsolidation({
|
|
175
|
+
idleMinutes,
|
|
176
|
+
minExchanges,
|
|
177
|
+
maxResults: maxSessions,
|
|
178
|
+
failBackoffMinutes,
|
|
179
|
+
});
|
|
180
|
+
let consolidated = 0;
|
|
181
|
+
let failed = 0;
|
|
182
|
+
let skipped = 0;
|
|
183
|
+
for (const candidate of candidates) {
|
|
184
|
+
try {
|
|
185
|
+
const result = await consolidateOneSession(store, candidate, opts);
|
|
186
|
+
if (result) {
|
|
187
|
+
consolidated++;
|
|
188
|
+
}
|
|
189
|
+
else {
|
|
190
|
+
store.updateConsolidationCursor(candidate.sessionKey, { success: false });
|
|
191
|
+
failed++;
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
catch (err) {
|
|
195
|
+
logger.warn({ err, sessionKey: candidate.sessionKey }, 'Consolidation pass error');
|
|
196
|
+
try {
|
|
197
|
+
store.updateConsolidationCursor(candidate.sessionKey, { success: false });
|
|
198
|
+
}
|
|
199
|
+
catch { /* ignore */ }
|
|
200
|
+
failed++;
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
return { consolidated, failed, skipped, candidates: candidates.length };
|
|
204
|
+
}
|
|
205
|
+
//# sourceMappingURL=episodic-consolidation.js.map
|
|
@@ -44,6 +44,8 @@ export declare class HeartbeatScheduler {
|
|
|
44
44
|
private denseBackfillInFlight;
|
|
45
45
|
private lastSalienceDecayDate;
|
|
46
46
|
private lastMemoryPulseDate;
|
|
47
|
+
private lastEpisodicConsolidationAt;
|
|
48
|
+
private episodicConsolidationInFlight;
|
|
47
49
|
/** Wire up the cron scheduler so daily plan suggestions can be applied. */
|
|
48
50
|
setCronScheduler(cs: CronScheduler): void;
|
|
49
51
|
private getLastAgentSiRun;
|
|
@@ -68,6 +70,14 @@ export declare class HeartbeatScheduler {
|
|
|
68
70
|
* Coverage climbs over hours/days without user action.
|
|
69
71
|
*/
|
|
70
72
|
private maybeIdleDenseBackfill;
|
|
73
|
+
/**
|
|
74
|
+
* Episodic consolidation pass. Turns idle session transcript ranges into
|
|
75
|
+
* durable episodes via a small Haiku call per session. Same shape as
|
|
76
|
+
* maybeIdleDenseBackfill: in-flight guard, cooldown, chat-lane busy check,
|
|
77
|
+
* bounded work per pass. Skipped silently when there's nothing eligible
|
|
78
|
+
* (which is the common case).
|
|
79
|
+
*/
|
|
80
|
+
private maybeRunEpisodicConsolidation;
|
|
71
81
|
/**
|
|
72
82
|
* Daily salience decay. Multiplies salience by 0.95 on chunks unaccessed
|
|
73
83
|
* for >30 days. Date-gated (one pass per calendar day), persisted in
|
|
@@ -54,6 +54,8 @@ export class HeartbeatScheduler {
|
|
|
54
54
|
denseBackfillInFlight = false;
|
|
55
55
|
lastSalienceDecayDate = '';
|
|
56
56
|
lastMemoryPulseDate = '';
|
|
57
|
+
lastEpisodicConsolidationAt = 0;
|
|
58
|
+
episodicConsolidationInFlight = false;
|
|
57
59
|
/** Wire up the cron scheduler so daily plan suggestions can be applied. */
|
|
58
60
|
setCronScheduler(cs) { this.cronScheduler = cs; }
|
|
59
61
|
getLastAgentSiRun(slug) {
|
|
@@ -158,6 +160,12 @@ export class HeartbeatScheduler {
|
|
|
158
160
|
// Pinned + soft-deleted + superseded chunks are exempt. One UPDATE per
|
|
159
161
|
// day, gated by a date stamp on HeartbeatState.
|
|
160
162
|
this.maybeRunSalienceDecay();
|
|
163
|
+
// Episodic consolidation — turn idle sessions' raw transcripts into
|
|
164
|
+
// durable, indexed episodes. ~5 min cooldown, capped at 3 sessions per
|
|
165
|
+
// pass to bound LLM cost. Best-effort; never blocks the tick.
|
|
166
|
+
this.maybeRunEpisodicConsolidation().catch(err => {
|
|
167
|
+
logger.debug({ err }, 'Episodic consolidation pass failed (non-fatal)');
|
|
168
|
+
});
|
|
161
169
|
// Claim verification sweep — auto-verify pending claims whose due
|
|
162
170
|
// times have passed (e.g. "I scheduled X for 8am" → check at 9am).
|
|
163
171
|
import('./claim-tracker.js').then(async ({ verifyDueClaims, drainLLMFallback }) => {
|
|
@@ -810,6 +818,43 @@ export class HeartbeatScheduler {
|
|
|
810
818
|
this.denseBackfillInFlight = false;
|
|
811
819
|
}
|
|
812
820
|
}
|
|
821
|
+
/**
|
|
822
|
+
* Episodic consolidation pass. Turns idle session transcript ranges into
|
|
823
|
+
* durable episodes via a small Haiku call per session. Same shape as
|
|
824
|
+
* maybeIdleDenseBackfill: in-flight guard, cooldown, chat-lane busy check,
|
|
825
|
+
* bounded work per pass. Skipped silently when there's nothing eligible
|
|
826
|
+
* (which is the common case).
|
|
827
|
+
*/
|
|
828
|
+
async maybeRunEpisodicConsolidation() {
|
|
829
|
+
if (this.episodicConsolidationInFlight)
|
|
830
|
+
return;
|
|
831
|
+
const sinceLastMs = Date.now() - this.lastEpisodicConsolidationAt;
|
|
832
|
+
if (sinceLastMs < 5 * 60 * 1000)
|
|
833
|
+
return;
|
|
834
|
+
const { lanes } = await import('./lanes.js');
|
|
835
|
+
if (lanes.status().chat.active > 0)
|
|
836
|
+
return;
|
|
837
|
+
const store = this.gateway.getMemoryStore();
|
|
838
|
+
if (!store)
|
|
839
|
+
return;
|
|
840
|
+
this.episodicConsolidationInFlight = true;
|
|
841
|
+
this.lastEpisodicConsolidationAt = Date.now();
|
|
842
|
+
try {
|
|
843
|
+
const { runEpisodicConsolidationPass } = await import('./episodic-consolidation.js');
|
|
844
|
+
const result = await runEpisodicConsolidationPass(store, {
|
|
845
|
+
idleMinutes: 20,
|
|
846
|
+
minExchanges: 3,
|
|
847
|
+
maxSessionsPerPass: 3,
|
|
848
|
+
failBackoffMinutes: 60,
|
|
849
|
+
});
|
|
850
|
+
if (result.consolidated > 0 || result.failed > 0) {
|
|
851
|
+
logger.info(result, 'Episodic consolidation pass complete');
|
|
852
|
+
}
|
|
853
|
+
}
|
|
854
|
+
finally {
|
|
855
|
+
this.episodicConsolidationInFlight = false;
|
|
856
|
+
}
|
|
857
|
+
}
|
|
813
858
|
/**
|
|
814
859
|
* Daily salience decay. Multiplies salience by 0.95 on chunks unaccessed
|
|
815
860
|
* for >30 days. Date-gated (one pass per calendar day), persisted in
|
package/dist/memory/store.d.ts
CHANGED
|
@@ -622,6 +622,93 @@ export declare class MemoryStore {
|
|
|
622
622
|
bothModes: number;
|
|
623
623
|
avgTopScore: number;
|
|
624
624
|
};
|
|
625
|
+
/**
|
|
626
|
+
* Find sessions whose latest turn is older than `idleMinutes` minutes,
|
|
627
|
+
* have at least `minExchanges` user/assistant turns combined since the
|
|
628
|
+
* last consolidation cursor, and aren't already up-to-date. Returns one
|
|
629
|
+
* row per session ranked oldest-idle first so we consolidate the
|
|
630
|
+
* least-fresh first when bounded by maxResults.
|
|
631
|
+
*/
|
|
632
|
+
getIdleSessionsForEpisodicConsolidation(opts: {
|
|
633
|
+
idleMinutes: number;
|
|
634
|
+
minExchanges: number;
|
|
635
|
+
maxResults: number;
|
|
636
|
+
failBackoffMinutes?: number;
|
|
637
|
+
}): Array<{
|
|
638
|
+
sessionKey: string;
|
|
639
|
+
startTranscriptId: number;
|
|
640
|
+
endTranscriptId: number;
|
|
641
|
+
startedAt: string;
|
|
642
|
+
endedAt: string;
|
|
643
|
+
exchanges: number;
|
|
644
|
+
}>;
|
|
645
|
+
/**
|
|
646
|
+
* Persist a consolidated episode and bump the per-session cursor so the
|
|
647
|
+
* same range isn't re-consolidated on the next pass. The summary text is
|
|
648
|
+
* also indexed into chunks (returned as chunkId) so hybrid recall surfaces
|
|
649
|
+
* episodes alongside raw transcripts.
|
|
650
|
+
*/
|
|
651
|
+
insertEpisode(entry: {
|
|
652
|
+
sessionKey: string;
|
|
653
|
+
startedAt: string;
|
|
654
|
+
endedAt: string;
|
|
655
|
+
summary: string;
|
|
656
|
+
topics: string[];
|
|
657
|
+
entities: string[];
|
|
658
|
+
outcome: string;
|
|
659
|
+
openLoops: string[];
|
|
660
|
+
transcriptIds: number[];
|
|
661
|
+
chunkId?: number | null;
|
|
662
|
+
}): {
|
|
663
|
+
episodeId: number;
|
|
664
|
+
chunkId: number | null;
|
|
665
|
+
};
|
|
666
|
+
/**
|
|
667
|
+
* Mark a consolidation pass result on the per-session cursor. On success
|
|
668
|
+
* we advance last_transcript_id and reset fail_count; on failure we bump
|
|
669
|
+
* fail_count + last_attempted_at so the backoff-aware idle scan skips
|
|
670
|
+
* this session for a while.
|
|
671
|
+
*/
|
|
672
|
+
updateConsolidationCursor(sessionKey: string, update: {
|
|
673
|
+
lastTranscriptId?: number;
|
|
674
|
+
success: boolean;
|
|
675
|
+
}): void;
|
|
676
|
+
/** Read the consolidation cursor for a session — used in tests and for diagnostics. */
|
|
677
|
+
getConsolidationCursor(sessionKey: string): {
|
|
678
|
+
sessionKey: string;
|
|
679
|
+
lastTranscriptId: number;
|
|
680
|
+
lastAttemptedAt: string | null;
|
|
681
|
+
lastSuccessAt: string | null;
|
|
682
|
+
failCount: number;
|
|
683
|
+
} | null;
|
|
684
|
+
/**
|
|
685
|
+
* List recent episodes for the dashboard. JSON columns are parsed back
|
|
686
|
+
* into arrays so callers don't have to.
|
|
687
|
+
*/
|
|
688
|
+
listRecentEpisodes(opts?: {
|
|
689
|
+
limit?: number;
|
|
690
|
+
sessionKey?: string;
|
|
691
|
+
sinceIso?: string;
|
|
692
|
+
}): Array<{
|
|
693
|
+
id: number;
|
|
694
|
+
sessionKey: string;
|
|
695
|
+
startedAt: string;
|
|
696
|
+
endedAt: string;
|
|
697
|
+
summary: string;
|
|
698
|
+
topics: string[];
|
|
699
|
+
entities: string[];
|
|
700
|
+
outcome: string;
|
|
701
|
+
openLoops: string[];
|
|
702
|
+
transcriptIds: number[];
|
|
703
|
+
chunkId: number | null;
|
|
704
|
+
createdAt: string;
|
|
705
|
+
}>;
|
|
706
|
+
/**
|
|
707
|
+
* Fetch a slice of transcripts by id range for consolidation. Used by
|
|
708
|
+
* the consolidation module to materialize the conversation it's about
|
|
709
|
+
* to summarize.
|
|
710
|
+
*/
|
|
711
|
+
getTranscriptsByIdRange(sessionKey: string, startId: number, endId: number): TranscriptTurn[];
|
|
625
712
|
/**
|
|
626
713
|
* Save a session summary for cross-session context.
|
|
627
714
|
*/
|
|
@@ -1542,7 +1629,7 @@ export declare class MemoryStore {
|
|
|
1542
1629
|
* Stored as JSON in `chunks.derived_from` so the dashboard can show
|
|
1543
1630
|
* "view source memories" — abstractions become auditable.
|
|
1544
1631
|
*/
|
|
1545
|
-
insertSummaryChunk(sourceFile: string, section: string, content: string, derivedFromIds?: number[]):
|
|
1632
|
+
insertSummaryChunk(sourceFile: string, section: string, content: string, derivedFromIds?: number[]): number;
|
|
1546
1633
|
upsertLead(lead: {
|
|
1547
1634
|
agentSlug: string;
|
|
1548
1635
|
email: string;
|
package/dist/memory/store.js
CHANGED
|
@@ -948,6 +948,42 @@ export class MemoryStore {
|
|
|
948
948
|
created_at TEXT DEFAULT (datetime('now'))
|
|
949
949
|
);
|
|
950
950
|
CREATE INDEX IF NOT EXISTS idx_recall_telemetry_created ON recall_telemetry(created_at DESC);
|
|
951
|
+
`);
|
|
952
|
+
// Episodes — durable, retrievable summaries of past sessions. Each
|
|
953
|
+
// episode is one chunked range of transcripts; the LLM extracts
|
|
954
|
+
// {summary, topics, entities, outcome, openLoops}. The summary text is
|
|
955
|
+
// also written into chunks so hybrid recall picks it up. transcript_ids
|
|
956
|
+
// is a JSON array; we don't normalize because the lineage is read-only.
|
|
957
|
+
this.conn.exec(`
|
|
958
|
+
CREATE TABLE IF NOT EXISTS episodes (
|
|
959
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
960
|
+
session_key TEXT NOT NULL,
|
|
961
|
+
started_at TEXT NOT NULL,
|
|
962
|
+
ended_at TEXT NOT NULL,
|
|
963
|
+
summary TEXT NOT NULL,
|
|
964
|
+
topics TEXT,
|
|
965
|
+
entities TEXT,
|
|
966
|
+
outcome TEXT,
|
|
967
|
+
open_loops TEXT,
|
|
968
|
+
transcript_ids TEXT,
|
|
969
|
+
chunk_id INTEGER,
|
|
970
|
+
created_at TEXT DEFAULT (datetime('now'))
|
|
971
|
+
);
|
|
972
|
+
CREATE INDEX IF NOT EXISTS idx_episodes_session ON episodes(session_key, started_at DESC);
|
|
973
|
+
CREATE INDEX IF NOT EXISTS idx_episodes_created ON episodes(created_at DESC);
|
|
974
|
+
`);
|
|
975
|
+
// Per-session consolidation cursor — tracks how far the LLM has
|
|
976
|
+
// summarized so we don't re-consolidate the same turns. Failure tracking
|
|
977
|
+
// (fail_count + last_attempted_at) lets us back off cleanly when the
|
|
978
|
+
// model rejects a session repeatedly without spamming retries.
|
|
979
|
+
this.conn.exec(`
|
|
980
|
+
CREATE TABLE IF NOT EXISTS consolidation_cursors (
|
|
981
|
+
session_key TEXT PRIMARY KEY,
|
|
982
|
+
last_transcript_id INTEGER NOT NULL DEFAULT 0,
|
|
983
|
+
last_attempted_at TEXT,
|
|
984
|
+
last_success_at TEXT,
|
|
985
|
+
fail_count INTEGER NOT NULL DEFAULT 0
|
|
986
|
+
);
|
|
951
987
|
`);
|
|
952
988
|
// Soft-delete via a separate table — keeps the chunks_au trigger
|
|
953
989
|
// out of the path so we don't have to fight with the FTS5 contentless
|
|
@@ -2940,6 +2976,206 @@ export class MemoryStore {
|
|
|
2940
2976
|
return { total: 0, semanticOnly: 0, lexicalOnly: 0, bothModes: 0, avgTopScore: 0 };
|
|
2941
2977
|
}
|
|
2942
2978
|
}
|
|
2979
|
+
// ── Episodes (durable session summaries) ──────────────────────────
|
|
2980
|
+
/**
|
|
2981
|
+
* Find sessions whose latest turn is older than `idleMinutes` minutes,
|
|
2982
|
+
* have at least `minExchanges` user/assistant turns combined since the
|
|
2983
|
+
* last consolidation cursor, and aren't already up-to-date. Returns one
|
|
2984
|
+
* row per session ranked oldest-idle first so we consolidate the
|
|
2985
|
+
* least-fresh first when bounded by maxResults.
|
|
2986
|
+
*/
|
|
2987
|
+
getIdleSessionsForEpisodicConsolidation(opts) {
|
|
2988
|
+
const idleMin = Math.max(1, opts.idleMinutes);
|
|
2989
|
+
const minEx = Math.max(1, opts.minExchanges);
|
|
2990
|
+
const max = Math.max(1, opts.maxResults);
|
|
2991
|
+
const backoff = Math.max(0, opts.failBackoffMinutes ?? 60);
|
|
2992
|
+
try {
|
|
2993
|
+
// Per-session: last cursor (or 0), count of new turns, MIN/MAX(id)
|
|
2994
|
+
// bounding the new range, and the timestamps. The fail-backoff
|
|
2995
|
+
// suppresses sessions whose last_attempted_at is recent enough that
|
|
2996
|
+
// the cursor's fail_count > 0 indicates we should wait.
|
|
2997
|
+
const rows = this.conn.prepare(`
|
|
2998
|
+
SELECT
|
|
2999
|
+
t.session_key AS session_key,
|
|
3000
|
+
MIN(t.id) AS start_id,
|
|
3001
|
+
MAX(t.id) AS end_id,
|
|
3002
|
+
MIN(t.created_at) AS started_at,
|
|
3003
|
+
MAX(t.created_at) AS ended_at,
|
|
3004
|
+
COUNT(*) AS exchanges
|
|
3005
|
+
FROM transcripts t
|
|
3006
|
+
LEFT JOIN consolidation_cursors c ON c.session_key = t.session_key
|
|
3007
|
+
WHERE t.id > COALESCE(c.last_transcript_id, 0)
|
|
3008
|
+
AND (
|
|
3009
|
+
c.fail_count IS NULL
|
|
3010
|
+
OR c.fail_count = 0
|
|
3011
|
+
OR c.last_attempted_at IS NULL
|
|
3012
|
+
OR c.last_attempted_at < datetime('now', ?)
|
|
3013
|
+
)
|
|
3014
|
+
GROUP BY t.session_key
|
|
3015
|
+
HAVING COUNT(*) >= ?
|
|
3016
|
+
AND MAX(t.created_at) < datetime('now', ?)
|
|
3017
|
+
ORDER BY MAX(t.created_at) ASC
|
|
3018
|
+
LIMIT ?
|
|
3019
|
+
`).all(`-${backoff} minutes`, minEx, `-${idleMin} minutes`, max);
|
|
3020
|
+
return rows.map(r => ({
|
|
3021
|
+
sessionKey: r.session_key,
|
|
3022
|
+
startTranscriptId: r.start_id,
|
|
3023
|
+
endTranscriptId: r.end_id,
|
|
3024
|
+
startedAt: r.started_at,
|
|
3025
|
+
endedAt: r.ended_at,
|
|
3026
|
+
exchanges: r.exchanges,
|
|
3027
|
+
}));
|
|
3028
|
+
}
|
|
3029
|
+
catch {
|
|
3030
|
+
return [];
|
|
3031
|
+
}
|
|
3032
|
+
}
|
|
3033
|
+
/**
|
|
3034
|
+
* Persist a consolidated episode and bump the per-session cursor so the
|
|
3035
|
+
* same range isn't re-consolidated on the next pass. The summary text is
|
|
3036
|
+
* also indexed into chunks (returned as chunkId) so hybrid recall surfaces
|
|
3037
|
+
* episodes alongside raw transcripts.
|
|
3038
|
+
*/
|
|
3039
|
+
insertEpisode(entry) {
|
|
3040
|
+
const result = this.conn
|
|
3041
|
+
.prepare(`INSERT INTO episodes
|
|
3042
|
+
(session_key, started_at, ended_at, summary, topics, entities, outcome, open_loops, transcript_ids, chunk_id)
|
|
3043
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
|
|
3044
|
+
.run(entry.sessionKey, entry.startedAt, entry.endedAt, entry.summary, JSON.stringify(entry.topics ?? []), JSON.stringify(entry.entities ?? []), entry.outcome ?? '', JSON.stringify(entry.openLoops ?? []), JSON.stringify(entry.transcriptIds ?? []), entry.chunkId ?? null);
|
|
3045
|
+
return {
|
|
3046
|
+
episodeId: result.lastInsertRowid,
|
|
3047
|
+
chunkId: entry.chunkId ?? null,
|
|
3048
|
+
};
|
|
3049
|
+
}
|
|
3050
|
+
/**
|
|
3051
|
+
* Mark a consolidation pass result on the per-session cursor. On success
|
|
3052
|
+
* we advance last_transcript_id and reset fail_count; on failure we bump
|
|
3053
|
+
* fail_count + last_attempted_at so the backoff-aware idle scan skips
|
|
3054
|
+
* this session for a while.
|
|
3055
|
+
*/
|
|
3056
|
+
updateConsolidationCursor(sessionKey, update) {
|
|
3057
|
+
const existing = this.conn
|
|
3058
|
+
.prepare('SELECT session_key FROM consolidation_cursors WHERE session_key = ?')
|
|
3059
|
+
.get(sessionKey);
|
|
3060
|
+
if (!existing) {
|
|
3061
|
+
this.conn
|
|
3062
|
+
.prepare(`INSERT INTO consolidation_cursors
|
|
3063
|
+
(session_key, last_transcript_id, last_attempted_at, last_success_at, fail_count)
|
|
3064
|
+
VALUES (?, ?, datetime('now'), ?, ?)`)
|
|
3065
|
+
.run(sessionKey, update.success ? (update.lastTranscriptId ?? 0) : 0, update.success ? new Date().toISOString() : null, update.success ? 0 : 1);
|
|
3066
|
+
return;
|
|
3067
|
+
}
|
|
3068
|
+
if (update.success) {
|
|
3069
|
+
this.conn
|
|
3070
|
+
.prepare(`UPDATE consolidation_cursors
|
|
3071
|
+
SET last_transcript_id = ?, last_attempted_at = datetime('now'),
|
|
3072
|
+
last_success_at = datetime('now'), fail_count = 0
|
|
3073
|
+
WHERE session_key = ?`)
|
|
3074
|
+
.run(update.lastTranscriptId ?? 0, sessionKey);
|
|
3075
|
+
}
|
|
3076
|
+
else {
|
|
3077
|
+
this.conn
|
|
3078
|
+
.prepare(`UPDATE consolidation_cursors
|
|
3079
|
+
SET last_attempted_at = datetime('now'), fail_count = fail_count + 1
|
|
3080
|
+
WHERE session_key = ?`)
|
|
3081
|
+
.run(sessionKey);
|
|
3082
|
+
}
|
|
3083
|
+
}
|
|
3084
|
+
/** Read the consolidation cursor for a session — used in tests and for diagnostics. */
|
|
3085
|
+
getConsolidationCursor(sessionKey) {
|
|
3086
|
+
const row = this.conn
|
|
3087
|
+
.prepare('SELECT * FROM consolidation_cursors WHERE session_key = ?')
|
|
3088
|
+
.get(sessionKey);
|
|
3089
|
+
if (!row)
|
|
3090
|
+
return null;
|
|
3091
|
+
return {
|
|
3092
|
+
sessionKey: row.session_key,
|
|
3093
|
+
lastTranscriptId: row.last_transcript_id,
|
|
3094
|
+
lastAttemptedAt: row.last_attempted_at,
|
|
3095
|
+
lastSuccessAt: row.last_success_at,
|
|
3096
|
+
failCount: row.fail_count,
|
|
3097
|
+
};
|
|
3098
|
+
}
|
|
3099
|
+
/**
|
|
3100
|
+
* List recent episodes for the dashboard. JSON columns are parsed back
|
|
3101
|
+
* into arrays so callers don't have to.
|
|
3102
|
+
*/
|
|
3103
|
+
listRecentEpisodes(opts = {}) {
|
|
3104
|
+
const limit = Math.max(1, Math.min(opts.limit ?? 30, 200));
|
|
3105
|
+
const params = [];
|
|
3106
|
+
let where = '';
|
|
3107
|
+
if (opts.sessionKey) {
|
|
3108
|
+
where += where ? ' AND' : ' WHERE';
|
|
3109
|
+
where += ' session_key = ?';
|
|
3110
|
+
params.push(opts.sessionKey);
|
|
3111
|
+
}
|
|
3112
|
+
if (opts.sinceIso) {
|
|
3113
|
+
where += where ? ' AND' : ' WHERE';
|
|
3114
|
+
where += ' created_at >= ?';
|
|
3115
|
+
params.push(opts.sinceIso);
|
|
3116
|
+
}
|
|
3117
|
+
params.push(limit);
|
|
3118
|
+
const rows = this.conn
|
|
3119
|
+
.prepare(`SELECT * FROM episodes${where} ORDER BY created_at DESC LIMIT ?`)
|
|
3120
|
+
.all(...params);
|
|
3121
|
+
const parseArray = (v) => {
|
|
3122
|
+
if (!v)
|
|
3123
|
+
return [];
|
|
3124
|
+
try {
|
|
3125
|
+
const x = JSON.parse(v);
|
|
3126
|
+
return Array.isArray(x) ? x.map(String) : [];
|
|
3127
|
+
}
|
|
3128
|
+
catch {
|
|
3129
|
+
return [];
|
|
3130
|
+
}
|
|
3131
|
+
};
|
|
3132
|
+
const parseNumArray = (v) => {
|
|
3133
|
+
if (!v)
|
|
3134
|
+
return [];
|
|
3135
|
+
try {
|
|
3136
|
+
const x = JSON.parse(v);
|
|
3137
|
+
return Array.isArray(x) ? x.filter(n => Number.isFinite(n)).map(Number) : [];
|
|
3138
|
+
}
|
|
3139
|
+
catch {
|
|
3140
|
+
return [];
|
|
3141
|
+
}
|
|
3142
|
+
};
|
|
3143
|
+
return rows.map(row => ({
|
|
3144
|
+
id: row.id,
|
|
3145
|
+
sessionKey: row.session_key,
|
|
3146
|
+
startedAt: row.started_at,
|
|
3147
|
+
endedAt: row.ended_at,
|
|
3148
|
+
summary: row.summary,
|
|
3149
|
+
topics: parseArray(row.topics),
|
|
3150
|
+
entities: parseArray(row.entities),
|
|
3151
|
+
outcome: row.outcome ?? '',
|
|
3152
|
+
openLoops: parseArray(row.open_loops),
|
|
3153
|
+
transcriptIds: parseNumArray(row.transcript_ids),
|
|
3154
|
+
chunkId: row.chunk_id,
|
|
3155
|
+
createdAt: row.created_at,
|
|
3156
|
+
}));
|
|
3157
|
+
}
|
|
3158
|
+
/**
|
|
3159
|
+
* Fetch a slice of transcripts by id range for consolidation. Used by
|
|
3160
|
+
* the consolidation module to materialize the conversation it's about
|
|
3161
|
+
* to summarize.
|
|
3162
|
+
*/
|
|
3163
|
+
getTranscriptsByIdRange(sessionKey, startId, endId) {
|
|
3164
|
+
const rows = this.conn
|
|
3165
|
+
.prepare(`SELECT id, session_key, role, content, model, created_at
|
|
3166
|
+
FROM transcripts
|
|
3167
|
+
WHERE session_key = ? AND id >= ? AND id <= ?
|
|
3168
|
+
ORDER BY id ASC`)
|
|
3169
|
+
.all(sessionKey, startId, endId);
|
|
3170
|
+
return rows.map(r => ({
|
|
3171
|
+
id: r.id,
|
|
3172
|
+
sessionKey: r.session_key,
|
|
3173
|
+
role: r.role,
|
|
3174
|
+
content: r.content,
|
|
3175
|
+
model: r.model,
|
|
3176
|
+
createdAt: r.created_at,
|
|
3177
|
+
}));
|
|
3178
|
+
}
|
|
2943
3179
|
// ── Session Summaries ─────────────────────────────────────────────
|
|
2944
3180
|
/**
|
|
2945
3181
|
* Save a session summary for cross-session context.
|
|
@@ -5318,14 +5554,16 @@ export class MemoryStore {
|
|
|
5318
5554
|
.prepare(`INSERT INTO chunks (source_file, section, content, chunk_type, content_hash, salience, consolidated, derived_from)
|
|
5319
5555
|
VALUES (?, ?, ?, 'summary', ?, 0.8, 0, ?)`)
|
|
5320
5556
|
.run(sourceFile, section, content, hash, derivedJson);
|
|
5557
|
+
const chunkId = result.lastInsertRowid;
|
|
5321
5558
|
// Immediately compute embedding so the summary is vector-searchable right away
|
|
5322
5559
|
if (embeddingsModule.isReady()) {
|
|
5323
5560
|
const vec = embeddingsModule.embed(content);
|
|
5324
5561
|
if (vec) {
|
|
5325
5562
|
this.conn.prepare('UPDATE chunks SET embedding = ? WHERE id = ?')
|
|
5326
|
-
.run(embeddingsModule.serializeEmbedding(vec),
|
|
5563
|
+
.run(embeddingsModule.serializeEmbedding(vec), chunkId);
|
|
5327
5564
|
}
|
|
5328
5565
|
}
|
|
5566
|
+
return chunkId;
|
|
5329
5567
|
}
|
|
5330
5568
|
// ── SDR Operational Data ─────────────────────────────────────────
|
|
5331
5569
|
// -- Leads --
|