@animalabs/agent-framework 0.14.0 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -1
- package/dist/src/agent.d.ts +42 -0
- package/dist/src/agent.d.ts.map +1 -1
- package/dist/src/agent.js +94 -0
- package/dist/src/agent.js.map +1 -1
- package/dist/src/api/server.d.ts +9 -0
- package/dist/src/api/server.d.ts.map +1 -1
- package/dist/src/api/server.js +156 -0
- package/dist/src/api/server.js.map +1 -1
- package/dist/src/api/types.d.ts +15 -0
- package/dist/src/api/types.d.ts.map +1 -1
- package/dist/src/framework.d.ts +423 -5
- package/dist/src/framework.d.ts.map +1 -1
- package/dist/src/framework.js +1906 -102
- package/dist/src/framework.js.map +1 -1
- package/dist/src/gate/event-gate.d.ts +16 -0
- package/dist/src/gate/event-gate.d.ts.map +1 -1
- package/dist/src/gate/event-gate.js +41 -3
- package/dist/src/gate/event-gate.js.map +1 -1
- package/dist/src/index.d.ts +4 -1
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +3 -1
- package/dist/src/index.js.map +1 -1
- package/dist/src/mcpl/channel-registry.d.ts +172 -0
- package/dist/src/mcpl/channel-registry.d.ts.map +1 -1
- package/dist/src/mcpl/channel-registry.js +444 -2
- package/dist/src/mcpl/channel-registry.js.map +1 -1
- package/dist/src/mcpl/server-connection.d.ts +9 -0
- package/dist/src/mcpl/server-connection.d.ts.map +1 -1
- package/dist/src/mcpl/server-connection.js +71 -13
- package/dist/src/mcpl/server-connection.js.map +1 -1
- package/dist/src/modules/history/index.d.ts +145 -0
- package/dist/src/modules/history/index.d.ts.map +1 -0
- package/dist/src/modules/history/index.js +1150 -0
- package/dist/src/modules/history/index.js.map +1 -0
- package/dist/src/modules/history/search-regex-worker.d.ts +23 -0
- package/dist/src/modules/history/search-regex-worker.d.ts.map +1 -0
- package/dist/src/modules/history/search-regex-worker.js +57 -0
- package/dist/src/modules/history/search-regex-worker.js.map +1 -0
- package/dist/src/modules/index.d.ts +1 -0
- package/dist/src/modules/index.d.ts.map +1 -1
- package/dist/src/modules/index.js +1 -0
- package/dist/src/modules/index.js.map +1 -1
- package/dist/src/modules/workspace/index.d.ts.map +1 -1
- package/dist/src/modules/workspace/index.js +42 -1
- package/dist/src/modules/workspace/index.js.map +1 -1
- package/dist/src/modules/workspace/types.d.ts +5 -0
- package/dist/src/modules/workspace/types.d.ts.map +1 -1
- package/dist/src/modules/workspace/types.js.map +1 -1
- package/dist/src/operator-log.d.ts +78 -0
- package/dist/src/operator-log.d.ts.map +1 -0
- package/dist/src/operator-log.js +102 -0
- package/dist/src/operator-log.js.map +1 -0
- package/dist/src/recovery/discord-awareness-outbox.d.ts +10 -0
- package/dist/src/recovery/discord-awareness-outbox.d.ts.map +1 -1
- package/dist/src/recovery/discord-awareness-outbox.js +20 -0
- package/dist/src/recovery/discord-awareness-outbox.js.map +1 -1
- package/dist/src/types/framework.d.ts +24 -0
- package/dist/src/types/framework.d.ts.map +1 -1
- package/dist/src/types/trace.d.ts +40 -0
- package/dist/src/types/trace.d.ts.map +1 -1
- package/dist/test/agent-runtime-settings.test.js +199 -1
- package/dist/test/agent-runtime-settings.test.js.map +1 -1
- package/dist/test/api-admin-gate.test.d.ts +2 -0
- package/dist/test/api-admin-gate.test.d.ts.map +1 -0
- package/dist/test/api-admin-gate.test.js +116 -0
- package/dist/test/api-admin-gate.test.js.map +1 -0
- package/dist/test/channel-label-history.test.d.ts +2 -0
- package/dist/test/channel-label-history.test.d.ts.map +1 -0
- package/dist/test/channel-label-history.test.js +424 -0
- package/dist/test/channel-label-history.test.js.map +1 -0
- package/dist/test/discord-awareness-outbox.test.js +16 -0
- package/dist/test/discord-awareness-outbox.test.js.map +1 -1
- package/dist/test/event-gate.test.js +44 -0
- package/dist/test/event-gate.test.js.map +1 -1
- package/dist/test/helpers/quiesce-crash-child.d.ts +2 -0
- package/dist/test/helpers/quiesce-crash-child.d.ts.map +1 -0
- package/dist/test/helpers/quiesce-crash-child.js +156 -0
- package/dist/test/helpers/quiesce-crash-child.js.map +1 -0
- package/dist/test/history-module-overview.test.d.ts +2 -0
- package/dist/test/history-module-overview.test.d.ts.map +1 -0
- package/dist/test/history-module-overview.test.js +711 -0
- package/dist/test/history-module-overview.test.js.map +1 -0
- package/dist/test/history-module.test.d.ts +2 -0
- package/dist/test/history-module.test.d.ts.map +1 -0
- package/dist/test/history-module.test.js +595 -0
- package/dist/test/history-module.test.js.map +1 -0
- package/dist/test/host-quiesce.test.d.ts +2 -0
- package/dist/test/host-quiesce.test.d.ts.map +1 -0
- package/dist/test/host-quiesce.test.js +863 -0
- package/dist/test/host-quiesce.test.js.map +1 -0
- package/dist/test/kv-unified-open-flight-retry.test.d.ts +2 -0
- package/dist/test/kv-unified-open-flight-retry.test.d.ts.map +1 -0
- package/dist/test/kv-unified-open-flight-retry.test.js +189 -0
- package/dist/test/kv-unified-open-flight-retry.test.js.map +1 -0
- package/dist/test/kv-unified-wiring.test.js +39 -0
- package/dist/test/kv-unified-wiring.test.js.map +1 -1
- package/dist/test/operator-surgery.test.d.ts +2 -0
- package/dist/test/operator-surgery.test.d.ts.map +1 -0
- package/dist/test/operator-surgery.test.js +398 -0
- package/dist/test/operator-surgery.test.js.map +1 -0
- package/dist/test/puppet-tool-call.test.js +118 -1
- package/dist/test/puppet-tool-call.test.js.map +1 -1
- package/dist/test/puppet-turn-reservation.test.d.ts +2 -0
- package/dist/test/puppet-turn-reservation.test.d.ts.map +1 -0
- package/dist/test/puppet-turn-reservation.test.js +139 -0
- package/dist/test/puppet-turn-reservation.test.js.map +1 -0
- package/dist/test/save-recent-image.test.js +29 -0
- package/dist/test/save-recent-image.test.js.map +1 -1
- package/dist/test/tool-result-spill.test.js +114 -0
- package/dist/test/tool-result-spill.test.js.map +1 -1
- package/dist/test/workspace-read-text.test.d.ts +2 -0
- package/dist/test/workspace-read-text.test.d.ts.map +1 -0
- package/dist/test/workspace-read-text.test.js +101 -0
- package/dist/test/workspace-read-text.test.js.map +1 -0
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +3 -3
- package/src/agent.ts +110 -0
- package/src/api/server.ts +159 -2
- package/src/api/types.ts +15 -0
- package/src/framework.ts +2196 -115
- package/src/gate/event-gate.ts +47 -4
- package/src/index.ts +5 -1
- package/src/mcpl/channel-registry.ts +495 -2
- package/src/mcpl/server-connection.ts +73 -13
- package/src/modules/history/index.ts +1277 -0
- package/src/modules/history/search-regex-worker.ts +76 -0
- package/src/modules/index.ts +2 -0
- package/src/modules/workspace/index.ts +42 -1
- package/src/modules/workspace/types.ts +5 -0
- package/src/operator-log.ts +132 -0
- package/src/recovery/discord-awareness-outbox.ts +19 -0
- package/src/types/framework.ts +27 -0
- package/src/types/trace.ts +46 -0
|
@@ -0,0 +1,1277 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* HistoryModule — read-only access to an agent's full uncompressed message
|
|
3
|
+
* history, backed by context-manager's native chronicle secondary index
|
|
4
|
+
* (`queryMessagesByTime` / `queryMessagesByChannel` /
|
|
5
|
+
* `queryMessagesByTimeAndChannel` / `getChannelMessageCounts` /
|
|
6
|
+
* `getChannelTokenStats`, @animalabs/context-manager >= 0.6.0). Those calls
|
|
7
|
+
* are O(log n + k) against chronicle's `/timestamp` and
|
|
8
|
+
* `/metadata/external/channelId` field indexes, not a full-store scan, so
|
|
9
|
+
* this module can answer "what happened in #foo last Tuesday" against a
|
|
10
|
+
* multi-million-message store without walking the whole log.
|
|
11
|
+
*
|
|
12
|
+
* Four tools:
|
|
13
|
+
* - `stats` — per-channel message counts (all-time) + token totals
|
|
14
|
+
* (range-scoped), for orienting before pulling raw content.
|
|
15
|
+
* - `extract` — paginated raw messages for a time range and/or channel.
|
|
16
|
+
* - `search` — substring/regex match over a narrowed candidate window,
|
|
17
|
+
* with an explicit truncation signal when the caller's
|
|
18
|
+
* filter was too broad for `maxScan`.
|
|
19
|
+
* - `overview` — a compression-summary table of contents (zero new LLM
|
|
20
|
+
* calls) for a time range, gap-filled with raw message
|
|
21
|
+
* counts wherever nothing has been summarized yet. See
|
|
22
|
+
* `handleOverview` for the fold-reduction and gap-fill
|
|
23
|
+
* algorithm.
|
|
24
|
+
*
|
|
25
|
+
* All four accept `channelId` as either a channel label (e.g. `#general`)
|
|
26
|
+
* or a raw internal channel id — `resolveChannel()` resolves a label via
|
|
27
|
+
* the bound `ChannelRegistry`'s durable label history before it reaches any
|
|
28
|
+
* context-manager call. A host that hasn't wired up MCPL (no registry
|
|
29
|
+
* bound) sees unchanged behavior: `channelId` is treated as already the raw
|
|
30
|
+
* internal id.
|
|
31
|
+
*
|
|
32
|
+
* Kept deliberately read-only, same posture as HealthModule: no side
|
|
33
|
+
* effects, no message mutation. `onProcess` is a no-op.
|
|
34
|
+
*
|
|
35
|
+
* Every query dispatch collapses onto `queryMessagesByTimeAndChannel` — its
|
|
36
|
+
* own implementation already handles "only a time range", "only a channel",
|
|
37
|
+
* or "neither" by delegating to the single-filter native queries (see
|
|
38
|
+
* context-manager's `test/message-store-history-index.test.ts`), so this
|
|
39
|
+
* module doesn't need to re-decide which of the three query methods to call.
|
|
40
|
+
*/
|
|
41
|
+
|
|
42
|
+
import { Worker } from 'node:worker_threads';
|
|
43
|
+
import { fileURLToPath } from 'node:url';
|
|
44
|
+
import { dirname, join } from 'node:path';
|
|
45
|
+
import type { ContextManager, StoredMessage, ChannelCount, ChannelTokenStats, TimeRangeSummaryEntry } from '@animalabs/context-manager';
|
|
46
|
+
import type { ContentBlock } from '@animalabs/membrane';
|
|
47
|
+
import type { Module, ModuleContext } from '../../types/module.js';
|
|
48
|
+
import type { ToolDefinition, ToolCall, ToolResult, ProcessEvent } from '../../types/events.js';
|
|
49
|
+
import type { EventResponse, ProcessState } from '../../types/module.js';
|
|
50
|
+
import type { SearchWorkerMessage, SearchWorkerMatch } from './search-regex-worker.js';
|
|
51
|
+
import type { ChannelRegistry } from '../../mcpl/channel-registry.js';
|
|
52
|
+
|
|
53
|
+
// ============================================================================
|
|
54
|
+
// Tool input shapes
|
|
55
|
+
// ============================================================================
|
|
56
|
+
|
|
57
|
+
interface StatsInput {
|
|
58
|
+
from?: string;
|
|
59
|
+
to?: string;
|
|
60
|
+
channelId?: string;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
interface ExtractInput {
|
|
64
|
+
from?: string;
|
|
65
|
+
to?: string;
|
|
66
|
+
channelId?: string;
|
|
67
|
+
limit?: number;
|
|
68
|
+
offset?: number;
|
|
69
|
+
format?: 'text' | 'raw';
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
interface SearchInput {
|
|
73
|
+
query: string;
|
|
74
|
+
regex?: boolean;
|
|
75
|
+
caseSensitive?: boolean;
|
|
76
|
+
from?: string;
|
|
77
|
+
to?: string;
|
|
78
|
+
channelId?: string;
|
|
79
|
+
limit?: number;
|
|
80
|
+
maxScan?: number;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
interface OverviewInput {
|
|
84
|
+
from?: string;
|
|
85
|
+
to?: string;
|
|
86
|
+
channelId?: string;
|
|
87
|
+
level?: number;
|
|
88
|
+
limit?: number;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
// ============================================================================
|
|
92
|
+
// Limits
|
|
93
|
+
// ============================================================================
|
|
94
|
+
|
|
95
|
+
const EXTRACT_DEFAULT_LIMIT = 50;
|
|
96
|
+
const EXTRACT_MAX_LIMIT = 200;
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* Real ceiling of the native chronicle pagination `offset` argument (a u32
|
|
100
|
+
* at the N-API boundary). `Number.MAX_SAFE_INTEGER` is NOT a safe ceiling
|
|
101
|
+
* for this field: a value between 0xFFFFFFFF and MAX_SAFE_INTEGER passes a
|
|
102
|
+
* naive upper-bound check unchanged, then wraps/truncates when it crosses
|
|
103
|
+
* into an unsigned 32-bit int on the native side — silently wrong
|
|
104
|
+
* pagination (repro: offset:4294967296 against a real store "succeeded"
|
|
105
|
+
* and returned page 0) instead of a clean failure. Clamping to this ceiling
|
|
106
|
+
* (same clamp-not-reject behavior `limit` already has) means anything
|
|
107
|
+
* beyond it lands on an offset no real store's message count will ever
|
|
108
|
+
* reach, i.e. a correctly-empty page, rather than wrapping onto a wrong one.
|
|
109
|
+
*/
|
|
110
|
+
const NATIVE_OFFSET_MAX = 0xffffffff; // 4294967295
|
|
111
|
+
|
|
112
|
+
const SEARCH_DEFAULT_LIMIT = 20;
|
|
113
|
+
const SEARCH_MAX_LIMIT = 100;
|
|
114
|
+
const SEARCH_DEFAULT_MAX_SCAN = 5000;
|
|
115
|
+
const SEARCH_MAX_MAX_SCAN = 50000;
|
|
116
|
+
|
|
117
|
+
/**
|
|
118
|
+
* `overview` response entry cap — unlike `stats`/`extract`/`search`, which
|
|
119
|
+
* all clamp their result sizes, `overview` had none: an unbounded call
|
|
120
|
+
* against a long-lived resident with thousands of minted summaries would
|
|
121
|
+
* return every overlapping summary's full `content` text inlined, with one
|
|
122
|
+
* `getChannelTokenStats` native call per entry, synchronously, with no
|
|
123
|
+
* limit — exactly the resource this tool exists to conserve. Default/max
|
|
124
|
+
* roughly match `extract`'s own scale.
|
|
125
|
+
*/
|
|
126
|
+
const OVERVIEW_DEFAULT_LIMIT = 50;
|
|
127
|
+
const OVERVIEW_MAX_LIMIT = 200;
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* A `<@id>`/`<@!id>` Discord-style mention — same shape `ChannelRegistry`
|
|
131
|
+
* uses for its own DM mention-form matching. A string matching this can
|
|
132
|
+
* never legitimately BE a raw internal channelId (unlike a bare word,
|
|
133
|
+
* which an agent could plausibly be echoing back), so `resolveChannel`
|
|
134
|
+
* treats an unresolved mention the same as an unresolved `#`/`@` label:
|
|
135
|
+
* a clean error, not a silent empty-result passthrough.
|
|
136
|
+
*/
|
|
137
|
+
const DM_MENTION_RE = /^<@!?\d+>$/;
|
|
138
|
+
|
|
139
|
+
/** Characters of surrounding context kept on each side of a search snippet. */
|
|
140
|
+
const SNIPPET_CONTEXT_CHARS = 80;
|
|
141
|
+
/** Fallback snippet length when a match position isn't meaningful to center on. */
|
|
142
|
+
const SNIPPET_FALLBACK_CHARS = 160;
|
|
143
|
+
|
|
144
|
+
/**
|
|
145
|
+
* Wall-clock deadline for a single regex-mode search call, enforced by
|
|
146
|
+
* forcibly terminating the worker thread doing the matching (see
|
|
147
|
+
* search-regex-worker.ts's header for why a worker — not a
|
|
148
|
+
* Promise.race/setTimeout — is required to actually interrupt a stuck
|
|
149
|
+
* synchronous RegExp.exec()). Generous enough for any legitimate pattern
|
|
150
|
+
* against a few thousand short strings; short enough that a catastrophic
|
|
151
|
+
* pattern doesn't tie up a worker (or an agent's turn) for long.
|
|
152
|
+
*/
|
|
153
|
+
const SEARCH_REGEX_TIMEOUT_MS = 2000;
|
|
154
|
+
|
|
155
|
+
/** Compiled sibling of search-regex-worker.ts — resolved at runtime the same
|
|
156
|
+
* way gate-script.ts locates gate-script-worker.js, so it tracks whatever
|
|
157
|
+
* directory this module's own compiled output lives in. */
|
|
158
|
+
const SEARCH_WORKER_PATH = join(dirname(fileURLToPath(import.meta.url)), 'search-regex-worker.js');
|
|
159
|
+
|
|
160
|
+
export class HistoryModule implements Module {
|
|
161
|
+
readonly name = 'history';
|
|
162
|
+
|
|
163
|
+
private ctx: ModuleContext | null = null;
|
|
164
|
+
private cm: ContextManager | null = null;
|
|
165
|
+
private channelRegistry: ChannelRegistry | null = null;
|
|
166
|
+
|
|
167
|
+
/**
|
|
168
|
+
* Wire the context-manager instance, and optionally the host's
|
|
169
|
+
* ChannelRegistry. Host calls this after ContextManager.open() so the
|
|
170
|
+
* module can issue the native index-backed queries — ModuleContext itself
|
|
171
|
+
* exposes no store/context-manager reference, only the narrower message
|
|
172
|
+
* CRUD surface (addMessage/getMessage/queryMessages), which can't do
|
|
173
|
+
* range or channel queries.
|
|
174
|
+
*
|
|
175
|
+
* `channelRegistry` is optional: a host without MCPL wired up has no
|
|
176
|
+
* registry to pass, and `resolveChannel()` falls back to treating
|
|
177
|
+
* `channelId` as already the raw internal id — today's behavior,
|
|
178
|
+
* unchanged.
|
|
179
|
+
*/
|
|
180
|
+
bind(contextManager: ContextManager, channelRegistry?: ChannelRegistry): void {
|
|
181
|
+
this.cm = contextManager;
|
|
182
|
+
this.channelRegistry = channelRegistry ?? null;
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
/**
|
|
186
|
+
* Resolve an agent-supplied channel spec — a label like `#general` or an
|
|
187
|
+
* already-raw internal channelId — to the internal id used by
|
|
188
|
+
* context-manager's channel-indexed queries, via the bound
|
|
189
|
+
* ChannelRegistry's durable label history (`resolveProseTargetDurable`,
|
|
190
|
+
* which survives the bot disconnecting from or restarting on a channel —
|
|
191
|
+
* exactly the case that matters for browsing OLD history).
|
|
192
|
+
*
|
|
193
|
+
* Falls through the input unchanged when no registry is bound (host
|
|
194
|
+
* without MCPL — today's behavior). On a registry MISS, the response
|
|
195
|
+
* depends on whether the spec actually looks like a label: `#general`,
|
|
196
|
+
* `@name`, or a `<@id>`/`<@!id>` mention (the syntaxes `resolveProseTarget`
|
|
197
|
+
* itself treats as unambiguously label-shaped — see its own leading
|
|
198
|
+
* `#`/`@` and mention-regex handling) is very likely a typo or a
|
|
199
|
+
* never-registered target, not a raw internal id, so a miss there throws
|
|
200
|
+
* a clean tool error (with any `candidates` suggestions folded in)
|
|
201
|
+
* instead of quietly treating it as an empty/nonexistent channel. A
|
|
202
|
+
* mention in particular can never legitimately BE a raw internal
|
|
203
|
+
* channelId, so the escape-hatch rationale below doesn't even apply to
|
|
204
|
+
* it. Anything else is
|
|
205
|
+
* passed through as an escape hatch — an agent can legitimately already
|
|
206
|
+
* hold a raw channelId (e.g. echoed back by a prior
|
|
207
|
+
* `stats`/`extract`/`search`/`overview` call), and erroring on that would
|
|
208
|
+
* break working addressing to "fix" a spec that was never a label at all.
|
|
209
|
+
*
|
|
210
|
+
* The thrown message is deliberately NOT `resolved.error` verbatim:
|
|
211
|
+
* `resolveProseTargetDurable`'s underlying live resolver
|
|
212
|
+
* (`resolveProseTarget`) writes its error text for a SEND context (e.g.
|
|
213
|
+
* a DM miss says '...use the send_dm tool') — sensible advice when
|
|
214
|
+
* you're trying to deliver a message, nonsense advice from a read-only
|
|
215
|
+
* history tool. This module always uses its own read-only-appropriate
|
|
216
|
+
* wording instead, regardless of which branch of the live resolver
|
|
217
|
+
* produced the miss.
|
|
218
|
+
*/
|
|
219
|
+
private resolveChannel(input: string | undefined): string | undefined {
|
|
220
|
+
if (!input || !this.channelRegistry) return input;
|
|
221
|
+
const resolved = this.channelRegistry.resolveProseTargetDurable(input);
|
|
222
|
+
if (!('error' in resolved)) return resolved.channelId;
|
|
223
|
+
if (input.startsWith('#') || input.startsWith('@') || DM_MENTION_RE.test(input)) {
|
|
224
|
+
const suggestions = resolved.candidates?.length ? ` Did you mean: ${resolved.candidates.join(', ')}?` : '';
|
|
225
|
+
throw new Error(
|
|
226
|
+
`No channel history found for "${input}" — it may never have been seen by this resident, or predates this feature.${suggestions}`,
|
|
227
|
+
);
|
|
228
|
+
}
|
|
229
|
+
return input;
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
/**
|
|
233
|
+
* Earliest message timestamp anywhere in the store, constrained to
|
|
234
|
+
* `toMs` when given (never look past the caller's own upper bound while
|
|
235
|
+
* anchoring). Used by `handleOverview` to anchor its gap-walk cursor when
|
|
236
|
+
* `from` is omitted — see the cursor-anchoring comment there for why
|
|
237
|
+
* falling back to "now" instead is wrong. Cheap: a single native
|
|
238
|
+
* time-indexed query for one message (oldest-first is
|
|
239
|
+
* `queryMessagesByTime`'s default order). Returns undefined for a
|
|
240
|
+
* genuinely empty store (or empty up to `toMs`).
|
|
241
|
+
*/
|
|
242
|
+
private earliestMessageMs(toMs: number | undefined): number | undefined {
|
|
243
|
+
const cm = this.cm as ContextManager;
|
|
244
|
+
const probe = cm.queryMessagesByTime({ toMs, limit: 1 });
|
|
245
|
+
return probe.messages[0]?.timestamp.getTime();
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
/**
|
|
249
|
+
* Every raw message sharing exactly `ms` (a single-point, both-ends-
|
|
250
|
+
* inclusive time query). Used by `handleOverview`'s boundary-tie
|
|
251
|
+
* disambiguation to see past `getChannelTokenStats`'s aggregate-only,
|
|
252
|
+
* sequence-blind view of a millisecond — see the correction pass there
|
|
253
|
+
* for why. Cheap: at most a handful of messages share one millisecond in
|
|
254
|
+
* practice.
|
|
255
|
+
*/
|
|
256
|
+
private boundaryMessagesAt(ms: number): StoredMessage[] {
|
|
257
|
+
const cm = this.cm as ContextManager;
|
|
258
|
+
return cm.queryMessagesByTime({ fromMs: ms, toMs: ms }).messages;
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
async start(ctx: ModuleContext): Promise<void> {
|
|
262
|
+
this.ctx = ctx;
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
async stop(): Promise<void> {
|
|
266
|
+
this.ctx = null;
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
getTools(): ToolDefinition[] {
|
|
270
|
+
return [
|
|
271
|
+
{
|
|
272
|
+
name: 'stats',
|
|
273
|
+
description:
|
|
274
|
+
'Orient before pulling raw history: per-channel message counts and token totals. ' +
|
|
275
|
+
'messageCountsAllTime is always all-time and all-channel-scope (the underlying index has no ' +
|
|
276
|
+
'range dimension) — it is NOT limited by from/to, only optionally filtered to one channelId. ' +
|
|
277
|
+
'tokenStatsForRange IS scoped to from/to when given. The two are reported separately, never ' +
|
|
278
|
+
'merged, because their range semantics differ.',
|
|
279
|
+
inputSchema: {
|
|
280
|
+
type: 'object' as const,
|
|
281
|
+
properties: {
|
|
282
|
+
from: { type: 'string', description: 'ISO 8601 inclusive lower bound for tokenStatsForRange. Omit for open-ended.' },
|
|
283
|
+
to: { type: 'string', description: 'ISO 8601 inclusive upper bound for tokenStatsForRange. Omit for open-ended.' },
|
|
284
|
+
channelId: { type: 'string', description: 'Restrict both parts of the response to one channel. Accepts a channel label (e.g. "#general") or the raw internal channel id.' },
|
|
285
|
+
},
|
|
286
|
+
},
|
|
287
|
+
},
|
|
288
|
+
{
|
|
289
|
+
name: 'extract',
|
|
290
|
+
description:
|
|
291
|
+
'Fetch raw, uncompressed messages for a time range and/or channel, paginated oldest-first. ' +
|
|
292
|
+
'Use after `stats` to pull the actual content. format:"text" flattens each message to a short ' +
|
|
293
|
+
'readable string (tool_use/tool_result/thinking/images rendered as bracketed labels); ' +
|
|
294
|
+
'format:"raw" returns the content blocks unmodified.',
|
|
295
|
+
inputSchema: {
|
|
296
|
+
type: 'object' as const,
|
|
297
|
+
properties: {
|
|
298
|
+
from: { type: 'string', description: 'ISO 8601 inclusive lower bound. Omit for open-ended.' },
|
|
299
|
+
to: { type: 'string', description: 'ISO 8601 inclusive upper bound. Omit for open-ended.' },
|
|
300
|
+
channelId: { type: 'string', description: 'Restrict to one channel. Accepts a channel label (e.g. "#general") or the raw internal channel id.' },
|
|
301
|
+
limit: { type: 'number', description: `Max messages to return (default ${EXTRACT_DEFAULT_LIMIT}, hard cap ${EXTRACT_MAX_LIMIT}). Must be a non-negative integer.` },
|
|
302
|
+
offset: { type: 'number', description: `Number of matching messages to skip (default 0, capped to ${NATIVE_OFFSET_MAX}). Must be a non-negative integer.` },
|
|
303
|
+
format: { type: 'string', enum: ['text', 'raw'], description: 'Content rendering (default "text").' },
|
|
304
|
+
},
|
|
305
|
+
},
|
|
306
|
+
},
|
|
307
|
+
{
|
|
308
|
+
name: 'search',
|
|
309
|
+
description:
|
|
310
|
+
'Search message history for a substring (default) or regex match, within an optional time ' +
|
|
311
|
+
'range/channel window. Narrows candidates via the same query as `extract` before matching, up ' +
|
|
312
|
+
'to maxScan candidates — if the narrowed window is larger than maxScan, the response reports ' +
|
|
313
|
+
'truncated:true up front rather than silently missing later matches; narrow the filter or raise ' +
|
|
314
|
+
'maxScan and retry. regex:true matching runs under a wall-clock deadline and is cleanly failed ' +
|
|
315
|
+
`(not silently empty) if a pattern is too slow — avoid nested-quantifier patterns like (a+)+.`,
|
|
316
|
+
inputSchema: {
|
|
317
|
+
type: 'object' as const,
|
|
318
|
+
properties: {
|
|
319
|
+
query: { type: 'string', description: 'Substring (or regex source, when regex:true) to search for.' },
|
|
320
|
+
regex: { type: 'boolean', description: 'Treat query as a regular expression (default false).' },
|
|
321
|
+
caseSensitive: { type: 'boolean', description: 'Case-sensitive match (default false).' },
|
|
322
|
+
from: { type: 'string', description: 'ISO 8601 inclusive lower bound. Omit for open-ended.' },
|
|
323
|
+
to: { type: 'string', description: 'ISO 8601 inclusive upper bound. Omit for open-ended.' },
|
|
324
|
+
channelId: { type: 'string', description: 'Restrict to one channel. Accepts a channel label (e.g. "#general") or the raw internal channel id.' },
|
|
325
|
+
limit: { type: 'number', description: `Max matches to return (default ${SEARCH_DEFAULT_LIMIT}, hard cap ${SEARCH_MAX_LIMIT}). Must be a non-negative integer.` },
|
|
326
|
+
maxScan: { type: 'number', description: `Max candidate messages to scan (default ${SEARCH_DEFAULT_MAX_SCAN}, hard cap ${SEARCH_MAX_MAX_SCAN}). Must be a non-negative integer.` },
|
|
327
|
+
},
|
|
328
|
+
required: ['query'],
|
|
329
|
+
},
|
|
330
|
+
},
|
|
331
|
+
{
|
|
332
|
+
name: 'overview',
|
|
333
|
+
description:
|
|
334
|
+
'Browse a time range when you don\'t know exactly what you\'re looking for yet: returns existing ' +
|
|
335
|
+
'compression summaries as a table of contents, at zero new LLM cost (purely a read over summaries ' +
|
|
336
|
+
'already produced by compression). Spans not yet summarized fall back to raw message counts, ' +
|
|
337
|
+
'marked summarized:false. Omit `level` to let each span show its own coarsest available summary ' +
|
|
338
|
+
'(spans can end up at DIFFERENT levels — the response then reports maxLevelAvailable rather than ' +
|
|
339
|
+
'a single `level`, since no one number would describe a mixed result); pass an exact `level` for ' +
|
|
340
|
+
'flat, single-granularity results instead (finer detail = lower level), and the response then ' +
|
|
341
|
+
'echoes that `level` back. `channelId` (a label like "#general" or the raw internal id) narrows ' +
|
|
342
|
+
'WHICH spans are shown (only spans with at least one message from that channel) and scopes each ' +
|
|
343
|
+
'entry\'s messageCount/tokensEstimate to that channel\'s own numbers (spanMessageCount/' +
|
|
344
|
+
'spanTokensEstimate carry the whole-span totals alongside, for context) — but does NOT scope a ' +
|
|
345
|
+
'kept entry\'s summary TEXT to that channel: compression doesn\'t chunk per-channel, so a ' +
|
|
346
|
+
'summary may describe other channels\' traffic too. A wide or unbounded from/to can be expensive ' +
|
|
347
|
+
'on a cold cache — same cost class as `stats`\'s tokenStatsForRange — since gap-filling still ' +
|
|
348
|
+
'walks the underlying message range wherever nothing has been summarized yet. Response spans are ' +
|
|
349
|
+
'capped at `limit`, keeping the MOST RECENT spans and reporting truncated:true + totalSpans when ' +
|
|
350
|
+
'more exist — narrow the range or raise `limit` to see further back. `from` after `to` is rejected ' +
|
|
351
|
+
'with a clear error rather than silently returning no spans. An entry marked boundaryUncertain:true ' +
|
|
352
|
+
'had a millisecond-timestamp collision at its edge with another span (rare — two distinct messages ' +
|
|
353
|
+
'landing in the same millisecond); its messageCount is still exact but tokensEstimate may be off by ' +
|
|
354
|
+
'a small, bounded amount.',
|
|
355
|
+
inputSchema: {
|
|
356
|
+
type: 'object' as const,
|
|
357
|
+
properties: {
|
|
358
|
+
from: { type: 'string', description: 'ISO 8601 inclusive lower bound. Omit for open-ended.' },
|
|
359
|
+
to: { type: 'string', description: 'ISO 8601 inclusive upper bound. Omit for open-ended.' },
|
|
360
|
+
channelId: { type: 'string', description: 'Only show spans with traffic from this channel. Accepts a channel label (e.g. "#general") or the raw internal channel id.' },
|
|
361
|
+
level: {
|
|
362
|
+
type: 'number',
|
|
363
|
+
description:
|
|
364
|
+
'Exact summary level to fetch, flat (no fold-reduction across levels). Omit to default to ' +
|
|
365
|
+
'each span\'s own coarsest currently-available summary (a mix of levels across the response ' +
|
|
366
|
+
'is expected and normal), reduced so a folded summary\'s now-superseded child does not also ' +
|
|
367
|
+
'appear alongside a coarser entry that already covers its span.',
|
|
368
|
+
},
|
|
369
|
+
limit: {
|
|
370
|
+
type: 'number',
|
|
371
|
+
description: `Max spans to return, keeping the most recent (default ${OVERVIEW_DEFAULT_LIMIT}, hard cap ${OVERVIEW_MAX_LIMIT}). Must be a non-negative integer.`,
|
|
372
|
+
},
|
|
373
|
+
},
|
|
374
|
+
},
|
|
375
|
+
},
|
|
376
|
+
];
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
async handleToolCall(call: ToolCall): Promise<ToolResult> {
|
|
380
|
+
try {
|
|
381
|
+
if (!this.cm) {
|
|
382
|
+
throw new Error('HistoryModule not bound — host must call bind(contextManager) before tool dispatch.');
|
|
383
|
+
}
|
|
384
|
+
switch (call.name) {
|
|
385
|
+
case 'stats':
|
|
386
|
+
return this.handleStats((call.input ?? {}) as StatsInput);
|
|
387
|
+
case 'extract':
|
|
388
|
+
return this.handleExtract((call.input ?? {}) as ExtractInput);
|
|
389
|
+
case 'search':
|
|
390
|
+
return await this.handleSearch((call.input ?? {}) as SearchInput);
|
|
391
|
+
case 'overview':
|
|
392
|
+
return this.handleOverview((call.input ?? {}) as OverviewInput);
|
|
393
|
+
default:
|
|
394
|
+
return { success: false, isError: true, error: `Unknown tool: ${call.name}` };
|
|
395
|
+
}
|
|
396
|
+
} catch (error) {
|
|
397
|
+
// Catches our own validation errors (bad ISO date, invalid regex,
|
|
398
|
+
// out-of-range limit/offset/maxScan, unbound module), a regex-search
|
|
399
|
+
// worker timeout or worker-side error (see handleSearch/
|
|
400
|
+
// searchWithRegexWorker — a ReDoS-shaped pattern surfaces here as a
|
|
401
|
+
// clean timeout error, never a hang), and context-manager's
|
|
402
|
+
// capability-absent error ("Chronicle history index unsupported...",
|
|
403
|
+
// thrown by queryMessagesByTime/queryMessagesByChannel/
|
|
404
|
+
// queryMessagesByTimeAndChannel/getChannelMessageCounts/
|
|
405
|
+
// getChannelTokenStats/getSummariesInRange/getMaxSummaryLevel on a
|
|
406
|
+
// chronicle/strategy build that predates the relevant capability) —
|
|
407
|
+
// all surfaced as a normal tool error rather than crashing the module.
|
|
408
|
+
return {
|
|
409
|
+
success: false,
|
|
410
|
+
isError: true,
|
|
411
|
+
error: error instanceof Error ? error.message : String(error),
|
|
412
|
+
};
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
async onProcess(_event: ProcessEvent, _state: ProcessState): Promise<EventResponse> {
|
|
417
|
+
return {};
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
// ==========================================================================
|
|
421
|
+
// stats
|
|
422
|
+
// ==========================================================================
|
|
423
|
+
|
|
424
|
+
private handleStats(input: StatsInput): ToolResult {
|
|
425
|
+
const channelId = this.resolveChannel(input.channelId);
|
|
426
|
+
const fromMs = parseIsoDate(input.from, 'from');
|
|
427
|
+
const toMs = parseIsoDate(input.to, 'to');
|
|
428
|
+
const cm = this.cm as ContextManager;
|
|
429
|
+
|
|
430
|
+
let messageCountsAllTime: ChannelCount[] = cm.getChannelMessageCounts();
|
|
431
|
+
let tokenStatsForRange: ChannelTokenStats = cm.getChannelTokenStats({ fromMs, toMs });
|
|
432
|
+
|
|
433
|
+
if (channelId) {
|
|
434
|
+
messageCountsAllTime = messageCountsAllTime.filter((c) => c.channelId === channelId);
|
|
435
|
+
tokenStatsForRange = {
|
|
436
|
+
// Totals stay whole-range (they're documented as store-wide-for-the-range,
|
|
437
|
+
// not per-channel); only the byChannel breakdown is narrowed. Labeling the
|
|
438
|
+
// field `byChannel` (rather than folding a channel-filtered total into
|
|
439
|
+
// totalMessages/totalTokensEstimate) keeps this from reading as a
|
|
440
|
+
// misleadingly-merged single-channel total.
|
|
441
|
+
totalMessages: tokenStatsForRange.totalMessages,
|
|
442
|
+
totalTokensEstimate: tokenStatsForRange.totalTokensEstimate,
|
|
443
|
+
byChannel: tokenStatsForRange.byChannel.filter((c) => c.channelId === channelId),
|
|
444
|
+
};
|
|
445
|
+
}
|
|
446
|
+
|
|
447
|
+
return {
|
|
448
|
+
success: true,
|
|
449
|
+
data: {
|
|
450
|
+
query: { from: input.from ?? null, to: input.to ?? null, channelId: input.channelId ?? null },
|
|
451
|
+
messageCountsAllTime,
|
|
452
|
+
tokenStatsForRange,
|
|
453
|
+
},
|
|
454
|
+
};
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
// ==========================================================================
|
|
458
|
+
// extract
|
|
459
|
+
// ==========================================================================
|
|
460
|
+
|
|
461
|
+
private handleExtract(input: ExtractInput): ToolResult {
|
|
462
|
+
const channelId = this.resolveChannel(input.channelId);
|
|
463
|
+
const fromMs = parseIsoDate(input.from, 'from');
|
|
464
|
+
const toMs = parseIsoDate(input.to, 'to');
|
|
465
|
+
const limit = clampCount(input.limit, EXTRACT_DEFAULT_LIMIT, EXTRACT_MAX_LIMIT, 'limit');
|
|
466
|
+
const offset = clampCount(input.offset, 0, NATIVE_OFFSET_MAX, 'offset');
|
|
467
|
+
const format = input.format ?? 'text';
|
|
468
|
+
|
|
469
|
+
const cm = this.cm as ContextManager;
|
|
470
|
+
const result = cm.queryMessagesByTimeAndChannel({
|
|
471
|
+
fromMs,
|
|
472
|
+
toMs,
|
|
473
|
+
channelId,
|
|
474
|
+
limit,
|
|
475
|
+
offset,
|
|
476
|
+
});
|
|
477
|
+
|
|
478
|
+
return {
|
|
479
|
+
success: true,
|
|
480
|
+
data: {
|
|
481
|
+
matchedCount: result.matchedCount,
|
|
482
|
+
returned: result.messages.length,
|
|
483
|
+
messages: result.messages.map((msg) => projectMessage(msg, format)),
|
|
484
|
+
},
|
|
485
|
+
};
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
// ==========================================================================
|
|
489
|
+
// search
|
|
490
|
+
// ==========================================================================
|
|
491
|
+
|
|
492
|
+
private async handleSearch(input: SearchInput): Promise<ToolResult> {
|
|
493
|
+
if (!input.query) {
|
|
494
|
+
throw new Error('search requires a non-empty "query".');
|
|
495
|
+
}
|
|
496
|
+
const channelId = this.resolveChannel(input.channelId);
|
|
497
|
+
const fromMs = parseIsoDate(input.from, 'from');
|
|
498
|
+
const toMs = parseIsoDate(input.to, 'to');
|
|
499
|
+
const limit = clampCount(input.limit, SEARCH_DEFAULT_LIMIT, SEARCH_MAX_LIMIT, 'limit');
|
|
500
|
+
const maxScan = clampCount(input.maxScan, SEARCH_DEFAULT_MAX_SCAN, SEARCH_MAX_MAX_SCAN, 'maxScan');
|
|
501
|
+
const caseSensitive = input.caseSensitive ?? false;
|
|
502
|
+
const flags = caseSensitive ? '' : 'i';
|
|
503
|
+
|
|
504
|
+
// Validate regex SYNTAX up front — an invalid pattern is a clean tool
|
|
505
|
+
// error, not a crash mid-scan. This does NOT bound match TIME (a
|
|
506
|
+
// syntactically valid pattern can still backtrack catastrophically),
|
|
507
|
+
// which is why regex-mode matching itself runs on a worker below rather
|
|
508
|
+
// than here.
|
|
509
|
+
if (input.regex) {
|
|
510
|
+
try {
|
|
511
|
+
new RegExp(input.query, flags);
|
|
512
|
+
} catch (error) {
|
|
513
|
+
throw new Error(`Invalid regex "${input.query}": ${error instanceof Error ? error.message : String(error)}`);
|
|
514
|
+
}
|
|
515
|
+
}
|
|
516
|
+
const needle = caseSensitive ? input.query : input.query.toLowerCase();
|
|
517
|
+
|
|
518
|
+
const cm = this.cm as ContextManager;
|
|
519
|
+
// Fetch one more than maxScan so an oversized candidate window is
|
|
520
|
+
// detected up front (before any matching is attempted), rather than
|
|
521
|
+
// silently scanning maxScan candidates and returning as if that were the
|
|
522
|
+
// whole window. See the tool description's `truncated` contract.
|
|
523
|
+
const probe = cm.queryMessagesByTimeAndChannel({
|
|
524
|
+
fromMs,
|
|
525
|
+
toMs,
|
|
526
|
+
channelId,
|
|
527
|
+
limit: maxScan + 1,
|
|
528
|
+
offset: 0,
|
|
529
|
+
});
|
|
530
|
+
const truncated = probe.messages.length > maxScan;
|
|
531
|
+
const candidates = truncated ? probe.messages.slice(0, maxScan) : probe.messages;
|
|
532
|
+
|
|
533
|
+
if (input.regex) {
|
|
534
|
+
// Regex matching against caller-supplied patterns is ReDoS-shaped: a
|
|
535
|
+
// pathological pattern (e.g. `(a+)+$`) can take catastrophically long
|
|
536
|
+
// against one candidate string, and a synchronous RegExp.exec() on
|
|
537
|
+
// this thread would block the WHOLE framework's event loop — every
|
|
538
|
+
// agent's turns, health checks, timers — for as long as it runs, with
|
|
539
|
+
// no way to interrupt it from this same thread. Route matching
|
|
540
|
+
// through a worker thread instead, which can be forcibly terminated
|
|
541
|
+
// on a deadline. See search-regex-worker.ts's header for the full
|
|
542
|
+
// rationale.
|
|
543
|
+
const { matches, scanned } = await this.searchWithRegexWorker(candidates, input.query, flags, limit);
|
|
544
|
+
return { success: true, data: { scanned, candidatePoolSize: candidates.length, truncated, matches } };
|
|
545
|
+
}
|
|
546
|
+
|
|
547
|
+
// Plain substring search (String.prototype.indexOf) is inherently
|
|
548
|
+
// linear in input length — no ReDoS-equivalent risk — so it stays
|
|
549
|
+
// in-process. This is also the common case, so it pays no worker-spawn
|
|
550
|
+
// overhead.
|
|
551
|
+
const matches: SearchMatch[] = [];
|
|
552
|
+
let scanned = 0;
|
|
553
|
+
for (const msg of candidates) {
|
|
554
|
+
// Check the limit BEFORE doing any work for this candidate — not
|
|
555
|
+
// after pushing a match — so limit:0 (a valid clampCount value: it's
|
|
556
|
+
// >= 0) correctly yields zero matches instead of one. Checking
|
|
557
|
+
// post-push would always let through the match that first reaches
|
|
558
|
+
// the limit. Same fix mirrored in search-regex-worker.ts's loop.
|
|
559
|
+
if (matches.length >= limit) break;
|
|
560
|
+
scanned++;
|
|
561
|
+
const text = flattenContent(msg.content);
|
|
562
|
+
const hit = matchSubstring(text, needle, caseSensitive);
|
|
563
|
+
if (!hit) continue;
|
|
564
|
+
matches.push({
|
|
565
|
+
id: String(msg.id),
|
|
566
|
+
timestamp: msg.timestamp.toISOString(),
|
|
567
|
+
participant: msg.participant,
|
|
568
|
+
channelId: getChannelId(msg) ?? null,
|
|
569
|
+
snippet: snippetAround(text, hit.index, hit.length),
|
|
570
|
+
});
|
|
571
|
+
}
|
|
572
|
+
|
|
573
|
+
return {
|
|
574
|
+
success: true,
|
|
575
|
+
data: { scanned, candidatePoolSize: candidates.length, truncated, matches },
|
|
576
|
+
};
|
|
577
|
+
}
|
|
578
|
+
|
|
579
|
+
/**
|
|
580
|
+
* Run regex matching for `search` on a worker thread with a hard
|
|
581
|
+
* wall-clock deadline, so a catastrophically-backtracking pattern can be
|
|
582
|
+
* forcibly killed instead of hanging the framework. One worker per call
|
|
583
|
+
* (not per candidate — spawn overhead would dominate at scale; not a
|
|
584
|
+
* persistent pool — a fresh worker per call means one bad pattern can
|
|
585
|
+
* never contaminate a later search). Always terminated on the way out,
|
|
586
|
+
* success or failure, so nothing lingers.
|
|
587
|
+
*
|
|
588
|
+
* On timeout or a worker-side error this THROWS (caught by
|
|
589
|
+
* handleToolCall's try/catch, same as every other error path in this
|
|
590
|
+
* module) rather than returning an empty match list — a timed-out search
|
|
591
|
+
* must never be indistinguishable from a clean "no matches" result.
|
|
592
|
+
*/
|
|
593
|
+
private async searchWithRegexWorker(
|
|
594
|
+
candidates: StoredMessage[],
|
|
595
|
+
pattern: string,
|
|
596
|
+
flags: string,
|
|
597
|
+
limit: number,
|
|
598
|
+
): Promise<{ matches: SearchMatch[]; scanned: number }> {
|
|
599
|
+
const texts = candidates.map((msg) => flattenContent(msg.content));
|
|
600
|
+
let worker: Worker | undefined;
|
|
601
|
+
try {
|
|
602
|
+
const { matches: rawMatches, scanned } = await new Promise<{ matches: SearchWorkerMatch[]; scanned: number }>(
|
|
603
|
+
(resolve, reject) => {
|
|
604
|
+
worker = new Worker(SEARCH_WORKER_PATH, { workerData: { texts, pattern, flags, limit } });
|
|
605
|
+
const timer = setTimeout(() => {
|
|
606
|
+
reject(
|
|
607
|
+
new Error(
|
|
608
|
+
`search timed out after ${SEARCH_REGEX_TIMEOUT_MS}ms while matching regex "${pattern}" against ` +
|
|
609
|
+
`up to ${texts.length} candidate(s) — the pattern may be catastrophically slow (exponential ` +
|
|
610
|
+
'backtracking) against this data; try a simpler pattern, a literal substring search ' +
|
|
611
|
+
'(regex:false), or a smaller maxScan.',
|
|
612
|
+
),
|
|
613
|
+
);
|
|
614
|
+
}, SEARCH_REGEX_TIMEOUT_MS);
|
|
615
|
+
timer.unref?.();
|
|
616
|
+
worker.once('message', (msg: SearchWorkerMessage) => {
|
|
617
|
+
clearTimeout(timer);
|
|
618
|
+
if (msg.type === 'error') reject(new Error(msg.error));
|
|
619
|
+
else resolve({ matches: msg.matches, scanned: msg.scanned });
|
|
620
|
+
});
|
|
621
|
+
worker.once('error', (err) => {
|
|
622
|
+
clearTimeout(timer);
|
|
623
|
+
reject(err);
|
|
624
|
+
});
|
|
625
|
+
},
|
|
626
|
+
);
|
|
627
|
+
|
|
628
|
+
const matches: SearchMatch[] = rawMatches.map(({ candidateIndex, matchIndex, matchLength }) => {
|
|
629
|
+
const msg = candidates[candidateIndex]!;
|
|
630
|
+
const text = texts[candidateIndex]!;
|
|
631
|
+
return {
|
|
632
|
+
id: String(msg.id),
|
|
633
|
+
timestamp: msg.timestamp.toISOString(),
|
|
634
|
+
participant: msg.participant,
|
|
635
|
+
channelId: getChannelId(msg) ?? null,
|
|
636
|
+
snippet: snippetAround(text, matchIndex, matchLength),
|
|
637
|
+
};
|
|
638
|
+
});
|
|
639
|
+
return { matches, scanned };
|
|
640
|
+
} finally {
|
|
641
|
+
// Always kill the worker — whether it finished, errored, or is still
|
|
642
|
+
// stuck mid-backtrack when the deadline hit. terminate() on an
|
|
643
|
+
// already-exited worker is a harmless no-op.
|
|
644
|
+
if (worker) void worker.terminate().catch(() => {});
|
|
645
|
+
}
|
|
646
|
+
}
|
|
647
|
+
|
|
648
|
+
// ==========================================================================
|
|
649
|
+
// overview
|
|
650
|
+
// ==========================================================================
|
|
651
|
+
|
|
652
|
+
private handleOverview(input: OverviewInput): ToolResult {
|
|
653
|
+
const channelId = this.resolveChannel(input.channelId);
|
|
654
|
+
const fromMs = parseIsoDate(input.from, 'from');
|
|
655
|
+
const toMs = parseIsoDate(input.to, 'to');
|
|
656
|
+
assertOrderedRange(fromMs, toMs, input.from, input.to);
|
|
657
|
+
const limit = clampCount(input.limit, OVERVIEW_DEFAULT_LIMIT, OVERVIEW_MAX_LIMIT, 'limit');
|
|
658
|
+
const cm = this.cm as ContextManager;
|
|
659
|
+
|
|
660
|
+
let entries: TimeRangeSummaryEntry[];
|
|
661
|
+
if (input.level !== undefined) {
|
|
662
|
+
// Caller asked for one exact level: flat, no fold-reduction — they
|
|
663
|
+
// want that granularity, not the coarsest-available mix below.
|
|
664
|
+
entries = cm.getSummariesInRange({ fromMs, toMs, level: input.level });
|
|
665
|
+
} else {
|
|
666
|
+
// No level given: fetch every level overlapping the range, then
|
|
667
|
+
// reduce to only "current" (non-superseded) entries. getSummariesInRange
|
|
668
|
+
// does NOT filter by fold status — a folded child and the coarser
|
|
669
|
+
// parent it was folded into can BOTH independently overlap the query
|
|
670
|
+
// range and BOTH come back in `all` (see TimeRangeSummaryEntry.parentId's
|
|
671
|
+
// doc comment in context-manager).
|
|
672
|
+
//
|
|
673
|
+
// Two complementary checks, because an ancestor CHAIN can be broken:
|
|
674
|
+
// `listSummariesInRange` silently skips an entry whose source range no
|
|
675
|
+
// longer resolves (a pruned/redacted source message, or a viewFilter
|
|
676
|
+
// exclusion — a real, named failure mode elsewhere in
|
|
677
|
+
// autobiographical.ts's contiguousMergeCandidates). If that skipped
|
|
678
|
+
// entry is an INTERMEDIATE level, its children's parentId points at an
|
|
679
|
+
// id that never made it into `all`, so the immediate-parent check
|
|
680
|
+
// alone can't see past the gap and a since-superseded child survives
|
|
681
|
+
// alongside the coarser grandparent that already covers its span.
|
|
682
|
+
// 1. Immediate-parent check (cheap, covers the intact-chain case).
|
|
683
|
+
// 2. Containment sweep (backstop, robust to a broken chain): drop an
|
|
684
|
+
// entry if any OTHER fetched entry is STRICTLY coarser (higher
|
|
685
|
+
// level) and its span fully contains this one — that entry
|
|
686
|
+
// already covers the same messages regardless of whether the
|
|
687
|
+
// parentId pointer chain between them survived intact.
|
|
688
|
+
// EXCEPTION: a zero-width entry (e.startMs === e.endMs — a tiny
|
|
689
|
+
// chunk that landed entirely within one millisecond) that merely
|
|
690
|
+
// TOUCHES a coarser entry's boundary at exactly that one point
|
|
691
|
+
// trivially satisfies "f.startMs <= e.startMs && f.endMs >=
|
|
692
|
+
// e.endMs" without f actually being an ancestor — a genuinely
|
|
693
|
+
// unrelated, later-starting/earlier-ending coarser summary can
|
|
694
|
+
// coincidentally share one endpoint with it. A real (non-zero-
|
|
695
|
+
// width) entry can't trigger this false positive: satisfying
|
|
696
|
+
// containment on BOTH ends while only touching at one means f
|
|
697
|
+
// must still genuinely extend past the entry's other, non-touching
|
|
698
|
+
// side. A genuine same-point fold is still caught by the
|
|
699
|
+
// immediate-parentId check above when the chain is intact.
|
|
700
|
+
// O(n^2) in the fetched set size, which is fine at summary-archive
|
|
701
|
+
// scale (bounded by how much has been compressed into this range, not
|
|
702
|
+
// by raw message count).
|
|
703
|
+
const all = cm.getSummariesInRange({ fromMs, toMs });
|
|
704
|
+
const idsInSet = new Set(all.map((e) => e.id));
|
|
705
|
+
entries = all.filter((e) => {
|
|
706
|
+
if (e.parentId && idsInSet.has(e.parentId)) return false;
|
|
707
|
+
return !all.some((f) => {
|
|
708
|
+
if (f.id === e.id || f.level <= e.level) return false;
|
|
709
|
+
if (!(f.startMs <= e.startMs && f.endMs >= e.endMs)) return false;
|
|
710
|
+
if (e.startMs === e.endMs && (e.startMs === f.startMs || e.endMs === f.endMs)) return false;
|
|
711
|
+
return true;
|
|
712
|
+
});
|
|
713
|
+
});
|
|
714
|
+
}
|
|
715
|
+
entries.sort((a, b) => a.startMs - b.startMs);
|
|
716
|
+
|
|
717
|
+
// Per-summary channel/token stats — same call `stats` uses, scoped to
|
|
718
|
+
// just this entry's own source span.
|
|
719
|
+
const summarized: Array<{
|
|
720
|
+
startMs: number;
|
|
721
|
+
endMs: number;
|
|
722
|
+
summarized: true;
|
|
723
|
+
content: string;
|
|
724
|
+
level: number;
|
|
725
|
+
stats: ChannelTokenStats;
|
|
726
|
+
boundaryUncertain?: boolean;
|
|
727
|
+
}> = entries.map((entry) => ({
|
|
728
|
+
startMs: entry.startMs,
|
|
729
|
+
endMs: entry.endMs,
|
|
730
|
+
summarized: true as const,
|
|
731
|
+
content: entry.content,
|
|
732
|
+
level: entry.level,
|
|
733
|
+
stats: cm.getChannelTokenStats({ fromMs: entry.startMs, toMs: entry.endMs }),
|
|
734
|
+
}));
|
|
735
|
+
|
|
736
|
+
// Gap-fill: walk the sorted summary spans and probe seams wide enough
|
|
737
|
+
// to be worth a getChannelTokenStats call. A gap with real traffic
|
|
738
|
+
// becomes a synthetic summarized:false entry; an empty gap (nothing
|
|
739
|
+
// happened there) is skipped rather than cluttering the response.
|
|
740
|
+
//
|
|
741
|
+
// Cursor anchoring when `from` is omitted: NOT "now" — with zero
|
|
742
|
+
// summaries minted yet (first-ever call, or a PassthroughStrategy host
|
|
743
|
+
// where getSummariesInRange always returns []) that would make the
|
|
744
|
+
// whole walk a zero-width no-op and silently report entries:[] even
|
|
745
|
+
// though the store is full of messages. And even WITH summaries
|
|
746
|
+
// present, starting at the first summary's startMs silently hides any
|
|
747
|
+
// older unsummarized traffic that predates every summary (e.g. history
|
|
748
|
+
// from before compression was enabled). Anchor instead at the earlier
|
|
749
|
+
// of "the first summary's start" and "the earliest real message in the
|
|
750
|
+
// store" — falling back to Date.now() only when NEITHER exists, i.e.
|
|
751
|
+
// the store is genuinely empty, where entries:[] is the correct answer.
|
|
752
|
+
const rangeEnd = toMs ?? Date.now();
|
|
753
|
+
const firstEntryStartMs = entries[0]?.startMs;
|
|
754
|
+
let cursor: number;
|
|
755
|
+
if (fromMs !== undefined) {
|
|
756
|
+
cursor = fromMs;
|
|
757
|
+
} else {
|
|
758
|
+
const earliestMessageMs = this.earliestMessageMs(toMs);
|
|
759
|
+
cursor =
|
|
760
|
+
earliestMessageMs !== undefined && firstEntryStartMs !== undefined
|
|
761
|
+
? Math.min(earliestMessageMs, firstEntryStartMs)
|
|
762
|
+
: earliestMessageMs ?? firstEntryStartMs ?? Date.now();
|
|
763
|
+
}
|
|
764
|
+
// Effective query start — captured BEFORE the gap loop below mutates
|
|
765
|
+
// `cursor` (it walks forward past each entry's endMs). Needed by the
|
|
766
|
+
// boundary-tie correction pass further down to clamp its output to
|
|
767
|
+
// what the caller actually asked for (see the comment there): when
|
|
768
|
+
// `fromMs` is omitted this equals the resolved anchor above, not
|
|
769
|
+
// `-Infinity` — an open-ended query still has a concrete effective
|
|
770
|
+
// start, it's just not caller-supplied.
|
|
771
|
+
const rangeStart = cursor;
|
|
772
|
+
|
|
773
|
+
const gaps: Array<{ startMs: number; endMs: number; summarized: false; stats: ChannelTokenStats; boundaryUncertain?: boolean }> = [];
|
|
774
|
+
// True once `cursor` has been advanced past at least one real summary
|
|
775
|
+
// entry's endMs — only THEN is `cursor` itself a value that a summary
|
|
776
|
+
// already claims (and so needs a +1 nudge below to probe half-open).
|
|
777
|
+
// The very first gap (before any entry) sits against `fromMs` or an
|
|
778
|
+
// earliest-message anchor, neither of which any summary has claimed,
|
|
779
|
+
// so it must NOT be nudged — nudging it would skip a real message
|
|
780
|
+
// sitting exactly at the anchor.
|
|
781
|
+
let cursorIsEntryBoundary = false;
|
|
782
|
+
for (const entry of entries) {
|
|
783
|
+
// getChannelTokenStats's range is BOTH-ENDS INCLUSIVE (chronicle's
|
|
784
|
+
// gte/lte), so probing the raw [cursor, entry.startMs] pair would
|
|
785
|
+
// re-count the neighboring summaries' own boundary messages (ts ===
|
|
786
|
+
// cursor from the entry before, ts === entry.startMs from this one)
|
|
787
|
+
// as if they were unsummarized — fabricating a phantom gap between
|
|
788
|
+
// EVERY pair of adjacent summaries, even when they cover 100% of
|
|
789
|
+
// traffic with zero real gap between them. Narrow to a half-open
|
|
790
|
+
// probe instead, and only skip the call when narrowing collapsed the
|
|
791
|
+
// width to zero or negative (`probeTo < probeFrom` — an entry that
|
|
792
|
+
// starts exactly where the previous one ended, or crosses it).
|
|
793
|
+
//
|
|
794
|
+
// Deliberately NO minimum-width threshold beyond that: an earlier
|
|
795
|
+
// version skipped any seam narrower than a fixed few seconds as
|
|
796
|
+
// "bookkeeping noise", but that check ran BEFORE the
|
|
797
|
+
// stats.totalMessages > 0 check below that actually decides whether
|
|
798
|
+
// real traffic exists — so a query window or a between-summaries
|
|
799
|
+
// seam narrower than the threshold silently reported entries:[] even
|
|
800
|
+
// when full of real messages (verified repro: 300 real messages,
|
|
801
|
+
// zero summaries, a 4-second window). The totalMessages > 0 check
|
|
802
|
+
// that follows is the one that's actually correct; nothing should
|
|
803
|
+
// gate the probe ahead of it.
|
|
804
|
+
const probeFrom = cursorIsEntryBoundary ? cursor + 1 : cursor;
|
|
805
|
+
const probeTo = entry.startMs - 1;
|
|
806
|
+
if (probeTo >= probeFrom) {
|
|
807
|
+
const stats = cm.getChannelTokenStats({ fromMs: probeFrom, toMs: probeTo });
|
|
808
|
+
if (stats.totalMessages > 0) gaps.push({ startMs: probeFrom, endMs: probeTo, summarized: false, stats });
|
|
809
|
+
}
|
|
810
|
+
cursor = Math.max(cursor, entry.endMs);
|
|
811
|
+
cursorIsEntryBoundary = true;
|
|
812
|
+
}
|
|
813
|
+
{
|
|
814
|
+
// Trailing gap: rangeEnd is either the caller's own inclusive `to`
|
|
815
|
+
// bound or Date.now() — never a summary boundary — so only the LOWER
|
|
816
|
+
// end gets the half-open nudge (same reasoning as inside the loop).
|
|
817
|
+
// Same no-threshold reasoning as above.
|
|
818
|
+
const probeFrom = cursorIsEntryBoundary ? cursor + 1 : cursor;
|
|
819
|
+
const probeTo = rangeEnd;
|
|
820
|
+
if (probeTo >= probeFrom) {
|
|
821
|
+
const stats = cm.getChannelTokenStats({ fromMs: probeFrom, toMs: probeTo });
|
|
822
|
+
if (stats.totalMessages > 0) gaps.push({ startMs: probeFrom, endMs: probeTo, summarized: false, stats });
|
|
823
|
+
}
|
|
824
|
+
}
|
|
825
|
+
|
|
826
|
+
// Boundary-tie disambiguation: getChannelTokenStats operates on
|
|
827
|
+
// wall-clock TIME ranges and has no visibility into chronicle's
|
|
828
|
+
// sequence numbers, so whenever a message that's genuinely NOT part of
|
|
829
|
+
// a summary's source span happens to share the EXACT millisecond with
|
|
830
|
+
// that summary's own first/last covered message (rapid-fire appends —
|
|
831
|
+
// wall-clock ms resolution isn't unique, sequence is), the per-span
|
|
832
|
+
// aggregate calls above silently fold it into the summary's counts —
|
|
833
|
+
// and the adjacent gap probe (which deliberately EXCLUDES that same ms
|
|
834
|
+
// via the half-open nudge, on the assumption the boundary ms belongs
|
|
835
|
+
// entirely to the neighboring summary) has no way to see it at all: it
|
|
836
|
+
// doesn't just get miscounted, it vanishes from the response entirely.
|
|
837
|
+
//
|
|
838
|
+
// Resolved using TimeRangeSummaryEntry's firstSequence/lastSequence
|
|
839
|
+
// (chronicle's per-record sequence number — strictly monotonic, never
|
|
840
|
+
// ties, unlike wall-clock ms) against the ACTUAL messages at each
|
|
841
|
+
// boundary ms (queried directly via boundaryMessagesAt — cheap, at
|
|
842
|
+
// most a handful of messages share one millisecond in practice).
|
|
843
|
+
//
|
|
844
|
+
// A "foreign" message found this way either belongs to ANOTHER fetched
|
|
845
|
+
// summary entry (its sequence falls in THAT entry's own range — its
|
|
846
|
+
// own inclusive-range getChannelTokenStats call already counts it
|
|
847
|
+
// correctly, so this entry just needs the erroneous count subtracted,
|
|
848
|
+
// nothing added elsewhere) or is genuinely unsummarized (belongs to the
|
|
849
|
+
// adjacent gap, extending it — or creating a new point-width gap
|
|
850
|
+
// fragment if no gap was probed there at all, which is exactly the
|
|
851
|
+
// "vanishes entirely" failure mode this fix exists for).
|
|
852
|
+
//
|
|
853
|
+
// MESSAGE COUNT/membership is corrected exactly — we know precisely
|
|
854
|
+
// which raw messages are foreign and their channelId. TOKEN weight for
|
|
855
|
+
// those specific messages is deliberately NOT recomputed:
|
|
856
|
+
// getChannelTokenStats can only report tokens for a TIME range, never
|
|
857
|
+
// for an arbitrary message subset, and reimplementing token estimation
|
|
858
|
+
// locally would silently drift from context-manager's own (mutable,
|
|
859
|
+
// calibration-adjusted) estimate — worse than admitting the gap. Any
|
|
860
|
+
// entry this pass touches is marked `boundaryUncertain: true` so a
|
|
861
|
+
// caller knows its tokensEstimate may be off by the small, bounded
|
|
862
|
+
// weight of the message(s) that moved, even though messageCount there
|
|
863
|
+
// is exact.
|
|
864
|
+
// Messages already routed into a gap by an EARLIER entry's pass in this
|
|
865
|
+
// same loop — without this, two entries that both touch the same
|
|
866
|
+
// boundary millisecond (e.g. summary A ends and summary B starts at the
|
|
867
|
+
// exact same ms, with one genuinely-unsummarized stray message also at
|
|
868
|
+
// that ms) each independently decide "not mine, not any other fetched
|
|
869
|
+
// SUMMARY's either" and BOTH call mergeForeignIntoGap for the same
|
|
870
|
+
// stray — mergeForeignIntoGap's own gap-lookup only checks for a gap
|
|
871
|
+
// immediately adjacent to ITS OWN entry's boundary, so it can't see the
|
|
872
|
+
// point-width gap the other entry's pass just created one step away,
|
|
873
|
+
// and the same message gets double-counted into two separate gap
|
|
874
|
+
// entries. Tracking ids already merged (not just already-seen) across
|
|
875
|
+
// the whole loop — not just within one entry's own pass — closes this.
|
|
876
|
+
const alreadyMergedForeignIds = new Set<string>();
|
|
877
|
+
for (let i = 0; i < entries.length; i++) {
|
|
878
|
+
const entry = entries[i]!;
|
|
879
|
+
const own = summarized[i]!;
|
|
880
|
+
// Defensive: firstSequence/lastSequence are new fields on
|
|
881
|
+
// TimeRangeSummaryEntry (added alongside this correction pass) — if
|
|
882
|
+
// this module ever ends up running against an OLDER published
|
|
883
|
+
// context-manager whose getSummariesInRange doesn't populate them
|
|
884
|
+
// (undefined, not a type error at runtime), every comparison below
|
|
885
|
+
// would silently evaluate false, foreignById would always be empty,
|
|
886
|
+
// and this whole pass would silently no-op — resurrecting the exact
|
|
887
|
+
// boundary-tie bug it exists to fix, with no error and no
|
|
888
|
+
// boundaryUncertain signal. Skip explicitly instead of trusting the
|
|
889
|
+
// type system alone.
|
|
890
|
+
if (typeof entry.firstSequence !== 'number' || typeof entry.lastSequence !== 'number') continue;
|
|
891
|
+
const boundaryMsList = entry.startMs === entry.endMs ? [entry.startMs] : [entry.startMs, entry.endMs];
|
|
892
|
+
const foreignById = new Map<string, StoredMessage>();
|
|
893
|
+
for (const ms of boundaryMsList) {
|
|
894
|
+
for (const m of this.boundaryMessagesAt(ms)) {
|
|
895
|
+
if (m.sequence < entry.firstSequence || m.sequence > entry.lastSequence) {
|
|
896
|
+
foreignById.set(String(m.id), m);
|
|
897
|
+
}
|
|
898
|
+
}
|
|
899
|
+
}
|
|
900
|
+
if (foreignById.size === 0) continue;
|
|
901
|
+
|
|
902
|
+
const genuinelyUnsummarized: StoredMessage[] = [];
|
|
903
|
+
for (const m of foreignById.values()) {
|
|
904
|
+
const owner = entries.find(
|
|
905
|
+
(e) =>
|
|
906
|
+
e.id !== entry.id &&
|
|
907
|
+
typeof e.firstSequence === 'number' &&
|
|
908
|
+
typeof e.lastSequence === 'number' &&
|
|
909
|
+
m.sequence >= e.firstSequence &&
|
|
910
|
+
m.sequence <= e.lastSequence,
|
|
911
|
+
);
|
|
912
|
+
if (!owner && !alreadyMergedForeignIds.has(String(m.id))) genuinelyUnsummarized.push(m);
|
|
913
|
+
}
|
|
914
|
+
|
|
915
|
+
own.stats = subtractMessagesFromStats(own.stats, [...foreignById.values()]);
|
|
916
|
+
own.boundaryUncertain = true;
|
|
917
|
+
|
|
918
|
+
// getSummariesInRange intentionally returns WHOLE summaries that
|
|
919
|
+
// overlap the query — a summary's own startMs/endMs can legitimately
|
|
920
|
+
// extend past the caller's requested [rangeStart, rangeEnd] (that's
|
|
921
|
+
// correct: the summary itself, and its corrected messageCount above,
|
|
922
|
+
// belong in the response either way). But a correction-pass GAP
|
|
923
|
+
// anchored at entry.startMs/entry.endMs must NOT be added when that
|
|
924
|
+
// boundary itself falls outside the query window — the ordinary gap
|
|
925
|
+
// walk elsewhere in this function never synthesizes a gap outside
|
|
926
|
+
// [rangeStart, rangeEnd], and this pass shouldn't either. Without
|
|
927
|
+
// this check, a narrow-window query landing entirely inside an
|
|
928
|
+
// already-summarized span could still surface phantom
|
|
929
|
+
// summarized:false rows anchored at that summary's FAR boundaries,
|
|
930
|
+
// outside anything the caller asked about.
|
|
931
|
+
const before = genuinelyUnsummarized.filter((m) => m.sequence < entry.firstSequence);
|
|
932
|
+
const after = genuinelyUnsummarized.filter((m) => m.sequence > entry.lastSequence);
|
|
933
|
+
const startInRange = entry.startMs >= rangeStart && entry.startMs <= rangeEnd;
|
|
934
|
+
const endInRange = entry.endMs >= rangeStart && entry.endMs <= rangeEnd;
|
|
935
|
+
if (before.length > 0 && startInRange) {
|
|
936
|
+
mergeForeignIntoGap(gaps, entry.startMs, 'before', before);
|
|
937
|
+
for (const m of before) alreadyMergedForeignIds.add(String(m.id));
|
|
938
|
+
}
|
|
939
|
+
if (after.length > 0 && endInRange) {
|
|
940
|
+
mergeForeignIntoGap(gaps, entry.endMs, 'after', after);
|
|
941
|
+
for (const m of after) alreadyMergedForeignIds.add(String(m.id));
|
|
942
|
+
}
|
|
943
|
+
}
|
|
944
|
+
|
|
945
|
+
// Merge summary-backed + gap-filled spans, apply the channel filter
|
|
946
|
+
// (drop any span with zero messages from the requested channel — but
|
|
947
|
+
// keep a kept summary's `content` unfiltered: compression doesn't chunk
|
|
948
|
+
// per-channel, so the text may legitimately describe other channels
|
|
949
|
+
// too), sort, and project to the response shape.
|
|
950
|
+
const merged = [...summarized, ...gaps].sort((a, b) => a.startMs - b.startMs);
|
|
951
|
+
const filtered = channelId
|
|
952
|
+
? merged.filter((e) => e.stats.byChannel.some((c) => c.channelId === channelId))
|
|
953
|
+
: merged;
|
|
954
|
+
|
|
955
|
+
// Cap the final entry list — mirrors search's truncated:true contract.
|
|
956
|
+
// Keeps the MOST RECENT `limit` spans when the cap is hit (a browse
|
|
957
|
+
// tool defaulting to "what's happened lately" should keep the tail, not
|
|
958
|
+
// silently cut off at the oldest end) while the survivors stay in their
|
|
959
|
+
// own chronological (oldest-first) order within the response, matching
|
|
960
|
+
// extract's/search's own ordering convention.
|
|
961
|
+
const totalSpans = filtered.length;
|
|
962
|
+
const truncated = totalSpans > limit;
|
|
963
|
+
// NOT `filtered.slice(-limit)`: `slice(-0)` is `slice(+0)` (negative
|
|
964
|
+
// zero coerces away), so `limit: 0` — a value clampCount explicitly
|
|
965
|
+
// accepts — would slice from index 0 and return the ENTIRE array while
|
|
966
|
+
// still reporting truncated:true. Same bug class already fixed twice
|
|
967
|
+
// elsewhere in this project (search's/extract's own limit:0 bugs);
|
|
968
|
+
// `filtered.length - limit` has no such cliff, at limit:0 it's
|
|
969
|
+
// `slice(filtered.length)` -> `[]`, the honest answer.
|
|
970
|
+
const capped = truncated ? filtered.slice(filtered.length - limit) : filtered;
|
|
971
|
+
|
|
972
|
+
return {
|
|
973
|
+
success: true,
|
|
974
|
+
data: {
|
|
975
|
+
range: { from: input.from ?? null, to: input.to ?? null },
|
|
976
|
+
// Only meaningful as a single number when `level` was pinned: the
|
|
977
|
+
// default (fold-reduced) result mixes spans at whatever level each
|
|
978
|
+
// one's own coarsest-available summary happens to be, so reporting
|
|
979
|
+
// a single `level` there would be actively misleading (an agent
|
|
980
|
+
// could see level:3 while looking at an L1 entry). Each summarized
|
|
981
|
+
// entry already carries its own accurate `level`; the top-level
|
|
982
|
+
// field instead reports the ceiling as `maxLevelAvailable` in that
|
|
983
|
+
// case, distinctly named so it can't be mistaken for a uniform
|
|
984
|
+
// per-entry level.
|
|
985
|
+
...(input.level !== undefined ? { level: input.level } : { maxLevelAvailable: cm.getMaxSummaryLevel() }),
|
|
986
|
+
truncated,
|
|
987
|
+
totalSpans,
|
|
988
|
+
entries: capped.map((e) => {
|
|
989
|
+
// When channelId narrows the response, the numbers an agent will
|
|
990
|
+
// actually reason over (messageCount/tokensEstimate) are scoped
|
|
991
|
+
// to THAT channel — not the whole span, which can otherwise
|
|
992
|
+
// include far more traffic from other channels sharing the same
|
|
993
|
+
// summarized/gap span. This is a deliberate departure from
|
|
994
|
+
// `stats`, which keeps its top-level totals whole-range and only
|
|
995
|
+
// narrows the byChannel breakdown (documented in handleStats) —
|
|
996
|
+
// here, an agent that asked to narrow by channel is reasoning
|
|
997
|
+
// about that channel specifically, so the whole-span totals stay
|
|
998
|
+
// available under distinctly-named spanMessageCount/
|
|
999
|
+
// spanTokensEstimate instead of silently double-serving as both.
|
|
1000
|
+
const channelStats = channelId ? e.stats.byChannel.find((c) => c.channelId === channelId) : undefined;
|
|
1001
|
+
return {
|
|
1002
|
+
from: new Date(e.startMs).toISOString(),
|
|
1003
|
+
to: new Date(e.endMs).toISOString(),
|
|
1004
|
+
summarized: e.summarized,
|
|
1005
|
+
...(e.summarized ? { content: e.content, level: e.level } : {}),
|
|
1006
|
+
// Set only when this entry's boundary-tie disambiguation (see
|
|
1007
|
+
// the correction pass above) found and corrected a
|
|
1008
|
+
// millisecond-timestamp collision at its edge. messageCount is
|
|
1009
|
+
// still exact there; tokensEstimate may be off by the small,
|
|
1010
|
+
// bounded weight of the message(s) that moved.
|
|
1011
|
+
...(e.boundaryUncertain ? { boundaryUncertain: true } : {}),
|
|
1012
|
+
messageCount: channelStats ? channelStats.messages : e.stats.totalMessages,
|
|
1013
|
+
tokensEstimate: channelStats ? channelStats.tokensEstimate : e.stats.totalTokensEstimate,
|
|
1014
|
+
...(channelId ? { spanMessageCount: e.stats.totalMessages, spanTokensEstimate: e.stats.totalTokensEstimate } : {}),
|
|
1015
|
+
byChannel: e.stats.byChannel,
|
|
1016
|
+
};
|
|
1017
|
+
}),
|
|
1018
|
+
},
|
|
1019
|
+
};
|
|
1020
|
+
}
|
|
1021
|
+
}
|
|
1022
|
+
|
|
1023
|
+
// ============================================================================
|
|
1024
|
+
// Helpers
|
|
1025
|
+
// ============================================================================
|
|
1026
|
+
|
|
1027
|
+
/** Parse an ISO 8601 date string to Unix ms. Throws a clear error (not a
|
|
1028
|
+
* crash) on an unparseable string; returns undefined for an omitted field
|
|
1029
|
+
* (open-ended bound). */
|
|
1030
|
+
function parseIsoDate(value: string | undefined, field: string): number | undefined {
|
|
1031
|
+
if (value === undefined) return undefined;
|
|
1032
|
+
const ms = Date.parse(value);
|
|
1033
|
+
if (Number.isNaN(ms)) {
|
|
1034
|
+
throw new Error(`Invalid ISO 8601 date for "${field}": ${JSON.stringify(value)}`);
|
|
1035
|
+
}
|
|
1036
|
+
return ms;
|
|
1037
|
+
}
|
|
1038
|
+
|
|
1039
|
+
/**
|
|
1040
|
+
* Reject an inverted range (`from` after `to`) with a clear error instead of
|
|
1041
|
+
* silently matching nothing. Used by `overview`, where "found nothing" is
|
|
1042
|
+
* reported as `entries: []` — the same shape a caller sees for a genuinely
|
|
1043
|
+
* quiet span — so a simple from/to mixup should never be allowed to look
|
|
1044
|
+
* like "you have no history here" instead of the input mistake it is.
|
|
1045
|
+
*/
|
|
1046
|
+
function assertOrderedRange(fromMs: number | undefined, toMs: number | undefined, fromRaw: string | undefined, toRaw: string | undefined): void {
|
|
1047
|
+
if (fromMs !== undefined && toMs !== undefined && fromMs > toMs) {
|
|
1048
|
+
throw new Error(`"from" (${fromRaw}) must not be after "to" (${toRaw}).`);
|
|
1049
|
+
}
|
|
1050
|
+
}
|
|
1051
|
+
|
|
1052
|
+
/**
|
|
1053
|
+
* Validate a pagination-ish numeric input (limit/offset/maxScan) and clamp
|
|
1054
|
+
* it to an upper bound. `Math.min(value, max)` alone is NOT sufficient here:
|
|
1055
|
+
* it only enforces an upper bound, so `Math.min(-1, 200)` is `-1`, not a
|
|
1056
|
+
* sane value — a negative limit/offset/maxScan would sail straight past the
|
|
1057
|
+
* "hard cap" and reach a native chronicle call expecting an unsigned
|
|
1058
|
+
* pagination argument (observed: `extract({limit:-1})` returning every
|
|
1059
|
+
* message in the store instead of being capped). Rejects (rather than
|
|
1060
|
+
* silently clamping) anything that isn't a finite non-negative integer, so
|
|
1061
|
+
* a caller mistake is surfaced as a clean tool error instead of silently
|
|
1062
|
+
* doing something other than what was asked.
|
|
1063
|
+
*/
|
|
1064
|
+
function clampCount(value: number | undefined, def: number, max: number, field: string): number {
|
|
1065
|
+
if (value === undefined) return def;
|
|
1066
|
+
if (typeof value !== 'number' || !Number.isFinite(value)) {
|
|
1067
|
+
throw new Error(`"${field}" must be a finite number, got ${JSON.stringify(value)}.`);
|
|
1068
|
+
}
|
|
1069
|
+
if (!Number.isInteger(value)) {
|
|
1070
|
+
throw new Error(`"${field}" must be an integer, got ${value}.`);
|
|
1071
|
+
}
|
|
1072
|
+
if (value < 0) {
|
|
1073
|
+
throw new Error(`"${field}" must be >= 0, got ${value}.`);
|
|
1074
|
+
}
|
|
1075
|
+
return Math.min(value, max);
|
|
1076
|
+
}
|
|
1077
|
+
|
|
1078
|
+
/** channelId lives at metadata.external.channelId — same path context-manager's
|
|
1079
|
+
* own native index reads from (message-store.ts's CHANNEL_FIELD). Metadata's
|
|
1080
|
+
* index-signature type means this needs an explicit narrow, same as
|
|
1081
|
+
* MessageStore.getChannelTokenStats does internally. */
|
|
1082
|
+
/**
|
|
1083
|
+
* Two channel-id metadata shapes exist in the wild: `metadata.channelId`
|
|
1084
|
+
* (what agent-framework's real MCPL ingestion — `handleMcplChannelIncoming`
|
|
1085
|
+
* — actually writes) and `metadata.external.channelId` (an older
|
|
1086
|
+
* convention). context-manager 0.9.1+ indexes and queries BOTH (see
|
|
1087
|
+
* MessageStore.extractChannelId there), so a channel-filtered result can
|
|
1088
|
+
* come from either shape — check the same order here so a displayed
|
|
1089
|
+
* `channelId` never reads `null` on a message that was, in fact, matched by
|
|
1090
|
+
* channel.
|
|
1091
|
+
*/
|
|
1092
|
+
function getChannelId(msg: StoredMessage): string | undefined {
|
|
1093
|
+
const metadata = msg.metadata as { channelId?: unknown; external?: { channelId?: unknown } } | undefined;
|
|
1094
|
+
if (typeof metadata?.channelId === 'string') return metadata.channelId;
|
|
1095
|
+
return typeof metadata?.external?.channelId === 'string' ? metadata.external.channelId : undefined;
|
|
1096
|
+
}
|
|
1097
|
+
|
|
1098
|
+
// ============================================================================
|
|
1099
|
+
// overview: boundary-tie correction helpers (see handleOverview)
|
|
1100
|
+
// ============================================================================
|
|
1101
|
+
|
|
1102
|
+
/**
|
|
1103
|
+
* Subtract the exact per-channel MESSAGE COUNT of `foreign` from `stats` —
|
|
1104
|
+
* used to correct a summary entry's own stats once boundary-tie
|
|
1105
|
+
* disambiguation finds messages counted in error (see handleOverview's
|
|
1106
|
+
* correction pass). Deliberately leaves totalTokensEstimate/
|
|
1107
|
+
* byChannel[].tokensEstimate untouched — see that pass's header comment
|
|
1108
|
+
* for why an exact token subtraction isn't achievable from the available
|
|
1109
|
+
* API.
|
|
1110
|
+
*/
|
|
1111
|
+
function subtractMessagesFromStats(stats: ChannelTokenStats, foreign: StoredMessage[]): ChannelTokenStats {
|
|
1112
|
+
if (foreign.length === 0) return stats;
|
|
1113
|
+
const countByChannel = new Map<string, number>();
|
|
1114
|
+
for (const m of foreign) {
|
|
1115
|
+
const ch = getChannelId(m);
|
|
1116
|
+
if (ch === undefined) continue; // unchanneled messages aren't tracked in byChannel at all
|
|
1117
|
+
countByChannel.set(ch, (countByChannel.get(ch) ?? 0) + 1);
|
|
1118
|
+
}
|
|
1119
|
+
const byChannel = stats.byChannel
|
|
1120
|
+
.map((c) => ({ ...c, messages: Math.max(0, c.messages - (countByChannel.get(c.channelId) ?? 0)) }))
|
|
1121
|
+
.filter((c) => c.messages > 0);
|
|
1122
|
+
return {
|
|
1123
|
+
totalMessages: Math.max(0, stats.totalMessages - foreign.length),
|
|
1124
|
+
totalTokensEstimate: stats.totalTokensEstimate,
|
|
1125
|
+
byChannel,
|
|
1126
|
+
};
|
|
1127
|
+
}
|
|
1128
|
+
|
|
1129
|
+
/**
|
|
1130
|
+
* Exact per-channel MESSAGE COUNT for a small raw message list, with
|
|
1131
|
+
* tokensEstimate left at 0 (unknown — see handleOverview's boundary-tie
|
|
1132
|
+
* correction pass). Only ever used to build a BRAND NEW synthetic gap
|
|
1133
|
+
* fragment out of boundary-tie foreign messages — a real query result
|
|
1134
|
+
* always goes through the real getChannelTokenStats aggregate instead.
|
|
1135
|
+
*/
|
|
1136
|
+
function statsFromMessages(messages: StoredMessage[]): ChannelTokenStats {
|
|
1137
|
+
const byChannel = new Map<string, number>();
|
|
1138
|
+
for (const m of messages) {
|
|
1139
|
+
const ch = getChannelId(m);
|
|
1140
|
+
if (ch === undefined) continue;
|
|
1141
|
+
byChannel.set(ch, (byChannel.get(ch) ?? 0) + 1);
|
|
1142
|
+
}
|
|
1143
|
+
return {
|
|
1144
|
+
totalMessages: messages.length,
|
|
1145
|
+
totalTokensEstimate: 0,
|
|
1146
|
+
byChannel: [...byChannel.entries()].map(([channelId, count]) => ({ channelId, messages: count, tokensEstimate: 0 })),
|
|
1147
|
+
};
|
|
1148
|
+
}
|
|
1149
|
+
|
|
1150
|
+
/** Add `b`'s counts into `a` — used to fold boundary-tie foreign messages
|
|
1151
|
+
* into an already-probed gap's stats. */
|
|
1152
|
+
function mergeStats(a: ChannelTokenStats, b: ChannelTokenStats): ChannelTokenStats {
|
|
1153
|
+
const byChannel = new Map<string, { messages: number; tokensEstimate: number }>();
|
|
1154
|
+
for (const c of a.byChannel) byChannel.set(c.channelId, { messages: c.messages, tokensEstimate: c.tokensEstimate });
|
|
1155
|
+
for (const c of b.byChannel) {
|
|
1156
|
+
const cur = byChannel.get(c.channelId) ?? { messages: 0, tokensEstimate: 0 };
|
|
1157
|
+
cur.messages += c.messages;
|
|
1158
|
+
cur.tokensEstimate += c.tokensEstimate;
|
|
1159
|
+
byChannel.set(c.channelId, cur);
|
|
1160
|
+
}
|
|
1161
|
+
return {
|
|
1162
|
+
totalMessages: a.totalMessages + b.totalMessages,
|
|
1163
|
+
totalTokensEstimate: a.totalTokensEstimate + b.totalTokensEstimate,
|
|
1164
|
+
byChannel: [...byChannel.entries()].map(([channelId, v]) => ({ channelId, ...v })),
|
|
1165
|
+
};
|
|
1166
|
+
}
|
|
1167
|
+
|
|
1168
|
+
/**
|
|
1169
|
+
* Fold boundary-tie `foreign` messages into whichever gap sits immediately
|
|
1170
|
+
* `side` of `boundaryMs`, extending that gap's span to include the tied
|
|
1171
|
+
* millisecond — or, if no such gap currently exists (the common case when
|
|
1172
|
+
* the seam had zero OTHER real traffic and so was never probed into
|
|
1173
|
+
* existence), create a new point-width gap fragment exactly at
|
|
1174
|
+
* `boundaryMs` to hold them. Always marks the result `boundaryUncertain:
|
|
1175
|
+
* true` (see handleOverview's boundary-tie correction pass for what that
|
|
1176
|
+
* means).
|
|
1177
|
+
*/
|
|
1178
|
+
function mergeForeignIntoGap(
|
|
1179
|
+
gaps: Array<{ startMs: number; endMs: number; summarized: false; stats: ChannelTokenStats; boundaryUncertain?: boolean }>,
|
|
1180
|
+
boundaryMs: number,
|
|
1181
|
+
side: 'before' | 'after',
|
|
1182
|
+
foreign: StoredMessage[],
|
|
1183
|
+
): void {
|
|
1184
|
+
const addend = statsFromMessages(foreign);
|
|
1185
|
+
const existing =
|
|
1186
|
+
side === 'before' ? gaps.find((g) => g.endMs === boundaryMs - 1) : gaps.find((g) => g.startMs === boundaryMs + 1);
|
|
1187
|
+
|
|
1188
|
+
if (existing) {
|
|
1189
|
+
existing.stats = mergeStats(existing.stats, addend);
|
|
1190
|
+
if (side === 'before') existing.endMs = boundaryMs;
|
|
1191
|
+
else existing.startMs = boundaryMs;
|
|
1192
|
+
existing.boundaryUncertain = true;
|
|
1193
|
+
} else {
|
|
1194
|
+
gaps.push({ startMs: boundaryMs, endMs: boundaryMs, summarized: false, stats: addend, boundaryUncertain: true });
|
|
1195
|
+
}
|
|
1196
|
+
}
|
|
1197
|
+
|
|
1198
|
+
function projectMessage(msg: StoredMessage, format: 'text' | 'raw'): Record<string, unknown> {
|
|
1199
|
+
return {
|
|
1200
|
+
id: String(msg.id),
|
|
1201
|
+
timestamp: msg.timestamp.toISOString(),
|
|
1202
|
+
participant: msg.participant,
|
|
1203
|
+
channelId: getChannelId(msg) ?? null,
|
|
1204
|
+
content: format === 'raw' ? msg.content : flattenContent(msg.content),
|
|
1205
|
+
};
|
|
1206
|
+
}
|
|
1207
|
+
|
|
1208
|
+
/** Flatten a message's content blocks to one short, readable string. Text
|
|
1209
|
+
* blocks verbatim; everything else (tool_use/tool_result/thinking/media) as
|
|
1210
|
+
* a short bracketed label — this is for agent readability and search
|
|
1211
|
+
* matching, not byte-faithful reconstruction. */
|
|
1212
|
+
function flattenContent(content: ContentBlock[]): string {
|
|
1213
|
+
return content.map(blockLabel).join(' ').trim();
|
|
1214
|
+
}
|
|
1215
|
+
|
|
1216
|
+
function blockLabel(block: ContentBlock): string {
|
|
1217
|
+
switch (block.type) {
|
|
1218
|
+
case 'text':
|
|
1219
|
+
return block.text;
|
|
1220
|
+
case 'tool_use':
|
|
1221
|
+
return `[tool_use: ${block.name}]`;
|
|
1222
|
+
case 'tool_result':
|
|
1223
|
+
return block.toolName ? `[tool_result: ${block.toolName}]` : '[tool_result]';
|
|
1224
|
+
case 'thinking':
|
|
1225
|
+
case 'redacted_thinking':
|
|
1226
|
+
return '[thinking]';
|
|
1227
|
+
case 'image':
|
|
1228
|
+
return '[image]';
|
|
1229
|
+
case 'generated_image':
|
|
1230
|
+
return '[image]';
|
|
1231
|
+
case 'document':
|
|
1232
|
+
return '[document]';
|
|
1233
|
+
case 'audio':
|
|
1234
|
+
return '[audio]';
|
|
1235
|
+
case 'video':
|
|
1236
|
+
return '[video]';
|
|
1237
|
+
default:
|
|
1238
|
+
// Exhaustiveness guard: a future ContentBlock variant falls back to a
|
|
1239
|
+
// generic label instead of a compile error at a call site far from here.
|
|
1240
|
+
return `[${(block as { type: string }).type}]`;
|
|
1241
|
+
}
|
|
1242
|
+
}
|
|
1243
|
+
|
|
1244
|
+
interface MatchHit {
|
|
1245
|
+
index: number;
|
|
1246
|
+
length: number;
|
|
1247
|
+
}
|
|
1248
|
+
|
|
1249
|
+
/** One `search` result entry — shared shape between the in-process substring
|
|
1250
|
+
* path and the worker-backed regex path. */
|
|
1251
|
+
interface SearchMatch {
|
|
1252
|
+
id: string;
|
|
1253
|
+
timestamp: string;
|
|
1254
|
+
participant: string;
|
|
1255
|
+
channelId: string | null;
|
|
1256
|
+
snippet: string;
|
|
1257
|
+
}
|
|
1258
|
+
|
|
1259
|
+
function matchSubstring(text: string, needle: string, caseSensitive: boolean): MatchHit | null {
|
|
1260
|
+
const haystack = caseSensitive ? text : text.toLowerCase();
|
|
1261
|
+
const index = haystack.indexOf(needle);
|
|
1262
|
+
if (index === -1) return null;
|
|
1263
|
+
return { index, length: needle.length };
|
|
1264
|
+
}
|
|
1265
|
+
|
|
1266
|
+
/** ~SNIPPET_CONTEXT_CHARS of surrounding context on each side of a match, or
|
|
1267
|
+
* the first ~SNIPPET_FALLBACK_CHARS of content when no meaningful match
|
|
1268
|
+
* position is given (kept simple on purpose — this is a readability aid,
|
|
1269
|
+
* not a highlighting engine). */
|
|
1270
|
+
function snippetAround(text: string, index: number, length: number): string {
|
|
1271
|
+
if (index < 0) return text.slice(0, SNIPPET_FALLBACK_CHARS);
|
|
1272
|
+
const start = Math.max(0, index - SNIPPET_CONTEXT_CHARS);
|
|
1273
|
+
const end = Math.min(text.length, index + length + SNIPPET_CONTEXT_CHARS);
|
|
1274
|
+
const prefix = start > 0 ? '…' : '';
|
|
1275
|
+
const suffix = end < text.length ? '…' : '';
|
|
1276
|
+
return prefix + text.slice(start, end) + suffix;
|
|
1277
|
+
}
|