@lacneu/openclaw-knowledge 3.1.2 → 3.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +368 -1
- package/README.md +131 -0
- package/dist/config.d.ts +4 -0
- package/dist/config.js +26 -0
- package/dist/config.js.map +1 -1
- package/dist/index.d.ts +61 -4
- package/dist/index.js +463 -50
- package/dist/index.js.map +1 -1
- package/dist/jina/classifier.d.ts +55 -0
- package/dist/jina/classifier.js +170 -0
- package/dist/jina/classifier.js.map +1 -0
- package/dist/jina/client.d.ts +30 -0
- package/dist/jina/client.js +131 -0
- package/dist/jina/client.js.map +1 -0
- package/dist/jina/errors.d.ts +42 -0
- package/dist/jina/errors.js +113 -0
- package/dist/jina/errors.js.map +1 -0
- package/dist/jina/reranker.d.ts +34 -0
- package/dist/jina/reranker.js +95 -0
- package/dist/jina/reranker.js.map +1 -0
- package/dist/jina/types.d.ts +78 -0
- package/dist/jina/types.js +12 -0
- package/dist/jina/types.js.map +1 -0
- package/dist/pgvector.d.ts +29 -0
- package/dist/pgvector.js +68 -0
- package/dist/pgvector.js.map +1 -1
- package/dist/router/heuristic.d.ts +29 -0
- package/dist/router/heuristic.js +104 -0
- package/dist/router/heuristic.js.map +1 -0
- package/dist/router/index.d.ts +33 -0
- package/dist/router/index.js +94 -0
- package/dist/router/index.js.map +1 -0
- package/dist/router/labels.d.ts +33 -0
- package/dist/router/labels.js +67 -0
- package/dist/router/labels.js.map +1 -0
- package/dist/router/types.d.ts +23 -0
- package/dist/router/types.js +7 -0
- package/dist/router/types.js.map +1 -0
- package/dist/tracing/events.d.ts +83 -0
- package/dist/tracing/events.js +86 -0
- package/dist/tracing/events.js.map +1 -0
- package/dist/types.d.ts +61 -1
- package/openclaw.plugin.json +97 -4
- package/package.json +3 -3
package/dist/index.js
CHANGED
|
@@ -3,67 +3,108 @@
|
|
|
3
3
|
// Queries two knowledge sources in parallel and injects relevant context
|
|
4
4
|
// into the agent's system prompt via `appendSystemContext`:
|
|
5
5
|
// 1. PostgreSQL pgvector — semantic vector search on document embeddings
|
|
6
|
+
// (optionally re-ordered by a Jina cross-encoder reranker)
|
|
6
7
|
// 2. LightRAG — knowledge graph with entity/relation multi-hop search
|
|
7
8
|
//
|
|
8
|
-
//
|
|
9
|
+
// As of v3.2.0:
|
|
10
|
+
// - An optional Jina-powered ROUTER decides which source(s) to call
|
|
11
|
+
// (or to skip retrieval entirely on heartbeats and meta-questions).
|
|
12
|
+
// - An optional Jina RERANKER re-orders pgvector results by relevance.
|
|
13
|
+
// Both features are opt-in via the `jina.*` config block and preserve
|
|
14
|
+
// pre-3.2.0 behavior when omitted.
|
|
15
|
+
//
|
|
16
|
+
// Hook: before_prompt_build (requires OpenClaw >= v2026.5.0)
|
|
9
17
|
// Depends on: pg (node-postgres)
|
|
10
18
|
//
|
|
11
19
|
// This is the canonical entry point for the plugin. Helpers live in sibling
|
|
12
|
-
// modules (`config.ts`, `embeddings.ts`, `pgvector.ts`, `lightrag.ts
|
|
13
|
-
//
|
|
20
|
+
// modules (`config.ts`, `embeddings.ts`, `pgvector.ts`, `lightrag.ts`,
|
|
21
|
+
// `jina/*`, `router/*`, `tracing/*`) so the business logic can be
|
|
22
|
+
// unit-tested without instantiating the full SDK.
|
|
14
23
|
import pg from "pg";
|
|
15
24
|
import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";
|
|
16
25
|
import { resolveConfig } from "./config.js";
|
|
17
26
|
import { embedQuery } from "./embeddings.js";
|
|
18
|
-
import { searchCollection, formatPgvectorResults } from "./pgvector.js";
|
|
27
|
+
import { searchCollection, formatPgvectorResults, rerankPgvectorResults, } from "./pgvector.js";
|
|
19
28
|
import { queryLightRAG, formatLightRAGResults } from "./lightrag.js";
|
|
29
|
+
import { decideRoute } from "./router/index.js";
|
|
30
|
+
import { JinaError, summarizeJinaError } from "./jina/errors.js";
|
|
31
|
+
import { emitEvent, emitTurnMetadata } from "./tracing/events.js";
|
|
20
32
|
// Re-export helpers so the test suite can import them directly without
|
|
21
33
|
// duplicating imports from every submodule.
|
|
22
34
|
export { resolveEnv, resolveConfig } from "./config.js";
|
|
23
35
|
export { embedQuery } from "./embeddings.js";
|
|
24
|
-
export { searchCollection, formatPgvectorResults } from "./pgvector.js";
|
|
36
|
+
export { searchCollection, formatPgvectorResults, rerankPgvectorResults, } from "./pgvector.js";
|
|
25
37
|
export { queryLightRAG, truncateLightRAG, formatLightRAGResults } from "./lightrag.js";
|
|
38
|
+
export { decideRoute } from "./router/index.js";
|
|
26
39
|
// ---------------------------------------------------------------------------
|
|
27
40
|
// Hook handler factory
|
|
28
|
-
//
|
|
29
|
-
// Extracted from `register` so tests can exercise the handler directly
|
|
30
|
-
// without mocking the full plugin API surface.
|
|
31
41
|
// ---------------------------------------------------------------------------
|
|
32
42
|
const MAX_CONSECUTIVE_ERRORS = 3;
|
|
33
43
|
const COOLDOWN_MS = 5 * 60 * 1000;
|
|
34
44
|
const MIN_QUERY_LENGTH = 3;
|
|
45
|
+
function newCooldown() {
|
|
46
|
+
return { consecutiveErrors: 0, cooldownUntil: 0 };
|
|
47
|
+
}
|
|
35
48
|
/**
|
|
36
49
|
* Build the `before_prompt_build` handler bound to a specific plugin state.
|
|
37
50
|
* Kept as a pure factory so the handler can be unit-tested with fake deps.
|
|
38
51
|
*/
|
|
39
52
|
export function createBeforePromptBuildHandler(deps) {
|
|
40
53
|
const { config, pool, logger } = deps;
|
|
41
|
-
// Per-instance state
|
|
42
|
-
//
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
54
|
+
// Per-instance cooldown state. Closed-over so two registrations of the
|
|
55
|
+
// hook never share counters.
|
|
56
|
+
const cooldowns = {
|
|
57
|
+
global: newCooldown(),
|
|
58
|
+
router: newCooldown(),
|
|
59
|
+
pgvector_reranker: newCooldown(),
|
|
60
|
+
};
|
|
61
|
+
return async function beforePromptBuild(event, ctx) {
|
|
46
62
|
if (!config.enabled)
|
|
47
63
|
return undefined;
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
if (Date.now() < cooldownUntil)
|
|
64
|
+
if (isInCooldown(cooldowns.global)) {
|
|
65
|
+
maybeResetCooldown(cooldowns.global, "global", logger);
|
|
66
|
+
if (isInCooldown(cooldowns.global))
|
|
52
67
|
return undefined;
|
|
53
|
-
consecutiveErrors = 0;
|
|
54
|
-
logger.info("openclaw-knowledge: resuming after cooldown");
|
|
55
68
|
}
|
|
56
|
-
const query =
|
|
69
|
+
const query = extractUserQuery(event);
|
|
57
70
|
if (!query || query.trim().length < MIN_QUERY_LENGTH)
|
|
58
71
|
return undefined;
|
|
72
|
+
emitTurnMetadata(logger, ctx?.runId, query.length);
|
|
73
|
+
// -----------------------------------------------------------------
|
|
74
|
+
// Router gate — decide which sources (if any) to consult.
|
|
75
|
+
// -----------------------------------------------------------------
|
|
76
|
+
const decision = await runRouterWithCooldown(config, ctx, query, cooldowns.router, logger);
|
|
77
|
+
// Project the abstract router decision onto the sources actually
|
|
78
|
+
// configured in this deployment. Without this projection, an
|
|
79
|
+
// exclusive route (e.g. LIGHTRAG_ONLY) on a single-source deployment
|
|
80
|
+
// (e.g. pgvector only) would produce zero tasks and strip context
|
|
81
|
+
// the deployment could otherwise have provided.
|
|
82
|
+
const effectiveRoute = projectRouteOnEnabledSources(decision.route, config.pgvectorEnabled, config.lightragEnabled);
|
|
83
|
+
emitEvent(logger, {
|
|
84
|
+
type: "router",
|
|
85
|
+
route: effectiveRoute,
|
|
86
|
+
reason: decision.reason,
|
|
87
|
+
score: decision.score,
|
|
88
|
+
queryLength: query.length,
|
|
89
|
+
trigger: ctx?.trigger,
|
|
90
|
+
});
|
|
91
|
+
if (effectiveRoute === "NONE")
|
|
92
|
+
return undefined;
|
|
93
|
+
// -----------------------------------------------------------------
|
|
94
|
+
// Source execution — guided by the route.
|
|
95
|
+
// -----------------------------------------------------------------
|
|
59
96
|
try {
|
|
60
97
|
const tasks = [];
|
|
61
|
-
if (
|
|
62
|
-
|
|
98
|
+
if (shouldUsePgvector(effectiveRoute) &&
|
|
99
|
+
config.pgvectorEnabled &&
|
|
100
|
+
pool) {
|
|
101
|
+
tasks.push(runPgvectorSource(pool, query, config, cooldowns.pgvector_reranker, logger));
|
|
63
102
|
}
|
|
64
|
-
if (config.lightragEnabled) {
|
|
103
|
+
if (shouldUseLightRAG(effectiveRoute) && config.lightragEnabled) {
|
|
65
104
|
tasks.push(runLightRAGSource(query, config));
|
|
66
105
|
}
|
|
106
|
+
if (tasks.length === 0)
|
|
107
|
+
return undefined;
|
|
67
108
|
const settled = await Promise.allSettled(tasks);
|
|
68
109
|
const sections = [];
|
|
69
110
|
let failedSources = 0;
|
|
@@ -82,14 +123,10 @@ export function createBeforePromptBuildHandler(deps) {
|
|
|
82
123
|
// cooldown tracking. A partial failure is fine — the other source's
|
|
83
124
|
// context is better than nothing.
|
|
84
125
|
if (failedSources > 0 && failedSources === tasks.length) {
|
|
85
|
-
|
|
86
|
-
if (consecutiveErrors >= MAX_CONSECUTIVE_ERRORS) {
|
|
87
|
-
cooldownUntil = Date.now() + COOLDOWN_MS;
|
|
88
|
-
logger.error(`openclaw-knowledge: ${consecutiveErrors} consecutive errors — cooling down 5 min`);
|
|
89
|
-
}
|
|
126
|
+
registerError(cooldowns.global, "global", logger);
|
|
90
127
|
return undefined;
|
|
91
128
|
}
|
|
92
|
-
consecutiveErrors = 0;
|
|
129
|
+
cooldowns.global.consecutiveErrors = 0;
|
|
93
130
|
if (sections.length === 0)
|
|
94
131
|
return undefined;
|
|
95
132
|
return {
|
|
@@ -105,24 +142,275 @@ export function createBeforePromptBuildHandler(deps) {
|
|
|
105
142
|
}
|
|
106
143
|
catch (err) {
|
|
107
144
|
// Catch-all: an unexpected crash must never propagate to the agent.
|
|
108
|
-
consecutiveErrors++;
|
|
109
145
|
const message = err instanceof Error ? err.message : String(err);
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
logger.error(`openclaw-knowledge: ${consecutiveErrors} consecutive errors — cooling down 5 min: ${message}`);
|
|
113
|
-
}
|
|
114
|
-
else {
|
|
115
|
-
logger.error(`openclaw-knowledge: ${message}`);
|
|
116
|
-
}
|
|
146
|
+
logger.error(`openclaw-knowledge: ${message}`);
|
|
147
|
+
registerError(cooldowns.global, "global", logger);
|
|
117
148
|
return undefined;
|
|
118
149
|
}
|
|
119
150
|
};
|
|
120
151
|
}
|
|
152
|
+
// ---------------------------------------------------------------------------
|
|
153
|
+
// Route gating helpers
|
|
154
|
+
// ---------------------------------------------------------------------------
|
|
155
|
+
function shouldUsePgvector(route) {
|
|
156
|
+
return route === "PGVECTOR_ONLY" || route === "ALL";
|
|
157
|
+
}
|
|
158
|
+
function shouldUseLightRAG(route) {
|
|
159
|
+
return route === "LIGHTRAG_ONLY" || route === "ALL";
|
|
160
|
+
}
|
|
161
|
+
/**
|
|
162
|
+
* Project a router decision onto the set of sources that are actually
|
|
163
|
+
* enabled in this deployment. This prevents "silent empty retrieval"
|
|
164
|
+
* when, for example, a pgvector-only deployment is told to use
|
|
165
|
+
* `LIGHTRAG_ONLY` for a multi-hop question — without this projection the
|
|
166
|
+
* task list would be empty and the agent would lose context that
|
|
167
|
+
* pgvector could have provided.
|
|
168
|
+
*
|
|
169
|
+
* Rules:
|
|
170
|
+
* - `NONE` → `NONE` (the router deliberately wants no retrieval).
|
|
171
|
+
* - `ALL` → `ALL` (downstream `shouldUseX` already skips disabled sources).
|
|
172
|
+
* - `PGVECTOR_ONLY` + pgvector disabled:
|
|
173
|
+
* - LightRAG available → `LIGHTRAG_ONLY` (best effort)
|
|
174
|
+
* - neither available → `NONE` (caller short-circuits)
|
|
175
|
+
* - `LIGHTRAG_ONLY` + LightRAG disabled: symmetric.
|
|
176
|
+
*
|
|
177
|
+
* Exported for unit testing.
|
|
178
|
+
*/
|
|
179
|
+
export function projectRouteOnEnabledSources(route, pgvectorEnabled, lightragEnabled) {
|
|
180
|
+
if (route === "NONE" || route === "ALL")
|
|
181
|
+
return route;
|
|
182
|
+
if (route === "PGVECTOR_ONLY") {
|
|
183
|
+
if (pgvectorEnabled)
|
|
184
|
+
return "PGVECTOR_ONLY";
|
|
185
|
+
return lightragEnabled ? "LIGHTRAG_ONLY" : "NONE";
|
|
186
|
+
}
|
|
187
|
+
// route === "LIGHTRAG_ONLY"
|
|
188
|
+
if (lightragEnabled)
|
|
189
|
+
return "LIGHTRAG_ONLY";
|
|
190
|
+
return pgvectorEnabled ? "PGVECTOR_ONLY" : "NONE";
|
|
191
|
+
}
|
|
192
|
+
/**
|
|
193
|
+
* Run `decideRoute` with isolated cooldown tracking. The router fails open
|
|
194
|
+
* by contract (returns ALL on any Jina error) — the cooldown here is only
|
|
195
|
+
* meant to suppress repeated log spam during a sustained outage, not to
|
|
196
|
+
* stop retrieval.
|
|
197
|
+
*/
|
|
198
|
+
async function runRouterWithCooldown(config, ctx, query, cooldown, logger) {
|
|
199
|
+
// Reset stale cooldown FIRST so we don't keep the classifier circuit
|
|
200
|
+
// open longer than necessary (the first turn after expiry must be
|
|
201
|
+
// able to attempt the classifier again).
|
|
202
|
+
maybeResetCooldown(cooldown, "router", logger);
|
|
203
|
+
// When the classifier circuit is open, we DOWNGRADE the mode to
|
|
204
|
+
// "heuristic" rather than short-circuiting to `ALL`. The cheap local
|
|
205
|
+
// rules (heartbeat / cron / memory trigger gating, meta-agent regex,
|
|
206
|
+
// CLI-trivial guard, keyword fast-paths) MUST still run during a Jina
|
|
207
|
+
// outage — otherwise a 5-min outage re-enables retrieval for every
|
|
208
|
+
// heartbeat, which is the exact waste the router is meant to prevent.
|
|
209
|
+
const classifierCircuitOpen = isInCooldown(cooldown);
|
|
210
|
+
const effectiveMode = classifierCircuitOpen
|
|
211
|
+
? "heuristic"
|
|
212
|
+
: config.routerMode;
|
|
213
|
+
try {
|
|
214
|
+
const d = await decideRoute({
|
|
215
|
+
enabled: config.routerEnabled,
|
|
216
|
+
mode: effectiveMode,
|
|
217
|
+
jinaApiKey: config.jinaApiKey,
|
|
218
|
+
classifierId: config.routerClassifierId || undefined,
|
|
219
|
+
}, {
|
|
220
|
+
query,
|
|
221
|
+
trigger: ctx?.trigger,
|
|
222
|
+
isCli: ctx?.messageProvider === "cli",
|
|
223
|
+
});
|
|
224
|
+
if (d.reason === "classifier_error") {
|
|
225
|
+
registerError(cooldown, "router", logger);
|
|
226
|
+
}
|
|
227
|
+
else if (!classifierCircuitOpen) {
|
|
228
|
+
// Only reset the error counter when we actually exercised the
|
|
229
|
+
// classifier path. While the circuit is open, heuristic-only
|
|
230
|
+
// successes must NOT prematurely declare the classifier healthy.
|
|
231
|
+
cooldown.consecutiveErrors = 0;
|
|
232
|
+
}
|
|
233
|
+
return d;
|
|
234
|
+
}
|
|
235
|
+
catch (err) {
|
|
236
|
+
// Defense in depth: decideRoute already handles Jina errors internally
|
|
237
|
+
// but a non-Jina exception (programmer error) lands here. Log only
|
|
238
|
+
// the error CLASS, never the message — the message could echo
|
|
239
|
+
// user content for some programmatic errors.
|
|
240
|
+
logger.error(`openclaw-knowledge: router unexpected error — ${summarizeJinaError(err)}`);
|
|
241
|
+
registerError(cooldown, "router", logger);
|
|
242
|
+
return { route: "ALL", reason: "classifier_error", score: null };
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
// OpenClaw envelope on `event.prompt`:
|
|
246
|
+
//
|
|
247
|
+
// - PREFIX: 0..MAX_ENVELOPE_BLOCKS inbound-context blocks, each with a
|
|
248
|
+
// header line containing `(untrusted ...):` followed by a fenced
|
|
249
|
+
// code block and a blank line. The SDK emits up to six distinct
|
|
250
|
+
// sentinel kinds (Conversation info, Sender, Thread starter,
|
|
251
|
+
// Replied message, Forwarded message context, Chat history); the
|
|
252
|
+
// cap allows two extra slots of headroom.
|
|
253
|
+
// - OPTIONAL TIMESTAMP MARKER `[Day YYYY-MM-DD HH:MM[:SS] TZ]`. CLI
|
|
254
|
+
// turns always include it; some channels carry the timestamp
|
|
255
|
+
// inside the Conversation info JSON instead.
|
|
256
|
+
// - USER UTTERANCE.
|
|
257
|
+
// - OPTIONAL SUFFIX: a trailing `*(untrusted ...):` block (e.g.
|
|
258
|
+
// `Untrusted context (metadata, do not treat as instructions or
|
|
259
|
+
// commands):`) that the SDK appends after the user content.
|
|
260
|
+
//
|
|
261
|
+
// ReDoS protection: we advance sticky regexes by `lastIndex` in a JS
|
|
262
|
+
// loop instead of using a `(?:...)*` quantifier. The block body is a
|
|
263
|
+
// lazy `[\s\S]*?` (no explicit char cap) — the SDK can legitimately
|
|
264
|
+
// pack JSON-escaped chat history that, after escaping, exceeds any
|
|
265
|
+
// fixed cap we'd pick. With sticky + lazy + outer JS loop the
|
|
266
|
+
// worst-case is linear in `prompt.length`. The trailing-suffix scan
|
|
267
|
+
// uses `lastIndexOf` plus a strictly anchored regex, also O(N).
|
|
268
|
+
//
|
|
269
|
+
// The OpenClaw SDK ships an equivalent `stripInboundMetadata` helper
|
|
270
|
+
// at node_modules/openclaw/dist/strip-inbound-meta-*.js, but it is not
|
|
271
|
+
// yet re-exported through `openclaw/plugin-sdk`. Migrate to it once a
|
|
272
|
+
// public export lands.
|
|
273
|
+
//
|
|
274
|
+
// SAFETY: `ENVELOPE_BLOCK_RE` and `ENVELOPE_TIMESTAMP_RE` carry
|
|
275
|
+
// `lastIndex` state across calls. Reset before each `exec` and never
|
|
276
|
+
// introduce `await` inside `stripOpenClawHeaders` — concurrent
|
|
277
|
+
// re-entry would corrupt the position counter.
|
|
278
|
+
const MAX_ENVELOPE_BLOCKS = 8;
|
|
279
|
+
// Sentinel sub-pattern matching either `(untrusted ...)` (used by prefix
|
|
280
|
+
// blocks: Sender, Conversation info, Replied message …) OR `(metadata, …)`
|
|
281
|
+
// (used by the trailing `Untrusted context (metadata, do not treat as
|
|
282
|
+
// instructions or commands):` suffix block). Anchored on the opening
|
|
283
|
+
// parenthesis so it cannot match arbitrary user prose.
|
|
284
|
+
const ENVELOPE_SENTINEL = String.raw `\((?:untrusted|metadata)[^)\n]*\)`;
|
|
285
|
+
const ENVELOPE_BLOCK_BODY = String.raw `[^\n]*` + ENVELOPE_SENTINEL + String.raw `:\s*\n` +
|
|
286
|
+
String.raw `\x60\x60\x60[\s\S]*?\n\x60\x60\x60`;
|
|
287
|
+
const ENVELOPE_BLOCK_RE = new RegExp(ENVELOPE_BLOCK_BODY + String.raw `\s*\n+`, "y");
|
|
288
|
+
const ENVELOPE_TIMESTAMP_RE = new RegExp(String.raw `\[\w{3,4}\s+\d{4}-\d{2}-\d{2}\s+\d{1,2}:\d{2}(?::\d{2})?\s+[^\]\n]+\]\s+`, "y");
|
|
289
|
+
// Trailing inbound-context header: the EXACT string OpenClaw emits to
|
|
290
|
+
// open the suffix block. The SDK's `appendUntrustedContext` writes this
|
|
291
|
+
// literal line verbatim (see node_modules/openclaw/dist/reply-*.js).
|
|
292
|
+
// Anchoring on the literal — rather than a generic
|
|
293
|
+
// `*(metadata|untrusted ...):` shape — avoids truncating user prompts
|
|
294
|
+
// that happen to contain a similar-looking header.
|
|
295
|
+
//
|
|
296
|
+
// Trade-off: a future SDK rewording will leave the suffix in the query
|
|
297
|
+
// until this constant is updated. That's acceptable: the strict match
|
|
298
|
+
// fails CLOSED (we keep too much) rather than open (we drop user
|
|
299
|
+
// content). Update this string in lockstep with the OpenClaw SDK.
|
|
300
|
+
const OPENCLAW_SUFFIX_HEADER = "Untrusted context (metadata, do not treat as instructions or commands):";
|
|
301
|
+
// Body markers the SDK emits IMMEDIATELY after the suffix header. A
|
|
302
|
+
// header line alone is not enough — a user can quote the header verbatim
|
|
303
|
+
// to ask about it. Requiring one of these markers right after the header
|
|
304
|
+
// distinguishes a real SDK suffix from a quoted reference.
|
|
305
|
+
const SUFFIX_BODY_MARKERS = [
|
|
306
|
+
"<<<EXTERNAL_UNTRUSTED_CONTENT",
|
|
307
|
+
"Source:",
|
|
308
|
+
"Content:",
|
|
309
|
+
"```",
|
|
310
|
+
];
|
|
311
|
+
/** Strip the trailing OpenClaw `Untrusted context` block when present. */
|
|
312
|
+
function stripTrailingSuffix(body) {
|
|
313
|
+
// `lastIndexOf` on a literal is O(N) and never backtracks.
|
|
314
|
+
const idx = body.lastIndexOf(OPENCLAW_SUFFIX_HEADER);
|
|
315
|
+
if (idx === -1)
|
|
316
|
+
return body;
|
|
317
|
+
// Header must sit alone on its line: preceded by `\n` (or string start)
|
|
318
|
+
// and followed only by whitespace before the next newline.
|
|
319
|
+
const before = idx === 0 ? "" : body[idx - 1];
|
|
320
|
+
if (before !== "\n" && before !== "")
|
|
321
|
+
return body;
|
|
322
|
+
const headerEnd = idx + OPENCLAW_SUFFIX_HEADER.length;
|
|
323
|
+
const newlineAfterHeader = body.indexOf("\n", headerEnd);
|
|
324
|
+
const restOfLine = newlineAfterHeader === -1 ? body.slice(headerEnd) : body.slice(headerEnd, newlineAfterHeader);
|
|
325
|
+
if (restOfLine.trim().length !== 0)
|
|
326
|
+
return body;
|
|
327
|
+
// The header alone is ambiguous (a user could be quoting it). Strip
|
|
328
|
+
// only when the body that follows begins with one of the markers the
|
|
329
|
+
// SDK actually emits.
|
|
330
|
+
const afterHeader = newlineAfterHeader === -1 ? "" : body.slice(newlineAfterHeader + 1).trimStart();
|
|
331
|
+
if (!SUFFIX_BODY_MARKERS.some((m) => afterHeader.startsWith(m)))
|
|
332
|
+
return body;
|
|
333
|
+
return body.slice(0, idx).trimEnd();
|
|
334
|
+
}
|
|
335
|
+
/**
|
|
336
|
+
* Strip the OpenClaw envelope (inbound-context blocks + timestamp
|
|
337
|
+
* marker) from the START of a raw user prompt and return only the user
|
|
338
|
+
* utterance. When no envelope is matched, the prompt is returned
|
|
339
|
+
* unchanged — the router then sees the full user content, which is the
|
|
340
|
+
* correct behavior for non-OpenClaw inputs.
|
|
341
|
+
*
|
|
342
|
+
* @internal exported for unit testing
|
|
343
|
+
*/
|
|
344
|
+
export function stripOpenClawHeaders(prompt) {
|
|
345
|
+
if (prompt.length === 0)
|
|
346
|
+
return prompt;
|
|
347
|
+
let pos = 0;
|
|
348
|
+
let blocksConsumed = 0;
|
|
349
|
+
let markerMatched = false;
|
|
350
|
+
// The SDK ships both orderings observed in production:
|
|
351
|
+
// - `block+ timestamp? user` (legacy CLI path)
|
|
352
|
+
// - `timestamp blocks+ user` (timestamp-first injection path)
|
|
353
|
+
// We tolerate any interleaving by attempting both regexes each turn
|
|
354
|
+
// and stopping when neither advances. The iteration cap is
|
|
355
|
+
// `MAX_ENVELOPE_BLOCKS + 2` to allow at most one leading and one
|
|
356
|
+
// trailing timestamp around the blocks.
|
|
357
|
+
for (let i = 0; i < MAX_ENVELOPE_BLOCKS + 2; i++) {
|
|
358
|
+
ENVELOPE_BLOCK_RE.lastIndex = pos;
|
|
359
|
+
if (ENVELOPE_BLOCK_RE.exec(prompt) !== null) {
|
|
360
|
+
pos = ENVELOPE_BLOCK_RE.lastIndex;
|
|
361
|
+
blocksConsumed++;
|
|
362
|
+
continue;
|
|
363
|
+
}
|
|
364
|
+
ENVELOPE_TIMESTAMP_RE.lastIndex = pos;
|
|
365
|
+
if (!markerMatched && ENVELOPE_TIMESTAMP_RE.exec(prompt) !== null) {
|
|
366
|
+
pos = ENVELOPE_TIMESTAMP_RE.lastIndex;
|
|
367
|
+
markerMatched = true;
|
|
368
|
+
continue;
|
|
369
|
+
}
|
|
370
|
+
break;
|
|
371
|
+
}
|
|
372
|
+
if (blocksConsumed === 0 && !markerMatched) {
|
|
373
|
+
// No prefix envelope detected — but a trailing suffix block may
|
|
374
|
+
// still be present (e.g. a webchat turn where only the
|
|
375
|
+
// `Untrusted context (metadata, ...)` block is appended). Probe
|
|
376
|
+
// for it before returning. When no suffix matches either, return
|
|
377
|
+
// the prompt unchanged.
|
|
378
|
+
const trailingStripped = stripTrailingSuffix(prompt);
|
|
379
|
+
return trailingStripped === prompt ? prompt : trailingStripped.trim();
|
|
380
|
+
}
|
|
381
|
+
return stripTrailingSuffix(prompt.slice(pos).trim());
|
|
382
|
+
}
|
|
383
|
+
/**
|
|
384
|
+
* Extract the user question from a `before_prompt_build` event.
|
|
385
|
+
*
|
|
386
|
+
* - When `event.prompt` is supplied (SDK 2026.5.0+), it is the
|
|
387
|
+
* authoritative source for the raw user utterance: this function
|
|
388
|
+
* strips the OpenClaw envelope and returns the result, even when the
|
|
389
|
+
* result is empty. `event.messages` is NOT consulted in this case
|
|
390
|
+
* because it carries the aggregated conversation window (multi-KB
|
|
391
|
+
* blob optimized for LLM consumption, not for plugin inspection).
|
|
392
|
+
* - When `event.prompt` is absent (older SDK), fall back to
|
|
393
|
+
* `extractQueryFromMessages(event.messages)`.
|
|
394
|
+
*
|
|
395
|
+
* The downstream `MIN_QUERY_LENGTH` check drops empty or near-empty
|
|
396
|
+
* results, so silently returning `""` from the `prompt` path is safe.
|
|
397
|
+
*
|
|
398
|
+
* @internal exported for unit testing
|
|
399
|
+
*/
|
|
400
|
+
export function extractUserQuery(event) {
|
|
401
|
+
if (typeof event.prompt === "string") {
|
|
402
|
+
return stripOpenClawHeaders(event.prompt);
|
|
403
|
+
}
|
|
404
|
+
return extractQueryFromMessages(event.messages);
|
|
405
|
+
}
|
|
121
406
|
/**
|
|
122
|
-
*
|
|
123
|
-
*
|
|
407
|
+
* Legacy extraction from `event.messages`, used only when the SDK does
|
|
408
|
+
* not populate `event.prompt`. On 2026.5.x+ the primary path is
|
|
409
|
+
* {@link extractUserQuery}.
|
|
410
|
+
*
|
|
411
|
+
* @internal exported for unit testing and backward compatibility
|
|
124
412
|
*/
|
|
125
|
-
function extractQueryFromMessages(messages) {
|
|
413
|
+
export function extractQueryFromMessages(messages) {
|
|
126
414
|
if (!Array.isArray(messages) || messages.length === 0)
|
|
127
415
|
return "";
|
|
128
416
|
for (let i = messages.length - 1; i >= 0; i--) {
|
|
@@ -142,16 +430,81 @@ function extractQueryFromMessages(messages) {
|
|
|
142
430
|
}
|
|
143
431
|
return "";
|
|
144
432
|
}
|
|
145
|
-
async function runPgvectorSource(pool, query, config) {
|
|
433
|
+
async function runPgvectorSource(pool, query, config, rerankerCooldown, logger) {
|
|
434
|
+
const startedAt = Date.now();
|
|
146
435
|
const vector = await embedQuery(query, config.geminiApiKey);
|
|
147
436
|
const searches = config.collections.map((col) => searchCollection(pool, col, vector, config.topK, config.scoreThreshold));
|
|
148
437
|
const allResults = (await Promise.all(searches)).flat();
|
|
149
438
|
allResults.sort((a, b) => b.score - a.score);
|
|
150
|
-
|
|
439
|
+
// Capture the recall size BEFORE the reranker runs. This is the
|
|
440
|
+
// number that monitors "how many candidates did pgvector find?"
|
|
441
|
+
// post-rerank, `data.length` may be smaller (truncated to topN), so
|
|
442
|
+
// we must not conflate the two in telemetry.
|
|
443
|
+
const rawCount = allResults.length;
|
|
444
|
+
// Optional cross-encoder rerank, gated on its own cooldown so a Jina
|
|
445
|
+
// hiccup doesn't poison the rest of the plugin.
|
|
446
|
+
//
|
|
447
|
+
// IMPORTANT: reset the cooldown BEFORE computing `rerankerActive`.
|
|
448
|
+
// Otherwise the first turn after the 5-min window expires would still
|
|
449
|
+
// see `consecutiveErrors=3`, skip the rerank, and only reset on the
|
|
450
|
+
// way out — leaving the operator with a "resuming" log message but a
|
|
451
|
+
// request that did NOT actually use the reranker.
|
|
452
|
+
maybeResetCooldown(rerankerCooldown, "pgvector_reranker", logger);
|
|
453
|
+
const rerankerActive = config.pgvectorRerankerEnabled &&
|
|
454
|
+
Boolean(config.jinaApiKey) &&
|
|
455
|
+
!isInCooldown(rerankerCooldown);
|
|
456
|
+
if (!rerankerActive) {
|
|
457
|
+
return {
|
|
458
|
+
source: "pgvector",
|
|
459
|
+
data: allResults,
|
|
460
|
+
rawCount,
|
|
461
|
+
reranked: false,
|
|
462
|
+
durationMs: Date.now() - startedAt,
|
|
463
|
+
};
|
|
464
|
+
}
|
|
465
|
+
try {
|
|
466
|
+
const reranked = await rerankPgvectorResults(allResults, {
|
|
467
|
+
apiKey: config.jinaApiKey,
|
|
468
|
+
query,
|
|
469
|
+
model: config.pgvectorRerankerModel,
|
|
470
|
+
topN: config.pgvectorRerankerTopN,
|
|
471
|
+
});
|
|
472
|
+
rerankerCooldown.consecutiveErrors = 0;
|
|
473
|
+
return {
|
|
474
|
+
source: "pgvector",
|
|
475
|
+
data: reranked,
|
|
476
|
+
rawCount,
|
|
477
|
+
reranked: true,
|
|
478
|
+
durationMs: Date.now() - startedAt,
|
|
479
|
+
};
|
|
480
|
+
}
|
|
481
|
+
catch (err) {
|
|
482
|
+
// Jina rerank failed → log a SANITIZED summary and fall back to
|
|
483
|
+
// cosine order. We do NOT log `err.message` because Jina error
|
|
484
|
+
// bodies (truncated to 200 chars in JinaApiError) may echo the
|
|
485
|
+
// query or document chunks — that would leak PHI / sensitive
|
|
486
|
+
// content into log files.
|
|
487
|
+
//
|
|
488
|
+
// We also intentionally DO NOT propagate the rejection to
|
|
489
|
+
// Promise.allSettled: pgvector retrieval itself succeeded, the
|
|
490
|
+
// reranker is bonus.
|
|
491
|
+
const isJina = err instanceof JinaError;
|
|
492
|
+
logger.error(`openclaw-knowledge: pgvector reranker failed — ${summarizeJinaError(err)}`);
|
|
493
|
+
if (isJina)
|
|
494
|
+
registerError(rerankerCooldown, "pgvector_reranker", logger);
|
|
495
|
+
return {
|
|
496
|
+
source: "pgvector",
|
|
497
|
+
data: allResults,
|
|
498
|
+
rawCount,
|
|
499
|
+
reranked: false,
|
|
500
|
+
durationMs: Date.now() - startedAt,
|
|
501
|
+
};
|
|
502
|
+
}
|
|
151
503
|
}
|
|
152
504
|
async function runLightRAGSource(query, config) {
|
|
505
|
+
const startedAt = Date.now();
|
|
153
506
|
const context = await queryLightRAG(config.lightragUrl, config.lightragApiKey, query, config.lightragQueryMode);
|
|
154
|
-
return { source: "lightrag", data: context };
|
|
507
|
+
return { source: "lightrag", data: context, durationMs: Date.now() - startedAt };
|
|
155
508
|
}
|
|
156
509
|
function renderSection(result, config, logger) {
|
|
157
510
|
if (result.source === "pgvector") {
|
|
@@ -159,7 +512,21 @@ function renderSection(result, config, logger) {
|
|
|
159
512
|
if (!formatted)
|
|
160
513
|
return null;
|
|
161
514
|
const topScore = result.data[0]?.score?.toFixed(2) ?? "n/a";
|
|
162
|
-
|
|
515
|
+
const rerankNote = result.reranked ? " [reranked]" : "";
|
|
516
|
+
logger.info(`openclaw-knowledge: pgvector — ${result.data.length} result(s)${rerankNote} (top: ${topScore})`);
|
|
517
|
+
emitEvent(logger, {
|
|
518
|
+
type: "pgvector",
|
|
519
|
+
collections: config.collections,
|
|
520
|
+
// `rawCount` is the recall size out of the vector index, captured
|
|
521
|
+
// BEFORE the reranker truncates to topN. `rerankedCount` is the
|
|
522
|
+
// final size that reaches the LLM (or `null` when the reranker
|
|
523
|
+
// is inactive). This split lets operators monitor recall vs.
|
|
524
|
+
// pruning independently.
|
|
525
|
+
rawCount: result.rawCount,
|
|
526
|
+
rerankedCount: result.reranked ? result.data.length : null,
|
|
527
|
+
topScore: result.data[0]?.score ?? null,
|
|
528
|
+
durationMs: result.durationMs,
|
|
529
|
+
});
|
|
163
530
|
return "### Document Search Results (pgvector)\n" + formatted;
|
|
164
531
|
}
|
|
165
532
|
if (result.source === "lightrag") {
|
|
@@ -167,15 +534,46 @@ function renderSection(result, config, logger) {
|
|
|
167
534
|
if (!formatted)
|
|
168
535
|
return null;
|
|
169
536
|
logger.info(`openclaw-knowledge: LightRAG — ${formatted.truncated.length}/${formatted.originalLength} chars (truncated from ${formatted.originalLength})`);
|
|
537
|
+
emitEvent(logger, {
|
|
538
|
+
type: "lightrag",
|
|
539
|
+
mode: config.lightragQueryMode,
|
|
540
|
+
contextChars: formatted.originalLength,
|
|
541
|
+
truncatedChars: formatted.truncated.length,
|
|
542
|
+
durationMs: result.durationMs,
|
|
543
|
+
});
|
|
170
544
|
return "### Knowledge Graph Context (LightRAG)\n" + formatted.truncated;
|
|
171
545
|
}
|
|
172
546
|
return null;
|
|
173
547
|
}
|
|
174
548
|
// ---------------------------------------------------------------------------
|
|
549
|
+
// Cooldown utilities
|
|
550
|
+
// ---------------------------------------------------------------------------
|
|
551
|
+
function isInCooldown(state) {
|
|
552
|
+
return state.consecutiveErrors >= MAX_CONSECUTIVE_ERRORS;
|
|
553
|
+
}
|
|
554
|
+
function maybeResetCooldown(state, scope, logger) {
|
|
555
|
+
if (!isInCooldown(state))
|
|
556
|
+
return;
|
|
557
|
+
if (Date.now() < state.cooldownUntil)
|
|
558
|
+
return;
|
|
559
|
+
state.consecutiveErrors = 0;
|
|
560
|
+
state.cooldownUntil = 0;
|
|
561
|
+
logger.info(`openclaw-knowledge: ${scope} — resuming after cooldown`);
|
|
562
|
+
}
|
|
563
|
+
function registerError(state, scope, logger) {
|
|
564
|
+
state.consecutiveErrors++;
|
|
565
|
+
if (state.consecutiveErrors >= MAX_CONSECUTIVE_ERRORS) {
|
|
566
|
+
state.cooldownUntil = Date.now() + COOLDOWN_MS;
|
|
567
|
+
logger.error(`openclaw-knowledge: ${state.consecutiveErrors} consecutive errors — ${scope} cooling down 5 min`);
|
|
568
|
+
emitEvent(logger, {
|
|
569
|
+
type: "cooldown",
|
|
570
|
+
scope,
|
|
571
|
+
consecutiveErrors: state.consecutiveErrors,
|
|
572
|
+
});
|
|
573
|
+
}
|
|
574
|
+
}
|
|
575
|
+
// ---------------------------------------------------------------------------
|
|
175
576
|
// Plugin registration helper
|
|
176
|
-
//
|
|
177
|
-
// Exposed so tests can exercise the full wiring (including api.on) without
|
|
178
|
-
// going through `definePluginEntry`, which is tied to the SDK runtime.
|
|
179
577
|
// ---------------------------------------------------------------------------
|
|
180
578
|
/**
|
|
181
579
|
* Register the plugin against a minimal shape-compatible subset of the
|
|
@@ -189,6 +587,15 @@ export function registerKnowledgePlugin(api) {
|
|
|
189
587
|
api.logger.warn("openclaw-knowledge: neither pgvector nor LightRAG configured — plugin disabled");
|
|
190
588
|
return;
|
|
191
589
|
}
|
|
590
|
+
// Sanity check: when the reranker is on, we want at least ~2× the topN
|
|
591
|
+
// as raw candidates to give the cross-encoder room to re-order.
|
|
592
|
+
if (config.pgvectorRerankerEnabled &&
|
|
593
|
+
config.topK < config.pgvectorRerankerTopN * 2) {
|
|
594
|
+
api.logger.warn(`openclaw-knowledge: topK=${config.topK} is small relative to ` +
|
|
595
|
+
`pgvectorRerankerTopN=${config.pgvectorRerankerTopN}. ` +
|
|
596
|
+
`Recommended: topK ≥ ${config.pgvectorRerankerTopN * 2} for the ` +
|
|
597
|
+
`reranker to meaningfully change ordering.`);
|
|
598
|
+
}
|
|
192
599
|
// Only instantiate the pg pool when pgvector is actually in play. Booting
|
|
193
600
|
// a pool with no valid connection string would keep the plugin disabled
|
|
194
601
|
// anyway and leak sockets on hot-reload.
|
|
@@ -206,12 +613,18 @@ export function registerKnowledgePlugin(api) {
|
|
|
206
613
|
}
|
|
207
614
|
const sources = [];
|
|
208
615
|
if (config.pgvectorEnabled) {
|
|
209
|
-
|
|
616
|
+
const rerankNote = config.pgvectorRerankerEnabled
|
|
617
|
+
? ` + reranker(${config.pgvectorRerankerModel})`
|
|
618
|
+
: "";
|
|
619
|
+
sources.push(`pgvector (${config.collections.join(", ")})${rerankNote}`);
|
|
210
620
|
}
|
|
211
621
|
if (config.lightragEnabled) {
|
|
212
622
|
sources.push(`LightRAG (${config.lightragQueryMode})`);
|
|
213
623
|
}
|
|
214
|
-
|
|
624
|
+
const routerNote = config.routerEnabled
|
|
625
|
+
? ` | router=${config.routerMode}${config.routerClassifierId ? "/few-shot" : "/zero-shot"}`
|
|
626
|
+
: "";
|
|
627
|
+
api.logger.info(`openclaw-knowledge: ready — sources: ${sources.join(" + ")}${routerNote}`);
|
|
215
628
|
const handler = createBeforePromptBuildHandler({
|
|
216
629
|
config,
|
|
217
630
|
pool,
|
|
@@ -231,7 +644,7 @@ export function registerKnowledgePlugin(api) {
|
|
|
231
644
|
export default definePluginEntry({
|
|
232
645
|
id: "openclaw-knowledge",
|
|
233
646
|
name: "Knowledge Base",
|
|
234
|
-
description: "Multi-source knowledge search for OpenClaw (pgvector + LightRAG)",
|
|
647
|
+
description: "Multi-source knowledge search for OpenClaw (pgvector + LightRAG) with optional Jina-powered router & reranker",
|
|
235
648
|
register(api) {
|
|
236
649
|
registerKnowledgePlugin(api);
|
|
237
650
|
},
|