@poolzin/pool-bot 2026.3.4 → 2026.3.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +10 -0
- package/assets/pool-bot-icon-dark.png +0 -0
- package/assets/pool-bot-logo-1.png +0 -0
- package/assets/pool-bot-mascot.png +0 -0
- package/dist/agents/pi-embedded-runner/tool-result-truncation.js +62 -7
- package/dist/agents/pi-tools.js +32 -2
- package/dist/agents/poolbot-tools.js +12 -0
- package/dist/agents/session-write-lock.js +93 -8
- package/dist/agents/tools/pdf-native-providers.js +102 -0
- package/dist/agents/tools/pdf-tool.helpers.js +86 -0
- package/dist/agents/tools/pdf-tool.js +508 -0
- package/dist/auto-reply/reply/get-reply.js +6 -0
- package/dist/auto-reply/reply/message-preprocess-hooks.js +17 -0
- package/dist/build-info.json +3 -3
- package/dist/cli/banner.js +20 -1
- package/dist/cli/security-cli.js +211 -2
- package/dist/cli/tagline.js +7 -0
- package/dist/config/types.cli.js +1 -0
- package/dist/config/types.security.js +33 -0
- package/dist/config/zod-schema.js +15 -0
- package/dist/config/zod-schema.providers-core.js +1 -0
- package/dist/config/zod-schema.security.js +113 -0
- package/dist/cron/normalize.js +3 -0
- package/dist/cron/service/jobs.js +48 -0
- package/dist/discord/monitor/message-handler.preflight.js +11 -2
- package/dist/gateway/http-common.js +6 -1
- package/dist/gateway/protocol/schema/cron.js +3 -0
- package/dist/gateway/server-channels.js +99 -14
- package/dist/gateway/server-cron.js +89 -0
- package/dist/gateway/server-health-probes.js +55 -0
- package/dist/gateway/server-http.js +5 -0
- package/dist/hooks/bundled/session-memory/handler.js +8 -2
- package/dist/hooks/fire-and-forget.js +6 -0
- package/dist/hooks/internal-hooks.js +64 -19
- package/dist/hooks/message-hook-mappers.js +179 -0
- package/dist/infra/abort-signal.js +12 -0
- package/dist/infra/boundary-file-read.js +118 -0
- package/dist/infra/boundary-path.js +594 -0
- package/dist/infra/file-identity.js +12 -0
- package/dist/infra/fs-safe.js +377 -12
- package/dist/infra/hardlink-guards.js +30 -0
- package/dist/infra/json-utf8-bytes.js +8 -0
- package/dist/infra/net/fetch-guard.js +63 -13
- package/dist/infra/net/proxy-env.js +17 -0
- package/dist/infra/net/ssrf.js +74 -272
- package/dist/infra/path-alias-guards.js +21 -0
- package/dist/infra/path-guards.js +13 -1
- package/dist/infra/ports-probe.js +19 -0
- package/dist/infra/prototype-keys.js +4 -0
- package/dist/infra/restart-stale-pids.js +254 -0
- package/dist/infra/safe-open-sync.js +71 -0
- package/dist/infra/secure-random.js +7 -0
- package/dist/media/ffmpeg-limits.js +4 -0
- package/dist/media/input-files.js +6 -2
- package/dist/media/temp-files.js +12 -0
- package/dist/memory/embedding-chunk-limits.js +5 -2
- package/dist/memory/embeddings-ollama.js +91 -138
- package/dist/memory/embeddings-remote-fetch.js +11 -10
- package/dist/memory/embeddings.js +25 -9
- package/dist/memory/manager-embedding-ops.js +1 -1
- package/dist/memory/post-json.js +23 -0
- package/dist/memory/qmd-manager.js +272 -77
- package/dist/memory/remote-http.js +33 -0
- package/dist/plugin-sdk/windows-spawn.js +214 -0
- package/dist/security/capability-guards.js +89 -0
- package/dist/security/capability-manager.js +76 -0
- package/dist/security/capability.js +147 -0
- package/dist/security/index.js +7 -0
- package/dist/security/middleware.js +105 -0
- package/dist/shared/net/ip-test-fixtures.js +1 -0
- package/dist/shared/net/ip.js +303 -0
- package/dist/shared/net/ipv4.js +8 -11
- package/dist/shared/pid-alive.js +59 -2
- package/dist/slack/monitor/context.js +1 -0
- package/dist/slack/monitor/message-handler/dispatch.js +14 -1
- package/dist/slack/monitor/provider.js +2 -0
- package/dist/test-helpers/ssrf.js +13 -0
- package/dist/tui/tui.js +9 -4
- package/dist/utils/fetch-timeout.js +12 -1
- package/docs/adr/003-feature-gap-analysis.md +112 -0
- package/package.json +10 -4
|
@@ -0,0 +1,254 @@
|
|
|
1
|
+
import { spawnSync } from "node:child_process";
|
|
2
|
+
import { resolveGatewayPort } from "../config/paths.js";
|
|
3
|
+
import { createSubsystemLogger } from "../logging/subsystem.js";
|
|
4
|
+
import { resolveLsofCommandSync } from "./ports-lsof.js";
|
|
5
|
+
const SPAWN_TIMEOUT_MS = 2000;
|
|
6
|
+
const STALE_SIGTERM_WAIT_MS = 600;
|
|
7
|
+
const STALE_SIGKILL_WAIT_MS = 400;
|
|
8
|
+
/**
|
|
9
|
+
* After SIGKILL, the kernel may not release the TCP port immediately.
|
|
10
|
+
* Poll until the port is confirmed free (or until the budget expires) before
|
|
11
|
+
* returning control to the caller (typically `triggerPoolBotRestart` →
|
|
12
|
+
* `systemctl restart`). Without this wait the new process races the dying
|
|
13
|
+
* process for the port and systemd enters an EADDRINUSE restart loop.
|
|
14
|
+
*
|
|
15
|
+
* POLL_SPAWN_TIMEOUT_MS is intentionally much shorter than SPAWN_TIMEOUT_MS
|
|
16
|
+
* so that a single slow or hung lsof invocation does not consume the entire
|
|
17
|
+
* polling budget. At 400 ms per call, up to five independent lsof attempts
|
|
18
|
+
* fit within PORT_FREE_TIMEOUT_MS = 2000 ms, each with a definitive outcome.
|
|
19
|
+
*/
|
|
20
|
+
const PORT_FREE_POLL_INTERVAL_MS = 50;
|
|
21
|
+
const PORT_FREE_TIMEOUT_MS = 2000;
|
|
22
|
+
const POLL_SPAWN_TIMEOUT_MS = 400;
|
|
23
|
+
const restartLog = createSubsystemLogger("restart");
|
|
24
|
+
let sleepSyncOverride = null;
|
|
25
|
+
let dateNowOverride = null;
|
|
26
|
+
function getTimeMs() {
|
|
27
|
+
return dateNowOverride ? dateNowOverride() : Date.now();
|
|
28
|
+
}
|
|
29
|
+
function sleepSync(ms) {
|
|
30
|
+
const timeoutMs = Math.max(0, Math.floor(ms));
|
|
31
|
+
if (timeoutMs <= 0) {
|
|
32
|
+
return;
|
|
33
|
+
}
|
|
34
|
+
if (sleepSyncOverride) {
|
|
35
|
+
sleepSyncOverride(timeoutMs);
|
|
36
|
+
return;
|
|
37
|
+
}
|
|
38
|
+
try {
|
|
39
|
+
const lock = new Int32Array(new SharedArrayBuffer(4));
|
|
40
|
+
Atomics.wait(lock, 0, 0, timeoutMs);
|
|
41
|
+
}
|
|
42
|
+
catch {
|
|
43
|
+
const start = Date.now();
|
|
44
|
+
while (Date.now() - start < timeoutMs) {
|
|
45
|
+
// Best-effort fallback when Atomics.wait is unavailable.
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Parse poolbot gateway PIDs from lsof -Fpc stdout.
|
|
51
|
+
* Pure function — no I/O. Excludes the current process.
|
|
52
|
+
*/
|
|
53
|
+
function parsePidsFromLsofOutput(stdout) {
|
|
54
|
+
const pids = [];
|
|
55
|
+
let currentPid;
|
|
56
|
+
let currentCmd;
|
|
57
|
+
for (const line of stdout.split(/\r?\n/).filter(Boolean)) {
|
|
58
|
+
if (line.startsWith("p")) {
|
|
59
|
+
if (currentPid != null && currentCmd && currentCmd.toLowerCase().includes("poolbot")) {
|
|
60
|
+
pids.push(currentPid);
|
|
61
|
+
}
|
|
62
|
+
const parsed = Number.parseInt(line.slice(1), 10);
|
|
63
|
+
currentPid = Number.isFinite(parsed) && parsed > 0 ? parsed : undefined;
|
|
64
|
+
currentCmd = undefined;
|
|
65
|
+
}
|
|
66
|
+
else if (line.startsWith("c")) {
|
|
67
|
+
currentCmd = line.slice(1);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
if (currentPid != null && currentCmd && currentCmd.toLowerCase().includes("poolbot")) {
|
|
71
|
+
pids.push(currentPid);
|
|
72
|
+
}
|
|
73
|
+
// Deduplicate: dual-stack listeners (IPv4 + IPv6) cause lsof to emit the
|
|
74
|
+
// same PID twice. Return each PID at most once to avoid double-killing.
|
|
75
|
+
return [...new Set(pids)].filter((pid) => pid !== process.pid);
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Find PIDs of gateway processes listening on the given port using synchronous lsof.
|
|
79
|
+
* Returns only PIDs that belong to poolbot gateway processes (not the current process).
|
|
80
|
+
*/
|
|
81
|
+
export function findGatewayPidsOnPortSync(port, spawnTimeoutMs = SPAWN_TIMEOUT_MS) {
|
|
82
|
+
if (process.platform === "win32") {
|
|
83
|
+
return [];
|
|
84
|
+
}
|
|
85
|
+
const lsof = resolveLsofCommandSync();
|
|
86
|
+
const res = spawnSync(lsof, ["-nP", `-iTCP:${port}`, "-sTCP:LISTEN", "-Fpc"], {
|
|
87
|
+
encoding: "utf8",
|
|
88
|
+
timeout: spawnTimeoutMs,
|
|
89
|
+
});
|
|
90
|
+
if (res.error) {
|
|
91
|
+
const code = res.error.code;
|
|
92
|
+
const detail = code && code.trim().length > 0
|
|
93
|
+
? code
|
|
94
|
+
: res.error instanceof Error
|
|
95
|
+
? res.error.message
|
|
96
|
+
: "unknown error";
|
|
97
|
+
restartLog.warn(`lsof failed during initial stale-pid scan for port ${port}: ${detail}`);
|
|
98
|
+
return [];
|
|
99
|
+
}
|
|
100
|
+
if (res.status === 1) {
|
|
101
|
+
return [];
|
|
102
|
+
}
|
|
103
|
+
if (res.status !== 0) {
|
|
104
|
+
restartLog.warn(`lsof exited with status ${res.status} during initial stale-pid scan for port ${port}; skipping stale pid check`);
|
|
105
|
+
return [];
|
|
106
|
+
}
|
|
107
|
+
return parsePidsFromLsofOutput(res.stdout);
|
|
108
|
+
}
|
|
109
|
+
function pollPortOnce(port) {
|
|
110
|
+
try {
|
|
111
|
+
const lsof = resolveLsofCommandSync();
|
|
112
|
+
const res = spawnSync(lsof, ["-nP", `-iTCP:${port}`, "-sTCP:LISTEN", "-Fpc"], {
|
|
113
|
+
encoding: "utf8",
|
|
114
|
+
timeout: POLL_SPAWN_TIMEOUT_MS,
|
|
115
|
+
});
|
|
116
|
+
if (res.error) {
|
|
117
|
+
// Spawn-level failure. ENOENT / EACCES means lsof is permanently
|
|
118
|
+
// unavailable on this system; other errors (e.g. timeout) are transient.
|
|
119
|
+
const code = res.error.code;
|
|
120
|
+
const permanent = code === "ENOENT" || code === "EACCES" || code === "EPERM";
|
|
121
|
+
return { free: null, permanent };
|
|
122
|
+
}
|
|
123
|
+
if (res.status === 1) {
|
|
124
|
+
// lsof canonical "no matching processes" exit — port is genuinely free.
|
|
125
|
+
// Guard: on Linux containers with restricted /proc (AppArmor, seccomp,
|
|
126
|
+
// user namespaces), lsof can exit 1 AND still emit some output for the
|
|
127
|
+
// processes it could read. Parse stdout when non-empty to avoid false-free.
|
|
128
|
+
if (res.stdout) {
|
|
129
|
+
const pids = parsePidsFromLsofOutput(res.stdout);
|
|
130
|
+
return pids.length === 0 ? { free: true } : { free: false };
|
|
131
|
+
}
|
|
132
|
+
return { free: true };
|
|
133
|
+
}
|
|
134
|
+
if (res.status !== 0) {
|
|
135
|
+
// status > 1: runtime/permission/flag error. Cannot confirm port state —
|
|
136
|
+
// treat as a transient failure and keep polling rather than falsely
|
|
137
|
+
// reporting the port as free (which would recreate the EADDRINUSE race).
|
|
138
|
+
return { free: null, permanent: false };
|
|
139
|
+
}
|
|
140
|
+
// status === 0: lsof found listeners. Parse pids from the stdout we
|
|
141
|
+
// already hold — no second lsof spawn, no new failure surface.
|
|
142
|
+
const pids = parsePidsFromLsofOutput(res.stdout);
|
|
143
|
+
return pids.length === 0 ? { free: true } : { free: false };
|
|
144
|
+
}
|
|
145
|
+
catch {
|
|
146
|
+
return { free: null, permanent: false };
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
/**
|
|
150
|
+
* Synchronously terminate stale gateway processes.
|
|
151
|
+
* Callers must pass a non-empty pids array.
|
|
152
|
+
* Sends SIGTERM, waits briefly, then SIGKILL for survivors.
|
|
153
|
+
*/
|
|
154
|
+
function terminateStaleProcessesSync(pids) {
|
|
155
|
+
const killed = [];
|
|
156
|
+
for (const pid of pids) {
|
|
157
|
+
try {
|
|
158
|
+
process.kill(pid, "SIGTERM");
|
|
159
|
+
killed.push(pid);
|
|
160
|
+
}
|
|
161
|
+
catch {
|
|
162
|
+
// ESRCH — already gone
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
if (killed.length === 0) {
|
|
166
|
+
return killed;
|
|
167
|
+
}
|
|
168
|
+
sleepSync(STALE_SIGTERM_WAIT_MS);
|
|
169
|
+
for (const pid of killed) {
|
|
170
|
+
try {
|
|
171
|
+
process.kill(pid, 0);
|
|
172
|
+
process.kill(pid, "SIGKILL");
|
|
173
|
+
}
|
|
174
|
+
catch {
|
|
175
|
+
// already gone
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
sleepSync(STALE_SIGKILL_WAIT_MS);
|
|
179
|
+
return killed;
|
|
180
|
+
}
|
|
181
|
+
/**
|
|
182
|
+
* Poll the given port until it is confirmed free, lsof is confirmed unavailable,
|
|
183
|
+
* or the wall-clock budget expires.
|
|
184
|
+
*
|
|
185
|
+
* Each poll invocation uses POLL_SPAWN_TIMEOUT_MS (400 ms), which is
|
|
186
|
+
* significantly shorter than PORT_FREE_TIMEOUT_MS (2000 ms). This ensures
|
|
187
|
+
* that a single slow or hung lsof call cannot consume the entire polling
|
|
188
|
+
* budget and cause the function to exit prematurely with an inconclusive
|
|
189
|
+
* result. Up to five independent lsof attempts fit within the budget.
|
|
190
|
+
*
|
|
191
|
+
* Exit conditions:
|
|
192
|
+
* - `pollPortOnce` returns `{ free: true }` → port confirmed free
|
|
193
|
+
* - `pollPortOnce` returns `{ free: null, permanent: true }` → lsof unavailable, bail
|
|
194
|
+
* - `pollPortOnce` returns `{ free: false }` → port busy, sleep + retry
|
|
195
|
+
* - `pollPortOnce` returns `{ free: null, permanent: false }` → transient error, sleep + retry
|
|
196
|
+
* - Wall-clock deadline exceeded → log warning, proceed anyway
|
|
197
|
+
*/
|
|
198
|
+
function waitForPortFreeSync(port) {
|
|
199
|
+
const deadline = getTimeMs() + PORT_FREE_TIMEOUT_MS;
|
|
200
|
+
while (getTimeMs() < deadline) {
|
|
201
|
+
const result = pollPortOnce(port);
|
|
202
|
+
if (result.free === true) {
|
|
203
|
+
return;
|
|
204
|
+
}
|
|
205
|
+
if (result.free === null && result.permanent) {
|
|
206
|
+
// lsof is permanently unavailable (ENOENT / EACCES) — bail immediately,
|
|
207
|
+
// no point spinning the remaining budget.
|
|
208
|
+
return;
|
|
209
|
+
}
|
|
210
|
+
// result.free === false: port still bound.
|
|
211
|
+
// result.free === null && !permanent: transient lsof error — keep polling.
|
|
212
|
+
sleepSync(PORT_FREE_POLL_INTERVAL_MS);
|
|
213
|
+
}
|
|
214
|
+
restartLog.warn(`port ${port} still in use after ${PORT_FREE_TIMEOUT_MS}ms; proceeding anyway`);
|
|
215
|
+
}
|
|
216
|
+
/**
|
|
217
|
+
* Inspect the gateway port and kill any stale gateway processes holding it.
|
|
218
|
+
* Blocks until the port is confirmed free (or the poll budget expires) so
|
|
219
|
+
* the supervisor (systemd / launchctl) does not race a zombie process for
|
|
220
|
+
* the port and enter an EADDRINUSE restart loop.
|
|
221
|
+
*
|
|
222
|
+
* Called before service restart commands to prevent port conflicts.
|
|
223
|
+
*/
|
|
224
|
+
export function cleanStaleGatewayProcessesSync() {
|
|
225
|
+
try {
|
|
226
|
+
const port = resolveGatewayPort(undefined, process.env);
|
|
227
|
+
const stalePids = findGatewayPidsOnPortSync(port);
|
|
228
|
+
if (stalePids.length === 0) {
|
|
229
|
+
return [];
|
|
230
|
+
}
|
|
231
|
+
restartLog.warn(`killing ${stalePids.length} stale gateway process(es) before restart: ${stalePids.join(", ")}`);
|
|
232
|
+
const killed = terminateStaleProcessesSync(stalePids);
|
|
233
|
+
// Wait for the port to be released before returning — called unconditionally
|
|
234
|
+
// even when `killed` is empty (all pids were already dead before SIGTERM).
|
|
235
|
+
// A process can exit before our signal arrives yet still leave its socket
|
|
236
|
+
// in TIME_WAIT / FIN_WAIT; polling is the only reliable way to confirm the
|
|
237
|
+
// kernel has fully released the port before systemd fires the new process.
|
|
238
|
+
waitForPortFreeSync(port);
|
|
239
|
+
return killed;
|
|
240
|
+
}
|
|
241
|
+
catch {
|
|
242
|
+
return [];
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
export const __testing = {
|
|
246
|
+
setSleepSyncOverride(fn) {
|
|
247
|
+
sleepSyncOverride = fn;
|
|
248
|
+
},
|
|
249
|
+
setDateNowOverride(fn) {
|
|
250
|
+
dateNowOverride = fn;
|
|
251
|
+
},
|
|
252
|
+
/** Invoke sleepSync directly (bypasses the override) for unit-testing the real Atomics path. */
|
|
253
|
+
callSleepSyncRaw: sleepSync,
|
|
254
|
+
};
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
import fs from "node:fs";
|
|
2
|
+
import { sameFileIdentity as hasSameFileIdentity } from "./file-identity.js";
|
|
3
|
+
function isExpectedPathError(error) {
|
|
4
|
+
const code = typeof error === "object" && error !== null && "code" in error ? String(error.code) : "";
|
|
5
|
+
return code === "ENOENT" || code === "ENOTDIR" || code === "ELOOP";
|
|
6
|
+
}
|
|
7
|
+
export function sameFileIdentity(left, right) {
|
|
8
|
+
return hasSameFileIdentity(left, right);
|
|
9
|
+
}
|
|
10
|
+
export function openVerifiedFileSync(params) {
|
|
11
|
+
const ioFs = params.ioFs ?? fs;
|
|
12
|
+
const allowedType = params.allowedType ?? "file";
|
|
13
|
+
const openReadFlags = ioFs.constants.O_RDONLY |
|
|
14
|
+
(typeof ioFs.constants.O_NOFOLLOW === "number" ? ioFs.constants.O_NOFOLLOW : 0);
|
|
15
|
+
let fd = null;
|
|
16
|
+
try {
|
|
17
|
+
if (params.rejectPathSymlink) {
|
|
18
|
+
const candidateStat = ioFs.lstatSync(params.filePath);
|
|
19
|
+
if (candidateStat.isSymbolicLink()) {
|
|
20
|
+
return { ok: false, reason: "validation" };
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
const realPath = params.resolvedPath ?? ioFs.realpathSync(params.filePath);
|
|
24
|
+
const preOpenStat = ioFs.lstatSync(realPath);
|
|
25
|
+
if (!isAllowedType(preOpenStat, allowedType)) {
|
|
26
|
+
return { ok: false, reason: "validation" };
|
|
27
|
+
}
|
|
28
|
+
if (params.rejectHardlinks && preOpenStat.isFile() && preOpenStat.nlink > 1) {
|
|
29
|
+
return { ok: false, reason: "validation" };
|
|
30
|
+
}
|
|
31
|
+
if (params.maxBytes !== undefined &&
|
|
32
|
+
preOpenStat.isFile() &&
|
|
33
|
+
preOpenStat.size > params.maxBytes) {
|
|
34
|
+
return { ok: false, reason: "validation" };
|
|
35
|
+
}
|
|
36
|
+
fd = ioFs.openSync(realPath, openReadFlags);
|
|
37
|
+
const openedStat = ioFs.fstatSync(fd);
|
|
38
|
+
if (!isAllowedType(openedStat, allowedType)) {
|
|
39
|
+
return { ok: false, reason: "validation" };
|
|
40
|
+
}
|
|
41
|
+
if (params.rejectHardlinks && openedStat.isFile() && openedStat.nlink > 1) {
|
|
42
|
+
return { ok: false, reason: "validation" };
|
|
43
|
+
}
|
|
44
|
+
if (params.maxBytes !== undefined && openedStat.isFile() && openedStat.size > params.maxBytes) {
|
|
45
|
+
return { ok: false, reason: "validation" };
|
|
46
|
+
}
|
|
47
|
+
if (!sameFileIdentity(preOpenStat, openedStat)) {
|
|
48
|
+
return { ok: false, reason: "validation" };
|
|
49
|
+
}
|
|
50
|
+
const opened = { ok: true, path: realPath, fd, stat: openedStat };
|
|
51
|
+
fd = null;
|
|
52
|
+
return opened;
|
|
53
|
+
}
|
|
54
|
+
catch (error) {
|
|
55
|
+
if (isExpectedPathError(error)) {
|
|
56
|
+
return { ok: false, reason: "path", error };
|
|
57
|
+
}
|
|
58
|
+
return { ok: false, reason: "io", error };
|
|
59
|
+
}
|
|
60
|
+
finally {
|
|
61
|
+
if (fd !== null) {
|
|
62
|
+
ioFs.closeSync(fd);
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
function isAllowedType(stat, allowedType) {
|
|
67
|
+
if (allowedType === "directory") {
|
|
68
|
+
return stat.isDirectory();
|
|
69
|
+
}
|
|
70
|
+
return stat.isFile();
|
|
71
|
+
}
|
|
@@ -153,8 +153,8 @@ function clampText(text, maxChars) {
|
|
|
153
153
|
return text;
|
|
154
154
|
return text.slice(0, maxChars);
|
|
155
155
|
}
|
|
156
|
-
async function extractPdfContent(params) {
|
|
157
|
-
const { buffer, limits } = params;
|
|
156
|
+
export async function extractPdfContent(params) {
|
|
157
|
+
const { buffer, limits, pageNumbers } = params;
|
|
158
158
|
const { getDocument } = await loadPdfJsModule();
|
|
159
159
|
const pdf = await getDocument({
|
|
160
160
|
data: new Uint8Array(buffer),
|
|
@@ -163,6 +163,8 @@ async function extractPdfContent(params) {
|
|
|
163
163
|
const maxPages = Math.min(pdf.numPages, limits.pdf.maxPages);
|
|
164
164
|
const textParts = [];
|
|
165
165
|
for (let pageNum = 1; pageNum <= maxPages; pageNum += 1) {
|
|
166
|
+
if (pageNumbers && !pageNumbers.includes(pageNum))
|
|
167
|
+
continue;
|
|
166
168
|
const page = await pdf.getPage(pageNum);
|
|
167
169
|
const textContent = await page.getTextContent();
|
|
168
170
|
const pageText = textContent.items
|
|
@@ -187,6 +189,8 @@ async function extractPdfContent(params) {
|
|
|
187
189
|
const { createCanvas } = canvasModule;
|
|
188
190
|
const images = [];
|
|
189
191
|
for (let pageNum = 1; pageNum <= maxPages; pageNum += 1) {
|
|
192
|
+
if (pageNumbers && !pageNumbers.includes(pageNum))
|
|
193
|
+
continue;
|
|
190
194
|
const page = await pdf.getPage(pageNum);
|
|
191
195
|
const viewport = page.getViewport({ scale: 1 });
|
|
192
196
|
const maxPixels = limits.pdf.maxPixels;
|
|
@@ -1,8 +1,11 @@
|
|
|
1
1
|
import { estimateUtf8Bytes, splitTextToUtf8ByteLimit } from "./embedding-input-limits.js";
|
|
2
2
|
import { resolveEmbeddingMaxInputTokens } from "./embedding-model-limits.js";
|
|
3
3
|
import { hashText } from "./internal.js";
|
|
4
|
-
export function enforceEmbeddingMaxInputTokens(provider, chunks) {
|
|
5
|
-
const
|
|
4
|
+
export function enforceEmbeddingMaxInputTokens(provider, chunks, hardMaxInputTokens) {
|
|
5
|
+
const providerMaxInputTokens = resolveEmbeddingMaxInputTokens(provider);
|
|
6
|
+
const maxInputTokens = typeof hardMaxInputTokens === "number" && hardMaxInputTokens > 0
|
|
7
|
+
? Math.min(providerMaxInputTokens, hardMaxInputTokens)
|
|
8
|
+
: providerMaxInputTokens;
|
|
6
9
|
const out = [];
|
|
7
10
|
for (const chunk of chunks) {
|
|
8
11
|
if (estimateUtf8Bytes(chunk.text) <= maxInputTokens) {
|
|
@@ -1,158 +1,111 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
* Supports models like nomic-embed-text, mxbai-embed-large, etc.
|
|
6
|
-
*
|
|
7
|
-
* Benefits:
|
|
8
|
-
* - 100% local and private (no data leaves the server)
|
|
9
|
-
* - Zero cost (no API fees)
|
|
10
|
-
* - Fast inference (no network latency)
|
|
11
|
-
* - Offline capable
|
|
12
|
-
*/
|
|
1
|
+
import { resolveEnvApiKey } from "../agents/model-auth.js";
|
|
2
|
+
import { formatErrorMessage } from "../infra/errors.js";
|
|
3
|
+
import { normalizeOptionalSecretInput } from "../utils/normalize-secret-input.js";
|
|
4
|
+
import { buildRemoteBaseUrlPolicy, withRemoteHttpResponse } from "./remote-http.js";
|
|
13
5
|
export const DEFAULT_OLLAMA_EMBEDDING_MODEL = "nomic-embed-text";
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
const
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
}
|
|
23
|
-
|
|
24
|
-
* Normalizes model name (removes ollama/ prefix if present)
|
|
25
|
-
*/
|
|
26
|
-
export function normalizeOllamaModel(model) {
|
|
6
|
+
const DEFAULT_OLLAMA_BASE_URL = "http://127.0.0.1:11434";
|
|
7
|
+
function sanitizeAndNormalizeEmbedding(vec) {
|
|
8
|
+
const sanitized = vec.map((value) => (Number.isFinite(value) ? value : 0));
|
|
9
|
+
const magnitude = Math.sqrt(sanitized.reduce((sum, value) => sum + value * value, 0));
|
|
10
|
+
if (magnitude < 1e-10) {
|
|
11
|
+
return sanitized;
|
|
12
|
+
}
|
|
13
|
+
return sanitized.map((value) => value / magnitude);
|
|
14
|
+
}
|
|
15
|
+
function normalizeOllamaModel(model) {
|
|
27
16
|
const trimmed = model.trim();
|
|
28
|
-
if (!trimmed)
|
|
17
|
+
if (!trimmed) {
|
|
29
18
|
return DEFAULT_OLLAMA_EMBEDDING_MODEL;
|
|
30
|
-
|
|
19
|
+
}
|
|
20
|
+
if (trimmed.startsWith("ollama/")) {
|
|
31
21
|
return trimmed.slice("ollama/".length);
|
|
22
|
+
}
|
|
32
23
|
return trimmed;
|
|
33
24
|
}
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
export async function checkOllamaAvailable(baseUrl) {
|
|
38
|
-
try {
|
|
39
|
-
const res = await fetch(`${baseUrl}/api/tags`, {
|
|
40
|
-
method: "GET",
|
|
41
|
-
signal: AbortSignal.timeout(5000), // 5 second timeout
|
|
42
|
-
});
|
|
43
|
-
return res.ok;
|
|
44
|
-
}
|
|
45
|
-
catch {
|
|
46
|
-
return false;
|
|
25
|
+
function resolveOllamaApiBase(configuredBaseUrl) {
|
|
26
|
+
if (!configuredBaseUrl) {
|
|
27
|
+
return DEFAULT_OLLAMA_BASE_URL;
|
|
47
28
|
}
|
|
29
|
+
const trimmed = configuredBaseUrl.replace(/\/+$/, "");
|
|
30
|
+
return trimmed.replace(/\/v1$/i, "");
|
|
48
31
|
}
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
try {
|
|
54
|
-
const res = await fetch(`${baseUrl}/api/tags`, {
|
|
55
|
-
method: "GET",
|
|
56
|
-
signal: AbortSignal.timeout(5000),
|
|
57
|
-
});
|
|
58
|
-
if (!res.ok)
|
|
59
|
-
return false;
|
|
60
|
-
const payload = (await res.json());
|
|
61
|
-
const models = payload.models ?? [];
|
|
62
|
-
return models.some((m) => m.name === model || m.name.startsWith(`${model}:`));
|
|
32
|
+
function resolveOllamaApiKey(options) {
|
|
33
|
+
const remoteApiKey = options.remote?.apiKey?.trim();
|
|
34
|
+
if (remoteApiKey) {
|
|
35
|
+
return remoteApiKey;
|
|
63
36
|
}
|
|
64
|
-
|
|
65
|
-
|
|
37
|
+
const providerApiKey = normalizeOptionalSecretInput(options.config.models?.providers?.ollama?.apiKey);
|
|
38
|
+
if (providerApiKey) {
|
|
39
|
+
return providerApiKey;
|
|
66
40
|
}
|
|
41
|
+
return resolveEnvApiKey("ollama")?.apiKey;
|
|
42
|
+
}
|
|
43
|
+
function resolveOllamaEmbeddingClient(options) {
|
|
44
|
+
const providerConfig = options.config.models?.providers?.ollama;
|
|
45
|
+
const rawBaseUrl = options.remote?.baseUrl?.trim() || providerConfig?.baseUrl?.trim();
|
|
46
|
+
const baseUrl = resolveOllamaApiBase(rawBaseUrl);
|
|
47
|
+
const model = normalizeOllamaModel(options.model);
|
|
48
|
+
const headerOverrides = Object.assign({}, providerConfig?.headers, options.remote?.headers);
|
|
49
|
+
const headers = {
|
|
50
|
+
"Content-Type": "application/json",
|
|
51
|
+
...headerOverrides,
|
|
52
|
+
};
|
|
53
|
+
const apiKey = resolveOllamaApiKey(options);
|
|
54
|
+
if (apiKey) {
|
|
55
|
+
headers.Authorization = `Bearer ${apiKey}`;
|
|
56
|
+
}
|
|
57
|
+
return {
|
|
58
|
+
baseUrl,
|
|
59
|
+
headers,
|
|
60
|
+
ssrfPolicy: buildRemoteBaseUrlPolicy(baseUrl),
|
|
61
|
+
model,
|
|
62
|
+
};
|
|
67
63
|
}
|
|
68
|
-
/**
|
|
69
|
-
* Creates an Ollama embedding provider
|
|
70
|
-
*
|
|
71
|
-
* Uses Ollama's /api/embeddings endpoint for generating embeddings
|
|
72
|
-
*/
|
|
73
64
|
export async function createOllamaEmbeddingProvider(options) {
|
|
74
|
-
const client =
|
|
75
|
-
const
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
65
|
+
const client = resolveOllamaEmbeddingClient(options);
|
|
66
|
+
const embedUrl = `${client.baseUrl.replace(/\/$/, "")}/api/embeddings`;
|
|
67
|
+
const embedOne = async (text) => {
|
|
68
|
+
const json = await withRemoteHttpResponse({
|
|
69
|
+
url: embedUrl,
|
|
70
|
+
ssrfPolicy: client.ssrfPolicy,
|
|
71
|
+
init: {
|
|
72
|
+
method: "POST",
|
|
73
|
+
headers: client.headers,
|
|
74
|
+
body: JSON.stringify({ model: client.model, prompt: text }),
|
|
75
|
+
},
|
|
76
|
+
onResponse: async (res) => {
|
|
77
|
+
if (!res.ok) {
|
|
78
|
+
throw new Error(`Ollama embeddings HTTP ${res.status}: ${await res.text()}`);
|
|
79
|
+
}
|
|
80
|
+
return (await res.json());
|
|
81
|
+
},
|
|
89
82
|
});
|
|
90
|
-
if (!
|
|
91
|
-
|
|
92
|
-
throw new Error(`ollama embeddings failed: ${res.status} ${errorText}`);
|
|
83
|
+
if (!Array.isArray(json.embedding)) {
|
|
84
|
+
throw new Error(`Ollama embeddings response missing embedding[]`);
|
|
93
85
|
}
|
|
94
|
-
|
|
95
|
-
return payload.embedding ?? [];
|
|
86
|
+
return sanitizeAndNormalizeEmbedding(json.embedding);
|
|
96
87
|
};
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
return
|
|
104
|
-
|
|
105
|
-
const results = await Promise.all(texts.map((text) => embedQuery(text)));
|
|
106
|
-
return results;
|
|
88
|
+
const provider = {
|
|
89
|
+
id: "ollama",
|
|
90
|
+
model: client.model,
|
|
91
|
+
embedQuery: embedOne,
|
|
92
|
+
embedBatch: async (texts) => {
|
|
93
|
+
// Ollama /api/embeddings accepts one prompt per request.
|
|
94
|
+
return await Promise.all(texts.map(embedOne));
|
|
95
|
+
},
|
|
107
96
|
};
|
|
108
97
|
return {
|
|
109
|
-
provider
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
98
|
+
provider,
|
|
99
|
+
client: {
|
|
100
|
+
...client,
|
|
101
|
+
embedBatch: async (texts) => {
|
|
102
|
+
try {
|
|
103
|
+
return await provider.embedBatch(texts);
|
|
104
|
+
}
|
|
105
|
+
catch (err) {
|
|
106
|
+
throw new Error(formatErrorMessage(err), { cause: err });
|
|
107
|
+
}
|
|
108
|
+
},
|
|
115
109
|
},
|
|
116
|
-
client,
|
|
117
110
|
};
|
|
118
111
|
}
|
|
119
|
-
/**
|
|
120
|
-
* Resolves Ollama client configuration
|
|
121
|
-
*/
|
|
122
|
-
export async function resolveOllamaEmbeddingClient(options) {
|
|
123
|
-
const remote = options.remote;
|
|
124
|
-
const remoteBaseUrl = remote?.baseUrl?.trim();
|
|
125
|
-
// Get base URL from options, config, or default
|
|
126
|
-
const providerConfig = options.config.models?.providers?.ollama;
|
|
127
|
-
const baseUrl = remoteBaseUrl ||
|
|
128
|
-
providerConfig?.baseUrl?.trim() ||
|
|
129
|
-
process.env.OLLAMA_BASE_URL?.trim() ||
|
|
130
|
-
DEFAULT_OLLAMA_BASE_URL;
|
|
131
|
-
// Normalize model name
|
|
132
|
-
const model = normalizeOllamaModel(options.model);
|
|
133
|
-
// Verify Ollama is available — throw so auto-selection can fall through
|
|
134
|
-
const available = await checkOllamaAvailable(baseUrl);
|
|
135
|
-
if (!available) {
|
|
136
|
-
throw new Error(`Ollama server not reachable at ${baseUrl}. ` + `Make sure Ollama is running: ollama serve`);
|
|
137
|
-
}
|
|
138
|
-
// Check if model is available (warn but don't block — user may pull later)
|
|
139
|
-
const modelAvailable = await checkOllamaModelAvailable(baseUrl, model);
|
|
140
|
-
if (!modelAvailable) {
|
|
141
|
-
console.warn(`[ollama-embeddings] Model "${model}" not found in Ollama. ` +
|
|
142
|
-
`Pull it with: ollama pull ${model}`);
|
|
143
|
-
}
|
|
144
|
-
return { baseUrl, model };
|
|
145
|
-
}
|
|
146
|
-
/**
|
|
147
|
-
* Get embedding dimensions for a model
|
|
148
|
-
*/
|
|
149
|
-
export function getOllamaEmbeddingDimensions(model) {
|
|
150
|
-
const normalizedModel = normalizeOllamaModel(model);
|
|
151
|
-
// Check for exact match or prefix match
|
|
152
|
-
for (const [key, dims] of Object.entries(OLLAMA_EMBEDDING_DIMENSIONS)) {
|
|
153
|
-
if (normalizedModel === key || normalizedModel.startsWith(`${key}:`)) {
|
|
154
|
-
return dims;
|
|
155
|
-
}
|
|
156
|
-
}
|
|
157
|
-
return undefined; // Unknown model
|
|
158
|
-
}
|
|
@@ -1,14 +1,15 @@
|
|
|
1
|
+
import { postJson } from "./post-json.js";
|
|
1
2
|
export async function fetchRemoteEmbeddingVectors(params) {
|
|
2
|
-
|
|
3
|
-
|
|
3
|
+
return await postJson({
|
|
4
|
+
url: params.url,
|
|
4
5
|
headers: params.headers,
|
|
5
|
-
|
|
6
|
+
ssrfPolicy: params.ssrfPolicy,
|
|
7
|
+
body: params.body,
|
|
8
|
+
errorPrefix: params.errorPrefix,
|
|
9
|
+
parse: (payload) => {
|
|
10
|
+
const typedPayload = payload;
|
|
11
|
+
const data = typedPayload.data ?? [];
|
|
12
|
+
return data.map((entry) => entry.embedding ?? []);
|
|
13
|
+
},
|
|
6
14
|
});
|
|
7
|
-
if (!res.ok) {
|
|
8
|
-
const text = await res.text();
|
|
9
|
-
throw new Error(`${params.errorPrefix}: ${res.status} ${text}`);
|
|
10
|
-
}
|
|
11
|
-
const payload = (await res.json());
|
|
12
|
-
const data = payload.data ?? [];
|
|
13
|
-
return data.map((entry) => entry.embedding ?? []);
|
|
14
15
|
}
|