mercury-agent 0.16.2 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -0
- package/container/Dockerfile +1 -0
- package/container/Dockerfile.base +1 -0
- package/docs/autopilot/config.yaml +43 -0
- package/docs/behavior-layers.md +26 -12
- package/docs/container-lifecycle.md +1 -0
- package/docs/extensions.md +43 -0
- package/docs/goals/football-reporter-profile/decisions.md +37 -1
- package/docs/goals/football-reporter-profile/roadmap.md +63 -5
- package/docs/permissions.md +15 -0
- package/docs/profile-guide.md +304 -18
- package/docs/skills-guide.md +24 -0
- package/examples/extensions/feed-watch/config.ts +36 -0
- package/examples/extensions/feed-watch/digest.ts +230 -32
- package/examples/extensions/feed-watch/index.ts +8 -0
- package/examples/extensions/feed-watch/items.ts +48 -4
- package/examples/extensions/overview/actions.ts +264 -0
- package/examples/extensions/overview/gate.ts +82 -0
- package/examples/extensions/overview/index.ts +123 -0
- package/examples/extensions/overview/skill/SKILL.md +80 -0
- package/examples/profiles/_template/config.yaml +34 -0
- package/examples/profiles/football-reporter/AGENTS.md +24 -6
- package/examples/profiles/football-reporter/README.md +9 -1
- package/examples/profiles/football-reporter/config.yaml +31 -2
- package/examples/profiles/football-reporter/tasks/daily-article.md +6 -4
- package/package.json +7 -3
- package/resources/templates/mercury.example.yaml +5 -0
- package/src/adapters/whatsapp-media.ts +80 -3
- package/src/agent/container-entry.ts +143 -21
- package/src/agent/container-env.ts +11 -0
- package/src/agent/container-error.ts +12 -1
- package/src/agent/container-runner.ts +91 -9
- package/src/agent/pi-jsonl-parser.ts +166 -5
- package/src/bridges/discord.ts +4 -0
- package/src/bridges/slack.ts +4 -0
- package/src/bridges/teams.ts +4 -0
- package/src/bridges/telegram.ts +33 -1
- package/src/cli/build-image-tags.ts +34 -0
- package/src/cli/mercury.ts +51 -4
- package/src/config-file.ts +6 -0
- package/src/config.ts +28 -3
- package/src/core/attachment-notes.ts +4 -0
- package/src/core/confirmation.ts +3 -1
- package/src/core/handler.ts +2 -1
- package/src/core/media.ts +104 -2
- package/src/core/outbox.ts +23 -3
- package/src/core/reply-context.ts +61 -0
- package/src/core/routes/chat.ts +70 -3
- package/src/core/routes/config-builtin.ts +7 -0
- package/src/core/routes/dashboard.ts +91 -5
- package/src/core/runtime.ts +170 -22
- package/src/core/task-output.ts +36 -18
- package/src/core/task-scheduler.ts +18 -1
- package/src/extensions/catalog.ts +1 -0
- package/src/extensions/hooks.ts +8 -0
- package/src/extensions/loader.ts +68 -7
- package/src/extensions/types.ts +35 -2
- package/src/main.ts +19 -0
- package/src/server.ts +15 -8
- package/src/storage/db.ts +231 -12
- package/src/text/sanitize-text.ts +118 -0
- package/src/types.ts +73 -1
package/src/cli/mercury.ts
CHANGED
|
@@ -17,7 +17,11 @@ import { homedir, tmpdir } from "node:os";
|
|
|
17
17
|
import { basename, dirname, join, resolve } from "node:path";
|
|
18
18
|
import { fileURLToPath } from "node:url";
|
|
19
19
|
import { Command } from "commander";
|
|
20
|
-
import {
|
|
20
|
+
import {
|
|
21
|
+
DEFAULT_AGENT_CONTAINER_IMAGE,
|
|
22
|
+
loadConfig,
|
|
23
|
+
resolveProjectPath,
|
|
24
|
+
} from "../config.js";
|
|
21
25
|
import { mergeRawMercuryConfig } from "../config-file.js";
|
|
22
26
|
import { reapProcessTree } from "../core/process-tree.js";
|
|
23
27
|
import { getCatalogEntryByName } from "../extensions/catalog.js";
|
|
@@ -31,6 +35,10 @@ import {
|
|
|
31
35
|
import { RESERVED_EXTENSION_NAMES } from "../extensions/reserved.js";
|
|
32
36
|
import { Db } from "../storage/db.js";
|
|
33
37
|
import { removeSpaceWorkspace } from "../storage/memory.js";
|
|
38
|
+
import {
|
|
39
|
+
LOCAL_BUILD_IMAGE_TAG,
|
|
40
|
+
resolveBuildImageTags,
|
|
41
|
+
} from "./build-image-tags.js";
|
|
34
42
|
import {
|
|
35
43
|
buildGlobalInstallArgs,
|
|
36
44
|
formatLockedDllMessage,
|
|
@@ -360,6 +368,29 @@ async function runAction(): Promise<void> {
|
|
|
360
368
|
|
|
361
369
|
function buildAction(): void {
|
|
362
370
|
// Build from package sources using a temp context — no files needed in user project
|
|
371
|
+
//
|
|
372
|
+
// The image is tagged with whatever `agentContainerImage` resolves to, not a
|
|
373
|
+
// fixed name: the runtime only ever looks for the configured reference, so a
|
|
374
|
+
// build under any other name is invisible to it. Config load is best-effort
|
|
375
|
+
// because this command is documented to run outside a project directory --
|
|
376
|
+
// there may be no .env and no mercury.yaml, and that is not an error.
|
|
377
|
+
let configuredImage: string = DEFAULT_AGENT_CONTAINER_IMAGE;
|
|
378
|
+
try {
|
|
379
|
+
// Inside the guard on purpose: loadEnvFile is a bare readFileSync, and an
|
|
380
|
+
// unreadable .env must not abort the one command that has to keep working
|
|
381
|
+
// in a degraded environment.
|
|
382
|
+
const buildEnvPath = join(CWD, ".env");
|
|
383
|
+
if (existsSync(buildEnvPath)) {
|
|
384
|
+
Object.assign(process.env, loadEnvFile(buildEnvPath));
|
|
385
|
+
}
|
|
386
|
+
configuredImage = loadConfig().agentContainerImage;
|
|
387
|
+
} catch (err) {
|
|
388
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
389
|
+
console.warn(`⚠️ Could not read configuration (${msg}).`);
|
|
390
|
+
console.warn(" Tagging the default image name instead.");
|
|
391
|
+
}
|
|
392
|
+
const imageTags = resolveBuildImageTags(configuredImage);
|
|
393
|
+
|
|
363
394
|
const tmpDir = join(tmpdir(), "mercury-build-context");
|
|
364
395
|
rmSync(tmpDir, { recursive: true, force: true });
|
|
365
396
|
mkdirSync(tmpDir, { recursive: true });
|
|
@@ -374,6 +405,7 @@ function buildAction(): void {
|
|
|
374
405
|
"src/agent/pi-failure-class.ts",
|
|
375
406
|
"src/agent/pi-jsonl-parser.ts",
|
|
376
407
|
"src/agent/preferences-prompt.ts",
|
|
408
|
+
"src/text/sanitize-text.ts",
|
|
377
409
|
"src/cli/mrctl.ts",
|
|
378
410
|
"src/cli/mrctl-http.ts",
|
|
379
411
|
"src/extensions/reserved.ts",
|
|
@@ -414,13 +446,13 @@ function buildAction(): void {
|
|
|
414
446
|
|
|
415
447
|
console.log(`📦 Building container image...`);
|
|
416
448
|
console.log(` Package root: ${PACKAGE_ROOT}`);
|
|
417
|
-
console.log(` Build context: ${tmpDir}
|
|
449
|
+
console.log(` Build context: ${tmpDir}`);
|
|
450
|
+
console.log(` Tags: ${imageTags.join(", ")}\n`);
|
|
418
451
|
const result = spawnSync(
|
|
419
452
|
"docker",
|
|
420
453
|
[
|
|
421
454
|
"build",
|
|
422
|
-
"-t",
|
|
423
|
-
"mercury-agent:latest",
|
|
455
|
+
...imageTags.flatMap((tag) => ["-t", tag]),
|
|
424
456
|
"-f",
|
|
425
457
|
join(tmpDir, "container/Dockerfile"),
|
|
426
458
|
tmpDir,
|
|
@@ -431,6 +463,21 @@ function buildAction(): void {
|
|
|
431
463
|
if (result.status !== 0) {
|
|
432
464
|
process.exit(result.status ?? 1);
|
|
433
465
|
}
|
|
466
|
+
|
|
467
|
+
console.log(`
|
|
468
|
+
✓ Built ${imageTags.join(" and ")}`);
|
|
469
|
+
console.log(
|
|
470
|
+
` Mercury runs ${imageTags[0]} — no registry pull needed for it.`,
|
|
471
|
+
);
|
|
472
|
+
if (imageTags[0] !== LOCAL_BUILD_IMAGE_TAG) {
|
|
473
|
+
console.log(
|
|
474
|
+
` Note: this shadows the registry copy. Docker only pulls a reference`,
|
|
475
|
+
);
|
|
476
|
+
console.log(
|
|
477
|
+
` that is absent locally, so re-run 'mercury build' after upgrading,`,
|
|
478
|
+
);
|
|
479
|
+
console.log(` or 'docker rmi' it to go back to pulling.`);
|
|
480
|
+
}
|
|
434
481
|
} finally {
|
|
435
482
|
// Clean up temp context
|
|
436
483
|
rmSync(tmpDir, { recursive: true, force: true });
|
package/src/config-file.ts
CHANGED
|
@@ -147,6 +147,7 @@ const mercuryFileSchema = z
|
|
|
147
147
|
.max(60 * 60 * 1000)
|
|
148
148
|
.optional(),
|
|
149
149
|
container_bwrap_docker_compat: z.boolean().optional(),
|
|
150
|
+
allow_unconfined: z.boolean().optional(),
|
|
150
151
|
override_pi_system_prompt: z.boolean().optional(),
|
|
151
152
|
env_passthrough: z.enum(["all", "claimed"]).optional(),
|
|
152
153
|
trace_runs: z.boolean().optional(),
|
|
@@ -284,6 +285,7 @@ const KNOWN_SECTION_KEYS: Record<string, Set<string>> = {
|
|
|
284
285
|
"image",
|
|
285
286
|
"container_timeout_ms",
|
|
286
287
|
"container_bwrap_docker_compat",
|
|
288
|
+
"allow_unconfined",
|
|
287
289
|
"override_pi_system_prompt",
|
|
288
290
|
"env_passthrough",
|
|
289
291
|
"trace_runs",
|
|
@@ -444,6 +446,9 @@ function flattenMercuryFile(f: MercuryFile): RawMercuryConfigInput {
|
|
|
444
446
|
if (f.agent?.container_bwrap_docker_compat != null) {
|
|
445
447
|
o.containerBwrapDockerCompat = f.agent.container_bwrap_docker_compat;
|
|
446
448
|
}
|
|
449
|
+
if (f.agent?.allow_unconfined != null) {
|
|
450
|
+
o.containerAllowUnconfined = f.agent.allow_unconfined;
|
|
451
|
+
}
|
|
447
452
|
if (f.agent?.override_pi_system_prompt != null) {
|
|
448
453
|
o.overridePiSystemPrompt = f.agent.override_pi_system_prompt;
|
|
449
454
|
}
|
|
@@ -532,6 +537,7 @@ const CAMEL_TO_ENV: Record<string, string> = {
|
|
|
532
537
|
containerNetwork: "MERCURY_CONTAINER_NETWORK",
|
|
533
538
|
containerApiHost: "MERCURY_CONTAINER_API_HOST",
|
|
534
539
|
containerBwrapDockerCompat: "MERCURY_CONTAINER_BWRAP_DOCKER_COMPAT",
|
|
540
|
+
containerAllowUnconfined: "MERCURY_CONTAINER_ALLOW_UNCONFINED",
|
|
535
541
|
containerEnvPassthrough: "MERCURY_CONTAINER_ENV_PASSTHROUGH",
|
|
536
542
|
overridePiSystemPrompt: "MERCURY_OVERRIDE_PI_SYSTEM_PROMPT",
|
|
537
543
|
maxConcurrency: "MERCURY_MAX_CONCURRENCY",
|
package/src/config.ts
CHANGED
|
@@ -51,6 +51,10 @@ const booleanFromEnv = z.union([z.boolean(), z.string()]).transform((val) => {
|
|
|
51
51
|
return lower === "true" || lower === "1";
|
|
52
52
|
});
|
|
53
53
|
|
|
54
|
+
/** Base agent image used when nothing overrides `agent.image`. */
|
|
55
|
+
export const DEFAULT_AGENT_CONTAINER_IMAGE =
|
|
56
|
+
"ghcr.io/avishai-tsabari/mercury-agent:latest";
|
|
57
|
+
|
|
54
58
|
const schema = z.object({
|
|
55
59
|
// ─── API Key Mode ───────────────────────────────────────────────────
|
|
56
60
|
apiKeyMode: z.enum(["platform", "byok"]).default("platform"),
|
|
@@ -136,9 +140,7 @@ const schema = z.object({
|
|
|
136
140
|
whatsappAuthDir: z.string().optional(),
|
|
137
141
|
|
|
138
142
|
// ─── Container / Agent ──────────────────────────────────────────────
|
|
139
|
-
agentContainerImage: z
|
|
140
|
-
.string()
|
|
141
|
-
.default("ghcr.io/avishai-tsabari/mercury-agent:latest"),
|
|
143
|
+
agentContainerImage: z.string().default(DEFAULT_AGENT_CONTAINER_IMAGE),
|
|
142
144
|
containerTimeoutMs: z.coerce
|
|
143
145
|
.number()
|
|
144
146
|
.int()
|
|
@@ -188,6 +190,29 @@ const schema = z.object({
|
|
|
188
190
|
* Ignored when containerRuntime is "runsc". See docs/container-lifecycle.md.
|
|
189
191
|
*/
|
|
190
192
|
containerBwrapDockerCompat: booleanFromEnv.default(false),
|
|
193
|
+
/**
|
|
194
|
+
* Permits the auto-detected Docker Desktop sandbox relaxation
|
|
195
|
+
* (`seccomp=unconfined`, `apparmor=unconfined`, `--cap-add SYS_ADMIN`), which
|
|
196
|
+
* bubblewrap needs in order to nest inside the agent container.
|
|
197
|
+
*
|
|
198
|
+
* Default false, and that refuses the spawn rather than quietly weakening it.
|
|
199
|
+
* The relaxation used to be applied automatically on every Mac and Windows
|
|
200
|
+
* host and announced only at `info`, so a self-hoster ran a materially weaker
|
|
201
|
+
* sandbox than the Linux deployment without ever being asked — while the
|
|
202
|
+
* container it protects executes model-directed shell commands against
|
|
203
|
+
* attacker-influenced input. This project's own rule is that a limit the
|
|
204
|
+
* actor cannot read is a trap; a *relaxation* the operator cannot see is
|
|
205
|
+
* worse, because the operator is the only one who can weigh it.
|
|
206
|
+
*
|
|
207
|
+
* Set MERCURY_CONTAINER_ALLOW_UNCONFINED=true to accept the trade knowingly (it is then
|
|
208
|
+
* logged at `warn` on every spawn), or MERCURY_CONTAINER_RUNTIME=runsc to
|
|
209
|
+
* avoid it entirely — gVisor needs no relaxation, but is Linux-only.
|
|
210
|
+
*
|
|
211
|
+
* Ignored when containerRuntime is "runsc" (no relaxation is requested) and
|
|
212
|
+
* when containerBwrapDockerCompat is set (that flag is already an explicit
|
|
213
|
+
* operator choice).
|
|
214
|
+
*/
|
|
215
|
+
containerAllowUnconfined: booleanFromEnv.default(false),
|
|
191
216
|
/**
|
|
192
217
|
* Docker network to attach inner (pi) containers to. When set, inner containers join this
|
|
193
218
|
* network and can reach the Mercury host container by its container name rather than via
|
|
@@ -34,6 +34,10 @@ function describe(d: DroppedAttachment): string {
|
|
|
34
34
|
return `a ${what} was not downloaded because the storage quota is exhausted.`;
|
|
35
35
|
case "download_failed":
|
|
36
36
|
return `a ${what} failed to download.`;
|
|
37
|
+
case "unsupported_type":
|
|
38
|
+
return `a ${what} was not accepted because its file type is not one this bot handles.`;
|
|
39
|
+
case "executable_content":
|
|
40
|
+
return `a ${what} was not accepted because its contents are executable code, whatever the filename said.`;
|
|
37
41
|
}
|
|
38
42
|
}
|
|
39
43
|
|
package/src/core/confirmation.ts
CHANGED
|
@@ -16,8 +16,10 @@
|
|
|
16
16
|
* make the match fail; typing a bare `yes` was the only thing that worked.
|
|
17
17
|
*/
|
|
18
18
|
|
|
19
|
+
import { REPLY_TO_BLOCK_RE } from "./reply-context.js";
|
|
20
|
+
|
|
19
21
|
/** Context blocks adapters append to inbound text. Order-independent. */
|
|
20
|
-
const APPENDED_BLOCKS = [
|
|
22
|
+
const APPENDED_BLOCKS = [REPLY_TO_BLOCK_RE];
|
|
21
23
|
|
|
22
24
|
/**
|
|
23
25
|
* Strip adapter-appended context blocks and normalize for comparison.
|
package/src/core/handler.ts
CHANGED
|
@@ -13,6 +13,7 @@ import {
|
|
|
13
13
|
resolveConversation,
|
|
14
14
|
} from "./conversation.js";
|
|
15
15
|
import { getDefaultDebounceMs, MessageDebouncer } from "./debounce.js";
|
|
16
|
+
import { deliverySuffix } from "./reply-context.js";
|
|
16
17
|
import { chatCommandWord } from "./router.js";
|
|
17
18
|
import type { MercuryCoreRuntime } from "./runtime.js";
|
|
18
19
|
import { formatSystemMessage, resolveLocale } from "./system-messages.js";
|
|
@@ -194,7 +195,7 @@ export function createMessageHandler(opts: MessageHandlerOptions) {
|
|
|
194
195
|
const elapsed = Math.round((Date.now() - startTime) / 1000);
|
|
195
196
|
const finalReply =
|
|
196
197
|
hadStatusMessage && reply
|
|
197
|
-
? `${reply}\n\
|
|
198
|
+
? `${reply}\n\n${deliverySuffix(elapsed)}`
|
|
198
199
|
: reply;
|
|
199
200
|
if (finalReply || files.length > 0) {
|
|
200
201
|
// Guarded, because this await used to be the one unprotected step
|
package/src/core/media.ts
CHANGED
|
@@ -8,7 +8,12 @@
|
|
|
8
8
|
import fs from "node:fs";
|
|
9
9
|
import path from "node:path";
|
|
10
10
|
import { logger } from "../logger.js";
|
|
11
|
-
import
|
|
11
|
+
import { stripInvisibleUnicode } from "../text/sanitize-text.js";
|
|
12
|
+
import type {
|
|
13
|
+
DroppedAttachment,
|
|
14
|
+
MediaType,
|
|
15
|
+
MessageAttachment,
|
|
16
|
+
} from "../types.js";
|
|
12
17
|
|
|
13
18
|
// ─── MIME Maps ──────────────────────────────────────────────────────────
|
|
14
19
|
|
|
@@ -96,6 +101,62 @@ export function mimeToMediaType(mimeType: string): MediaType {
|
|
|
96
101
|
return "document";
|
|
97
102
|
}
|
|
98
103
|
|
|
104
|
+
// ─── Ingest Gates ───────────────────────────────────────────────────────
|
|
105
|
+
|
|
106
|
+
/**
|
|
107
|
+
* Is this filename's extension one the bot knows about?
|
|
108
|
+
*
|
|
109
|
+
* `EXT_TO_MIME` is the single source of truth — there is deliberately no
|
|
110
|
+
* parallel allowlist config, because that would be two sources of truth for
|
|
111
|
+
* one question. Anything the table does not name yields
|
|
112
|
+
* `application/octet-stream`, which is what this reports on.
|
|
113
|
+
*
|
|
114
|
+
* Ships **warn-only**: callers log what they would have rejected and accept
|
|
115
|
+
* the file anyway. Flipping to a rejection is a separate, deliberate change
|
|
116
|
+
* once the log shows the table matches real traffic.
|
|
117
|
+
*/
|
|
118
|
+
export function isAllowedExtension(filename: string): boolean {
|
|
119
|
+
// `extToMime` splits on "." and takes the last part, which is the whole
|
|
120
|
+
// string when there is no dot — so a file named literally `pdf` would
|
|
121
|
+
// otherwise read as allowed. This predicate is what the deferred flip to
|
|
122
|
+
// rejection will consume, so the hole is closed here rather than there.
|
|
123
|
+
if (!filename.includes(".")) return false;
|
|
124
|
+
return extToMime(filename) !== "application/octet-stream";
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/**
|
|
128
|
+
* Known executable headers, matched against the first bytes of the file.
|
|
129
|
+
*
|
|
130
|
+
* A denylist, not a content/extension match: `.txt`, `.csv` and friends have
|
|
131
|
+
* no signature at all, and the whole OOXML family shares `PK\x03\x04`, so a
|
|
132
|
+
* positive identification is not available for the formats that matter. This
|
|
133
|
+
* asserts only that the bytes are *not* one of the shapes below.
|
|
134
|
+
*
|
|
135
|
+
* It does not prevent code execution — nothing in the container executes an
|
|
136
|
+
* attachment. It is blast-radius containment against a future parser, and the
|
|
137
|
+
* correct posture for the relay case where a file reaches a human. A payload
|
|
138
|
+
* with `GIF89a` or `%PDF-` prepended defeats it entirely; this targets crude
|
|
139
|
+
* mislabelling, not a motivated attacker.
|
|
140
|
+
*/
|
|
141
|
+
const EXECUTABLE_HEADERS: readonly (readonly number[])[] = [
|
|
142
|
+
[0x4d, 0x5a], // MZ — DOS/Windows PE
|
|
143
|
+
[0x7f, 0x45, 0x4c, 0x46], // \x7fELF — Linux/BSD
|
|
144
|
+
[0xfe, 0xed, 0xfa, 0xce], // Mach-O 32-bit BE
|
|
145
|
+
[0xce, 0xfa, 0xed, 0xfe], // Mach-O 32-bit LE
|
|
146
|
+
[0xfe, 0xed, 0xfa, 0xcf], // Mach-O 64-bit BE
|
|
147
|
+
[0xcf, 0xfa, 0xed, 0xfe], // Mach-O 64-bit LE
|
|
148
|
+
[0xca, 0xfe, 0xba, 0xbe], // Java class / Mach-O fat binary
|
|
149
|
+
[0x23, 0x21], // #! — shebang script
|
|
150
|
+
];
|
|
151
|
+
|
|
152
|
+
/** Do the first bytes of this buffer match a known executable header? */
|
|
153
|
+
export function hasExecutableHeader(buffer: Buffer): boolean {
|
|
154
|
+
return EXECUTABLE_HEADERS.some(
|
|
155
|
+
(sig) =>
|
|
156
|
+
buffer.length >= sig.length && sig.every((byte, i) => buffer[i] === byte),
|
|
157
|
+
);
|
|
158
|
+
}
|
|
159
|
+
|
|
99
160
|
// ─── URL-based Media Downloader ─────────────────────────────────────────
|
|
100
161
|
|
|
101
162
|
/**
|
|
@@ -114,10 +175,35 @@ export async function downloadMediaFromUrl(
|
|
|
114
175
|
maxSizeBytes: number;
|
|
115
176
|
outputDir: string;
|
|
116
177
|
headers?: Record<string, string>;
|
|
178
|
+
/**
|
|
179
|
+
* Called when this function drops the file instead of returning it, so the
|
|
180
|
+
* caller can tell the agent why. Same shape `downloadWhatsAppMedia` uses.
|
|
181
|
+
* Currently fired only for the ingest gates below — the size and
|
|
182
|
+
* download-failure paths still return a bare `null`. Closing that is
|
|
183
|
+
* tracked in `docs/ideas/url-download-drops-not-reported-to-agent.md`.
|
|
184
|
+
*/
|
|
185
|
+
onSkipped?: (info: DroppedAttachment) => void;
|
|
117
186
|
},
|
|
118
187
|
): Promise<MessageAttachment | null> {
|
|
119
188
|
const { maxSizeBytes, outputDir, headers } = options;
|
|
120
189
|
|
|
190
|
+
// Part A — extension allowlist, WARN-ONLY. Runs before the fetch so the log
|
|
191
|
+
// reflects what a rejection would have skipped downloading. Deliberately
|
|
192
|
+
// does not drop, does not call onSkipped, and changes nothing the sender or
|
|
193
|
+
// the agent sees; the flip to rejection is a separate change gated on this
|
|
194
|
+
// log. Only meaningful when the sender supplied a filename — without one the
|
|
195
|
+
// stored name is derived from the MIME type and there is no extension to judge.
|
|
196
|
+
if (options.filename && !isAllowedExtension(options.filename)) {
|
|
197
|
+
logger.warn(
|
|
198
|
+
"Attachment extension not in EXT_TO_MIME (warn-only, accepted)",
|
|
199
|
+
{
|
|
200
|
+
filename: options.filename,
|
|
201
|
+
mimeType: options.mimeType,
|
|
202
|
+
type: options.type,
|
|
203
|
+
},
|
|
204
|
+
);
|
|
205
|
+
}
|
|
206
|
+
|
|
121
207
|
// Check expected size before downloading
|
|
122
208
|
if (options.expectedSizeBytes && options.expectedSizeBytes > maxSizeBytes) {
|
|
123
209
|
logger.warn("Skipping large media file", {
|
|
@@ -164,6 +250,19 @@ export async function downloadMediaFromUrl(
|
|
|
164
250
|
return null;
|
|
165
251
|
}
|
|
166
252
|
|
|
253
|
+
// Part B — binary-header denylist, enforced. Checked on the bytes already
|
|
254
|
+
// in memory, before anything reaches disk: never write-then-unlink.
|
|
255
|
+
if (hasExecutableHeader(buffer)) {
|
|
256
|
+
logger.warn("Dropping attachment with executable header", {
|
|
257
|
+
filename: options.filename,
|
|
258
|
+
mimeType: options.mimeType,
|
|
259
|
+
type: options.type,
|
|
260
|
+
sizeBytes: buffer.length,
|
|
261
|
+
});
|
|
262
|
+
options.onSkipped?.({ type: options.type, reason: "executable_content" });
|
|
263
|
+
return null;
|
|
264
|
+
}
|
|
265
|
+
|
|
167
266
|
// Ensure output directory exists
|
|
168
267
|
fs.mkdirSync(outputDir, { recursive: true });
|
|
169
268
|
|
|
@@ -190,7 +289,10 @@ export async function downloadMediaFromUrl(
|
|
|
190
289
|
path: filePath,
|
|
191
290
|
type: options.type,
|
|
192
291
|
mimeType: options.mimeType,
|
|
193
|
-
|
|
292
|
+
// The stored path is already ASCII-only via `safeName`; this is the
|
|
293
|
+
// display name that reaches the prompt, so it is stripped rather than
|
|
294
|
+
// rewritten — the sender's real filename stays readable.
|
|
295
|
+
filename: stripInvisibleUnicode(options.filename),
|
|
194
296
|
sizeBytes: buffer.length,
|
|
195
297
|
};
|
|
196
298
|
} catch (error) {
|
package/src/core/outbox.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import fs from "node:fs";
|
|
2
2
|
import path from "node:path";
|
|
3
3
|
import { logger } from "../logger.js";
|
|
4
|
+
import { stripOutboundTagCarriers } from "../text/sanitize-text.js";
|
|
4
5
|
import type { EgressFile } from "../types.js";
|
|
5
6
|
import { extToMime } from "./media.js";
|
|
6
7
|
|
|
@@ -38,8 +39,23 @@ export function scanOutbox(
|
|
|
38
39
|
|
|
39
40
|
for (const entry of entries) {
|
|
40
41
|
if (!entry.isFile()) continue;
|
|
41
|
-
if (entry.name.startsWith(".")) continue;
|
|
42
42
|
|
|
43
|
+
// The display name is the only string here a human ever sees, and the agent
|
|
44
|
+
// inside the container chose it — so it is remote-supplied text on the way
|
|
45
|
+
// out, and every bridge hands it to its platform API verbatim. Strip the
|
|
46
|
+
// Tags Block carriers for the same reason `consumeResult` strips the reply:
|
|
47
|
+
// an invisible payload rides out looking like `report.pdf`.
|
|
48
|
+
//
|
|
49
|
+
// Derived BEFORE the dotfile guard so one value drives both the filter and
|
|
50
|
+
// what ships. Stripping afterwards let a name like `\u{E0001}.env` pass a
|
|
51
|
+
// check reading the raw name and then be presented as `.env` — the guard and
|
|
52
|
+
// the emitted filename disagreeing about which string is the filename is the
|
|
53
|
+
// same shape as the bug this fixes.
|
|
54
|
+
const displayName = stripOutboundTagCarriers(entry.name) || "attachment";
|
|
55
|
+
if (displayName.startsWith(".")) continue;
|
|
56
|
+
|
|
57
|
+
// The real on-disk path, always the raw name: every bridge reads the bytes
|
|
58
|
+
// from it and a stripped path would not exist.
|
|
43
59
|
const filePath = path.join(outboxDir, entry.name);
|
|
44
60
|
|
|
45
61
|
let stat: fs.Stats;
|
|
@@ -64,10 +80,14 @@ export function scanOutbox(
|
|
|
64
80
|
continue;
|
|
65
81
|
}
|
|
66
82
|
|
|
83
|
+
// `extToMime` reads the stripped name on purpose: it takes the segment after
|
|
84
|
+
// the last dot, so a payload appended to `report.pdf` used to yield an
|
|
85
|
+
// extension of `pdf` + 40 invisible characters and a mime of
|
|
86
|
+
// `application/octet-stream`.
|
|
67
87
|
files.push({
|
|
68
88
|
path: filePath,
|
|
69
|
-
filename:
|
|
70
|
-
mimeType: extToMime(
|
|
89
|
+
filename: displayName,
|
|
90
|
+
mimeType: extToMime(displayName),
|
|
71
91
|
sizeBytes: stat.size,
|
|
72
92
|
});
|
|
73
93
|
}
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Transport decoration that rides inside an inbound prompt.
|
|
3
|
+
*
|
|
4
|
+
* Two pieces of text arrive glued to what the person typed and are not part
|
|
5
|
+
* of it:
|
|
6
|
+
*
|
|
7
|
+
* - the `<reply_to …>…</reply_to>` block an adapter appends when the message
|
|
8
|
+
* is a swipe-reply (`adapters/whatsapp.ts` `buildReplyContext`,
|
|
9
|
+
* `bridges/telegram.ts` `buildTelegramReplyContext`) — the platform's own
|
|
10
|
+
* rendering of the quoted message;
|
|
11
|
+
* - the `_(responded in Ns)_` suffix `handler.ts` adds to a delivered reply
|
|
12
|
+
* after the reply is stored, which therefore comes back inside a quote of
|
|
13
|
+
* that reply but never existed in the bot's history.
|
|
14
|
+
*
|
|
15
|
+
* One regex, one place. `confirmation.ts`, the reply-isolation branch and the
|
|
16
|
+
* quote de-duplication in `runtime.ts` all used to carry their own copy of the
|
|
17
|
+
* block pattern; a fix to one was invisible to the others.
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* An adapter-appended quote block. Global and case-insensitive — safe with
|
|
22
|
+
* `String.replace`, stateful with `RegExp.test`/`exec` (reset `lastIndex` or
|
|
23
|
+
* clone before using it that way).
|
|
24
|
+
*/
|
|
25
|
+
export const REPLY_TO_BLOCK_RE = /<reply_to\b[\s\S]*?<\/reply_to>/gi;
|
|
26
|
+
|
|
27
|
+
/** The block together with the blank lines the adapter put before it. */
|
|
28
|
+
const REPLY_TO_BLOCK_WITH_LEAD_RE = /\n*<reply_to\b[\s\S]*?<\/reply_to>/gi;
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* The delivery suffix, when it is the last thing inside a quote block. Only a
|
|
32
|
+
* trailing one is matched: a person quoting a message that *mentions* the
|
|
33
|
+
* phrase mid-body keeps it.
|
|
34
|
+
*/
|
|
35
|
+
const TRAILING_DELIVERY_SUFFIX_RE =
|
|
36
|
+
/\s*_\(responded in \d+s\)_[ \t]*(?=\r?\n?\s*<\/reply_to>)/i;
|
|
37
|
+
|
|
38
|
+
/** The suffix `handler.ts` appends when a status message was shown. */
|
|
39
|
+
export function deliverySuffix(elapsedSeconds: number): string {
|
|
40
|
+
return `_(responded in ${elapsedSeconds}s)_`;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/** Remove every quote block, with the newlines that led into it. */
|
|
44
|
+
export function stripReplyToBlocks(text: string): string {
|
|
45
|
+
return text.replace(REPLY_TO_BLOCK_WITH_LEAD_RE, "");
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/** True when the text carries at least one quote block. */
|
|
49
|
+
export function hasReplyToBlock(text: string): boolean {
|
|
50
|
+
return new RegExp(REPLY_TO_BLOCK_RE.source, "i").test(text);
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Drop a trailing delivery suffix from inside each quote block. The block
|
|
55
|
+
* itself, and everything the person actually quoted, is left as it was.
|
|
56
|
+
*/
|
|
57
|
+
export function scrubDeliverySuffix(text: string): string {
|
|
58
|
+
return text.replace(REPLY_TO_BLOCK_RE, (block) =>
|
|
59
|
+
block.replace(TRAILING_DELIVERY_SUFFIX_RE, ""),
|
|
60
|
+
);
|
|
61
|
+
}
|
package/src/core/routes/chat.ts
CHANGED
|
@@ -3,10 +3,20 @@ import path from "node:path";
|
|
|
3
3
|
import { Hono } from "hono";
|
|
4
4
|
import { logger } from "../../logger.js";
|
|
5
5
|
import { ensureSpaceWorkspace } from "../../storage/memory.js";
|
|
6
|
-
import
|
|
6
|
+
import { stripInvisibleUnicode } from "../../text/sanitize-text.js";
|
|
7
|
+
import type {
|
|
8
|
+
DroppedAttachment,
|
|
9
|
+
IngressMessage,
|
|
10
|
+
MessageAttachment,
|
|
11
|
+
} from "../../types.js";
|
|
7
12
|
import { authorizeApiSecret } from "../auth.js";
|
|
8
13
|
import { type AutoSpaceConfig, resolveConversation } from "../conversation.js";
|
|
9
|
-
import {
|
|
14
|
+
import {
|
|
15
|
+
extToMime,
|
|
16
|
+
hasExecutableHeader,
|
|
17
|
+
isAllowedExtension,
|
|
18
|
+
mimeToMediaType,
|
|
19
|
+
} from "../media.js";
|
|
10
20
|
import type { MercuryCoreRuntime } from "../runtime.js";
|
|
11
21
|
import { isOverQuota } from "../storage-guard.js";
|
|
12
22
|
|
|
@@ -130,6 +140,7 @@ export function createChatRoute(core: MercuryCoreRuntime): Hono {
|
|
|
130
140
|
|
|
131
141
|
// Save incoming files to inbox/
|
|
132
142
|
const attachments: MessageAttachment[] = [];
|
|
143
|
+
const dropped: DroppedAttachment[] = [];
|
|
133
144
|
if (Array.isArray(body.files)) {
|
|
134
145
|
if (body.files.length > 20) {
|
|
135
146
|
return c.json({ error: "Too many files (max 20)" }, 400);
|
|
@@ -141,10 +152,65 @@ export function createChatRoute(core: MercuryCoreRuntime): Hono {
|
|
|
141
152
|
const inboxDir = path.join(workspace, "inbox");
|
|
142
153
|
fs.mkdirSync(inboxDir, { recursive: true });
|
|
143
154
|
|
|
155
|
+
// Same per-file cap the five platform bridges apply in
|
|
156
|
+
// downloadMediaFromUrl — /chat enforced it zero times. Skip-and-log
|
|
157
|
+
// matches downloadMediaFromUrl returning null: one oversized file must
|
|
158
|
+
// not fail the whole turn.
|
|
159
|
+
// `|| 10` mirrors the zod default in config.ts: an AppConfig assembled
|
|
160
|
+
// without it would otherwise yield NaN, and `length > NaN` is always
|
|
161
|
+
// false — a cap that silently does not apply, which is the exact shape
|
|
162
|
+
// of the bug this fix exists to close.
|
|
163
|
+
const maxFileBytes = (core.config.mediaMaxSizeMb || 10) * 1024 * 1024;
|
|
164
|
+
|
|
144
165
|
for (const file of body.files as ChatFileInput[]) {
|
|
145
166
|
if (!file.name || !file.data) continue;
|
|
146
167
|
try {
|
|
168
|
+
// Measured after decoding, not from the encoded length: base64 with
|
|
169
|
+
// embedded newlines inflates that estimate, which would reject files
|
|
170
|
+
// that are actually under the cap. The decode is safe to run because
|
|
171
|
+
// the global body limit already bounds how much can arrive at all.
|
|
147
172
|
const buffer = Buffer.from(file.data, "base64");
|
|
173
|
+
if (buffer.length > maxFileBytes) {
|
|
174
|
+
logger.warn("Skipping oversized chat file", {
|
|
175
|
+
name: file.name,
|
|
176
|
+
sizeBytes: buffer.length,
|
|
177
|
+
maxBytes: maxFileBytes,
|
|
178
|
+
});
|
|
179
|
+
// Tell the agent the file is missing and why. Without this the
|
|
180
|
+
// caller sees a normal reply and the agent invents an explanation
|
|
181
|
+
// for the gap — the failure attachment-notes.ts was written for.
|
|
182
|
+
dropped.push({
|
|
183
|
+
type: mimeToMediaType(extToMime(file.name)),
|
|
184
|
+
reason: "too_large",
|
|
185
|
+
sizeBytes: buffer.length,
|
|
186
|
+
maxBytes: maxFileBytes,
|
|
187
|
+
});
|
|
188
|
+
continue;
|
|
189
|
+
}
|
|
190
|
+
// Part A — extension allowlist, WARN-ONLY. Logs what a rejection
|
|
191
|
+
// would have dropped and accepts the file regardless; the flip is a
|
|
192
|
+
// separate change gated on this log.
|
|
193
|
+
if (!isAllowedExtension(file.name)) {
|
|
194
|
+
logger.warn(
|
|
195
|
+
"Attachment extension not in EXT_TO_MIME (warn-only, accepted)",
|
|
196
|
+
{ name: file.name },
|
|
197
|
+
);
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
// Part B — binary-header denylist, enforced. Checked on the decoded
|
|
201
|
+
// bytes, before anything reaches disk.
|
|
202
|
+
if (hasExecutableHeader(buffer)) {
|
|
203
|
+
logger.warn("Dropping chat file with executable header", {
|
|
204
|
+
name: file.name,
|
|
205
|
+
sizeBytes: buffer.length,
|
|
206
|
+
});
|
|
207
|
+
dropped.push({
|
|
208
|
+
type: mimeToMediaType(extToMime(file.name)),
|
|
209
|
+
reason: "executable_content",
|
|
210
|
+
});
|
|
211
|
+
continue;
|
|
212
|
+
}
|
|
213
|
+
|
|
148
214
|
const safeName = path
|
|
149
215
|
.basename(file.name)
|
|
150
216
|
.replace(/[^a-zA-Z0-9._-]/g, "_");
|
|
@@ -157,7 +223,7 @@ export function createChatRoute(core: MercuryCoreRuntime): Hono {
|
|
|
157
223
|
path: filePath,
|
|
158
224
|
type: mimeToMediaType(mimeType),
|
|
159
225
|
mimeType,
|
|
160
|
-
filename: file.name,
|
|
226
|
+
filename: stripInvisibleUnicode(file.name),
|
|
161
227
|
sizeBytes: buffer.length,
|
|
162
228
|
});
|
|
163
229
|
} catch (err) {
|
|
@@ -189,6 +255,7 @@ export function createChatRoute(core: MercuryCoreRuntime): Hono {
|
|
|
189
255
|
isDM: true,
|
|
190
256
|
isReplyToBot: false,
|
|
191
257
|
attachments,
|
|
258
|
+
droppedAttachments: dropped.length > 0 ? dropped : undefined,
|
|
192
259
|
};
|
|
193
260
|
|
|
194
261
|
logger.info("API chat inbound", {
|
|
@@ -16,6 +16,7 @@ export const BUILTIN_CONFIG_KEYS = new Set([
|
|
|
16
16
|
"rate_limit.admin",
|
|
17
17
|
"debounce.idle_timeout_ms",
|
|
18
18
|
"messages.locale",
|
|
19
|
+
"persona.exclusive",
|
|
19
20
|
]);
|
|
20
21
|
|
|
21
22
|
/**
|
|
@@ -55,6 +56,8 @@ export const BUILTIN_CONFIG_DESCRIPTIONS: Record<string, string> = {
|
|
|
55
56
|
"Milliseconds to wait for additional messages before processing a batch. 0 disables debounce. Platform default: 2000 for WhatsApp/Telegram, 0 for others.",
|
|
56
57
|
"messages.locale":
|
|
57
58
|
"Language for Mercury-generated system messages (rate limits, errors, denials) in this space: 'en' or 'he'. Unset falls back to the deployment default.",
|
|
59
|
+
"persona.exclusive":
|
|
60
|
+
"When true, this space's AGENTS.md is the only persona the model hears: the owner's global character and the global AGENTS.md are left out of its prompt. Unset/false keeps both (the default).",
|
|
58
61
|
};
|
|
59
62
|
|
|
60
63
|
const BUILTIN_VALIDATORS: Record<string, (v: string) => string | null> = {
|
|
@@ -126,6 +129,10 @@ const BUILTIN_VALIDATORS: Record<string, (v: string) => string | null> = {
|
|
|
126
129
|
["en", "he"].includes(v)
|
|
127
130
|
? null
|
|
128
131
|
: "Invalid messages.locale value. Valid: en, he",
|
|
132
|
+
"persona.exclusive": (v) =>
|
|
133
|
+
["true", "false"].includes(v)
|
|
134
|
+
? null
|
|
135
|
+
: "Invalid persona.exclusive value. Valid: true, false",
|
|
129
136
|
};
|
|
130
137
|
|
|
131
138
|
export function isBuiltinConfigKey(key: string): boolean {
|