@nanhara/hara 0.122.0 → 0.122.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +62 -0
- package/README.md +24 -11
- package/SECURITY.md +48 -9
- package/dist/agent/loop.js +33 -10
- package/dist/agent/touched.js +4 -0
- package/dist/checkpoints.js +103 -17
- package/dist/cli.js +16 -0
- package/dist/config.js +141 -12
- package/dist/context/agents-md.js +44 -9
- package/dist/context/mentions.js +10 -4
- package/dist/context/subdir-hints.js +40 -7
- package/dist/cron/runner.js +372 -37
- package/dist/cron/store.js +11 -3
- package/dist/exec/jobs.js +88 -20
- package/dist/fs-read.js +318 -3
- package/dist/fs-walk.js +8 -2
- package/dist/fs-write.js +197 -11
- package/dist/gateway/discord.js +2 -4
- package/dist/gateway/feishu.js +4 -6
- package/dist/gateway/flows-pending.js +38 -31
- package/dist/gateway/matrix.js +3 -5
- package/dist/gateway/mattermost.js +2 -4
- package/dist/gateway/outbound-files.js +379 -0
- package/dist/gateway/serve.js +121 -73
- package/dist/gateway/signal.js +4 -6
- package/dist/gateway/slack.js +4 -6
- package/dist/gateway/telegram.js +3 -5
- package/dist/gateway/tmux-routes.js +11 -3
- package/dist/gateway/wecom.js +7 -8
- package/dist/gateway/weixin.js +22 -12
- package/dist/hooks.js +12 -6
- package/dist/index.js +142 -61
- package/dist/mcp/client.js +164 -12
- package/dist/memory/store.js +68 -22
- package/dist/org/planner.js +36 -10
- package/dist/org/review-chain.js +360 -24
- package/dist/org/roles.js +4 -2
- package/dist/profile/profile.js +152 -27
- package/dist/recall.js +4 -2
- package/dist/runtime.js +37 -0
- package/dist/sandbox.js +142 -33
- package/dist/search/semindex.js +182 -53
- package/dist/search/zvec-store.js +121 -42
- package/dist/security/permissions.js +326 -19
- package/dist/security/private-state.js +299 -0
- package/dist/security/project-trust.js +6 -0
- package/dist/security/sensitive-files.js +723 -0
- package/dist/security/subprocess-env.js +210 -0
- package/dist/serve/server.js +2 -1
- package/dist/skills/skills.js +16 -7
- package/dist/tools/builtin.js +53 -27
- package/dist/tools/cron.js +6 -0
- package/dist/tools/edit.js +15 -5
- package/dist/tools/external_agent.js +110 -16
- package/dist/tools/memory.js +38 -8
- package/dist/tools/patch.js +37 -17
- package/dist/tools/search.js +100 -40
- package/dist/tools/send.js +11 -5
- package/package.json +11 -10
- package/runtime-bootstrap.cjs +72 -0
package/dist/gateway/discord.js
CHANGED
|
@@ -4,8 +4,6 @@
|
|
|
4
4
|
// the cross-platform gateway plumbing (send_file, in-chat system context, stuck-guard, image attach/describe)
|
|
5
5
|
// works unchanged. NOTE: receiving message text needs the privileged "Message Content Intent" enabled for the
|
|
6
6
|
// bot in the Discord developer portal.
|
|
7
|
-
import { readFileSync } from "node:fs";
|
|
8
|
-
import { basename } from "node:path";
|
|
9
7
|
import { InboundMediaBudget, savePrivateResponse } from "./media.js";
|
|
10
8
|
import { chunkText } from "./telegram.js";
|
|
11
9
|
const REST = "https://discord.com/api/v10";
|
|
@@ -75,10 +73,10 @@ export function discordAdapter(token) {
|
|
|
75
73
|
}).catch(() => { });
|
|
76
74
|
}
|
|
77
75
|
},
|
|
78
|
-
async sendFile(chatId,
|
|
76
|
+
async sendFile(chatId, file) {
|
|
79
77
|
const form = new FormData();
|
|
80
78
|
form.append("payload_json", JSON.stringify({}));
|
|
81
|
-
form.append("files[0]", new Blob([
|
|
79
|
+
form.append("files[0]", new Blob([new Uint8Array(file.bytes)]), file.safeName);
|
|
82
80
|
await fetch(`${REST}/channels/${chatId}/messages`, { method: "POST", headers: auth, body: form }).catch(() => { });
|
|
83
81
|
},
|
|
84
82
|
async start(onMessage, signal, shouldDownload) {
|
package/dist/gateway/feishu.js
CHANGED
|
@@ -8,8 +8,6 @@
|
|
|
8
8
|
// every binaries release fail. This form works under both resolutions.
|
|
9
9
|
import * as larkNs from "@larksuiteoapi/node-sdk";
|
|
10
10
|
const lark = (larkNs.default ?? larkNs);
|
|
11
|
-
import { createReadStream } from "node:fs";
|
|
12
|
-
import { basename } from "node:path";
|
|
13
11
|
import { chunkText } from "./telegram.js";
|
|
14
12
|
import { InboundMediaBudget, cleanupTransientMedia, savePrivateMedia } from "./media.js";
|
|
15
13
|
/** Normalize a Feishu message's parsed content by type → text + any media keys (pure; download done by caller). */
|
|
@@ -203,18 +201,18 @@ export function feishuAdapter(appId, appSecret) {
|
|
|
203
201
|
/* best-effort cleanup — an unrecallable message just stays */
|
|
204
202
|
}
|
|
205
203
|
},
|
|
206
|
-
async sendFile(chatId,
|
|
207
|
-
const name =
|
|
204
|
+
async sendFile(chatId, file) {
|
|
205
|
+
const name = file.safeName;
|
|
208
206
|
const isImg = /\.(png|jpe?g|gif|webp)$/i.test(name);
|
|
209
207
|
if (isImg) {
|
|
210
|
-
const up = await client.im.image.create({ data: { image_type: "message", image:
|
|
208
|
+
const up = await client.im.image.create({ data: { image_type: "message", image: file.bytes } });
|
|
211
209
|
const key = up?.image_key ?? up?.data?.image_key;
|
|
212
210
|
if (!key)
|
|
213
211
|
throw new Error("Feishu image upload returned no image_key");
|
|
214
212
|
await sendMsg(chatId, "image", { image_key: key });
|
|
215
213
|
}
|
|
216
214
|
else {
|
|
217
|
-
const up = await client.im.file.create({ data: { file_type: "stream", file_name: name, file:
|
|
215
|
+
const up = await client.im.file.create({ data: { file_type: "stream", file_name: name, file: file.bytes } });
|
|
218
216
|
const key = up?.file_key ?? up?.data?.file_key;
|
|
219
217
|
if (!key)
|
|
220
218
|
throw new Error("Feishu file upload returned no file_key");
|
|
@@ -22,6 +22,7 @@ import { createOpenAIProvider } from "../providers/openai.js";
|
|
|
22
22
|
import { getValidQwenAuth } from "../providers/qwen-oauth.js";
|
|
23
23
|
import { resolvePlatform } from "../providers/registry.js";
|
|
24
24
|
import { validateAgainstSchema } from "../agent/structured.js";
|
|
25
|
+
import { terminateSubprocessTree } from "../security/subprocess-env.js";
|
|
25
26
|
const FILE = () => join(homedir(), ".hara", "flows-pending.json");
|
|
26
27
|
const LOCK = () => FILE() + ".lock";
|
|
27
28
|
const sleepCell = new Int32Array(new SharedArrayBuffer(4));
|
|
@@ -387,18 +388,6 @@ function boundedProcessDuration(value, fallback, max) {
|
|
|
387
388
|
export function approvedOrgTimeoutMs(value = process.env.HARA_GATEWAY_ORG_TIMEOUT_MS) {
|
|
388
389
|
return boundedProcessDuration(value, DEFAULT_APPROVED_ORG_TIMEOUT_MS, MAX_APPROVED_ORG_TIMEOUT_MS);
|
|
389
390
|
}
|
|
390
|
-
function signalChildGroup(child, signal) {
|
|
391
|
-
try {
|
|
392
|
-
if (process.platform !== "win32" && child.pid) {
|
|
393
|
-
process.kill(-child.pid, signal);
|
|
394
|
-
return true;
|
|
395
|
-
}
|
|
396
|
-
return child.kill(signal);
|
|
397
|
-
}
|
|
398
|
-
catch {
|
|
399
|
-
return false;
|
|
400
|
-
}
|
|
401
|
-
}
|
|
402
391
|
/** Run an approved delegation as one bounded process group. Exported for lifecycle regression tests. */
|
|
403
392
|
export function runApprovedOrgProcess(command, args, options = {}) {
|
|
404
393
|
if (options.signal?.aborted) {
|
|
@@ -425,10 +414,13 @@ export function runApprovedOrgProcess(command, args, options = {}) {
|
|
|
425
414
|
let code = null;
|
|
426
415
|
let exitSignal = null;
|
|
427
416
|
let stopReason;
|
|
428
|
-
let killTimer;
|
|
429
417
|
let drainTimer;
|
|
430
|
-
let
|
|
418
|
+
let forceIssued = false;
|
|
419
|
+
let terminationError;
|
|
420
|
+
let cancelTermination;
|
|
431
421
|
const cap = (data) => {
|
|
422
|
+
if (settled || stopReason)
|
|
423
|
+
return;
|
|
432
424
|
output = (output + data.toString()).slice(-8_000);
|
|
433
425
|
};
|
|
434
426
|
child.stdout?.on("data", cap);
|
|
@@ -441,43 +433,58 @@ export function runApprovedOrgProcess(command, args, options = {}) {
|
|
|
441
433
|
return;
|
|
442
434
|
settled = true;
|
|
443
435
|
clearTimeout(timeoutTimer);
|
|
444
|
-
if (killTimer)
|
|
445
|
-
clearTimeout(killTimer);
|
|
446
436
|
if (drainTimer)
|
|
447
437
|
clearTimeout(drainTimer);
|
|
448
|
-
|
|
449
|
-
|
|
438
|
+
// Preserve an already-scheduled force signal when the direct child closed after TERM. See the shared
|
|
439
|
+
// helper: close cancels only its API fallback, never the process-group SIGKILL.
|
|
440
|
+
cancelTermination?.();
|
|
450
441
|
options.signal?.removeEventListener("abort", abortRun);
|
|
451
442
|
child.stdout?.destroy();
|
|
452
443
|
child.stderr?.destroy();
|
|
453
444
|
resolveResult({ output, code, signal: exitSignal, ...(stopReason ? { stopReason } : {}), ...extra });
|
|
454
445
|
};
|
|
455
|
-
const finishFromExit = () =>
|
|
446
|
+
const finishFromExit = () => {
|
|
447
|
+
if (stopReason && !forceIssued)
|
|
448
|
+
return;
|
|
449
|
+
finish(terminationError ? { error: terminationError } : {});
|
|
450
|
+
};
|
|
456
451
|
const terminate = (reason) => {
|
|
457
|
-
if (settled)
|
|
452
|
+
if (settled || cancelTermination)
|
|
458
453
|
return;
|
|
459
454
|
stopReason ??= reason;
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
455
|
+
cancelTermination = terminateSubprocessTree(child, {
|
|
456
|
+
processGroup: process.platform !== "win32",
|
|
457
|
+
graceMs: killGraceMs,
|
|
458
|
+
fallbackMs: 250,
|
|
459
|
+
onForce: () => {
|
|
460
|
+
forceIssued = true;
|
|
461
|
+
if (exited)
|
|
462
|
+
finishFromExit();
|
|
463
|
+
},
|
|
464
|
+
onFallback: finishFromExit,
|
|
465
|
+
});
|
|
469
466
|
};
|
|
470
467
|
const abortRun = () => terminate("shutdown");
|
|
471
468
|
const timeoutTimer = setTimeout(() => terminate("timeout"), timeoutMs);
|
|
472
469
|
options.signal?.addEventListener("abort", abortRun, { once: true });
|
|
473
470
|
if (options.signal?.aborted)
|
|
474
471
|
abortRun();
|
|
475
|
-
child.once("error", (error) =>
|
|
472
|
+
child.once("error", (error) => {
|
|
473
|
+
if (!stopReason)
|
|
474
|
+
return finish({ error: error.message });
|
|
475
|
+
terminationError = error.message;
|
|
476
|
+
exited = true;
|
|
477
|
+
if (forceIssued)
|
|
478
|
+
finishFromExit();
|
|
479
|
+
});
|
|
476
480
|
child.once("exit", (nextCode, nextSignal) => {
|
|
477
481
|
exited = true;
|
|
478
482
|
code = nextCode;
|
|
479
483
|
exitSignal = nextSignal;
|
|
480
|
-
|
|
484
|
+
if (stopReason)
|
|
485
|
+
finishFromExit();
|
|
486
|
+
else
|
|
487
|
+
drainTimer = setTimeout(finishFromExit, 100);
|
|
481
488
|
});
|
|
482
489
|
child.once("close", (nextCode, nextSignal) => {
|
|
483
490
|
exited = true;
|
package/dist/gateway/matrix.js
CHANGED
|
@@ -7,8 +7,6 @@
|
|
|
7
7
|
// LIMITATION (v1): NO end-to-end encryption. Encrypted rooms (m.room.encrypted events) are skipped — only
|
|
8
8
|
// plaintext rooms work. E2EE would need libolm + a crypto store (see hermes' matrix-nio adapter), which breaks
|
|
9
9
|
// the zero-dep constraint. Invite this bot into UNENCRYPTED rooms only.
|
|
10
|
-
import { readFileSync } from "node:fs";
|
|
11
|
-
import { basename } from "node:path";
|
|
12
10
|
import { InboundMediaBudget, savePrivateResponse } from "./media.js";
|
|
13
11
|
import { chunkText } from "./telegram.js";
|
|
14
12
|
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
@@ -159,15 +157,15 @@ export function matrixAdapter(homeserver, token, selfUserId) {
|
|
|
159
157
|
}).catch(() => { });
|
|
160
158
|
}
|
|
161
159
|
},
|
|
162
|
-
async sendFile(chatId,
|
|
163
|
-
const name =
|
|
160
|
+
async sendFile(chatId, file) {
|
|
161
|
+
const name = file.safeName;
|
|
164
162
|
const mime = mimeFromExt(name);
|
|
165
163
|
try {
|
|
166
164
|
// 1) upload bytes → content_uri (mxc://)
|
|
167
165
|
const up = await fetch(`${base}/_matrix/media/v3/upload?filename=${encodeURIComponent(name)}`, {
|
|
168
166
|
method: "POST",
|
|
169
167
|
headers: { ...auth, "content-type": mime },
|
|
170
|
-
body:
|
|
168
|
+
body: new Uint8Array(file.bytes),
|
|
171
169
|
});
|
|
172
170
|
if (!up.ok)
|
|
173
171
|
return;
|
|
@@ -4,8 +4,6 @@
|
|
|
4
4
|
// allow users via HARA_GATEWAY_ALLOWED (Mattermost user ids). Same ChatAdapter shape as Telegram/Discord, so all
|
|
5
5
|
// the cross-platform gateway plumbing (send_file, in-chat system context, stuck-guard, image attach/describe)
|
|
6
6
|
// works unchanged. Auth is a WS "authentication_challenge" rather than an HTTP header.
|
|
7
|
-
import { readFileSync } from "node:fs";
|
|
8
|
-
import { basename } from "node:path";
|
|
9
7
|
import { InboundMediaBudget, savePrivateResponse } from "./media.js";
|
|
10
8
|
import { chunkText } from "./telegram.js";
|
|
11
9
|
const WSImpl = globalThis.WebSocket;
|
|
@@ -85,11 +83,11 @@ export function mattermostAdapter(serverUrl, token) {
|
|
|
85
83
|
}).catch(() => { });
|
|
86
84
|
}
|
|
87
85
|
},
|
|
88
|
-
async sendFile(chatId,
|
|
86
|
+
async sendFile(chatId, file) {
|
|
89
87
|
// upload the file (multipart) → get a file_id, then create a post referencing it
|
|
90
88
|
const form = new FormData();
|
|
91
89
|
form.append("channel_id", String(chatId));
|
|
92
|
-
form.append("files", new Blob([
|
|
90
|
+
form.append("files", new Blob([new Uint8Array(file.bytes)]), file.safeName);
|
|
93
91
|
const up = await fetch(`${api}/files`, { method: "POST", headers: auth, body: form }).catch(() => null);
|
|
94
92
|
if (!up || !up.ok)
|
|
95
93
|
return;
|
|
@@ -0,0 +1,379 @@
|
|
|
1
|
+
// Immutable, owner-only snapshots for files leaving the machine through a gateway adapter. `send_file`
|
|
2
|
+
// runs in a child process and the adapter sends later in the daemon; queueing an original pathname would
|
|
3
|
+
// leave a TOCTOU window where that pathname could be exchanged for a symlink or a different file.
|
|
4
|
+
import { randomUUID } from "node:crypto";
|
|
5
|
+
import { chmodSync, constants, lstatSync, mkdirSync, readdirSync, realpathSync, renameSync, rmdirSync, unlinkSync, } from "node:fs";
|
|
6
|
+
import { open } from "node:fs/promises";
|
|
7
|
+
import { basename, dirname, extname, join, resolve } from "node:path";
|
|
8
|
+
import { openVerifiedRegularFileNoFollow, verifyOpenedRegularFileSync } from "../fs-read.js";
|
|
9
|
+
export const OUTBOUND_FILE_MAX_BYTES = 20 * 1024 * 1024;
|
|
10
|
+
export const OUTBOUND_BATCH_MAX_BYTES = 20 * 1024 * 1024;
|
|
11
|
+
export const OUTBOUND_BATCH_MAX_FILES = 4;
|
|
12
|
+
const OUTBOX_MAX_BYTES = 256 * 1024;
|
|
13
|
+
const UUID = "[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}";
|
|
14
|
+
const OWNED_SNAPSHOT = new RegExp(`^(?:\\.${UUID}\\.part|[a-z0-9][a-z0-9_-]{0,48}-${UUID}(?:\\.[a-z0-9]{1,12})?)$`, "i");
|
|
15
|
+
const SNAPSHOT_SUFFIX = new RegExp(`-${UUID}(?=\\.|$)`, "i");
|
|
16
|
+
// Cleanup receives only the cleanup-only path from serve.ts. Remember the identity that produced its payload so
|
|
17
|
+
// a later path replacement is never mistaken for the consumed snapshot.
|
|
18
|
+
const consumedSnapshotIdentities = new Map();
|
|
19
|
+
const outboundQueueLocks = new Map();
|
|
20
|
+
export function outboundSnapshotDir(outbox) {
|
|
21
|
+
return `${resolve(outbox)}.files`;
|
|
22
|
+
}
|
|
23
|
+
function ownedByProcess(path) {
|
|
24
|
+
if (typeof process.getuid !== "function")
|
|
25
|
+
return true;
|
|
26
|
+
return lstatSync(path).uid === process.getuid();
|
|
27
|
+
}
|
|
28
|
+
function ensureSnapshotDir(outbox) {
|
|
29
|
+
const dir = outboundSnapshotDir(outbox);
|
|
30
|
+
try {
|
|
31
|
+
mkdirSync(dir, { mode: 0o700 });
|
|
32
|
+
}
|
|
33
|
+
catch (error) {
|
|
34
|
+
if (error?.code !== "EEXIST")
|
|
35
|
+
throw error;
|
|
36
|
+
}
|
|
37
|
+
const info = lstatSync(dir);
|
|
38
|
+
if (!info.isDirectory() || info.isSymbolicLink() || !ownedByProcess(dir)) {
|
|
39
|
+
throw new Error(`unsafe gateway snapshot directory: ${dir}`);
|
|
40
|
+
}
|
|
41
|
+
chmodSync(dir, 0o700);
|
|
42
|
+
return dir;
|
|
43
|
+
}
|
|
44
|
+
function safeExtension(source) {
|
|
45
|
+
const extension = extname(basename(source)).toLowerCase();
|
|
46
|
+
return /^\.[a-z0-9]{1,12}$/.test(extension) ? extension : "";
|
|
47
|
+
}
|
|
48
|
+
function safeStem(source) {
|
|
49
|
+
const extension = extname(basename(source));
|
|
50
|
+
const raw = basename(source, extension)
|
|
51
|
+
.normalize("NFKD")
|
|
52
|
+
.replace(/[^a-z0-9_-]+/gi, "-")
|
|
53
|
+
.replace(/^-+|-+$/g, "")
|
|
54
|
+
.slice(0, 48);
|
|
55
|
+
return raw || "attachment";
|
|
56
|
+
}
|
|
57
|
+
function existingBatchUsage(dir) {
|
|
58
|
+
let bytes = 0;
|
|
59
|
+
let files = 0;
|
|
60
|
+
for (const name of readdirSync(dir)) {
|
|
61
|
+
if (!OWNED_SNAPSHOT.test(name))
|
|
62
|
+
continue;
|
|
63
|
+
const path = join(dir, name);
|
|
64
|
+
const info = lstatSync(path);
|
|
65
|
+
if (!info.isFile() || info.isSymbolicLink() || info.nlink > 1 || !ownedByProcess(path)) {
|
|
66
|
+
throw new Error(`unsafe gateway snapshot entry: ${path}`);
|
|
67
|
+
}
|
|
68
|
+
files++;
|
|
69
|
+
bytes += info.size;
|
|
70
|
+
if (!Number.isSafeInteger(bytes))
|
|
71
|
+
throw new Error("gateway send batch size is invalid");
|
|
72
|
+
}
|
|
73
|
+
return { bytes, files };
|
|
74
|
+
}
|
|
75
|
+
async function appendOutbox(outbox, snapshot) {
|
|
76
|
+
const path = resolve(outbox);
|
|
77
|
+
let before;
|
|
78
|
+
try {
|
|
79
|
+
before = lstatSync(path);
|
|
80
|
+
if (!before.isFile() || before.isSymbolicLink() || before.nlink > 1 || !ownedByProcess(path)) {
|
|
81
|
+
throw new Error(`unsafe gateway outbox: ${outbox}`);
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
catch (error) {
|
|
85
|
+
if (error?.code !== "ENOENT")
|
|
86
|
+
throw error;
|
|
87
|
+
}
|
|
88
|
+
const noFollow = typeof constants.O_NOFOLLOW === "number" ? constants.O_NOFOLLOW : 0;
|
|
89
|
+
const handle = await open(path, constants.O_WRONLY | constants.O_APPEND | constants.O_CREAT | noFollow, 0o600);
|
|
90
|
+
try {
|
|
91
|
+
const info = await handle.stat();
|
|
92
|
+
const current = lstatSync(path);
|
|
93
|
+
if (!info.isFile()
|
|
94
|
+
|| info.nlink > 1
|
|
95
|
+
|| current.isSymbolicLink()
|
|
96
|
+
|| current.dev !== info.dev
|
|
97
|
+
|| current.ino !== info.ino
|
|
98
|
+
|| (before && (before.dev !== info.dev || before.ino !== info.ino))
|
|
99
|
+
|| !ownedByProcess(path))
|
|
100
|
+
throw new Error(`unsafe gateway outbox: ${outbox}`);
|
|
101
|
+
await handle.chmod(0o600);
|
|
102
|
+
await handle.writeFile(snapshot + "\n", "utf8");
|
|
103
|
+
await handle.sync();
|
|
104
|
+
}
|
|
105
|
+
finally {
|
|
106
|
+
await handle.close().catch(() => { });
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
/** Copy a verified source fd to a private immutable snapshot, then append only that snapshot to the queue. */
|
|
110
|
+
async function queueOutboundSnapshotLocked(sourcePath, outbox) {
|
|
111
|
+
const source = resolve(sourcePath);
|
|
112
|
+
const verified = await openVerifiedRegularFileNoFollow(source, {
|
|
113
|
+
action: "send",
|
|
114
|
+
rejectHardLinks: true,
|
|
115
|
+
protectSensitive: true,
|
|
116
|
+
});
|
|
117
|
+
let destination;
|
|
118
|
+
let temporary;
|
|
119
|
+
let snapshot;
|
|
120
|
+
let published = false;
|
|
121
|
+
try {
|
|
122
|
+
if (verified.info.size > OUTBOUND_FILE_MAX_BYTES) {
|
|
123
|
+
throw new Error(`file exceeds the ${OUTBOUND_FILE_MAX_BYTES}-byte gateway send limit`);
|
|
124
|
+
}
|
|
125
|
+
const dir = ensureSnapshotDir(outbox);
|
|
126
|
+
const usage = existingBatchUsage(dir);
|
|
127
|
+
if (usage.files >= OUTBOUND_BATCH_MAX_FILES) {
|
|
128
|
+
throw new Error(`gateway send batch exceeds the ${OUTBOUND_BATCH_MAX_FILES}-file limit`);
|
|
129
|
+
}
|
|
130
|
+
if (verified.info.size > OUTBOUND_BATCH_MAX_BYTES - usage.bytes) {
|
|
131
|
+
throw new Error(`gateway send batch exceeds the ${OUTBOUND_BATCH_MAX_BYTES}-byte limit`);
|
|
132
|
+
}
|
|
133
|
+
const id = randomUUID();
|
|
134
|
+
temporary = join(dir, `.${id}.part`);
|
|
135
|
+
// Keep a recognizable, sanitized stem for a friendlier attachment name while the UUID preserves uniqueness.
|
|
136
|
+
snapshot = join(dir, `${safeStem(source)}-${id}${safeExtension(source)}`);
|
|
137
|
+
destination = await open(temporary, "wx", 0o600);
|
|
138
|
+
const buffer = Buffer.allocUnsafe(64 * 1024);
|
|
139
|
+
let position = 0;
|
|
140
|
+
while (position < verified.info.size) {
|
|
141
|
+
const want = Math.min(buffer.length, verified.info.size - position);
|
|
142
|
+
const { bytesRead } = await verified.handle.read(buffer, 0, want, position);
|
|
143
|
+
if (bytesRead <= 0)
|
|
144
|
+
throw new Error(`source changed while snapshotting ${source}`);
|
|
145
|
+
let written = 0;
|
|
146
|
+
while (written < bytesRead) {
|
|
147
|
+
const result = await destination.write(buffer, written, bytesRead - written, position + written);
|
|
148
|
+
if (result.bytesWritten <= 0)
|
|
149
|
+
throw new Error("failed to write gateway snapshot");
|
|
150
|
+
written += result.bytesWritten;
|
|
151
|
+
}
|
|
152
|
+
position += bytesRead;
|
|
153
|
+
}
|
|
154
|
+
const latest = await verified.handle.stat();
|
|
155
|
+
verifyOpenedRegularFileSync(source, latest, {
|
|
156
|
+
action: "send",
|
|
157
|
+
rejectHardLinks: true,
|
|
158
|
+
protectSensitive: true,
|
|
159
|
+
});
|
|
160
|
+
if (latest.dev !== verified.info.dev
|
|
161
|
+
|| latest.ino !== verified.info.ino
|
|
162
|
+
|| latest.size !== verified.info.size
|
|
163
|
+
|| latest.mtimeMs !== verified.info.mtimeMs
|
|
164
|
+
|| latest.ctimeMs !== verified.info.ctimeMs)
|
|
165
|
+
throw new Error(`source changed while snapshotting ${source}`);
|
|
166
|
+
await destination.sync();
|
|
167
|
+
await destination.chmod(0o600);
|
|
168
|
+
await destination.close();
|
|
169
|
+
destination = undefined;
|
|
170
|
+
renameSync(temporary, snapshot);
|
|
171
|
+
published = true;
|
|
172
|
+
await appendOutbox(outbox, snapshot);
|
|
173
|
+
return snapshot;
|
|
174
|
+
}
|
|
175
|
+
catch (error) {
|
|
176
|
+
if (published && snapshot) {
|
|
177
|
+
try {
|
|
178
|
+
unlinkSync(snapshot);
|
|
179
|
+
}
|
|
180
|
+
catch { /* best-effort cleanup */ }
|
|
181
|
+
}
|
|
182
|
+
if (temporary) {
|
|
183
|
+
try {
|
|
184
|
+
unlinkSync(temporary);
|
|
185
|
+
}
|
|
186
|
+
catch { /* best-effort cleanup */ }
|
|
187
|
+
}
|
|
188
|
+
throw error;
|
|
189
|
+
}
|
|
190
|
+
finally {
|
|
191
|
+
await destination?.close().catch(() => { });
|
|
192
|
+
await verified.handle.close().catch(() => { });
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
/** Serialize admissions for one outbox so parallel send_file calls cannot race past the aggregate budget. */
|
|
196
|
+
export async function queueOutboundSnapshot(sourcePath, outbox) {
|
|
197
|
+
const key = resolve(outbox);
|
|
198
|
+
const previous = outboundQueueLocks.get(key) ?? Promise.resolve();
|
|
199
|
+
let release;
|
|
200
|
+
const gate = new Promise((resolveGate) => { release = resolveGate; });
|
|
201
|
+
const current = previous.then(() => gate, () => gate);
|
|
202
|
+
outboundQueueLocks.set(key, current);
|
|
203
|
+
await previous.catch(() => { });
|
|
204
|
+
try {
|
|
205
|
+
return await queueOutboundSnapshotLocked(sourcePath, outbox);
|
|
206
|
+
}
|
|
207
|
+
finally {
|
|
208
|
+
release();
|
|
209
|
+
if (outboundQueueLocks.get(key) === current)
|
|
210
|
+
outboundQueueLocks.delete(key);
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
async function readOutbox(outbox) {
|
|
214
|
+
let verified;
|
|
215
|
+
try {
|
|
216
|
+
verified = await openVerifiedRegularFileNoFollow(resolve(outbox), {
|
|
217
|
+
action: "read gateway outbox",
|
|
218
|
+
rejectHardLinks: true,
|
|
219
|
+
protectSensitive: false,
|
|
220
|
+
});
|
|
221
|
+
if (verified.info.size > OUTBOX_MAX_BYTES)
|
|
222
|
+
throw new Error("gateway outbox is too large");
|
|
223
|
+
const bytes = Buffer.alloc(verified.info.size);
|
|
224
|
+
let offset = 0;
|
|
225
|
+
while (offset < bytes.length) {
|
|
226
|
+
const { bytesRead } = await verified.handle.read(bytes, offset, bytes.length - offset, offset);
|
|
227
|
+
if (!bytesRead)
|
|
228
|
+
break;
|
|
229
|
+
offset += bytesRead;
|
|
230
|
+
}
|
|
231
|
+
return bytes.subarray(0, offset).toString("utf8").split("\n").filter(Boolean);
|
|
232
|
+
}
|
|
233
|
+
catch {
|
|
234
|
+
return [];
|
|
235
|
+
}
|
|
236
|
+
finally {
|
|
237
|
+
await verified?.handle.close().catch(() => { });
|
|
238
|
+
try {
|
|
239
|
+
const current = lstatSync(resolve(outbox));
|
|
240
|
+
if (verified && !current.isSymbolicLink() && current.dev === verified.info.dev && current.ino === verified.info.ino) {
|
|
241
|
+
unlinkSync(resolve(outbox));
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
catch { /* missing or replaced outbox is left alone */ }
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
function attachmentName(snapshotPath) {
|
|
248
|
+
return basename(snapshotPath).replace(SNAPSHOT_SUFFIX, "") || "attachment";
|
|
249
|
+
}
|
|
250
|
+
/**
|
|
251
|
+
* Drain an outbox and materialize only verified snapshots owned by this queue. The returned bytes are the
|
|
252
|
+
* security boundary: adapters never reopen `snapshotPath`, so replacing it after this function returns cannot
|
|
253
|
+
* alter or disclose what is uploaded. Per-file, batch-byte, and file-count caps bound the in-memory payload.
|
|
254
|
+
*/
|
|
255
|
+
export async function consumeOutboundSnapshots(outbox) {
|
|
256
|
+
const dir = outboundSnapshotDir(outbox);
|
|
257
|
+
const queued = await readOutbox(outbox); // always drain/unlink the outbox, even if its snapshot dir is absent
|
|
258
|
+
let dirReal;
|
|
259
|
+
try {
|
|
260
|
+
const info = lstatSync(dir);
|
|
261
|
+
if (!info.isDirectory() || info.isSymbolicLink() || !ownedByProcess(dir))
|
|
262
|
+
return [];
|
|
263
|
+
dirReal = realpathSync.native(dir);
|
|
264
|
+
}
|
|
265
|
+
catch {
|
|
266
|
+
return [];
|
|
267
|
+
}
|
|
268
|
+
const accepted = [];
|
|
269
|
+
let acceptedBytes = 0;
|
|
270
|
+
for (const raw of queued) {
|
|
271
|
+
if (accepted.length >= OUTBOUND_BATCH_MAX_FILES || acceptedBytes >= OUTBOUND_BATCH_MAX_BYTES)
|
|
272
|
+
break;
|
|
273
|
+
const candidate = resolve(raw);
|
|
274
|
+
if (dirname(candidate) !== dir || !OWNED_SNAPSHOT.test(basename(candidate)))
|
|
275
|
+
continue;
|
|
276
|
+
let verified;
|
|
277
|
+
try {
|
|
278
|
+
verified = await openVerifiedRegularFileNoFollow(candidate, {
|
|
279
|
+
action: "send gateway snapshot",
|
|
280
|
+
rejectHardLinks: true,
|
|
281
|
+
protectSensitive: false,
|
|
282
|
+
});
|
|
283
|
+
if (dirname(verified.canonicalPath) !== dirReal
|
|
284
|
+
|| verified.info.size > OUTBOUND_FILE_MAX_BYTES
|
|
285
|
+
|| verified.info.size > OUTBOUND_BATCH_MAX_BYTES - acceptedBytes
|
|
286
|
+
|| !ownedByProcess(candidate))
|
|
287
|
+
continue;
|
|
288
|
+
const bytes = Buffer.allocUnsafe(verified.info.size);
|
|
289
|
+
let offset = 0;
|
|
290
|
+
while (offset < bytes.length) {
|
|
291
|
+
const { bytesRead } = await verified.handle.read(bytes, offset, bytes.length - offset, offset);
|
|
292
|
+
if (!bytesRead)
|
|
293
|
+
break;
|
|
294
|
+
offset += bytesRead;
|
|
295
|
+
}
|
|
296
|
+
if (offset !== bytes.length)
|
|
297
|
+
continue;
|
|
298
|
+
const latest = await verified.handle.stat();
|
|
299
|
+
verifyOpenedRegularFileSync(candidate, latest, {
|
|
300
|
+
action: "send gateway snapshot",
|
|
301
|
+
rejectHardLinks: true,
|
|
302
|
+
protectSensitive: false,
|
|
303
|
+
});
|
|
304
|
+
if (latest.dev !== verified.info.dev
|
|
305
|
+
|| latest.ino !== verified.info.ino
|
|
306
|
+
|| latest.size !== verified.info.size
|
|
307
|
+
|| latest.mtimeMs !== verified.info.mtimeMs
|
|
308
|
+
|| latest.ctimeMs !== verified.info.ctimeMs)
|
|
309
|
+
continue;
|
|
310
|
+
consumedSnapshotIdentities.set(candidate, { dev: latest.dev, ino: latest.ino });
|
|
311
|
+
accepted.push({ snapshotPath: candidate, safeName: attachmentName(candidate), bytes });
|
|
312
|
+
acceptedBytes += bytes.length;
|
|
313
|
+
}
|
|
314
|
+
catch {
|
|
315
|
+
/* malformed/replaced queue entry is never delivered */
|
|
316
|
+
}
|
|
317
|
+
finally {
|
|
318
|
+
await verified?.handle.close().catch(() => { });
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
return accepted;
|
|
322
|
+
}
|
|
323
|
+
/** Remove only files in this queue's private snapshot directory, never arbitrary outbox entries. */
|
|
324
|
+
export function cleanupOutboundSnapshots(outbox, preserve = []) {
|
|
325
|
+
const dir = outboundSnapshotDir(outbox);
|
|
326
|
+
const keep = new Set(preserve.map((path) => resolve(path)));
|
|
327
|
+
try {
|
|
328
|
+
const info = lstatSync(dir);
|
|
329
|
+
if (!info.isDirectory() || info.isSymbolicLink() || !ownedByProcess(dir))
|
|
330
|
+
return;
|
|
331
|
+
for (const name of readdirSync(dir)) {
|
|
332
|
+
if (!OWNED_SNAPSHOT.test(name))
|
|
333
|
+
continue;
|
|
334
|
+
const path = join(dir, name);
|
|
335
|
+
if (keep.has(resolve(path)))
|
|
336
|
+
continue;
|
|
337
|
+
try {
|
|
338
|
+
const entry = lstatSync(path);
|
|
339
|
+
if ((entry.isFile() || entry.isSymbolicLink()) && ownedByProcess(path))
|
|
340
|
+
unlinkSync(path);
|
|
341
|
+
}
|
|
342
|
+
catch {
|
|
343
|
+
/* raced cleanup is harmless */
|
|
344
|
+
}
|
|
345
|
+
}
|
|
346
|
+
rmdirSync(dir);
|
|
347
|
+
}
|
|
348
|
+
catch {
|
|
349
|
+
/* best-effort cleanup */
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
/** Delete one previously verified delivered snapshot and remove its now-empty queue directory. */
|
|
353
|
+
export function cleanupOutboundSnapshot(path) {
|
|
354
|
+
const candidate = resolve(path);
|
|
355
|
+
const expected = consumedSnapshotIdentities.get(candidate);
|
|
356
|
+
consumedSnapshotIdentities.delete(candidate);
|
|
357
|
+
const dir = dirname(candidate);
|
|
358
|
+
if (!dir.endsWith(".files") || !OWNED_SNAPSHOT.test(basename(candidate)))
|
|
359
|
+
return;
|
|
360
|
+
try {
|
|
361
|
+
const parent = lstatSync(dir);
|
|
362
|
+
if (!parent.isDirectory() || parent.isSymbolicLink() || !ownedByProcess(dir))
|
|
363
|
+
return;
|
|
364
|
+
const entry = lstatSync(candidate);
|
|
365
|
+
if (expected
|
|
366
|
+
&& entry.isFile()
|
|
367
|
+
&& !entry.isSymbolicLink()
|
|
368
|
+
&& entry.nlink === 1
|
|
369
|
+
&& entry.dev === expected.dev
|
|
370
|
+
&& entry.ino === expected.ino
|
|
371
|
+
&& ownedByProcess(candidate))
|
|
372
|
+
unlinkSync(candidate);
|
|
373
|
+
if (readdirSync(dir).length === 0)
|
|
374
|
+
rmdirSync(dir);
|
|
375
|
+
}
|
|
376
|
+
catch {
|
|
377
|
+
/* already removed or raced cleanup */
|
|
378
|
+
}
|
|
379
|
+
}
|