@mono-agent/agent-app 0.11.2 → 0.11.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +674 -0
- package/README.md +222 -26
- package/dist/adapter-send-tools-main.js +7 -1
- package/dist/adapter-send-tools-main.js.map +1 -1
- package/dist/adapter-send-tools.d.ts +19 -1
- package/dist/adapter-send-tools.d.ts.map +1 -1
- package/dist/adapter-send-tools.js +127 -13
- package/dist/adapter-send-tools.js.map +1 -1
- package/dist/app-config.d.ts.map +1 -1
- package/dist/app-config.js +15 -2
- package/dist/app-config.js.map +1 -1
- package/dist/app-controller-channels.d.ts +8 -0
- package/dist/app-controller-channels.d.ts.map +1 -0
- package/dist/app-controller-channels.js +207 -0
- package/dist/app-controller-channels.js.map +1 -0
- package/dist/app-controller-continuation.d.ts +26 -0
- package/dist/app-controller-continuation.d.ts.map +1 -0
- package/dist/app-controller-continuation.js +198 -0
- package/dist/app-controller-continuation.js.map +1 -0
- package/dist/app-controller-lifecycle.d.ts +6 -0
- package/dist/app-controller-lifecycle.d.ts.map +1 -0
- package/dist/app-controller-lifecycle.js +81 -0
- package/dist/app-controller-lifecycle.js.map +1 -0
- package/dist/app-controller-maintenance.d.ts +13 -0
- package/dist/app-controller-maintenance.d.ts.map +1 -0
- package/dist/app-controller-maintenance.js +180 -0
- package/dist/app-controller-maintenance.js.map +1 -0
- package/dist/app-controller-memory-health.d.ts +12 -0
- package/dist/app-controller-memory-health.d.ts.map +1 -0
- package/dist/app-controller-memory-health.js +160 -0
- package/dist/app-controller-memory-health.js.map +1 -0
- package/dist/app-controller-memory.d.ts +9 -0
- package/dist/app-controller-memory.d.ts.map +1 -0
- package/dist/app-controller-memory.js +83 -0
- package/dist/app-controller-memory.js.map +1 -0
- package/dist/app-controller-responder.d.ts +20 -0
- package/dist/app-controller-responder.d.ts.map +1 -0
- package/dist/app-controller-responder.js +246 -0
- package/dist/app-controller-responder.js.map +1 -0
- package/dist/app-controller-traceability.d.ts +25 -0
- package/dist/app-controller-traceability.d.ts.map +1 -0
- package/dist/app-controller-traceability.js +418 -0
- package/dist/app-controller-traceability.js.map +1 -0
- package/dist/app-controller-utils.d.ts +38 -0
- package/dist/app-controller-utils.d.ts.map +1 -0
- package/dist/app-controller-utils.js +185 -0
- package/dist/app-controller-utils.js.map +1 -0
- package/dist/app-controller.d.ts +400 -0
- package/dist/app-controller.d.ts.map +1 -0
- package/dist/app-controller.js +336 -0
- package/dist/app-controller.js.map +1 -0
- package/dist/app.d.ts +7 -115
- package/dist/app.d.ts.map +1 -1
- package/dist/app.js +6 -1912
- package/dist/app.js.map +1 -1
- package/dist/artifact-retention.d.ts +4 -0
- package/dist/artifact-retention.d.ts.map +1 -1
- package/dist/artifact-retention.js +38 -0
- package/dist/artifact-retention.js.map +1 -1
- package/dist/backfill.d.ts +3 -1
- package/dist/backfill.d.ts.map +1 -1
- package/dist/backfill.js +5 -1
- package/dist/backfill.js.map +1 -1
- package/dist/background-runtime.d.ts +27 -0
- package/dist/background-runtime.d.ts.map +1 -1
- package/dist/background-runtime.js +383 -205
- package/dist/background-runtime.js.map +1 -1
- package/dist/background-worker-lease.d.ts.map +1 -1
- package/dist/background-worker-lease.js +41 -241
- package/dist/background-worker-lease.js.map +1 -1
- package/dist/background.d.ts +31 -9
- package/dist/background.d.ts.map +1 -1
- package/dist/background.js +543 -272
- package/dist/background.js.map +1 -1
- package/dist/channel-drivers/cron.d.ts +11 -0
- package/dist/channel-drivers/cron.d.ts.map +1 -0
- package/dist/channel-drivers/cron.js +168 -0
- package/dist/channel-drivers/cron.js.map +1 -0
- package/dist/channel-drivers/live.d.ts +11 -0
- package/dist/channel-drivers/live.d.ts.map +1 -0
- package/dist/channel-drivers/live.js +47 -0
- package/dist/channel-drivers/live.js.map +1 -0
- package/dist/channel-drivers/native-notify.d.ts +27 -0
- package/dist/channel-drivers/native-notify.d.ts.map +1 -0
- package/dist/channel-drivers/native-notify.js +94 -0
- package/dist/channel-drivers/native-notify.js.map +1 -0
- package/dist/channel-drivers/openai-api.d.ts +7 -0
- package/dist/channel-drivers/openai-api.d.ts.map +1 -0
- package/dist/channel-drivers/openai-api.js +63 -0
- package/dist/channel-drivers/openai-api.js.map +1 -0
- package/dist/channel-drivers/shared.d.ts +4 -0
- package/dist/channel-drivers/shared.d.ts.map +1 -0
- package/dist/channel-drivers/shared.js +5 -0
- package/dist/channel-drivers/shared.js.map +1 -0
- package/dist/channel-drivers/slack.d.ts +14 -0
- package/dist/channel-drivers/slack.d.ts.map +1 -0
- package/dist/channel-drivers/slack.js +227 -0
- package/dist/channel-drivers/slack.js.map +1 -0
- package/dist/channel-drivers/telegram.d.ts +11 -0
- package/dist/channel-drivers/telegram.d.ts.map +1 -0
- package/dist/channel-drivers/telegram.js +244 -0
- package/dist/channel-drivers/telegram.js.map +1 -0
- package/dist/channel-drivers/tui.d.ts +15 -0
- package/dist/channel-drivers/tui.d.ts.map +1 -0
- package/dist/channel-drivers/tui.js +108 -0
- package/dist/channel-drivers/tui.js.map +1 -0
- package/dist/channel-drivers/webhook.d.ts +7 -0
- package/dist/channel-drivers/webhook.d.ts.map +1 -0
- package/dist/channel-drivers/webhook.js +115 -0
- package/dist/channel-drivers/webhook.js.map +1 -0
- package/dist/channels.d.ts +21 -81
- package/dist/channels.d.ts.map +1 -1
- package/dist/channels.js +11 -1174
- package/dist/channels.js.map +1 -1
- package/dist/cli-args.d.ts +121 -0
- package/dist/cli-args.d.ts.map +1 -0
- package/dist/cli-args.js +532 -0
- package/dist/cli-args.js.map +1 -0
- package/dist/cli-background-command.d.ts +59 -0
- package/dist/cli-background-command.d.ts.map +1 -0
- package/dist/cli-background-command.js +475 -0
- package/dist/cli-background-command.js.map +1 -0
- package/dist/cli-cancellation.d.ts +6 -0
- package/dist/cli-cancellation.d.ts.map +1 -0
- package/dist/cli-cancellation.js +23 -0
- package/dist/cli-cancellation.js.map +1 -0
- package/dist/cli-commands.d.ts +12 -0
- package/dist/cli-commands.d.ts.map +1 -0
- package/dist/cli-commands.js +306 -0
- package/dist/cli-commands.js.map +1 -0
- package/dist/cli-help.d.ts +11 -0
- package/dist/cli-help.d.ts.map +1 -0
- package/dist/cli-help.js +262 -0
- package/dist/cli-help.js.map +1 -0
- package/dist/cli-init-command.d.ts +60 -0
- package/dist/cli-init-command.d.ts.map +1 -0
- package/dist/cli-init-command.js +1910 -0
- package/dist/cli-init-command.js.map +1 -0
- package/dist/cli-install-skill-command.d.ts +3 -0
- package/dist/cli-install-skill-command.d.ts.map +1 -0
- package/dist/cli-install-skill-command.js +51 -0
- package/dist/cli-install-skill-command.js.map +1 -0
- package/dist/cli-sandbox-command.d.ts +10 -0
- package/dist/cli-sandbox-command.d.ts.map +1 -0
- package/dist/cli-sandbox-command.js +100 -0
- package/dist/cli-sandbox-command.js.map +1 -0
- package/dist/cli-validate-config-command.d.ts +36 -0
- package/dist/cli-validate-config-command.d.ts.map +1 -0
- package/dist/cli-validate-config-command.js +317 -0
- package/dist/cli-validate-config-command.js.map +1 -0
- package/dist/cli.d.ts +6 -248
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +6 -3792
- package/dist/cli.js.map +1 -1
- package/dist/config-reference.d.ts +2 -1
- package/dist/config-reference.d.ts.map +1 -1
- package/dist/config-reference.js +34 -8
- package/dist/config-reference.js.map +1 -1
- package/dist/configured-agent.d.ts +23 -0
- package/dist/configured-agent.d.ts.map +1 -1
- package/dist/configured-agent.js +92 -13
- package/dist/configured-agent.js.map +1 -1
- package/dist/continuation-dispatch-worker.d.ts +21 -0
- package/dist/continuation-dispatch-worker.d.ts.map +1 -0
- package/dist/continuation-dispatch-worker.js +425 -0
- package/dist/continuation-dispatch-worker.js.map +1 -0
- package/dist/continuation-http-routes.d.ts +15 -0
- package/dist/continuation-http-routes.d.ts.map +1 -0
- package/dist/continuation-http-routes.js +265 -0
- package/dist/continuation-http-routes.js.map +1 -0
- package/dist/continuation-operator-api.d.ts +18 -0
- package/dist/continuation-operator-api.d.ts.map +1 -0
- package/dist/continuation-operator-api.js +164 -0
- package/dist/continuation-operator-api.js.map +1 -0
- package/dist/continuation-origin-store.d.ts +19 -0
- package/dist/continuation-origin-store.d.ts.map +1 -0
- package/dist/continuation-origin-store.js +273 -0
- package/dist/continuation-origin-store.js.map +1 -0
- package/dist/continuation-service-errors.d.ts +18 -0
- package/dist/continuation-service-errors.d.ts.map +1 -0
- package/dist/continuation-service-errors.js +35 -0
- package/dist/continuation-service-errors.js.map +1 -0
- package/dist/continuation-service-helpers.d.ts +39 -0
- package/dist/continuation-service-helpers.d.ts.map +1 -0
- package/dist/continuation-service-helpers.js +308 -0
- package/dist/continuation-service-helpers.js.map +1 -0
- package/dist/continuation-service-origin.d.ts +55 -0
- package/dist/continuation-service-origin.d.ts.map +1 -0
- package/dist/continuation-service-origin.js +325 -0
- package/dist/continuation-service-origin.js.map +1 -0
- package/dist/continuation-service-types.d.ts +110 -0
- package/dist/continuation-service-types.d.ts.map +1 -0
- package/dist/continuation-service-types.js +11 -0
- package/dist/continuation-service-types.js.map +1 -0
- package/dist/continuation-service.d.ts +4 -97
- package/dist/continuation-service.d.ts.map +1 -1
- package/dist/continuation-service.js +9 -1525
- package/dist/continuation-service.js.map +1 -1
- package/dist/continuation-store-fs.d.ts +15 -0
- package/dist/continuation-store-fs.d.ts.map +1 -0
- package/dist/continuation-store-fs.js +247 -0
- package/dist/continuation-store-fs.js.map +1 -0
- package/dist/continuation-store-policy.d.ts +21 -0
- package/dist/continuation-store-policy.d.ts.map +1 -0
- package/dist/continuation-store-policy.js +278 -0
- package/dist/continuation-store-policy.js.map +1 -0
- package/dist/continuation-store-records.d.ts +9 -0
- package/dist/continuation-store-records.d.ts.map +1 -0
- package/dist/continuation-store-records.js +221 -0
- package/dist/continuation-store-records.js.map +1 -0
- package/dist/continuation-store-types.d.ts +173 -0
- package/dist/continuation-store-types.d.ts.map +1 -0
- package/dist/continuation-store-types.js +26 -0
- package/dist/continuation-store-types.js.map +1 -0
- package/dist/continuation-store.d.ts +4 -138
- package/dist/continuation-store.d.ts.map +1 -1
- package/dist/continuation-store.js +145 -1049
- package/dist/continuation-store.js.map +1 -1
- package/dist/doctor.d.ts +10 -5
- package/dist/doctor.d.ts.map +1 -1
- package/dist/doctor.js +629 -58
- package/dist/doctor.js.map +1 -1
- package/dist/init.d.ts +2 -0
- package/dist/init.d.ts.map +1 -1
- package/dist/init.js +92 -239
- package/dist/init.js.map +1 -1
- package/dist/interaction-bridge.d.ts +19 -0
- package/dist/interaction-bridge.d.ts.map +1 -1
- package/dist/interaction-bridge.js +130 -0
- package/dist/interaction-bridge.js.map +1 -1
- package/dist/launchd-logs.d.ts +91 -0
- package/dist/launchd-logs.d.ts.map +1 -0
- package/dist/launchd-logs.js +1499 -0
- package/dist/launchd-logs.js.map +1 -0
- package/dist/launchd.d.ts +27 -0
- package/dist/launchd.d.ts.map +1 -1
- package/dist/launchd.js +89 -10
- package/dist/launchd.js.map +1 -1
- package/dist/memory-command.d.ts +2 -0
- package/dist/memory-command.d.ts.map +1 -1
- package/dist/memory-command.js +161 -48
- package/dist/memory-command.js.map +1 -1
- package/dist/memory-recall-settings.d.ts +64 -0
- package/dist/memory-recall-settings.d.ts.map +1 -0
- package/dist/memory-recall-settings.js +51 -0
- package/dist/memory-recall-settings.js.map +1 -0
- package/dist/memory-recall.d.ts +7 -95
- package/dist/memory-recall.d.ts.map +1 -1
- package/dist/memory-recall.js +8 -143
- package/dist/memory-recall.js.map +1 -1
- package/dist/memory-rituals.d.ts +3 -3
- package/dist/memory-rituals.d.ts.map +1 -1
- package/dist/memory-rituals.js +16 -111
- package/dist/memory-rituals.js.map +1 -1
- package/dist/modules/catalog.js +2 -2
- package/dist/modules/catalog.js.map +1 -1
- package/dist/modules/known-tools.d.ts +8 -6
- package/dist/modules/known-tools.d.ts.map +1 -1
- package/dist/modules/known-tools.js +8 -6
- package/dist/modules/known-tools.js.map +1 -1
- package/dist/notify-destinations.d.ts +5 -0
- package/dist/notify-destinations.d.ts.map +1 -1
- package/dist/notify-destinations.js +7 -1
- package/dist/notify-destinations.js.map +1 -1
- package/dist/owner-private-lock.d.ts +95 -0
- package/dist/owner-private-lock.d.ts.map +1 -0
- package/dist/owner-private-lock.js +566 -0
- package/dist/owner-private-lock.js.map +1 -0
- package/dist/pi-oauth-login.d.ts +2 -0
- package/dist/pi-oauth-login.d.ts.map +1 -1
- package/dist/pi-oauth-login.js +39 -5
- package/dist/pi-oauth-login.js.map +1 -1
- package/dist/posted-message-index.d.ts +35 -16
- package/dist/posted-message-index.d.ts.map +1 -1
- package/dist/posted-message-index.js +1068 -52
- package/dist/posted-message-index.js.map +1 -1
- package/dist/posted-reply-history.d.ts +17 -0
- package/dist/posted-reply-history.d.ts.map +1 -0
- package/dist/posted-reply-history.js +169 -0
- package/dist/posted-reply-history.js.map +1 -0
- package/dist/proactive-notify.d.ts.map +1 -1
- package/dist/proactive-notify.js +6 -4
- package/dist/proactive-notify.js.map +1 -1
- package/dist/project-skills.d.ts +4 -0
- package/dist/project-skills.d.ts.map +1 -1
- package/dist/project-skills.js +99 -130
- package/dist/project-skills.js.map +1 -1
- package/dist/provider-setup.d.ts +0 -1
- package/dist/provider-setup.d.ts.map +1 -1
- package/dist/provider-setup.js +0 -3
- package/dist/provider-setup.js.map +1 -1
- package/dist/readiness-probe-worker.d.ts +61 -1
- package/dist/readiness-probe-worker.d.ts.map +1 -1
- package/dist/readiness-probe-worker.js +54 -58
- package/dist/readiness-probe-worker.js.map +1 -1
- package/dist/readiness-probe.d.ts +6 -9
- package/dist/readiness-probe.d.ts.map +1 -1
- package/dist/readiness-probe.js +30 -56
- package/dist/readiness-probe.js.map +1 -1
- package/dist/redact-secrets.d.ts +15 -0
- package/dist/redact-secrets.d.ts.map +1 -0
- package/dist/redact-secrets.js +65 -0
- package/dist/redact-secrets.js.map +1 -0
- package/dist/run-history.js +34 -24
- package/dist/run-history.js.map +1 -1
- package/dist/runtime-provenance.d.ts +8 -0
- package/dist/runtime-provenance.d.ts.map +1 -0
- package/dist/runtime-provenance.js +225 -0
- package/dist/runtime-provenance.js.map +1 -0
- package/dist/sandbox-manager.d.ts +4 -1
- package/dist/sandbox-manager.d.ts.map +1 -1
- package/dist/sandbox-manager.js +114 -273
- package/dist/sandbox-manager.js.map +1 -1
- package/dist/secure-file-replace.d.ts +49 -0
- package/dist/secure-file-replace.d.ts.map +1 -0
- package/dist/secure-file-replace.js +270 -0
- package/dist/secure-file-replace.js.map +1 -0
- package/dist/seen-conversations.d.ts +32 -5
- package/dist/seen-conversations.d.ts.map +1 -1
- package/dist/seen-conversations.js +145 -15
- package/dist/seen-conversations.js.map +1 -1
- package/dist/sessions.d.ts +20 -0
- package/dist/sessions.d.ts.map +1 -1
- package/dist/sessions.js +41 -6
- package/dist/sessions.js.map +1 -1
- package/dist/wizard/index.d.ts +2 -2
- package/dist/wizard/index.d.ts.map +1 -1
- package/dist/wizard/index.js +1 -1
- package/dist/wizard/index.js.map +1 -1
- package/dist/wizard/presets.d.ts +2 -1
- package/dist/wizard/presets.d.ts.map +1 -1
- package/dist/wizard/presets.js +2 -1
- package/dist/wizard/presets.js.map +1 -1
- package/dist/wizard/run.d.ts +0 -26
- package/dist/wizard/run.d.ts.map +1 -1
- package/dist/wizard/run.js +0 -53
- package/dist/wizard/run.js.map +1 -1
- package/package.json +22 -17
- package/schema/mono-agent.config.schema.json +45 -10
- package/skills/mono-agent-composer/SKILL.md +3 -3
- package/skills/mono-agent-composer/references/config-blueprint.md +123 -29
- package/skills/mono-agent-composer/references/discovery-questions.md +10 -8
- package/skills/mono-agent-composer/references/feature-coverage.md +90 -76
- package/skills/mono-agent-composer/references/package-map.md +6 -4
- package/skills/mono-agent-composer/references/playbooks.md +48 -27
- package/skills/mono-agent-composer/references/validation.md +16 -3
- package/dist/notify-runtime.d.ts +0 -26
- package/dist/notify-runtime.d.ts.map +0 -1
- package/dist/notify-runtime.js +0 -27
- package/dist/notify-runtime.js.map +0 -1
- package/dist/notify-tool.d.ts +0 -51
- package/dist/notify-tool.d.ts.map +0 -1
- package/dist/notify-tool.js +0 -182
- package/dist/notify-tool.js.map +0 -1
- package/dist/recipes/base.d.ts +0 -15
- package/dist/recipes/base.d.ts.map +0 -1
- package/dist/recipes/base.js +0 -51
- package/dist/recipes/base.js.map +0 -1
- package/dist/recipes/catalog.d.ts +0 -4
- package/dist/recipes/catalog.d.ts.map +0 -1
- package/dist/recipes/catalog.js +0 -525
- package/dist/recipes/catalog.js.map +0 -1
- package/dist/recipes/index.d.ts +0 -11
- package/dist/recipes/index.d.ts.map +0 -1
- package/dist/recipes/index.js +0 -14
- package/dist/recipes/index.js.map +0 -1
- package/dist/recipes/types.d.ts +0 -70
- package/dist/recipes/types.d.ts.map +0 -1
- package/dist/recipes/types.js +0 -15
- package/dist/recipes/types.js.map +0 -1
- package/dist/setup.d.ts +0 -29
- package/dist/setup.d.ts.map +0 -1
- package/dist/setup.js +0 -97
- package/dist/setup.js.map +0 -1
|
@@ -1,5 +1,7 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { constants as fsConstants } from "node:fs";
|
|
3
|
+
import { lstat, mkdir, open } from "node:fs/promises";
|
|
4
|
+
import { dirname, join, parse, resolve, sep } from "node:path";
|
|
3
5
|
/**
|
|
4
6
|
* Maps a message the agent POSTED (`channelId` + Slack `ts`) back to the
|
|
5
7
|
* conversation that produced it, so a later in-thread reply can be resolved to
|
|
@@ -13,17 +15,30 @@ import { dirname, join } from "node:path";
|
|
|
13
15
|
* producing conversationId`; the consumer (inbound dispatch) looks it up and
|
|
14
16
|
* aliases the reply onto the producing conversation.
|
|
15
17
|
*
|
|
16
|
-
* Storage is
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
18
|
+
* Storage is a JSONL file inside the run-artifact dir. Appenders, readers,
|
|
19
|
+
* startup maintenance, recovery, and compaction serialize through one
|
|
20
|
+
* owner-only SQLite coordinator. Compaction appends a durable recovery trailer
|
|
21
|
+
* to the verified index inode, then rewrites that same inode through its pinned
|
|
22
|
+
* descriptor; it never promotes or later reopens a source pathname. The index
|
|
23
|
+
* and coordinator are opened no-follow and must remain current-owner, regular,
|
|
24
|
+
* single-link files. These files are ignored by the `.summary.json` artifact
|
|
25
|
+
* scanners (see `seen-conversations.ts`), so they never collide with run-artifact
|
|
26
|
+
* tooling.
|
|
22
27
|
*/
|
|
23
28
|
export const POSTED_MESSAGE_INDEX_FILENAME = "posted-message-index.jsonl";
|
|
24
29
|
/** Daily-rollover bucket suffix the responder appends (`…#2026-06-22`). */
|
|
25
30
|
const ROLLOVER_BUCKET = /#\d{4}-\d{2}-\d{2}$/u;
|
|
26
31
|
const DEFAULT_COMPACT_MAX_ENTRIES = 5000;
|
|
32
|
+
const COMPACT_HEADROOM_DIVISOR = 10;
|
|
33
|
+
const INDEX_LOCK_WAIT_MS = 2000;
|
|
34
|
+
const MAX_LOCK_FILE_BYTES = 64 * 1024;
|
|
35
|
+
const SQLITE_SIDECAR_SUFFIXES = ["-journal", "-wal", "-shm"];
|
|
36
|
+
const COMPACTION_TRAILER_MARKER = "mono-agent-posted-index-compaction";
|
|
37
|
+
const COMPACTION_TRAILER_SCHEMA_VERSION = 1;
|
|
38
|
+
const COMPACTION_TRAILER_MAX_BYTES = 4096;
|
|
39
|
+
const COMPACTION_SEPARATOR = Buffer.from("\n", "utf8");
|
|
40
|
+
const inProcessIndexTails = new Map();
|
|
41
|
+
const inProcessIndexStates = new Map();
|
|
27
42
|
/** The single index-file path both producer and consumer agree on. */
|
|
28
43
|
export function resolvePostedMessageIndexPath(artifactDir) {
|
|
29
44
|
return join(artifactDir, POSTED_MESSAGE_INDEX_FILENAME);
|
|
@@ -33,13 +48,18 @@ export function basePostedConversationId(conversationId) {
|
|
|
33
48
|
return conversationId.replace(ROLLOVER_BUCKET, "");
|
|
34
49
|
}
|
|
35
50
|
/**
|
|
36
|
-
* Record that `conversationId` posted a message at `(channelId, ts)`.
|
|
37
|
-
*
|
|
38
|
-
*
|
|
39
|
-
*
|
|
40
|
-
* the
|
|
51
|
+
* Record that `conversationId` posted a message at `(channelId, ts)`. Appenders in
|
|
52
|
+
* both the adapter and its stdio child share a filesystem lock with compaction, so
|
|
53
|
+
* a rewrite cannot discard a completed concurrent append. Once the cap is reached,
|
|
54
|
+
* compaction drops a batch of oldest entries before appending; this keeps the file
|
|
55
|
+
* at or below the cap without a full rewrite on every later send.
|
|
56
|
+
*
|
|
57
|
+
* Best-effort: a failed index write or lock acquisition must never fail the Slack
|
|
58
|
+
* post, so this function swallows errors. The stored conversationId is de-bucketed
|
|
59
|
+
* so the consumer can let the responder re-bucket to the reply's own day
|
|
60
|
+
* (consistent with daily session rollover).
|
|
41
61
|
*/
|
|
42
|
-
export async function appendPostedMessage(indexPath, entry, now = () => new Date()) {
|
|
62
|
+
export async function appendPostedMessage(indexPath, entry, now = () => new Date(), maxEntries = DEFAULT_COMPACT_MAX_ENTRIES, compactionHooks = {}) {
|
|
43
63
|
const channelId = entry.channelId.trim();
|
|
44
64
|
const ts = entry.ts.trim();
|
|
45
65
|
const conversationId = basePostedConversationId(entry.conversationId.trim());
|
|
@@ -53,23 +73,42 @@ export async function appendPostedMessage(indexPath, entry, now = () => new Date
|
|
|
53
73
|
writtenAt: now().toISOString(),
|
|
54
74
|
};
|
|
55
75
|
const line = `${JSON.stringify(record)}\n`;
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
if (
|
|
61
|
-
return;
|
|
76
|
+
const cap = normalizedAppendCap(maxEntries);
|
|
77
|
+
await withPostedMessageIndexLock(indexPath, async (lock) => {
|
|
78
|
+
const { directoryIdentity } = lock;
|
|
79
|
+
const identity = await ensureOwnerOnlyFile(indexPath, directoryIdentity, true);
|
|
80
|
+
if (identity === undefined) {
|
|
81
|
+
return;
|
|
82
|
+
}
|
|
83
|
+
let state = await loadPostedMessageIndexState(indexPath, directoryIdentity, identity);
|
|
84
|
+
if (state === undefined) {
|
|
85
|
+
return;
|
|
86
|
+
}
|
|
87
|
+
if (state.count >= cap) {
|
|
88
|
+
const compacted = await compactPostedMessageIndexUnlocked(indexPath, amortizedCompactTarget(cap), lock, compactionHooks);
|
|
89
|
+
if (compacted === undefined) {
|
|
90
|
+
// Preserve the existing bounded file rather than append past the cap when
|
|
91
|
+
// its rewrite cannot be completed safely.
|
|
92
|
+
return;
|
|
93
|
+
}
|
|
94
|
+
state = compacted;
|
|
95
|
+
inProcessIndexStates.set(indexPath, compacted);
|
|
62
96
|
}
|
|
63
|
-
// Artifact dir not created yet (agent never wrote a summary here). Create it
|
|
64
|
-
// and retry once; still best-effort.
|
|
65
97
|
try {
|
|
66
|
-
await
|
|
67
|
-
|
|
98
|
+
const appended = await appendSecureIndexLine(indexPath, line, state, directoryIdentity);
|
|
99
|
+
inProcessIndexStates.set(indexPath, {
|
|
100
|
+
count: state.count + 1,
|
|
101
|
+
size: appended.size,
|
|
102
|
+
identity: state.identity,
|
|
103
|
+
changeToken: appended.changeToken,
|
|
104
|
+
});
|
|
68
105
|
}
|
|
69
106
|
catch {
|
|
70
|
-
//
|
|
107
|
+
// Best-effort. Discard any in-process hint so the next writer reconciles
|
|
108
|
+
// count and size from a securely-opened JSONL snapshot.
|
|
109
|
+
inProcessIndexStates.delete(indexPath);
|
|
71
110
|
}
|
|
72
|
-
}
|
|
111
|
+
});
|
|
73
112
|
}
|
|
74
113
|
/**
|
|
75
114
|
* Resolve the producing conversationId for a posted message, newest write wins.
|
|
@@ -82,31 +121,46 @@ export async function lookupProducingConversation(indexPath, channelId, ts) {
|
|
|
82
121
|
if (wantChannel.length === 0 || wantTs.length === 0) {
|
|
83
122
|
return undefined;
|
|
84
123
|
}
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
124
|
+
return await withPostedMessageIndexLock(indexPath, async ({ directoryIdentity }) => {
|
|
125
|
+
let match;
|
|
126
|
+
const entries = (await tryReadIndexSnapshot(indexPath, directoryIdentity))?.entries ?? [];
|
|
127
|
+
for (const entry of entries) {
|
|
128
|
+
if (entry.channelId !== wantChannel || entry.ts !== wantTs) {
|
|
129
|
+
continue;
|
|
130
|
+
}
|
|
131
|
+
if (match === undefined || entry.writtenAt >= match.writtenAt) {
|
|
132
|
+
match = entry;
|
|
133
|
+
}
|
|
89
134
|
}
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
}
|
|
93
|
-
}
|
|
94
|
-
return match?.conversationId;
|
|
135
|
+
return match?.conversationId;
|
|
136
|
+
});
|
|
95
137
|
}
|
|
96
138
|
/**
|
|
97
139
|
* Bound file growth by rewriting the index with only the newest `maxEntries`
|
|
98
|
-
* (by write time, de-duped to the newest entry per `channel+ts`).
|
|
99
|
-
*
|
|
100
|
-
*
|
|
140
|
+
* (by write time, de-duped to the newest entry per `channel+ts`). Slack-driver
|
|
141
|
+
* startup invokes this exact routine, and every appender also uses the same
|
|
142
|
+
* unlocked implementation after taking this cross-process lock. Best-effort.
|
|
101
143
|
*/
|
|
102
|
-
export async function compactPostedMessageIndex(indexPath, maxEntries = DEFAULT_COMPACT_MAX_ENTRIES) {
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
144
|
+
export async function compactPostedMessageIndex(indexPath, maxEntries = DEFAULT_COMPACT_MAX_ENTRIES, hooks = {}) {
|
|
145
|
+
await withPostedMessageIndexLock(indexPath, async (lock) => {
|
|
146
|
+
const compacted = await compactPostedMessageIndexUnlocked(indexPath, normalizedCompactionCap(maxEntries), lock, hooks);
|
|
147
|
+
if (compacted !== undefined) {
|
|
148
|
+
inProcessIndexStates.set(indexPath, compacted);
|
|
149
|
+
}
|
|
150
|
+
});
|
|
151
|
+
}
|
|
152
|
+
async function compactPostedMessageIndexUnlocked(indexPath, maxEntries, lock, hooks = {}) {
|
|
153
|
+
const { directoryIdentity } = lock;
|
|
154
|
+
const snapshot = await tryReadIndexSnapshot(indexPath, directoryIdentity);
|
|
155
|
+
if (snapshot === undefined) {
|
|
156
|
+
return undefined;
|
|
157
|
+
}
|
|
158
|
+
if (snapshot.entries.length <= maxEntries) {
|
|
159
|
+
return snapshot;
|
|
106
160
|
}
|
|
107
161
|
// Newest entry per (channel, ts), then newest-first, then cap.
|
|
108
162
|
const latest = new Map();
|
|
109
|
-
for (const entry of entries) {
|
|
163
|
+
for (const entry of snapshot.entries) {
|
|
110
164
|
const key = `${entry.channelId} ${entry.ts}`;
|
|
111
165
|
const prior = latest.get(key);
|
|
112
166
|
if (prior === undefined || entry.writtenAt >= prior.writtenAt) {
|
|
@@ -116,24 +170,654 @@ export async function compactPostedMessageIndex(indexPath, maxEntries = DEFAULT_
|
|
|
116
170
|
const kept = [...latest.values()]
|
|
117
171
|
.sort((a, b) => (a.writtenAt < b.writtenAt ? 1 : a.writtenAt > b.writtenAt ? -1 : 0))
|
|
118
172
|
.slice(0, maxEntries);
|
|
119
|
-
const
|
|
120
|
-
|
|
173
|
+
const nextBytes = encodePostedMessageEntries(kept);
|
|
174
|
+
let target;
|
|
175
|
+
let appendTarget;
|
|
176
|
+
try {
|
|
177
|
+
await assertDirectoryIdentity(dirname(indexPath), directoryIdentity);
|
|
178
|
+
await hooks.beforeReplace?.();
|
|
179
|
+
target = await openPinnedIndexForRewrite(indexPath, snapshot.identity, snapshot.size, directoryIdentity);
|
|
180
|
+
appendTarget = await openPinnedIndexForAppend(indexPath, snapshot.identity, snapshot.size, directoryIdentity);
|
|
181
|
+
await hooks.beforePrepare?.();
|
|
182
|
+
const prepared = await prepareCompaction(target, appendTarget, indexPath, snapshot.identity, lock, snapshot.size, nextBytes, kept.length, hooks.afterPrepareBody);
|
|
183
|
+
await assertFileIdentity(indexPath, snapshot.identity);
|
|
184
|
+
await assertLockFileIdentity(lock.path, lock.identity);
|
|
185
|
+
await assertNoSqliteSidecars(lock.path, directoryIdentity);
|
|
186
|
+
await assertDirectoryIdentity(dirname(indexPath), directoryIdentity);
|
|
187
|
+
await hooks.afterPromotionChecks?.(indexPath);
|
|
188
|
+
// There is intentionally no pathname rename or sidecar source here. The
|
|
189
|
+
// recovery bytes and destination are both pinned inside the verified index
|
|
190
|
+
// inode. A later path swap cannot change the prepared bytes or redirect the
|
|
191
|
+
// rewrite into the raced pathname. The footer also binds recovery to this
|
|
192
|
+
// exact coordinator inode, so a replacement lock domain cannot append while
|
|
193
|
+
// the transaction is prepared. The footer remains beyond the maximum rewrite
|
|
194
|
+
// extent until the new prefix is durable, so process death at any rewrite
|
|
195
|
+
// boundary is recoverable before the next locked reader or writer.
|
|
196
|
+
const rewritten = await rewritePinnedIndex(target, indexPath, snapshot.identity, prepared.body, hooks.afterRewriteSync);
|
|
197
|
+
await assertFileIdentity(indexPath, snapshot.identity);
|
|
198
|
+
await assertLockFileIdentity(lock.path, lock.identity);
|
|
199
|
+
await assertNoSqliteSidecars(lock.path, directoryIdentity);
|
|
200
|
+
await assertDirectoryIdentity(dirname(indexPath), directoryIdentity);
|
|
201
|
+
await fsyncDirectory(dirname(indexPath), directoryIdentity);
|
|
202
|
+
return {
|
|
203
|
+
count: kept.length,
|
|
204
|
+
size: rewritten.size,
|
|
205
|
+
identity: snapshot.identity,
|
|
206
|
+
changeToken: rewritten.changeToken,
|
|
207
|
+
};
|
|
208
|
+
}
|
|
209
|
+
catch {
|
|
210
|
+
// Best-effort and fail-closed. No catch path unlinks or renames a pathname. A
|
|
211
|
+
// complete trailer remains on the intended inode for locked recovery; an
|
|
212
|
+
// interrupted prepare happens before any original byte is rewritten and is
|
|
213
|
+
// treated as ordinary ignored JSONL tail data on the next operation.
|
|
214
|
+
return undefined;
|
|
215
|
+
}
|
|
216
|
+
finally {
|
|
217
|
+
await appendTarget?.close().catch(() => undefined);
|
|
218
|
+
await target?.close().catch(() => undefined);
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
async function recoverPreparedCompaction(indexPath, lock) {
|
|
222
|
+
const { directoryIdentity } = lock;
|
|
223
|
+
let target;
|
|
224
|
+
try {
|
|
225
|
+
const targetIdentity = await ensureOwnerOnlyFile(indexPath, directoryIdentity, false);
|
|
226
|
+
if (targetIdentity === undefined) {
|
|
227
|
+
return true;
|
|
228
|
+
}
|
|
229
|
+
target = await openPinnedIndexForRewrite(indexPath, targetIdentity, undefined, directoryIdentity);
|
|
230
|
+
const prepared = await readPreparedCompaction(target, indexPath, targetIdentity, lock.identity);
|
|
231
|
+
if (prepared === undefined) {
|
|
232
|
+
if (await terminateIncompleteCompactionTail(target, indexPath, targetIdentity)) {
|
|
233
|
+
inProcessIndexStates.delete(indexPath);
|
|
234
|
+
}
|
|
235
|
+
await assertFileIdentity(indexPath, targetIdentity);
|
|
236
|
+
await assertLockFileIdentity(lock.path, lock.identity);
|
|
237
|
+
await assertNoSqliteSidecars(lock.path, directoryIdentity);
|
|
238
|
+
await assertDirectoryIdentity(dirname(indexPath), directoryIdentity);
|
|
239
|
+
return true;
|
|
240
|
+
}
|
|
241
|
+
await assertFileIdentity(indexPath, targetIdentity);
|
|
242
|
+
await assertLockFileIdentity(lock.path, lock.identity);
|
|
243
|
+
await assertNoSqliteSidecars(lock.path, directoryIdentity);
|
|
244
|
+
await assertDirectoryIdentity(dirname(indexPath), directoryIdentity);
|
|
245
|
+
const rewritten = await rewritePinnedIndex(target, indexPath, targetIdentity, prepared.body);
|
|
246
|
+
await assertFileIdentity(indexPath, targetIdentity);
|
|
247
|
+
await assertLockFileIdentity(lock.path, lock.identity);
|
|
248
|
+
await assertNoSqliteSidecars(lock.path, directoryIdentity);
|
|
249
|
+
await fsyncDirectory(dirname(indexPath), directoryIdentity);
|
|
250
|
+
inProcessIndexStates.set(indexPath, {
|
|
251
|
+
count: prepared.record.entryCount,
|
|
252
|
+
size: rewritten.size,
|
|
253
|
+
identity: targetIdentity,
|
|
254
|
+
changeToken: rewritten.changeToken,
|
|
255
|
+
});
|
|
256
|
+
return true;
|
|
257
|
+
}
|
|
258
|
+
catch {
|
|
259
|
+
inProcessIndexStates.delete(indexPath);
|
|
260
|
+
return false;
|
|
261
|
+
}
|
|
262
|
+
finally {
|
|
263
|
+
await target?.close().catch(() => undefined);
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
async function prepareCompaction(handle, appendHandle, indexPath, targetIdentity, lock, originalSize, body, entryCount, afterBody) {
|
|
267
|
+
const before = await handle.stat();
|
|
268
|
+
assertSecureFile(before, indexPath);
|
|
269
|
+
assertSameIdentity(targetIdentity, before, indexPath);
|
|
270
|
+
const appendBefore = await appendHandle.stat();
|
|
271
|
+
assertSecureFile(appendBefore, indexPath);
|
|
272
|
+
assertSameIdentity(targetIdentity, appendBefore, indexPath);
|
|
273
|
+
if (before.size !== originalSize) {
|
|
274
|
+
throw new Error(`Posted-message index ${indexPath} changed before compaction prepare.`);
|
|
275
|
+
}
|
|
276
|
+
if (appendBefore.size !== originalSize) {
|
|
277
|
+
throw new Error(`Posted-message index ${indexPath} changed before compaction append.`);
|
|
278
|
+
}
|
|
279
|
+
const record = {
|
|
280
|
+
marker: COMPACTION_TRAILER_MARKER,
|
|
281
|
+
schemaVersion: COMPACTION_TRAILER_SCHEMA_VERSION,
|
|
282
|
+
targetDev: targetIdentity.dev,
|
|
283
|
+
targetIno: targetIdentity.ino,
|
|
284
|
+
lockDev: lock.identity.dev,
|
|
285
|
+
lockIno: lock.identity.ino,
|
|
286
|
+
originalSize,
|
|
287
|
+
bodyLength: body.byteLength,
|
|
288
|
+
bodySha256: sha256(body),
|
|
289
|
+
entryCount,
|
|
290
|
+
};
|
|
291
|
+
const footer = encodeCompactionTrailer(record);
|
|
292
|
+
const bodyOffset = compactionRecoveryBodyOffset(originalSize, body.byteLength);
|
|
293
|
+
const footerOffset = bodyOffset + body.byteLength;
|
|
294
|
+
const preparedSize = footerOffset + footer.byteLength;
|
|
295
|
+
if (!Number.isSafeInteger(bodyOffset)
|
|
296
|
+
|| !Number.isSafeInteger(footerOffset)
|
|
297
|
+
|| !Number.isSafeInteger(preparedSize)) {
|
|
298
|
+
throw new Error(`Posted-message index ${indexPath} compaction offsets are unsafe.`);
|
|
299
|
+
}
|
|
300
|
+
const padding = Buffer.alloc(bodyOffset - originalSize, 0x0a);
|
|
301
|
+
const recoveryBytes = Buffer.concat([padding, body]);
|
|
302
|
+
await assertLockFileIdentity(lock.path, lock.identity);
|
|
303
|
+
await assertNoSqliteSidecars(lock.path, lock.directoryIdentity);
|
|
304
|
+
await appendBuffer(appendHandle, recoveryBytes);
|
|
305
|
+
// Recovery bytes must be durable before the footer commits them. If this sync
|
|
306
|
+
// or the following hook is interrupted, the original prefix is untouched and
|
|
307
|
+
// the uncommitted JSONL-compatible tail can be retried safely.
|
|
308
|
+
await appendHandle.sync();
|
|
309
|
+
const bodyWritten = await handle.stat();
|
|
310
|
+
assertSecureFile(bodyWritten, indexPath);
|
|
311
|
+
assertSameIdentity(targetIdentity, bodyWritten, indexPath);
|
|
312
|
+
if (bodyWritten.size !== footerOffset) {
|
|
313
|
+
throw new Error(`Posted-message index ${indexPath} compaction body was interleaved.`);
|
|
314
|
+
}
|
|
315
|
+
await afterBody?.();
|
|
316
|
+
const beforeFooter = await handle.stat();
|
|
317
|
+
assertSecureFile(beforeFooter, indexPath);
|
|
318
|
+
assertSameIdentity(targetIdentity, beforeFooter, indexPath);
|
|
319
|
+
if (beforeFooter.size !== footerOffset) {
|
|
320
|
+
throw new Error(`Posted-message index ${indexPath} changed before compaction commit.`);
|
|
321
|
+
}
|
|
322
|
+
await assertLockFileIdentity(lock.path, lock.identity);
|
|
323
|
+
await assertNoSqliteSidecars(lock.path, lock.directoryIdentity);
|
|
324
|
+
await appendBuffer(appendHandle, footer);
|
|
325
|
+
await appendHandle.sync();
|
|
326
|
+
const committed = await handle.stat();
|
|
327
|
+
assertSecureFile(committed, indexPath);
|
|
328
|
+
assertSameIdentity(targetIdentity, committed, indexPath);
|
|
329
|
+
if (committed.size !== preparedSize) {
|
|
330
|
+
throw new Error(`Posted-message index ${indexPath} compaction footer was interleaved.`);
|
|
331
|
+
}
|
|
332
|
+
const prepared = await readPreparedCompaction(handle, indexPath, targetIdentity, lock.identity);
|
|
333
|
+
if (prepared === undefined
|
|
334
|
+
|| !sameCompactionTrailerRecord(prepared.record, record)
|
|
335
|
+
|| !prepared.body.equals(body)) {
|
|
336
|
+
throw new Error(`Posted-message index ${indexPath} did not durably prepare compaction.`);
|
|
337
|
+
}
|
|
338
|
+
return prepared;
|
|
339
|
+
}
|
|
340
|
+
async function openPinnedIndexForRewrite(indexPath, expectedIdentity, expectedSize, directoryIdentity) {
|
|
341
|
+
await assertFileIdentity(indexPath, expectedIdentity);
|
|
342
|
+
const handle = await open(indexPath, fsConstants.O_RDWR | noFollowFlag() | nonBlockFlag());
|
|
343
|
+
try {
|
|
344
|
+
const info = await handle.stat();
|
|
345
|
+
assertSecureFile(info, indexPath);
|
|
346
|
+
assertSameIdentity(expectedIdentity, info, indexPath);
|
|
347
|
+
if (expectedSize !== undefined && info.size !== expectedSize) {
|
|
348
|
+
throw new Error(`Posted-message index ${indexPath} changed before compaction.`);
|
|
349
|
+
}
|
|
350
|
+
await assertFileIdentity(indexPath, expectedIdentity);
|
|
351
|
+
await assertDirectoryIdentity(dirname(indexPath), directoryIdentity);
|
|
352
|
+
return handle;
|
|
353
|
+
}
|
|
354
|
+
catch (error) {
|
|
355
|
+
await handle.close().catch(() => undefined);
|
|
356
|
+
throw error;
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
async function openPinnedIndexForAppend(indexPath, expectedIdentity, expectedSize, directoryIdentity) {
|
|
360
|
+
await assertFileIdentity(indexPath, expectedIdentity);
|
|
361
|
+
const handle = await open(indexPath, fsConstants.O_WRONLY | fsConstants.O_APPEND | noFollowFlag() | nonBlockFlag());
|
|
362
|
+
try {
|
|
363
|
+
const info = await handle.stat();
|
|
364
|
+
assertSecureFile(info, indexPath);
|
|
365
|
+
assertSameIdentity(expectedIdentity, info, indexPath);
|
|
366
|
+
if (info.size !== expectedSize) {
|
|
367
|
+
throw new Error(`Posted-message index ${indexPath} changed before compaction append open.`);
|
|
368
|
+
}
|
|
369
|
+
await assertFileIdentity(indexPath, expectedIdentity);
|
|
370
|
+
await assertDirectoryIdentity(dirname(indexPath), directoryIdentity);
|
|
371
|
+
return handle;
|
|
372
|
+
}
|
|
373
|
+
catch (error) {
|
|
374
|
+
await handle.close().catch(() => undefined);
|
|
375
|
+
throw error;
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
async function rewritePinnedIndex(handle, indexPath, expectedIdentity, body, afterPrefixSync) {
|
|
379
|
+
const before = await handle.stat();
|
|
380
|
+
assertSecureFile(before, indexPath);
|
|
381
|
+
assertSameIdentity(expectedIdentity, before, indexPath);
|
|
382
|
+
// Keep the recovery footer intact until the complete replacement prefix is
|
|
383
|
+
// durable. A crash before truncate is replayed; a crash after truncate sees
|
|
384
|
+
// the already-synced replacement bytes at their final length.
|
|
385
|
+
await writeBufferAt(handle, body, 0);
|
|
386
|
+
await handle.sync();
|
|
387
|
+
await afterPrefixSync?.();
|
|
388
|
+
await handle.truncate(body.byteLength);
|
|
389
|
+
await handle.sync();
|
|
390
|
+
const after = await handle.stat();
|
|
391
|
+
assertSecureFile(after, indexPath);
|
|
392
|
+
assertSameIdentity(expectedIdentity, after, indexPath);
|
|
393
|
+
if (after.size !== body.byteLength) {
|
|
394
|
+
throw new Error(`Posted-message index ${indexPath} compaction rewrite was incomplete.`);
|
|
395
|
+
}
|
|
396
|
+
return {
|
|
397
|
+
size: after.size,
|
|
398
|
+
changeToken: await preciseFileChangeToken(handle, after, indexPath),
|
|
399
|
+
};
|
|
400
|
+
}
|
|
401
|
+
async function readPreparedCompaction(handle, indexPath, expectedIdentity, expectedLockIdentity) {
|
|
402
|
+
const info = await handle.stat();
|
|
403
|
+
assertSecureFile(info, indexPath);
|
|
404
|
+
assertSameIdentity(expectedIdentity, info, indexPath);
|
|
405
|
+
const located = await readLastCompactionTrailerLine(handle, info.size);
|
|
406
|
+
if (located === undefined) {
|
|
407
|
+
return undefined;
|
|
408
|
+
}
|
|
409
|
+
const record = decodeCompactionTrailer(located.line);
|
|
410
|
+
if (record === undefined) {
|
|
411
|
+
return undefined;
|
|
412
|
+
}
|
|
413
|
+
const footer = encodeCompactionTrailer(record);
|
|
414
|
+
if (!footer.subarray(0, footer.byteLength - 1).equals(located.line)) {
|
|
415
|
+
throw new Error(`Posted-message index ${indexPath} has a non-canonical compaction trailer.`);
|
|
416
|
+
}
|
|
417
|
+
const bodyOffset = compactionRecoveryBodyOffset(record.originalSize, record.bodyLength);
|
|
418
|
+
const expectedFooterOffset = bodyOffset + record.bodyLength;
|
|
419
|
+
const expectedPreparedSize = expectedFooterOffset + footer.byteLength;
|
|
420
|
+
if (!Number.isSafeInteger(expectedFooterOffset)
|
|
421
|
+
|| !Number.isSafeInteger(expectedPreparedSize)
|
|
422
|
+
|| located.offset !== expectedFooterOffset
|
|
423
|
+
|| info.size !== expectedPreparedSize
|
|
424
|
+
|| record.targetDev !== expectedIdentity.dev
|
|
425
|
+
|| record.targetIno !== expectedIdentity.ino
|
|
426
|
+
|| record.lockDev !== expectedLockIdentity.dev
|
|
427
|
+
|| record.lockIno !== expectedLockIdentity.ino) {
|
|
428
|
+
throw new Error(`Posted-message index ${indexPath} has a mismatched compaction trailer.`);
|
|
429
|
+
}
|
|
430
|
+
const body = await readBufferAt(handle, record.bodyLength, bodyOffset);
|
|
431
|
+
if (body === undefined || sha256(body) !== record.bodySha256) {
|
|
432
|
+
throw new Error(`Posted-message index ${indexPath} failed its compaction body digest.`);
|
|
433
|
+
}
|
|
434
|
+
const entries = parseEntries(body.toString("utf8"));
|
|
435
|
+
if (entries.length !== record.entryCount || !encodePostedMessageEntries(entries).equals(body)) {
|
|
436
|
+
throw new Error(`Posted-message index ${indexPath} has invalid compaction entries.`);
|
|
437
|
+
}
|
|
438
|
+
return { record, body };
|
|
439
|
+
}
|
|
440
|
+
async function readLastCompactionTrailerLine(handle, size) {
|
|
441
|
+
if (size === 0) {
|
|
442
|
+
return undefined;
|
|
443
|
+
}
|
|
444
|
+
const readLength = Math.min(size, COMPACTION_TRAILER_MAX_BYTES + 1);
|
|
445
|
+
const tail = await readBufferAt(handle, readLength, size - readLength);
|
|
446
|
+
if (tail === undefined || tail[tail.byteLength - 1] !== 0x0a) {
|
|
447
|
+
return undefined;
|
|
448
|
+
}
|
|
449
|
+
const lineEnd = tail.byteLength - 1;
|
|
450
|
+
const precedingNewline = tail.lastIndexOf(0x0a, lineEnd - 1);
|
|
451
|
+
if (precedingNewline < 0 && readLength < size) {
|
|
452
|
+
return undefined;
|
|
453
|
+
}
|
|
454
|
+
const lineStart = precedingNewline + 1;
|
|
455
|
+
const line = tail.subarray(lineStart, lineEnd);
|
|
456
|
+
if (line.byteLength === 0 || line.byteLength > COMPACTION_TRAILER_MAX_BYTES) {
|
|
457
|
+
return undefined;
|
|
458
|
+
}
|
|
459
|
+
return { line, offset: size - readLength + lineStart };
|
|
460
|
+
}
|
|
461
|
+
async function terminateIncompleteCompactionTail(handle, indexPath, expectedIdentity) {
|
|
462
|
+
const before = await handle.stat();
|
|
463
|
+
assertSecureFile(before, indexPath);
|
|
464
|
+
assertSameIdentity(expectedIdentity, before, indexPath);
|
|
465
|
+
if (before.size === 0) {
|
|
466
|
+
return false;
|
|
467
|
+
}
|
|
468
|
+
const last = await readBufferAt(handle, 1, before.size - 1);
|
|
469
|
+
if (last === undefined || last[0] === 0x0a) {
|
|
470
|
+
return false;
|
|
471
|
+
}
|
|
472
|
+
await writeBufferAt(handle, COMPACTION_SEPARATOR, before.size);
|
|
473
|
+
await handle.sync();
|
|
474
|
+
const after = await handle.stat();
|
|
475
|
+
assertSecureFile(after, indexPath);
|
|
476
|
+
assertSameIdentity(expectedIdentity, after, indexPath);
|
|
477
|
+
if (after.size !== before.size + COMPACTION_SEPARATOR.byteLength) {
|
|
478
|
+
throw new Error(`Posted-message index ${indexPath} tail repair was incomplete.`);
|
|
479
|
+
}
|
|
480
|
+
return true;
|
|
481
|
+
}
|
|
482
|
+
function encodeCompactionTrailer(record) {
|
|
483
|
+
const encoded = Buffer.from(`${JSON.stringify({
|
|
484
|
+
...record,
|
|
485
|
+
checksum: compactionTrailerChecksum(record),
|
|
486
|
+
})}\n`, "utf8");
|
|
487
|
+
if (encoded.byteLength > COMPACTION_TRAILER_MAX_BYTES) {
|
|
488
|
+
throw new Error("Posted-message index compaction trailer is too large.");
|
|
489
|
+
}
|
|
490
|
+
return encoded;
|
|
491
|
+
}
|
|
492
|
+
function decodeCompactionTrailer(bytes) {
|
|
493
|
+
const text = bytes.toString("utf8");
|
|
494
|
+
let value;
|
|
495
|
+
try {
|
|
496
|
+
value = JSON.parse(text);
|
|
497
|
+
}
|
|
498
|
+
catch {
|
|
499
|
+
return undefined;
|
|
500
|
+
}
|
|
501
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
502
|
+
return undefined;
|
|
503
|
+
}
|
|
504
|
+
const raw = value;
|
|
505
|
+
if (raw.marker !== COMPACTION_TRAILER_MARKER) {
|
|
506
|
+
return undefined;
|
|
507
|
+
}
|
|
508
|
+
const expectedKeys = [
|
|
509
|
+
"bodyLength",
|
|
510
|
+
"bodySha256",
|
|
511
|
+
"checksum",
|
|
512
|
+
"entryCount",
|
|
513
|
+
"lockDev",
|
|
514
|
+
"lockIno",
|
|
515
|
+
"marker",
|
|
516
|
+
"originalSize",
|
|
517
|
+
"schemaVersion",
|
|
518
|
+
"targetDev",
|
|
519
|
+
"targetIno",
|
|
520
|
+
];
|
|
521
|
+
if (JSON.stringify(Object.keys(raw).sort()) !== JSON.stringify(expectedKeys)) {
|
|
522
|
+
throw new Error("Posted-message index compaction trailer has unexpected fields.");
|
|
523
|
+
}
|
|
524
|
+
if (raw.schemaVersion !== COMPACTION_TRAILER_SCHEMA_VERSION
|
|
525
|
+
|| !isNonNegativeSafeInteger(raw.targetDev)
|
|
526
|
+
|| !isNonNegativeSafeInteger(raw.targetIno)
|
|
527
|
+
|| !isNonNegativeSafeInteger(raw.lockDev)
|
|
528
|
+
|| !isNonNegativeSafeInteger(raw.lockIno)
|
|
529
|
+
|| !isNonNegativeSafeInteger(raw.originalSize)
|
|
530
|
+
|| !isNonNegativeSafeInteger(raw.bodyLength)
|
|
531
|
+
|| typeof raw.bodySha256 !== "string"
|
|
532
|
+
|| !isNonNegativeSafeInteger(raw.entryCount)
|
|
533
|
+
|| typeof raw.checksum !== "string") {
|
|
534
|
+
throw new Error("Posted-message index compaction trailer has invalid fields.");
|
|
535
|
+
}
|
|
536
|
+
const record = {
|
|
537
|
+
marker: COMPACTION_TRAILER_MARKER,
|
|
538
|
+
schemaVersion: COMPACTION_TRAILER_SCHEMA_VERSION,
|
|
539
|
+
targetDev: raw.targetDev,
|
|
540
|
+
targetIno: raw.targetIno,
|
|
541
|
+
lockDev: raw.lockDev,
|
|
542
|
+
lockIno: raw.lockIno,
|
|
543
|
+
originalSize: raw.originalSize,
|
|
544
|
+
bodyLength: raw.bodyLength,
|
|
545
|
+
bodySha256: raw.bodySha256,
|
|
546
|
+
entryCount: raw.entryCount,
|
|
547
|
+
};
|
|
548
|
+
if (!/^[0-9a-f]{64}$/u.test(record.bodySha256)
|
|
549
|
+
|| raw.checksum !== compactionTrailerChecksum(record)) {
|
|
550
|
+
throw new Error("Posted-message index compaction trailer failed its checksum.");
|
|
551
|
+
}
|
|
552
|
+
return record;
|
|
553
|
+
}
|
|
554
|
+
function compactionTrailerChecksum(record) {
|
|
555
|
+
return sha256(Buffer.from(JSON.stringify(record), "utf8"));
|
|
556
|
+
}
|
|
557
|
+
function sameCompactionTrailerRecord(left, right) {
|
|
558
|
+
return JSON.stringify(left) === JSON.stringify(right);
|
|
559
|
+
}
|
|
560
|
+
function compactionRecoveryBodyOffset(originalSize, bodyLength) {
|
|
561
|
+
const afterOriginal = originalSize + COMPACTION_SEPARATOR.byteLength;
|
|
562
|
+
const offset = Math.max(afterOriginal, bodyLength);
|
|
563
|
+
if (!Number.isSafeInteger(afterOriginal) || !Number.isSafeInteger(offset)) {
|
|
564
|
+
throw new Error("Posted-message index compaction recovery offset is unsafe.");
|
|
565
|
+
}
|
|
566
|
+
return offset;
|
|
567
|
+
}
|
|
568
|
+
function encodePostedMessageEntries(entries) {
|
|
569
|
+
const body = entries.map((entry) => JSON.stringify(entry)).join("\n");
|
|
570
|
+
return Buffer.from(entries.length === 0 ? "" : `${body}\n`, "utf8");
|
|
571
|
+
}
|
|
572
|
+
async function appendBuffer(handle, buffer) {
|
|
573
|
+
let written = 0;
|
|
574
|
+
while (written < buffer.byteLength) {
|
|
575
|
+
const result = await handle.write(buffer, written, buffer.byteLength - written, null);
|
|
576
|
+
if (result.bytesWritten <= 0) {
|
|
577
|
+
throw new Error("Posted-message index append made no progress.");
|
|
578
|
+
}
|
|
579
|
+
written += result.bytesWritten;
|
|
580
|
+
}
|
|
581
|
+
}
|
|
582
|
+
async function writeBufferAt(handle, buffer, position) {
|
|
583
|
+
let written = 0;
|
|
584
|
+
while (written < buffer.byteLength) {
|
|
585
|
+
const result = await handle.write(buffer, written, buffer.byteLength - written, position + written);
|
|
586
|
+
if (result.bytesWritten <= 0) {
|
|
587
|
+
throw new Error("Posted-message index write made no progress.");
|
|
588
|
+
}
|
|
589
|
+
written += result.bytesWritten;
|
|
590
|
+
}
|
|
591
|
+
}
|
|
592
|
+
async function readBufferAt(handle, length, position) {
|
|
593
|
+
const buffer = Buffer.alloc(length);
|
|
594
|
+
let read = 0;
|
|
595
|
+
while (read < length) {
|
|
596
|
+
const result = await handle.read(buffer, read, length - read, position + read);
|
|
597
|
+
if (result.bytesRead <= 0) {
|
|
598
|
+
return undefined;
|
|
599
|
+
}
|
|
600
|
+
read += result.bytesRead;
|
|
601
|
+
}
|
|
602
|
+
return buffer;
|
|
603
|
+
}
|
|
604
|
+
function sha256(bytes) {
|
|
605
|
+
return createHash("sha256").update(bytes).digest("hex");
|
|
606
|
+
}
|
|
607
|
+
function isNonNegativeSafeInteger(value) {
|
|
608
|
+
return Number.isSafeInteger(value) && value >= 0;
|
|
609
|
+
}
|
|
610
|
+
function normalizedAppendCap(maxEntries) {
|
|
611
|
+
return Number.isSafeInteger(maxEntries) && maxEntries > 0
|
|
612
|
+
? maxEntries
|
|
613
|
+
: DEFAULT_COMPACT_MAX_ENTRIES;
|
|
614
|
+
}
|
|
615
|
+
function normalizedCompactionCap(maxEntries) {
|
|
616
|
+
return Number.isSafeInteger(maxEntries) && maxEntries >= 0
|
|
617
|
+
? maxEntries
|
|
618
|
+
: DEFAULT_COMPACT_MAX_ENTRIES;
|
|
619
|
+
}
|
|
620
|
+
function amortizedCompactTarget(maxEntries) {
|
|
621
|
+
const headroom = Math.max(1, Math.ceil(maxEntries / COMPACT_HEADROOM_DIVISOR));
|
|
622
|
+
return Math.max(0, maxEntries - headroom);
|
|
623
|
+
}
|
|
624
|
+
async function loadPostedMessageIndexState(indexPath, directoryIdentity, expectedIdentity) {
|
|
625
|
+
const current = await secureFileVersion(indexPath, expectedIdentity, directoryIdentity);
|
|
626
|
+
const cached = inProcessIndexStates.get(indexPath);
|
|
627
|
+
// Size plus inode is not a freshness proof: another locked process can compact
|
|
628
|
+
// the same inode back to the same byte length. Precise change metadata preserves
|
|
629
|
+
// the O(1) hot path while forcing a secure recount after any such rewrite.
|
|
630
|
+
if (cached !== undefined
|
|
631
|
+
&& cached.size === current.size
|
|
632
|
+
&& sameFileChangeToken(cached.changeToken, current.changeToken)
|
|
633
|
+
&& sameIdentity(cached.identity, expectedIdentity)) {
|
|
634
|
+
return cached;
|
|
635
|
+
}
|
|
636
|
+
const snapshot = await tryReadIndexSnapshot(indexPath, directoryIdentity);
|
|
637
|
+
if (snapshot === undefined) {
|
|
638
|
+
return undefined;
|
|
639
|
+
}
|
|
640
|
+
assertSameIdentity(expectedIdentity, snapshot.identity, indexPath);
|
|
641
|
+
inProcessIndexStates.set(indexPath, snapshot);
|
|
642
|
+
return snapshot;
|
|
643
|
+
}
|
|
644
|
+
async function withPostedMessageIndexLock(indexPath, action) {
|
|
645
|
+
// Avoid blocking this process's event loop on its own synchronous SQLite lock;
|
|
646
|
+
// the OS-backed lock below then serializes the adapter against stdio children.
|
|
647
|
+
const prior = inProcessIndexTails.get(indexPath) ?? Promise.resolve();
|
|
648
|
+
let releaseQueue;
|
|
649
|
+
const queueSlot = new Promise((resolve) => {
|
|
650
|
+
releaseQueue = resolve;
|
|
651
|
+
});
|
|
652
|
+
const tail = prior.then(() => queueSlot);
|
|
653
|
+
inProcessIndexTails.set(indexPath, tail);
|
|
654
|
+
await prior;
|
|
655
|
+
try {
|
|
656
|
+
const lock = await acquirePostedMessageIndexLock(indexPath);
|
|
657
|
+
if (lock === undefined) {
|
|
658
|
+
return undefined;
|
|
659
|
+
}
|
|
660
|
+
try {
|
|
661
|
+
if (!await recoverPreparedCompaction(indexPath, lock)) {
|
|
662
|
+
return undefined;
|
|
663
|
+
}
|
|
664
|
+
return await action(lock);
|
|
665
|
+
}
|
|
666
|
+
finally {
|
|
667
|
+
await lock.release();
|
|
668
|
+
}
|
|
669
|
+
}
|
|
670
|
+
catch {
|
|
671
|
+
// Best-effort; a posted-message index failure never fails the Slack post.
|
|
672
|
+
return undefined;
|
|
673
|
+
}
|
|
674
|
+
finally {
|
|
675
|
+
releaseQueue();
|
|
676
|
+
if (inProcessIndexTails.get(indexPath) === tail) {
|
|
677
|
+
inProcessIndexTails.delete(indexPath);
|
|
678
|
+
}
|
|
679
|
+
}
|
|
680
|
+
}
|
|
681
|
+
async function acquirePostedMessageIndexLock(indexPath) {
|
|
682
|
+
let DatabaseSync;
|
|
683
|
+
let directoryIdentity;
|
|
684
|
+
try {
|
|
685
|
+
directoryIdentity = await ensureIndexDirectory(indexPath);
|
|
686
|
+
({ DatabaseSync } = await import("node:sqlite"));
|
|
687
|
+
}
|
|
688
|
+
catch {
|
|
689
|
+
return undefined;
|
|
690
|
+
}
|
|
691
|
+
const lockPath = `${indexPath}.lock.sqlite`;
|
|
692
|
+
let lockIdentity;
|
|
121
693
|
try {
|
|
122
|
-
await
|
|
123
|
-
|
|
694
|
+
const ensured = await ensureOwnerOnlyFile(lockPath, directoryIdentity, true);
|
|
695
|
+
if (ensured === undefined) {
|
|
696
|
+
return undefined;
|
|
697
|
+
}
|
|
698
|
+
lockIdentity = ensured;
|
|
699
|
+
await assertLockFileIdentity(lockPath, lockIdentity);
|
|
700
|
+
await assertNoSqliteSidecars(lockPath, directoryIdentity);
|
|
124
701
|
}
|
|
125
702
|
catch {
|
|
126
|
-
|
|
703
|
+
return undefined;
|
|
704
|
+
}
|
|
705
|
+
const deadline = Date.now() + INDEX_LOCK_WAIT_MS;
|
|
706
|
+
while (true) {
|
|
707
|
+
let database;
|
|
708
|
+
try {
|
|
709
|
+
await assertLockFileIdentity(lockPath, lockIdentity);
|
|
710
|
+
await assertNoSqliteSidecars(lockPath, directoryIdentity);
|
|
711
|
+
database = new DatabaseSync(lockPath, { timeout: 0 });
|
|
712
|
+
// MEMORY preserves SQLite's kernel-backed cross-process lock while avoiding
|
|
713
|
+
// attacker-preparable, umask-dependent -journal/-wal/-shm filesystem paths.
|
|
714
|
+
// This database is lock-only: no schema or count state is ever mutated, so
|
|
715
|
+
// process death never relies on a MEMORY journal for data recovery.
|
|
716
|
+
database.exec("PRAGMA journal_mode=MEMORY");
|
|
717
|
+
await assertLockFileIdentity(lockPath, lockIdentity);
|
|
718
|
+
await assertNoSqliteSidecars(lockPath, directoryIdentity);
|
|
719
|
+
// The kernel releases this transaction lock automatically on close or
|
|
720
|
+
// process death; there is no stale-path cleanup race.
|
|
721
|
+
database.exec("BEGIN IMMEDIATE");
|
|
722
|
+
await assertLockFileIdentity(lockPath, lockIdentity);
|
|
723
|
+
await assertNoSqliteSidecars(lockPath, directoryIdentity);
|
|
724
|
+
return postedMessageIndexLock(database, lockPath, lockIdentity, directoryIdentity);
|
|
725
|
+
}
|
|
726
|
+
catch (error) {
|
|
727
|
+
try {
|
|
728
|
+
database?.close();
|
|
729
|
+
}
|
|
730
|
+
catch {
|
|
731
|
+
// Best-effort cleanup after a failed acquisition.
|
|
732
|
+
}
|
|
733
|
+
if (!isSqliteBusy(error) || Date.now() >= deadline) {
|
|
734
|
+
return undefined;
|
|
735
|
+
}
|
|
736
|
+
await delay(8 + Math.floor(Math.random() * 8));
|
|
737
|
+
}
|
|
127
738
|
}
|
|
128
739
|
}
|
|
129
|
-
|
|
130
|
-
let
|
|
740
|
+
function postedMessageIndexLock(database, lockPath, lockIdentity, directoryIdentity) {
|
|
741
|
+
let released = false;
|
|
742
|
+
return {
|
|
743
|
+
directoryIdentity,
|
|
744
|
+
path: lockPath,
|
|
745
|
+
identity: lockIdentity,
|
|
746
|
+
async release() {
|
|
747
|
+
if (released) {
|
|
748
|
+
return;
|
|
749
|
+
}
|
|
750
|
+
released = true;
|
|
751
|
+
try {
|
|
752
|
+
if (database.isTransaction) {
|
|
753
|
+
database.exec("ROLLBACK");
|
|
754
|
+
}
|
|
755
|
+
}
|
|
756
|
+
catch {
|
|
757
|
+
// close() is the authoritative kernel-lock release. SQLite is deliberately
|
|
758
|
+
// lock-only, so there is no mutable coordinator state to recover.
|
|
759
|
+
}
|
|
760
|
+
try {
|
|
761
|
+
database.close();
|
|
762
|
+
}
|
|
763
|
+
catch {
|
|
764
|
+
// The connection is no longer reusable.
|
|
765
|
+
}
|
|
766
|
+
await assertLockFileIdentity(lockPath, lockIdentity);
|
|
767
|
+
await assertNoSqliteSidecars(lockPath, directoryIdentity);
|
|
768
|
+
},
|
|
769
|
+
};
|
|
770
|
+
}
|
|
771
|
+
function isSqliteBusy(error) {
|
|
772
|
+
return error instanceof Error && /database is locked|database is busy/iu.test(error.message);
|
|
773
|
+
}
|
|
774
|
+
async function delay(ms) {
|
|
775
|
+
await new Promise((resolve) => setTimeout(resolve, ms));
|
|
776
|
+
}
|
|
777
|
+
async function tryReadIndexSnapshot(indexPath, directoryIdentity) {
|
|
778
|
+
let handle;
|
|
131
779
|
try {
|
|
132
|
-
|
|
780
|
+
const identity = await ensureOwnerOnlyFile(indexPath, directoryIdentity, false);
|
|
781
|
+
if (identity === undefined) {
|
|
782
|
+
return undefined;
|
|
783
|
+
}
|
|
784
|
+
const before = await lstat(indexPath);
|
|
785
|
+
assertSecureFile(before, indexPath);
|
|
786
|
+
assertSameIdentity(identity, before, indexPath);
|
|
787
|
+
handle = await open(indexPath, fsConstants.O_RDONLY | noFollowFlag() | nonBlockFlag());
|
|
788
|
+
const opened = await handle.stat();
|
|
789
|
+
assertSecureFile(opened, indexPath);
|
|
790
|
+
assertSameIdentity(identity, opened, indexPath);
|
|
791
|
+
const openedToken = await preciseFileChangeToken(handle, opened, indexPath);
|
|
792
|
+
const bytes = await handle.readFile();
|
|
793
|
+
const after = await handle.stat();
|
|
794
|
+
assertSecureFile(after, indexPath);
|
|
795
|
+
assertSameIdentity(opened, after, indexPath);
|
|
796
|
+
const afterToken = await preciseFileChangeToken(handle, after, indexPath);
|
|
797
|
+
if (opened.size !== after.size
|
|
798
|
+
|| !sameFileChangeToken(openedToken, afterToken)
|
|
799
|
+
|| bytes.byteLength !== after.size) {
|
|
800
|
+
throw new Error(`Posted-message index ${indexPath} changed while its snapshot was read.`);
|
|
801
|
+
}
|
|
802
|
+
await assertFileIdentity(indexPath, identity);
|
|
803
|
+
await assertDirectoryIdentity(dirname(indexPath), directoryIdentity);
|
|
804
|
+
const entries = parseEntries(bytes.toString("utf8"));
|
|
805
|
+
return {
|
|
806
|
+
entries,
|
|
807
|
+
count: entries.length,
|
|
808
|
+
size: bytes.byteLength,
|
|
809
|
+
identity,
|
|
810
|
+
changeToken: afterToken,
|
|
811
|
+
};
|
|
133
812
|
}
|
|
134
813
|
catch {
|
|
135
|
-
return
|
|
814
|
+
return undefined;
|
|
136
815
|
}
|
|
816
|
+
finally {
|
|
817
|
+
await handle?.close().catch(() => undefined);
|
|
818
|
+
}
|
|
819
|
+
}
|
|
820
|
+
function parseEntries(raw) {
|
|
137
821
|
const out = [];
|
|
138
822
|
for (const line of raw.split("\n")) {
|
|
139
823
|
const trimmed = line.trim();
|
|
@@ -147,6 +831,338 @@ async function readEntries(indexPath) {
|
|
|
147
831
|
}
|
|
148
832
|
return out;
|
|
149
833
|
}
|
|
834
|
+
async function appendSecureIndexLine(indexPath, line, expected, directoryIdentity) {
|
|
835
|
+
let handle;
|
|
836
|
+
try {
|
|
837
|
+
await assertFileIdentity(indexPath, expected.identity);
|
|
838
|
+
handle = await open(indexPath, fsConstants.O_WRONLY | fsConstants.O_APPEND | noFollowFlag() | nonBlockFlag());
|
|
839
|
+
const opened = await handle.stat();
|
|
840
|
+
assertSecureFile(opened, indexPath);
|
|
841
|
+
assertSameIdentity(expected.identity, opened, indexPath);
|
|
842
|
+
if (opened.size !== expected.size) {
|
|
843
|
+
throw new Error(`Posted-message index ${indexPath} changed before append.`);
|
|
844
|
+
}
|
|
845
|
+
const result = await handle.write(line);
|
|
846
|
+
const lineBytes = Buffer.byteLength(line);
|
|
847
|
+
if (result.bytesWritten !== lineBytes) {
|
|
848
|
+
throw new Error(`Posted-message index ${indexPath} append was incomplete.`);
|
|
849
|
+
}
|
|
850
|
+
const after = await handle.stat();
|
|
851
|
+
assertSecureFile(after, indexPath);
|
|
852
|
+
assertSameIdentity(opened, after, indexPath);
|
|
853
|
+
if (after.size !== expected.size + lineBytes) {
|
|
854
|
+
throw new Error(`Posted-message index ${indexPath} changed during append.`);
|
|
855
|
+
}
|
|
856
|
+
const changeToken = await preciseFileChangeToken(handle, after, indexPath);
|
|
857
|
+
await assertFileIdentity(indexPath, expected.identity);
|
|
858
|
+
await assertDirectoryIdentity(dirname(indexPath), directoryIdentity);
|
|
859
|
+
return {
|
|
860
|
+
size: after.size,
|
|
861
|
+
changeToken,
|
|
862
|
+
};
|
|
863
|
+
}
|
|
864
|
+
finally {
|
|
865
|
+
await handle?.close().catch(() => undefined);
|
|
866
|
+
}
|
|
867
|
+
}
|
|
868
|
+
async function secureFileVersion(path, expectedIdentity, directoryIdentity) {
|
|
869
|
+
let handle;
|
|
870
|
+
try {
|
|
871
|
+
await assertFileIdentity(path, expectedIdentity);
|
|
872
|
+
handle = await open(path, fsConstants.O_RDONLY | noFollowFlag() | nonBlockFlag());
|
|
873
|
+
const info = await handle.stat();
|
|
874
|
+
assertSecureFile(info, path);
|
|
875
|
+
assertSameIdentity(expectedIdentity, info, path);
|
|
876
|
+
const changeToken = await preciseFileChangeToken(handle, info, path);
|
|
877
|
+
await assertFileIdentity(path, expectedIdentity);
|
|
878
|
+
await assertDirectoryIdentity(dirname(path), directoryIdentity);
|
|
879
|
+
return {
|
|
880
|
+
size: info.size,
|
|
881
|
+
changeToken,
|
|
882
|
+
};
|
|
883
|
+
}
|
|
884
|
+
finally {
|
|
885
|
+
await handle?.close().catch(() => undefined);
|
|
886
|
+
}
|
|
887
|
+
}
|
|
888
|
+
async function preciseFileChangeToken(handle, expected, path) {
|
|
889
|
+
const precise = await handle.stat({ bigint: true });
|
|
890
|
+
if (precise.dev !== BigInt(expected.dev)
|
|
891
|
+
|| precise.ino !== BigInt(expected.ino)
|
|
892
|
+
|| precise.size !== BigInt(expected.size)) {
|
|
893
|
+
throw new Error(`Posted-message index ${path} changed while its metadata was read.`);
|
|
894
|
+
}
|
|
895
|
+
return { ctimeNs: precise.ctimeNs, mtimeNs: precise.mtimeNs };
|
|
896
|
+
}
|
|
897
|
+
function sameFileChangeToken(left, right) {
|
|
898
|
+
return left.ctimeNs === right.ctimeNs && left.mtimeNs === right.mtimeNs;
|
|
899
|
+
}
|
|
900
|
+
async function ensureOwnerOnlyFile(path, directoryIdentity, createIfMissing) {
|
|
901
|
+
const directory = dirname(path);
|
|
902
|
+
await assertDirectoryIdentity(directory, directoryIdentity);
|
|
903
|
+
let created = false;
|
|
904
|
+
let createHandle;
|
|
905
|
+
if (createIfMissing) {
|
|
906
|
+
try {
|
|
907
|
+
try {
|
|
908
|
+
createHandle = await open(path, fsConstants.O_RDWR | fsConstants.O_CREAT | fsConstants.O_EXCL | noFollowFlag() | nonBlockFlag(), 0o600);
|
|
909
|
+
created = true;
|
|
910
|
+
await createHandle.chmod(0o600);
|
|
911
|
+
await createHandle.sync();
|
|
912
|
+
}
|
|
913
|
+
catch (error) {
|
|
914
|
+
if (!isErrno(error, "EEXIST")) {
|
|
915
|
+
throw error;
|
|
916
|
+
}
|
|
917
|
+
}
|
|
918
|
+
}
|
|
919
|
+
finally {
|
|
920
|
+
await createHandle?.close().catch(() => undefined);
|
|
921
|
+
}
|
|
922
|
+
}
|
|
923
|
+
if (!created && !createIfMissing) {
|
|
924
|
+
try {
|
|
925
|
+
await lstat(path);
|
|
926
|
+
}
|
|
927
|
+
catch (error) {
|
|
928
|
+
if (isErrno(error, "ENOENT")) {
|
|
929
|
+
return undefined;
|
|
930
|
+
}
|
|
931
|
+
throw error;
|
|
932
|
+
}
|
|
933
|
+
}
|
|
934
|
+
const before = await lstat(path);
|
|
935
|
+
assertSecureFileShape(before, path);
|
|
936
|
+
let handle;
|
|
937
|
+
try {
|
|
938
|
+
handle = await open(path, fsConstants.O_RDWR | noFollowFlag() | nonBlockFlag());
|
|
939
|
+
const opened = await handle.stat();
|
|
940
|
+
assertSecureFileShape(opened, path);
|
|
941
|
+
assertSameIdentity(before, opened, path);
|
|
942
|
+
if (process.platform !== "win32" && (opened.mode & 0o777) !== 0o600) {
|
|
943
|
+
await handle.chmod(0o600);
|
|
944
|
+
await handle.sync();
|
|
945
|
+
}
|
|
946
|
+
const secured = await handle.stat();
|
|
947
|
+
assertSecureFile(secured, path);
|
|
948
|
+
assertSameIdentity(opened, secured, path);
|
|
949
|
+
const after = await lstat(path);
|
|
950
|
+
assertSecureFile(after, path);
|
|
951
|
+
assertSameIdentity(secured, after, path);
|
|
952
|
+
await assertDirectoryIdentity(directory, directoryIdentity);
|
|
953
|
+
if (created) {
|
|
954
|
+
await fsyncDirectory(directory, directoryIdentity);
|
|
955
|
+
}
|
|
956
|
+
return identityOf(secured);
|
|
957
|
+
}
|
|
958
|
+
finally {
|
|
959
|
+
await handle?.close().catch(() => undefined);
|
|
960
|
+
}
|
|
961
|
+
}
|
|
962
|
+
async function ensureIndexDirectory(indexPath) {
|
|
963
|
+
const directory = dirname(indexPath);
|
|
964
|
+
await createDirectoryPathWithoutSymlinks(directory);
|
|
965
|
+
const identity = await existingIndexDirectoryIdentityRequired(indexPath);
|
|
966
|
+
return await secureOwnerOnlyDirectory(directory, identity);
|
|
967
|
+
}
|
|
968
|
+
async function existingIndexDirectoryIdentity(indexPath) {
|
|
969
|
+
try {
|
|
970
|
+
return await existingIndexDirectoryIdentityRequired(indexPath);
|
|
971
|
+
}
|
|
972
|
+
catch (error) {
|
|
973
|
+
if (isErrno(error, "ENOENT")) {
|
|
974
|
+
return undefined;
|
|
975
|
+
}
|
|
976
|
+
throw error;
|
|
977
|
+
}
|
|
978
|
+
}
|
|
979
|
+
async function existingIndexDirectoryIdentityRequired(indexPath) {
|
|
980
|
+
const directory = dirname(indexPath);
|
|
981
|
+
const info = await lstat(directory);
|
|
982
|
+
assertSecureDirectory(info, directory);
|
|
983
|
+
return identityOf(info);
|
|
984
|
+
}
|
|
985
|
+
async function createDirectoryPathWithoutSymlinks(path) {
|
|
986
|
+
const absolute = resolve(path);
|
|
987
|
+
const parsed = parse(absolute);
|
|
988
|
+
const segments = absolute.slice(parsed.root.length).split(sep).filter(Boolean);
|
|
989
|
+
let current = parsed.root;
|
|
990
|
+
for (const segment of segments) {
|
|
991
|
+
current = join(current, segment);
|
|
992
|
+
let info;
|
|
993
|
+
try {
|
|
994
|
+
info = await lstat(current);
|
|
995
|
+
}
|
|
996
|
+
catch (error) {
|
|
997
|
+
if (!isErrno(error, "ENOENT")) {
|
|
998
|
+
throw error;
|
|
999
|
+
}
|
|
1000
|
+
try {
|
|
1001
|
+
await mkdir(current, { mode: 0o700 });
|
|
1002
|
+
}
|
|
1003
|
+
catch (mkdirError) {
|
|
1004
|
+
if (!isErrno(mkdirError, "EEXIST")) {
|
|
1005
|
+
throw mkdirError;
|
|
1006
|
+
}
|
|
1007
|
+
}
|
|
1008
|
+
info = await lstat(current);
|
|
1009
|
+
}
|
|
1010
|
+
// macOS exposes root-owned compatibility links such as /var -> /private/var.
|
|
1011
|
+
// User-controlled links anywhere in the configured path remain fail-closed.
|
|
1012
|
+
if (info.isSymbolicLink()) {
|
|
1013
|
+
const uid = process.getuid?.();
|
|
1014
|
+
if (uid === undefined || info.uid !== 0 || uid === 0) {
|
|
1015
|
+
throw new Error(`Posted-message index path component ${current} must not be a user-controlled symbolic link.`);
|
|
1016
|
+
}
|
|
1017
|
+
continue;
|
|
1018
|
+
}
|
|
1019
|
+
if (!info.isDirectory()) {
|
|
1020
|
+
throw new Error(`Posted-message index path component ${current} must be a directory.`);
|
|
1021
|
+
}
|
|
1022
|
+
assertSafeDirectoryComponent(info, current);
|
|
1023
|
+
}
|
|
1024
|
+
}
|
|
1025
|
+
async function secureOwnerOnlyDirectory(path, expected) {
|
|
1026
|
+
if (process.platform === "win32") {
|
|
1027
|
+
await assertDirectoryIdentity(path, expected);
|
|
1028
|
+
return expected;
|
|
1029
|
+
}
|
|
1030
|
+
let handle;
|
|
1031
|
+
try {
|
|
1032
|
+
handle = await open(path, fsConstants.O_RDONLY | (fsConstants.O_DIRECTORY ?? 0) | noFollowFlag());
|
|
1033
|
+
const opened = await handle.stat();
|
|
1034
|
+
assertSecureDirectory(opened, path);
|
|
1035
|
+
assertSameIdentity(expected, opened, path);
|
|
1036
|
+
if ((opened.mode & 0o777) !== 0o700) {
|
|
1037
|
+
await handle.chmod(0o700);
|
|
1038
|
+
await handle.sync();
|
|
1039
|
+
}
|
|
1040
|
+
const secured = await handle.stat();
|
|
1041
|
+
assertSecureDirectory(secured, path);
|
|
1042
|
+
if ((secured.mode & 0o777) !== 0o700) {
|
|
1043
|
+
throw new Error(`Posted-message index directory ${path} must have owner-only mode 0700.`);
|
|
1044
|
+
}
|
|
1045
|
+
assertSameIdentity(opened, secured, path);
|
|
1046
|
+
const after = await lstat(path);
|
|
1047
|
+
assertSecureDirectory(after, path);
|
|
1048
|
+
assertSameIdentity(secured, after, path);
|
|
1049
|
+
return identityOf(secured);
|
|
1050
|
+
}
|
|
1051
|
+
finally {
|
|
1052
|
+
await handle?.close().catch(() => undefined);
|
|
1053
|
+
}
|
|
1054
|
+
}
|
|
1055
|
+
async function assertNoSqliteSidecars(lockPath, directoryIdentity) {
|
|
1056
|
+
await assertDirectoryIdentity(dirname(lockPath), directoryIdentity);
|
|
1057
|
+
for (const suffix of SQLITE_SIDECAR_SUFFIXES) {
|
|
1058
|
+
const path = `${lockPath}${suffix}`;
|
|
1059
|
+
try {
|
|
1060
|
+
await lstat(path);
|
|
1061
|
+
}
|
|
1062
|
+
catch (error) {
|
|
1063
|
+
if (isErrno(error, "ENOENT")) {
|
|
1064
|
+
continue;
|
|
1065
|
+
}
|
|
1066
|
+
throw error;
|
|
1067
|
+
}
|
|
1068
|
+
throw new Error(`Posted-message index SQLite sidecar path ${path} must not exist.`);
|
|
1069
|
+
}
|
|
1070
|
+
}
|
|
1071
|
+
function assertSecureDirectory(info, path) {
|
|
1072
|
+
if (info.isSymbolicLink() || !info.isDirectory()) {
|
|
1073
|
+
throw new Error(`Posted-message index directory ${path} must be a non-symlink directory.`);
|
|
1074
|
+
}
|
|
1075
|
+
assertOwnedByCurrentUser(info, path);
|
|
1076
|
+
if (process.platform !== "win32" && (info.mode & 0o022) !== 0) {
|
|
1077
|
+
throw new Error(`Posted-message index directory ${path} must not be group/world writable.`);
|
|
1078
|
+
}
|
|
1079
|
+
}
|
|
1080
|
+
function assertSafeDirectoryComponent(info, path) {
|
|
1081
|
+
if (process.platform === "win32" || (info.mode & 0o022) === 0) {
|
|
1082
|
+
return;
|
|
1083
|
+
}
|
|
1084
|
+
const uid = process.getuid?.();
|
|
1085
|
+
const rootOwnedSticky = uid !== undefined && info.uid === 0 && (info.mode & 0o1000) !== 0;
|
|
1086
|
+
if (!rootOwnedSticky) {
|
|
1087
|
+
throw new Error(`Posted-message index path component ${path} must not be group/world writable.`);
|
|
1088
|
+
}
|
|
1089
|
+
}
|
|
1090
|
+
function assertSecureFileShape(info, path) {
|
|
1091
|
+
if (info.isSymbolicLink() || !info.isFile()) {
|
|
1092
|
+
throw new Error(`Posted-message index path ${path} must be a non-symlink regular file.`);
|
|
1093
|
+
}
|
|
1094
|
+
assertOwnedByCurrentUser(info, path);
|
|
1095
|
+
if (info.nlink !== 1) {
|
|
1096
|
+
throw new Error(`Posted-message index file ${path} must have exactly one hard link.`);
|
|
1097
|
+
}
|
|
1098
|
+
}
|
|
1099
|
+
function assertSecureFile(info, path) {
|
|
1100
|
+
assertSecureFileShape(info, path);
|
|
1101
|
+
if (process.platform !== "win32" && (info.mode & 0o777) !== 0o600) {
|
|
1102
|
+
throw new Error(`Posted-message index file ${path} must have owner-only mode 0600.`);
|
|
1103
|
+
}
|
|
1104
|
+
}
|
|
1105
|
+
function assertOwnedByCurrentUser(info, path) {
|
|
1106
|
+
const uid = process.getuid?.();
|
|
1107
|
+
if (uid !== undefined && info.uid !== uid) {
|
|
1108
|
+
throw new Error(`Posted-message index path ${path} must be owned by the current user.`);
|
|
1109
|
+
}
|
|
1110
|
+
}
|
|
1111
|
+
async function assertDirectoryIdentity(path, expected) {
|
|
1112
|
+
const info = await lstat(path);
|
|
1113
|
+
assertSecureDirectory(info, path);
|
|
1114
|
+
assertSameIdentity(expected, info, path);
|
|
1115
|
+
}
|
|
1116
|
+
async function assertFileIdentity(path, expected) {
|
|
1117
|
+
const info = await lstat(path);
|
|
1118
|
+
assertSecureFile(info, path);
|
|
1119
|
+
assertSameIdentity(expected, info, path);
|
|
1120
|
+
}
|
|
1121
|
+
async function assertLockFileIdentity(path, expected) {
|
|
1122
|
+
const info = await lstat(path);
|
|
1123
|
+
assertSecureFile(info, path);
|
|
1124
|
+
assertSameIdentity(expected, info, path);
|
|
1125
|
+
if (info.size > MAX_LOCK_FILE_BYTES) {
|
|
1126
|
+
throw new Error(`Posted-message index lock ${path} is unexpectedly large.`);
|
|
1127
|
+
}
|
|
1128
|
+
}
|
|
1129
|
+
function assertSameIdentity(before, after, path) {
|
|
1130
|
+
if (!sameIdentity(before, after)) {
|
|
1131
|
+
throw new Error(`Posted-message index path ${path} changed while it was in use.`);
|
|
1132
|
+
}
|
|
1133
|
+
}
|
|
1134
|
+
function sameIdentity(left, right) {
|
|
1135
|
+
return left.dev === right.dev && left.ino === right.ino;
|
|
1136
|
+
}
|
|
1137
|
+
function identityOf(info) {
|
|
1138
|
+
return { dev: info.dev, ino: info.ino };
|
|
1139
|
+
}
|
|
1140
|
+
async function fsyncDirectory(path, expected) {
|
|
1141
|
+
if (process.platform === "win32") {
|
|
1142
|
+
await assertDirectoryIdentity(path, expected);
|
|
1143
|
+
return;
|
|
1144
|
+
}
|
|
1145
|
+
let handle;
|
|
1146
|
+
try {
|
|
1147
|
+
handle = await open(path, fsConstants.O_RDONLY | (fsConstants.O_DIRECTORY ?? 0) | noFollowFlag());
|
|
1148
|
+
const info = await handle.stat();
|
|
1149
|
+
assertSecureDirectory(info, path);
|
|
1150
|
+
assertSameIdentity(expected, info, path);
|
|
1151
|
+
await handle.sync();
|
|
1152
|
+
}
|
|
1153
|
+
finally {
|
|
1154
|
+
await handle?.close().catch(() => undefined);
|
|
1155
|
+
}
|
|
1156
|
+
}
|
|
1157
|
+
function noFollowFlag() {
|
|
1158
|
+
return fsConstants.O_NOFOLLOW ?? 0;
|
|
1159
|
+
}
|
|
1160
|
+
function nonBlockFlag() {
|
|
1161
|
+
return fsConstants.O_NONBLOCK ?? 0;
|
|
1162
|
+
}
|
|
1163
|
+
function isErrno(error, code) {
|
|
1164
|
+
return error?.code === code;
|
|
1165
|
+
}
|
|
150
1166
|
function parseEntry(line) {
|
|
151
1167
|
let json;
|
|
152
1168
|
try {
|