@vellumai/assistant 0.12.2 → 0.12.3-staging.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +2 -0
- package/AGENTS.md +1 -1
- package/docs/architecture/integrations.md +7 -0
- package/docs/architecture/memory.md +17 -2
- package/docs/credential-execution-service.md +1 -1
- package/docs/desktop-browser-cli.md +7 -3
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/__tests__/secret-detection.test.ts +1 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/secret-detection.ts +4 -1
- package/node_modules/@vellumai/environments/src/shell.test.ts +21 -0
- package/node_modules/@vellumai/environments/src/shell.ts +24 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/__tests__/secret-detection.test.ts +1 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/secret-detection.ts +4 -1
- package/node_modules/@vellumai/gateway-client/src/inbound-contract.ts +8 -2
- package/node_modules/@vellumai/service-contracts/src/__tests__/secret-detection.test.ts +1 -0
- package/node_modules/@vellumai/service-contracts/src/secret-detection.ts +4 -1
- package/openapi.yaml +54 -5
- package/package.json +1 -1
- package/scripts/smoke-desktop-browser-cli.ts +1 -0
- package/scripts/sync-llm-catalog.ts +3 -0
- package/scripts/voice-ttft-spike.ts +2 -2
- package/src/__tests__/agent-loop.test.ts +259 -0
- package/src/__tests__/agent-wake-delegation-prompt.test.ts +64 -2
- package/src/__tests__/anthropic-provider.test.ts +55 -0
- package/src/__tests__/approval-interception-trust-gates.test.ts +40 -0
- package/src/__tests__/attachments-store.test.ts +22 -3
- package/src/__tests__/channel-approval.test.ts +9 -14
- package/src/__tests__/channel-reply-delivery.test.ts +56 -0
- package/src/__tests__/chat-credential-redaction.test.ts +23 -0
- package/src/__tests__/computer-use-screenshot-attachments.test.ts +411 -0
- package/src/__tests__/computer-use-screenshot-selection.test.ts +50 -0
- package/src/__tests__/conversation-agent-loop-inference-profile.test.ts +2 -0
- package/src/__tests__/conversation-agent-loop-overflow.test.ts +2 -0
- package/src/__tests__/conversation-agent-loop.test.ts +257 -6
- package/src/__tests__/conversation-attachments.test.ts +106 -0
- package/src/__tests__/conversation-fork-crud.test.ts +191 -0
- package/src/__tests__/conversation-rendered-delegation-state.test.ts +276 -0
- package/src/__tests__/credential-execution-client.test.ts +70 -53
- package/src/__tests__/db-conversation-tool-surface.test.ts +221 -0
- package/src/__tests__/events-tail-route.test.ts +33 -0
- package/src/__tests__/history-repair.test.ts +53 -0
- package/src/__tests__/inference-profile-session-handler.test.ts +26 -0
- package/src/__tests__/list-messages-tool-merge.test.ts +85 -1
- package/src/__tests__/llm-catalog-parity.test.ts +27 -4
- package/src/__tests__/oauth-apps-routes.test.ts +1 -0
- package/src/__tests__/oauth-commands-routes.test.ts +114 -101
- package/src/__tests__/oauth-connect-orchestrator.test.ts +2 -0
- package/src/__tests__/oauth-provider-serializer.test.ts +1 -0
- package/src/__tests__/oauth-providers-routes.test.ts +2 -0
- package/src/__tests__/persist-media-references.test.ts +50 -0
- package/src/__tests__/plugin-import-boundary-guard.test.ts +0 -1
- package/src/__tests__/run-conversation-turn-persistence.test.ts +138 -1
- package/src/__tests__/schedule-routes.test.ts +20 -0
- package/src/__tests__/scheduler-result-notification.test.ts +23 -4
- package/src/__tests__/script-proxy-certs.test.ts +1 -1
- package/src/__tests__/secret-routes-platform-proxy.test.ts +86 -31
- package/src/__tests__/secret-routes-scrub.test.ts +22 -27
- package/src/__tests__/secret-scanner.test.ts +20 -0
- package/src/__tests__/secure-keys-managed-failover.test.ts +8 -0
- package/src/__tests__/secure-keys.test.ts +7 -3
- package/src/__tests__/server-tool-pairing.test.ts +107 -0
- package/src/__tests__/skills.test.ts +5 -4
- package/src/__tests__/subagent-tool-gate-mode.test.ts +226 -0
- package/src/__tests__/terminal-tools.test.ts +8 -0
- package/src/__tests__/tool-result-follow-up.test.ts +99 -0
- package/src/__tests__/tool-result-metadata-plumbing.test.ts +63 -0
- package/src/__tests__/unicode.test.ts +36 -0
- package/src/agent/loop.ts +37 -12
- package/src/agent/tool-result-follow-up.ts +70 -0
- package/src/api/attachment-provenance.test.ts +68 -0
- package/src/api/computer-use-tool.test.ts +55 -0
- package/src/api/computer-use-tool.ts +27 -0
- package/src/api/events/assistant-outbound-attachment.ts +3 -0
- package/src/api/events/desktop-activity-changed.ts +10 -0
- package/src/api/events/question-request.ts +1 -0
- package/src/api/index.ts +11 -0
- package/src/api/responses/conversation-message.ts +3 -0
- package/src/approvals/approval-primitive.ts +5 -2
- package/src/approvals/scoped-approval-grants.ts +6 -2
- package/src/browser/virtual-desktop-target.ts +1 -2
- package/src/cli/commands/__tests__/cli-test-harness.ts +21 -3
- package/src/cli/commands/__tests__/plugins.test.ts +20 -1
- package/src/cli/commands/__tests__/schedules.test.ts +14 -0
- package/src/cli/commands/bash.help.ts +4 -3
- package/src/cli/commands/browser.help.ts +5 -1
- package/src/cli/commands/credentials.help.ts +3 -3
- package/src/cli/commands/oauth/index.help.ts +10 -0
- package/src/cli/commands/oauth/providers.ts +12 -0
- package/src/cli/commands/oauth/request.test.ts +179 -2
- package/src/cli/commands/oauth/request.ts +41 -10
- package/src/cli/commands/plugins.ts +12 -5
- package/src/cli/commands/schedules.ts +2 -0
- package/src/cli/lib/__tests__/inspect-plugin.test.ts +54 -0
- package/src/cli/lib/__tests__/install-from-github.test.ts +41 -0
- package/src/cli/lib/__tests__/local-plugin-upgrade.test.ts +133 -51
- package/src/cli/lib/__tests__/plugin-catalog-cache.test.ts +8 -2
- package/src/cli/lib/bundled-marketplace.json +855 -0
- package/src/cli/lib/bundled-plugin-packages.json +783 -1
- package/src/cli/lib/inspect-plugin.ts +11 -4
- package/src/cli/lib/upgrade-plugin.ts +19 -11
- package/src/cli/output.ts +18 -4
- package/src/config/__tests__/default-provider.test.ts +4 -0
- package/src/config/bundled-skills/schedule/SKILL.md +6 -6
- package/src/config/loader.ts +9 -17
- package/src/config/platform-identity.ts +8 -8
- package/src/config/profile-text-generation.test.ts +51 -0
- package/src/config/profile-text-generation.ts +51 -0
- package/src/config/schemas/__tests__/memory-v3.test.ts +12 -0
- package/src/config/schemas/llm.ts +7 -2
- package/src/config/schemas/mcp.ts +5 -1
- package/src/config/schemas/memory-v3.ts +12 -0
- package/src/credential-execution/ces-runtime.ts +2 -2
- package/src/credential-execution/executable-discovery.ts +36 -134
- package/src/credential-execution/process-manager.test.ts +20 -24
- package/src/credential-execution/process-manager.ts +24 -19
- package/src/daemon/__tests__/conversation-tool-setup.test.ts +43 -0
- package/src/daemon/__tests__/turn-tail-deleted-conversation.test.ts +36 -0
- package/src/daemon/assistant-attachments.ts +20 -12
- package/src/daemon/chat-credential-redaction.ts +16 -1
- package/src/daemon/conversation-agent-loop-handlers.ts +74 -3
- package/src/daemon/conversation-agent-loop.ts +12 -0
- package/src/daemon/conversation-attachments.ts +93 -12
- package/src/daemon/conversation-tool-setup.ts +71 -3
- package/src/daemon/conversation-turn-finalize.ts +26 -13
- package/src/daemon/conversation.ts +72 -17
- package/src/daemon/daemon-control.ts +3 -7
- package/src/daemon/lifecycle.ts +4 -4
- package/src/daemon/mcp-reload-service.ts +1 -1
- package/src/daemon/message-types/shared.ts +1 -0
- package/src/daemon/orphan-reaper.ts +4 -3
- package/src/daemon/persist-media-references.ts +20 -3
- package/src/daemon/tool-setup-types.ts +6 -0
- package/src/daemon/wake-conversation-ops.ts +50 -15
- package/src/desktop/desktop-automation-lease.test.ts +230 -0
- package/src/desktop/desktop-automation-lease.ts +90 -6
- package/src/desktop/desktop-help.ts +15 -0
- package/src/desktop/desktop-stream-bridge.test.ts +65 -10
- package/src/desktop/desktop-stream-bridge.ts +3 -3
- package/src/desktop/desktop-wallpaper-renderer.ts +132 -0
- package/src/desktop/desktop-wallpaper-worker.ts +17 -0
- package/src/desktop/desktop-wallpaper.test.ts +48 -7
- package/src/desktop/desktop-wallpaper.ts +34 -102
- package/src/desktop/virtual-desktop-feature.ts +1 -1
- package/src/mcp/__tests__/manager-tool-caps.test.ts +111 -0
- package/src/mcp/__tests__/startup.test.ts +30 -7
- package/src/mcp/__tests__/tool-caps.test.ts +107 -0
- package/src/mcp/manager.ts +168 -101
- package/src/mcp/startup.ts +33 -13
- package/src/mcp/tool-caps.ts +176 -0
- package/src/messaging/provider-message-metadata.ts +3 -3
- package/src/notifications/__tests__/copy-composer.test.ts +70 -0
- package/src/notifications/copy-composer.ts +11 -3
- package/src/notifications/schedule-result-producer.ts +4 -6
- package/src/oauth/AGENTS.md +2 -0
- package/src/oauth/__tests__/identity-verifier.test.ts +36 -1
- package/src/oauth/identity-verifier.ts +24 -0
- package/src/oauth/oauth-store.ts +11 -1
- package/src/oauth/provider-serializer.ts +1 -0
- package/src/oauth/seed-providers.ts +12 -0
- package/src/permissions/confirmation-guardian-request.test.ts +16 -0
- package/src/permissions/confirmation-guardian-request.ts +2 -3
- package/src/permissions/question-prompter.test.ts +31 -0
- package/src/permissions/question-prompter.ts +2 -0
- package/src/persistence/attachments-store.ts +84 -44
- package/src/persistence/conversation-crud.ts +111 -20
- package/src/persistence/conversation-plugin-facade.ts +30 -0
- package/src/persistence/conversation-tool-surface.ts +135 -0
- package/src/persistence/conversation-types.test.ts +32 -0
- package/src/persistence/conversation-types.ts +26 -4
- package/src/persistence/migrations/378-create-conversation-tool-surfaces.test.ts +78 -0
- package/src/persistence/migrations/378-create-conversation-tool-surfaces.ts +29 -0
- package/src/persistence/migrations/379-oauth-providers-response-ok-field.test.ts +92 -0
- package/src/persistence/migrations/379-oauth-providers-response-ok-field.ts +32 -0
- package/src/persistence/migrations/380-conversation-tool-surfaces-delegate-independent-tasks.test.ts +90 -0
- package/src/persistence/migrations/380-conversation-tool-surfaces-delegate-independent-tasks.ts +34 -0
- package/src/persistence/schema/conversation-tool-surfaces.ts +32 -0
- package/src/persistence/schema/index.ts +1 -0
- package/src/persistence/schema/oauth.ts +1 -0
- package/src/persistence/steps.ts +24 -3
- package/src/plugin-api/conversation-turn.ts +31 -7
- package/src/plugin-api/index.ts +11 -1
- package/src/plugin-api/plugin-channel-turn-trust.test.ts +133 -0
- package/src/plugin-api/plugin-channel-turn-trust.ts +71 -0
- package/src/plugins/defaults/memory/AGENTS.md +39 -5
- package/src/plugins/defaults/memory/__tests__/buffer-file.test.ts +365 -0
- package/src/plugins/defaults/memory/__tests__/buffer-format.test.ts +43 -0
- package/src/plugins/defaults/memory/__tests__/conversation-memory-purge.test.ts +1 -0
- package/src/plugins/defaults/memory/__tests__/db-memory-attach.test.ts +2 -0
- package/src/plugins/defaults/memory/__tests__/fixtures/buffer-appender.ts +17 -0
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-job.test.ts +74 -0
- package/src/plugins/defaults/memory/__tests__/memory-run-evidence.test.ts +161 -0
- package/src/plugins/defaults/memory/__tests__/relocated-memory-test-rows.ts +10 -0
- package/src/plugins/defaults/memory/buffer-file.ts +354 -0
- package/src/plugins/defaults/memory/buffer-format.ts +40 -0
- package/src/plugins/defaults/memory/context-search/agent-runner.ts +1 -2
- package/src/plugins/defaults/memory/context-search/format.ts +2 -1
- package/src/plugins/defaults/memory/context-search/sources/memory-v2.ts +2 -1
- package/src/plugins/defaults/memory/context-search/sources/workspace.ts +2 -1
- package/src/plugins/defaults/memory/conversation-memory-purge.ts +4 -0
- package/src/plugins/defaults/memory/graph/capability-seed.ts +1 -2
- package/src/plugins/defaults/memory/graph/tool-handlers.ts +1 -42
- package/src/plugins/defaults/memory/host-utils.ts +0 -10
- package/src/plugins/defaults/memory/injectors.ts +4 -3
- package/src/plugins/defaults/memory/memory-retrospective-job.ts +72 -183
- package/src/plugins/defaults/memory/memory-run-evidence.ts +213 -0
- package/src/plugins/defaults/memory/src/memory-item-routes.test.ts +1 -1
- package/src/plugins/defaults/memory/substrate/__tests__/consolidation-job.test.ts +407 -99
- package/src/plugins/defaults/memory/substrate/__tests__/consolidation-prompt-flag-gating-guard.test.ts +10 -0
- package/src/plugins/defaults/memory/substrate/__tests__/prompts-consolidation.test.ts +107 -7
- package/src/plugins/defaults/memory/substrate/consolidation-job.ts +307 -86
- package/src/plugins/defaults/memory/substrate/consolidation-tool-surface.ts +34 -0
- package/src/plugins/defaults/memory/substrate/page-index.ts +2 -1
- package/src/plugins/defaults/memory/substrate/prompts/consolidation.ts +89 -49
- package/src/plugins/defaults/memory/substrate/sweep-job.ts +1 -1
- package/src/plugins/defaults/memory/tools.ts +1 -1
- package/src/plugins/defaults/memory/v1/graph/consolidation.ts +2 -2
- package/src/plugins/defaults/memory/v1/graph/extraction.ts +2 -1
- package/src/plugins/defaults/memory/v1/graph/retriever.ts +1 -1
- package/src/plugins/defaults/memory/v2/__tests__/migration.test.ts +5 -0
- package/src/plugins/defaults/memory/v2/__tests__/reranker.test.ts +5 -2
- package/src/plugins/defaults/memory/v2/reranker.ts +2 -1
- package/src/plugins/defaults/memory/v3/__tests__/injection.test.ts +81 -1
- package/src/plugins/defaults/memory/v3/__tests__/orchestrate.test.ts +87 -0
- package/src/plugins/defaults/memory/v3/__tests__/plugin-schema.test.ts +12 -0
- package/src/plugins/defaults/memory/v3/__tests__/pool-log-store.test.ts +183 -4
- package/src/plugins/defaults/memory/v3/__tests__/shadow-plugin.test.ts +103 -2
- package/src/plugins/defaults/memory/v3/card.ts +2 -1
- package/src/plugins/defaults/memory/v3/injector.ts +212 -178
- package/src/plugins/defaults/memory/v3/orchestrate.ts +112 -25
- package/src/plugins/defaults/memory/v3/plugin-schema.ts +54 -4
- package/src/plugins/defaults/memory/v3/pool-log-store.ts +253 -1
- package/src/plugins/defaults/memory/v3/pool-select.test.ts +35 -2
- package/src/plugins/defaults/memory/v3/pool-select.ts +31 -19
- package/src/plugins/defaults/memory/v3/sections.ts +2 -1
- package/src/plugins/defaults/memory/v3/shadow-plugin.ts +41 -14
- package/src/plugins/defaults/tool-error/hooks/post-tool-use.ts +4 -1
- package/src/plugins/defaults/tool-result-truncate/terminal.ts +1 -46
- package/src/prompts/__tests__/parallel-tasks-section.test.ts +25 -0
- package/src/prompts/delegation-gate.ts +57 -0
- package/src/prompts/system-prompt.ts +14 -32
- package/src/providers/inference/adapter-factory.ts +6 -0
- package/src/providers/jev/client.test.ts +260 -0
- package/src/providers/jev/client.ts +518 -0
- package/src/providers/model-catalog.ts +58 -3
- package/src/providers/server-tool-pairing.ts +16 -7
- package/src/runtime/AGENTS.md +2 -2
- package/src/runtime/__tests__/agent-wake.test.ts +94 -1
- package/src/runtime/agent-wake.ts +31 -4
- package/src/runtime/guardian-action-service.ts +2 -17
- package/src/runtime/guardian-reply-router.ts +1 -8
- package/src/runtime/http-server.ts +2 -2
- package/src/runtime/migrations/__tests__/vbundle-import-policy.test.ts +89 -0
- package/src/runtime/migrations/vbundle-import-policy.ts +33 -11
- package/src/runtime/routes/__tests__/inference-profiles-routes.test.ts +24 -2
- package/src/runtime/routes/channel-route-shared.ts +1 -9
- package/src/runtime/routes/conversation-routes.ts +29 -4
- package/src/runtime/routes/desktop-setup-routes.test.ts +2 -2
- package/src/runtime/routes/desktop-setup-routes.ts +8 -5
- package/src/runtime/routes/guardian-approval-interception.ts +24 -0
- package/src/runtime/routes/inbound-message-handler.ts +2 -3
- package/src/runtime/routes/inbound-stages/background-dispatch.test.ts +1 -1
- package/src/runtime/routes/inbound-stages/background-dispatch.ts +7 -4
- package/src/runtime/routes/inference-profile-session-handler.ts +11 -1
- package/src/runtime/routes/inference-profiles-routes.ts +9 -0
- package/src/runtime/routes/oauth-commands-routes.ts +57 -54
- package/src/runtime/routes/oauth-providers.ts +4 -0
- package/src/runtime/routes/oauth-request-hints.test.ts +246 -0
- package/src/runtime/routes/oauth-request-hints.ts +149 -0
- package/src/runtime/routes/schedule-routes.ts +2 -0
- package/src/runtime/routes/secret-routes.ts +99 -119
- package/src/schedule/__tests__/worker-mcp-bootstrap.test.ts +107 -0
- package/src/schedule/__tests__/worker-mcp-readiness.test.ts +173 -0
- package/src/schedule/__tests__/worker-mcp-tools.test.ts +13 -47
- package/src/schedule/run-script.ts +2 -2
- package/src/schedule/scheduler.ts +47 -11
- package/src/schedule/tool-surface-readiness.ts +65 -0
- package/src/schedule/worker-mcp.ts +85 -0
- package/src/schedule/worker.ts +9 -31
- package/src/security/secure-keys.ts +60 -48
- package/src/tools/ask-question/ask-question-tool.test.ts +167 -5
- package/src/tools/ask-question/ask-question-tool.ts +119 -23
- package/src/tools/browser/browser-execution.ts +7 -1
- package/src/tools/host-terminal/host-shell.ts +12 -6
- package/src/tools/shared/filesystem/file-ops-service.ts +1 -31
- package/src/tools/shared/shell-output.test.ts +10 -0
- package/src/tools/shared/shell-output.ts +14 -2
- package/src/tools/skills/sandbox-runner.ts +13 -2
- package/src/tools/skills/scaffold-managed.ts +2 -1
- package/src/tools/terminal/__tests__/safe-env.test.ts +33 -4
- package/src/tools/terminal/__tests__/sanitized-bash.test.ts +0 -14
- package/src/tools/terminal/safe-env.ts +35 -24
- package/src/tools/terminal/sanitized-bash.ts +15 -2
- package/src/tools/terminal/shell-launch.test.ts +162 -0
- package/src/tools/terminal/shell.test.ts +29 -0
- package/src/tools/terminal/shell.ts +13 -7
- package/src/util/browser-human-verification.ts +14 -0
- package/src/util/host-process.test.ts +17 -1
- package/src/util/host-process.ts +24 -0
- package/src/util/unicode.ts +29 -0
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
import { beforeEach, describe, expect, mock, test } from "bun:test";
|
|
2
|
+
|
|
3
|
+
type InboundTrustReadResult =
|
|
4
|
+
| {
|
|
5
|
+
ok: true;
|
|
6
|
+
verdict: {
|
|
7
|
+
trustClass: string;
|
|
8
|
+
canonicalSenderId: string | null;
|
|
9
|
+
contactId?: string;
|
|
10
|
+
status?: string;
|
|
11
|
+
resolutionFailed?: boolean;
|
|
12
|
+
};
|
|
13
|
+
admissionPolicy: string | null;
|
|
14
|
+
}
|
|
15
|
+
| { ok: false };
|
|
16
|
+
|
|
17
|
+
let nextRead: InboundTrustReadResult = { ok: false };
|
|
18
|
+
|
|
19
|
+
mock.module("../calls/inbound-trust-reader.js", () => ({
|
|
20
|
+
readInboundTrust: async () => nextRead,
|
|
21
|
+
}));
|
|
22
|
+
|
|
23
|
+
import {
|
|
24
|
+
PluginTurnNotAdmittedError,
|
|
25
|
+
resolvePluginChannelTurnTrust,
|
|
26
|
+
} from "./plugin-channel-turn-trust.js";
|
|
27
|
+
|
|
28
|
+
const CHANNEL = {
|
|
29
|
+
sourceChannel: "plugin" as const,
|
|
30
|
+
externalChatId: "imessage:+12025550142",
|
|
31
|
+
externalUserId: "imessage:+12025550142",
|
|
32
|
+
displayName: "Ada",
|
|
33
|
+
};
|
|
34
|
+
|
|
35
|
+
beforeEach(() => {
|
|
36
|
+
nextRead = { ok: false };
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
describe("resolvePluginChannelTurnTrust", () => {
|
|
40
|
+
test("returns the gateway verdict as the turn trust context", async () => {
|
|
41
|
+
nextRead = {
|
|
42
|
+
ok: true,
|
|
43
|
+
verdict: {
|
|
44
|
+
trustClass: "trusted_contact",
|
|
45
|
+
canonicalSenderId: "imessage:+12025550142",
|
|
46
|
+
contactId: "c-ada",
|
|
47
|
+
status: "active",
|
|
48
|
+
},
|
|
49
|
+
admissionPolicy: "trusted_contacts",
|
|
50
|
+
};
|
|
51
|
+
|
|
52
|
+
const trust = await resolvePluginChannelTurnTrust(CHANNEL);
|
|
53
|
+
expect(trust.trustClass).toBe("trusted_contact");
|
|
54
|
+
expect(trust.sourceChannel).toBe("plugin");
|
|
55
|
+
expect(trust.requesterChatId).toBe(CHANNEL.externalChatId);
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
test("fails closed when the gateway could not vouch for the sender", async () => {
|
|
59
|
+
nextRead = {
|
|
60
|
+
ok: true,
|
|
61
|
+
verdict: {
|
|
62
|
+
trustClass: "unknown",
|
|
63
|
+
canonicalSenderId: null,
|
|
64
|
+
resolutionFailed: true,
|
|
65
|
+
},
|
|
66
|
+
admissionPolicy: "strangers",
|
|
67
|
+
};
|
|
68
|
+
|
|
69
|
+
await expect(resolvePluginChannelTurnTrust(CHANNEL)).rejects.toMatchObject({
|
|
70
|
+
name: "PluginTurnNotAdmittedError",
|
|
71
|
+
reason: "trust_resolution_failed",
|
|
72
|
+
});
|
|
73
|
+
});
|
|
74
|
+
|
|
75
|
+
test("fails closed when the gateway trust read fails", async () => {
|
|
76
|
+
nextRead = { ok: false };
|
|
77
|
+
await expect(resolvePluginChannelTurnTrust(CHANNEL)).rejects.toMatchObject({
|
|
78
|
+
name: "PluginTurnNotAdmittedError",
|
|
79
|
+
reason: "trust_resolution_failed",
|
|
80
|
+
});
|
|
81
|
+
});
|
|
82
|
+
|
|
83
|
+
test("rejects a sender below the channel admission floor", async () => {
|
|
84
|
+
nextRead = {
|
|
85
|
+
ok: true,
|
|
86
|
+
verdict: { trustClass: "unknown", canonicalSenderId: null },
|
|
87
|
+
admissionPolicy: "guardian_only",
|
|
88
|
+
};
|
|
89
|
+
|
|
90
|
+
await expect(resolvePluginChannelTurnTrust(CHANNEL)).rejects.toEqual(
|
|
91
|
+
expect.objectContaining({
|
|
92
|
+
name: "PluginTurnNotAdmittedError",
|
|
93
|
+
reason: "admission_policy_guardian_only",
|
|
94
|
+
}),
|
|
95
|
+
);
|
|
96
|
+
});
|
|
97
|
+
|
|
98
|
+
test("rejects a blocked member regardless of floor", async () => {
|
|
99
|
+
nextRead = {
|
|
100
|
+
ok: true,
|
|
101
|
+
verdict: {
|
|
102
|
+
trustClass: "trusted_contact",
|
|
103
|
+
canonicalSenderId: "imessage:+12025550142",
|
|
104
|
+
contactId: "c-ada",
|
|
105
|
+
status: "blocked",
|
|
106
|
+
},
|
|
107
|
+
admissionPolicy: "strangers",
|
|
108
|
+
};
|
|
109
|
+
|
|
110
|
+
await expect(resolvePluginChannelTurnTrust(CHANNEL)).rejects.toMatchObject({
|
|
111
|
+
name: "PluginTurnNotAdmittedError",
|
|
112
|
+
reason: "member_blocked",
|
|
113
|
+
});
|
|
114
|
+
});
|
|
115
|
+
|
|
116
|
+
test("admits when the gateway reports no admission policy", async () => {
|
|
117
|
+
nextRead = {
|
|
118
|
+
ok: true,
|
|
119
|
+
verdict: { trustClass: "unknown", canonicalSenderId: null },
|
|
120
|
+
admissionPolicy: null,
|
|
121
|
+
};
|
|
122
|
+
|
|
123
|
+
const trust = await resolvePluginChannelTurnTrust(CHANNEL);
|
|
124
|
+
expect(trust.trustClass).toBe("unknown");
|
|
125
|
+
});
|
|
126
|
+
|
|
127
|
+
test("PluginTurnNotAdmittedError names the deny reason", () => {
|
|
128
|
+
const err = new PluginTurnNotAdmittedError("admission_policy_no_one");
|
|
129
|
+
expect(err).toBeInstanceOf(Error);
|
|
130
|
+
expect(err.reason).toBe("admission_policy_no_one");
|
|
131
|
+
expect(err.message).toContain("admission_policy_no_one");
|
|
132
|
+
});
|
|
133
|
+
});
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Gateway admission and trust for plugin turns addressed by a channel chat.
|
|
3
|
+
*
|
|
4
|
+
* A channel address means an inbound sender, not an internal plugin job.
|
|
5
|
+
* Trust comes from the gateway verdict (`resolve_inbound_trust`); the turn
|
|
6
|
+
* runs only when that sender clears the channel admission floor.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import { meetsAdmissionFloor } from "@vellumai/gateway-client";
|
|
10
|
+
|
|
11
|
+
import { readInboundTrust } from "../calls/inbound-trust-reader.js";
|
|
12
|
+
import type { ChannelId } from "../channels/types.js";
|
|
13
|
+
import type { TrustContext } from "../daemon/trust-context-types.js";
|
|
14
|
+
import { trustContextFromVerdict } from "../runtime/trust-verdict-consumer.js";
|
|
15
|
+
|
|
16
|
+
interface PluginChannelTurnAddress {
|
|
17
|
+
sourceChannel: ChannelId;
|
|
18
|
+
externalChatId: string;
|
|
19
|
+
externalUserId?: string | null;
|
|
20
|
+
displayName?: string | null;
|
|
21
|
+
username?: string | null;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export class PluginTurnNotAdmittedError extends Error {
|
|
25
|
+
readonly reason: string;
|
|
26
|
+
|
|
27
|
+
constructor(reason: string) {
|
|
28
|
+
super(`This inbound plugin turn was not admitted (${reason}).`);
|
|
29
|
+
this.name = "PluginTurnNotAdmittedError";
|
|
30
|
+
this.reason = reason;
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export async function resolvePluginChannelTurnTrust(
|
|
35
|
+
channel: PluginChannelTurnAddress,
|
|
36
|
+
): Promise<TrustContext> {
|
|
37
|
+
const actorExternalId = channel.externalUserId?.trim() || undefined;
|
|
38
|
+
const read = await readInboundTrust({
|
|
39
|
+
channelType: channel.sourceChannel,
|
|
40
|
+
actorExternalId,
|
|
41
|
+
});
|
|
42
|
+
if (!read.ok) {
|
|
43
|
+
throw new PluginTurnNotAdmittedError("trust_resolution_failed");
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
if (read.verdict.resolutionFailed) {
|
|
47
|
+
throw new PluginTurnNotAdmittedError("trust_resolution_failed");
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
const memberStatus = read.verdict.status;
|
|
51
|
+
if (memberStatus === "blocked" || memberStatus === "revoked") {
|
|
52
|
+
throw new PluginTurnNotAdmittedError(`member_${memberStatus}`);
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
const trustClass = read.verdict.trustClass;
|
|
56
|
+
if (
|
|
57
|
+
read.admissionPolicy != null &&
|
|
58
|
+
!meetsAdmissionFloor(read.admissionPolicy, trustClass)
|
|
59
|
+
) {
|
|
60
|
+
throw new PluginTurnNotAdmittedError(
|
|
61
|
+
`admission_policy_${read.admissionPolicy}`,
|
|
62
|
+
);
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
return trustContextFromVerdict(read.verdict, {
|
|
66
|
+
sourceChannel: channel.sourceChannel,
|
|
67
|
+
conversationExternalId: channel.externalChatId,
|
|
68
|
+
actorDisplayName: channel.displayName ?? undefined,
|
|
69
|
+
actorUsername: channel.username ?? undefined,
|
|
70
|
+
});
|
|
71
|
+
}
|
|
@@ -68,6 +68,13 @@ Everything else under the plugin root is **spine**:
|
|
|
68
68
|
the writer plus the one matcher every reader uses. It lives at the root
|
|
69
69
|
precisely so `substrate/`, `graph/`, and `graph-topology/` can all reach it
|
|
70
70
|
without a tier importing spine. Do not add a second matcher anywhere),
|
|
71
|
+
`buffer-file` (the two writers of `memory/buffer.md`: the append every
|
|
72
|
+
`remember()`-shaped path uses and the consume the consolidation job runs
|
|
73
|
+
after a pass. Do not write the buffer from anywhere else, and never
|
|
74
|
+
rewrite it from a read that is not inside `consumeBufferEntries`'s
|
|
75
|
+
synchronous critical section), `memory-run-evidence` (readers of what a
|
|
76
|
+
background memory run durably produced from its persisted messages; every
|
|
77
|
+
job that gates a state transition on a verified write uses them),
|
|
71
78
|
`segmenter`, `message-media`, `worker`, `worker-control`,
|
|
72
79
|
`memory-recall-log-store`, `activation-session-store` (the onboarding
|
|
73
80
|
activation rail — **not** a memory tier despite the name),
|
|
@@ -213,7 +220,8 @@ not break.
|
|
|
213
220
|
behind a re-injected copy), and a reset whose ledger clear fails reports
|
|
214
221
|
that the same way a missing marker does.
|
|
215
222
|
- **The store is plugin-owned.** `memory_v3_injected_sections`,
|
|
216
|
-
`memory_v3_pools`,
|
|
223
|
+
`memory_v3_pools`, `memory_v3_pool_inputs`, `memory_v3_pool_texts`, and
|
|
224
|
+
the `section_key` column of `memory_v3_selections`
|
|
217
225
|
are created by `v3/plugin-schema.ts` (`ensureMemoryV3PluginSchema` from the
|
|
218
226
|
`init` hook, and each store's once-per-connection ensure on first use),
|
|
219
227
|
never by the global migration chain, and every read goes through its
|
|
@@ -365,13 +373,16 @@ Persisted rows; a rename orphans every existing install.
|
|
|
365
373
|
| `memory_v2_injection_events` | v2 scoring feedback |
|
|
366
374
|
| `memory_v3_selections` | v3 selection log (`section_key` column plugin-added, see below) |
|
|
367
375
|
| `memory_v3_pools` | v3 selector pool audit (plugin-created, see below) |
|
|
376
|
+
| `memory_v3_pool_inputs` | v3 selector input capture, opt-in (plugin-created, see below) |
|
|
377
|
+
| `memory_v3_pool_texts` | v3 pooled candidate texts by content hash, opt-in (see below) |
|
|
368
378
|
| `memory_v3_injected_sections` | v3 section dedup, prune bytes + recency (plugin-created, below) |
|
|
369
379
|
| `memory_v3_ever_injected` | v3 card dedup (superseded, frozen) |
|
|
370
380
|
| `memory_retrospective_state` | retrospective (tier-agnostic) |
|
|
371
381
|
| `activation_sessions` | onboarding activation rail |
|
|
372
382
|
|
|
373
|
-
`memory_v3_pools`
|
|
374
|
-
|
|
383
|
+
`memory_v3_pools`, `memory_v3_pool_inputs`, `memory_v3_pool_texts`, and
|
|
384
|
+
`memory_v3_injected_sections` are the plugin's own tables, created by the
|
|
385
|
+
plugin rather than by the global migration chain
|
|
375
386
|
(`v3/plugin-schema.ts`): the memory plugin's `init` hook ensures both on every
|
|
376
387
|
boot, and each store ensures again on the first use of a connection in its
|
|
377
388
|
process (the memory worker is a separate process), idempotently and fail-open,
|
|
@@ -407,8 +418,13 @@ per distinct matched section, at most `memory.v3.finderSectionsPerPage` in
|
|
|
407
418
|
surfacing order (needle, dense, reply, span) plus its entity and rare-term
|
|
408
419
|
lines, and selecting a line selects that section; the selection log keeps
|
|
409
420
|
one row per slug, so the pool row is where the per-section verdicts live. A
|
|
410
|
-
turn whose selector never judged a pool
|
|
411
|
-
|
|
421
|
+
turn whose selector never judged a pool persists it with `selector_ran = 0`:
|
|
422
|
+
an empty pool when the injection gate hard-skipped it or nothing was pooled,
|
|
423
|
+
and the pool as the selector was given it, stable-prefix cards chosen and
|
|
424
|
+
finder lines not, when the selector's provider failed and the orchestrator
|
|
425
|
+
kept that prefix unjudged. That turn writes no `memory_v3_selections` rows:
|
|
426
|
+
the hot set's frecency and the learned-edge graph read that table as
|
|
427
|
+
judgments, and an unjudged page is not one. A
|
|
412
428
|
turn that logged no selections is still reachable by its stamped
|
|
413
429
|
`message_id`, so the inspector shows negative verdicts too. The pool row and
|
|
414
430
|
the turn's `memory_v3_selections` rows are
|
|
@@ -419,6 +435,24 @@ diagnostics, roughly 10KB each, with no retention job; a conversation delete
|
|
|
419
435
|
purges them with the other conversation-keyed tables
|
|
420
436
|
(`conversation-memory-purge.ts`).
|
|
421
437
|
|
|
438
|
+
`memory_v3_pool_inputs` and `memory_v3_pool_texts` (`v3/pool-log-store.ts`,
|
|
439
|
+
written only under `memory.v3.poolLog.captureInput`, off by default) extend
|
|
440
|
+
the pool row with the selector's exact input, for offline selector
|
|
441
|
+
evaluation and training data. One `memory_v3_pool_inputs` row per turn
|
|
442
|
+
holds the context strings the selector was given (situation, recent context,
|
|
443
|
+
current message, the previous assistant reply), the selector prompt's
|
|
444
|
+
content hash, the gate reason, the keep-all flag, and the content hash of
|
|
445
|
+
every pooled candidate's rendered text in pool order, aligned index for
|
|
446
|
+
index with the pool row's `candidates_json` (pool position `i + 1` is the
|
|
447
|
+
number the selector saw candidate `i` under). `memory_v3_pool_texts` holds
|
|
448
|
+
each rendered text once by that hash: a stable-prefix card is its
|
|
449
|
+
pre-rendered card, a finder line is exactly the line the selector saw minus
|
|
450
|
+
its number (`renderFinderLine` in `v3/pool-select.ts`), and a card repeated
|
|
451
|
+
across turns is stored once. The input row is written in the transaction
|
|
452
|
+
that writes the pool and selection rows and purged with the conversation;
|
|
453
|
+
the texts table is page-derived and keyed by content, so it is never purged
|
|
454
|
+
per conversation.
|
|
455
|
+
|
|
422
456
|
**v3 finder lanes.** The per-turn finder lanes (`v3/orchestrate.ts`) surface
|
|
423
457
|
candidates in a fixed order, needle, dense, reply, span, entity, rare, then
|
|
424
458
|
edge and learned, and each only ever adds lines. The entity lane
|
|
@@ -0,0 +1,365 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Tests for `buffer-file.ts`: the two writers of `memory/buffer.md`.
|
|
3
|
+
*
|
|
4
|
+
* The consume cases pin the invariant the module exists for: an entry saved
|
|
5
|
+
* at any point while a consolidation run is in flight is either the run's
|
|
6
|
+
* own material or still in the buffer afterwards, never destroyed. The
|
|
7
|
+
* cross-process case is the reason to believe it: a child process appends
|
|
8
|
+
* as fast as it can while this process consumes in a loop, and every tagged
|
|
9
|
+
* entry must end up consumed or present. The harness's accounting is proven
|
|
10
|
+
* against a naive read-modify-write consume with the losing interleaving
|
|
11
|
+
* forced, not raced, so the proof does not depend on scheduling.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import {
|
|
15
|
+
appendFileSync,
|
|
16
|
+
chmodSync,
|
|
17
|
+
closeSync,
|
|
18
|
+
existsSync,
|
|
19
|
+
mkdtempSync,
|
|
20
|
+
openSync,
|
|
21
|
+
readdirSync,
|
|
22
|
+
readFileSync,
|
|
23
|
+
rmSync,
|
|
24
|
+
writeFileSync,
|
|
25
|
+
} from "node:fs";
|
|
26
|
+
import { tmpdir } from "node:os";
|
|
27
|
+
import { join } from "node:path";
|
|
28
|
+
import {
|
|
29
|
+
afterAll,
|
|
30
|
+
beforeAll,
|
|
31
|
+
beforeEach,
|
|
32
|
+
describe,
|
|
33
|
+
expect,
|
|
34
|
+
test,
|
|
35
|
+
} from "bun:test";
|
|
36
|
+
|
|
37
|
+
import {
|
|
38
|
+
appendBufferAndArchive,
|
|
39
|
+
consumeBufferEntries,
|
|
40
|
+
} from "../buffer-file.js";
|
|
41
|
+
import {
|
|
42
|
+
type BufferEntryLines,
|
|
43
|
+
bufferEntryText,
|
|
44
|
+
formatRememberEntry,
|
|
45
|
+
joinBufferEntries,
|
|
46
|
+
splitBufferContent,
|
|
47
|
+
} from "../buffer-format.js";
|
|
48
|
+
|
|
49
|
+
let tmp: string;
|
|
50
|
+
beforeAll(() => {
|
|
51
|
+
tmp = mkdtempSync(join(tmpdir(), "memory-buffer-file-test-"));
|
|
52
|
+
});
|
|
53
|
+
afterAll(() => {
|
|
54
|
+
rmSync(tmp, { recursive: true, force: true });
|
|
55
|
+
});
|
|
56
|
+
|
|
57
|
+
let memoryDir: string;
|
|
58
|
+
let bufferPath: string;
|
|
59
|
+
beforeEach(() => {
|
|
60
|
+
memoryDir = mkdtempSync(join(tmp, "memory-"));
|
|
61
|
+
bufferPath = join(memoryDir, "buffer.md");
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
const A = "- [Apr 27, 9:00 AM] Alice prefers VS Code.";
|
|
65
|
+
const B = "- [Apr 27, 9:01 AM] Bob shared a snippet:\n line two\n line three";
|
|
66
|
+
const C = "- [Apr 27, 9:02 AM] Carol loves jazz.";
|
|
67
|
+
const D = "- [Apr 27, 9:03 AM] Dave runs marathons.";
|
|
68
|
+
|
|
69
|
+
function file(...entries: string[]): string {
|
|
70
|
+
return entries.map((e) => `${e}\n`).join("");
|
|
71
|
+
}
|
|
72
|
+
function entries(content: string): BufferEntryLines[] {
|
|
73
|
+
return splitBufferContent(content);
|
|
74
|
+
}
|
|
75
|
+
function texts(): string[] {
|
|
76
|
+
return entries(readFileSync(bufferPath, "utf-8")).map(bufferEntryText);
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
describe("appendBufferAndArchive", () => {
|
|
80
|
+
test("appends to the buffer and the dated archive, seeding the archive header once", () => {
|
|
81
|
+
const now = new Date(2026, 3, 27, 9, 0);
|
|
82
|
+
const entry = formatRememberEntry("Alice prefers VS Code.", now);
|
|
83
|
+
const first = appendBufferAndArchive({ rootDir: memoryDir, entry, now });
|
|
84
|
+
appendBufferAndArchive({ rootDir: memoryDir, entry, now });
|
|
85
|
+
|
|
86
|
+
expect(first.bufferPath).toBe(bufferPath);
|
|
87
|
+
expect(readFileSync(bufferPath, "utf-8")).toBe(entry + entry);
|
|
88
|
+
expect(first.archivePath).toBe(join(memoryDir, "archive", "2026-04-27.md"));
|
|
89
|
+
expect(readFileSync(first.archivePath, "utf-8")).toBe(
|
|
90
|
+
`# Apr 27, 2026\n\n${entry}${entry}`,
|
|
91
|
+
);
|
|
92
|
+
});
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
describe("consumeBufferEntries", () => {
|
|
96
|
+
test("removes exactly the consumed entries and keeps the rest verbatim and in order", async () => {
|
|
97
|
+
writeFileSync(bufferPath, file(A, B, C, D));
|
|
98
|
+
const pass = entries(file(A, B));
|
|
99
|
+
|
|
100
|
+
const result = await consumeBufferEntries(bufferPath, pass, {
|
|
101
|
+
lateAppendGraceMs: 0,
|
|
102
|
+
});
|
|
103
|
+
|
|
104
|
+
expect(result).toEqual({
|
|
105
|
+
removed: 2,
|
|
106
|
+
alreadyAbsent: 0,
|
|
107
|
+
lateAppendBytesRecovered: 0,
|
|
108
|
+
unrecoveredLateAppendBytes: 0,
|
|
109
|
+
lateAppendDrainFailed: false,
|
|
110
|
+
});
|
|
111
|
+
expect(readFileSync(bufferPath, "utf-8")).toBe(file(C, D));
|
|
112
|
+
});
|
|
113
|
+
|
|
114
|
+
test("entries appended after the snapshot survive, including one that duplicates a consumed entry", async () => {
|
|
115
|
+
// The snapshot held one copy of A; the same fact was remembered again
|
|
116
|
+
// during the run. Multiset semantics: one copy goes, one stays.
|
|
117
|
+
writeFileSync(bufferPath, file(A, B));
|
|
118
|
+
const pass = entries(file(A, B));
|
|
119
|
+
appendFileSync(bufferPath, file(C, A));
|
|
120
|
+
|
|
121
|
+
const result = await consumeBufferEntries(bufferPath, pass, {
|
|
122
|
+
lateAppendGraceMs: 0,
|
|
123
|
+
});
|
|
124
|
+
|
|
125
|
+
expect(result.removed).toBe(2);
|
|
126
|
+
expect(texts()).toEqual([C, A]);
|
|
127
|
+
});
|
|
128
|
+
|
|
129
|
+
test("a consumed entry missing from the live file is counted, not searched for", async () => {
|
|
130
|
+
writeFileSync(bufferPath, file(B, C));
|
|
131
|
+
const pass = entries(file(A, B));
|
|
132
|
+
|
|
133
|
+
const result = await consumeBufferEntries(bufferPath, pass, {
|
|
134
|
+
lateAppendGraceMs: 0,
|
|
135
|
+
});
|
|
136
|
+
|
|
137
|
+
expect(result).toMatchObject({ removed: 1, alreadyAbsent: 1 });
|
|
138
|
+
expect(texts()).toEqual([C]);
|
|
139
|
+
});
|
|
140
|
+
|
|
141
|
+
test("a missing buffer file consumes nothing and creates nothing", async () => {
|
|
142
|
+
const result = await consumeBufferEntries(bufferPath, entries(file(A)), {
|
|
143
|
+
lateAppendGraceMs: 0,
|
|
144
|
+
});
|
|
145
|
+
|
|
146
|
+
expect(result).toEqual({
|
|
147
|
+
removed: 0,
|
|
148
|
+
alreadyAbsent: 1,
|
|
149
|
+
lateAppendBytesRecovered: 0,
|
|
150
|
+
unrecoveredLateAppendBytes: 0,
|
|
151
|
+
lateAppendDrainFailed: false,
|
|
152
|
+
});
|
|
153
|
+
expect(existsSync(bufferPath)).toBe(false);
|
|
154
|
+
});
|
|
155
|
+
|
|
156
|
+
test("consuming everything leaves an empty file and no temp file behind", async () => {
|
|
157
|
+
writeFileSync(bufferPath, file(A, B));
|
|
158
|
+
|
|
159
|
+
await consumeBufferEntries(bufferPath, entries(file(A, B)), {
|
|
160
|
+
lateAppendGraceMs: 0,
|
|
161
|
+
});
|
|
162
|
+
|
|
163
|
+
expect(readFileSync(bufferPath, "utf-8")).toBe("");
|
|
164
|
+
expect(readdirSync(memoryDir)).toEqual(["buffer.md"]);
|
|
165
|
+
});
|
|
166
|
+
|
|
167
|
+
test("an append that landed on the replaced inode is drained back into the buffer", async () => {
|
|
168
|
+
// An appender that opened the file before the rename writes to the old
|
|
169
|
+
// inode. The consumer holds that inode open and copies the late bytes
|
|
170
|
+
// over. The test plays the appender: open before, write after the
|
|
171
|
+
// synchronous phase, before the grace-window drain.
|
|
172
|
+
writeFileSync(bufferPath, file(A, B));
|
|
173
|
+
const lateFd = openSync(bufferPath, "a");
|
|
174
|
+
const consuming = consumeBufferEntries(bufferPath, entries(file(A)), {
|
|
175
|
+
lateAppendGraceMs: 50,
|
|
176
|
+
});
|
|
177
|
+
// The synchronous phase (read, rewrite, rename) ran before the first
|
|
178
|
+
// await inside consume, so this write lands on the orphaned inode.
|
|
179
|
+
appendFileSync(lateFd, file(C));
|
|
180
|
+
closeSync(lateFd);
|
|
181
|
+
|
|
182
|
+
const result = await consuming;
|
|
183
|
+
|
|
184
|
+
expect(result.removed).toBe(1);
|
|
185
|
+
expect(result.lateAppendBytesRecovered).toBe(file(C).length);
|
|
186
|
+
expect(result.unrecoveredLateAppendBytes).toBe(0);
|
|
187
|
+
expect(texts()).toEqual([B, C]);
|
|
188
|
+
});
|
|
189
|
+
|
|
190
|
+
test("a late append the drain cannot copy back is reported, not thrown, and the pass stays consumed", async () => {
|
|
191
|
+
// The rename has committed when the drain runs, so a failing copy must
|
|
192
|
+
// not surface as "nothing happened". The test makes the rewritten
|
|
193
|
+
// buffer read-only before the late bytes land, so every copy attempt
|
|
194
|
+
// fails.
|
|
195
|
+
writeFileSync(bufferPath, file(A, B));
|
|
196
|
+
const lateFd = openSync(bufferPath, "a");
|
|
197
|
+
const consuming = consumeBufferEntries(bufferPath, entries(file(A)), {
|
|
198
|
+
lateAppendGraceMs: 50,
|
|
199
|
+
});
|
|
200
|
+
chmodSync(bufferPath, 0o444);
|
|
201
|
+
appendFileSync(lateFd, file(C));
|
|
202
|
+
closeSync(lateFd);
|
|
203
|
+
|
|
204
|
+
try {
|
|
205
|
+
const result = await consuming;
|
|
206
|
+
expect(result.removed).toBe(1);
|
|
207
|
+
expect(result.lateAppendBytesRecovered).toBe(0);
|
|
208
|
+
expect(result.unrecoveredLateAppendBytes).toBe(file(C).length);
|
|
209
|
+
expect(result.lateAppendDrainFailed).toBe(false);
|
|
210
|
+
expect(texts()).toEqual([B]);
|
|
211
|
+
} finally {
|
|
212
|
+
chmodSync(bufferPath, 0o644);
|
|
213
|
+
}
|
|
214
|
+
});
|
|
215
|
+
});
|
|
216
|
+
|
|
217
|
+
/**
|
|
218
|
+
* The cross-process invariant. A child appends `count` tagged entries as
|
|
219
|
+
* fast as it can through `appendFileSync`; this process consumes the leading
|
|
220
|
+
* entries (up to `passSize` at a time) in a loop through `consume`. Returns
|
|
221
|
+
* the tags that were neither consumed by a pass nor left in the buffer: the
|
|
222
|
+
* entries the consume destroyed.
|
|
223
|
+
*/
|
|
224
|
+
async function lostUnder(
|
|
225
|
+
consume: (path: string, pass: BufferEntryLines[]) => Promise<unknown>,
|
|
226
|
+
count: number,
|
|
227
|
+
passSize: number,
|
|
228
|
+
): Promise<{ lost: string[]; duplicated: string[]; passes: number }> {
|
|
229
|
+
const tag = `t${Math.random().toString(36).slice(2, 8)}`;
|
|
230
|
+
writeFileSync(bufferPath, "");
|
|
231
|
+
const child = Bun.spawn(
|
|
232
|
+
[
|
|
233
|
+
process.execPath,
|
|
234
|
+
join(import.meta.dir, "fixtures", "buffer-appender.ts"),
|
|
235
|
+
bufferPath,
|
|
236
|
+
String(count),
|
|
237
|
+
tag,
|
|
238
|
+
],
|
|
239
|
+
{ stdout: "ignore", stderr: "inherit" },
|
|
240
|
+
);
|
|
241
|
+
const consumed: string[] = [];
|
|
242
|
+
let passes = 0;
|
|
243
|
+
let childDone = false;
|
|
244
|
+
void child.exited.then(() => {
|
|
245
|
+
childDone = true;
|
|
246
|
+
});
|
|
247
|
+
while (!childDone) {
|
|
248
|
+
const snapshot = entries(readFileSync(bufferPath, "utf-8"));
|
|
249
|
+
const pass = snapshot.slice(0, passSize);
|
|
250
|
+
if (pass.length === 0) {
|
|
251
|
+
await Bun.sleep(1);
|
|
252
|
+
continue;
|
|
253
|
+
}
|
|
254
|
+
await consume(bufferPath, pass);
|
|
255
|
+
consumed.push(...pass.map(bufferEntryText));
|
|
256
|
+
passes += 1;
|
|
257
|
+
}
|
|
258
|
+
await child.exited;
|
|
259
|
+
return { ...accountFor(count, tag, consumed), passes };
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
/** A tagged entry as the appender writes it. */
|
|
263
|
+
function taggedEntry(tag: string, i: number): string {
|
|
264
|
+
return `- [Apr 27, 9:00 AM] ${tag}-${i}`;
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
/**
|
|
268
|
+
* The harness's verdict: of `count` tagged entries, which were neither
|
|
269
|
+
* consumed by a pass nor left in the buffer (lost), and which were both
|
|
270
|
+
* (duplicated).
|
|
271
|
+
*/
|
|
272
|
+
function accountFor(
|
|
273
|
+
count: number,
|
|
274
|
+
tag: string,
|
|
275
|
+
consumed: readonly string[],
|
|
276
|
+
): { lost: string[]; duplicated: string[] } {
|
|
277
|
+
const seen = new Map<string, number>();
|
|
278
|
+
for (const text of [...consumed, ...texts()]) {
|
|
279
|
+
seen.set(text, (seen.get(text) ?? 0) + 1);
|
|
280
|
+
}
|
|
281
|
+
const lost: string[] = [];
|
|
282
|
+
const duplicated: string[] = [];
|
|
283
|
+
for (let i = 0; i < count; i++) {
|
|
284
|
+
const text = taggedEntry(tag, i);
|
|
285
|
+
const n = seen.get(text) ?? 0;
|
|
286
|
+
if (n === 0) {
|
|
287
|
+
lost.push(text);
|
|
288
|
+
} else if (n > 1) {
|
|
289
|
+
duplicated.push(text);
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
return { lost, duplicated };
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
/**
|
|
296
|
+
* A read-modify-write consume, the shape the production consume is measured
|
|
297
|
+
* against: an append that lands between its read and its write is destroyed.
|
|
298
|
+
* `betweenReadAndWrite` runs in exactly that window.
|
|
299
|
+
*/
|
|
300
|
+
async function naiveConsume(
|
|
301
|
+
path: string,
|
|
302
|
+
pass: BufferEntryLines[],
|
|
303
|
+
betweenReadAndWrite?: () => void,
|
|
304
|
+
): Promise<void> {
|
|
305
|
+
const pending = pass.map(bufferEntryText);
|
|
306
|
+
const remaining = entries(readFileSync(path, "utf-8")).filter((entry) => {
|
|
307
|
+
const index = pending.indexOf(bufferEntryText(entry));
|
|
308
|
+
if (index === -1) {
|
|
309
|
+
return true;
|
|
310
|
+
}
|
|
311
|
+
pending.splice(index, 1);
|
|
312
|
+
return false;
|
|
313
|
+
});
|
|
314
|
+
betweenReadAndWrite?.();
|
|
315
|
+
writeFileSync(path, joinBufferEntries(remaining), "utf-8");
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
describe("consumeBufferEntries under a concurrent appender in another process", () => {
|
|
319
|
+
const APPENDS = 4000;
|
|
320
|
+
|
|
321
|
+
test("no entry appended during consumption is lost or duplicated", async () => {
|
|
322
|
+
// Runs the consume exactly as production does, grace window included:
|
|
323
|
+
// the window is what makes the invariant hold, and a loaded machine
|
|
324
|
+
// can deschedule the appender between its open and its write for
|
|
325
|
+
// longer than a token grace would cover. Larger passes keep the run
|
|
326
|
+
// short with the full grace paid per pass.
|
|
327
|
+
const outcome = await lostUnder(
|
|
328
|
+
(path, pass) => consumeBufferEntries(path, pass),
|
|
329
|
+
APPENDS,
|
|
330
|
+
500,
|
|
331
|
+
);
|
|
332
|
+
expect(outcome.passes).toBeGreaterThan(0);
|
|
333
|
+
expect(outcome.lost).toEqual([]);
|
|
334
|
+
expect(outcome.duplicated).toEqual([]);
|
|
335
|
+
}, 60_000);
|
|
336
|
+
|
|
337
|
+
test("the accounting reports an entry the naive consume destroys, and one a pass duplicates", async () => {
|
|
338
|
+
// Sensitivity check for the harness: the case above is only evidence if
|
|
339
|
+
// the accounting can see a loss. The losing interleaving is forced (an
|
|
340
|
+
// append lands between the naive consume's read and its write) rather
|
|
341
|
+
// than raced against another process, so whether it happens never
|
|
342
|
+
// depends on scheduling.
|
|
343
|
+
const tag = "sens";
|
|
344
|
+
writeFileSync(bufferPath, file(taggedEntry(tag, 0)));
|
|
345
|
+
const pass = entries(readFileSync(bufferPath, "utf-8"));
|
|
346
|
+
|
|
347
|
+
await naiveConsume(bufferPath, pass, () => {
|
|
348
|
+
appendFileSync(bufferPath, `${taggedEntry(tag, 1)}\n`, "utf-8");
|
|
349
|
+
});
|
|
350
|
+
|
|
351
|
+
expect(texts()).toEqual([]);
|
|
352
|
+
const consumed = pass.map(bufferEntryText);
|
|
353
|
+
expect(accountFor(2, tag, consumed)).toEqual({
|
|
354
|
+
lost: [taggedEntry(tag, 1)],
|
|
355
|
+
duplicated: [],
|
|
356
|
+
});
|
|
357
|
+
|
|
358
|
+
// A pass that consumed an entry the buffer still holds is a duplicate.
|
|
359
|
+
appendFileSync(bufferPath, `${taggedEntry(tag, 0)}\n`, "utf-8");
|
|
360
|
+
expect(accountFor(2, tag, consumed)).toEqual({
|
|
361
|
+
lost: [taggedEntry(tag, 1)],
|
|
362
|
+
duplicated: [taggedEntry(tag, 0)],
|
|
363
|
+
});
|
|
364
|
+
});
|
|
365
|
+
});
|