@vellumai/assistant 0.10.7-dev.202607101252.74ea83d → 0.10.7-dev.202607101459.27388eb
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs/workspace-tools.md +4 -1
- package/package.json +1 -1
- package/src/__tests__/conversation-agent-loop.test.ts +25 -0
- package/src/__tests__/credential-security-invariants.test.ts +1 -0
- package/src/__tests__/llm-request-log-clickhouse-routing.test.ts +117 -0
- package/src/__tests__/llm-request-log-sink-clickhouse.test.ts +219 -0
- package/src/__tests__/mtime-cache.test.ts +13 -2
- package/src/agent/loop.ts +16 -2
- package/src/cli/lib/__tests__/install-from-github.test.ts +36 -4
- package/src/cli/lib/install-from-github.ts +42 -1
- package/src/daemon/conversation-agent-loop-handlers.ts +18 -4
- package/src/daemon/conversation-tool-setup.ts +10 -5
- package/src/daemon/external-plugins-bootstrap.ts +7 -0
- package/src/persistence/__tests__/message-content-file.test.ts +130 -35
- package/src/persistence/conversation-crud.ts +19 -0
- package/src/persistence/llm-request-log-sink-clickhouse.ts +312 -0
- package/src/persistence/llm-request-log-store.ts +40 -2
- package/src/persistence/message-content-file.ts +129 -73
- package/src/persistence/migrations/325-create-config-setting-events.ts +40 -0
- package/src/persistence/schema/infrastructure.ts +21 -0
- package/src/persistence/steps.ts +2 -0
- package/src/plugin-api/index.ts +9 -0
- package/src/plugins/mtime-cache.ts +93 -57
- package/src/providers/content-block-schema.ts +149 -0
- package/src/telemetry/__tests__/config-setting-events-store.test.ts +134 -0
- package/src/telemetry/config-setting-events-store.ts +97 -0
- package/src/telemetry/turn-trace-store.test.ts +86 -1
- package/src/telemetry/turn-trace-store.ts +29 -1
- package/src/telemetry/types.ts +36 -2
- package/src/telemetry/usage-telemetry-reporter.test.ts +106 -7
- package/src/telemetry/usage-telemetry-reporter.ts +58 -3
- package/src/tools/registry.ts +99 -0
- package/src/tts/synthesize-text.ts +15 -5
package/docs/workspace-tools.md
CHANGED
|
@@ -130,14 +130,17 @@ on the incoming tool.
|
|
|
130
130
|
initializeTools() # core tools register
|
|
131
131
|
→ loadWorkspaceTools() # initial workspace tool reconcile
|
|
132
132
|
→ MCP tool registration
|
|
133
|
-
→ loadUserPlugins()
|
|
133
|
+
→ loadUserPlugins() # populates the plugin mtime-cache (no registry writes)
|
|
134
|
+
→ loadPluginTools() # registry pulls the active user-plugin tool set
|
|
134
135
|
→ bootstrapPlugins()
|
|
135
136
|
|
|
136
137
|
# on every conversation turn (createResolveToolsCallback):
|
|
137
138
|
resolveTools(history)
|
|
138
139
|
→ loadWorkspaceTools() # reconcile registry against disk (fire-and-forget)
|
|
140
|
+
→ loadPluginTools() # pull user-plugin tools from the mtime-cache (same pattern)
|
|
139
141
|
→ getWorkspaceToolDefinitions() # re-read workspace tools from the registry
|
|
140
142
|
→ getMcpToolDefinitions() # re-read MCP tools (same pattern)
|
|
143
|
+
→ getPluginToolDefinitions() # re-read plugin tools from the registry
|
|
141
144
|
```
|
|
142
145
|
|
|
143
146
|
Workspace tools register _after_ core tools and _before_ every other
|
package/package.json
CHANGED
|
@@ -1606,6 +1606,31 @@ describe("session-agent-loop", () => {
|
|
|
1606
1606
|
expect(mainAgentCall?.[2]).toBe(3);
|
|
1607
1607
|
expect(mainAgentCall?.[3]).toBe("gpt-4.1-2026-03-01");
|
|
1608
1608
|
});
|
|
1609
|
+
|
|
1610
|
+
test("persists the served model onto the assistant row's metadata at finalize", async () => {
|
|
1611
|
+
const events: ServerMessage[] = [];
|
|
1612
|
+
|
|
1613
|
+
const ctx = makeCtx({
|
|
1614
|
+
providerResponses: [
|
|
1615
|
+
{
|
|
1616
|
+
content: [{ type: "text", text: "Hi there." }],
|
|
1617
|
+
model: "gpt-4.1-2026-03-01",
|
|
1618
|
+
usage: { inputTokens: 12, outputTokens: 3 },
|
|
1619
|
+
stopReason: "end_turn",
|
|
1620
|
+
},
|
|
1621
|
+
],
|
|
1622
|
+
});
|
|
1623
|
+
|
|
1624
|
+
await runAgentLoopImpl(ctx, "hello", "msg-1", (msg) => events.push(msg));
|
|
1625
|
+
|
|
1626
|
+
// The finalize write carries the `message_complete` event's model
|
|
1627
|
+
// (`response.model`) as metadata alongside the content, in one write.
|
|
1628
|
+
const finalizeCall = updateMessageContentMock.mock.calls.find(
|
|
1629
|
+
(call) => (call as unknown[])[2] !== undefined,
|
|
1630
|
+
) as unknown[] | undefined;
|
|
1631
|
+
expect(finalizeCall).toBeDefined();
|
|
1632
|
+
expect(finalizeCall?.[2]).toEqual({ model: "gpt-4.1-2026-03-01" });
|
|
1633
|
+
});
|
|
1609
1634
|
});
|
|
1610
1635
|
|
|
1611
1636
|
describe("checkpoint handoff (infinite loop prevention)", () => {
|
|
@@ -208,6 +208,7 @@ describe("Invariant 2: no generic plaintext secret read API", () => {
|
|
|
208
208
|
"media/image-credentials.ts", // shared image-gen credential resolver (provider API key lookup)
|
|
209
209
|
"persistence/embeddings/embedding-backend.ts", // embedding backend API key lookup
|
|
210
210
|
"persistence/llm-request-log-source-clickhouse.ts", // ClickHouse read source — lazy lookup of clickhouse:url + clickhouse:password + vellum:platform_assistant_id for self-scoped mirror reads
|
|
211
|
+
"persistence/llm-request-log-sink-clickhouse.ts", // ClickHouse write sink — lazy lookup of clickhouse:url + clickhouse:password + vellum:platform_assistant_id for self-scoped log writes
|
|
211
212
|
"persistence/compaction-log-store-clickhouse.ts", // ClickHouse compaction log writer — lazy lookup of clickhouse:url + clickhouse:password + vellum:platform_assistant_id for self-scoped event writes
|
|
212
213
|
"daemon/providers-setup.ts", // provider initialization API key lookup
|
|
213
214
|
"workspace/migrations/006-services-config.ts", // services config migration reads provider API keys
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Tests that the store's write path routes to ClickHouse instead of local
|
|
3
|
+
* SQLite when a ClickHouse sink is configured — and stays on SQLite
|
|
4
|
+
* otherwise. The sink module is mocked so the routing decision is observable
|
|
5
|
+
* without a live ClickHouse.
|
|
6
|
+
*/
|
|
7
|
+
import { afterAll, beforeEach, describe, expect, mock, test } from "bun:test";
|
|
8
|
+
|
|
9
|
+
mock.module("../util/logger.js", () => ({
|
|
10
|
+
getLogger: () =>
|
|
11
|
+
new Proxy({} as Record<string, unknown>, { get: () => () => {} }),
|
|
12
|
+
}));
|
|
13
|
+
|
|
14
|
+
// Logging stays enabled; `getConfigReadOnly` is only consulted by the store.s
|
|
15
|
+
// disabled gate here (the sink factory itself is mocked below).
|
|
16
|
+
mock.module("../config/loader.js", () => ({
|
|
17
|
+
getConfig: () => ({ llmRequestLogs: { readSource: "local" } }),
|
|
18
|
+
getConfigReadOnly: () => ({ llmRequestLogs: { readSource: "local" } }),
|
|
19
|
+
}));
|
|
20
|
+
|
|
21
|
+
// Mutable fake sink: when non-null, the store must route to it.
|
|
22
|
+
interface CapturedRow {
|
|
23
|
+
id: string;
|
|
24
|
+
conversationId: string;
|
|
25
|
+
callSite: string | null;
|
|
26
|
+
}
|
|
27
|
+
let capturedRows: CapturedRow[] = [];
|
|
28
|
+
let sinkActive = false;
|
|
29
|
+
mock.module("../persistence/llm-request-log-sink-clickhouse.js", () => ({
|
|
30
|
+
getClickHouseLlmRequestLogSink: () =>
|
|
31
|
+
sinkActive
|
|
32
|
+
? {
|
|
33
|
+
recordBestEffort: (row: CapturedRow) => {
|
|
34
|
+
capturedRows.push(row);
|
|
35
|
+
},
|
|
36
|
+
}
|
|
37
|
+
: null,
|
|
38
|
+
}));
|
|
39
|
+
|
|
40
|
+
afterAll(() => {
|
|
41
|
+
sinkActive = false;
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
import { getLogsDb } from "../persistence/db-connection.js";
|
|
45
|
+
import { initializeDb } from "../persistence/db-init.js";
|
|
46
|
+
import {
|
|
47
|
+
getRequestLogsByConversationId,
|
|
48
|
+
recordRequestLog,
|
|
49
|
+
recordSyntheticAgentErrorMessageLog,
|
|
50
|
+
} from "../persistence/llm-request-log-store.js";
|
|
51
|
+
import { llmRequestLogs } from "../persistence/schema/index.js";
|
|
52
|
+
|
|
53
|
+
await initializeDb();
|
|
54
|
+
|
|
55
|
+
function resetLogs(): void {
|
|
56
|
+
getLogsDb()!.delete(llmRequestLogs).run();
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
describe("write routing between SQLite and ClickHouse", () => {
|
|
60
|
+
beforeEach(() => {
|
|
61
|
+
resetLogs();
|
|
62
|
+
capturedRows = [];
|
|
63
|
+
sinkActive = false;
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
test("writes to local SQLite when no ClickHouse sink is configured", () => {
|
|
67
|
+
const id = recordRequestLog(
|
|
68
|
+
"conv-local",
|
|
69
|
+
'{"req":1}',
|
|
70
|
+
'{"res":1}',
|
|
71
|
+
undefined,
|
|
72
|
+
"anthropic",
|
|
73
|
+
"mainAgent",
|
|
74
|
+
);
|
|
75
|
+
expect(id).not.toBeNull();
|
|
76
|
+
expect(capturedRows).toHaveLength(0);
|
|
77
|
+
const rows = getRequestLogsByConversationId("conv-local");
|
|
78
|
+
expect(rows).toHaveLength(1);
|
|
79
|
+
expect(rows[0]!.provider).toBe("anthropic");
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
test("routes to ClickHouse and skips SQLite when a sink is configured", () => {
|
|
83
|
+
sinkActive = true;
|
|
84
|
+
const id = recordRequestLog(
|
|
85
|
+
"conv-ch",
|
|
86
|
+
'{"req":1}',
|
|
87
|
+
'{"res":1}',
|
|
88
|
+
"msg-ch",
|
|
89
|
+
"openai",
|
|
90
|
+
"mainAgent",
|
|
91
|
+
);
|
|
92
|
+
expect(id).not.toBeNull();
|
|
93
|
+
// Nothing landed in the local table.
|
|
94
|
+
expect(getRequestLogsByConversationId("conv-ch")).toEqual([]);
|
|
95
|
+
// The row went to the sink, carrying the same id and call site.
|
|
96
|
+
expect(capturedRows).toHaveLength(1);
|
|
97
|
+
expect(capturedRows[0]!.id).toBe(id!);
|
|
98
|
+
expect(capturedRows[0]!.conversationId).toBe("conv-ch");
|
|
99
|
+
expect(capturedRows[0]!.callSite).toBe("mainAgent");
|
|
100
|
+
});
|
|
101
|
+
|
|
102
|
+
test("routes synthetic error rows to ClickHouse too", () => {
|
|
103
|
+
sinkActive = true;
|
|
104
|
+
const id = recordSyntheticAgentErrorMessageLog({
|
|
105
|
+
conversationId: "conv-ch-2",
|
|
106
|
+
messageId: "msg-ch-2",
|
|
107
|
+
exitReason: "budget_yield_unrecovered",
|
|
108
|
+
noticeText: "Out of budget.",
|
|
109
|
+
preparedRequest: null,
|
|
110
|
+
createdAt: 1_700_000_000_000,
|
|
111
|
+
});
|
|
112
|
+
expect(id).not.toBeNull();
|
|
113
|
+
expect(getRequestLogsByConversationId("conv-ch-2")).toEqual([]);
|
|
114
|
+
expect(capturedRows).toHaveLength(1);
|
|
115
|
+
expect(capturedRows[0]!.callSite).toBe("syntheticAgentErrorMessage");
|
|
116
|
+
});
|
|
117
|
+
});
|
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Tests the ClickHouse WRITE sink for LLM request logs: the wire shape it
|
|
3
|
+
* POSTs (lazy CREATE TABLE + JSONEachRow INSERT, null → '' sentinels,
|
|
4
|
+
* assistant_id injected from credentials) and the config-gated factory.
|
|
5
|
+
*/
|
|
6
|
+
import { afterEach, describe, expect, mock, test } from "bun:test";
|
|
7
|
+
|
|
8
|
+
function makeLoggerStub(): Record<string, unknown> {
|
|
9
|
+
const stub: Record<string, unknown> = {};
|
|
10
|
+
for (const m of [
|
|
11
|
+
"info",
|
|
12
|
+
"warn",
|
|
13
|
+
"error",
|
|
14
|
+
"debug",
|
|
15
|
+
"trace",
|
|
16
|
+
"fatal",
|
|
17
|
+
"silent",
|
|
18
|
+
"child",
|
|
19
|
+
]) {
|
|
20
|
+
stub[m] = m === "child" ? () => makeLoggerStub() : () => {};
|
|
21
|
+
}
|
|
22
|
+
return stub;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
mock.module("../util/logger.js", () => ({
|
|
26
|
+
getLogger: () => makeLoggerStub(),
|
|
27
|
+
}));
|
|
28
|
+
|
|
29
|
+
// Mutable config the factory reads via `getConfigReadOnly`.
|
|
30
|
+
let currentConfig: unknown = { llmRequestLogs: { readSource: "local" } };
|
|
31
|
+
mock.module("../config/loader.js", () => ({
|
|
32
|
+
getConfigReadOnly: () => currentConfig,
|
|
33
|
+
}));
|
|
34
|
+
|
|
35
|
+
afterEach(() => {
|
|
36
|
+
currentConfig = { llmRequestLogs: { readSource: "local" } };
|
|
37
|
+
resetClickHouseLlmRequestLogSinkForTests();
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
import {
|
|
41
|
+
ClickHouseLlmRequestLogSink,
|
|
42
|
+
getClickHouseLlmRequestLogSink,
|
|
43
|
+
resetClickHouseLlmRequestLogSinkForTests,
|
|
44
|
+
} from "../persistence/llm-request-log-sink-clickhouse.js";
|
|
45
|
+
|
|
46
|
+
const DEFAULT_CONFIG = {
|
|
47
|
+
database: "default",
|
|
48
|
+
table: "llm_request_logs",
|
|
49
|
+
user: "default",
|
|
50
|
+
};
|
|
51
|
+
|
|
52
|
+
interface FakeFetchCall {
|
|
53
|
+
url: string;
|
|
54
|
+
init: RequestInit | undefined;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
function fakeFetch(recorder: FakeFetchCall[], status = 200): typeof fetch {
|
|
58
|
+
return ((url: string | URL | Request, init?: RequestInit) => {
|
|
59
|
+
recorder.push({
|
|
60
|
+
url: typeof url === "string" ? url : url.toString(),
|
|
61
|
+
init,
|
|
62
|
+
});
|
|
63
|
+
return Promise.resolve(new Response("", { status }));
|
|
64
|
+
}) as typeof fetch;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
function makeSink(recorder: FakeFetchCall[], status = 200) {
|
|
68
|
+
return new ClickHouseLlmRequestLogSink(DEFAULT_CONFIG, {
|
|
69
|
+
resolveUrl: async () => "https://ch.example.test:8443",
|
|
70
|
+
resolvePassword: async () => "hunter2",
|
|
71
|
+
resolveAssistantId: async () => "asst-fixture-001",
|
|
72
|
+
fetchImpl: fakeFetch(recorder, status),
|
|
73
|
+
});
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
describe("ClickHouseLlmRequestLogSink.insert", () => {
|
|
77
|
+
test("creates the table then inserts the JSONEachRow wire shape", async () => {
|
|
78
|
+
const calls: FakeFetchCall[] = [];
|
|
79
|
+
const sink = makeSink(calls);
|
|
80
|
+
|
|
81
|
+
await sink.insert({
|
|
82
|
+
id: "log-1",
|
|
83
|
+
conversationId: "conv-1",
|
|
84
|
+
messageId: "msg-1",
|
|
85
|
+
provider: "anthropic",
|
|
86
|
+
requestPayload: '{"req":1}',
|
|
87
|
+
responsePayload: '{"res":1}',
|
|
88
|
+
createdAt: 1_700_000_000_123,
|
|
89
|
+
agentLoopExitReason: "no_tool_calls",
|
|
90
|
+
callSite: "mainAgent",
|
|
91
|
+
});
|
|
92
|
+
|
|
93
|
+
expect(calls).toHaveLength(2);
|
|
94
|
+
// 1) DDL as the request body, no `query` param.
|
|
95
|
+
expect(String(calls[0]!.init?.body)).toContain(
|
|
96
|
+
"CREATE TABLE IF NOT EXISTS",
|
|
97
|
+
);
|
|
98
|
+
expect(new URL(calls[0]!.url).searchParams.get("query")).toBeNull();
|
|
99
|
+
|
|
100
|
+
// 2) INSERT statement in the `query` param, row JSON in the body.
|
|
101
|
+
const insertUrl = new URL(calls[1]!.url);
|
|
102
|
+
expect(insertUrl.searchParams.get("query")).toContain(
|
|
103
|
+
"INSERT INTO `llm_request_logs` FORMAT JSONEachRow",
|
|
104
|
+
);
|
|
105
|
+
expect(insertUrl.searchParams.get("database")).toBe("default");
|
|
106
|
+
const row = JSON.parse(String(calls[1]!.init?.body));
|
|
107
|
+
expect(row).toEqual({
|
|
108
|
+
assistant_id: "asst-fixture-001",
|
|
109
|
+
id: "log-1",
|
|
110
|
+
conversation_id: "conv-1",
|
|
111
|
+
message_id: "msg-1",
|
|
112
|
+
provider: "anthropic",
|
|
113
|
+
request_payload: '{"req":1}',
|
|
114
|
+
response_payload: '{"res":1}',
|
|
115
|
+
created_at: 1_700_000_000_123,
|
|
116
|
+
agent_loop_exit_reason: "no_tool_calls",
|
|
117
|
+
call_site: "mainAgent",
|
|
118
|
+
});
|
|
119
|
+
// Basic auth is sent.
|
|
120
|
+
expect(
|
|
121
|
+
(calls[1]!.init?.headers as Record<string, string>).Authorization,
|
|
122
|
+
).toStartWith("Basic ");
|
|
123
|
+
});
|
|
124
|
+
|
|
125
|
+
test("maps null string fields to the '' sentinel", async () => {
|
|
126
|
+
const calls: FakeFetchCall[] = [];
|
|
127
|
+
const sink = makeSink(calls);
|
|
128
|
+
|
|
129
|
+
await sink.insert({
|
|
130
|
+
id: "log-2",
|
|
131
|
+
conversationId: "conv-2",
|
|
132
|
+
messageId: null,
|
|
133
|
+
provider: null,
|
|
134
|
+
requestPayload: "{}",
|
|
135
|
+
responsePayload: "{}",
|
|
136
|
+
createdAt: 1,
|
|
137
|
+
agentLoopExitReason: null,
|
|
138
|
+
callSite: null,
|
|
139
|
+
});
|
|
140
|
+
|
|
141
|
+
const row = JSON.parse(String(calls[1]!.init?.body));
|
|
142
|
+
expect(row.message_id).toBe("");
|
|
143
|
+
expect(row.provider).toBe("");
|
|
144
|
+
expect(row.agent_loop_exit_reason).toBe("");
|
|
145
|
+
expect(row.call_site).toBe("");
|
|
146
|
+
});
|
|
147
|
+
|
|
148
|
+
test("insert rejects on a ClickHouse error status", async () => {
|
|
149
|
+
const calls: FakeFetchCall[] = [];
|
|
150
|
+
const sink = makeSink(calls, 500);
|
|
151
|
+
await expect(
|
|
152
|
+
sink.insert({
|
|
153
|
+
id: "log-3",
|
|
154
|
+
conversationId: "conv-3",
|
|
155
|
+
messageId: null,
|
|
156
|
+
provider: null,
|
|
157
|
+
requestPayload: "{}",
|
|
158
|
+
responsePayload: "{}",
|
|
159
|
+
createdAt: 1,
|
|
160
|
+
agentLoopExitReason: null,
|
|
161
|
+
callSite: null,
|
|
162
|
+
}),
|
|
163
|
+
).rejects.toThrow(/ClickHouse llm-request-log write failed/);
|
|
164
|
+
});
|
|
165
|
+
|
|
166
|
+
test("recordBestEffort never throws synchronously on a failing backend", () => {
|
|
167
|
+
const calls: FakeFetchCall[] = [];
|
|
168
|
+
const sink = makeSink(calls, 500);
|
|
169
|
+
expect(() =>
|
|
170
|
+
sink.recordBestEffort({
|
|
171
|
+
id: "log-4",
|
|
172
|
+
conversationId: "conv-4",
|
|
173
|
+
messageId: null,
|
|
174
|
+
provider: null,
|
|
175
|
+
requestPayload: "{}",
|
|
176
|
+
responsePayload: "{}",
|
|
177
|
+
createdAt: 1,
|
|
178
|
+
agentLoopExitReason: null,
|
|
179
|
+
callSite: null,
|
|
180
|
+
}),
|
|
181
|
+
).not.toThrow();
|
|
182
|
+
});
|
|
183
|
+
});
|
|
184
|
+
|
|
185
|
+
describe("getClickHouseLlmRequestLogSink factory", () => {
|
|
186
|
+
test("returns null when readSource is 'local'", () => {
|
|
187
|
+
currentConfig = { llmRequestLogs: { readSource: "local" } };
|
|
188
|
+
expect(getClickHouseLlmRequestLogSink()).toBeNull();
|
|
189
|
+
});
|
|
190
|
+
|
|
191
|
+
test("returns null when llmRequestLogs is absent", () => {
|
|
192
|
+
currentConfig = {};
|
|
193
|
+
expect(getClickHouseLlmRequestLogSink()).toBeNull();
|
|
194
|
+
});
|
|
195
|
+
|
|
196
|
+
test("returns a sink when readSource is 'clickhouse'", () => {
|
|
197
|
+
currentConfig = {
|
|
198
|
+
llmRequestLogs: { readSource: "clickhouse", clickhouse: DEFAULT_CONFIG },
|
|
199
|
+
};
|
|
200
|
+
expect(getClickHouseLlmRequestLogSink()).toBeInstanceOf(
|
|
201
|
+
ClickHouseLlmRequestLogSink,
|
|
202
|
+
);
|
|
203
|
+
});
|
|
204
|
+
|
|
205
|
+
test("re-instantiates when the clickhouse config changes", () => {
|
|
206
|
+
currentConfig = {
|
|
207
|
+
llmRequestLogs: { readSource: "clickhouse", clickhouse: DEFAULT_CONFIG },
|
|
208
|
+
};
|
|
209
|
+
const first = getClickHouseLlmRequestLogSink();
|
|
210
|
+
currentConfig = {
|
|
211
|
+
llmRequestLogs: {
|
|
212
|
+
readSource: "clickhouse",
|
|
213
|
+
clickhouse: { ...DEFAULT_CONFIG, table: "other_logs" },
|
|
214
|
+
},
|
|
215
|
+
};
|
|
216
|
+
const second = getClickHouseLlmRequestLogSink();
|
|
217
|
+
expect(second).not.toBe(first);
|
|
218
|
+
});
|
|
219
|
+
});
|
|
@@ -43,9 +43,11 @@ import {
|
|
|
43
43
|
} from "../plugins/mtime-cache.js";
|
|
44
44
|
import { getSourceVersionsPath } from "../plugins/source-versions.js";
|
|
45
45
|
import {
|
|
46
|
+
__resetRegistryForTesting,
|
|
46
47
|
getAllToolDefinitions,
|
|
47
48
|
getPluginToolDefinitions,
|
|
48
49
|
getToolOwner,
|
|
50
|
+
loadPluginTools,
|
|
49
51
|
} from "../tools/registry.js";
|
|
50
52
|
|
|
51
53
|
// ─── Test fixtures ───────────────────────────────────────────────────────────
|
|
@@ -171,6 +173,9 @@ beforeEach(() => {
|
|
|
171
173
|
watchState = null;
|
|
172
174
|
sentinelTouchSeq = 0;
|
|
173
175
|
resetPluginCacheForTests();
|
|
176
|
+
// The registry pulls plugin tools via loadPluginTools(); clear its side
|
|
177
|
+
// (tools + pulled fingerprints) so registrations never leak across tests.
|
|
178
|
+
__resetRegistryForTesting();
|
|
174
179
|
});
|
|
175
180
|
|
|
176
181
|
afterAll(() => {
|
|
@@ -357,9 +362,11 @@ describe("plugin mtime cache (per-surface)", () => {
|
|
|
357
362
|
touchFile(toolFile);
|
|
358
363
|
publishSourceChanges();
|
|
359
364
|
await getUserHooksFor("user-prompt-submit");
|
|
365
|
+
await loadPluginTools();
|
|
360
366
|
|
|
361
367
|
// Both the cache and the global registry serve the fresh definition —
|
|
362
|
-
// the
|
|
368
|
+
// the moved source mtime changed the plugin's fingerprint, so the pull
|
|
369
|
+
// reconcile re-registered its tool set.
|
|
363
370
|
expect(getCachedUserTools()[0]?.description).toBe("v2");
|
|
364
371
|
expect(
|
|
365
372
|
getAllToolDefinitions().find((t) => t.name === "edited-tool")
|
|
@@ -632,11 +639,13 @@ const TOOL_SRC = (name: string) =>
|
|
|
632
639
|
* production sequence: the monitor's pass rewrites the sentinel, and the
|
|
633
640
|
* next hook dispatch's one-stat gate applies the diff. The hook name is
|
|
634
641
|
* irrelevant — the reconcile runs regardless of whether a plugin defines
|
|
635
|
-
* that hook.
|
|
642
|
+
* that hook. Finish with the registry pull the per-turn tool resolver
|
|
643
|
+
* fires, so registry-facing assertions see the reconciled tool set.
|
|
636
644
|
*/
|
|
637
645
|
async function publishAndDispatch(): Promise<void> {
|
|
638
646
|
publishSourceChanges();
|
|
639
647
|
await getUserHooksFor("user-prompt-submit");
|
|
648
|
+
await loadPluginTools();
|
|
640
649
|
}
|
|
641
650
|
|
|
642
651
|
describe("plugin runtime activation", () => {
|
|
@@ -701,6 +710,7 @@ describe("plugin runtime activation", () => {
|
|
|
701
710
|
writeMarkerHook(dir, "shutdown", shutdownMarker, "bye");
|
|
702
711
|
|
|
703
712
|
await populateCacheAtBoot();
|
|
713
|
+
await loadPluginTools(); // boot pull (initializePlugins does this)
|
|
704
714
|
expect(getToolOwner("temp-tool")?.kind).toBe("plugin");
|
|
705
715
|
|
|
706
716
|
rmSync(dir, { recursive: true, force: true });
|
|
@@ -740,6 +750,7 @@ describe("plugin runtime activation", () => {
|
|
|
740
750
|
writeMarkerHook(dir, "shutdown", shutdownMarker, "disabled");
|
|
741
751
|
|
|
742
752
|
await populateCacheAtBoot();
|
|
753
|
+
await loadPluginTools(); // boot pull (initializePlugins does this)
|
|
743
754
|
expect(getToolOwner("disable-tool")?.kind).toBe("plugin");
|
|
744
755
|
|
|
745
756
|
writeFileSync(join(dir, ".disabled"), "");
|
package/src/agent/loop.ts
CHANGED
|
@@ -240,7 +240,15 @@ export type AgentEvent =
|
|
|
240
240
|
| { type: "llm_call_started"; callSite?: LLMCallSite }
|
|
241
241
|
| { type: "text_delta"; text: string }
|
|
242
242
|
| { type: "thinking_delta"; thinking: string }
|
|
243
|
-
|
|
243
|
+
/**
|
|
244
|
+
* Emitted once per LLM call when the assistant message is finalized.
|
|
245
|
+
* `model` is the provider-reported model that served the call
|
|
246
|
+
* (`response.model`) — the same value `llm_usage` records — so downstream
|
|
247
|
+
* persistence can attribute the message to the model that actually ran,
|
|
248
|
+
* including per-call reroutes by a `pre-model-call` hook. Absent on
|
|
249
|
+
* synthesized emissions that have no provider response.
|
|
250
|
+
*/
|
|
251
|
+
| { type: "message_complete"; message: Message; model?: string }
|
|
244
252
|
| { type: "max_tokens_reached"; stopReason: string }
|
|
245
253
|
| {
|
|
246
254
|
type: "tool_use";
|
|
@@ -1902,6 +1910,7 @@ export class AgentLoop {
|
|
|
1902
1910
|
await onEvent({
|
|
1903
1911
|
type: "message_complete",
|
|
1904
1912
|
message: safeAssistantMessage,
|
|
1913
|
+
model: response.model,
|
|
1905
1914
|
});
|
|
1906
1915
|
history = maxTokensMessages;
|
|
1907
1916
|
continue;
|
|
@@ -1914,6 +1923,7 @@ export class AgentLoop {
|
|
|
1914
1923
|
await onEvent({
|
|
1915
1924
|
type: "message_complete",
|
|
1916
1925
|
message: safeAssistantMessage,
|
|
1926
|
+
model: response.model,
|
|
1917
1927
|
});
|
|
1918
1928
|
await stopTurn("max_tokens_reached");
|
|
1919
1929
|
break;
|
|
@@ -1998,7 +2008,11 @@ export class AgentLoop {
|
|
|
1998
2008
|
|
|
1999
2009
|
history.push(assistantMessage);
|
|
2000
2010
|
|
|
2001
|
-
await onEvent({
|
|
2011
|
+
await onEvent({
|
|
2012
|
+
type: "message_complete",
|
|
2013
|
+
message: assistantMessage,
|
|
2014
|
+
model: response.model,
|
|
2015
|
+
});
|
|
2002
2016
|
|
|
2003
2017
|
if (toolUseBlocks.length === 0 || !this.toolExecutor) {
|
|
2004
2018
|
// The model stopped requesting tools and `post-model-call` settled on
|
|
@@ -78,14 +78,14 @@ function makeContentsFetch(opts: {
|
|
|
78
78
|
const prefix = apiPath ? `${apiPath}/` : "";
|
|
79
79
|
const direct = new Map<string, boolean>();
|
|
80
80
|
for (const key of Object.keys(tree)) {
|
|
81
|
-
if (!key.startsWith(prefix)) continue;
|
|
81
|
+
if (!key.startsWith(prefix)) {continue;}
|
|
82
82
|
const remainder = key.slice(prefix.length);
|
|
83
|
-
if (!remainder) continue;
|
|
83
|
+
if (!remainder) {continue;}
|
|
84
84
|
const seg = remainder.split("/")[0]!;
|
|
85
85
|
const isDir = remainder.includes("/") || tree[`${prefix}${seg}`] === null;
|
|
86
86
|
direct.set(seg, (direct.get(seg) ?? false) || isDir);
|
|
87
87
|
}
|
|
88
|
-
if (direct.size === 0) return null;
|
|
88
|
+
if (direct.size === 0) {return null;}
|
|
89
89
|
return Array.from(direct.entries()).map(([name, isDir]) => {
|
|
90
90
|
const path = `${prefix}${name}`;
|
|
91
91
|
return isDir
|
|
@@ -157,7 +157,7 @@ function fakeGitRunner(opts: {
|
|
|
157
157
|
opts.calls?.push([...args]);
|
|
158
158
|
switch (args[0]) {
|
|
159
159
|
case "fetch": {
|
|
160
|
-
if (opts.fetchError) throw opts.fetchError;
|
|
160
|
+
if (opts.fetchError) {throw opts.fetchError;}
|
|
161
161
|
mkdirSync(join(cwd, ".git"), { recursive: true });
|
|
162
162
|
writeFileSync(join(cwd, ".git", "config"), "[core]\n");
|
|
163
163
|
for (const [rel, content] of Object.entries(opts.tree)) {
|
|
@@ -1097,6 +1097,38 @@ describe("installPlugin — direct (untrusted) install", () => {
|
|
|
1097
1097
|
).rejects.toBeInstanceOf(PluginSourceUnavailableError);
|
|
1098
1098
|
expect(readdirSync(pluginsDir)).toEqual([]);
|
|
1099
1099
|
});
|
|
1100
|
+
|
|
1101
|
+
test("synthesizes a minimal package.json when the upstream repo ships none", async () => {
|
|
1102
|
+
// GIVEN a plugin tree with no package.json at all
|
|
1103
|
+
const fetch = makeContentsFetch({ tree: {} });
|
|
1104
|
+
const runGit = fakeGitRunner({
|
|
1105
|
+
tree: { "hooks/init.ts": "//", "README.md": "# bare" },
|
|
1106
|
+
commit: "f".repeat(40),
|
|
1107
|
+
});
|
|
1108
|
+
|
|
1109
|
+
// WHEN we install directly
|
|
1110
|
+
await installPlugin(
|
|
1111
|
+
{
|
|
1112
|
+
name: "bare-plugin",
|
|
1113
|
+
directSource: {
|
|
1114
|
+
owner: "owner",
|
|
1115
|
+
repo: "bare",
|
|
1116
|
+
rootPath: "",
|
|
1117
|
+
ref: "HEAD",
|
|
1118
|
+
},
|
|
1119
|
+
},
|
|
1120
|
+
{ fetch, runGit, workspacePluginsDir: pluginsDir },
|
|
1121
|
+
);
|
|
1122
|
+
|
|
1123
|
+
// THEN a minimal package.json is synthesized with the install name and
|
|
1124
|
+
// the default plugin-api peer dependency range
|
|
1125
|
+
const pkgPath = join(pluginsDir, "bare-plugin", "package.json");
|
|
1126
|
+
expect(existsSync(pkgPath)).toBe(true);
|
|
1127
|
+
const pkg = JSON.parse(readFileSync(pkgPath, "utf-8"));
|
|
1128
|
+
expect(pkg.name).toBe("bare-plugin");
|
|
1129
|
+
expect(pkg.version).toBe("0.0.0");
|
|
1130
|
+
expect(pkg.peerDependencies["@vellumai/plugin-api"]).toBeDefined();
|
|
1131
|
+
});
|
|
1100
1132
|
});
|
|
1101
1133
|
|
|
1102
1134
|
describe("sanitizePluginName", () => {
|
|
@@ -714,10 +714,23 @@ export async function materializePluginTree(
|
|
|
714
714
|
// An external clone is often a foreign-ecosystem plugin (e.g. a Claude Code
|
|
715
715
|
// plugin) that the Vellum loader can't run as-is. When we curate an adapter
|
|
716
716
|
// stub for it, overlay the stub and run its transform so the materialized
|
|
717
|
-
// tree is a valid Vellum plugin. Raw clones (no stub) are left untouched
|
|
717
|
+
// tree is a valid Vellum plugin. Raw clones (no stub) are left untouched,
|
|
718
|
+
// except for a minimal package.json synthesis when the upstream repo shipped
|
|
719
|
+
// none — the Vellum loader hard-requires one and would silently skip the
|
|
720
|
+
// plugin without it. The synthesis runs only for direct installs
|
|
721
|
+
// (stubRef === null); the upgrade path re-materializes baselines through
|
|
722
|
+
// this function with a non-null stubRef, and synthesizing a package.json
|
|
723
|
+
// not present at install time would corrupt the fingerprint comparison.
|
|
718
724
|
if (cloned.fileCount > 0 && opts.stubRef !== null) {
|
|
719
725
|
await applyAdapterStub(opts.name, opts.stubRef, opts.destDir, deps);
|
|
720
726
|
}
|
|
727
|
+
if (
|
|
728
|
+
cloned.fileCount > 0 &&
|
|
729
|
+
opts.stubRef === null &&
|
|
730
|
+
!existsSync(join(opts.destDir, "package.json"))
|
|
731
|
+
) {
|
|
732
|
+
synthesizeMinimalPackageJson(opts.name, opts.destDir);
|
|
733
|
+
}
|
|
721
734
|
return cloned;
|
|
722
735
|
}
|
|
723
736
|
|
|
@@ -974,6 +987,34 @@ function normalizeInstalledManifest(
|
|
|
974
987
|
writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, "utf8");
|
|
975
988
|
}
|
|
976
989
|
|
|
990
|
+
/**
|
|
991
|
+
* Write a minimal Vellum-compatible `package.json` into a staged plugin that
|
|
992
|
+
* shipped no manifest of its own. The Vellum external plugin loader
|
|
993
|
+
* (`buildPluginFromDir`) hard-requires a `package.json` validated against
|
|
994
|
+
* `PluginPackageJsonSchema` and silently skips the plugin when it's missing.
|
|
995
|
+
*
|
|
996
|
+
* The synthesized manifest carries the install name and the default
|
|
997
|
+
* `@vellumai/plugin-api` peer dependency range. No foreign-ecosystem manifest
|
|
998
|
+
* data is read — the install name is the only identity we trust for an
|
|
999
|
+
* untrusted direct install.
|
|
1000
|
+
*/
|
|
1001
|
+
function synthesizeMinimalPackageJson(
|
|
1002
|
+
name: string,
|
|
1003
|
+
stagingDir: string,
|
|
1004
|
+
): void {
|
|
1005
|
+
const manifestPath = join(stagingDir, "package.json");
|
|
1006
|
+
|
|
1007
|
+
const manifest: PackageManifest = {
|
|
1008
|
+
name,
|
|
1009
|
+
version: "0.0.0",
|
|
1010
|
+
peerDependencies: {
|
|
1011
|
+
"@vellumai/plugin-api": PLUGIN_API_PEER_RANGE,
|
|
1012
|
+
},
|
|
1013
|
+
};
|
|
1014
|
+
|
|
1015
|
+
writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, "utf8");
|
|
1016
|
+
}
|
|
1017
|
+
|
|
977
1018
|
/**
|
|
978
1019
|
* Resolve the absolute path of the adapter script named by the (overlaid stub)
|
|
979
1020
|
* `package.json`'s `scripts.postinstall`, or `null` when there is no stub
|