@vellumai/assistant 0.10.0-dev.202606200539.c052d10 → 0.10.0-dev.202606201539.0cebd50
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/__tests__/agent-loop-callsite-precedence.test.ts +1 -40
- package/src/__tests__/agent-wake-override-profile.test.ts +2 -0
- package/src/__tests__/app-source-watcher.test.ts +30 -10
- package/src/__tests__/config-schema.test.ts +34 -0
- package/src/__tests__/conversation-agent-loop-disk-pressure.test.ts +3 -0
- package/src/__tests__/conversation-agent-loop-inference-profile.test.ts +3 -0
- package/src/__tests__/conversation-agent-loop-overflow.test.ts +3 -0
- package/src/__tests__/conversation-agent-loop.test.ts +3 -0
- package/src/__tests__/conversation-process-callsite.test.ts +0 -14
- package/src/__tests__/db-llm-request-log-provider-migration.test.ts +6 -1
- package/src/__tests__/db-migration-rollback.test.ts +69 -1
- package/src/__tests__/heartbeat-disk-pressure.test.ts +3 -0
- package/src/__tests__/heartbeat-service.test.ts +6 -0
- package/src/__tests__/list-messages-attachments.test.ts +41 -0
- package/src/__tests__/plugin-source-watcher.test.ts +33 -1
- package/src/__tests__/usage-cache-backfill-migration.test.ts +17 -2
- package/src/acp/__tests__/session-manager.test.ts +72 -1
- package/src/acp/index.ts +10 -0
- package/src/acp/session-manager.ts +35 -0
- package/src/agent/loop.ts +28 -22
- package/src/cli/commands/db/status.ts +35 -1
- package/src/config/schemas/memory-lifecycle.ts +5 -3
- package/src/config/schemas/timeouts.ts +24 -0
- package/src/daemon/app-source-watcher.ts +31 -18
- package/src/daemon/conversation-agent-loop.ts +8 -5
- package/src/daemon/conversation.ts +30 -41
- package/src/daemon/handlers/conversations.ts +7 -0
- package/src/daemon/plugin-source-watcher.ts +5 -0
- package/src/daemon/workspace-tools-watcher.ts +4 -0
- package/src/heartbeat/__tests__/heartbeat-service.test.ts +6 -0
- package/src/heartbeat/heartbeat-service.ts +3 -4
- package/src/memory/__tests__/db-maintenance.test.ts +27 -35
- package/src/memory/__tests__/db-memory-attach.test.ts +126 -0
- package/src/memory/conversation-crud.ts +9 -3
- package/src/memory/db-connection.ts +49 -0
- package/src/memory/db-init.ts +50 -14
- package/src/memory/db-maintenance.ts +43 -38
- package/src/memory/job-handlers/cleanup.ts +6 -0
- package/src/memory/migrations/{100-core-tables.ts → 000-core-tables.ts} +6 -10
- package/src/memory/migrations/103-complex-migrations.ts +13 -12
- package/src/memory/migrations/104-core-indexes.ts +1 -1
- package/src/memory/migrations/113-late-migrations.ts +16 -14
- package/src/memory/migrations/297-move-llm-request-logs-to-logs-db.ts +130 -0
- package/src/memory/migrations/__tests__/297-move-llm-request-logs.test.ts +159 -0
- package/src/memory/migrations/__tests__/run-migrations.test.ts +147 -5
- package/src/memory/migrations/index.ts +5 -4
- package/src/memory/migrations/run-migrations.ts +162 -12
- package/src/memory/migrations/validate-migration-state.ts +14 -56
- package/src/plugin-api/index.ts +7 -0
- package/src/plugin-api/vision-support.ts +75 -0
- package/src/prompts/system-prompt.ts +1 -1
- package/src/runtime/__tests__/agent-wake.test.ts +6 -4
- package/src/runtime/agent-wake.ts +15 -7
- package/src/runtime/routes/conversation-routes.ts +24 -3
- package/src/runtime/routes/migration-routes.ts +35 -39
- package/src/schedule/scheduler.ts +5 -9
- package/src/tools/ask-question/ask-question-tool.test.ts +60 -52
- package/src/tools/ask-question/ask-question-tool.ts +14 -73
- package/src/util/fs-watcher-error.ts +36 -0
- package/src/util/memory-db-path.ts +23 -0
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import type { DrizzleDb } from "../db-connection.js";
|
|
2
1
|
import { migrateJobDeferrals } from "./001-job-deferrals.js";
|
|
3
2
|
import { migrateToolInvocationsFk } from "./002-tool-invocations-fk.js";
|
|
4
3
|
import { migrateMemoryEntityRelationDedup } from "./004-entity-relation-dedup.js";
|
|
@@ -7,17 +6,19 @@ import { migrateMemoryItemsScopeSaltedFingerprints } from "./006-scope-salted-fi
|
|
|
7
6
|
import { migrateAssistantIdToSelf } from "./007-assistant-id-to-self.js";
|
|
8
7
|
import { migrateRemoveAssistantIdColumns } from "./008-remove-assistant-id-columns.js";
|
|
9
8
|
import { migrateLlmUsageEventsDropAssistantId } from "./009-llm-usage-events-drop-assistant-id.js";
|
|
9
|
+
import type { MigrationStep } from "./run-migrations.js";
|
|
10
10
|
|
|
11
11
|
/**
|
|
12
|
-
* Complex multi-step migrations that go beyond simple ALTER TABLE.
|
|
12
|
+
* Complex multi-step migrations that go beyond simple ALTER TABLE. Spread into
|
|
13
|
+
* the boot migration list so each runs — and is checkpointed — as its own step.
|
|
13
14
|
*/
|
|
14
|
-
export
|
|
15
|
-
migrateJobDeferrals
|
|
16
|
-
migrateToolInvocationsFk
|
|
17
|
-
migrateMemoryEntityRelationDedup
|
|
18
|
-
migrateMemoryItemsFingerprintScopeUnique
|
|
19
|
-
migrateMemoryItemsScopeSaltedFingerprints
|
|
20
|
-
migrateAssistantIdToSelf
|
|
21
|
-
migrateRemoveAssistantIdColumns
|
|
22
|
-
migrateLlmUsageEventsDropAssistantId
|
|
23
|
-
|
|
15
|
+
export const complexMigrationSteps: MigrationStep[] = [
|
|
16
|
+
migrateJobDeferrals,
|
|
17
|
+
migrateToolInvocationsFk,
|
|
18
|
+
migrateMemoryEntityRelationDedup,
|
|
19
|
+
migrateMemoryItemsFingerprintScopeUnique,
|
|
20
|
+
migrateMemoryItemsScopeSaltedFingerprints,
|
|
21
|
+
migrateAssistantIdToSelf,
|
|
22
|
+
migrateRemoveAssistantIdColumns,
|
|
23
|
+
migrateLlmUsageEventsDropAssistantId,
|
|
24
|
+
];
|
|
@@ -95,7 +95,7 @@ export function createCoreIndexes(database: DrizzleDb): void {
|
|
|
95
95
|
/*sql*/ `DROP INDEX IF EXISTS idx_memory_embeddings_provider_model`,
|
|
96
96
|
);
|
|
97
97
|
// Ensure a unique constraint exists on (target_type, target_id, provider, model).
|
|
98
|
-
// New databases get this via the table-level UNIQUE in
|
|
98
|
+
// New databases get this via the table-level UNIQUE in 000-core-tables.ts (autoindex),
|
|
99
99
|
// but for pre-100 databases where CREATE TABLE IF NOT EXISTS was a no-op, the autoindex
|
|
100
100
|
// doesn't exist. Always create the named index — it's a no-op if it already exists and
|
|
101
101
|
// harmless if an autoindex also covers these columns.
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import type { DrizzleDb } from "../db-connection.js";
|
|
2
1
|
import { migrateMemoryFtsBackfill } from "./003-memory-fts-backfill.js";
|
|
3
2
|
import { migrateGuardianActionTables } from "./013-guardian-action-tables.js";
|
|
4
3
|
import { migrateMemorySegmentsIndexes } from "./016-memory-segments-indexes.js";
|
|
@@ -10,21 +9,24 @@ import { migrateAddOriginInterface } from "./022-add-origin-interface.js";
|
|
|
10
9
|
import { migrateMemoryItemSourcesIndexes } from "./023-memory-item-sources-indexes.js";
|
|
11
10
|
import { migrateEmbeddingVectorBlob } from "./024-embedding-vector-blob.js";
|
|
12
11
|
import { migrateEmbeddingsNullableVectorJson } from "./026a-embeddings-nullable-vector-json.js";
|
|
12
|
+
import type { MigrationStep } from "./run-migrations.js";
|
|
13
13
|
|
|
14
14
|
/**
|
|
15
15
|
* Late-stage migrations that must run after all tables and indexes exist:
|
|
16
16
|
* guardian action tables, FTS backfill, index migrations, and channel renames.
|
|
17
|
+
* Spread into the boot migration list so each runs — and is checkpointed — as
|
|
18
|
+
* its own step.
|
|
17
19
|
*/
|
|
18
|
-
export
|
|
19
|
-
migrateGuardianActionTables
|
|
20
|
-
migrateMemoryFtsBackfill
|
|
21
|
-
migrateMemorySegmentsIndexes
|
|
22
|
-
migrateMemoryItemsIndexes
|
|
23
|
-
migrateRemainingTableIndexes
|
|
24
|
-
migrateRenameChannelToVellum
|
|
25
|
-
migrateConversationStatusIndexes
|
|
26
|
-
migrateAddOriginInterface
|
|
27
|
-
migrateMemoryItemSourcesIndexes
|
|
28
|
-
migrateEmbeddingVectorBlob
|
|
29
|
-
migrateEmbeddingsNullableVectorJson
|
|
30
|
-
|
|
20
|
+
export const lateMigrationSteps: MigrationStep[] = [
|
|
21
|
+
migrateGuardianActionTables,
|
|
22
|
+
migrateMemoryFtsBackfill,
|
|
23
|
+
migrateMemorySegmentsIndexes,
|
|
24
|
+
migrateMemoryItemsIndexes,
|
|
25
|
+
migrateRemainingTableIndexes,
|
|
26
|
+
migrateRenameChannelToVellum,
|
|
27
|
+
migrateConversationStatusIndexes,
|
|
28
|
+
migrateAddOriginInterface,
|
|
29
|
+
migrateMemoryItemSourcesIndexes,
|
|
30
|
+
migrateEmbeddingVectorBlob,
|
|
31
|
+
migrateEmbeddingsNullableVectorJson,
|
|
32
|
+
];
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
import {
|
|
2
|
+
type DrizzleDb,
|
|
3
|
+
getSqliteFrom,
|
|
4
|
+
LOGS_DB_SCHEMA,
|
|
5
|
+
} from "../db-connection.js";
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Column names of `llm_request_logs`, in a fixed order used for the
|
|
9
|
+
* cross-database copy. Listed explicitly (rather than `SELECT *`) so the copy
|
|
10
|
+
* is insensitive to the physical column order of the `main` table, which varies
|
|
11
|
+
* with the historical sequence of `ALTER TABLE ... ADD COLUMN` migrations.
|
|
12
|
+
*
|
|
13
|
+
* The first columns are the original base columns; the rest were added by later
|
|
14
|
+
* column migrations.
|
|
15
|
+
*/
|
|
16
|
+
const COLUMN_NAMES = [
|
|
17
|
+
"id",
|
|
18
|
+
"conversation_id",
|
|
19
|
+
"message_id",
|
|
20
|
+
"provider",
|
|
21
|
+
"request_payload",
|
|
22
|
+
"response_payload",
|
|
23
|
+
"created_at",
|
|
24
|
+
"agent_loop_exit_reason",
|
|
25
|
+
"call_site",
|
|
26
|
+
];
|
|
27
|
+
const COLUMNS = COLUMN_NAMES.join(", ");
|
|
28
|
+
|
|
29
|
+
const CREATE_TABLE = (schema: string): string => /*sql*/ `
|
|
30
|
+
CREATE TABLE IF NOT EXISTS ${schema}.llm_request_logs (
|
|
31
|
+
id TEXT PRIMARY KEY,
|
|
32
|
+
conversation_id TEXT NOT NULL,
|
|
33
|
+
message_id TEXT,
|
|
34
|
+
provider TEXT,
|
|
35
|
+
request_payload TEXT NOT NULL,
|
|
36
|
+
response_payload TEXT NOT NULL,
|
|
37
|
+
created_at INTEGER NOT NULL,
|
|
38
|
+
agent_loop_exit_reason TEXT,
|
|
39
|
+
call_site TEXT
|
|
40
|
+
)
|
|
41
|
+
`;
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Create the three indexes in `schema`. The index names are schema-qualified;
|
|
45
|
+
* the table name in `CREATE INDEX` must be unqualified and is resolved within
|
|
46
|
+
* the index's schema — so the table must already resolve to `schema` (i.e. no
|
|
47
|
+
* same-named table shadowing it in `main`) when this runs.
|
|
48
|
+
*/
|
|
49
|
+
function createIndexes(raw: ReturnType<typeof getSqliteFrom>, schema: string) {
|
|
50
|
+
raw.exec(
|
|
51
|
+
`CREATE INDEX IF NOT EXISTS ${schema}.idx_llm_request_logs_message_id ON llm_request_logs(message_id)`,
|
|
52
|
+
);
|
|
53
|
+
raw.exec(
|
|
54
|
+
`CREATE INDEX IF NOT EXISTS ${schema}.idx_llm_request_logs_created_at ON llm_request_logs(created_at)`,
|
|
55
|
+
);
|
|
56
|
+
raw.exec(
|
|
57
|
+
`CREATE INDEX IF NOT EXISTS ${schema}.idx_llm_request_logs_conv_created ON llm_request_logs(conversation_id, created_at)`,
|
|
58
|
+
);
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
function tableExists(
|
|
62
|
+
raw: ReturnType<typeof getSqliteFrom>,
|
|
63
|
+
schema: string,
|
|
64
|
+
): boolean {
|
|
65
|
+
return (
|
|
66
|
+
raw
|
|
67
|
+
.query(
|
|
68
|
+
`SELECT name FROM ${schema}.sqlite_master WHERE type='table' AND name='llm_request_logs'`,
|
|
69
|
+
)
|
|
70
|
+
.get() != null
|
|
71
|
+
);
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* Copy every row of `llm_request_logs` from `main` into the `logs` database,
|
|
76
|
+
* substituting `NULL` for any target column missing in the source. The newer
|
|
77
|
+
* columns (message_id/provider/agent_loop_exit_reason/call_site) are all
|
|
78
|
+
* nullable, so NULL is correct for a legacy row that predates them.
|
|
79
|
+
*/
|
|
80
|
+
function copyRowsFromMain(raw: ReturnType<typeof getSqliteFrom>): void {
|
|
81
|
+
const presentColumns = new Set(
|
|
82
|
+
(
|
|
83
|
+
raw
|
|
84
|
+
.query(`SELECT name FROM pragma_table_info('llm_request_logs', 'main')`)
|
|
85
|
+
.all() as Array<{ name: string }>
|
|
86
|
+
).map((r) => r.name),
|
|
87
|
+
);
|
|
88
|
+
const selectList = COLUMN_NAMES.map((c) =>
|
|
89
|
+
presentColumns.has(c) ? c : "NULL",
|
|
90
|
+
).join(", ");
|
|
91
|
+
raw.exec(/*sql*/ `
|
|
92
|
+
INSERT OR IGNORE INTO ${LOGS_DB_SCHEMA}.llm_request_logs (${COLUMNS})
|
|
93
|
+
SELECT ${selectList} FROM main.llm_request_logs
|
|
94
|
+
`);
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* Keep `llm_request_logs` housed in the attached append-only `logs` database
|
|
99
|
+
* (`assistant-logs.db`) rather than the main DB. Keeping this heavy table — and
|
|
100
|
+
* the request/response payloads it stores — in its own file stops it from
|
|
101
|
+
* bloating the main DB and its WAL, and lets the two files VACUUM and
|
|
102
|
+
* checkpoint independently. Once it lives only in `logs`, the unqualified name
|
|
103
|
+
* used by the Drizzle store resolves to the attached copy, so query code is
|
|
104
|
+
* unchanged.
|
|
105
|
+
*
|
|
106
|
+
* This step is idempotent and runs on every startup (it is not checkpoint-
|
|
107
|
+
* gated). It must, because the earlier `createWatchersAndLogsTables` migration
|
|
108
|
+
* recreates an empty `main.llm_request_logs` on every boot via
|
|
109
|
+
* `CREATE TABLE IF NOT EXISTS`; this step re-relocates and drops that shadow so
|
|
110
|
+
* the unqualified name keeps resolving to `logs`.
|
|
111
|
+
*
|
|
112
|
+
* Ordering within the step matters:
|
|
113
|
+
* 1. Create the table in `logs` (safe whether or not `main` has it).
|
|
114
|
+
* 2. If `main` still has the table, copy its rows over (`INSERT OR IGNORE` on
|
|
115
|
+
* the id PK, so a re-run is a no-op) and drop it.
|
|
116
|
+
* 3. Create the indexes — only now is `main` guaranteed not to shadow the
|
|
117
|
+
* table, so the unqualified reference resolves to `logs`.
|
|
118
|
+
*/
|
|
119
|
+
export function migrateMoveLlmRequestLogsToLogsDb(database: DrizzleDb): void {
|
|
120
|
+
const raw = getSqliteFrom(database);
|
|
121
|
+
|
|
122
|
+
raw.exec(CREATE_TABLE(LOGS_DB_SCHEMA));
|
|
123
|
+
|
|
124
|
+
if (tableExists(raw, "main")) {
|
|
125
|
+
copyRowsFromMain(raw);
|
|
126
|
+
raw.exec(`DROP TABLE main.llm_request_logs`);
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
createIndexes(raw, LOGS_DB_SCHEMA);
|
|
130
|
+
}
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Tests for migration 297 — keeping `llm_request_logs` in the attached
|
|
3
|
+
* `logs` database.
|
|
4
|
+
*
|
|
5
|
+
* Covers:
|
|
6
|
+
* 1. End state after a full initializeDb(): the table lives in `logs`, not
|
|
7
|
+
* `main`, and the Drizzle store round-trips against it.
|
|
8
|
+
* 2. The relocation itself: rows in a `main.llm_request_logs` are copied into
|
|
9
|
+
* `logs` and the main-DB copy is dropped, with indexes built in `logs`.
|
|
10
|
+
* 3. A legacy base-only `main` table (predating the newer columns) is copied
|
|
11
|
+
* without error, NULL-filling the absent columns.
|
|
12
|
+
*
|
|
13
|
+
* The step is idempotent and not checkpoint-gated, so each test can drive it
|
|
14
|
+
* directly without clearing any checkpoint.
|
|
15
|
+
*/
|
|
16
|
+
import { describe, expect, test } from "bun:test";
|
|
17
|
+
|
|
18
|
+
const { getDb, getSqlite, LOGS_DB_SCHEMA } =
|
|
19
|
+
await import("../../db-connection.js");
|
|
20
|
+
const { initializeDb } = await import("../../db-init.js");
|
|
21
|
+
const { migrateMoveLlmRequestLogsToLogsDb } =
|
|
22
|
+
await import("../297-move-llm-request-logs-to-logs-db.js");
|
|
23
|
+
const { recordRequestLog, getRequestLogById } =
|
|
24
|
+
await import("../../llm-request-log-store.js");
|
|
25
|
+
|
|
26
|
+
initializeDb();
|
|
27
|
+
|
|
28
|
+
function tableSchemas(name: string): string[] {
|
|
29
|
+
return getSqlite()
|
|
30
|
+
.query<{ schema: string }, []>(
|
|
31
|
+
`SELECT 'main' AS schema FROM main.sqlite_master WHERE type='table' AND name='${name}'
|
|
32
|
+
UNION ALL
|
|
33
|
+
SELECT '${LOGS_DB_SCHEMA}' FROM ${LOGS_DB_SCHEMA}.sqlite_master WHERE type='table' AND name='${name}'`,
|
|
34
|
+
)
|
|
35
|
+
.all()
|
|
36
|
+
.map((r) => r.schema);
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
describe("migration 297 — llm_request_logs lives in the logs database", () => {
|
|
40
|
+
test("after init, the table is in logs and not in main", () => {
|
|
41
|
+
expect(tableSchemas("llm_request_logs")).toEqual([LOGS_DB_SCHEMA]);
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
test("the store round-trips against the relocated table", () => {
|
|
45
|
+
const id = recordRequestLog(
|
|
46
|
+
"conv-297",
|
|
47
|
+
JSON.stringify({ req: 1 }),
|
|
48
|
+
JSON.stringify({ res: 1 }),
|
|
49
|
+
"msg-297",
|
|
50
|
+
"anthropic",
|
|
51
|
+
"mainAgent",
|
|
52
|
+
);
|
|
53
|
+
const row = getRequestLogById(id);
|
|
54
|
+
expect(row?.conversationId).toBe("conv-297");
|
|
55
|
+
expect(row?.messageId).toBe("msg-297");
|
|
56
|
+
expect(row?.provider).toBe("anthropic");
|
|
57
|
+
expect(row?.callSite).toBe("mainAgent");
|
|
58
|
+
|
|
59
|
+
// The written row physically lives in the logs database.
|
|
60
|
+
const inLogs = getSqlite()
|
|
61
|
+
.query<
|
|
62
|
+
{ c: number },
|
|
63
|
+
[string]
|
|
64
|
+
>(`SELECT COUNT(*) AS c FROM ${LOGS_DB_SCHEMA}.llm_request_logs WHERE id = ?`)
|
|
65
|
+
.get(id);
|
|
66
|
+
expect(inLogs?.c).toBe(1);
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
test("relocates rows from a main-DB table and drops it", () => {
|
|
70
|
+
const sqlite = getSqlite();
|
|
71
|
+
|
|
72
|
+
// Simulate the shadow that createWatchersAndLogsTables recreates each boot:
|
|
73
|
+
// a main-DB table with a row, alongside the logs copy.
|
|
74
|
+
sqlite.exec(`DROP TABLE IF EXISTS ${LOGS_DB_SCHEMA}.llm_request_logs`);
|
|
75
|
+
sqlite.exec(`
|
|
76
|
+
CREATE TABLE main.llm_request_logs (
|
|
77
|
+
id TEXT PRIMARY KEY,
|
|
78
|
+
conversation_id TEXT NOT NULL,
|
|
79
|
+
message_id TEXT,
|
|
80
|
+
provider TEXT,
|
|
81
|
+
request_payload TEXT NOT NULL,
|
|
82
|
+
response_payload TEXT NOT NULL,
|
|
83
|
+
created_at INTEGER NOT NULL,
|
|
84
|
+
agent_loop_exit_reason TEXT,
|
|
85
|
+
call_site TEXT
|
|
86
|
+
)
|
|
87
|
+
`);
|
|
88
|
+
sqlite.exec(
|
|
89
|
+
`INSERT INTO main.llm_request_logs (id, conversation_id, request_payload, response_payload, created_at, provider) VALUES ('legacy-1', 'conv-legacy', '{}', '{}', 123, 'openai')`,
|
|
90
|
+
);
|
|
91
|
+
|
|
92
|
+
migrateMoveLlmRequestLogsToLogsDb(getDb());
|
|
93
|
+
|
|
94
|
+
// Main copy gone, logs copy has the row.
|
|
95
|
+
expect(tableSchemas("llm_request_logs")).toEqual([LOGS_DB_SCHEMA]);
|
|
96
|
+
const moved = sqlite
|
|
97
|
+
.query<
|
|
98
|
+
{ conversation_id: string; provider: string },
|
|
99
|
+
[]
|
|
100
|
+
>(`SELECT conversation_id, provider FROM ${LOGS_DB_SCHEMA}.llm_request_logs WHERE id = 'legacy-1'`)
|
|
101
|
+
.get();
|
|
102
|
+
expect(moved?.conversation_id).toBe("conv-legacy");
|
|
103
|
+
expect(moved?.provider).toBe("openai");
|
|
104
|
+
|
|
105
|
+
// Indexes were created in the logs database.
|
|
106
|
+
const indexes = sqlite
|
|
107
|
+
.query<{ name: string }, []>(
|
|
108
|
+
`SELECT name FROM ${LOGS_DB_SCHEMA}.sqlite_master WHERE type='index' AND tbl_name='llm_request_logs'`,
|
|
109
|
+
)
|
|
110
|
+
.all()
|
|
111
|
+
.map((r) => r.name);
|
|
112
|
+
expect(indexes).toContain("idx_llm_request_logs_message_id");
|
|
113
|
+
expect(indexes).toContain("idx_llm_request_logs_created_at");
|
|
114
|
+
expect(indexes).toContain("idx_llm_request_logs_conv_created");
|
|
115
|
+
});
|
|
116
|
+
|
|
117
|
+
test("copies a legacy base-only table, NULL-filling newer columns", () => {
|
|
118
|
+
const sqlite = getSqlite();
|
|
119
|
+
|
|
120
|
+
// A workspace upgrading from a build that predates the
|
|
121
|
+
// message_id/provider/agent_loop_exit_reason/call_site columns: the main
|
|
122
|
+
// table has only the original base columns.
|
|
123
|
+
sqlite.exec(`DROP TABLE IF EXISTS ${LOGS_DB_SCHEMA}.llm_request_logs`);
|
|
124
|
+
sqlite.exec(`
|
|
125
|
+
CREATE TABLE main.llm_request_logs (
|
|
126
|
+
id TEXT PRIMARY KEY,
|
|
127
|
+
conversation_id TEXT NOT NULL,
|
|
128
|
+
request_payload TEXT NOT NULL,
|
|
129
|
+
response_payload TEXT NOT NULL,
|
|
130
|
+
created_at INTEGER NOT NULL
|
|
131
|
+
)
|
|
132
|
+
`);
|
|
133
|
+
sqlite.exec(
|
|
134
|
+
`INSERT INTO main.llm_request_logs (id, conversation_id, request_payload, response_payload, created_at) VALUES ('base-1', 'conv-base', '{}', '{}', 7)`,
|
|
135
|
+
);
|
|
136
|
+
|
|
137
|
+
// Must not throw on the absent columns.
|
|
138
|
+
migrateMoveLlmRequestLogsToLogsDb(getDb());
|
|
139
|
+
|
|
140
|
+
expect(tableSchemas("llm_request_logs")).toEqual([LOGS_DB_SCHEMA]);
|
|
141
|
+
const moved = sqlite
|
|
142
|
+
.query<
|
|
143
|
+
{
|
|
144
|
+
conversation_id: string;
|
|
145
|
+
message_id: string | null;
|
|
146
|
+
provider: string | null;
|
|
147
|
+
call_site: string | null;
|
|
148
|
+
},
|
|
149
|
+
[]
|
|
150
|
+
>(
|
|
151
|
+
`SELECT conversation_id, message_id, provider, call_site FROM ${LOGS_DB_SCHEMA}.llm_request_logs WHERE id = 'base-1'`,
|
|
152
|
+
)
|
|
153
|
+
.get();
|
|
154
|
+
expect(moved?.conversation_id).toBe("conv-base");
|
|
155
|
+
expect(moved?.message_id).toBeNull();
|
|
156
|
+
expect(moved?.provider).toBeNull();
|
|
157
|
+
expect(moved?.call_site).toBeNull();
|
|
158
|
+
});
|
|
159
|
+
});
|
|
@@ -3,8 +3,13 @@ import { describe, expect, test } from "bun:test";
|
|
|
3
3
|
|
|
4
4
|
import { drizzle } from "drizzle-orm/bun-sqlite";
|
|
5
5
|
|
|
6
|
+
import { getSqliteFrom } from "../../db-connection.js";
|
|
6
7
|
import * as schema from "../../schema.js";
|
|
7
|
-
import {
|
|
8
|
+
import {
|
|
9
|
+
clearMigrationStepCheckpoints,
|
|
10
|
+
type MigrationStep,
|
|
11
|
+
runMigrationSteps,
|
|
12
|
+
} from "../run-migrations.js";
|
|
8
13
|
|
|
9
14
|
function createTestDb() {
|
|
10
15
|
const sqlite = new Database(":memory:");
|
|
@@ -14,10 +19,7 @@ function createTestDb() {
|
|
|
14
19
|
}
|
|
15
20
|
|
|
16
21
|
describe("runMigrationSteps — checkpointing", () => {
|
|
17
|
-
|
|
18
|
-
// current behavior, which re-runs every forward step on every boot. Promote
|
|
19
|
-
// to `test` when true checkpointing of initial migrations lands.
|
|
20
|
-
test.skip("does not re-execute a step that was already applied on a prior run", () => {
|
|
22
|
+
test("does not re-execute a step that was already applied on a prior run", () => {
|
|
21
23
|
/**
|
|
22
24
|
* Booting an already-migrated database must not re-run forward migration
|
|
23
25
|
* steps. With true checkpointing each step's body executes at most once
|
|
@@ -49,4 +51,144 @@ describe("runMigrationSteps — checkpointing", () => {
|
|
|
49
51
|
// THEN no step body executes a second time
|
|
50
52
|
expect(calls).toEqual({ a: 1, b: 1, c: 1 });
|
|
51
53
|
});
|
|
54
|
+
|
|
55
|
+
test("records step completions in the shared memory_checkpoints ledger", () => {
|
|
56
|
+
/**
|
|
57
|
+
* Step bookkeeping lives in the same memory_checkpoints table the registry
|
|
58
|
+
* uses, under the `step:` namespace — one ledger for all applied state.
|
|
59
|
+
*/
|
|
60
|
+
|
|
61
|
+
// GIVEN a single named step
|
|
62
|
+
const db = createTestDb();
|
|
63
|
+
runMigrationSteps(db, [
|
|
64
|
+
function dummyMigrationA() {
|
|
65
|
+
// no-op
|
|
66
|
+
},
|
|
67
|
+
]);
|
|
68
|
+
|
|
69
|
+
// THEN its completion is recorded under the step: namespace in memory_checkpoints
|
|
70
|
+
const row = getSqliteFrom(db)
|
|
71
|
+
.query(
|
|
72
|
+
`SELECT value FROM memory_checkpoints WHERE key = 'step:dummyMigrationA'`,
|
|
73
|
+
)
|
|
74
|
+
.get() as { value: string } | null;
|
|
75
|
+
expect(row?.value).toBe("1");
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
test("runs a newly appended step on a later boot while skipping applied ones", () => {
|
|
79
|
+
/**
|
|
80
|
+
* Checkpointing must not block migrations added in a later release: an
|
|
81
|
+
* already-applied step is skipped, but a brand-new step still runs.
|
|
82
|
+
*/
|
|
83
|
+
|
|
84
|
+
// GIVEN a database already migrated with two steps
|
|
85
|
+
const calls = { a: 0, b: 0, c: 0 };
|
|
86
|
+
const stepA: MigrationStep = function dummyMigrationA() {
|
|
87
|
+
calls.a++;
|
|
88
|
+
};
|
|
89
|
+
const stepB: MigrationStep = function dummyMigrationB() {
|
|
90
|
+
calls.b++;
|
|
91
|
+
};
|
|
92
|
+
const db = createTestDb();
|
|
93
|
+
runMigrationSteps(db, [stepA, stepB]);
|
|
94
|
+
|
|
95
|
+
// AND a third step appended to the list
|
|
96
|
+
const stepC: MigrationStep = function dummyMigrationC() {
|
|
97
|
+
calls.c++;
|
|
98
|
+
};
|
|
99
|
+
|
|
100
|
+
// WHEN the expanded list runs against the already-migrated database
|
|
101
|
+
runMigrationSteps(db, [stepA, stepB, stepC]);
|
|
102
|
+
|
|
103
|
+
// THEN only the newly appended step executes
|
|
104
|
+
expect(calls).toEqual({ a: 1, b: 1, c: 1 });
|
|
105
|
+
});
|
|
106
|
+
|
|
107
|
+
test("recovers crashed migrations before running steps", () => {
|
|
108
|
+
/**
|
|
109
|
+
* runMigrationSteps clears stalled `started`/`rolling_back` checkpoints
|
|
110
|
+
* before the loop so a migration interrupted mid-flight re-runs this boot.
|
|
111
|
+
*/
|
|
112
|
+
|
|
113
|
+
// GIVEN a database with a stalled registry checkpoint left by a crash
|
|
114
|
+
const db = createTestDb();
|
|
115
|
+
const raw = getSqliteFrom(db);
|
|
116
|
+
raw.run(
|
|
117
|
+
`CREATE TABLE IF NOT EXISTS memory_checkpoints (key TEXT PRIMARY KEY, value TEXT NOT NULL, updated_at INTEGER NOT NULL)`,
|
|
118
|
+
);
|
|
119
|
+
raw.run(
|
|
120
|
+
`INSERT INTO memory_checkpoints (key, value, updated_at) VALUES ('migration_stalled_v1', 'started', 0)`,
|
|
121
|
+
);
|
|
122
|
+
|
|
123
|
+
// WHEN the runner executes
|
|
124
|
+
runMigrationSteps(db, [
|
|
125
|
+
function dummyMigrationA() {
|
|
126
|
+
// no-op
|
|
127
|
+
},
|
|
128
|
+
]);
|
|
129
|
+
|
|
130
|
+
// THEN the stalled checkpoint has been cleared so its migration can re-run
|
|
131
|
+
const row = raw
|
|
132
|
+
.query(
|
|
133
|
+
`SELECT 1 FROM memory_checkpoints WHERE key = 'migration_stalled_v1'`,
|
|
134
|
+
)
|
|
135
|
+
.get();
|
|
136
|
+
expect(row).toBeNull();
|
|
137
|
+
});
|
|
138
|
+
|
|
139
|
+
test("does not checkpoint a failed step so it retries on the next boot", () => {
|
|
140
|
+
/**
|
|
141
|
+
* A step whose body throws is reported in `failed` and left uncheckpointed,
|
|
142
|
+
* so the next boot retries it instead of silently skipping it.
|
|
143
|
+
*/
|
|
144
|
+
|
|
145
|
+
// GIVEN a step that throws on its first run and succeeds afterwards
|
|
146
|
+
const calls = { flaky: 0 };
|
|
147
|
+
const steps: MigrationStep[] = [
|
|
148
|
+
function flakyStep() {
|
|
149
|
+
calls.flaky++;
|
|
150
|
+
if (calls.flaky === 1) {
|
|
151
|
+
throw new Error("transient failure");
|
|
152
|
+
}
|
|
153
|
+
},
|
|
154
|
+
];
|
|
155
|
+
const db = createTestDb();
|
|
156
|
+
const first = runMigrationSteps(db, steps);
|
|
157
|
+
|
|
158
|
+
// AND the first run reports the failure
|
|
159
|
+
expect(first.failed).toEqual(["flakyStep"]);
|
|
160
|
+
|
|
161
|
+
// WHEN the step runs again on the next boot
|
|
162
|
+
const second = runMigrationSteps(db, steps);
|
|
163
|
+
|
|
164
|
+
// THEN it retries, succeeds, and is then checkpointed
|
|
165
|
+
expect(calls.flaky).toBe(2);
|
|
166
|
+
expect(second.failed).toEqual([]);
|
|
167
|
+
expect(runMigrationSteps(db, steps).skipped).toEqual(["flakyStep"]);
|
|
168
|
+
});
|
|
169
|
+
|
|
170
|
+
test("clearMigrationStepCheckpoints forces every step to re-run", () => {
|
|
171
|
+
/**
|
|
172
|
+
* Clearing the step namespace (as rollback does) makes the next run
|
|
173
|
+
* re-execute and re-record all steps.
|
|
174
|
+
*/
|
|
175
|
+
|
|
176
|
+
// GIVEN a database migrated once
|
|
177
|
+
const calls = { a: 0 };
|
|
178
|
+
const steps: MigrationStep[] = [
|
|
179
|
+
function dummyMigrationA() {
|
|
180
|
+
calls.a++;
|
|
181
|
+
},
|
|
182
|
+
];
|
|
183
|
+
const db = createTestDb();
|
|
184
|
+
runMigrationSteps(db, steps);
|
|
185
|
+
expect(calls.a).toBe(1);
|
|
186
|
+
|
|
187
|
+
// WHEN the step checkpoints are cleared
|
|
188
|
+
clearMigrationStepCheckpoints(db);
|
|
189
|
+
|
|
190
|
+
// THEN the next run re-executes the step
|
|
191
|
+
runMigrationSteps(db, steps);
|
|
192
|
+
expect(calls.a).toBe(2);
|
|
193
|
+
});
|
|
52
194
|
});
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
export { migrateCoreTables } from "./000-core-tables.js";
|
|
1
2
|
export { migrateJobDeferrals } from "./001-job-deferrals.js";
|
|
2
3
|
export { migrateToolInvocationsFk } from "./002-tool-invocations-fk.js";
|
|
3
4
|
export { migrateMemoryFtsBackfill } from "./003-memory-fts-backfill.js";
|
|
@@ -43,10 +44,9 @@ export { createActorTokenRecordsTable } from "./038-actor-token-records.js";
|
|
|
43
44
|
export { createActorRefreshTokenRecordsTable } from "./039-actor-refresh-token-records.js";
|
|
44
45
|
export { migrateInviteCodeHashColumn } from "./040-invite-code-hash-column.js";
|
|
45
46
|
export { createApprovalPromptTsTrackerTable } from "./041-approval-prompt-ts-tracker.js";
|
|
46
|
-
export { createCoreTables } from "./100-core-tables.js";
|
|
47
47
|
export { createWatchersAndLogsTables } from "./101-watchers-and-logs.js";
|
|
48
48
|
export { addCoreColumns } from "./102-alter-table-columns.js";
|
|
49
|
-
export {
|
|
49
|
+
export { complexMigrationSteps } from "./103-complex-migrations.js";
|
|
50
50
|
export { createCoreIndexes } from "./104-core-indexes.js";
|
|
51
51
|
export { createContactsAndTriageTables } from "./105-contacts-and-triage.js";
|
|
52
52
|
export { createCallSessionsTables } from "./106-call-sessions.js";
|
|
@@ -56,7 +56,7 @@ export { createExternalConversationBindingsTables } from "./109-external-convers
|
|
|
56
56
|
export { createChannelGuardianTables } from "./110-channel-guardian.js";
|
|
57
57
|
export { createMediaAssetsTables } from "./111-media-assets.js";
|
|
58
58
|
export { createAssistantInboxTables } from "./112-assistant-inbox.js";
|
|
59
|
-
export {
|
|
59
|
+
export { lateMigrationSteps } from "./113-late-migrations.js";
|
|
60
60
|
export { createNotificationTables } from "./114-notifications.js";
|
|
61
61
|
export { createSequenceTables } from "./115-sequences.js";
|
|
62
62
|
export { createMessagesFts } from "./116-messages-fts.js";
|
|
@@ -288,13 +288,14 @@ export { migrateWorkflowJournalLeafTokens } from "./293-workflow-journal-leaf-to
|
|
|
288
288
|
export { migrateDropExternalUserId } from "./294-drop-external-user-id.js";
|
|
289
289
|
export { dropApprovalPromptTsTrackerTable } from "./295-drop-approval-prompt-ts-tracker.js";
|
|
290
290
|
export { migrateRewriteBalancedEconomyProfilePins } from "./296-rewrite-balanced-economy-profile-pins.js";
|
|
291
|
+
export { migrateMoveLlmRequestLogsToLogsDb } from "./297-move-llm-request-logs-to-logs-db.js";
|
|
291
292
|
export {
|
|
292
293
|
MIGRATION_REGISTRY,
|
|
293
294
|
type MigrationRegistryEntry,
|
|
294
295
|
type MigrationValidationResult,
|
|
295
296
|
} from "./registry.js";
|
|
297
|
+
export { recoverCrashedMigrations } from "./run-migrations.js";
|
|
296
298
|
export {
|
|
297
|
-
recoverCrashedMigrations,
|
|
298
299
|
rollbackMemoryMigration,
|
|
299
300
|
validateMigrationState,
|
|
300
301
|
withCrashRecovery,
|