@vellumai/assistant 0.10.0-dev.202606200539.c052d10 → 0.10.0-dev.202606201539.0cebd50

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. package/package.json +1 -1
  2. package/src/__tests__/agent-loop-callsite-precedence.test.ts +1 -40
  3. package/src/__tests__/agent-wake-override-profile.test.ts +2 -0
  4. package/src/__tests__/app-source-watcher.test.ts +30 -10
  5. package/src/__tests__/config-schema.test.ts +34 -0
  6. package/src/__tests__/conversation-agent-loop-disk-pressure.test.ts +3 -0
  7. package/src/__tests__/conversation-agent-loop-inference-profile.test.ts +3 -0
  8. package/src/__tests__/conversation-agent-loop-overflow.test.ts +3 -0
  9. package/src/__tests__/conversation-agent-loop.test.ts +3 -0
  10. package/src/__tests__/conversation-process-callsite.test.ts +0 -14
  11. package/src/__tests__/db-llm-request-log-provider-migration.test.ts +6 -1
  12. package/src/__tests__/db-migration-rollback.test.ts +69 -1
  13. package/src/__tests__/heartbeat-disk-pressure.test.ts +3 -0
  14. package/src/__tests__/heartbeat-service.test.ts +6 -0
  15. package/src/__tests__/list-messages-attachments.test.ts +41 -0
  16. package/src/__tests__/plugin-source-watcher.test.ts +33 -1
  17. package/src/__tests__/usage-cache-backfill-migration.test.ts +17 -2
  18. package/src/acp/__tests__/session-manager.test.ts +72 -1
  19. package/src/acp/index.ts +10 -0
  20. package/src/acp/session-manager.ts +35 -0
  21. package/src/agent/loop.ts +28 -22
  22. package/src/cli/commands/db/status.ts +35 -1
  23. package/src/config/schemas/memory-lifecycle.ts +5 -3
  24. package/src/config/schemas/timeouts.ts +24 -0
  25. package/src/daemon/app-source-watcher.ts +31 -18
  26. package/src/daemon/conversation-agent-loop.ts +8 -5
  27. package/src/daemon/conversation.ts +30 -41
  28. package/src/daemon/handlers/conversations.ts +7 -0
  29. package/src/daemon/plugin-source-watcher.ts +5 -0
  30. package/src/daemon/workspace-tools-watcher.ts +4 -0
  31. package/src/heartbeat/__tests__/heartbeat-service.test.ts +6 -0
  32. package/src/heartbeat/heartbeat-service.ts +3 -4
  33. package/src/memory/__tests__/db-maintenance.test.ts +27 -35
  34. package/src/memory/__tests__/db-memory-attach.test.ts +126 -0
  35. package/src/memory/conversation-crud.ts +9 -3
  36. package/src/memory/db-connection.ts +49 -0
  37. package/src/memory/db-init.ts +50 -14
  38. package/src/memory/db-maintenance.ts +43 -38
  39. package/src/memory/job-handlers/cleanup.ts +6 -0
  40. package/src/memory/migrations/{100-core-tables.ts → 000-core-tables.ts} +6 -10
  41. package/src/memory/migrations/103-complex-migrations.ts +13 -12
  42. package/src/memory/migrations/104-core-indexes.ts +1 -1
  43. package/src/memory/migrations/113-late-migrations.ts +16 -14
  44. package/src/memory/migrations/297-move-llm-request-logs-to-logs-db.ts +130 -0
  45. package/src/memory/migrations/__tests__/297-move-llm-request-logs.test.ts +159 -0
  46. package/src/memory/migrations/__tests__/run-migrations.test.ts +147 -5
  47. package/src/memory/migrations/index.ts +5 -4
  48. package/src/memory/migrations/run-migrations.ts +162 -12
  49. package/src/memory/migrations/validate-migration-state.ts +14 -56
  50. package/src/plugin-api/index.ts +7 -0
  51. package/src/plugin-api/vision-support.ts +75 -0
  52. package/src/prompts/system-prompt.ts +1 -1
  53. package/src/runtime/__tests__/agent-wake.test.ts +6 -4
  54. package/src/runtime/agent-wake.ts +15 -7
  55. package/src/runtime/routes/conversation-routes.ts +24 -3
  56. package/src/runtime/routes/migration-routes.ts +35 -39
  57. package/src/schedule/scheduler.ts +5 -9
  58. package/src/tools/ask-question/ask-question-tool.test.ts +60 -52
  59. package/src/tools/ask-question/ask-question-tool.ts +14 -73
  60. package/src/util/fs-watcher-error.ts +36 -0
  61. package/src/util/memory-db-path.ts +23 -0
@@ -1,4 +1,3 @@
1
- import type { DrizzleDb } from "../db-connection.js";
2
1
  import { migrateJobDeferrals } from "./001-job-deferrals.js";
3
2
  import { migrateToolInvocationsFk } from "./002-tool-invocations-fk.js";
4
3
  import { migrateMemoryEntityRelationDedup } from "./004-entity-relation-dedup.js";
@@ -7,17 +6,19 @@ import { migrateMemoryItemsScopeSaltedFingerprints } from "./006-scope-salted-fi
7
6
  import { migrateAssistantIdToSelf } from "./007-assistant-id-to-self.js";
8
7
  import { migrateRemoveAssistantIdColumns } from "./008-remove-assistant-id-columns.js";
9
8
  import { migrateLlmUsageEventsDropAssistantId } from "./009-llm-usage-events-drop-assistant-id.js";
9
+ import type { MigrationStep } from "./run-migrations.js";
10
10
 
11
11
  /**
12
- * Complex multi-step migrations that go beyond simple ALTER TABLE.
12
+ * Complex multi-step migrations that go beyond simple ALTER TABLE. Spread into
13
+ * the boot migration list so each runs — and is checkpointed — as its own step.
13
14
  */
14
- export function runComplexMigrations(database: DrizzleDb): void {
15
- migrateJobDeferrals(database);
16
- migrateToolInvocationsFk(database);
17
- migrateMemoryEntityRelationDedup(database);
18
- migrateMemoryItemsFingerprintScopeUnique(database);
19
- migrateMemoryItemsScopeSaltedFingerprints(database);
20
- migrateAssistantIdToSelf(database);
21
- migrateRemoveAssistantIdColumns(database);
22
- migrateLlmUsageEventsDropAssistantId(database);
23
- }
15
+ export const complexMigrationSteps: MigrationStep[] = [
16
+ migrateJobDeferrals,
17
+ migrateToolInvocationsFk,
18
+ migrateMemoryEntityRelationDedup,
19
+ migrateMemoryItemsFingerprintScopeUnique,
20
+ migrateMemoryItemsScopeSaltedFingerprints,
21
+ migrateAssistantIdToSelf,
22
+ migrateRemoveAssistantIdColumns,
23
+ migrateLlmUsageEventsDropAssistantId,
24
+ ];
@@ -95,7 +95,7 @@ export function createCoreIndexes(database: DrizzleDb): void {
95
95
  /*sql*/ `DROP INDEX IF EXISTS idx_memory_embeddings_provider_model`,
96
96
  );
97
97
  // Ensure a unique constraint exists on (target_type, target_id, provider, model).
98
- // New databases get this via the table-level UNIQUE in 100-core-tables.ts (autoindex),
98
+ // New databases get this via the table-level UNIQUE in 000-core-tables.ts (autoindex),
99
99
  // but for pre-100 databases where CREATE TABLE IF NOT EXISTS was a no-op, the autoindex
100
100
  // doesn't exist. Always create the named index — it's a no-op if it already exists and
101
101
  // harmless if an autoindex also covers these columns.
@@ -1,4 +1,3 @@
1
- import type { DrizzleDb } from "../db-connection.js";
2
1
  import { migrateMemoryFtsBackfill } from "./003-memory-fts-backfill.js";
3
2
  import { migrateGuardianActionTables } from "./013-guardian-action-tables.js";
4
3
  import { migrateMemorySegmentsIndexes } from "./016-memory-segments-indexes.js";
@@ -10,21 +9,24 @@ import { migrateAddOriginInterface } from "./022-add-origin-interface.js";
10
9
  import { migrateMemoryItemSourcesIndexes } from "./023-memory-item-sources-indexes.js";
11
10
  import { migrateEmbeddingVectorBlob } from "./024-embedding-vector-blob.js";
12
11
  import { migrateEmbeddingsNullableVectorJson } from "./026a-embeddings-nullable-vector-json.js";
12
+ import type { MigrationStep } from "./run-migrations.js";
13
13
 
14
14
  /**
15
15
  * Late-stage migrations that must run after all tables and indexes exist:
16
16
  * guardian action tables, FTS backfill, index migrations, and channel renames.
17
+ * Spread into the boot migration list so each runs — and is checkpointed — as
18
+ * its own step.
17
19
  */
18
- export function runLateMigrations(database: DrizzleDb): void {
19
- migrateGuardianActionTables(database);
20
- migrateMemoryFtsBackfill(database);
21
- migrateMemorySegmentsIndexes(database);
22
- migrateMemoryItemsIndexes(database);
23
- migrateRemainingTableIndexes(database);
24
- migrateRenameChannelToVellum(database);
25
- migrateConversationStatusIndexes(database);
26
- migrateAddOriginInterface(database);
27
- migrateMemoryItemSourcesIndexes(database);
28
- migrateEmbeddingVectorBlob(database);
29
- migrateEmbeddingsNullableVectorJson(database);
30
- }
20
+ export const lateMigrationSteps: MigrationStep[] = [
21
+ migrateGuardianActionTables,
22
+ migrateMemoryFtsBackfill,
23
+ migrateMemorySegmentsIndexes,
24
+ migrateMemoryItemsIndexes,
25
+ migrateRemainingTableIndexes,
26
+ migrateRenameChannelToVellum,
27
+ migrateConversationStatusIndexes,
28
+ migrateAddOriginInterface,
29
+ migrateMemoryItemSourcesIndexes,
30
+ migrateEmbeddingVectorBlob,
31
+ migrateEmbeddingsNullableVectorJson,
32
+ ];
@@ -0,0 +1,130 @@
1
+ import {
2
+ type DrizzleDb,
3
+ getSqliteFrom,
4
+ LOGS_DB_SCHEMA,
5
+ } from "../db-connection.js";
6
+
7
+ /**
8
+ * Column names of `llm_request_logs`, in a fixed order used for the
9
+ * cross-database copy. Listed explicitly (rather than `SELECT *`) so the copy
10
+ * is insensitive to the physical column order of the `main` table, which varies
11
+ * with the historical sequence of `ALTER TABLE ... ADD COLUMN` migrations.
12
+ *
13
+ * The first columns are the original base columns; the rest were added by later
14
+ * column migrations.
15
+ */
16
+ const COLUMN_NAMES = [
17
+ "id",
18
+ "conversation_id",
19
+ "message_id",
20
+ "provider",
21
+ "request_payload",
22
+ "response_payload",
23
+ "created_at",
24
+ "agent_loop_exit_reason",
25
+ "call_site",
26
+ ];
27
+ const COLUMNS = COLUMN_NAMES.join(", ");
28
+
29
+ const CREATE_TABLE = (schema: string): string => /*sql*/ `
30
+ CREATE TABLE IF NOT EXISTS ${schema}.llm_request_logs (
31
+ id TEXT PRIMARY KEY,
32
+ conversation_id TEXT NOT NULL,
33
+ message_id TEXT,
34
+ provider TEXT,
35
+ request_payload TEXT NOT NULL,
36
+ response_payload TEXT NOT NULL,
37
+ created_at INTEGER NOT NULL,
38
+ agent_loop_exit_reason TEXT,
39
+ call_site TEXT
40
+ )
41
+ `;
42
+
43
+ /**
44
+ * Create the three indexes in `schema`. The index names are schema-qualified;
45
+ * the table name in `CREATE INDEX` must be unqualified and is resolved within
46
+ * the index's schema — so the table must already resolve to `schema` (i.e. no
47
+ * same-named table shadowing it in `main`) when this runs.
48
+ */
49
+ function createIndexes(raw: ReturnType<typeof getSqliteFrom>, schema: string) {
50
+ raw.exec(
51
+ `CREATE INDEX IF NOT EXISTS ${schema}.idx_llm_request_logs_message_id ON llm_request_logs(message_id)`,
52
+ );
53
+ raw.exec(
54
+ `CREATE INDEX IF NOT EXISTS ${schema}.idx_llm_request_logs_created_at ON llm_request_logs(created_at)`,
55
+ );
56
+ raw.exec(
57
+ `CREATE INDEX IF NOT EXISTS ${schema}.idx_llm_request_logs_conv_created ON llm_request_logs(conversation_id, created_at)`,
58
+ );
59
+ }
60
+
61
+ function tableExists(
62
+ raw: ReturnType<typeof getSqliteFrom>,
63
+ schema: string,
64
+ ): boolean {
65
+ return (
66
+ raw
67
+ .query(
68
+ `SELECT name FROM ${schema}.sqlite_master WHERE type='table' AND name='llm_request_logs'`,
69
+ )
70
+ .get() != null
71
+ );
72
+ }
73
+
74
+ /**
75
+ * Copy every row of `llm_request_logs` from `main` into the `logs` database,
76
+ * substituting `NULL` for any target column missing in the source. The newer
77
+ * columns (message_id/provider/agent_loop_exit_reason/call_site) are all
78
+ * nullable, so NULL is correct for a legacy row that predates them.
79
+ */
80
+ function copyRowsFromMain(raw: ReturnType<typeof getSqliteFrom>): void {
81
+ const presentColumns = new Set(
82
+ (
83
+ raw
84
+ .query(`SELECT name FROM pragma_table_info('llm_request_logs', 'main')`)
85
+ .all() as Array<{ name: string }>
86
+ ).map((r) => r.name),
87
+ );
88
+ const selectList = COLUMN_NAMES.map((c) =>
89
+ presentColumns.has(c) ? c : "NULL",
90
+ ).join(", ");
91
+ raw.exec(/*sql*/ `
92
+ INSERT OR IGNORE INTO ${LOGS_DB_SCHEMA}.llm_request_logs (${COLUMNS})
93
+ SELECT ${selectList} FROM main.llm_request_logs
94
+ `);
95
+ }
96
+
97
+ /**
98
+ * Keep `llm_request_logs` housed in the attached append-only `logs` database
99
+ * (`assistant-logs.db`) rather than the main DB. Keeping this heavy table — and
100
+ * the request/response payloads it stores — in its own file stops it from
101
+ * bloating the main DB and its WAL, and lets the two files VACUUM and
102
+ * checkpoint independently. Once it lives only in `logs`, the unqualified name
103
+ * used by the Drizzle store resolves to the attached copy, so query code is
104
+ * unchanged.
105
+ *
106
+ * This step is idempotent and runs on every startup (it is not checkpoint-
107
+ * gated). It must, because the earlier `createWatchersAndLogsTables` migration
108
+ * recreates an empty `main.llm_request_logs` on every boot via
109
+ * `CREATE TABLE IF NOT EXISTS`; this step re-relocates and drops that shadow so
110
+ * the unqualified name keeps resolving to `logs`.
111
+ *
112
+ * Ordering within the step matters:
113
+ * 1. Create the table in `logs` (safe whether or not `main` has it).
114
+ * 2. If `main` still has the table, copy its rows over (`INSERT OR IGNORE` on
115
+ * the id PK, so a re-run is a no-op) and drop it.
116
+ * 3. Create the indexes — only now is `main` guaranteed not to shadow the
117
+ * table, so the unqualified reference resolves to `logs`.
118
+ */
119
+ export function migrateMoveLlmRequestLogsToLogsDb(database: DrizzleDb): void {
120
+ const raw = getSqliteFrom(database);
121
+
122
+ raw.exec(CREATE_TABLE(LOGS_DB_SCHEMA));
123
+
124
+ if (tableExists(raw, "main")) {
125
+ copyRowsFromMain(raw);
126
+ raw.exec(`DROP TABLE main.llm_request_logs`);
127
+ }
128
+
129
+ createIndexes(raw, LOGS_DB_SCHEMA);
130
+ }
@@ -0,0 +1,159 @@
1
+ /**
2
+ * Tests for migration 297 — keeping `llm_request_logs` in the attached
3
+ * `logs` database.
4
+ *
5
+ * Covers:
6
+ * 1. End state after a full initializeDb(): the table lives in `logs`, not
7
+ * `main`, and the Drizzle store round-trips against it.
8
+ * 2. The relocation itself: rows in a `main.llm_request_logs` are copied into
9
+ * `logs` and the main-DB copy is dropped, with indexes built in `logs`.
10
+ * 3. A legacy base-only `main` table (predating the newer columns) is copied
11
+ * without error, NULL-filling the absent columns.
12
+ *
13
+ * The step is idempotent and not checkpoint-gated, so each test can drive it
14
+ * directly without clearing any checkpoint.
15
+ */
16
+ import { describe, expect, test } from "bun:test";
17
+
18
+ const { getDb, getSqlite, LOGS_DB_SCHEMA } =
19
+ await import("../../db-connection.js");
20
+ const { initializeDb } = await import("../../db-init.js");
21
+ const { migrateMoveLlmRequestLogsToLogsDb } =
22
+ await import("../297-move-llm-request-logs-to-logs-db.js");
23
+ const { recordRequestLog, getRequestLogById } =
24
+ await import("../../llm-request-log-store.js");
25
+
26
+ initializeDb();
27
+
28
+ function tableSchemas(name: string): string[] {
29
+ return getSqlite()
30
+ .query<{ schema: string }, []>(
31
+ `SELECT 'main' AS schema FROM main.sqlite_master WHERE type='table' AND name='${name}'
32
+ UNION ALL
33
+ SELECT '${LOGS_DB_SCHEMA}' FROM ${LOGS_DB_SCHEMA}.sqlite_master WHERE type='table' AND name='${name}'`,
34
+ )
35
+ .all()
36
+ .map((r) => r.schema);
37
+ }
38
+
39
+ describe("migration 297 — llm_request_logs lives in the logs database", () => {
40
+ test("after init, the table is in logs and not in main", () => {
41
+ expect(tableSchemas("llm_request_logs")).toEqual([LOGS_DB_SCHEMA]);
42
+ });
43
+
44
+ test("the store round-trips against the relocated table", () => {
45
+ const id = recordRequestLog(
46
+ "conv-297",
47
+ JSON.stringify({ req: 1 }),
48
+ JSON.stringify({ res: 1 }),
49
+ "msg-297",
50
+ "anthropic",
51
+ "mainAgent",
52
+ );
53
+ const row = getRequestLogById(id);
54
+ expect(row?.conversationId).toBe("conv-297");
55
+ expect(row?.messageId).toBe("msg-297");
56
+ expect(row?.provider).toBe("anthropic");
57
+ expect(row?.callSite).toBe("mainAgent");
58
+
59
+ // The written row physically lives in the logs database.
60
+ const inLogs = getSqlite()
61
+ .query<
62
+ { c: number },
63
+ [string]
64
+ >(`SELECT COUNT(*) AS c FROM ${LOGS_DB_SCHEMA}.llm_request_logs WHERE id = ?`)
65
+ .get(id);
66
+ expect(inLogs?.c).toBe(1);
67
+ });
68
+
69
+ test("relocates rows from a main-DB table and drops it", () => {
70
+ const sqlite = getSqlite();
71
+
72
+ // Simulate the shadow that createWatchersAndLogsTables recreates each boot:
73
+ // a main-DB table with a row, alongside the logs copy.
74
+ sqlite.exec(`DROP TABLE IF EXISTS ${LOGS_DB_SCHEMA}.llm_request_logs`);
75
+ sqlite.exec(`
76
+ CREATE TABLE main.llm_request_logs (
77
+ id TEXT PRIMARY KEY,
78
+ conversation_id TEXT NOT NULL,
79
+ message_id TEXT,
80
+ provider TEXT,
81
+ request_payload TEXT NOT NULL,
82
+ response_payload TEXT NOT NULL,
83
+ created_at INTEGER NOT NULL,
84
+ agent_loop_exit_reason TEXT,
85
+ call_site TEXT
86
+ )
87
+ `);
88
+ sqlite.exec(
89
+ `INSERT INTO main.llm_request_logs (id, conversation_id, request_payload, response_payload, created_at, provider) VALUES ('legacy-1', 'conv-legacy', '{}', '{}', 123, 'openai')`,
90
+ );
91
+
92
+ migrateMoveLlmRequestLogsToLogsDb(getDb());
93
+
94
+ // Main copy gone, logs copy has the row.
95
+ expect(tableSchemas("llm_request_logs")).toEqual([LOGS_DB_SCHEMA]);
96
+ const moved = sqlite
97
+ .query<
98
+ { conversation_id: string; provider: string },
99
+ []
100
+ >(`SELECT conversation_id, provider FROM ${LOGS_DB_SCHEMA}.llm_request_logs WHERE id = 'legacy-1'`)
101
+ .get();
102
+ expect(moved?.conversation_id).toBe("conv-legacy");
103
+ expect(moved?.provider).toBe("openai");
104
+
105
+ // Indexes were created in the logs database.
106
+ const indexes = sqlite
107
+ .query<{ name: string }, []>(
108
+ `SELECT name FROM ${LOGS_DB_SCHEMA}.sqlite_master WHERE type='index' AND tbl_name='llm_request_logs'`,
109
+ )
110
+ .all()
111
+ .map((r) => r.name);
112
+ expect(indexes).toContain("idx_llm_request_logs_message_id");
113
+ expect(indexes).toContain("idx_llm_request_logs_created_at");
114
+ expect(indexes).toContain("idx_llm_request_logs_conv_created");
115
+ });
116
+
117
+ test("copies a legacy base-only table, NULL-filling newer columns", () => {
118
+ const sqlite = getSqlite();
119
+
120
+ // A workspace upgrading from a build that predates the
121
+ // message_id/provider/agent_loop_exit_reason/call_site columns: the main
122
+ // table has only the original base columns.
123
+ sqlite.exec(`DROP TABLE IF EXISTS ${LOGS_DB_SCHEMA}.llm_request_logs`);
124
+ sqlite.exec(`
125
+ CREATE TABLE main.llm_request_logs (
126
+ id TEXT PRIMARY KEY,
127
+ conversation_id TEXT NOT NULL,
128
+ request_payload TEXT NOT NULL,
129
+ response_payload TEXT NOT NULL,
130
+ created_at INTEGER NOT NULL
131
+ )
132
+ `);
133
+ sqlite.exec(
134
+ `INSERT INTO main.llm_request_logs (id, conversation_id, request_payload, response_payload, created_at) VALUES ('base-1', 'conv-base', '{}', '{}', 7)`,
135
+ );
136
+
137
+ // Must not throw on the absent columns.
138
+ migrateMoveLlmRequestLogsToLogsDb(getDb());
139
+
140
+ expect(tableSchemas("llm_request_logs")).toEqual([LOGS_DB_SCHEMA]);
141
+ const moved = sqlite
142
+ .query<
143
+ {
144
+ conversation_id: string;
145
+ message_id: string | null;
146
+ provider: string | null;
147
+ call_site: string | null;
148
+ },
149
+ []
150
+ >(
151
+ `SELECT conversation_id, message_id, provider, call_site FROM ${LOGS_DB_SCHEMA}.llm_request_logs WHERE id = 'base-1'`,
152
+ )
153
+ .get();
154
+ expect(moved?.conversation_id).toBe("conv-base");
155
+ expect(moved?.message_id).toBeNull();
156
+ expect(moved?.provider).toBeNull();
157
+ expect(moved?.call_site).toBeNull();
158
+ });
159
+ });
@@ -3,8 +3,13 @@ import { describe, expect, test } from "bun:test";
3
3
 
4
4
  import { drizzle } from "drizzle-orm/bun-sqlite";
5
5
 
6
+ import { getSqliteFrom } from "../../db-connection.js";
6
7
  import * as schema from "../../schema.js";
7
- import { type MigrationStep, runMigrationSteps } from "../run-migrations.js";
8
+ import {
9
+ clearMigrationStepCheckpoints,
10
+ type MigrationStep,
11
+ runMigrationSteps,
12
+ } from "../run-migrations.js";
8
13
 
9
14
  function createTestDb() {
10
15
  const sqlite = new Database(":memory:");
@@ -14,10 +19,7 @@ function createTestDb() {
14
19
  }
15
20
 
16
21
  describe("runMigrationSteps — checkpointing", () => {
17
- // Skipped: documents the post-checkpointing contract and fails against the
18
- // current behavior, which re-runs every forward step on every boot. Promote
19
- // to `test` when true checkpointing of initial migrations lands.
20
- test.skip("does not re-execute a step that was already applied on a prior run", () => {
22
+ test("does not re-execute a step that was already applied on a prior run", () => {
21
23
  /**
22
24
  * Booting an already-migrated database must not re-run forward migration
23
25
  * steps. With true checkpointing each step's body executes at most once
@@ -49,4 +51,144 @@ describe("runMigrationSteps — checkpointing", () => {
49
51
  // THEN no step body executes a second time
50
52
  expect(calls).toEqual({ a: 1, b: 1, c: 1 });
51
53
  });
54
+
55
+ test("records step completions in the shared memory_checkpoints ledger", () => {
56
+ /**
57
+ * Step bookkeeping lives in the same memory_checkpoints table the registry
58
+ * uses, under the `step:` namespace — one ledger for all applied state.
59
+ */
60
+
61
+ // GIVEN a single named step
62
+ const db = createTestDb();
63
+ runMigrationSteps(db, [
64
+ function dummyMigrationA() {
65
+ // no-op
66
+ },
67
+ ]);
68
+
69
+ // THEN its completion is recorded under the step: namespace in memory_checkpoints
70
+ const row = getSqliteFrom(db)
71
+ .query(
72
+ `SELECT value FROM memory_checkpoints WHERE key = 'step:dummyMigrationA'`,
73
+ )
74
+ .get() as { value: string } | null;
75
+ expect(row?.value).toBe("1");
76
+ });
77
+
78
+ test("runs a newly appended step on a later boot while skipping applied ones", () => {
79
+ /**
80
+ * Checkpointing must not block migrations added in a later release: an
81
+ * already-applied step is skipped, but a brand-new step still runs.
82
+ */
83
+
84
+ // GIVEN a database already migrated with two steps
85
+ const calls = { a: 0, b: 0, c: 0 };
86
+ const stepA: MigrationStep = function dummyMigrationA() {
87
+ calls.a++;
88
+ };
89
+ const stepB: MigrationStep = function dummyMigrationB() {
90
+ calls.b++;
91
+ };
92
+ const db = createTestDb();
93
+ runMigrationSteps(db, [stepA, stepB]);
94
+
95
+ // AND a third step appended to the list
96
+ const stepC: MigrationStep = function dummyMigrationC() {
97
+ calls.c++;
98
+ };
99
+
100
+ // WHEN the expanded list runs against the already-migrated database
101
+ runMigrationSteps(db, [stepA, stepB, stepC]);
102
+
103
+ // THEN only the newly appended step executes
104
+ expect(calls).toEqual({ a: 1, b: 1, c: 1 });
105
+ });
106
+
107
+ test("recovers crashed migrations before running steps", () => {
108
+ /**
109
+ * runMigrationSteps clears stalled `started`/`rolling_back` checkpoints
110
+ * before the loop so a migration interrupted mid-flight re-runs this boot.
111
+ */
112
+
113
+ // GIVEN a database with a stalled registry checkpoint left by a crash
114
+ const db = createTestDb();
115
+ const raw = getSqliteFrom(db);
116
+ raw.run(
117
+ `CREATE TABLE IF NOT EXISTS memory_checkpoints (key TEXT PRIMARY KEY, value TEXT NOT NULL, updated_at INTEGER NOT NULL)`,
118
+ );
119
+ raw.run(
120
+ `INSERT INTO memory_checkpoints (key, value, updated_at) VALUES ('migration_stalled_v1', 'started', 0)`,
121
+ );
122
+
123
+ // WHEN the runner executes
124
+ runMigrationSteps(db, [
125
+ function dummyMigrationA() {
126
+ // no-op
127
+ },
128
+ ]);
129
+
130
+ // THEN the stalled checkpoint has been cleared so its migration can re-run
131
+ const row = raw
132
+ .query(
133
+ `SELECT 1 FROM memory_checkpoints WHERE key = 'migration_stalled_v1'`,
134
+ )
135
+ .get();
136
+ expect(row).toBeNull();
137
+ });
138
+
139
+ test("does not checkpoint a failed step so it retries on the next boot", () => {
140
+ /**
141
+ * A step whose body throws is reported in `failed` and left uncheckpointed,
142
+ * so the next boot retries it instead of silently skipping it.
143
+ */
144
+
145
+ // GIVEN a step that throws on its first run and succeeds afterwards
146
+ const calls = { flaky: 0 };
147
+ const steps: MigrationStep[] = [
148
+ function flakyStep() {
149
+ calls.flaky++;
150
+ if (calls.flaky === 1) {
151
+ throw new Error("transient failure");
152
+ }
153
+ },
154
+ ];
155
+ const db = createTestDb();
156
+ const first = runMigrationSteps(db, steps);
157
+
158
+ // AND the first run reports the failure
159
+ expect(first.failed).toEqual(["flakyStep"]);
160
+
161
+ // WHEN the step runs again on the next boot
162
+ const second = runMigrationSteps(db, steps);
163
+
164
+ // THEN it retries, succeeds, and is then checkpointed
165
+ expect(calls.flaky).toBe(2);
166
+ expect(second.failed).toEqual([]);
167
+ expect(runMigrationSteps(db, steps).skipped).toEqual(["flakyStep"]);
168
+ });
169
+
170
+ test("clearMigrationStepCheckpoints forces every step to re-run", () => {
171
+ /**
172
+ * Clearing the step namespace (as rollback does) makes the next run
173
+ * re-execute and re-record all steps.
174
+ */
175
+
176
+ // GIVEN a database migrated once
177
+ const calls = { a: 0 };
178
+ const steps: MigrationStep[] = [
179
+ function dummyMigrationA() {
180
+ calls.a++;
181
+ },
182
+ ];
183
+ const db = createTestDb();
184
+ runMigrationSteps(db, steps);
185
+ expect(calls.a).toBe(1);
186
+
187
+ // WHEN the step checkpoints are cleared
188
+ clearMigrationStepCheckpoints(db);
189
+
190
+ // THEN the next run re-executes the step
191
+ runMigrationSteps(db, steps);
192
+ expect(calls.a).toBe(2);
193
+ });
52
194
  });
@@ -1,3 +1,4 @@
1
+ export { migrateCoreTables } from "./000-core-tables.js";
1
2
  export { migrateJobDeferrals } from "./001-job-deferrals.js";
2
3
  export { migrateToolInvocationsFk } from "./002-tool-invocations-fk.js";
3
4
  export { migrateMemoryFtsBackfill } from "./003-memory-fts-backfill.js";
@@ -43,10 +44,9 @@ export { createActorTokenRecordsTable } from "./038-actor-token-records.js";
43
44
  export { createActorRefreshTokenRecordsTable } from "./039-actor-refresh-token-records.js";
44
45
  export { migrateInviteCodeHashColumn } from "./040-invite-code-hash-column.js";
45
46
  export { createApprovalPromptTsTrackerTable } from "./041-approval-prompt-ts-tracker.js";
46
- export { createCoreTables } from "./100-core-tables.js";
47
47
  export { createWatchersAndLogsTables } from "./101-watchers-and-logs.js";
48
48
  export { addCoreColumns } from "./102-alter-table-columns.js";
49
- export { runComplexMigrations } from "./103-complex-migrations.js";
49
+ export { complexMigrationSteps } from "./103-complex-migrations.js";
50
50
  export { createCoreIndexes } from "./104-core-indexes.js";
51
51
  export { createContactsAndTriageTables } from "./105-contacts-and-triage.js";
52
52
  export { createCallSessionsTables } from "./106-call-sessions.js";
@@ -56,7 +56,7 @@ export { createExternalConversationBindingsTables } from "./109-external-convers
56
56
  export { createChannelGuardianTables } from "./110-channel-guardian.js";
57
57
  export { createMediaAssetsTables } from "./111-media-assets.js";
58
58
  export { createAssistantInboxTables } from "./112-assistant-inbox.js";
59
- export { runLateMigrations } from "./113-late-migrations.js";
59
+ export { lateMigrationSteps } from "./113-late-migrations.js";
60
60
  export { createNotificationTables } from "./114-notifications.js";
61
61
  export { createSequenceTables } from "./115-sequences.js";
62
62
  export { createMessagesFts } from "./116-messages-fts.js";
@@ -288,13 +288,14 @@ export { migrateWorkflowJournalLeafTokens } from "./293-workflow-journal-leaf-to
288
288
  export { migrateDropExternalUserId } from "./294-drop-external-user-id.js";
289
289
  export { dropApprovalPromptTsTrackerTable } from "./295-drop-approval-prompt-ts-tracker.js";
290
290
  export { migrateRewriteBalancedEconomyProfilePins } from "./296-rewrite-balanced-economy-profile-pins.js";
291
+ export { migrateMoveLlmRequestLogsToLogsDb } from "./297-move-llm-request-logs-to-logs-db.js";
291
292
  export {
292
293
  MIGRATION_REGISTRY,
293
294
  type MigrationRegistryEntry,
294
295
  type MigrationValidationResult,
295
296
  } from "./registry.js";
297
+ export { recoverCrashedMigrations } from "./run-migrations.js";
296
298
  export {
297
- recoverCrashedMigrations,
298
299
  rollbackMemoryMigration,
299
300
  validateMigrationState,
300
301
  withCrashRecovery,