@axiom-lattice/pg-stores 3.2.1 → 3.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +10 -10
- package/CHANGELOG.md +87 -0
- package/dist/index.d.mts +109 -2
- package/dist/index.d.ts +109 -2
- package/dist/index.js +395 -23
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +395 -23
- package/dist/index.mjs.map +1 -1
- package/package.json +3 -3
- package/src/__tests__/PostgreSQLLeaseStore.test.ts +43 -0
- package/src/__tests__/PostgreSQLOutboxStore.test.ts +77 -0
- package/src/__tests__/PostgreSQLScheduleStorage.test.ts +162 -0
- package/src/__tests__/ThreadMessageQueueStore.integration.test.ts +82 -0
- package/src/__tests__/ThreadMessageQueueStore.test.ts +71 -12
- package/src/__tests__/queue_migrations.test.ts +18 -0
- package/src/createPgStoreConfig.ts +10 -12
- package/src/migrations/add_queue_claim_lease.ts +35 -0
- package/src/migrations/distributed_lease_migration.ts +32 -0
- package/src/migrations/outbox_migration.ts +30 -0
- package/src/migrations/queue_migrations.ts +28 -0
- package/src/stores/PostgreSQLLeaseStore.ts +126 -0
- package/src/stores/PostgreSQLOutboxStore.ts +136 -0
- package/src/stores/PostgreSQLScheduleStorage.ts +54 -0
- package/src/stores/ThreadMessageQueueStore.ts +49 -16
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@axiom-lattice/pg-stores",
|
|
3
|
-
"version": "3.
|
|
3
|
+
"version": "3.3.0",
|
|
4
4
|
"description": "PG stores implementation for Axiom Lattice framework",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"module": "dist/index.mjs",
|
|
@@ -25,8 +25,8 @@
|
|
|
25
25
|
"@langchain/core": "1.1.30",
|
|
26
26
|
"pg": "^8.16.3",
|
|
27
27
|
"uuid": "^9.0.1",
|
|
28
|
-
"@axiom-lattice/
|
|
29
|
-
"@axiom-lattice/
|
|
28
|
+
"@axiom-lattice/core": "6.3.0",
|
|
29
|
+
"@axiom-lattice/protocols": "4.7.0"
|
|
30
30
|
},
|
|
31
31
|
"devDependencies": {
|
|
32
32
|
"@types/jest": "^29.5.14",
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { PostgreSQLLeaseStore } from "../stores/PostgreSQLLeaseStore";
|
|
3
|
+
|
|
4
|
+
const TEST_DATABASE_URL = process.env.TEST_DATABASE_URL;
|
|
5
|
+
const maybeDescribe = TEST_DATABASE_URL ? describe : describe.skip;
|
|
6
|
+
|
|
7
|
+
maybeDescribe("PostgreSQLLeaseStore", () => {
|
|
8
|
+
it("grants exactly one of two concurrent acquires", async () => {
|
|
9
|
+
const store = new PostgreSQLLeaseStore({ poolConfig: TEST_DATABASE_URL! });
|
|
10
|
+
await store.initialize();
|
|
11
|
+
const key = `test-${randomUUID()}`;
|
|
12
|
+
const [a, b] = await Promise.all([
|
|
13
|
+
store.tryAcquire(key, "a", 5000),
|
|
14
|
+
store.tryAcquire(key, "b", 5000),
|
|
15
|
+
]);
|
|
16
|
+
expect([a, b].filter(Boolean)).toHaveLength(1);
|
|
17
|
+
expect((await store.get(key))?.holder).toBe(a ? "a" : "b");
|
|
18
|
+
await store.release(key, a ? "a" : "b");
|
|
19
|
+
await store.dispose();
|
|
20
|
+
});
|
|
21
|
+
|
|
22
|
+
it("lets another holder take over an expired lease", async () => {
|
|
23
|
+
const store = new PostgreSQLLeaseStore({ poolConfig: TEST_DATABASE_URL! });
|
|
24
|
+
await store.initialize();
|
|
25
|
+
const key = `test-${randomUUID()}`;
|
|
26
|
+
expect(await store.tryAcquire(key, "a", 1)).toBe(true);
|
|
27
|
+
await new Promise((r) => setTimeout(r, 50));
|
|
28
|
+
expect(await store.tryAcquire(key, "b", 5000)).toBe(true);
|
|
29
|
+
expect((await store.get(key))?.holder).toBe("b");
|
|
30
|
+
await store.dispose();
|
|
31
|
+
});
|
|
32
|
+
|
|
33
|
+
it("is idempotent for the same holder re-acquiring its own live lease", async () => {
|
|
34
|
+
const store = new PostgreSQLLeaseStore({ poolConfig: TEST_DATABASE_URL! });
|
|
35
|
+
await store.initialize();
|
|
36
|
+
const key = `test-${randomUUID()}`;
|
|
37
|
+
expect(await store.tryAcquire(key, "a", 5000)).toBe(true);
|
|
38
|
+
const first = await store.get(key);
|
|
39
|
+
expect(await store.tryAcquire(key, "a", 5000)).toBe(true);
|
|
40
|
+
expect((await store.get(key))?.acquiredAt).toBe(first?.acquiredAt);
|
|
41
|
+
await store.dispose();
|
|
42
|
+
});
|
|
43
|
+
});
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
import { Pool } from "pg";
|
|
2
|
+
import { PostgreSQLOutboxStore } from "../stores/PostgreSQLOutboxStore";
|
|
3
|
+
|
|
4
|
+
const TEST_DATABASE_URL = process.env.TEST_DATABASE_URL;
|
|
5
|
+
const maybeDescribe = TEST_DATABASE_URL ? describe : describe.skip;
|
|
6
|
+
|
|
7
|
+
maybeDescribe("PostgreSQLOutboxStore", () => {
|
|
8
|
+
it("appends events and pages them with a cursor", async () => {
|
|
9
|
+
const store = new PostgreSQLOutboxStore({ poolConfig: TEST_DATABASE_URL! });
|
|
10
|
+
await store.initialize();
|
|
11
|
+
|
|
12
|
+
const before = await store.latestId();
|
|
13
|
+
const first = await store.append({ topic: "t", payload: { n: 1 }, originReplicaId: "a" });
|
|
14
|
+
const second = await store.append({ topic: "t", payload: { n: 2 }, originReplicaId: "b" });
|
|
15
|
+
|
|
16
|
+
const appended = await store.listAfter(before, 10);
|
|
17
|
+
expect(appended.map((e) => e.id)).toEqual([first.id, second.id]);
|
|
18
|
+
expect(Number(first.id)).toBeLessThan(Number(second.id));
|
|
19
|
+
|
|
20
|
+
const all = await store.listAfter(null, 1000);
|
|
21
|
+
const allIds = all.map((e) => e.id);
|
|
22
|
+
const numeric = allIds.map(Number);
|
|
23
|
+
expect(numeric).toEqual([...numeric].sort((a, b) => a - b));
|
|
24
|
+
expect(allIds).toEqual(expect.arrayContaining([first.id, second.id]));
|
|
25
|
+
expect(allIds.indexOf(first.id)).toBeLessThan(allIds.indexOf(second.id));
|
|
26
|
+
|
|
27
|
+
const afterFirst = await store.listAfter(first.id, 1000);
|
|
28
|
+
expect(afterFirst.every((e) => Number(e.id) > Number(first.id))).toBe(true);
|
|
29
|
+
expect(afterFirst.map((e) => e.id)).toContain(second.id);
|
|
30
|
+
|
|
31
|
+
expect(await store.latestId()).toBe(second.id);
|
|
32
|
+
|
|
33
|
+
await store.dispose();
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
it("assigns distinct, increasing ids to concurrent appends", async () => {
|
|
37
|
+
const store = new PostgreSQLOutboxStore({ poolConfig: TEST_DATABASE_URL! });
|
|
38
|
+
await store.initialize();
|
|
39
|
+
|
|
40
|
+
const events = await Promise.all(
|
|
41
|
+
Array.from({ length: 5 }, (_, i) =>
|
|
42
|
+
store.append({ topic: "t", payload: { i }, originReplicaId: "c" }),
|
|
43
|
+
),
|
|
44
|
+
);
|
|
45
|
+
const ids = events.map((e) => Number(e.id)).sort((a, b) => a - b);
|
|
46
|
+
expect(new Set(ids).size).toBe(5);
|
|
47
|
+
for (let i = 1; i < ids.length; i++) {
|
|
48
|
+
expect(ids[i]).toBeGreaterThan(ids[i - 1]);
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
await store.dispose();
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
it("prunes rows older than the threshold and keeps recent ones", async () => {
|
|
55
|
+
const store = new PostgreSQLOutboxStore({ poolConfig: TEST_DATABASE_URL! });
|
|
56
|
+
await store.initialize();
|
|
57
|
+
const pool = new Pool({ connectionString: TEST_DATABASE_URL! });
|
|
58
|
+
|
|
59
|
+
const recent = await store.append({ topic: "t", payload: { n: 1 }, originReplicaId: "a" });
|
|
60
|
+
const old = await pool.query(
|
|
61
|
+
`INSERT INTO lattice_outbox (topic, payload, origin_replica_id, created_at)
|
|
62
|
+
VALUES ('t', '{}'::jsonb, 'a', now() - interval '2 days')
|
|
63
|
+
RETURNING id`,
|
|
64
|
+
);
|
|
65
|
+
const oldId = String(old.rows[0].id);
|
|
66
|
+
|
|
67
|
+
const removed = await store.pruneOlderThan(24 * 60 * 60 * 1000);
|
|
68
|
+
|
|
69
|
+
expect(removed).toBeGreaterThanOrEqual(1);
|
|
70
|
+
const remaining = (await store.listAfter(null, 1000)).map((event) => event.id);
|
|
71
|
+
expect(remaining).toContain(recent.id);
|
|
72
|
+
expect(remaining).not.toContain(oldId);
|
|
73
|
+
|
|
74
|
+
await pool.end();
|
|
75
|
+
await store.dispose();
|
|
76
|
+
});
|
|
77
|
+
});
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
|
|
3
|
+
import {
|
|
4
|
+
ScheduleExecutionType,
|
|
5
|
+
ScheduledTaskDefinition,
|
|
6
|
+
ScheduledTaskStatus,
|
|
7
|
+
} from "@axiom-lattice/protocols";
|
|
8
|
+
|
|
9
|
+
import { PostgreSQLScheduleStorage } from "../stores/PostgreSQLScheduleStorage";
|
|
10
|
+
|
|
11
|
+
const TEST_DATABASE_URL = process.env.TEST_DATABASE_URL;
|
|
12
|
+
const maybeDescribe = TEST_DATABASE_URL ? describe : describe.skip;
|
|
13
|
+
|
|
14
|
+
const buildTask = (
|
|
15
|
+
taskId: string,
|
|
16
|
+
overrides: Partial<ScheduledTaskDefinition> = {}
|
|
17
|
+
): ScheduledTaskDefinition => {
|
|
18
|
+
const now = Date.now();
|
|
19
|
+
return {
|
|
20
|
+
taskId,
|
|
21
|
+
taskType: "test",
|
|
22
|
+
payload: {},
|
|
23
|
+
tenantId: "tenant-1",
|
|
24
|
+
executionType: ScheduleExecutionType.ONCE,
|
|
25
|
+
executeAt: now - 1000,
|
|
26
|
+
status: ScheduledTaskStatus.PENDING,
|
|
27
|
+
runCount: 0,
|
|
28
|
+
retryCount: 0,
|
|
29
|
+
maxRetries: 0,
|
|
30
|
+
createdAt: now,
|
|
31
|
+
updatedAt: now,
|
|
32
|
+
...overrides,
|
|
33
|
+
};
|
|
34
|
+
};
|
|
35
|
+
|
|
36
|
+
maybeDescribe("PostgreSQLScheduleStorage.claimScheduledRun", () => {
|
|
37
|
+
it("claims a pending task exactly once", async () => {
|
|
38
|
+
const store = new PostgreSQLScheduleStorage({
|
|
39
|
+
poolConfig: TEST_DATABASE_URL!,
|
|
40
|
+
});
|
|
41
|
+
await store.initialize();
|
|
42
|
+
const taskId = `test-${randomUUID()}`;
|
|
43
|
+
await store.save(buildTask(taskId));
|
|
44
|
+
|
|
45
|
+
expect(await store.claimScheduledRun(taskId)).toBe(true);
|
|
46
|
+
expect(await store.claimScheduledRun(taskId)).toBe(false);
|
|
47
|
+
|
|
48
|
+
const stored = await store.get(taskId);
|
|
49
|
+
expect(stored?.status).toBe(ScheduledTaskStatus.RUNNING);
|
|
50
|
+
|
|
51
|
+
await store.delete(taskId);
|
|
52
|
+
await store.dispose();
|
|
53
|
+
});
|
|
54
|
+
|
|
55
|
+
it("grants exactly one of two concurrent claims", async () => {
|
|
56
|
+
const store = new PostgreSQLScheduleStorage({
|
|
57
|
+
poolConfig: TEST_DATABASE_URL!,
|
|
58
|
+
});
|
|
59
|
+
await store.initialize();
|
|
60
|
+
const taskId = `test-${randomUUID()}`;
|
|
61
|
+
await store.save(buildTask(taskId));
|
|
62
|
+
|
|
63
|
+
const [a, b] = await Promise.all([
|
|
64
|
+
store.claimScheduledRun(taskId),
|
|
65
|
+
store.claimScheduledRun(taskId),
|
|
66
|
+
]);
|
|
67
|
+
expect([a, b].filter(Boolean)).toHaveLength(1);
|
|
68
|
+
|
|
69
|
+
await store.delete(taskId);
|
|
70
|
+
await store.dispose();
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
it("returns false for an unknown task", async () => {
|
|
74
|
+
const store = new PostgreSQLScheduleStorage({
|
|
75
|
+
poolConfig: TEST_DATABASE_URL!,
|
|
76
|
+
});
|
|
77
|
+
await store.initialize();
|
|
78
|
+
|
|
79
|
+
expect(await store.claimScheduledRun(`missing-${randomUUID()}`)).toBe(false);
|
|
80
|
+
|
|
81
|
+
await store.dispose();
|
|
82
|
+
});
|
|
83
|
+
});
|
|
84
|
+
|
|
85
|
+
maybeDescribe("PostgreSQLScheduleStorage.requeueStaleRunning", () => {
|
|
86
|
+
it("requeues a running task older than the threshold", async () => {
|
|
87
|
+
const store = new PostgreSQLScheduleStorage({
|
|
88
|
+
poolConfig: TEST_DATABASE_URL!,
|
|
89
|
+
});
|
|
90
|
+
await store.initialize();
|
|
91
|
+
const taskId = `test-${randomUUID()}`;
|
|
92
|
+
await store.save(
|
|
93
|
+
buildTask(taskId, {
|
|
94
|
+
status: ScheduledTaskStatus.RUNNING,
|
|
95
|
+
updatedAt: Date.now() - 60_000,
|
|
96
|
+
})
|
|
97
|
+
);
|
|
98
|
+
|
|
99
|
+
expect(await store.requeueStaleRunning(30_000)).toBe(1);
|
|
100
|
+
|
|
101
|
+
const stored = await store.get(taskId);
|
|
102
|
+
expect(stored?.status).toBe(ScheduledTaskStatus.PENDING);
|
|
103
|
+
|
|
104
|
+
await store.delete(taskId);
|
|
105
|
+
await store.dispose();
|
|
106
|
+
});
|
|
107
|
+
|
|
108
|
+
it("leaves a recently updated running task alone", async () => {
|
|
109
|
+
const store = new PostgreSQLScheduleStorage({
|
|
110
|
+
poolConfig: TEST_DATABASE_URL!,
|
|
111
|
+
});
|
|
112
|
+
await store.initialize();
|
|
113
|
+
const taskId = `test-${randomUUID()}`;
|
|
114
|
+
await store.save(
|
|
115
|
+
buildTask(taskId, {
|
|
116
|
+
status: ScheduledTaskStatus.RUNNING,
|
|
117
|
+
updatedAt: Date.now() - 1_000,
|
|
118
|
+
})
|
|
119
|
+
);
|
|
120
|
+
|
|
121
|
+
expect(await store.requeueStaleRunning(30_000)).toBe(0);
|
|
122
|
+
|
|
123
|
+
const stored = await store.get(taskId);
|
|
124
|
+
expect(stored?.status).toBe(ScheduledTaskStatus.RUNNING);
|
|
125
|
+
|
|
126
|
+
await store.delete(taskId);
|
|
127
|
+
await store.dispose();
|
|
128
|
+
});
|
|
129
|
+
|
|
130
|
+
it("never touches pending or terminal tasks", async () => {
|
|
131
|
+
const store = new PostgreSQLScheduleStorage({
|
|
132
|
+
poolConfig: TEST_DATABASE_URL!,
|
|
133
|
+
});
|
|
134
|
+
await store.initialize();
|
|
135
|
+
const pendingId = `test-${randomUUID()}`;
|
|
136
|
+
const completedId = `test-${randomUUID()}`;
|
|
137
|
+
await store.save(
|
|
138
|
+
buildTask(pendingId, {
|
|
139
|
+
status: ScheduledTaskStatus.PENDING,
|
|
140
|
+
updatedAt: Date.now() - 60_000,
|
|
141
|
+
})
|
|
142
|
+
);
|
|
143
|
+
await store.save(
|
|
144
|
+
buildTask(completedId, {
|
|
145
|
+
status: ScheduledTaskStatus.COMPLETED,
|
|
146
|
+
updatedAt: Date.now() - 60_000,
|
|
147
|
+
})
|
|
148
|
+
);
|
|
149
|
+
|
|
150
|
+
expect(await store.requeueStaleRunning(30_000)).toBe(0);
|
|
151
|
+
expect((await store.get(pendingId))?.status).toBe(
|
|
152
|
+
ScheduledTaskStatus.PENDING
|
|
153
|
+
);
|
|
154
|
+
expect((await store.get(completedId))?.status).toBe(
|
|
155
|
+
ScheduledTaskStatus.COMPLETED
|
|
156
|
+
);
|
|
157
|
+
|
|
158
|
+
await store.delete(pendingId);
|
|
159
|
+
await store.delete(completedId);
|
|
160
|
+
await store.dispose();
|
|
161
|
+
});
|
|
162
|
+
});
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { ThreadMessageQueueStore } from "../stores/ThreadMessageQueueStore";
|
|
3
|
+
|
|
4
|
+
const TEST_DATABASE_URL = process.env.TEST_DATABASE_URL;
|
|
5
|
+
const maybeDescribe = TEST_DATABASE_URL ? describe : describe.skip;
|
|
6
|
+
|
|
7
|
+
maybeDescribe("ThreadMessageQueueStore claim/lease (PostgreSQL)", () => {
|
|
8
|
+
let store: ThreadMessageQueueStore;
|
|
9
|
+
const threadIds: string[] = [];
|
|
10
|
+
|
|
11
|
+
function newThreadId(): string {
|
|
12
|
+
const threadId = `claim-${randomUUID()}`;
|
|
13
|
+
threadIds.push(threadId);
|
|
14
|
+
return threadId;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
beforeAll(async () => {
|
|
18
|
+
store = new ThreadMessageQueueStore({ poolConfig: TEST_DATABASE_URL!, autoMigrate: true });
|
|
19
|
+
await store.initialize();
|
|
20
|
+
});
|
|
21
|
+
|
|
22
|
+
afterAll(async () => {
|
|
23
|
+
for (const threadId of threadIds) {
|
|
24
|
+
await store.clearMessages(threadId);
|
|
25
|
+
}
|
|
26
|
+
await store.dispose();
|
|
27
|
+
});
|
|
28
|
+
|
|
29
|
+
it("grants disjoint sets to concurrent claimants", async () => {
|
|
30
|
+
const threadId = newThreadId();
|
|
31
|
+
for (let i = 0; i < 4; i += 1) {
|
|
32
|
+
await store.addMessage({
|
|
33
|
+
threadId,
|
|
34
|
+
tenantId: "tenant",
|
|
35
|
+
assistantId: "assistant",
|
|
36
|
+
content: { id: randomUUID(), message: `message-${i}`, queueMessage: { input: {} } },
|
|
37
|
+
id: randomUUID(),
|
|
38
|
+
});
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
const [a, b] = await Promise.all([
|
|
42
|
+
store.claimPendingMessages(threadId, undefined, { owner: "r1", ttlMs: 5000, limit: 2 }),
|
|
43
|
+
store.claimPendingMessages(threadId, undefined, { owner: "r2", ttlMs: 5000, limit: 2 }),
|
|
44
|
+
]);
|
|
45
|
+
|
|
46
|
+
const ids = [...a, ...b].map((m) => m.id);
|
|
47
|
+
expect(ids).toHaveLength(4);
|
|
48
|
+
expect(new Set(ids).size).toBe(4);
|
|
49
|
+
expect(await store.getPendingMessages(threadId)).toHaveLength(0);
|
|
50
|
+
expect(await store.getProcessingMessages(threadId)).toHaveLength(4);
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
it("reclaims only expired processing rows", async () => {
|
|
54
|
+
const liveThread = newThreadId();
|
|
55
|
+
const expiredThread = newThreadId();
|
|
56
|
+
await store.addMessage({ threadId: liveThread, tenantId: "tenant", assistantId: "assistant", content: { id: randomUUID(), message: "live", queueMessage: { input: {} } }, id: randomUUID() });
|
|
57
|
+
await store.addMessage({ threadId: expiredThread, tenantId: "tenant", assistantId: "assistant", content: { id: randomUUID(), message: "expired", queueMessage: { input: {} } }, id: randomUUID() });
|
|
58
|
+
|
|
59
|
+
await store.claimPendingMessages(liveThread, undefined, { owner: "r1", ttlMs: 60000 });
|
|
60
|
+
await store.claimPendingMessages(expiredThread, undefined, { owner: "r1", ttlMs: 1 });
|
|
61
|
+
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
62
|
+
|
|
63
|
+
expect(await store.reclaimExpiredProcessing(liveThread)).toBe(0);
|
|
64
|
+
expect(await store.reclaimExpiredProcessing(expiredThread)).toBe(1);
|
|
65
|
+
expect(await store.getPendingMessages(expiredThread)).toHaveLength(1);
|
|
66
|
+
expect(await store.getPendingMessages(liveThread)).toHaveLength(0);
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
it("round-trips a bare-string message content", async () => {
|
|
70
|
+
const threadId = newThreadId();
|
|
71
|
+
await store.addMessage({
|
|
72
|
+
threadId,
|
|
73
|
+
tenantId: "ten",
|
|
74
|
+
assistantId: "a",
|
|
75
|
+
content: "hello",
|
|
76
|
+
id: randomUUID(),
|
|
77
|
+
});
|
|
78
|
+
|
|
79
|
+
const [pending] = await store.getPendingMessages(threadId);
|
|
80
|
+
expect(pending.content).toBe("hello");
|
|
81
|
+
});
|
|
82
|
+
});
|
|
@@ -46,7 +46,7 @@ describe('ThreadMessageQueueStore', () => {
|
|
|
46
46
|
'persists trusted metadata through %s',
|
|
47
47
|
async (method) => {
|
|
48
48
|
const row = {
|
|
49
|
-
id: 'input-1', message_content:
|
|
49
|
+
id: 'input-1', message_content: 'message', message_type: 'human',
|
|
50
50
|
sequence_order: 1, created_at: new Date(), trusted_run_context: trusted,
|
|
51
51
|
execution_mode: 'followup',
|
|
52
52
|
};
|
|
@@ -144,7 +144,7 @@ describe('ThreadMessageQueueStore', () => {
|
|
|
144
144
|
thread_id: mockThreadId,
|
|
145
145
|
tenant_id: mockTenantId,
|
|
146
146
|
assistant_id: mockAssistantId,
|
|
147
|
-
message_content:
|
|
147
|
+
message_content: 'test content',
|
|
148
148
|
message_type: 'human',
|
|
149
149
|
sequence_order: 1,
|
|
150
150
|
status: 'pending',
|
|
@@ -176,7 +176,7 @@ describe('ThreadMessageQueueStore', () => {
|
|
|
176
176
|
rows: [{
|
|
177
177
|
id: 'msg-5',
|
|
178
178
|
sequence_order: 5,
|
|
179
|
-
message_content:
|
|
179
|
+
message_content: 'content',
|
|
180
180
|
}]
|
|
181
181
|
});
|
|
182
182
|
|
|
@@ -205,7 +205,7 @@ describe('ThreadMessageQueueStore', () => {
|
|
|
205
205
|
thread_id: mockThreadId,
|
|
206
206
|
tenant_id: mockTenantId,
|
|
207
207
|
assistant_id: mockAssistantId,
|
|
208
|
-
message_content:
|
|
208
|
+
message_content: 'test content',
|
|
209
209
|
message_type: 'human',
|
|
210
210
|
sequence_order: 1,
|
|
211
211
|
status: 'pending',
|
|
@@ -241,7 +241,7 @@ describe('ThreadMessageQueueStore', () => {
|
|
|
241
241
|
id: 'msg-head',
|
|
242
242
|
sequence_order: 1,
|
|
243
243
|
priority: 100,
|
|
244
|
-
message_content:
|
|
244
|
+
message_content: 'head content',
|
|
245
245
|
}]
|
|
246
246
|
}); // Insert
|
|
247
247
|
|
|
@@ -267,7 +267,7 @@ describe('ThreadMessageQueueStore', () => {
|
|
|
267
267
|
id: 'msg-head',
|
|
268
268
|
sequence_order: 0,
|
|
269
269
|
priority: 100,
|
|
270
|
-
message_content:
|
|
270
|
+
message_content: 'head content',
|
|
271
271
|
}]
|
|
272
272
|
});
|
|
273
273
|
|
|
@@ -288,14 +288,14 @@ describe('ThreadMessageQueueStore', () => {
|
|
|
288
288
|
const mockMessages = [
|
|
289
289
|
{
|
|
290
290
|
id: 'msg-1',
|
|
291
|
-
message_content:
|
|
291
|
+
message_content: 'first',
|
|
292
292
|
message_type: 'human',
|
|
293
293
|
sequence_order: 1,
|
|
294
294
|
created_at: new Date('2024-01-01'),
|
|
295
295
|
},
|
|
296
296
|
{
|
|
297
297
|
id: 'msg-2',
|
|
298
|
-
message_content:
|
|
298
|
+
message_content: 'second',
|
|
299
299
|
message_type: 'human',
|
|
300
300
|
sequence_order: 2,
|
|
301
301
|
created_at: new Date('2024-01-02'),
|
|
@@ -335,7 +335,7 @@ describe('ThreadMessageQueueStore', () => {
|
|
|
335
335
|
rows: [
|
|
336
336
|
{
|
|
337
337
|
id: 'msg-1',
|
|
338
|
-
message_content:
|
|
338
|
+
message_content: {
|
|
339
339
|
id: 'msg-1',
|
|
340
340
|
message: 'Clarification answers submitted',
|
|
341
341
|
queueMessage: {
|
|
@@ -344,7 +344,7 @@ describe('ThreadMessageQueueStore', () => {
|
|
|
344
344
|
message: 'Clarification answers submitted',
|
|
345
345
|
},
|
|
346
346
|
},
|
|
347
|
-
}
|
|
347
|
+
},
|
|
348
348
|
command: JSON.stringify(command),
|
|
349
349
|
custom_run_config: JSON.stringify(customRunConfig),
|
|
350
350
|
message_type: 'human',
|
|
@@ -364,7 +364,7 @@ describe('ThreadMessageQueueStore', () => {
|
|
|
364
364
|
|
|
365
365
|
it('hydrates trusted metadata for pending and processing rows', async () => {
|
|
366
366
|
const row = {
|
|
367
|
-
id: 'input-1', message_content:
|
|
367
|
+
id: 'input-1', message_content: 'message', message_type: 'human',
|
|
368
368
|
sequence_order: 1, created_at: new Date(), trusted_run_context: JSON.stringify(trusted),
|
|
369
369
|
execution_mode: 'followup',
|
|
370
370
|
};
|
|
@@ -380,7 +380,7 @@ describe('ThreadMessageQueueStore', () => {
|
|
|
380
380
|
|
|
381
381
|
it('rejects hostile trusted JSON read from PostgreSQL', async () => {
|
|
382
382
|
mockQuery.mockResolvedValueOnce({ rows: [{
|
|
383
|
-
id: 'hostile', message_content:
|
|
383
|
+
id: 'hostile', message_content: 'message', message_type: 'human',
|
|
384
384
|
sequence_order: 1, created_at: new Date(),
|
|
385
385
|
trusted_run_context: { projectRoom: { tenantId: mockTenantId } }, execution_mode: 'followup',
|
|
386
386
|
}] });
|
|
@@ -547,4 +547,63 @@ describe('ThreadMessageQueueStore', () => {
|
|
|
547
547
|
});
|
|
548
548
|
});
|
|
549
549
|
|
|
550
|
+
describe('claimPendingMessages', () => {
|
|
551
|
+
it('claims with lease columns bounded by an ordered SKIP LOCKED subquery', async () => {
|
|
552
|
+
const row = {
|
|
553
|
+
id: 'msg-1', message_content: 'content', message_type: 'human',
|
|
554
|
+
sequence_order: 1, created_at: new Date(),
|
|
555
|
+
};
|
|
556
|
+
mockQuery.mockResolvedValueOnce({ rows: [row] });
|
|
557
|
+
|
|
558
|
+
const claimed = await store.claimPendingMessages(mockThreadId, undefined, { owner: 'r1', ttlMs: 5000, limit: 2 });
|
|
559
|
+
|
|
560
|
+
expect(claimed.map((message) => message.id)).toEqual(['msg-1']);
|
|
561
|
+
const sql = mockQuery.mock.calls[0][0] as string;
|
|
562
|
+
expect(sql).toContain("status = 'processing'");
|
|
563
|
+
expect(sql).toContain('locked_by = $2');
|
|
564
|
+
expect(sql).toContain("lease_expires_at = now() + ($3::bigint || ' milliseconds')::interval");
|
|
565
|
+
expect(sql).toContain('ORDER BY priority DESC, sequence_order ASC LIMIT 2');
|
|
566
|
+
expect(sql).toContain('FOR UPDATE SKIP LOCKED');
|
|
567
|
+
expect(mockQuery.mock.calls[0][1]).toEqual([mockThreadId, 'r1', 5000]);
|
|
568
|
+
});
|
|
569
|
+
|
|
570
|
+
it('applies scope predicates without colliding with the owner parameter', async () => {
|
|
571
|
+
mockQuery.mockResolvedValueOnce({ rows: [] });
|
|
572
|
+
|
|
573
|
+
await store.claimPendingMessages(mockThreadId, {
|
|
574
|
+
tenantId: mockTenantId, assistantId: mockAssistantId, workspaceId: undefined, projectId: undefined,
|
|
575
|
+
}, { owner: 'r1', ttlMs: 5000 });
|
|
576
|
+
|
|
577
|
+
const [sql, params] = mockQuery.mock.calls[0] as [string, unknown[]];
|
|
578
|
+
expect(sql).toMatch(/tenant_id = \$4.*assistant_id = \$5.*workspace_id IS NULL.*project_id IS NULL/s);
|
|
579
|
+
expect(params).toEqual([mockThreadId, 'r1', 5000, mockTenantId, mockAssistantId]);
|
|
580
|
+
});
|
|
581
|
+
});
|
|
582
|
+
|
|
583
|
+
describe('reclaimExpiredProcessing', () => {
|
|
584
|
+
it('resets only expired leases and returns the row count', async () => {
|
|
585
|
+
mockQuery.mockResolvedValueOnce({ rowCount: 2 });
|
|
586
|
+
|
|
587
|
+
await expect(store.reclaimExpiredProcessing(mockThreadId)).resolves.toBe(2);
|
|
588
|
+
|
|
589
|
+
const sql = mockQuery.mock.calls[0][0] as string;
|
|
590
|
+
expect(sql).toContain("SET status = 'pending', locked_by = NULL, locked_at = NULL, lease_expires_at = NULL");
|
|
591
|
+
expect(sql).toContain('lease_expires_at <= now()');
|
|
592
|
+
expect(mockQuery.mock.calls[0][1]).toEqual([mockThreadId]);
|
|
593
|
+
});
|
|
594
|
+
});
|
|
595
|
+
|
|
596
|
+
describe('resetProcessingToPending', () => {
|
|
597
|
+
it('clears the lease columns for reset rows', async () => {
|
|
598
|
+
mockQuery.mockResolvedValueOnce({ rowCount: 1 });
|
|
599
|
+
|
|
600
|
+
await store.resetProcessingToPending(mockThreadId);
|
|
601
|
+
|
|
602
|
+
const sql = mockQuery.mock.calls[0][0] as string;
|
|
603
|
+
expect(sql).toContain('locked_by = NULL');
|
|
604
|
+
expect(sql).toContain('locked_at = NULL');
|
|
605
|
+
expect(sql).toContain('lease_expires_at = NULL');
|
|
606
|
+
});
|
|
607
|
+
});
|
|
608
|
+
|
|
550
609
|
});
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import { registerThreadMessageQueueMigrations } from "../migrations/queue_migrations";
|
|
2
|
+
|
|
3
|
+
describe("registerThreadMessageQueueMigrations", () => {
|
|
4
|
+
it("registers every thread-message-queue migration, including the claim lease", () => {
|
|
5
|
+
const names: string[] = [];
|
|
6
|
+
const fake = {
|
|
7
|
+
register: (migration: { name: string }): void => {
|
|
8
|
+
names.push(migration.name);
|
|
9
|
+
},
|
|
10
|
+
};
|
|
11
|
+
registerThreadMessageQueueMigrations(fake as never);
|
|
12
|
+
// The shared list must stay complete; bump this count when adding a queue
|
|
13
|
+
// migration (and register it in the shared helper, never inline).
|
|
14
|
+
expect(names).toHaveLength(7);
|
|
15
|
+
expect(names).toContain("create_lattice_thread_message_queue_table");
|
|
16
|
+
expect(names).toContain("add_queue_claim_lease_columns");
|
|
17
|
+
});
|
|
18
|
+
});
|
|
@@ -58,12 +58,7 @@ import { createDatabaseConfigsTable } from "./migrations/database_config_migrati
|
|
|
58
58
|
import { createConnectionConfigsTable } from "./migrations/connection_config_migrations";
|
|
59
59
|
import { createMetricsConfigsTable } from "./migrations/metrics_config_migrations";
|
|
60
60
|
import { createMcpServerConfigsTable } from "./migrations/mcp_server_config_migrations";
|
|
61
|
-
import {
|
|
62
|
-
import { addPriorityAndCommandColumns } from "./migrations/add_priority_command_columns";
|
|
63
|
-
import { addCustomRunConfigColumn } from "./migrations/add_custom_run_config_column";
|
|
64
|
-
import { alterMessageQueueIdColumn } from "./migrations/alter_message_queue_id_column";
|
|
65
|
-
import { addWorkspaceProjectToQueue } from "./migrations/add_workspace_project_to_queue";
|
|
66
|
-
import { addTrustedRunContextColumn } from "./migrations/add_trusted_run_context_column";
|
|
61
|
+
import { registerThreadMessageQueueMigrations } from "./migrations/queue_migrations";
|
|
67
62
|
import { createWorkflowTrackingTables, addStepThreadId, addWorkflowRunsTenantStatusUpdatedIndex } from "./migrations/workflow_tracking_migrations";
|
|
68
63
|
import { evalMigrations } from "./migrations/eval_migrations";
|
|
69
64
|
import { createA2AApiKeysTable } from "./migrations/a2a_api_key_migration";
|
|
@@ -90,6 +85,8 @@ import { createCapabilityBundlesTable } from "./migrations/capability_bundle_mig
|
|
|
90
85
|
import { createProjectRoomReadStates, createProjectRoomTables } from "./migrations/project_room_migration";
|
|
91
86
|
import { addProjectRoomMessageRetentionIndex } from "./migrations/project_room_message_retention_migration";
|
|
92
87
|
import { createUserPushSubscriptions } from "./migrations/user_push_subscription_migration";
|
|
88
|
+
import { createDistributedLeases } from "./migrations/distributed_lease_migration";
|
|
89
|
+
import { createOutboxTable } from "./migrations/outbox_migration";
|
|
93
90
|
import {
|
|
94
91
|
createModelProvidersTable,
|
|
95
92
|
createModelProviderModelsTable,
|
|
@@ -104,6 +101,8 @@ import { PostgreSQLProjectRoomMessageStore } from "./stores/PostgreSQLProjectRoo
|
|
|
104
101
|
import { PostgreSQLProjectRoomReadStateStore } from "./stores/PostgreSQLProjectRoomReadStateStore";
|
|
105
102
|
import { PostgreSQLUserPushSubscriptionStore } from "./stores/PostgreSQLUserPushSubscriptionStore";
|
|
106
103
|
import { PostgreSQLModelProviderStore } from "./stores/PostgreSQLModelProviderStore";
|
|
104
|
+
import { PostgreSQLLeaseStore } from "./stores/PostgreSQLLeaseStore";
|
|
105
|
+
import { PostgreSQLOutboxStore } from "./stores/PostgreSQLOutboxStore";
|
|
107
106
|
|
|
108
107
|
export async function createPgStoreConfig(connectionString: string) {
|
|
109
108
|
const pool = new Pool({ connectionString });
|
|
@@ -134,9 +133,7 @@ export async function createPgStoreConfig(connectionString: string) {
|
|
|
134
133
|
mm.register(createChannelInstallationsTable); // v21
|
|
135
134
|
mm.register(addThreadTenantId); // v23
|
|
136
135
|
mm.register(addProjectConfigColumn); // v24
|
|
137
|
-
mm
|
|
138
|
-
mm.register(addPriorityAndCommandColumns); // v101
|
|
139
|
-
mm.register(alterMessageQueueIdColumn); // v102
|
|
136
|
+
registerThreadMessageQueueMigrations(mm); // v100, v101, v102, v114, v131, v173, v186
|
|
140
137
|
mm.register(createWorkflowTrackingTables); // v103
|
|
141
138
|
for (const m of evalMigrations) { // v104-v108
|
|
142
139
|
mm.register(m);
|
|
@@ -146,9 +143,7 @@ export async function createPgStoreConfig(connectionString: string) {
|
|
|
146
143
|
mm.register(createDatabaseConfigsTable); // v111
|
|
147
144
|
mm.register(changeAssistantPrimaryKey); // v112
|
|
148
145
|
mm.register(addStepThreadId); // v113
|
|
149
|
-
mm.register(addCustomRunConfigColumn); // v114
|
|
150
146
|
mm.register(createA2AApiKeysTable); // v130
|
|
151
|
-
mm.register(addWorkspaceProjectToQueue); // v131
|
|
152
147
|
mm.register(addAssistantOwnerUserId); // v132
|
|
153
148
|
for (const m of taskMigrations) { // v133, v139, v140, v141
|
|
154
149
|
mm.register(m);
|
|
@@ -172,7 +167,6 @@ export async function createPgStoreConfig(connectionString: string) {
|
|
|
172
167
|
mm.register(createCapabilityBundlesTable); // v170
|
|
173
168
|
mm.register(addTaskWorkItemPendingIndexesMigration); // v171
|
|
174
169
|
mm.register(createProjectRoomTables); // v172
|
|
175
|
-
mm.register(addTrustedRunContextColumn); // v173
|
|
176
170
|
mm.register(addProjectLifecycleEventIndex); // v175
|
|
177
171
|
mm.register(addAgentWebAppsIdentity); // v176
|
|
178
172
|
mm.register(createProjectRoomReadStates); // v177
|
|
@@ -182,6 +176,8 @@ export async function createPgStoreConfig(connectionString: string) {
|
|
|
182
176
|
mm.register(createModelProviderModelsTable); // v182
|
|
183
177
|
mm.register(addModelProviderApiStyle); // v183
|
|
184
178
|
mm.register(addModelProviderModelStale); // v184
|
|
179
|
+
mm.register(createDistributedLeases); // v185
|
|
180
|
+
mm.register(createOutboxTable); // v187
|
|
185
181
|
|
|
186
182
|
await mm.migrate();
|
|
187
183
|
|
|
@@ -228,6 +224,8 @@ export async function createPgStoreConfig(connectionString: string) {
|
|
|
228
224
|
projectRoomMessage: new PostgreSQLProjectRoomMessageStore(opts),
|
|
229
225
|
projectRoomReadState: new PostgreSQLProjectRoomReadStateStore(opts),
|
|
230
226
|
userPushSubscription: new PostgreSQLUserPushSubscriptionStore(opts),
|
|
227
|
+
lease: new PostgreSQLLeaseStore(opts),
|
|
228
|
+
outbox: new PostgreSQLOutboxStore(opts),
|
|
231
229
|
vectorStoreProvider: new PGVectorStoreProvider(pool, connectionString),
|
|
232
230
|
checkpoint,
|
|
233
231
|
};
|