@poncho-ai/harness 0.55.0 → 0.58.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +5 -5
- package/CHANGELOG.md +54 -0
- package/dist/index.d.ts +210 -2
- package/dist/index.js +632 -43
- package/package.json +1 -1
- package/src/index.ts +18 -0
- package/src/orchestrator/entries-dual-write.ts +265 -0
- package/src/orchestrator/index.ts +7 -0
- package/src/orchestrator/orchestrator.ts +179 -13
- package/src/orchestrator/run-conversation-turn.ts +108 -0
- package/src/state.ts +66 -1
- package/src/storage/engine.ts +18 -0
- package/src/storage/entries.ts +70 -2
- package/src/storage/memory-engine.ts +57 -2
- package/src/storage/schema.ts +30 -0
- package/src/storage/sql-dialect.ts +127 -2
- package/src/storage/store-adapters.ts +8 -0
- package/test/entries-dual-write.test.ts +172 -0
- package/test/entries-read-cutover.test.ts +96 -0
- package/test/entries-store.test.ts +165 -0
|
@@ -36,6 +36,15 @@ import {
|
|
|
36
36
|
executeConversationTurn,
|
|
37
37
|
flushTurnDraft,
|
|
38
38
|
} from "./turn.js";
|
|
39
|
+
import {
|
|
40
|
+
appendEntriesSafe,
|
|
41
|
+
assistantMessageEntry,
|
|
42
|
+
compactionEntry,
|
|
43
|
+
harnessMessageEntries,
|
|
44
|
+
newHarnessMessagesThisTurn,
|
|
45
|
+
userMessageEntry,
|
|
46
|
+
verifyEntriesParity,
|
|
47
|
+
} from "./entries-dual-write.js";
|
|
39
48
|
|
|
40
49
|
const log = createLogger("orchestrator");
|
|
41
50
|
|
|
@@ -185,6 +194,14 @@ export const runConversationTurn = async (
|
|
|
185
194
|
await opts.conversationStore.update(conversation);
|
|
186
195
|
};
|
|
187
196
|
|
|
197
|
+
// Snapshot the harness-message array as it stood BEFORE this turn so the
|
|
198
|
+
// finalize path can diff out the messages this turn appended (dual-write).
|
|
199
|
+
const preTurnHarnessMessages = conversation._harnessMessages
|
|
200
|
+
? [...conversation._harnessMessages]
|
|
201
|
+
: undefined;
|
|
202
|
+
// The stable per-turn id used to group dual-write entries.
|
|
203
|
+
const turnId = assistantId;
|
|
204
|
+
|
|
188
205
|
// Persist the user turn immediately so a crash mid-run still records what
|
|
189
206
|
// the user said. Fire-and-forget — don't block the run.
|
|
190
207
|
conversation.messages = [...historyMessages, userMessage];
|
|
@@ -197,6 +214,15 @@ export const runConversationTurn = async (
|
|
|
197
214
|
);
|
|
198
215
|
});
|
|
199
216
|
|
|
217
|
+
// DUAL-WRITE (additive, mirrors the user-turn blob write above): append a
|
|
218
|
+
// user_message entry. Fire-and-forget — never blocks or breaks the turn.
|
|
219
|
+
void appendEntriesSafe(
|
|
220
|
+
opts.conversationStore,
|
|
221
|
+
conversation,
|
|
222
|
+
[userMessageEntry(userMessage, turnId)],
|
|
223
|
+
log,
|
|
224
|
+
);
|
|
225
|
+
|
|
200
226
|
try {
|
|
201
227
|
const execution = await executeConversationTurn({
|
|
202
228
|
harness: opts.harness,
|
|
@@ -247,6 +273,48 @@ export const runConversationTurn = async (
|
|
|
247
273
|
...existingHistory,
|
|
248
274
|
...preRunMessages.slice(0, removedCount),
|
|
249
275
|
];
|
|
276
|
+
|
|
277
|
+
// DUAL-WRITE (mirrors the compactedHistory blob write above): the
|
|
278
|
+
// compacted array is [summaryMessage, ...keptMessages]. BEST-EFFORT
|
|
279
|
+
// firstKeptSeq: the entry-log seqs of the kept harness messages
|
|
280
|
+
// aren't known here, so we derive a sentinel from the kept-count by
|
|
281
|
+
// reading the current max harness_message seq and pointing at the
|
|
282
|
+
// tail. We read the existing entries to compute it.
|
|
283
|
+
const summaryMessage = event.compactedMessages[0];
|
|
284
|
+
const keptCount = Math.max(0, event.compactedMessages.length - 1);
|
|
285
|
+
if (summaryMessage) {
|
|
286
|
+
void (async () => {
|
|
287
|
+
try {
|
|
288
|
+
const existing = await opts.conversationStore.readEntries(
|
|
289
|
+
opts.conversationId,
|
|
290
|
+
{ types: ["harness_message"] },
|
|
291
|
+
);
|
|
292
|
+
// firstKeptSeq = seq of the (keptCount)-th-from-last existing
|
|
293
|
+
// harness message, so rebuild keeps exactly that many.
|
|
294
|
+
const harnessSeqs = existing.map((e) => e.seq);
|
|
295
|
+
const firstKeptSeq =
|
|
296
|
+
harnessSeqs.length >= keptCount && keptCount > 0
|
|
297
|
+
? harnessSeqs[harnessSeqs.length - keptCount]!
|
|
298
|
+
: (harnessSeqs[harnessSeqs.length - 1] ?? 0) + 1;
|
|
299
|
+
await appendEntriesSafe(
|
|
300
|
+
opts.conversationStore,
|
|
301
|
+
conversation,
|
|
302
|
+
[
|
|
303
|
+
compactionEntry(summaryMessage, firstKeptSeq, {
|
|
304
|
+
tokensBefore: conversation.contextTokens,
|
|
305
|
+
}),
|
|
306
|
+
],
|
|
307
|
+
log,
|
|
308
|
+
);
|
|
309
|
+
} catch (err) {
|
|
310
|
+
log.error(
|
|
311
|
+
`[entries-dual-write] compaction append failed: ${
|
|
312
|
+
err instanceof Error ? err.message : String(err)
|
|
313
|
+
}`,
|
|
314
|
+
);
|
|
315
|
+
}
|
|
316
|
+
})();
|
|
317
|
+
}
|
|
250
318
|
}
|
|
251
319
|
}
|
|
252
320
|
if (event.type === "step:completed") {
|
|
@@ -387,6 +455,46 @@ export const runConversationTurn = async (
|
|
|
387
455
|
{ shouldRebuildCanonical },
|
|
388
456
|
);
|
|
389
457
|
await opts.conversationStore.update(conversation);
|
|
458
|
+
|
|
459
|
+
// DUAL-WRITE at finalize (mirrors applyTurnMetadata's _harnessMessages
|
|
460
|
+
// write + the final assistant bubble in conversation.messages):
|
|
461
|
+
// 1. harness_message entries for the messages this turn appended,
|
|
462
|
+
// 2. the final assistant_message entry.
|
|
463
|
+
// Best-effort + fire-and-forget; never blocks the return.
|
|
464
|
+
const finalAssistant =
|
|
465
|
+
conversation.messages[conversation.messages.length - 1];
|
|
466
|
+
const { messages: newHarness, approximate } = newHarnessMessagesThisTurn(
|
|
467
|
+
preTurnHarnessMessages,
|
|
468
|
+
conversation._harnessMessages,
|
|
469
|
+
);
|
|
470
|
+
if (approximate) {
|
|
471
|
+
log.warn(
|
|
472
|
+
`[entries-dual-write] ${opts.conversationId} harness-message diff approximate ` +
|
|
473
|
+
`(blob array shrank this turn — likely compaction); appended full context`,
|
|
474
|
+
);
|
|
475
|
+
}
|
|
476
|
+
const finalizeEntries = [
|
|
477
|
+
...harnessMessageEntries(newHarness, turnId),
|
|
478
|
+
...(finalAssistant && finalAssistant.role === "assistant"
|
|
479
|
+
? [assistantMessageEntry(finalAssistant, turnId, latestRunId)]
|
|
480
|
+
: []),
|
|
481
|
+
];
|
|
482
|
+
void appendEntriesSafe(
|
|
483
|
+
opts.conversationStore,
|
|
484
|
+
conversation,
|
|
485
|
+
finalizeEntries,
|
|
486
|
+
log,
|
|
487
|
+
).then(() =>
|
|
488
|
+
verifyEntriesParity(
|
|
489
|
+
opts.conversationStore,
|
|
490
|
+
opts.conversationId,
|
|
491
|
+
{
|
|
492
|
+
harnessMessages: conversation._harnessMessages,
|
|
493
|
+
displayMessages: conversation.messages,
|
|
494
|
+
},
|
|
495
|
+
log,
|
|
496
|
+
),
|
|
497
|
+
);
|
|
390
498
|
}
|
|
391
499
|
|
|
392
500
|
return {
|
package/src/state.ts
CHANGED
|
@@ -1,4 +1,9 @@
|
|
|
1
1
|
import type { Message } from "@poncho-ai/sdk";
|
|
2
|
+
import {
|
|
3
|
+
type ConversationEntry,
|
|
4
|
+
type NewConversationEntry,
|
|
5
|
+
rebuildConversationFromEntries,
|
|
6
|
+
} from "./storage/entries.js";
|
|
2
7
|
|
|
3
8
|
export interface ConversationState {
|
|
4
9
|
runId: string;
|
|
@@ -142,6 +147,23 @@ export interface ConversationStore {
|
|
|
142
147
|
clearCallbackLock(conversationId: string): Promise<Conversation | undefined>;
|
|
143
148
|
/** List thread conversations anchored under `parentConversationId`. */
|
|
144
149
|
listThreads(parentConversationId: string): Promise<ConversationSummary[]>;
|
|
150
|
+
/**
|
|
151
|
+
* Append entries to a conversation's append-only log (Phase 3 substrate).
|
|
152
|
+
* Assigns a per-conversation monotonic `seq` and a `createdAt` to each
|
|
153
|
+
* entry, persists them in order, and returns the stored entries with those
|
|
154
|
+
* fields filled in. Additive — no existing read path consumes these yet.
|
|
155
|
+
*/
|
|
156
|
+
appendEntries(
|
|
157
|
+
conversationId: string,
|
|
158
|
+
agentId: string,
|
|
159
|
+
tenantId: string | null,
|
|
160
|
+
entries: NewConversationEntry[],
|
|
161
|
+
): Promise<ConversationEntry[]>;
|
|
162
|
+
/** Read a conversation's entries ordered by `seq` ascending. */
|
|
163
|
+
readEntries(
|
|
164
|
+
conversationId: string,
|
|
165
|
+
opts?: { types?: string[]; afterSeq?: number; limit?: number },
|
|
166
|
+
): Promise<ConversationEntry[]>;
|
|
145
167
|
}
|
|
146
168
|
|
|
147
169
|
export type StateProviderName =
|
|
@@ -201,6 +223,7 @@ export class InMemoryStateStore implements StateStore {
|
|
|
201
223
|
|
|
202
224
|
export class InMemoryConversationStore implements ConversationStore {
|
|
203
225
|
private readonly conversations = new Map<string, Conversation>();
|
|
226
|
+
private readonly entries = new Map<string, ConversationEntry[]>();
|
|
204
227
|
private readonly ttlMs?: number;
|
|
205
228
|
|
|
206
229
|
constructor(ttlSeconds?: number) {
|
|
@@ -250,7 +273,12 @@ export class InMemoryConversationStore implements ConversationStore {
|
|
|
250
273
|
|
|
251
274
|
async get(conversationId: string): Promise<Conversation | undefined> {
|
|
252
275
|
this.purgeExpired();
|
|
253
|
-
|
|
276
|
+
const c = this.conversations.get(conversationId);
|
|
277
|
+
if (!c) return undefined;
|
|
278
|
+
// Phase 3c read cutover: rebuild reader-facing fields from the entry log
|
|
279
|
+
// (blob fallback for un-migrated conversations). Clone first — the map
|
|
280
|
+
// holds a live mutable reference and the rebuild overrides fields.
|
|
281
|
+
return rebuildConversationFromEntries({ ...c }, (id) => this.readEntries(id));
|
|
254
282
|
}
|
|
255
283
|
|
|
256
284
|
// In-memory stores already hold the full conversation object, so there's
|
|
@@ -372,6 +400,43 @@ export class InMemoryConversationStore implements ConversationStore {
|
|
|
372
400
|
channelMeta: c.channelMeta,
|
|
373
401
|
}));
|
|
374
402
|
}
|
|
403
|
+
|
|
404
|
+
async appendEntries(
|
|
405
|
+
conversationId: string,
|
|
406
|
+
_agentId: string,
|
|
407
|
+
_tenantId: string | null,
|
|
408
|
+
entries: NewConversationEntry[],
|
|
409
|
+
): Promise<ConversationEntry[]> {
|
|
410
|
+
const list = this.entries.get(conversationId) ?? [];
|
|
411
|
+
// seq is per-conversation: max existing seq + 1, then consecutive.
|
|
412
|
+
let nextSeq = list.reduce((max, e) => (e.seq > max ? e.seq : max), 0) + 1;
|
|
413
|
+
const now = Date.now();
|
|
414
|
+
const stored: ConversationEntry[] = entries.map(
|
|
415
|
+
(e) => ({ ...e, seq: nextSeq++, createdAt: now }) as ConversationEntry,
|
|
416
|
+
);
|
|
417
|
+
this.entries.set(conversationId, [...list, ...stored]);
|
|
418
|
+
return stored;
|
|
419
|
+
}
|
|
420
|
+
|
|
421
|
+
async readEntries(
|
|
422
|
+
conversationId: string,
|
|
423
|
+
opts?: { types?: string[]; afterSeq?: number; limit?: number },
|
|
424
|
+
): Promise<ConversationEntry[]> {
|
|
425
|
+
let list = (this.entries.get(conversationId) ?? [])
|
|
426
|
+
.slice()
|
|
427
|
+
.sort((a, b) => a.seq - b.seq);
|
|
428
|
+
if (opts?.types && opts.types.length > 0) {
|
|
429
|
+
const allowed = new Set(opts.types);
|
|
430
|
+
list = list.filter((e) => allowed.has(e.type));
|
|
431
|
+
}
|
|
432
|
+
if (typeof opts?.afterSeq === "number") {
|
|
433
|
+
list = list.filter((e) => e.seq > opts.afterSeq!);
|
|
434
|
+
}
|
|
435
|
+
if (typeof opts?.limit === "number") {
|
|
436
|
+
list = list.slice(0, opts.limit);
|
|
437
|
+
}
|
|
438
|
+
return list;
|
|
439
|
+
}
|
|
375
440
|
}
|
|
376
441
|
|
|
377
442
|
export type ConversationSummary = {
|
package/src/storage/engine.ts
CHANGED
|
@@ -5,6 +5,7 @@ import type {
|
|
|
5
5
|
ConversationSummary,
|
|
6
6
|
PendingSubagentResult,
|
|
7
7
|
} from "../state.js";
|
|
8
|
+
import type { ConversationEntry, NewConversationEntry } from "./entries.js";
|
|
8
9
|
import type { MainMemory } from "../memory.js";
|
|
9
10
|
import type { TodoItem } from "../todo-tools.js";
|
|
10
11
|
import type { Reminder, ReminderCreateInput, ReminderStatus } from "../reminder-store.js";
|
|
@@ -77,6 +78,23 @@ export interface StorageEngine {
|
|
|
77
78
|
clearCallbackLock(conversationId: string): Promise<Conversation | undefined>;
|
|
78
79
|
/** List thread conversations anchored under `parentConversationId`. */
|
|
79
80
|
listThreads(parentConversationId: string): Promise<ConversationSummary[]>;
|
|
81
|
+
/**
|
|
82
|
+
* Append entries to a conversation's append-only log (Phase 3 substrate).
|
|
83
|
+
* Assigns a per-conversation monotonic `seq` and `createdAt` to each entry,
|
|
84
|
+
* persists them in order, and returns the stored entries. Additive — no
|
|
85
|
+
* read path consumes these yet.
|
|
86
|
+
*/
|
|
87
|
+
appendEntries(
|
|
88
|
+
conversationId: string,
|
|
89
|
+
agentId: string,
|
|
90
|
+
tenantId: string | null,
|
|
91
|
+
entries: NewConversationEntry[],
|
|
92
|
+
): Promise<ConversationEntry[]>;
|
|
93
|
+
/** Read a conversation's entries ordered by `seq` ascending. */
|
|
94
|
+
readEntries(
|
|
95
|
+
conversationId: string,
|
|
96
|
+
opts?: { types?: string[]; afterSeq?: number; limit?: number },
|
|
97
|
+
): Promise<ConversationEntry[]>;
|
|
80
98
|
};
|
|
81
99
|
|
|
82
100
|
// --- Memory (replaces MemoryStore) ---
|
package/src/storage/entries.ts
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
|
-
import type
|
|
2
|
-
import type { PendingSubagentResult } from "../state.js";
|
|
1
|
+
import { createLogger, type Message } from "@poncho-ai/sdk";
|
|
2
|
+
import type { Conversation, PendingSubagentResult } from "../state.js";
|
|
3
|
+
|
|
4
|
+
const entriesReadLog = createLogger("entries-read");
|
|
3
5
|
|
|
4
6
|
/**
|
|
5
7
|
* Append-only conversation entries (Phase 3 substrate).
|
|
@@ -100,6 +102,19 @@ export type ConversationEntry =
|
|
|
100
102
|
| SubagentResultEntry
|
|
101
103
|
| CallbackStartedEntry;
|
|
102
104
|
|
|
105
|
+
/**
|
|
106
|
+
* An entry to append, before the engine assigns `seq` and `createdAt`. This
|
|
107
|
+
* is a DISTRIBUTIVE omit — `Omit<ConversationEntry, K>` over a union would
|
|
108
|
+
* collapse to only the keys common to every member (dropping `message`,
|
|
109
|
+
* `summaryMessage`, etc.), so we distribute over the union with a
|
|
110
|
+
* conditional type to omit those fields from each member individually.
|
|
111
|
+
*/
|
|
112
|
+
export type NewConversationEntry = ConversationEntry extends infer T
|
|
113
|
+
? T extends ConversationEntry
|
|
114
|
+
? Omit<T, "seq" | "createdAt">
|
|
115
|
+
: never
|
|
116
|
+
: never;
|
|
117
|
+
|
|
103
118
|
/**
|
|
104
119
|
* Rebuild the LLM-visible message context from the entry log.
|
|
105
120
|
*
|
|
@@ -202,3 +217,56 @@ export function getPendingSubagentResults(
|
|
|
202
217
|
.filter((e) => !consumed.has(e.seq))
|
|
203
218
|
.map((e) => e.result);
|
|
204
219
|
}
|
|
220
|
+
|
|
221
|
+
// A very large tail so the rebuilt display snapshot is the full transcript.
|
|
222
|
+
// Display callers slice to whatever window they actually render.
|
|
223
|
+
const FULL_TRANSCRIPT_TAIL = 100_000;
|
|
224
|
+
|
|
225
|
+
/**
|
|
226
|
+
* Phase 3c read cutover: rebuild a conversation's reader-facing fields from
|
|
227
|
+
* the append-only entry log, with a blob fallback for conversations that
|
|
228
|
+
* predate dual-write.
|
|
229
|
+
*
|
|
230
|
+
* Call this in every conversation `get`/`getWithArchive` path AFTER the
|
|
231
|
+
* Conversation has been constructed from the stored row/blob. It:
|
|
232
|
+
* - reads the entry log via `readEntries`,
|
|
233
|
+
* - if NON-EMPTY, overrides `_harnessMessages`, `messages`, and
|
|
234
|
+
* `pendingSubagentResults` with entry-derived values,
|
|
235
|
+
* - if EMPTY (un-migrated conversation), leaves the blob-derived fields
|
|
236
|
+
* untouched (fallback),
|
|
237
|
+
* - on ANY error, logs and falls back to the blob (never throws — this is
|
|
238
|
+
* a hot read path).
|
|
239
|
+
*
|
|
240
|
+
* `_continuationMessages` and `pendingApprovals` are NOT modeled as entries
|
|
241
|
+
* yet and are intentionally left as blob fields.
|
|
242
|
+
*
|
|
243
|
+
* Kill-switch: set `PONCHO_READ_ENTRIES=0` to instantly revert to pure blob
|
|
244
|
+
* reads without a deploy (rebuild is ON by default).
|
|
245
|
+
*
|
|
246
|
+
* NOTE: mutates `conversation` in place and returns it. Callers that hand
|
|
247
|
+
* back a shared/mutable Conversation reference (the in-memory stores) MUST
|
|
248
|
+
* pass a clone, or the override will corrupt their stored object.
|
|
249
|
+
*/
|
|
250
|
+
export async function rebuildConversationFromEntries(
|
|
251
|
+
conversation: Conversation,
|
|
252
|
+
readEntries: (conversationId: string) => Promise<ConversationEntry[]>,
|
|
253
|
+
): Promise<Conversation> {
|
|
254
|
+
// Kill-switch: ON by default; PONCHO_READ_ENTRIES="0" reverts to blob reads.
|
|
255
|
+
if (process.env.PONCHO_READ_ENTRIES === "0") return conversation;
|
|
256
|
+
|
|
257
|
+
try {
|
|
258
|
+
const entries = await readEntries(conversation.conversationId);
|
|
259
|
+
if (entries.length === 0) return conversation; // fallback: pre-dual-write
|
|
260
|
+
conversation._harnessMessages = buildLlmContext(entries);
|
|
261
|
+
conversation.messages = buildDisplaySnapshot(entries, FULL_TRANSCRIPT_TAIL).messages;
|
|
262
|
+
conversation.pendingSubagentResults = getPendingSubagentResults(entries);
|
|
263
|
+
return conversation;
|
|
264
|
+
} catch (err) {
|
|
265
|
+
entriesReadLog.warn(
|
|
266
|
+
`[entries-read] ${conversation.conversationId} rebuild failed, using blob: ${
|
|
267
|
+
err instanceof Error ? err.message : String(err)
|
|
268
|
+
}`,
|
|
269
|
+
);
|
|
270
|
+
return conversation;
|
|
271
|
+
}
|
|
272
|
+
}
|
|
@@ -14,6 +14,11 @@ import type { MainMemory } from "../memory.js";
|
|
|
14
14
|
import type { TodoItem } from "../todo-tools.js";
|
|
15
15
|
import type { Reminder, ReminderCreateInput, ReminderStatus } from "../reminder-store.js";
|
|
16
16
|
import type { StorageEngine, VfsDirEntry, VfsStat } from "./engine.js";
|
|
17
|
+
import {
|
|
18
|
+
type ConversationEntry,
|
|
19
|
+
type NewConversationEntry,
|
|
20
|
+
rebuildConversationFromEntries,
|
|
21
|
+
} from "./entries.js";
|
|
17
22
|
|
|
18
23
|
// ---------------------------------------------------------------------------
|
|
19
24
|
// Internal VFS entry type
|
|
@@ -59,6 +64,8 @@ export class InMemoryEngine implements StorageEngine {
|
|
|
59
64
|
|
|
60
65
|
// Conversation data
|
|
61
66
|
private convs = new Map<string, Conversation>();
|
|
67
|
+
// Append-only conversation entries (Phase 3 substrate)
|
|
68
|
+
private entries = new Map<string, ConversationEntry[]>();
|
|
62
69
|
// Memory data
|
|
63
70
|
private mem = new Map<string, MainMemory>();
|
|
64
71
|
// Todos data
|
|
@@ -100,13 +107,24 @@ export class InMemoryEngine implements StorageEngine {
|
|
|
100
107
|
},
|
|
101
108
|
|
|
102
109
|
get: async (conversationId: string): Promise<Conversation | undefined> => {
|
|
103
|
-
|
|
110
|
+
const c = this.convs.get(conversationId);
|
|
111
|
+
if (!c) return undefined;
|
|
112
|
+
// Phase 3c read cutover: rebuild reader-facing fields from the entry
|
|
113
|
+
// log (blob fallback for un-migrated conversations). Clone first — the
|
|
114
|
+
// map holds a live mutable reference and the rebuild overrides fields.
|
|
115
|
+
return rebuildConversationFromEntries({ ...c }, (id) =>
|
|
116
|
+
this.conversations.readEntries(id),
|
|
117
|
+
);
|
|
104
118
|
},
|
|
105
119
|
|
|
106
120
|
// In-memory storage has no separate archive blob, so both variants
|
|
107
121
|
// return the same conversation object.
|
|
108
122
|
getWithArchive: async (conversationId: string): Promise<Conversation | undefined> => {
|
|
109
|
-
|
|
123
|
+
const c = this.convs.get(conversationId);
|
|
124
|
+
if (!c) return undefined;
|
|
125
|
+
return rebuildConversationFromEntries({ ...c }, (id) =>
|
|
126
|
+
this.conversations.readEntries(id),
|
|
127
|
+
);
|
|
110
128
|
},
|
|
111
129
|
|
|
112
130
|
getStatusSnapshot: async (
|
|
@@ -239,6 +257,43 @@ export class InMemoryEngine implements StorageEngine {
|
|
|
239
257
|
results.sort((a, b) => b.updatedAt - a.updatedAt);
|
|
240
258
|
return results;
|
|
241
259
|
},
|
|
260
|
+
|
|
261
|
+
appendEntries: async (
|
|
262
|
+
conversationId: string,
|
|
263
|
+
_agentId: string,
|
|
264
|
+
_tenantId: string | null,
|
|
265
|
+
entries: NewConversationEntry[],
|
|
266
|
+
): Promise<ConversationEntry[]> => {
|
|
267
|
+
const list = this.entries.get(conversationId) ?? [];
|
|
268
|
+
// seq is per-conversation: max existing seq + 1, then consecutive.
|
|
269
|
+
let nextSeq = list.reduce((max, e) => (e.seq > max ? e.seq : max), 0) + 1;
|
|
270
|
+
const now = Date.now();
|
|
271
|
+
const stored: ConversationEntry[] = entries.map(
|
|
272
|
+
(e) => ({ ...e, seq: nextSeq++, createdAt: now }) as ConversationEntry,
|
|
273
|
+
);
|
|
274
|
+
this.entries.set(conversationId, [...list, ...stored]);
|
|
275
|
+
return stored;
|
|
276
|
+
},
|
|
277
|
+
|
|
278
|
+
readEntries: async (
|
|
279
|
+
conversationId: string,
|
|
280
|
+
opts?: { types?: string[]; afterSeq?: number; limit?: number },
|
|
281
|
+
): Promise<ConversationEntry[]> => {
|
|
282
|
+
let list = (this.entries.get(conversationId) ?? [])
|
|
283
|
+
.slice()
|
|
284
|
+
.sort((a, b) => a.seq - b.seq);
|
|
285
|
+
if (opts?.types && opts.types.length > 0) {
|
|
286
|
+
const allowed = new Set(opts.types);
|
|
287
|
+
list = list.filter((e) => allowed.has(e.type));
|
|
288
|
+
}
|
|
289
|
+
if (typeof opts?.afterSeq === "number") {
|
|
290
|
+
list = list.filter((e) => e.seq > opts.afterSeq!);
|
|
291
|
+
}
|
|
292
|
+
if (typeof opts?.limit === "number") {
|
|
293
|
+
list = list.slice(0, opts.limit);
|
|
294
|
+
}
|
|
295
|
+
return list;
|
|
296
|
+
},
|
|
242
297
|
};
|
|
243
298
|
|
|
244
299
|
// -----------------------------------------------------------------------
|
package/src/storage/schema.ts
CHANGED
|
@@ -214,4 +214,34 @@ export const migrations: Migration[] = [
|
|
|
214
214
|
];
|
|
215
215
|
},
|
|
216
216
|
},
|
|
217
|
+
{
|
|
218
|
+
version: 8,
|
|
219
|
+
name: "conversation_entries",
|
|
220
|
+
// Append-only conversation log (Phase 3 substrate). Additive: no
|
|
221
|
+
// existing table or behavior changes. `seq` is a per-conversation
|
|
222
|
+
// monotonic order assigned by the application (NOT an autoincrement
|
|
223
|
+
// serial), so the same seq space restarts at 1 for every conversation.
|
|
224
|
+
// The UNIQUE (conversation_id, seq) constraint backstops the
|
|
225
|
+
// app-assigned ordering against concurrent writers.
|
|
226
|
+
up: (d) => {
|
|
227
|
+
const jsonType = d === "sqlite" ? "TEXT" : "JSONB";
|
|
228
|
+
const tsDefault = d === "sqlite" ? "datetime('now')" : "NOW()";
|
|
229
|
+
const autoTs = `DEFAULT (${tsDefault})`;
|
|
230
|
+
return [
|
|
231
|
+
`CREATE TABLE IF NOT EXISTS conversation_entries (
|
|
232
|
+
seq INTEGER NOT NULL,
|
|
233
|
+
id TEXT NOT NULL UNIQUE,
|
|
234
|
+
agent_id TEXT NOT NULL,
|
|
235
|
+
tenant_id TEXT NOT NULL DEFAULT '__default__',
|
|
236
|
+
conversation_id TEXT NOT NULL,
|
|
237
|
+
type TEXT NOT NULL,
|
|
238
|
+
payload ${jsonType} NOT NULL,
|
|
239
|
+
created_at TIMESTAMP ${autoTs},
|
|
240
|
+
UNIQUE (conversation_id, seq)
|
|
241
|
+
)`,
|
|
242
|
+
`CREATE INDEX IF NOT EXISTS idx_conversation_entries_seq
|
|
243
|
+
ON conversation_entries (conversation_id, seq)`,
|
|
244
|
+
];
|
|
245
|
+
},
|
|
246
|
+
},
|
|
217
247
|
];
|
|
@@ -22,6 +22,11 @@ import type { MainMemory } from "../memory.js";
|
|
|
22
22
|
import type { TodoItem } from "../todo-tools.js";
|
|
23
23
|
import type { Reminder, ReminderCreateInput, ReminderStatus } from "../reminder-store.js";
|
|
24
24
|
import type { StorageEngine, VfsDirEntry, VfsStat } from "./engine.js";
|
|
25
|
+
import {
|
|
26
|
+
type ConversationEntry,
|
|
27
|
+
type NewConversationEntry,
|
|
28
|
+
rebuildConversationFromEntries,
|
|
29
|
+
} from "./entries.js";
|
|
25
30
|
import { type DialectTag, migrations } from "./schema.js";
|
|
26
31
|
|
|
27
32
|
// ---------------------------------------------------------------------------
|
|
@@ -324,7 +329,12 @@ export abstract class SqlStorageEngine implements StorageEngine {
|
|
|
324
329
|
? JSON.parse(row.continuation_messages)
|
|
325
330
|
: row.continuation_messages;
|
|
326
331
|
}
|
|
327
|
-
|
|
332
|
+
// Phase 3c read cutover: rebuild reader-facing fields from the
|
|
333
|
+
// append-only entry log, falling back to the blob for un-migrated
|
|
334
|
+
// conversations. parseConversation returns a fresh object, so no clone.
|
|
335
|
+
return rebuildConversationFromEntries(conv, (id) =>
|
|
336
|
+
this.conversations.readEntries(id),
|
|
337
|
+
);
|
|
328
338
|
},
|
|
329
339
|
|
|
330
340
|
getStatusSnapshot: async (
|
|
@@ -407,7 +417,11 @@ export abstract class SqlStorageEngine implements StorageEngine {
|
|
|
407
417
|
? JSON.parse(row.continuation_messages)
|
|
408
418
|
: row.continuation_messages;
|
|
409
419
|
}
|
|
410
|
-
|
|
420
|
+
// Phase 3c read cutover: rebuild reader-facing fields from the entry
|
|
421
|
+
// log (blob fallback for un-migrated conversations).
|
|
422
|
+
return rebuildConversationFromEntries(conv, (id) =>
|
|
423
|
+
this.conversations.readEntries(id),
|
|
424
|
+
);
|
|
411
425
|
},
|
|
412
426
|
|
|
413
427
|
create: async (
|
|
@@ -628,6 +642,117 @@ export abstract class SqlStorageEngine implements StorageEngine {
|
|
|
628
642
|
]);
|
|
629
643
|
return rows.map((r) => this.rowToSummary(r));
|
|
630
644
|
},
|
|
645
|
+
|
|
646
|
+
appendEntries: async (
|
|
647
|
+
conversationId: string,
|
|
648
|
+
agentId: string,
|
|
649
|
+
tenantId: string | null,
|
|
650
|
+
entries: NewConversationEntry[],
|
|
651
|
+
): Promise<ConversationEntry[]> => {
|
|
652
|
+
if (entries.length === 0) return [];
|
|
653
|
+
const tid = normalizeTenant(tenantId);
|
|
654
|
+
|
|
655
|
+
// CONCURRENCY: seq is assigned by the app as COALESCE(MAX(seq),0)+1 per
|
|
656
|
+
// conversation, inside a transaction. Two concurrent writers can read
|
|
657
|
+
// the same MAX and collide on the UNIQUE (conversation_id, seq)
|
|
658
|
+
// constraint; we let the loser's transaction fail and retry the whole
|
|
659
|
+
// compute-and-insert up to 3 times. This MAX+1+retry scheme is correct
|
|
660
|
+
// for the modest concurrency here (a single conversation rarely has
|
|
661
|
+
// simultaneous appenders). A later hardening — a per-conversation
|
|
662
|
+
// advisory lock — can replace it before any read path depends on these
|
|
663
|
+
// entries.
|
|
664
|
+
const maxAttempts = 3;
|
|
665
|
+
let lastErr: unknown;
|
|
666
|
+
for (let attempt = 0; attempt < maxAttempts; attempt++) {
|
|
667
|
+
const stored: ConversationEntry[] = [];
|
|
668
|
+
try {
|
|
669
|
+
await this.executor.transaction(async () => {
|
|
670
|
+
stored.length = 0; // reset on retry within the same closure
|
|
671
|
+
const row = await this.executor.get<{ max_seq: number | null }>(
|
|
672
|
+
rewrite(
|
|
673
|
+
"SELECT MAX(seq) AS max_seq FROM conversation_entries WHERE conversation_id = $1",
|
|
674
|
+
this.dialect,
|
|
675
|
+
),
|
|
676
|
+
[conversationId],
|
|
677
|
+
);
|
|
678
|
+
let nextSeq = Number(row?.max_seq ?? 0) + 1;
|
|
679
|
+
const now = Date.now();
|
|
680
|
+
for (const entry of entries) {
|
|
681
|
+
const seq = nextSeq++;
|
|
682
|
+
const createdAt = now;
|
|
683
|
+
// Persist the full entry object (minus seq/createdAt, which are
|
|
684
|
+
// columns) as the JSON payload. seq, id, created_at are columns.
|
|
685
|
+
const payload = JSON.stringify(entry);
|
|
686
|
+
await this.executor.run(
|
|
687
|
+
rewrite(
|
|
688
|
+
`INSERT INTO conversation_entries
|
|
689
|
+
(seq, id, agent_id, tenant_id, conversation_id, type, payload, created_at)
|
|
690
|
+
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`,
|
|
691
|
+
this.dialect,
|
|
692
|
+
),
|
|
693
|
+
[
|
|
694
|
+
seq,
|
|
695
|
+
entry.id,
|
|
696
|
+
agentId,
|
|
697
|
+
tid,
|
|
698
|
+
conversationId,
|
|
699
|
+
entry.type,
|
|
700
|
+
payload,
|
|
701
|
+
new Date(createdAt).toISOString(),
|
|
702
|
+
],
|
|
703
|
+
);
|
|
704
|
+
stored.push({ ...entry, seq, createdAt } as ConversationEntry);
|
|
705
|
+
}
|
|
706
|
+
});
|
|
707
|
+
return stored;
|
|
708
|
+
} catch (err) {
|
|
709
|
+
lastErr = err;
|
|
710
|
+
// Retry on any failure (the likely cause is a UNIQUE collision from
|
|
711
|
+
// a concurrent writer); the next attempt re-reads MAX(seq).
|
|
712
|
+
}
|
|
713
|
+
}
|
|
714
|
+
throw lastErr;
|
|
715
|
+
},
|
|
716
|
+
|
|
717
|
+
readEntries: async (
|
|
718
|
+
conversationId: string,
|
|
719
|
+
opts?: { types?: string[]; afterSeq?: number; limit?: number },
|
|
720
|
+
): Promise<ConversationEntry[]> => {
|
|
721
|
+
const params: unknown[] = [conversationId];
|
|
722
|
+
let sql = "SELECT seq, id, payload, created_at FROM conversation_entries WHERE conversation_id = $1";
|
|
723
|
+
if (opts?.types && opts.types.length > 0) {
|
|
724
|
+
const placeholders = opts.types.map(
|
|
725
|
+
(_t, i) => `$${params.length + 1 + i}`,
|
|
726
|
+
);
|
|
727
|
+
sql += ` AND type IN (${placeholders.join(", ")})`;
|
|
728
|
+
params.push(...opts.types);
|
|
729
|
+
}
|
|
730
|
+
if (typeof opts?.afterSeq === "number") {
|
|
731
|
+
sql += ` AND seq > $${params.length + 1}`;
|
|
732
|
+
params.push(opts.afterSeq);
|
|
733
|
+
}
|
|
734
|
+
sql += " ORDER BY seq ASC";
|
|
735
|
+
if (typeof opts?.limit === "number") {
|
|
736
|
+
sql += ` LIMIT $${params.length + 1}`;
|
|
737
|
+
params.push(opts.limit);
|
|
738
|
+
}
|
|
739
|
+
const rows = await this.executor.all<{
|
|
740
|
+
seq: number;
|
|
741
|
+
id: string;
|
|
742
|
+
payload: unknown;
|
|
743
|
+
created_at: string;
|
|
744
|
+
}>(rewrite(sql, this.dialect), params);
|
|
745
|
+
return rows.map((r) => {
|
|
746
|
+
const parsed =
|
|
747
|
+
typeof r.payload === "string" ? JSON.parse(r.payload) : r.payload;
|
|
748
|
+
return {
|
|
749
|
+
...parsed,
|
|
750
|
+
id: r.id,
|
|
751
|
+
seq: Number(r.seq),
|
|
752
|
+
createdAt: new Date(r.created_at).getTime(),
|
|
753
|
+
} as ConversationEntry;
|
|
754
|
+
});
|
|
755
|
+
},
|
|
631
756
|
};
|
|
632
757
|
|
|
633
758
|
// -----------------------------------------------------------------------
|
|
@@ -58,6 +58,14 @@ export function createConversationStoreFromEngine(
|
|
|
58
58
|
engine.conversations.clearCallbackLock(conversationId),
|
|
59
59
|
listThreads: (parentConversationId: string) =>
|
|
60
60
|
engine.conversations.listThreads(parentConversationId),
|
|
61
|
+
appendEntries: (
|
|
62
|
+
conversationId: string,
|
|
63
|
+
agentId: string,
|
|
64
|
+
tenantId: string | null,
|
|
65
|
+
entries,
|
|
66
|
+
) => engine.conversations.appendEntries(conversationId, agentId, tenantId, entries),
|
|
67
|
+
readEntries: (conversationId: string, opts) =>
|
|
68
|
+
engine.conversations.readEntries(conversationId, opts),
|
|
61
69
|
};
|
|
62
70
|
}
|
|
63
71
|
|