@yahyahammad/venastine 0.1.0 → 0.3.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +19 -14
- package/CONFIG_ARCHITECTURE.md +2002 -0
- package/NOTICE +1 -1
- package/README.md +249 -93
- package/agents/builtin/build.md +73 -0
- package/agents/builtin/general.md +80 -0
- package/agents/builtin/plan.md +23 -5
- package/agents/builtin/test.md +65 -0
- package/agents/builtin/writer.md +67 -0
- package/agents/manager.py +261 -195
- package/agents/subagent_tool.py +280 -23
- package/agents/tui_commands.py +31 -21
- package/bin/venastine.mjs +528 -382
- package/bin/venastine.py +71 -0
- package/cli_commands.py +835 -0
- package/config.py +95 -1198
- package/config.yaml +1090 -0
- package/config_edit.py +1301 -0
- package/config_schema.py +966 -0
- package/config_update.py +749 -0
- package/core/agent_activity.py +256 -0
- package/core/ansi.py +452 -0
- package/core/approval.py +27 -5
- package/core/call_view.py +364 -0
- package/core/cancel.py +121 -0
- package/core/chatgpt_auth.py +747 -0
- package/core/claims_view.py +107 -0
- package/core/client.py +1249 -82
- package/core/commands.py +454 -0
- package/core/compaction.py +1735 -959
- package/core/config_loader.py +516 -97
- package/core/connect.py +686 -0
- package/core/events.py +68 -6
- package/core/file_mentions.py +280 -0
- package/core/interaction.py +87 -4
- package/core/line_pattern.py +128 -0
- package/core/loop.py +2566 -1353
- package/core/memory.py +482 -370
- package/core/pipeline_models.py +23 -9
- package/core/provider_errors.py +234 -0
- package/core/reasoning/confidence_scoring.py +70 -54
- package/core/reasoning/json_retry.py +19 -1
- package/core/reasoning/orchestrator.py +232 -53
- package/core/reasoning/output_writer.py +90 -31
- package/core/reasoning/payload_validation.py +133 -7
- package/core/reasoning/pipeline_storage.py +427 -361
- package/core/reasoning/review.py +659 -615
- package/core/reasoning/scholar.py +38 -14
- package/core/reasoning/source_corpus.py +110 -4
- package/core/replay.py +315 -167
- package/core/session.py +36 -12
- package/core/session_view.py +259 -0
- package/core/session_wake.py +259 -0
- package/core/shell_sessions.py +1553 -0
- package/core/workspace_trust.py +1 -1
- package/credentials.py +308 -180
- package/database.py +82 -1
- package/{PRIVACY.md → docs/PRIVACY.md} +19 -7
- package/docs/SECURITY.md +108 -0
- package/docs/THIRD_PARTY_NOTICES.md +148 -0
- package/env_secrets.py +118 -77
- package/json_store.py +236 -16
- package/logging_setup.py +121 -11
- package/main.py +1672 -198
- package/mcp_client/client.py +257 -23
- package/mcp_client/config.py +484 -379
- package/mcp_client/registration.py +163 -2
- package/memories/manager.py +15 -0
- package/memories/tui_commands.py +21 -14
- package/package.json +303 -73
- package/project_init/config_files.py +4 -0
- package/project_init/generator.py +201 -37
- package/project_init/manifest.py +418 -406
- package/project_init/tui_commands.py +7 -4
- package/prompts/revalidate.md +1 -1
- package/prompts/revise.md +17 -15
- package/prompts/system_prompts.py +64 -10
- package/providers.json.example +94 -92
- package/pyproject.toml +149 -100
- package/requirements.txt +252 -29
- package/safety/policy_enforcement.py +1082 -682
- package/security/askpass.py +419 -0
- package/security/capability.py +37 -7
- package/security/confine.py +484 -0
- package/security/hosted.py +182 -0
- package/security/permissions.py +54 -7
- package/security/posture.py +124 -12
- package/security/protected_paths.py +198 -6
- package/security/sandbox.py +3262 -139
- package/security/secrets.py +855 -0
- package/skills/tui_commands.py +3 -4
- package/storage.py +1338 -1005
- package/tools/base.py +465 -305
- package/tools/builtin/_math_common.py +306 -16
- package/tools/builtin/_net_common.py +267 -61
- package/tools/builtin/arxiv.py +681 -223
- package/tools/builtin/ask_user.py +231 -213
- package/tools/builtin/discrete_math.py +35 -4
- package/tools/builtin/fetch_url.py +239 -21
- package/tools/builtin/file_ops.py +341 -33
- package/tools/builtin/geometry.py +184 -148
- package/tools/builtin/get_time.py +2 -2
- package/tools/builtin/linear_algebra.py +66 -23
- package/tools/builtin/load_skill.py +10 -0
- package/tools/builtin/logic.py +171 -12
- package/tools/builtin/pin.py +46 -12
- package/tools/builtin/probability_stats.py +224 -228
- package/tools/builtin/project_docs.py +80 -21
- package/tools/builtin/remember.py +21 -4
- package/tools/builtin/shell.py +586 -67
- package/tools/builtin/shell_sessions.py +613 -0
- package/tools/builtin/symbolic_math.py +80 -10
- package/tools/builtin/todo.py +5 -3
- package/tools/builtin/web_search.py +329 -151
- package/tools/context.py +23 -0
- package/tools/isolation.py +387 -314
- package/tools/registry.py +374 -29
- package/tui/app.py +5281 -397
- package/tui/app.tcss +290 -9
- package/tui/commands.py +26 -61
- package/tui/diffs.py +286 -265
- package/tui/history.py +128 -0
- package/tui/hosted_driver.py +102 -0
- package/tui/markdown.py +1329 -0
- package/tui/meters.py +364 -0
- package/tui/screens.py +346 -92
- package/tui/themes.py +599 -366
- package/tui/widgets.py +2673 -116
- package/SECURITY.md +0 -88
- package/THIRD_PARTY_NOTICES.md +0 -100
package/storage.py
CHANGED
|
@@ -1,1005 +1,1338 @@
|
|
|
1
|
-
import json
|
|
2
|
-
import logging
|
|
3
|
-
from datetime import
|
|
4
|
-
from typing import
|
|
5
|
-
from uuid import UUID, uuid4
|
|
6
|
-
|
|
7
|
-
from
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
from
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
#
|
|
22
|
-
#
|
|
23
|
-
#
|
|
24
|
-
#
|
|
25
|
-
#
|
|
26
|
-
#
|
|
27
|
-
#
|
|
28
|
-
#
|
|
29
|
-
#
|
|
30
|
-
#
|
|
31
|
-
#
|
|
32
|
-
#
|
|
33
|
-
#
|
|
34
|
-
#
|
|
35
|
-
|
|
36
|
-
#
|
|
37
|
-
|
|
38
|
-
#
|
|
39
|
-
#
|
|
40
|
-
#
|
|
41
|
-
#
|
|
42
|
-
#
|
|
43
|
-
#
|
|
44
|
-
# the
|
|
45
|
-
#
|
|
46
|
-
#
|
|
47
|
-
#
|
|
48
|
-
#
|
|
49
|
-
#
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
#
|
|
62
|
-
#
|
|
63
|
-
#
|
|
64
|
-
#
|
|
65
|
-
#
|
|
66
|
-
|
|
67
|
-
#
|
|
68
|
-
#
|
|
69
|
-
#
|
|
70
|
-
#
|
|
71
|
-
#
|
|
72
|
-
#
|
|
73
|
-
#
|
|
74
|
-
|
|
75
|
-
#
|
|
76
|
-
#
|
|
77
|
-
#
|
|
78
|
-
#
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
the
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
"""
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
""
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
"""
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
def
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
""
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
.
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
.
|
|
539
|
-
|
|
540
|
-
)
|
|
541
|
-
return [
|
|
542
|
-
{
|
|
543
|
-
"id":
|
|
544
|
-
"
|
|
545
|
-
|
|
546
|
-
#
|
|
547
|
-
#
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
)
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
is
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
"""
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
for
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
the
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
is
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
"""
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1
|
+
import json
|
|
2
|
+
import logging
|
|
3
|
+
from datetime import UTC, datetime
|
|
4
|
+
from typing import Any, Optional
|
|
5
|
+
from uuid import UUID, uuid4
|
|
6
|
+
|
|
7
|
+
from sqlalchemy import (
|
|
8
|
+
func as sa_func, # real sqlalchemy; sqlmodel is faked in tests, this is not
|
|
9
|
+
)
|
|
10
|
+
from sqlmodel import JSON, Field, Session, SQLModel, select
|
|
11
|
+
|
|
12
|
+
from database import engine # your SQLAlchemy engine, assumed to exist here
|
|
13
|
+
|
|
14
|
+
logger = logging.getLogger(__name__)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class ConversationThread(SQLModel, table=True):
|
|
18
|
+
id: UUID = Field(default_factory=uuid4, primary_key=True)
|
|
19
|
+
created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
|
|
20
|
+
extra_data: dict[str, Any] = Field(default_factory=dict, sa_type=JSON)
|
|
21
|
+
# ROADMAP_v2 §27 (T1). What this thread IS: "chat" (a conversation a
|
|
22
|
+
# human had), "research_pass" (one pass of the ten-pass pipeline) or
|
|
23
|
+
# "subagent" (a spawned agent, the §20 reviewer, or the compactor).
|
|
24
|
+
#
|
|
25
|
+
# A COLUMN, not an extra_data key. That field exists and would need no
|
|
26
|
+
# migration, but it is an opaque JSON blob -- filtering on it means
|
|
27
|
+
# loading every row and testing in Python, and list_threads() is the
|
|
28
|
+
# one query that must stay cheap on a database gaining ~10 rows per
|
|
29
|
+
# research run.
|
|
30
|
+
#
|
|
31
|
+
# NOT indexed, deliberately. ensure_columns() ALTERs in a column and
|
|
32
|
+
# does not build indexes, and create_all() will not add one to a table
|
|
33
|
+
# that already exists -- so index=True here would produce an index on
|
|
34
|
+
# fresh databases and none on migrated ones. A divergence that shows
|
|
35
|
+
# up only as a performance difference is worse than no index at all;
|
|
36
|
+
# the WHERE clause is what T1 was buying, not the index.
|
|
37
|
+
kind: str = Field(default="chat")
|
|
38
|
+
# (#32) When this thread was last written to -- ANY archived row,
|
|
39
|
+
# user, assistant or tool alike. The picker orders by this rather
|
|
40
|
+
# than by creation, because "most recent first" should surface the
|
|
41
|
+
# conversation you were just in, not the one you merely opened last.
|
|
42
|
+
#
|
|
43
|
+
# Nullable deliberately, twice over. Semantically: NULL means "never
|
|
44
|
+
# received a message", and the reader falls back to created_at --
|
|
45
|
+
# which is also what every row predating this column reads as, since
|
|
46
|
+
# the additive migration never backfills (M7). Mechanically: the
|
|
47
|
+
# honest default is per-row Python (a timestamp), which has no SQL
|
|
48
|
+
# literal, so the migrator could not have added it NOT NULL anyway
|
|
49
|
+
# (#26's WARNING path). save_message is the single writer of
|
|
50
|
+
# MessageLog rows in production and stamps this in the same session
|
|
51
|
+
# it writes the message, so the two cannot drift.
|
|
52
|
+
last_activity_at: Optional[datetime] = Field(default=None)
|
|
53
|
+
# ROADMAP_v2 §47. WHO spawned this thread, and at which call.
|
|
54
|
+
#
|
|
55
|
+
# Before these, the only trace of the edge was the child's uuid
|
|
56
|
+
# stringified inside the parent's tool-result row -- a repr inside a
|
|
57
|
+
# JSON blob, not a field, not queryable, and skipped by replay's T4
|
|
58
|
+
# anyway. A sidebar row and a `▸ spawn_subagent` line could describe
|
|
59
|
+
# a run nobody could then open.
|
|
60
|
+
#
|
|
61
|
+
# COLUMNS, not `extra_data`, for `kind`'s reason exactly (T1): the
|
|
62
|
+
# question asked of them is "which threads are children of this
|
|
63
|
+
# one", and answering that from an opaque JSON blob means loading
|
|
64
|
+
# every row and testing in Python. Not indexed, also for T1's
|
|
65
|
+
# reason -- ensure_columns() ALTERs a column in and builds no index,
|
|
66
|
+
# and create_all() adds none to a table that already exists, so
|
|
67
|
+
# index=True would mean an index on fresh databases and none on
|
|
68
|
+
# migrated ones.
|
|
69
|
+
#
|
|
70
|
+
# Nullable and additive, so database.ensure_columns() adds them to an
|
|
71
|
+
# existing app.db exactly as it added `pinned`, `kind`, `thinking`
|
|
72
|
+
# and `last_activity_at`. NULL is the honest value for every thread
|
|
73
|
+
# that predates them and for every one nothing spawned.
|
|
74
|
+
parent_thread_id: Optional[UUID] = Field(default=None)
|
|
75
|
+
# The parent's `tool_call_id` for the call that made this thread --
|
|
76
|
+
# the SAME id MessageLog stores on the parent's side, which is what
|
|
77
|
+
# lets one specific `▸ spawn_subagent` line find one specific child
|
|
78
|
+
# when a turn spawned three. None for a run no tool call made: the
|
|
79
|
+
# compactor, the reviewer, /init's initializer.
|
|
80
|
+
parent_call_id: Optional[str] = Field(default=None)
|
|
81
|
+
# Which agent ran, for a reader who has only the row. `kind` says
|
|
82
|
+
# what a thread IS; this says who it was. A viewer opening a stored
|
|
83
|
+
# thread cannot ask the span, because the span is long gone.
|
|
84
|
+
agent_name: Optional[str] = Field(default=None)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
class MessageLog(SQLModel, table=True):
|
|
88
|
+
id: UUID = Field(default_factory=uuid4, primary_key=True)
|
|
89
|
+
thread_id: UUID = Field(foreign_key="conversationthread.id", index=True)
|
|
90
|
+
role: str = Field(description="system, user, assistant, or tool")
|
|
91
|
+
content: str # always JSON-encoded, regardless of the original type
|
|
92
|
+
name: Optional[str] = None
|
|
93
|
+
tool_call_id: Optional[str] = None
|
|
94
|
+
created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
|
|
95
|
+
# ROADMAP_v2 §21. Compaction-exempt within this thread (D26's pin
|
|
96
|
+
# tool). This is the first field this project has ever added to a
|
|
97
|
+
# table that already exists on disk, which is why database.py grew
|
|
98
|
+
# ensure_columns() -- create_all() would have left every existing
|
|
99
|
+
# app.db without it and failed at the next SELECT (M7).
|
|
100
|
+
pinned: bool = Field(default=False)
|
|
101
|
+
# §44. The turn's reasoning, JSON-encoded, or None -- assistant rows
|
|
102
|
+
# only, and only where the provider returned any.
|
|
103
|
+
#
|
|
104
|
+
# A SEPARATE COLUMN rather than a key inside `content`, because these
|
|
105
|
+
# are different KINDS of thing with different rules. `content` is what
|
|
106
|
+
# the model said; this is the provider's own signed blocks, which go
|
|
107
|
+
# back on the wire unchanged or not at all, and which every reader
|
|
108
|
+
# that measures the conversation (the compactor's input, /summary's
|
|
109
|
+
# distillation) has to be able to leave out without parsing around
|
|
110
|
+
# them. Nullable and additive, so database.ensure_columns() adds it to
|
|
111
|
+
# an existing database exactly as it added `pinned` -- see the note on
|
|
112
|
+
# that field's migration in _ordered_rows.
|
|
113
|
+
thinking: Optional[str] = None
|
|
114
|
+
# Batch 91. Why the run this row STARTED failed, or None -- user rows
|
|
115
|
+
# only, written by mark_turn_failed when an exception escapes the loop.
|
|
116
|
+
#
|
|
117
|
+
# A column on the user row rather than a row of its own, and that is
|
|
118
|
+
# the whole design. A failed run leaves no assistant turn behind (D20
|
|
119
|
+
# persists one only when a call returns), so without this a thread
|
|
120
|
+
# whose first call failed read as a request nobody answered. An `error`
|
|
121
|
+
# ROLE would have been a harness notice stored as conversation -- the
|
|
122
|
+
# compactor's `_as_text` and /summary render every role, so it would
|
|
123
|
+
# have been summarized into what the model later reads, which is M8's
|
|
124
|
+
# mistake. A column is read by nothing that measures the conversation:
|
|
125
|
+
# provider translation and the compactor take `content` alone. On the
|
|
126
|
+
# user row because that is what says WHICH turn failed, which a
|
|
127
|
+
# thread-level field could not for a chat. Nullable and additive, so
|
|
128
|
+
# database.ensure_columns() adds it like `thinking`.
|
|
129
|
+
error: Optional[str] = None
|
|
130
|
+
# ROADMAP_v3 §49 (SS5). What wrote this row when the HARNESS did, as a
|
|
131
|
+
# JSON object -- {"kind": "session_wake" | "session_killed_at_quit" |
|
|
132
|
+
# "session_held", "sessions": [{"id", "call_id", "shape"}, ...]} -- or
|
|
133
|
+
# None for everything a person or the model wrote.
|
|
134
|
+
#
|
|
135
|
+
# A column on a USER row, for batch 91's reason one field over. A
|
|
136
|
+
# finished background session has to reach the model as a message that
|
|
137
|
+
# starts a turn, and a tool result cannot carry it: every tool_use is
|
|
138
|
+
# answered in the step that made it (D20, NA11), long before the session
|
|
139
|
+
# ends. A user-role row is the one shape every provider accepts there,
|
|
140
|
+
# and a compacted thread already sends one the person never wrote (M8's
|
|
141
|
+
# summary). A `harness` ROLE would need translating and summarizing as
|
|
142
|
+
# something new; a column is read by nothing that builds a request, so
|
|
143
|
+
# the row goes out as the plain user message it is, while replay and
|
|
144
|
+
# the compactor can still tell it was not the person.
|
|
145
|
+
#
|
|
146
|
+
# NEVER beside a tool_call_id. `pinned_through` treats any kept row's
|
|
147
|
+
# tool_call_id as an answered call, so a wake row carrying its session's
|
|
148
|
+
# start call id would hide the real tool_result -- the HTTP 400 M4
|
|
149
|
+
# exists to prevent. The call id lives inside this JSON instead.
|
|
150
|
+
harness: Optional[str] = None
|
|
151
|
+
# ROADMAP_v3 §81 (IM2). The images a tool result carried, as a JSON list
|
|
152
|
+
# of {"media_type", "data"} with `data` base64 -- tool rows only, and only
|
|
153
|
+
# where `view_image` attached one. NULL everywhere else.
|
|
154
|
+
#
|
|
155
|
+
# A COLUMN rather than blocks inside `content`, for `thinking`'s reason:
|
|
156
|
+
# `content` is read as a string by every consumer of the conversation --
|
|
157
|
+
# replay, the transcript, /copy, the compactor, the summarizer -- and
|
|
158
|
+
# only two readers have any use for the bytes, provider translation and
|
|
159
|
+
# the compactor's size proxy. In the database rather than in a blob store
|
|
160
|
+
# or as a path to the workspace file: a path would make a resumed thread
|
|
161
|
+
# send whatever the file holds NOW, and a second store would be a second
|
|
162
|
+
# place docs/PRIVACY.md's "wipe everything" has to name. Nullable and
|
|
163
|
+
# additive, so database.ensure_columns() adds it like `thinking`.
|
|
164
|
+
images: Optional[str] = None
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
class CompactionCheckpoint(SQLModel, table=True):
|
|
168
|
+
"""One compaction of one thread (ROADMAP_v2 §21).
|
|
169
|
+
|
|
170
|
+
THE ARCHIVE IS NEVER EDITED. Compaction only ever ADDS a row here;
|
|
171
|
+
MessageLog is append-only and stays complete forever, which is what
|
|
172
|
+
§21 AC1 is about and what makes an aggressive compaction trigger safe --
|
|
173
|
+
nothing is ever destroyed, only hidden from the next call. See
|
|
174
|
+
archive_history() for the read that proves it.
|
|
175
|
+
|
|
176
|
+
`summary_text` represents every message in this thread through
|
|
177
|
+
`covers_up_to_message_id` EXCEPT the pinned ones (§21 AC2), which is why
|
|
178
|
+
the derived view has to re-include them separately (M9) -- a single
|
|
179
|
+
watermark cannot express "everything up to K except rows 5 and 6".
|
|
180
|
+
|
|
181
|
+
`strategy` records which way this summary was produced: "rederive"
|
|
182
|
+
(summarized the original messages, so exactly one summarization step
|
|
183
|
+
sits between an original message and what the model sees) or "chain"
|
|
184
|
+
(summarized the previous summary plus what followed it, bounded cost,
|
|
185
|
+
compounding loss). M2 makes rederive the default and chain the
|
|
186
|
+
configurable/fallback case; storing it per row means a thread's
|
|
187
|
+
history of the two is auditable rather than inferred from settings
|
|
188
|
+
that may since have changed.
|
|
189
|
+
"""
|
|
190
|
+
|
|
191
|
+
id: UUID = Field(default_factory=uuid4, primary_key=True)
|
|
192
|
+
thread_id: UUID = Field(foreign_key="conversationthread.id", index=True)
|
|
193
|
+
summary_text: str
|
|
194
|
+
covers_up_to_message_id: UUID
|
|
195
|
+
strategy: str = Field(default="rederive")
|
|
196
|
+
created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
class ThreadSummary(SQLModel, table=True):
|
|
200
|
+
"""A DESCRIPTION of one whole thread (ROADMAP_v2 §21c, M18).
|
|
201
|
+
|
|
202
|
+
NOT A COMPACTION, and the distinction is the entire reason this is a
|
|
203
|
+
separate table rather than a row in CompactionCheckpoint with a
|
|
204
|
+
different `strategy`. The two carry almost the same columns and mean
|
|
205
|
+
opposite things:
|
|
206
|
+
|
|
207
|
+
* a CompactionCheckpoint CHANGES WHAT THE MODEL SEES. `latest_checkpoint`
|
|
208
|
+
resolves by timestamp and feeds ConversationMemory._derived_view(), so
|
|
209
|
+
writing one is an edit to the live conversation.
|
|
210
|
+
* a ThreadSummary changes nothing. It is read by `/summary` to show a
|
|
211
|
+
person, and by `/ref` to inject into a DIFFERENT thread. The thread it
|
|
212
|
+
describes is untouched.
|
|
213
|
+
|
|
214
|
+
So a summary row in the checkpoint table would silently compact the
|
|
215
|
+
thread it was only meant to describe -- and it would win, because it
|
|
216
|
+
would be the newest row. That is the failure this table exists to make
|
|
217
|
+
impossible rather than to warn about.
|
|
218
|
+
|
|
219
|
+
`covers_up_to_message_id` is the last archive row the summary accounts
|
|
220
|
+
for, which is what makes staleness answerable: storage.advances() already
|
|
221
|
+
knows whether a thread has moved past a given position, so a `/ref` of an
|
|
222
|
+
unchanged thread costs no model call and a `/ref` of a grown one is
|
|
223
|
+
re-summarized. Same column name as the checkpoint's on purpose -- it means
|
|
224
|
+
the same thing there, and `advances()` serves both.
|
|
225
|
+
"""
|
|
226
|
+
|
|
227
|
+
id: UUID = Field(default_factory=uuid4, primary_key=True)
|
|
228
|
+
thread_id: UUID = Field(foreign_key="conversationthread.id", index=True)
|
|
229
|
+
summary_text: str
|
|
230
|
+
covers_up_to_message_id: UUID
|
|
231
|
+
created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
class UserMemory(SQLModel, table=True):
|
|
235
|
+
"""One durable fact, carried BETWEEN threads (ROADMAP_v2 §21b).
|
|
236
|
+
|
|
237
|
+
Unlike everything else in this file, a row here is not part of any one
|
|
238
|
+
conversation: `source_thread_id` records where it was learned, but the
|
|
239
|
+
row outlives that thread and is injected into later ones. That is the
|
|
240
|
+
whole point, and it is also why D26 gates `remember` and leaves `pin`
|
|
241
|
+
ungated -- a wrong pin costs some context budget inside a conversation
|
|
242
|
+
the user is watching, while a wrong memory silently shapes
|
|
243
|
+
conversations that have not started yet.
|
|
244
|
+
|
|
245
|
+
SCOPE IS TWO COLUMNS, not one (M12). §21's schema block carries only
|
|
246
|
+
`scope`, but D25 says a project-scoped memory is "keyed to the same
|
|
247
|
+
resolved project path the workspace-trust store uses" -- and with no
|
|
248
|
+
path recorded, `"project"` cannot tell two projects apart, which makes
|
|
249
|
+
§21 AC6 untestable rather than merely unenforced. `project_path` is the
|
|
250
|
+
realpath for `scope="project"` and None for `scope="global"`, taken
|
|
251
|
+
from config_loader.get_project_path() so "which project" means one
|
|
252
|
+
thing across trust, config and memory.
|
|
253
|
+
|
|
254
|
+
`category` is free text the model may set for its own grouping. It is
|
|
255
|
+
deliberately not an enum: the harness has no basis to decide what
|
|
256
|
+
kinds of fact are worth distinguishing, and a wrong enum would push
|
|
257
|
+
every real memory into "other".
|
|
258
|
+
"""
|
|
259
|
+
|
|
260
|
+
id: UUID = Field(default_factory=uuid4, primary_key=True)
|
|
261
|
+
content: str
|
|
262
|
+
category: Optional[str] = None
|
|
263
|
+
# D25: "project" (default) or "global". Narrow by default because the
|
|
264
|
+
# two error directions are not symmetric -- a memory that should have
|
|
265
|
+
# been project-scoped but landed globally contaminates every later
|
|
266
|
+
# conversation with a stale fact about a codebase you may not be
|
|
267
|
+
# working in, and presents as the model simply being confident.
|
|
268
|
+
scope: str = Field(default="project", index=True)
|
|
269
|
+
project_path: Optional[str] = Field(default=None, index=True)
|
|
270
|
+
source_thread_id: UUID = Field(foreign_key="conversationthread.id")
|
|
271
|
+
created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
#: ROADMAP_v2 §27 (T1). The three things a thread can be. Not an Enum:
|
|
275
|
+
#: the column is a plain string so an unmigrated row reads back as data
|
|
276
|
+
#: rather than raising, and config.py's "plain values only" rule means a
|
|
277
|
+
#: vocabulary shared by storage and its callers belongs beside the table
|
|
278
|
+
#: that stores it.
|
|
279
|
+
THREAD_KIND_CHAT = "chat"
|
|
280
|
+
THREAD_KIND_RESEARCH_PASS = "research_pass"
|
|
281
|
+
THREAD_KIND_SUBAGENT = "subagent"
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
def create_thread(kind: str = THREAD_KIND_CHAT, *,
|
|
285
|
+
parent_thread_id: Optional[UUID] = None,
|
|
286
|
+
parent_call_id: Optional[str] = None,
|
|
287
|
+
agent_name: Optional[str] = None) -> UUID:
|
|
288
|
+
"""Starts a brand-new conversation thread and returns its id.
|
|
289
|
+
|
|
290
|
+
`kind` defaults to "chat" (§27 AC1), so every existing caller keeps
|
|
291
|
+
creating conversations and only the paths that know they are creating
|
|
292
|
+
something else pass anything here.
|
|
293
|
+
|
|
294
|
+
The three lineage arguments (§47) are keyword-only and all default to
|
|
295
|
+
None, for the same reason: a caller that does not know it is creating
|
|
296
|
+
a child is not one, and should not be able to say so by accident in a
|
|
297
|
+
positional slot.
|
|
298
|
+
"""
|
|
299
|
+
with Session(engine) as session:
|
|
300
|
+
thread = ConversationThread(
|
|
301
|
+
kind=kind, parent_thread_id=parent_thread_id,
|
|
302
|
+
parent_call_id=parent_call_id, agent_name=agent_name)
|
|
303
|
+
session.add(thread)
|
|
304
|
+
session.commit()
|
|
305
|
+
session.refresh(thread)
|
|
306
|
+
return thread.id
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
def get_thread(thread_id: UUID) -> Optional[dict]:
|
|
310
|
+
"""The thread row as a plain column dict, or None if unknown (#31).
|
|
311
|
+
|
|
312
|
+
Used to confirm a thread_id is real before resuming it, and by
|
|
313
|
+
ConversationMemory to read extra_data and kind on resume. A PLAIN
|
|
314
|
+
DICT, like every other public read here: returning a detached ORM
|
|
315
|
+
instance worked only because nothing had expired it -- what a
|
|
316
|
+
detached instance hands back depends on what happened to be loaded,
|
|
317
|
+
which is exactly the question _ordered_rows' docstring says copying
|
|
318
|
+
columns removes.
|
|
319
|
+
|
|
320
|
+
THE `getattr` FALLBACKS COVER A NULL, NOT A MISSING COLUMN, and this
|
|
321
|
+
paragraph said the second thing until batch 75. It read: "a row read
|
|
322
|
+
from a database whose ALTER has not run yet has no attribute at all".
|
|
323
|
+
It cannot -- `session.get` SELECTs every mapped column, so a database
|
|
324
|
+
missing one raises OperationalError before any attribute is read, and
|
|
325
|
+
the class always has the attribute anyway. What these actually absorb
|
|
326
|
+
is a NULL on a row written before the column existed (and a hand-built
|
|
327
|
+
stand-in in a test), which is why each pairs the lookup with a default
|
|
328
|
+
the caller can use. The migration case belongs to
|
|
329
|
+
`database.ensure_columns()`, which runs at startup.
|
|
330
|
+
|
|
331
|
+
`kind` IS THE EXCEPTION, and the sentence above was wrong about it
|
|
332
|
+
until #257. ensure_columns has added `kind` with DEFAULT 'chat' since
|
|
333
|
+
§27 shipped it, and SQLite reads every row older than that ALTER as
|
|
334
|
+
the default -- so a migrated row arrives as 'chat' before any reader
|
|
335
|
+
runs (the lineage columns and `last_activity_at` have no SQL default
|
|
336
|
+
and really are NULL there). A NULL `kind` takes SQL written by hand,
|
|
337
|
+
on a database first migrated between §27 and #26 (2026-08-06 to
|
|
338
|
+
08-22), where the column was added nullable. That is all the
|
|
339
|
+
`or THREAD_KIND_CHAT` here, in child_threads and in list_threads
|
|
340
|
+
absorbs; tests/test_storage_e2e.py builds that database and pins both
|
|
341
|
+
the default and the fallback. list_threads filters in SQL, so its
|
|
342
|
+
default picker never meets such a row at all.
|
|
343
|
+
"""
|
|
344
|
+
with Session(engine) as session:
|
|
345
|
+
thread = session.get(ConversationThread, thread_id)
|
|
346
|
+
if thread is None:
|
|
347
|
+
return None
|
|
348
|
+
return {
|
|
349
|
+
"id": thread.id,
|
|
350
|
+
"created_at": thread.created_at,
|
|
351
|
+
"extra_data": dict(thread.extra_data or {}),
|
|
352
|
+
"kind": getattr(thread, "kind", None) or THREAD_KIND_CHAT,
|
|
353
|
+
"last_activity_at": getattr(thread, "last_activity_at", None),
|
|
354
|
+
# getattr for the kind fallback's reason, one column over --
|
|
355
|
+
# see the docstring for what that reason actually is. NULL on
|
|
356
|
+
# a row written before §47 means "no known parent", and a
|
|
357
|
+
# viewer showing that is better than one that raises.
|
|
358
|
+
"parent_thread_id": getattr(thread, "parent_thread_id", None),
|
|
359
|
+
"parent_call_id": getattr(thread, "parent_call_id", None),
|
|
360
|
+
"agent_name": getattr(thread, "agent_name", None),
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
|
|
364
|
+
def child_threads(parent_thread_id: UUID) -> list[dict]:
|
|
365
|
+
"""The threads spawned by this one, oldest first (§47).
|
|
366
|
+
|
|
367
|
+
Each entry: ``{"id", "created_at", "kind", "parent_call_id",
|
|
368
|
+
"agent_name"}``. Empty for a thread that spawned nothing, which is
|
|
369
|
+
most of them.
|
|
370
|
+
|
|
371
|
+
ONE QUERY, not one per spawn line. A resumed conversation resolves
|
|
372
|
+
every `▸ spawn_subagent` line it draws, and asking per line would be
|
|
373
|
+
a query per row of a screen.
|
|
374
|
+
|
|
375
|
+
Oldest first because that is the order the calls happened in, so a
|
|
376
|
+
reader comparing this against the transcript walks both the same
|
|
377
|
+
way. `parent_call_id` is what pairs an entry to a specific line;
|
|
378
|
+
ordering is for the human, not for the lookup.
|
|
379
|
+
|
|
380
|
+
ORDERED BY `(created_at, id)`, which is `_ordered_rows`' rule and was
|
|
381
|
+
missing here (batch 75). That rule reads: timestamps have microsecond
|
|
382
|
+
resolution so ties are unlikely, but "unlikely" is not an ordering
|
|
383
|
+
guarantee. This is the query where it stopped being merely unlikely --
|
|
384
|
+
§47's NA9 makes three `create_thread` calls land from three threads at
|
|
385
|
+
once, which is the one place in this file that can actually tie.
|
|
386
|
+
|
|
387
|
+
NO `kind` FILTER, deliberately. Every child of a chat thread is
|
|
388
|
+
machinery of some sort, and which kinds exist is §27's question
|
|
389
|
+
rather than this one's -- a filter here would have to be widened
|
|
390
|
+
again the moment a sixth source appears, which is how §27's own
|
|
391
|
+
picker missed the compactor.
|
|
392
|
+
"""
|
|
393
|
+
with Session(engine) as session:
|
|
394
|
+
statement = (
|
|
395
|
+
select(ConversationThread)
|
|
396
|
+
.where(ConversationThread.parent_thread_id == parent_thread_id)
|
|
397
|
+
.order_by(ConversationThread.created_at.asc(),
|
|
398
|
+
ConversationThread.id.asc())
|
|
399
|
+
)
|
|
400
|
+
return [
|
|
401
|
+
{
|
|
402
|
+
"id": t.id,
|
|
403
|
+
"created_at": t.created_at,
|
|
404
|
+
# A hand-written NULL only (#257); see get_thread.
|
|
405
|
+
"kind": getattr(t, "kind", None) or THREAD_KIND_CHAT,
|
|
406
|
+
"parent_call_id": getattr(t, "parent_call_id", None),
|
|
407
|
+
"agent_name": getattr(t, "agent_name", None),
|
|
408
|
+
}
|
|
409
|
+
for t in session.exec(statement).all()
|
|
410
|
+
]
|
|
411
|
+
|
|
412
|
+
|
|
413
|
+
def get_thread_extra(thread_id: UUID) -> dict[str, Any]:
|
|
414
|
+
"""Copy of the thread's extra_data JSON column (goal mode §18, todo
|
|
415
|
+
list §23). Raises ValueError for an unknown thread, same contract as
|
|
416
|
+
the other accessors."""
|
|
417
|
+
with Session(engine) as session:
|
|
418
|
+
thread = session.get(ConversationThread, thread_id)
|
|
419
|
+
if thread is None:
|
|
420
|
+
raise ValueError(f"No conversation thread found with id {thread_id}")
|
|
421
|
+
return dict(thread.extra_data or {})
|
|
422
|
+
|
|
423
|
+
|
|
424
|
+
def update_thread_extra(thread_id: UUID, key: str, value: Any) -> None:
|
|
425
|
+
"""Set one key of the thread's extra_data; value=None deletes it.
|
|
426
|
+
Reassigns a fresh dict rather than mutating in place so SQLModel sees
|
|
427
|
+
the JSON column change."""
|
|
428
|
+
with Session(engine) as session:
|
|
429
|
+
thread = session.get(ConversationThread, thread_id)
|
|
430
|
+
if thread is None:
|
|
431
|
+
raise ValueError(f"No conversation thread found with id {thread_id}")
|
|
432
|
+
extra = dict(thread.extra_data or {})
|
|
433
|
+
if value is None:
|
|
434
|
+
extra.pop(key, None)
|
|
435
|
+
else:
|
|
436
|
+
extra[key] = value
|
|
437
|
+
thread.extra_data = extra
|
|
438
|
+
session.add(thread)
|
|
439
|
+
session.commit()
|
|
440
|
+
|
|
441
|
+
|
|
442
|
+
def save_message(
|
|
443
|
+
thread_id: UUID,
|
|
444
|
+
role: str,
|
|
445
|
+
content: Any,
|
|
446
|
+
name: Optional[str] = None,
|
|
447
|
+
tool_call_id: Optional[str] = None,
|
|
448
|
+
thinking: Optional[dict] = None,
|
|
449
|
+
harness: Optional[dict] = None,
|
|
450
|
+
images: Optional[list] = None,
|
|
451
|
+
) -> None:
|
|
452
|
+
"""Append one archived row and stamp the thread's activity (#32).
|
|
453
|
+
|
|
454
|
+
THE single production writer of MessageLog rows, which is what makes
|
|
455
|
+
it the right place to maintain `last_activity_at`: the stamp rides in
|
|
456
|
+
the same session and commit as the insert, so a message can never be
|
|
457
|
+
persisted without its thread moving to the top of the picker. Any
|
|
458
|
+
role counts (Q8b) -- a thread whose last event was tool output is
|
|
459
|
+
still the one you were just in. An unknown thread_id writes the
|
|
460
|
+
orphan message exactly as before; there is simply no row to stamp.
|
|
461
|
+
"""
|
|
462
|
+
with Session(engine) as session:
|
|
463
|
+
new_message = MessageLog(
|
|
464
|
+
thread_id=thread_id,
|
|
465
|
+
role=role,
|
|
466
|
+
content=json.dumps(content),
|
|
467
|
+
name=name,
|
|
468
|
+
tool_call_id=tool_call_id,
|
|
469
|
+
# Encoded separately from `content` and stored as NULL when
|
|
470
|
+
# absent, so "this turn did no reasoning" costs no bytes and
|
|
471
|
+
# reads back as None rather than as an empty structure.
|
|
472
|
+
thinking=json.dumps(thinking) if thinking else None,
|
|
473
|
+
# ROADMAP_v3 §49 (SS5), on `thinking`'s terms: NULL for every
|
|
474
|
+
# row a person or the model wrote, which is nearly all of them.
|
|
475
|
+
harness=json.dumps(harness) if harness else None,
|
|
476
|
+
# ROADMAP_v3 §81 (IM2), on `thinking`'s terms again: NULL for
|
|
477
|
+
# every row that attached no image.
|
|
478
|
+
images=json.dumps(images) if images else None,
|
|
479
|
+
)
|
|
480
|
+
session.add(new_message)
|
|
481
|
+
thread = session.get(ConversationThread, thread_id)
|
|
482
|
+
if thread is not None:
|
|
483
|
+
thread.last_activity_at = datetime.now(UTC)
|
|
484
|
+
session.add(thread)
|
|
485
|
+
session.commit()
|
|
486
|
+
|
|
487
|
+
|
|
488
|
+
def list_threads(
|
|
489
|
+
kind: Optional[str] = THREAD_KIND_CHAT, limit: Optional[int] = None,
|
|
490
|
+
) -> list[dict]:
|
|
491
|
+
"""Conversation threads, most recently ACTIVE first (#32).
|
|
492
|
+
|
|
493
|
+
Each entry: ``{"id": UUID, "created_at": datetime, "kind": str,
|
|
494
|
+
"preview": str}``. Used by the CLI / TUI layer for thread browsing —
|
|
495
|
+
core/memory.py does NOT call this.
|
|
496
|
+
|
|
497
|
+
`limit` caps the result AFTER ordering (#30/#32 together). The default
|
|
498
|
+
of None is uncached and changes nothing for existing callers -- a
|
|
499
|
+
caller that wants everything still gets everything. The TUI picker
|
|
500
|
+
passes a cap because a modal listing thousands of conversations is its
|
|
501
|
+
own usability defect; with activity ordering the cap costs the least,
|
|
502
|
+
since the threads most likely to be wanted are exactly the ones that
|
|
503
|
+
survive it. Anything older stays reachable by id (/resume, --thread).
|
|
504
|
+
|
|
505
|
+
The order key is `last_activity_at`, falling back to `created_at`
|
|
506
|
+
when it is NULL -- never-received-a-message threads, and every row
|
|
507
|
+
that predates the column (the additive migration never backfills).
|
|
508
|
+
§27 made the picker correct by filtering and identifiable by preview;
|
|
509
|
+
ordering is the third axis: resume yesterday's long thread, work in
|
|
510
|
+
it an hour, and it stays pinned at the top rather than sinking below
|
|
511
|
+
three threads opened and abandoned this morning.
|
|
512
|
+
|
|
513
|
+
ROADMAP_v2 §27 AC2: CONVERSATIONS ONLY by default. One research run
|
|
514
|
+
creates ~10 pass threads plus up to 4 retry-round threads plus the
|
|
515
|
+
reviewer's, and every automatic compaction creates one more — so the
|
|
516
|
+
unfiltered list was mostly machinery, which is what made the picker
|
|
517
|
+
unusable. `kind=None` returns everything, for a caller that genuinely
|
|
518
|
+
wants the machinery (see §27's note on reaching a run's pass threads).
|
|
519
|
+
|
|
520
|
+
`preview` is the thread's first user message, truncated (§27's picker
|
|
521
|
+
decision). A row of bare uuid + timestamp is filterable but still not
|
|
522
|
+
recognisable, and identifying a conversation was the actual complaint.
|
|
523
|
+
It costs ONE extra query for the whole list rather than one per row --
|
|
524
|
+
and ONE ROW per thread, not one row per user message (#30).
|
|
525
|
+
"""
|
|
526
|
+
with Session(engine) as session:
|
|
527
|
+
statement = select(ConversationThread)
|
|
528
|
+
if kind is not None:
|
|
529
|
+
statement = statement.where(ConversationThread.kind == kind)
|
|
530
|
+
statement = statement.order_by(
|
|
531
|
+
sa_func.coalesce(
|
|
532
|
+
ConversationThread.last_activity_at,
|
|
533
|
+
ConversationThread.created_at,
|
|
534
|
+
).desc(),
|
|
535
|
+
ConversationThread.created_at.desc(),
|
|
536
|
+
)
|
|
537
|
+
if limit is not None:
|
|
538
|
+
statement = statement.limit(limit)
|
|
539
|
+
threads = session.exec(statement).all()
|
|
540
|
+
previews = _first_user_messages(session, [t.id for t in threads])
|
|
541
|
+
return [
|
|
542
|
+
{
|
|
543
|
+
"id": t.id,
|
|
544
|
+
"created_at": t.created_at,
|
|
545
|
+
# The fallback absorbs a hand-written NULL only: the
|
|
546
|
+
# migration fills a row older than the column with 'chat'
|
|
547
|
+
# (#257). get_thread's docstring has the whole of it. NOT a
|
|
548
|
+
# database whose ALTER has not run: the mapped class always
|
|
549
|
+
# has the attribute, and a database missing the column
|
|
550
|
+
# raises OperationalError at the SELECT above, before any
|
|
551
|
+
# attribute is read.
|
|
552
|
+
"kind": getattr(t, "kind", None) or THREAD_KIND_CHAT,
|
|
553
|
+
"preview": previews.get(t.id, ""),
|
|
554
|
+
}
|
|
555
|
+
for t in threads
|
|
556
|
+
]
|
|
557
|
+
|
|
558
|
+
|
|
559
|
+
#: How much of a thread's first user message list_threads() carries. Long
|
|
560
|
+
#: enough to tell two conversations apart, short enough for one row.
|
|
561
|
+
_PREVIEW_CHARS = 70
|
|
562
|
+
|
|
563
|
+
|
|
564
|
+
def thread_preview(thread_id: UUID) -> str:
|
|
565
|
+
"""The picker's preview for ONE thread (#171).
|
|
566
|
+
|
|
567
|
+
The same first-user-message text `list_threads()` computes -- read
|
|
568
|
+
from the ARCHIVE, so a compacted thread is labelled by what the user
|
|
569
|
+
actually said, never by §21a's synthesized summary (the reason T3
|
|
570
|
+
makes replay do the same) -- through the same one-row window query,
|
|
571
|
+
so a caller with a single id pays one row too.
|
|
572
|
+
|
|
573
|
+
A PUBLIC wrapper rather than a list_threads() scan because the CLI's
|
|
574
|
+
--ref takes ANY thread id (#171): the picker list is chat-kind-
|
|
575
|
+
filtered and capped at 200, and a pass thread or an older
|
|
576
|
+
conversation is exactly the thing worth referencing by name. Unknown
|
|
577
|
+
id -> "", which the reader renders as "(no preview)".
|
|
578
|
+
"""
|
|
579
|
+
with Session(engine) as session:
|
|
580
|
+
return _first_user_messages(session, [thread_id]).get(thread_id, "")
|
|
581
|
+
|
|
582
|
+
|
|
583
|
+
def _first_user_messages(session, thread_ids: list[UUID]) -> dict[UUID, str]:
|
|
584
|
+
"""thread id -> truncated first user message, for the given threads.
|
|
585
|
+
|
|
586
|
+
ONE query for the whole list, walked oldest-first so the first row seen
|
|
587
|
+
per thread is the earliest one. Reads the ARCHIVE (every MessageLog row
|
|
588
|
+
of the thread) rather than the derived view, for the same reason §27's
|
|
589
|
+
T3 makes replay do it: on a compacted thread the view begins with the
|
|
590
|
+
synthesized summary, and previewing THAT would label harness-generated
|
|
591
|
+
text as something the user said.
|
|
592
|
+
|
|
593
|
+
The "one query" claim is now true of ROWS too (#30): the first-user
|
|
594
|
+
message is picked by `ROW_NUMBER() OVER (PARTITION BY thread_id ...)`,
|
|
595
|
+
so SQLite materializes one user message per thread instead of loading
|
|
596
|
+
every user message of every thread into ORM objects and discarding all
|
|
597
|
+
but one in Python -- measured at 40x over-read on a realistic database
|
|
598
|
+
(~98% of the picker's cost). A plain LIMIT-per-thread loop would need
|
|
599
|
+
one query per thread; the window function keeps the count at one.
|
|
600
|
+
"""
|
|
601
|
+
if not thread_ids:
|
|
602
|
+
return {}
|
|
603
|
+
ranked = (
|
|
604
|
+
select(
|
|
605
|
+
MessageLog.thread_id.label("thread_id"),
|
|
606
|
+
MessageLog.content.label("content"),
|
|
607
|
+
sa_func.row_number().over(
|
|
608
|
+
partition_by=MessageLog.thread_id,
|
|
609
|
+
order_by=(MessageLog.created_at.asc(), MessageLog.id.asc()),
|
|
610
|
+
).label("rn"),
|
|
611
|
+
)
|
|
612
|
+
.where(MessageLog.role == "user")
|
|
613
|
+
.where(MessageLog.thread_id.in_(thread_ids))
|
|
614
|
+
.cte("first_user_messages_ranked")
|
|
615
|
+
)
|
|
616
|
+
rows = session.exec(
|
|
617
|
+
select(ranked.c.thread_id, ranked.c.content)
|
|
618
|
+
.where(ranked.c.rn == 1)
|
|
619
|
+
).all()
|
|
620
|
+
out: dict[UUID, str] = {}
|
|
621
|
+
for thread_id, raw in rows:
|
|
622
|
+
try:
|
|
623
|
+
text = json.loads(raw)
|
|
624
|
+
except (TypeError, ValueError):
|
|
625
|
+
text = raw
|
|
626
|
+
text = " ".join(str(text).split())
|
|
627
|
+
out[thread_id] = (
|
|
628
|
+
text[:_PREVIEW_CHARS] + "…" if len(text) > _PREVIEW_CHARS else text
|
|
629
|
+
)
|
|
630
|
+
return out
|
|
631
|
+
|
|
632
|
+
|
|
633
|
+
def _decode_thinking(raw):
|
|
634
|
+
"""A stored thinking record, or None for anything unusable.
|
|
635
|
+
|
|
636
|
+
Degrades rather than raising, like every other read on this path: the
|
|
637
|
+
column is a convenience for replay and for echoing blocks back to the
|
|
638
|
+
model that wrote them, and a row whose JSON cannot be parsed must cost
|
|
639
|
+
that convenience rather than the ability to resume the thread at all.
|
|
640
|
+
"""
|
|
641
|
+
if not raw:
|
|
642
|
+
return None
|
|
643
|
+
try:
|
|
644
|
+
value = json.loads(raw)
|
|
645
|
+
except (TypeError, ValueError):
|
|
646
|
+
return None
|
|
647
|
+
return value if isinstance(value, dict) and value.get("blocks") else None
|
|
648
|
+
|
|
649
|
+
|
|
650
|
+
def _decode_harness(raw):
|
|
651
|
+
"""A stored harness record (ROADMAP_v3 §49, SS5), or None for anything
|
|
652
|
+
unusable -- `_decode_thinking`'s rule for `_decode_thinking`'s reason: a
|
|
653
|
+
mark that cannot be read costs the mark, never the thread."""
|
|
654
|
+
if not raw:
|
|
655
|
+
return None
|
|
656
|
+
try:
|
|
657
|
+
value = json.loads(raw)
|
|
658
|
+
except (TypeError, ValueError):
|
|
659
|
+
return None
|
|
660
|
+
return value if isinstance(value, dict) and value.get("kind") else None
|
|
661
|
+
|
|
662
|
+
|
|
663
|
+
def _decode_images(raw):
|
|
664
|
+
"""A tool row's stored images (ROADMAP_v3 §81, IM2), or None.
|
|
665
|
+
|
|
666
|
+
`_decode_thinking`'s rule: an entry that cannot be read costs that
|
|
667
|
+
entry, never the thread. An entry is usable when it names a media type
|
|
668
|
+
and carries data, both as non-empty strings -- what translation needs to
|
|
669
|
+
build a block, and nothing it would have to guess at.
|
|
670
|
+
"""
|
|
671
|
+
if not raw:
|
|
672
|
+
return None
|
|
673
|
+
try:
|
|
674
|
+
value = json.loads(raw)
|
|
675
|
+
except (TypeError, ValueError):
|
|
676
|
+
return None
|
|
677
|
+
if not isinstance(value, list):
|
|
678
|
+
return None
|
|
679
|
+
usable = [entry for entry in value
|
|
680
|
+
if isinstance(entry, dict)
|
|
681
|
+
and isinstance(entry.get("media_type"), str)
|
|
682
|
+
and entry["media_type"]
|
|
683
|
+
and isinstance(entry.get("data"), str)
|
|
684
|
+
and entry["data"]]
|
|
685
|
+
return usable or None
|
|
686
|
+
|
|
687
|
+
|
|
688
|
+
def _to_neutral(msg: dict) -> dict:
|
|
689
|
+
"""Reconstructs the exact neutral shape core/memory.py's add_* methods
|
|
690
|
+
write, per msg.role -- this is the counterpart to memory.py only ever
|
|
691
|
+
persisting the role-specific payload (see that file's docstring for
|
|
692
|
+
why there's no separate normalization step on the read side either).
|
|
693
|
+
|
|
694
|
+
ONE reconstruction point. Every read below is a filter over the same
|
|
695
|
+
rows fed through this function, so a change to the neutral shape
|
|
696
|
+
cannot reach one read and miss another -- which is the same reason
|
|
697
|
+
memory.py fixes shapes on the write side rather than per-role at each
|
|
698
|
+
reader. tests/conftest.py's FakeStorage mirrors this function; if it
|
|
699
|
+
changes, that changes identically.
|
|
700
|
+
"""
|
|
701
|
+
decoded = json.loads(msg["content"])
|
|
702
|
+
role = msg["role"]
|
|
703
|
+
|
|
704
|
+
if role == "assistant":
|
|
705
|
+
# decoded is {"text": ..., "tool_calls": [...]} -- exactly
|
|
706
|
+
# what add_assistant_message persisted, nothing nested.
|
|
707
|
+
payload = {
|
|
708
|
+
"role": "assistant",
|
|
709
|
+
"text": decoded.get("text", ""),
|
|
710
|
+
"tool_calls": decoded.get("tool_calls", []),
|
|
711
|
+
}
|
|
712
|
+
# §44, from its own column rather than from `content`. Present
|
|
713
|
+
# ONLY when the row carried reasoning, so every message this
|
|
714
|
+
# project has ever written reconstructs to exactly the shape it
|
|
715
|
+
# did before -- which is what keeps _messages_for_provider,
|
|
716
|
+
# compactable_span and the fake's mirror unchanged for them.
|
|
717
|
+
thinking = _decode_thinking(msg.get("thinking"))
|
|
718
|
+
if thinking is not None:
|
|
719
|
+
payload["thinking"] = thinking
|
|
720
|
+
elif role == "tool":
|
|
721
|
+
# decoded is the plain result string add_tool_result persisted.
|
|
722
|
+
payload = {"role": "tool", "tool_call_id": msg["tool_call_id"], "content": decoded}
|
|
723
|
+
# ROADMAP_v3 §81 (IM2), on §44's terms for `thinking`: present ONLY
|
|
724
|
+
# when the row attached an image, so every tool row written before
|
|
725
|
+
# reconstructs to exactly the shape it did. Read by translation and
|
|
726
|
+
# the compactor's size proxy; never by anything that renders text.
|
|
727
|
+
images = _decode_images(msg.get("images"))
|
|
728
|
+
if images is not None:
|
|
729
|
+
payload["images"] = images
|
|
730
|
+
else:
|
|
731
|
+
# user (and any future plain-content role): decoded is
|
|
732
|
+
# already the right value for "content".
|
|
733
|
+
payload = {"role": role, "content": decoded}
|
|
734
|
+
if msg["tool_call_id"]:
|
|
735
|
+
payload["tool_call_id"] = msg["tool_call_id"]
|
|
736
|
+
# Batch 91, on §44's terms for `thinking`: present ONLY when the row
|
|
737
|
+
# records a failed run, so every message written before reconstructs
|
|
738
|
+
# to exactly the shape it did. Nothing that builds a request reads
|
|
739
|
+
# it -- _messages_for_provider takes a user row's `content` alone,
|
|
740
|
+
# and so does the compactor's _as_text.
|
|
741
|
+
if msg.get("error"):
|
|
742
|
+
payload["error"] = msg["error"]
|
|
743
|
+
# ROADMAP_v3 §49 (SS5), on the same terms: present ONLY when the HARNESS
|
|
744
|
+
# wrote the row, so every other message reconstructs exactly as it did.
|
|
745
|
+
# Read by replay and the compactor's labelling; never by translation.
|
|
746
|
+
harness = _decode_harness(msg.get("harness"))
|
|
747
|
+
if harness is not None:
|
|
748
|
+
payload["harness"] = harness
|
|
749
|
+
|
|
750
|
+
if msg["name"]:
|
|
751
|
+
payload["name"] = msg["name"]
|
|
752
|
+
|
|
753
|
+
return payload
|
|
754
|
+
|
|
755
|
+
|
|
756
|
+
def _ordered_rows(thread_id: UUID) -> list[dict]:
|
|
757
|
+
"""Every row of a thread as plain column values, oldest first.
|
|
758
|
+
|
|
759
|
+
Ordered by (created_at, id) rather than created_at alone. Timestamps
|
|
760
|
+
have microsecond resolution so ties are unlikely, but "unlikely" is
|
|
761
|
+
not an ordering guarantee, and §21's watermark arithmetic slices this
|
|
762
|
+
list by position -- an unstable sort there would move the fold
|
|
763
|
+
boundary between two reads of the same thread.
|
|
764
|
+
|
|
765
|
+
Plain dicts rather than ORM instances because every caller below
|
|
766
|
+
reads these after the Session has closed, and what a detached
|
|
767
|
+
instance will still hand back depends on what happened to be loaded.
|
|
768
|
+
Copying six columns removes the question.
|
|
769
|
+
|
|
770
|
+
`pinned` is coerced to bool: a database migrated by
|
|
771
|
+
database.ensure_columns() gets the column with a DEFAULT, but a row
|
|
772
|
+
written by an older build through some other path could still read
|
|
773
|
+
back NULL, and None is not the same thing as "not pinned" to a
|
|
774
|
+
caller doing `if row["pinned"]`.
|
|
775
|
+
"""
|
|
776
|
+
with Session(engine) as session:
|
|
777
|
+
statement = (
|
|
778
|
+
select(MessageLog)
|
|
779
|
+
.where(MessageLog.thread_id == thread_id)
|
|
780
|
+
.order_by(MessageLog.created_at.asc(), MessageLog.id.asc())
|
|
781
|
+
)
|
|
782
|
+
return [
|
|
783
|
+
{
|
|
784
|
+
"id": m.id, "role": m.role, "content": m.content,
|
|
785
|
+
"name": m.name, "tool_call_id": m.tool_call_id,
|
|
786
|
+
"pinned": bool(m.pinned),
|
|
787
|
+
# Not coerced, unlike `pinned`: NULL is the ordinary state
|
|
788
|
+
# of this column and _decode_thinking answers None for it.
|
|
789
|
+
"thinking": m.thinking,
|
|
790
|
+
# Batch 91, and NULL is ordinary here too: a run failed.
|
|
791
|
+
"error": m.error,
|
|
792
|
+
# ROADMAP_v3 §49 (SS5). NULL for every row a person or the
|
|
793
|
+
# model wrote.
|
|
794
|
+
"harness": m.harness,
|
|
795
|
+
# ROADMAP_v3 §81 (IM2). NULL for every row that attached no
|
|
796
|
+
# image; the compactor's size proxy reads it (IM8).
|
|
797
|
+
"images": m.images,
|
|
798
|
+
}
|
|
799
|
+
for m in session.exec(statement).all()
|
|
800
|
+
]
|
|
801
|
+
|
|
802
|
+
|
|
803
|
+
def _split_at(rows: list[dict], message_id: Optional[UUID]) -> int:
|
|
804
|
+
"""Index just past `message_id` in `rows`, or 0 when it isn't there.
|
|
805
|
+
|
|
806
|
+
Resolving a watermark by POSITION in the already-loaded, already-ordered
|
|
807
|
+
list rather than by a `(created_at, id) > (t, i)` SQL predicate. The
|
|
808
|
+
comparison a tuple-ordered watermark needs is awkward to express
|
|
809
|
+
portably over a UUID column, and a thread is at most a few thousand
|
|
810
|
+
rows that this function's callers are loading anyway. Simplicity wins
|
|
811
|
+
over a query that has to be right about UUID collation.
|
|
812
|
+
|
|
813
|
+
A watermark that names a row this thread does not have returns 0 --
|
|
814
|
+
i.e. nothing is treated as covered. That is the safe direction: the
|
|
815
|
+
caller shows the whole thread rather than silently hiding a prefix
|
|
816
|
+
on the strength of an id it could not find.
|
|
817
|
+
"""
|
|
818
|
+
if message_id is None:
|
|
819
|
+
return 0
|
|
820
|
+
for index, row in enumerate(rows):
|
|
821
|
+
if row["id"] == message_id:
|
|
822
|
+
return index + 1
|
|
823
|
+
return 0
|
|
824
|
+
|
|
825
|
+
|
|
826
|
+
def get_session_history(
|
|
827
|
+
thread_id: UUID, after_message_id: Optional[UUID] = None,
|
|
828
|
+
) -> list[dict]:
|
|
829
|
+
"""The thread's messages in neutral shape, oldest first.
|
|
830
|
+
|
|
831
|
+
`after_message_id` (ROADMAP_v2 §21) loads only the UNCOMPACTED TAIL --
|
|
832
|
+
everything strictly after that row. The rows before it are not gone;
|
|
833
|
+
they are represented by a CompactionCheckpoint summary that
|
|
834
|
+
core/memory.py prepends, and they remain readable in full via
|
|
835
|
+
archive_history(). None (the default) returns the whole thread, which
|
|
836
|
+
is what every pre-§21 caller gets and why their behaviour is
|
|
837
|
+
unchanged.
|
|
838
|
+
"""
|
|
839
|
+
rows = _ordered_rows(thread_id)
|
|
840
|
+
return [_to_neutral(m) for m in rows[_split_at(rows, after_message_id):]]
|
|
841
|
+
|
|
842
|
+
|
|
843
|
+
def archive_history(thread_id: UUID) -> list[dict]:
|
|
844
|
+
"""Every message ever written to this thread, compaction or not.
|
|
845
|
+
|
|
846
|
+
This is ROADMAP_v2 §21 AC1 made greppable: compaction adds
|
|
847
|
+
CompactionCheckpoint rows and never edits or deletes a MessageLog
|
|
848
|
+
row, so this read is complete no matter how many times a thread has
|
|
849
|
+
been compacted. It is a distinct name rather than a comment on
|
|
850
|
+
get_session_history because the guarantee is what callers want to
|
|
851
|
+
find -- and because a future watermark default on that function
|
|
852
|
+
would silently take this one with it.
|
|
853
|
+
"""
|
|
854
|
+
return get_session_history(thread_id)
|
|
855
|
+
|
|
856
|
+
|
|
857
|
+
def _turn_opener_index(rows: list[dict], span: list[dict]) -> list[int]:
|
|
858
|
+
"""`[i]` for the user row that OPENS the turn a MID-TURN watermark cut
|
|
859
|
+
into, or `[]`. Batch 120, register item 40.
|
|
860
|
+
|
|
861
|
+
A sub-turn fold puts the watermark inside a turn, so the turn's own
|
|
862
|
+
request falls on the covered side and would be summarized away -- and
|
|
863
|
+
the model would then read a paraphrase of the question it is answering
|
|
864
|
+
rather than the question. That is M5's objection to folding the current
|
|
865
|
+
turn at all, and re-adding the request verbatim is what answers it.
|
|
866
|
+
|
|
867
|
+
The TEST IS THE FIRST KEPT ROW, not a flag on the checkpoint. When the
|
|
868
|
+
watermark is turn-aligned the tail opens with a `user` row and nothing
|
|
869
|
+
needs re-adding; when it is mid-turn the tail opens with an `assistant`
|
|
870
|
+
row, because that is the only boundary _sub_turn_span will place. So
|
|
871
|
+
the state is derivable from the rows, which means no column, no
|
|
872
|
+
migration, and no checkpoint written before this existed that lies
|
|
873
|
+
about its own shape.
|
|
874
|
+
"""
|
|
875
|
+
if not span or len(span) >= len(rows):
|
|
876
|
+
return []
|
|
877
|
+
if rows[len(span)]["role"] == "user":
|
|
878
|
+
return []
|
|
879
|
+
for index in range(len(span) - 1, -1, -1):
|
|
880
|
+
if span[index]["role"] == "user":
|
|
881
|
+
return [index]
|
|
882
|
+
return []
|
|
883
|
+
|
|
884
|
+
|
|
885
|
+
def pinned_through(thread_id: UUID, message_id: Optional[UUID]) -> list[dict]:
|
|
886
|
+
"""Rows at or before the watermark the derived view must carry
|
|
887
|
+
verbatim: the pinned set, made whole (§21 M9), plus the opening request
|
|
888
|
+
of a turn a mid-turn watermark cut into (batch 120).
|
|
889
|
+
|
|
890
|
+
A summary covers a contiguous span, but §21 AC2 says a pinned message is
|
|
891
|
+
never summarized -- so pinned rows INSIDE the covered span have to
|
|
892
|
+
come back verbatim, and a single `covers_up_to_message_id` cannot say
|
|
893
|
+
"everything through K except these". The derived view is therefore
|
|
894
|
+
summary + these rows + tail.
|
|
895
|
+
|
|
896
|
+
Empty when nothing in the span is pinned, which is every thread until
|
|
897
|
+
someone calls pin().
|
|
898
|
+
|
|
899
|
+
THE PINNED SPAN IS MADE WHOLE HERE, which is M4's rule applied to the
|
|
900
|
+
second boundary. The derived view has two of them and only the
|
|
901
|
+
checkpoint watermark is turn-aligned: `pin` is dispatched from inside a
|
|
902
|
+
turn, so the set it writes always ends mid-turn. D20 has already
|
|
903
|
+
persisted the assistant message carrying the `tool_use` for `pin`
|
|
904
|
+
itself, and the answering `tool` row is written after dispatch returns
|
|
905
|
+
-- so it is never pinned. While the pin is still in the uncompacted
|
|
906
|
+
tail this is invisible, because the tail carries both rows. The first
|
|
907
|
+
compaction whose watermark passes the pinned region activates it: the
|
|
908
|
+
assistant row comes back with an unanswered `tool_use`, its result
|
|
909
|
+
having gone into the summary text instead, and a `tool_result` with no
|
|
910
|
+
preceding `tool_use` is an HTTP 400 on Anthropic -- on a thread that
|
|
911
|
+
was working a moment ago, with no visible connection to a `pin` several
|
|
912
|
+
turns earlier. Durable, too: the checkpoint is a persisted row, so
|
|
913
|
+
every later call and every resume rebuilds the same unsendable view.
|
|
914
|
+
|
|
915
|
+
Fixed at the PRODUCER, where a pinned span becomes view content.
|
|
916
|
+
Stopping pin_last at the last complete turn would change what "pin the
|
|
917
|
+
last N turns" means and leave any other non-turn-aligned pinned set
|
|
918
|
+
open; dropping unanswered tool_calls in _derived_view is consumer-side
|
|
919
|
+
-- this project's canonical bug shape -- and would hide from the model
|
|
920
|
+
a call it actually made.
|
|
921
|
+
"""
|
|
922
|
+
rows = _ordered_rows(thread_id)
|
|
923
|
+
span = rows[:_split_at(rows, message_id)]
|
|
924
|
+
neutral = [_to_neutral(m) for m in span]
|
|
925
|
+
return [neutral[i] for i in _pinned_indices(rows, span, neutral)]
|
|
926
|
+
|
|
927
|
+
|
|
928
|
+
def _pinned_indices(rows: list, span: list, neutral: list) -> list[int]:
|
|
929
|
+
"""The positions in `span` that `pinned_through` returns, in archive
|
|
930
|
+
order: the pinned rows, the tool results that make them whole, and the
|
|
931
|
+
opener of a turn a mid-turn watermark cut into.
|
|
932
|
+
|
|
933
|
+
A PURE FUNCTION SINCE BATCH 216 (#231), split out of `pinned_through`
|
|
934
|
+
so `tests/test_fake_storage_mirror.py` can compare it against
|
|
935
|
+
`FakeStorage`'s copy the way it compares `_to_neutral` and `_split_at`
|
|
936
|
+
-- the fake transcribes this algorithm, and a transcription nothing
|
|
937
|
+
compares is a convention somebody has to remember. `rows` and `span`
|
|
938
|
+
need only `role` and `pinned`; `neutral` is `span` reconstructed.
|
|
939
|
+
"""
|
|
940
|
+
keep = [i for i, m in enumerate(span) if m["pinned"]]
|
|
941
|
+
# `have`, the set() below and the role check are all BACKSTOPS, and the
|
|
942
|
+
# measurement says so: each was deleted separately against the full
|
|
943
|
+
# suite and each survived. What actually keeps the result duplicate-free
|
|
944
|
+
# is the `not m["pinned"]` filter -- a pinned tool row is already in
|
|
945
|
+
# `keep` and can never be re-added -- and reaching any of the three
|
|
946
|
+
# needs two rows sharing one tool_call_id, or a non-tool row carrying
|
|
947
|
+
# one, neither of which has a producer: add_tool_result writes exactly
|
|
948
|
+
# one row per call, and add_user_message passes no tool_call_id.
|
|
949
|
+
#
|
|
950
|
+
# Kept rather than trimmed, and the reasoning recorded rather than
|
|
951
|
+
# asserted: the failure they guard against is a duplicate tool_result,
|
|
952
|
+
# which is the same wire error from the other direction as the one this
|
|
953
|
+
# function exists to prevent. That is the wrong place to trade a free
|
|
954
|
+
# backstop for a reachability argument.
|
|
955
|
+
have = {neutral[i].get("tool_call_id") for i in keep}
|
|
956
|
+
need = {call["id"] for i in keep
|
|
957
|
+
for call in (neutral[i].get("tool_calls") or [])
|
|
958
|
+
if call["id"] not in have}
|
|
959
|
+
keep += [i for i, m in enumerate(span)
|
|
960
|
+
if not m["pinned"] and m["role"] == "tool"
|
|
961
|
+
and neutral[i].get("tool_call_id") in need]
|
|
962
|
+
|
|
963
|
+
# Batch 120. ONE INDEX, AND THE SORT BELOW PLACES IT -- which is the
|
|
964
|
+
# whole reason the turn opener lives here rather than in a function of
|
|
965
|
+
# its own beside this one. The view needs these rows in ARCHIVE ORDER,
|
|
966
|
+
# and `_derived_view` has no positions to merge two lists by: a pinned
|
|
967
|
+
# row from an older turn belongs before the request and a pinned row
|
|
968
|
+
# from inside this turn belongs after it, and either sibling-function
|
|
969
|
+
# arrangement gets one of those wrong. `set()` covers the opener
|
|
970
|
+
# already being pinned.
|
|
971
|
+
keep += _turn_opener_index(rows, span)
|
|
972
|
+
|
|
973
|
+
return sorted(set(keep))
|
|
974
|
+
|
|
975
|
+
|
|
976
|
+
def history_through(
|
|
977
|
+
thread_id: UUID, message_id: Optional[UUID],
|
|
978
|
+
after_message_id: Optional[UUID] = None,
|
|
979
|
+
) -> list[dict]:
|
|
980
|
+
"""The span a compaction is about to summarize, in neutral shape.
|
|
981
|
+
|
|
982
|
+
Pinned rows are EXCLUDED (§21 AC2: a pinned message is never included in
|
|
983
|
+
what the compactor is asked to summarize). `after_message_id` is what
|
|
984
|
+
makes chaining possible -- M2's chain strategy summarizes only what
|
|
985
|
+
followed the previous checkpoint, while rederive passes None and
|
|
986
|
+
summarizes the whole span from the top.
|
|
987
|
+
"""
|
|
988
|
+
rows = _ordered_rows(thread_id)
|
|
989
|
+
start = _split_at(rows, after_message_id)
|
|
990
|
+
end = _split_at(rows, message_id)
|
|
991
|
+
return [_to_neutral(m) for m in rows[start:end] if not m["pinned"]]
|
|
992
|
+
|
|
993
|
+
|
|
994
|
+
def turn_start_ids(thread_id: UUID) -> list[UUID]:
|
|
995
|
+
"""Ids of the rows that BEGIN each turn, oldest first.
|
|
996
|
+
|
|
997
|
+
A turn starts at a user message: the wrapper adds one, then the loop
|
|
998
|
+
appends assistant and tool rows until it stops. §21's M4 needs this
|
|
999
|
+
because a fold boundary that lands anywhere else can separate an
|
|
1000
|
+
assistant message from its tool results, and a tool_result with no
|
|
1001
|
+
preceding tool_use is an HTTP 400 on Anthropic -- a hard wire error,
|
|
1002
|
+
not a degraded answer. M5's "keep the last N turns" needs it too.
|
|
1003
|
+
"""
|
|
1004
|
+
return [m["id"] for m in _ordered_rows(thread_id) if m["role"] == "user"]
|
|
1005
|
+
|
|
1006
|
+
|
|
1007
|
+
def message_ids_from(thread_id: UUID, message_id: UUID) -> list[UUID]:
|
|
1008
|
+
"""Ids of `message_id` and everything after it. Resolves pin()'s
|
|
1009
|
+
ordinal ("the last N turns") into the real row ids the pinned flag is
|
|
1010
|
+
written against -- §21 keeps MessageLog.id an implementation detail
|
|
1011
|
+
the model never sees, so the ordinal-to-id mapping has to happen
|
|
1012
|
+
somewhere below the tool."""
|
|
1013
|
+
rows = _ordered_rows(thread_id)
|
|
1014
|
+
start = _split_at(rows, message_id)
|
|
1015
|
+
if not start:
|
|
1016
|
+
return []
|
|
1017
|
+
return [message_id] + [m["id"] for m in rows[start:]]
|
|
1018
|
+
|
|
1019
|
+
|
|
1020
|
+
def set_pinned(message_ids: list[UUID], pinned: bool = True) -> int:
|
|
1021
|
+
"""Flag rows compaction-exempt. Returns how many rows changed."""
|
|
1022
|
+
if not message_ids:
|
|
1023
|
+
return 0
|
|
1024
|
+
changed = 0
|
|
1025
|
+
with Session(engine) as session:
|
|
1026
|
+
for message_id in message_ids:
|
|
1027
|
+
row = session.get(MessageLog, message_id)
|
|
1028
|
+
if row is None or row.pinned == pinned:
|
|
1029
|
+
continue
|
|
1030
|
+
row.pinned = pinned
|
|
1031
|
+
session.add(row)
|
|
1032
|
+
changed += 1
|
|
1033
|
+
session.commit()
|
|
1034
|
+
return changed
|
|
1035
|
+
|
|
1036
|
+
|
|
1037
|
+
def mark_turn_failed(thread_id: UUID, error: str) -> bool:
|
|
1038
|
+
"""Record on the thread's NEWEST user row why the run it started failed.
|
|
1039
|
+
|
|
1040
|
+
Batch 91; see MessageLog.error for why this is a column. The newest user
|
|
1041
|
+
row is always the right one: every entry point writes the message that
|
|
1042
|
+
starts a run BEFORE the loop begins -- run_agent_conversation,
|
|
1043
|
+
continue_conversation, stream_deep_research_mode and the TUI turn -- so
|
|
1044
|
+
by the time a failure escapes the loop, the row that started it is the
|
|
1045
|
+
last user row there is. Ordered by (created_at, id), `_ordered_rows`'
|
|
1046
|
+
own order, so the two can never disagree about which row is newest.
|
|
1047
|
+
|
|
1048
|
+
Returns False for a thread with no user row. Does not stamp
|
|
1049
|
+
last_activity_at the way save_message does: a failure is not a message.
|
|
1050
|
+
"""
|
|
1051
|
+
with Session(engine) as session:
|
|
1052
|
+
statement = (
|
|
1053
|
+
select(MessageLog)
|
|
1054
|
+
.where(MessageLog.thread_id == thread_id)
|
|
1055
|
+
.where(MessageLog.role == "user")
|
|
1056
|
+
.order_by(MessageLog.created_at.desc(), MessageLog.id.desc())
|
|
1057
|
+
)
|
|
1058
|
+
row = session.exec(statement).first()
|
|
1059
|
+
if row is None:
|
|
1060
|
+
return False
|
|
1061
|
+
row.error = error
|
|
1062
|
+
session.add(row)
|
|
1063
|
+
session.commit()
|
|
1064
|
+
return True
|
|
1065
|
+
|
|
1066
|
+
|
|
1067
|
+
def _current_watermark(thread_id: UUID) -> Optional[UUID]:
|
|
1068
|
+
"""The live checkpoint's watermark, or None if never compacted."""
|
|
1069
|
+
checkpoint = latest_checkpoint(thread_id)
|
|
1070
|
+
return checkpoint["covers_up_to_message_id"] if checkpoint else None
|
|
1071
|
+
|
|
1072
|
+
|
|
1073
|
+
def advances(thread_id: UUID, current: Optional[UUID], proposed: UUID) -> bool:
|
|
1074
|
+
"""Is `proposed` strictly later in this thread than `current`?
|
|
1075
|
+
|
|
1076
|
+
The ordering test behind §21's one real monotonic invariant: a
|
|
1077
|
+
compaction watermark only ever moves FORWARD, and -- #27's half --
|
|
1078
|
+
it must name a row THIS THREAD actually has. `current` of None
|
|
1079
|
+
(no checkpoint yet) skips the ordering comparison but not the
|
|
1080
|
+
containment one: a watermark we cannot place is one we must not
|
|
1081
|
+
write, and "first compaction" is exactly when nothing else would
|
|
1082
|
+
catch a foreign id.
|
|
1083
|
+
|
|
1084
|
+
A `proposed` this thread does not contain returns False -- the safe
|
|
1085
|
+
direction, since a watermark we cannot place is one we must not write.
|
|
1086
|
+
"""
|
|
1087
|
+
rows = _ordered_rows(thread_id)
|
|
1088
|
+
if current is None:
|
|
1089
|
+
return _split_at(rows, proposed) > 0
|
|
1090
|
+
return _split_at(rows, proposed) > _split_at(rows, current) > 0
|
|
1091
|
+
|
|
1092
|
+
|
|
1093
|
+
def latest_checkpoint(thread_id: UUID) -> Optional[dict]:
|
|
1094
|
+
"""The most recent compaction of this thread as a plain column dict,
|
|
1095
|
+
or None if it has never been compacted (which is every thread until
|
|
1096
|
+
the trigger first fires).
|
|
1097
|
+
|
|
1098
|
+
A dict rather than a detached ORM instance (#31): two of the three
|
|
1099
|
+
callers read it after their Session has closed, and one of them is
|
|
1100
|
+
the compaction path -- the one place in the project where a wrong
|
|
1101
|
+
read rewrites a live conversation. Copying the columns removes the
|
|
1102
|
+
question of what a detached instance still hands back.
|
|
1103
|
+
"""
|
|
1104
|
+
with Session(engine) as session:
|
|
1105
|
+
statement = (
|
|
1106
|
+
select(CompactionCheckpoint)
|
|
1107
|
+
.where(CompactionCheckpoint.thread_id == thread_id)
|
|
1108
|
+
.order_by(CompactionCheckpoint.created_at.desc(),
|
|
1109
|
+
CompactionCheckpoint.id.desc())
|
|
1110
|
+
)
|
|
1111
|
+
row = next(iter(session.exec(statement).all()), None)
|
|
1112
|
+
if row is None:
|
|
1113
|
+
return None
|
|
1114
|
+
return {
|
|
1115
|
+
"id": row.id,
|
|
1116
|
+
"thread_id": row.thread_id,
|
|
1117
|
+
"summary_text": row.summary_text,
|
|
1118
|
+
"covers_up_to_message_id": row.covers_up_to_message_id,
|
|
1119
|
+
"strategy": row.strategy,
|
|
1120
|
+
"created_at": row.created_at,
|
|
1121
|
+
}
|
|
1122
|
+
|
|
1123
|
+
|
|
1124
|
+
def save_checkpoint(
|
|
1125
|
+
thread_id: UUID, summary_text: str, covers_up_to_message_id: UUID,
|
|
1126
|
+
strategy: str = "rederive",
|
|
1127
|
+
) -> UUID:
|
|
1128
|
+
"""Record one compaction. Adds a row; touches nothing else (§21 AC1).
|
|
1129
|
+
|
|
1130
|
+
REFUSES A WATERMARK THAT DOES NOT ADVANCE, returning the existing
|
|
1131
|
+
checkpoint's id instead. `latest_checkpoint` resolves ties by
|
|
1132
|
+
timestamp, so a backwards watermark written here does not merely fail
|
|
1133
|
+
to help -- it becomes the live one, and the thread UN-COMPACTS: the
|
|
1134
|
+
derived view jumps back to showing everything after the earlier
|
|
1135
|
+
boundary. That is what §21a shipped, and the caller-side guard in
|
|
1136
|
+
core/compaction.py is not enough on its own, because it only covers
|
|
1137
|
+
the one caller that exists today.
|
|
1138
|
+
|
|
1139
|
+
Enforced rather than documented, for the same reason D24's declared-
|
|
1140
|
+
permission check is: the invariant is cheap to state and its violation
|
|
1141
|
+
is silent and durable.
|
|
1142
|
+
"""
|
|
1143
|
+
if not advances(thread_id, _current_watermark(thread_id),
|
|
1144
|
+
covers_up_to_message_id):
|
|
1145
|
+
existing = latest_checkpoint(thread_id)
|
|
1146
|
+
logger.warning(
|
|
1147
|
+
"Refused a compaction checkpoint for thread %s whose watermark "
|
|
1148
|
+
"does not advance on the existing one; keeping the existing "
|
|
1149
|
+
"checkpoint. This would have made the thread larger, not "
|
|
1150
|
+
"smaller.", thread_id)
|
|
1151
|
+
return existing["id"] if existing is not None else None
|
|
1152
|
+
|
|
1153
|
+
with Session(engine) as session:
|
|
1154
|
+
checkpoint = CompactionCheckpoint(
|
|
1155
|
+
thread_id=thread_id,
|
|
1156
|
+
summary_text=summary_text,
|
|
1157
|
+
covers_up_to_message_id=covers_up_to_message_id,
|
|
1158
|
+
strategy=strategy,
|
|
1159
|
+
)
|
|
1160
|
+
session.add(checkpoint)
|
|
1161
|
+
session.commit()
|
|
1162
|
+
session.refresh(checkpoint)
|
|
1163
|
+
return checkpoint.id
|
|
1164
|
+
|
|
1165
|
+
|
|
1166
|
+
# ---------------------------------------------------------------------------
|
|
1167
|
+
# ---- Thread summaries (ROADMAP_v2 §21c) -----------------------------------
|
|
1168
|
+
# ---------------------------------------------------------------------------
|
|
1169
|
+
|
|
1170
|
+
def last_message_id(thread_id: UUID) -> Optional[UUID]:
|
|
1171
|
+
"""The id of this thread's most recent archive row, or None if empty.
|
|
1172
|
+
|
|
1173
|
+
§21c needs a position meaning "everything, as of now", to store as a
|
|
1174
|
+
summary's watermark and to compare against a stored one. Nothing existing
|
|
1175
|
+
answers it: `message_ids_from` needs a start id, `turn_start_ids` returns
|
|
1176
|
+
turn starts (and the last row is usually not one), and `_ordered_rows` is
|
|
1177
|
+
private for the reasons its docstring gives.
|
|
1178
|
+
|
|
1179
|
+
Reads the ARCHIVE, like everything else here -- the ordering is
|
|
1180
|
+
`(created_at, id)`, so this is the same last row `advances()` compares
|
|
1181
|
+
positions within.
|
|
1182
|
+
"""
|
|
1183
|
+
rows = _ordered_rows(thread_id)
|
|
1184
|
+
return rows[-1]["id"] if rows else None
|
|
1185
|
+
|
|
1186
|
+
|
|
1187
|
+
def latest_thread_summary(thread_id: UUID) -> Optional[dict]:
|
|
1188
|
+
"""The most recent summary of this thread as a plain column dict, or
|
|
1189
|
+
None if never summarized (#31 -- same reasoning as latest_checkpoint).
|
|
1190
|
+
|
|
1191
|
+
Newest by `(created_at, id)`, matching `latest_checkpoint` -- but note
|
|
1192
|
+
that the resemblance ends at the query. A stale row winning here shows
|
|
1193
|
+
someone an out-of-date description, which `advances()` then detects and
|
|
1194
|
+
corrects; a stale row winning in `latest_checkpoint` silently rewrites a
|
|
1195
|
+
live conversation. See ThreadSummary's docstring.
|
|
1196
|
+
"""
|
|
1197
|
+
with Session(engine) as session:
|
|
1198
|
+
statement = (
|
|
1199
|
+
select(ThreadSummary)
|
|
1200
|
+
.where(ThreadSummary.thread_id == thread_id)
|
|
1201
|
+
.order_by(ThreadSummary.created_at.desc(), ThreadSummary.id.desc())
|
|
1202
|
+
)
|
|
1203
|
+
row = next(iter(session.exec(statement).all()), None)
|
|
1204
|
+
if row is None:
|
|
1205
|
+
return None
|
|
1206
|
+
return {
|
|
1207
|
+
"id": row.id,
|
|
1208
|
+
"thread_id": row.thread_id,
|
|
1209
|
+
"summary_text": row.summary_text,
|
|
1210
|
+
"covers_up_to_message_id": row.covers_up_to_message_id,
|
|
1211
|
+
"created_at": row.created_at,
|
|
1212
|
+
}
|
|
1213
|
+
|
|
1214
|
+
|
|
1215
|
+
def save_thread_summary(
|
|
1216
|
+
thread_id: UUID, summary_text: str, covers_up_to_message_id: UUID,
|
|
1217
|
+
) -> UUID:
|
|
1218
|
+
"""Record one summary of one thread. Adds a row; touches nothing else.
|
|
1219
|
+
|
|
1220
|
+
NO ADVANCE GUARD, deliberately, and this is the one place the difference
|
|
1221
|
+
from `save_checkpoint` above is load-bearing rather than decorative. That
|
|
1222
|
+
function REFUSES a watermark that does not advance, because a backwards
|
|
1223
|
+
compaction watermark becomes the live one and un-compacts the thread. A
|
|
1224
|
+
summary row changes what NO model sees: the worst a redundant or
|
|
1225
|
+
backwards one can do is describe the thread as it was a moment ago, which
|
|
1226
|
+
the next `advances()` check notices.
|
|
1227
|
+
|
|
1228
|
+
So the guard is absent because it would be cargo -- not because the
|
|
1229
|
+
monotonic property does not matter here. Do not add one, and do not route
|
|
1230
|
+
a summary through `save_checkpoint` to get one.
|
|
1231
|
+
"""
|
|
1232
|
+
with Session(engine) as session:
|
|
1233
|
+
summary = ThreadSummary(
|
|
1234
|
+
thread_id=thread_id,
|
|
1235
|
+
summary_text=summary_text,
|
|
1236
|
+
covers_up_to_message_id=covers_up_to_message_id,
|
|
1237
|
+
)
|
|
1238
|
+
session.add(summary)
|
|
1239
|
+
session.commit()
|
|
1240
|
+
session.refresh(summary)
|
|
1241
|
+
return summary.id
|
|
1242
|
+
|
|
1243
|
+
|
|
1244
|
+
# ---------------------------------------------------------------------------
|
|
1245
|
+
# ---- Durable memory (ROADMAP_v2 §21b) -------------------------------------
|
|
1246
|
+
# ---------------------------------------------------------------------------
|
|
1247
|
+
|
|
1248
|
+
def save_memory(
|
|
1249
|
+
content: str, source_thread_id: UUID, scope: str = "project",
|
|
1250
|
+
category: Optional[str] = None, project_path: Optional[str] = None,
|
|
1251
|
+
) -> UUID:
|
|
1252
|
+
"""Record one durable fact. Returns its id.
|
|
1253
|
+
|
|
1254
|
+
`project_path` is required in substance for scope="project" and
|
|
1255
|
+
meaningless for "global" -- the caller resolves it, because this file
|
|
1256
|
+
knows nothing about config_loader or the trust store. memories/manager.py
|
|
1257
|
+
is the one place that decides.
|
|
1258
|
+
"""
|
|
1259
|
+
with Session(engine) as session:
|
|
1260
|
+
row = UserMemory(
|
|
1261
|
+
content=content,
|
|
1262
|
+
category=category,
|
|
1263
|
+
scope=scope,
|
|
1264
|
+
project_path=project_path if scope == "project" else None,
|
|
1265
|
+
source_thread_id=source_thread_id,
|
|
1266
|
+
)
|
|
1267
|
+
session.add(row)
|
|
1268
|
+
session.commit()
|
|
1269
|
+
session.refresh(row)
|
|
1270
|
+
return row.id
|
|
1271
|
+
|
|
1272
|
+
|
|
1273
|
+
def list_memories(
|
|
1274
|
+
project_path: Optional[str] = None, scope: Optional[str] = None,
|
|
1275
|
+
) -> list[dict]:
|
|
1276
|
+
"""Memories visible from `project_path`, NEWEST FIRST.
|
|
1277
|
+
|
|
1278
|
+
Returns global memories plus the project ones recorded against exactly
|
|
1279
|
+
this path -- which is §21 AC6: a memory written in one project does not
|
|
1280
|
+
surface in a thread opened from another. `project_path=None` returns
|
|
1281
|
+
global memories only, which is the honest answer when no project has
|
|
1282
|
+
been resolved rather than a reason to show everything.
|
|
1283
|
+
|
|
1284
|
+
`scope` filters to one kind, for the management commands. Absent means
|
|
1285
|
+
both, which is what prompt assembly wants.
|
|
1286
|
+
|
|
1287
|
+
Both predicates are pushed INTO THE QUERY (#28): `UserMemory` declares
|
|
1288
|
+
indexes on `scope` and `project_path`, and this function used to load
|
|
1289
|
+
every row and filter in Python -- the exact pattern T1 rejected, run
|
|
1290
|
+
against two columns whose indexes had no reader. SQLite answers the
|
|
1291
|
+
visibility rule below from those indexes now.
|
|
1292
|
+
|
|
1293
|
+
Plain dicts rather than ORM instances, for the same reason
|
|
1294
|
+
_ordered_rows returns them: every caller reads these after the Session
|
|
1295
|
+
has closed.
|
|
1296
|
+
"""
|
|
1297
|
+
with Session(engine) as session:
|
|
1298
|
+
statement = select(UserMemory)
|
|
1299
|
+
if scope is not None:
|
|
1300
|
+
statement = statement.where(UserMemory.scope == scope)
|
|
1301
|
+
# Visibility: anything not project-scoped travels; a project memory
|
|
1302
|
+
# only matches its own path. A NULL path therefore matches nothing
|
|
1303
|
+
# except when the caller has no resolved project either -- same as
|
|
1304
|
+
# the Python filter this replaced, edge included deliberately.
|
|
1305
|
+
statement = statement.where(
|
|
1306
|
+
(UserMemory.scope != "project")
|
|
1307
|
+
| (UserMemory.project_path == project_path))
|
|
1308
|
+
statement = statement.order_by(
|
|
1309
|
+
UserMemory.created_at.desc(), UserMemory.id.desc())
|
|
1310
|
+
return [
|
|
1311
|
+
{
|
|
1312
|
+
"id": row.id, "content": row.content,
|
|
1313
|
+
"category": row.category, "scope": row.scope,
|
|
1314
|
+
"project_path": row.project_path,
|
|
1315
|
+
"source_thread_id": row.source_thread_id,
|
|
1316
|
+
"created_at": row.created_at,
|
|
1317
|
+
}
|
|
1318
|
+
for row in session.exec(statement).all()
|
|
1319
|
+
]
|
|
1320
|
+
|
|
1321
|
+
|
|
1322
|
+
def forget_memory(memory_id: UUID) -> bool:
|
|
1323
|
+
"""Delete one memory. True if a row was removed.
|
|
1324
|
+
|
|
1325
|
+
A REAL DELETE, unlike everything else here. MessageLog is append-only
|
|
1326
|
+
because a conversation is a record of what happened; a UserMemory is a
|
|
1327
|
+
standing assertion the harness keeps repeating, and "this is no longer
|
|
1328
|
+
true" has no useful archived form. Returning False for an unknown id
|
|
1329
|
+
rather than raising: the id came from a human reading a list, and a
|
|
1330
|
+
typo deserves "no such memory", not a traceback.
|
|
1331
|
+
"""
|
|
1332
|
+
with Session(engine) as session:
|
|
1333
|
+
row = session.get(UserMemory, memory_id)
|
|
1334
|
+
if row is None:
|
|
1335
|
+
return False
|
|
1336
|
+
session.delete(row)
|
|
1337
|
+
session.commit()
|
|
1338
|
+
return True
|