@cortexkit/pi-magic-context 0.49.26 → 0.49.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/auto-search-worker.js +4 -4
- package/dist/chunk-embedding-write-worker.js +8 -9
- package/dist/{compartment-chunk-embedding-vt6afvdm.js → compartment-chunk-embedding-pkk5beqp.js} +3 -3
- package/dist/{embedding-backfill-jc5fqt5s.js → embedding-backfill-25fp3b8w.js} +7 -7
- package/dist/{index-0dr64gsy.js → index-0s14xaca.js} +2 -2
- package/dist/{index-n8e5e9fp.js → index-0x1c054v.js} +1 -1
- package/dist/{index-hxp6q5f8.js → index-48twbev2.js} +50 -36
- package/dist/{index-j3pz3a1b.js → index-4zemngt7.js} +1 -1
- package/dist/{index-r1hvaza0.js → index-5d506f7y.js} +719 -23
- package/dist/{index-z24d75tz.js → index-6981g1xv.js} +0 -2
- package/dist/{index-zpmz6x24.js → index-bkcwcghx.js} +1308 -104
- package/dist/{index-g0d51qzc.js → index-bvpar0bg.js} +2 -5
- package/dist/{index-w1avzkdt.js → index-g026ztby.js} +1 -1
- package/dist/{index-rjnc6kkf.js → index-kagezf2h.js} +3 -3
- package/dist/{index-dnadppqx.js → index-kmk3xpf9.js} +1 -1
- package/dist/{index-zz7dnyh6.js → index-sdsnb5b6.js} +8 -8
- package/dist/index-v0ht940m.js +488 -0
- package/dist/{index-kbswhkc8.js → index-vea2d9ka.js} +1 -1
- package/dist/{index-w5v0ntbw.js → index-wjbqw6r1.js} +1 -1
- package/dist/{index-z6k6jsmh.js → index-xa90f88m.js} +3 -3
- package/dist/{index-wpy2rzb2.js → index-xw070axm.js} +3 -3
- package/dist/index.js +726 -688
- package/dist/{message-fts-rowid-map-xe3ss2g3.js → message-fts-rowid-map-3mbvdgn9.js} +2 -4
- package/dist/{message-index-bounded-2zffszp9.js → message-index-bounded-avbaa7wa.js} +19 -22
- package/dist/migration-worker.js +7 -8
- package/dist/{project-embedding-registry-bn9ts1h0.js → project-embedding-registry-hc4ahjdt.js} +6 -6
- package/dist/raw-ordinal-warmup-worker.js +7 -30
- package/dist/{read-session-chunk-p0cexvvs.js → read-session-chunk-1rfprgc1.js} +4 -4
- package/dist/{read-session-raw-mq03ws9v.js → read-session-raw-g9b1zdn5.js} +1 -1
- package/dist/{search-archive-jw2wm4tw.js → search-archive-mpjeecwj.js} +5 -5
- package/dist/{search-vmcbbwm5.js → search-vkkfg27q.js} +13 -14
- package/dist/{storage-tags-r48vxthe.js → storage-tags-9qaasrky.js} +2 -2
- package/dist/subagent-entry.js +17 -18
- package/dist/tag-snapshot-worker-pi.js +9 -10
- package/package.json +1 -1
- package/dist/index-87xm7z44.js +0 -2100
- package/dist/index-n4wyxcgr.js +0 -712
- package/dist/index-wrr9rp9a.js +0 -16
- package/dist/message-index-jv1m3g8d.js +0 -58
package/dist/index.js
CHANGED
|
@@ -4,7 +4,6 @@ import {
|
|
|
4
4
|
harnessOwnsOpenCodeStore2
|
|
5
5
|
} from "./index-0h2mkab2.js";
|
|
6
6
|
import {
|
|
7
|
-
getDataDir,
|
|
8
7
|
ensureCortexKitArtifactGitignore,
|
|
9
8
|
getProjectMagicContextHistorianDir,
|
|
10
9
|
getMagicContextStorageDir,
|
|
@@ -46,11 +45,21 @@ import {
|
|
|
46
45
|
cortexKitProjectConfigBasePath,
|
|
47
46
|
migrateMagicContextConfigLocations,
|
|
48
47
|
getProtectedTokensTierOverrides,
|
|
48
|
+
LkgRecoveryFenceError,
|
|
49
|
+
clearLkgFenceProcessState,
|
|
50
|
+
lkgFenceRequiresRebuild,
|
|
51
|
+
lkgRecoveryFenceUnclean,
|
|
52
|
+
ownsLkgDirtyFence,
|
|
53
|
+
invalidateCleanLkgRecoveryFence,
|
|
54
|
+
markLkgRecoveryDirty,
|
|
49
55
|
rememberUnpublishedAutoSearchNoHint,
|
|
50
56
|
getUnpublishedAutoSearchNoHint,
|
|
51
57
|
forgetUnpublishedAutoSearchNoHint,
|
|
52
58
|
registerAutoSearchWork,
|
|
53
59
|
cancelAutoSearchWork,
|
|
60
|
+
readSessionActivity,
|
|
61
|
+
observeSessionActivity,
|
|
62
|
+
backfillSessionActivity,
|
|
54
63
|
readSessionMeta,
|
|
55
64
|
getOrCreateSessionMeta,
|
|
56
65
|
updateSessionMeta,
|
|
@@ -251,7 +260,7 @@ import {
|
|
|
251
260
|
registerTodoStateLifecycle,
|
|
252
261
|
syncCtxMemoryToolEnabled,
|
|
253
262
|
registerMagicContextTools
|
|
254
|
-
} from "./index-
|
|
263
|
+
} from "./index-bkcwcghx.js";
|
|
255
264
|
import"./index-4knyh1kt.js";
|
|
256
265
|
import {
|
|
257
266
|
canonicalModelIdentity,
|
|
@@ -277,6 +286,7 @@ import {
|
|
|
277
286
|
isFailClosedBlockingError,
|
|
278
287
|
shouldBypassFailClosedBlock,
|
|
279
288
|
createFailClosedController,
|
|
289
|
+
piHarnessKindFromExecutable,
|
|
280
290
|
getErrorMessage,
|
|
281
291
|
describeError,
|
|
282
292
|
startBootDeadline,
|
|
@@ -292,8 +302,9 @@ import {
|
|
|
292
302
|
openCurrentDatabase,
|
|
293
303
|
openDatabaseAsync,
|
|
294
304
|
closeDatabase
|
|
295
|
-
} from "./index-
|
|
305
|
+
} from "./index-5d506f7y.js";
|
|
296
306
|
import {
|
|
307
|
+
sameProjectDirectory,
|
|
297
308
|
pruneMissingProjectIdentities,
|
|
298
309
|
setHomeProjectPermission,
|
|
299
310
|
ProjectIdentityError,
|
|
@@ -304,7 +315,7 @@ import {
|
|
|
304
315
|
resolveProjectIdentityForSession,
|
|
305
316
|
beginBootQuietPeriod,
|
|
306
317
|
scheduleAfterBootQuiet
|
|
307
|
-
} from "./index-
|
|
318
|
+
} from "./index-48twbev2.js";
|
|
308
319
|
import {
|
|
309
320
|
detectThinkingBindingMismatch,
|
|
310
321
|
detectLatestTurnThinkingMismatch,
|
|
@@ -462,11 +473,12 @@ import {
|
|
|
462
473
|
getCachedAbsoluteMessageCount2,
|
|
463
474
|
primeInMemoryTailRawMessageCache2,
|
|
464
475
|
readRawSessionMessageIdOrdinalsForRange2,
|
|
476
|
+
compareRawSessionMessageOrder2,
|
|
465
477
|
getRawSessionMessageCount2,
|
|
466
478
|
getRawSessionTagKeysThrough2,
|
|
467
479
|
getLegacyProtectedTailStartOrdinal2,
|
|
468
480
|
readSessionChunk2
|
|
469
|
-
} from "./index-
|
|
481
|
+
} from "./index-kagezf2h.js";
|
|
470
482
|
import {
|
|
471
483
|
UNKNOWN_FIT_RATIO,
|
|
472
484
|
hasMeasuredDecisionCalibration,
|
|
@@ -553,7 +565,7 @@ import {
|
|
|
553
565
|
getToolTagNumberByOwner2,
|
|
554
566
|
getNullOwnerToolTag2,
|
|
555
567
|
adoptNullOwnerToolTag2
|
|
556
|
-
} from "./index-
|
|
568
|
+
} from "./index-0x1c054v.js";
|
|
557
569
|
import {
|
|
558
570
|
textMentionsRecentCommit,
|
|
559
571
|
hasMeaningfulUserText2,
|
|
@@ -587,36 +599,7 @@ import {
|
|
|
587
599
|
observeTemporalDecisions,
|
|
588
600
|
freezeTemporalDecisions,
|
|
589
601
|
tagIdentityRevisionKey
|
|
590
|
-
} from "./index-
|
|
591
|
-
import {
|
|
592
|
-
piHarnessKindFromExecutable
|
|
593
|
-
} from "./index-n4wyxcgr.js";
|
|
594
|
-
import {
|
|
595
|
-
LkgRecoveryFenceError,
|
|
596
|
-
clearLkgFenceProcessState,
|
|
597
|
-
takeLkgRecoveryLockSweepPage,
|
|
598
|
-
lkgFenceRequiresRebuild,
|
|
599
|
-
lkgRecoveryFenceUnclean,
|
|
600
|
-
ownsLkgDirtyFence,
|
|
601
|
-
invalidateCleanLkgRecoveryFence,
|
|
602
|
-
markLkgRecoveryDirty,
|
|
603
|
-
gaDatabasePath,
|
|
604
|
-
readSessionActivity,
|
|
605
|
-
observeSessionActivity,
|
|
606
|
-
backfillSessionActivity,
|
|
607
|
-
clearCompressionDepth,
|
|
608
|
-
clearCompressionDepthRange,
|
|
609
|
-
cleanSearchText,
|
|
610
|
-
matchedSearchSnippet,
|
|
611
|
-
getMessageIndexSourceIdentity2,
|
|
612
|
-
isMessageIndexSourceCurrent2,
|
|
613
|
-
getLastIndexedOrdinal2,
|
|
614
|
-
getMessageIndexReconciliationStartOrdinal2,
|
|
615
|
-
isMessageIndexReconciledThrough2,
|
|
616
|
-
indexSingleMessage2,
|
|
617
|
-
indexMessagesAfterOrdinal2,
|
|
618
|
-
sweepOrphanedOpenCodeMessageIndexesCooperatively2
|
|
619
|
-
} from "./index-87xm7z44.js";
|
|
602
|
+
} from "./index-g026ztby.js";
|
|
620
603
|
import {
|
|
621
604
|
BoundedSessionMap,
|
|
622
605
|
resolveOpenCodeDbPath,
|
|
@@ -633,11 +616,11 @@ import {
|
|
|
633
616
|
import {
|
|
634
617
|
runSessionWarmupWorker,
|
|
635
618
|
RAW_SUMMARY_TEXT_MAX_CHARS2
|
|
636
|
-
} from "./index-
|
|
619
|
+
} from "./index-6981g1xv.js";
|
|
637
620
|
import {
|
|
638
621
|
formatSynapseLaneDescriptor
|
|
639
622
|
} from "./index-gwxaz0z5.js";
|
|
640
|
-
import"./index-
|
|
623
|
+
import"./index-bvpar0bg.js";
|
|
641
624
|
import {
|
|
642
625
|
buildCanonicalChunkTextFromFts2,
|
|
643
626
|
buildCompartmentSummaryFallbackText2,
|
|
@@ -647,7 +630,7 @@ import {
|
|
|
647
630
|
replaceCompartmentChunkEmbeddings2,
|
|
648
631
|
deleteChunkEmbedBackoffForSession2,
|
|
649
632
|
chunkWindowSourceKey2
|
|
650
|
-
} from "./index-
|
|
633
|
+
} from "./index-kmk3xpf9.js";
|
|
651
634
|
import {
|
|
652
635
|
recordSessionProjectIdentity,
|
|
653
636
|
GIT_SWEEP_LEASE_RENEWAL_MS,
|
|
@@ -656,7 +639,7 @@ import {
|
|
|
656
639
|
markGitSweepSuccessAndRelease,
|
|
657
640
|
parkGitSweepNonIndexable,
|
|
658
641
|
releaseGitSweepLease
|
|
659
|
-
} from "./index-
|
|
642
|
+
} from "./index-4zemngt7.js";
|
|
660
643
|
import {
|
|
661
644
|
embedPauseBySession,
|
|
662
645
|
embedRunStateBySession,
|
|
@@ -682,7 +665,20 @@ import {
|
|
|
682
665
|
embedSessionCompartmentChunks2,
|
|
683
666
|
getEmbeddingCoverageStatus2,
|
|
684
667
|
getAutoEmbeddingSessionCoverage2
|
|
685
|
-
} from "./index-
|
|
668
|
+
} from "./index-xa90f88m.js";
|
|
669
|
+
import {
|
|
670
|
+
clearCompressionDepth,
|
|
671
|
+
clearCompressionDepthRange,
|
|
672
|
+
cleanSearchText,
|
|
673
|
+
matchedSearchSnippet,
|
|
674
|
+
getMessageIndexSourceIdentity,
|
|
675
|
+
isMessageIndexSourceCurrent,
|
|
676
|
+
getLastIndexedOrdinal,
|
|
677
|
+
getMessageIndexReconciliationStartOrdinal,
|
|
678
|
+
isMessageIndexReconciledThrough,
|
|
679
|
+
indexSingleMessage,
|
|
680
|
+
indexMessagesAfterOrdinal
|
|
681
|
+
} from "./index-v0ht940m.js";
|
|
686
682
|
import {
|
|
687
683
|
rememberV1HiddenRun
|
|
688
684
|
} from "./index-6x556c2q.js";
|
|
@@ -697,7 +693,7 @@ import {
|
|
|
697
693
|
expandWorkspaceIdentitySetWithAliases,
|
|
698
694
|
sourceNameForMemory,
|
|
699
695
|
computeWorkspaceEpochFingerprint
|
|
700
|
-
} from "./index-
|
|
696
|
+
} from "./index-0s14xaca.js";
|
|
701
697
|
import {
|
|
702
698
|
configureContextDatabasePragmas2
|
|
703
699
|
} from "./index-fapenpch.js";
|
|
@@ -747,7 +743,7 @@ import {
|
|
|
747
743
|
normalizeVerificationFiles,
|
|
748
744
|
indexCommitsForProject,
|
|
749
745
|
embedUnembeddedCommits
|
|
750
|
-
} from "./index-
|
|
746
|
+
} from "./index-sdsnb5b6.js";
|
|
751
747
|
import {
|
|
752
748
|
setEmbeddingSessionBusy2,
|
|
753
749
|
isEmbeddingHostBusy2
|
|
@@ -783,7 +779,7 @@ import {
|
|
|
783
779
|
} from "./index-wqpjhx1p.js";
|
|
784
780
|
import {
|
|
785
781
|
cosineSimilarity
|
|
786
|
-
} from "./index-
|
|
782
|
+
} from "./index-wjbqw6r1.js";
|
|
787
783
|
import"./index-6wdwv3a6.js";
|
|
788
784
|
import {
|
|
789
785
|
DREAMER_PRIMER_INVESTIGATOR_ALLOWED_TOOLS,
|
|
@@ -811,10 +807,10 @@ import {
|
|
|
811
807
|
drainPiTagSnapshotSession,
|
|
812
808
|
assertPiTagSnapshotReady,
|
|
813
809
|
ensurePiTagSnapshotReady
|
|
814
|
-
} from "./index-
|
|
810
|
+
} from "./index-xw070axm.js";
|
|
815
811
|
import"./index-jn44h81p.js";
|
|
816
812
|
import"./index-1cvyaw9a.js";
|
|
817
|
-
import"./index-
|
|
813
|
+
import"./index-vea2d9ka.js";
|
|
818
814
|
import {
|
|
819
815
|
__toESM
|
|
820
816
|
} from "./index-22jkk6wg.js";
|
|
@@ -824,9 +820,6 @@ import {
|
|
|
824
820
|
import {
|
|
825
821
|
normalizeProtectedToolName
|
|
826
822
|
} from "./index-674f6qs2.js";
|
|
827
|
-
import {
|
|
828
|
-
sameProjectDirectory
|
|
829
|
-
} from "./index-wrr9rp9a.js";
|
|
830
823
|
import {
|
|
831
824
|
logSlowWriteTransaction,
|
|
832
825
|
WriteTransactionSteps
|
|
@@ -3556,7 +3549,7 @@ function startHostStallProfilerSwitch() {
|
|
|
3556
3549
|
// package.json
|
|
3557
3550
|
var package_default = {
|
|
3558
3551
|
name: "@cortexkit/pi-magic-context",
|
|
3559
|
-
version: "0.49.
|
|
3552
|
+
version: "0.49.27",
|
|
3560
3553
|
type: "module",
|
|
3561
3554
|
description: "Pi and OMP coding agent extension for Magic Context — cross-session memory and context management",
|
|
3562
3555
|
main: "dist/index.js",
|
|
@@ -16930,6 +16923,21 @@ function verifyAssistantText(messages) {
|
|
|
16930
16923
|
const start = text.lastIndexOf("<verify>");
|
|
16931
16924
|
return start >= 0 ? text.slice(start) : text;
|
|
16932
16925
|
}
|
|
16926
|
+
function validateVerifyCompletion(messages, text, expectedIds, responseText = text) {
|
|
16927
|
+
let parsed;
|
|
16928
|
+
try {
|
|
16929
|
+
parsed = validateVerifyManifest(text, expectedIds);
|
|
16930
|
+
} catch (error) {
|
|
16931
|
+
throw providerOutputFailureFromInvalidManifest(messages, responseText) ?? error;
|
|
16932
|
+
}
|
|
16933
|
+
const ids = [...parsed.verified, ...parsed.updated, ...parsed.archived, ...parsed.skipped].map((entry) => entry.id);
|
|
16934
|
+
if (ids.length !== expectedIds.size || ids.some((id) => !expectedIds.has(id))) {
|
|
16935
|
+
const providerFailure = providerOutputFailureFromInvalidManifest(messages, responseText);
|
|
16936
|
+
if (providerFailure)
|
|
16937
|
+
throw providerFailure;
|
|
16938
|
+
}
|
|
16939
|
+
return parsed;
|
|
16940
|
+
}
|
|
16933
16941
|
var IDENTICAL_PROVIDER_FAILURE_BATCH_LIMIT = 2;
|
|
16934
16942
|
var VERIFY_BATCH_FLOOR_MS = 30000;
|
|
16935
16943
|
function computeVerifyBatchSliceMs(remainingMs, _batchesRemaining) {
|
|
@@ -17088,6 +17096,7 @@ async function verifyOneBatch(args, batch, sliceMs, signal) {
|
|
|
17088
17096
|
const timer = setTimeout(() => batchController.abort(new Error(`verify slice timed out after ${sliceMs}ms`)), sliceMs);
|
|
17089
17097
|
const batchSignal = batchController.signal;
|
|
17090
17098
|
const tokenBudget = verifyBatchTokenBudget(batch.length, args.tokenBudget);
|
|
17099
|
+
const expectedIds = new Set(batch.map((memory) => memory.id));
|
|
17091
17100
|
let agentSessionId = null;
|
|
17092
17101
|
let hiddenSessionId = null;
|
|
17093
17102
|
let promptSettled = false;
|
|
@@ -17140,13 +17149,10 @@ async function verifyOneBatch(args, batch, sliceMs, signal) {
|
|
|
17140
17149
|
onBudgetUpdate({ spent: 0, finalizeFired: false, sessionId });
|
|
17141
17150
|
},
|
|
17142
17151
|
parse: (text, completion) => {
|
|
17143
|
-
const providerFailure = providerOutputFailureFromInvalidManifest(completion.messages ?? [], text);
|
|
17144
|
-
if (providerFailure)
|
|
17145
|
-
throw providerFailure;
|
|
17146
17152
|
const manifest = verifyAssistantText(completion.messages ?? []) ?? (text.trim() ? text : null);
|
|
17147
17153
|
if (!manifest)
|
|
17148
17154
|
throw new Error("verify returned no output");
|
|
17149
|
-
return
|
|
17155
|
+
return validateVerifyCompletion(completion.messages ?? [], manifest, expectedIds, text);
|
|
17150
17156
|
}
|
|
17151
17157
|
});
|
|
17152
17158
|
recordInvocation4(args, startedAt, {
|
|
@@ -17208,10 +17214,7 @@ async function verifyOneBatch(args, batch, sliceMs, signal) {
|
|
|
17208
17214
|
const text = verifyAssistantText(messages);
|
|
17209
17215
|
if (!text)
|
|
17210
17216
|
throw new Error("verify returned no output");
|
|
17211
|
-
|
|
17212
|
-
if (providerFailure)
|
|
17213
|
-
throw providerFailure;
|
|
17214
|
-
return validateVerifyManifest(text, new Set(batch.map((memory) => memory.id)));
|
|
17217
|
+
return validateVerifyCompletion(messages, text, expectedIds);
|
|
17215
17218
|
}
|
|
17216
17219
|
});
|
|
17217
17220
|
promptSettled = true;
|
|
@@ -19481,541 +19484,215 @@ function clearDreamerTickFailure(db) {
|
|
|
19481
19484
|
db.prepare("DELETE FROM schema_migrations_meta WHERE key = ?").run(DREAMER_TICK_FAILURE_KEY);
|
|
19482
19485
|
}
|
|
19483
19486
|
|
|
19484
|
-
// ../plugin/src/
|
|
19485
|
-
var
|
|
19486
|
-
var
|
|
19487
|
-
var
|
|
19488
|
-
|
|
19489
|
-
|
|
19487
|
+
// ../plugin/src/plugin/dream-timer.ts
|
|
19488
|
+
var DREAM_TIMER_INTERVAL_MS = 15 * 60 * 1000;
|
|
19489
|
+
var GIT_COMMIT_BACKLOG_DRAIN_MAX_MS = 5 * 60 * 1000;
|
|
19490
|
+
var BOOT_PROJECT_JITTER_SLOT_MS = 1000;
|
|
19491
|
+
var activeTimer = null;
|
|
19492
|
+
var tickInFlight = false;
|
|
19493
|
+
var startupTickTimer = null;
|
|
19494
|
+
var startupTimers = new Map;
|
|
19495
|
+
var startupJitters = new Map;
|
|
19496
|
+
var nextStartupJitterSlot = 0;
|
|
19497
|
+
function directoryStillExists(directory) {
|
|
19498
|
+
try {
|
|
19499
|
+
return statSync3(directory).isDirectory();
|
|
19500
|
+
} catch {
|
|
19490
19501
|
return false;
|
|
19491
|
-
const e = error;
|
|
19492
|
-
if (typeof e.code === "string") {
|
|
19493
|
-
if (e.code === "SQLITE_BUSY" || e.code === "SQLITE_LOCKED")
|
|
19494
|
-
return true;
|
|
19495
|
-
}
|
|
19496
|
-
if (typeof e.message === "string") {
|
|
19497
|
-
if (/database is locked/i.test(e.message))
|
|
19498
|
-
return true;
|
|
19499
|
-
if (/sqlite_(busy|locked)/i.test(e.message))
|
|
19500
|
-
return true;
|
|
19501
19502
|
}
|
|
19502
|
-
return false;
|
|
19503
|
-
}
|
|
19504
|
-
var INCREMENTAL_DEBOUNCE_MS = 100;
|
|
19505
|
-
var RECONCILIATION_BATCH_SIZE = 100;
|
|
19506
|
-
|
|
19507
|
-
class MagicContextMessageIndexHeapHolder {
|
|
19508
|
-
reconciledSessions = new Set;
|
|
19509
|
-
reconciledSnapshots = new Map;
|
|
19510
|
-
stalledSessions = new Set;
|
|
19511
|
-
reconciliationScheduledSessions = new Set;
|
|
19512
|
-
clearAndReindexScheduledSessions = new Map;
|
|
19513
|
-
sessionLocks = new Map;
|
|
19514
|
-
incrementalTimers = new Map;
|
|
19515
|
-
pendingIncrementalKeys = new Set;
|
|
19516
|
-
completedIncrementalKeys = new Set;
|
|
19517
|
-
activeReconcilerBuffers = new Map;
|
|
19518
|
-
}
|
|
19519
|
-
var heapHolder = new MagicContextMessageIndexHeapHolder;
|
|
19520
|
-
var PREPARE_RETRY_MS = 60000;
|
|
19521
|
-
async function sourceReady(source, sessionId) {
|
|
19522
|
-
const prepare = source?.prepare;
|
|
19523
|
-
return typeof prepare === "function" ? prepare(sessionId) : true;
|
|
19524
19503
|
}
|
|
19525
|
-
function
|
|
19526
|
-
|
|
19527
|
-
|
|
19528
|
-
|
|
19529
|
-
|
|
19504
|
+
function openTimerDatabaseOrNull(context) {
|
|
19505
|
+
let db;
|
|
19506
|
+
try {
|
|
19507
|
+
db = openCurrentDatabase();
|
|
19508
|
+
} catch (error) {
|
|
19509
|
+
log2(`[dreamer] storage fatal; skipping ${context}: ${getErrorMessage(error)}`);
|
|
19510
|
+
return null;
|
|
19530
19511
|
}
|
|
19531
|
-
|
|
19532
|
-
}
|
|
19533
|
-
|
|
19534
|
-
|
|
19512
|
+
if (!db) {
|
|
19513
|
+
log2(`[dreamer] storage unavailable; skipping ${context} (the cache schema is newer than this binary supports — restart/upgrade OpenCode/Pi/Magic Context to recover)`);
|
|
19514
|
+
return null;
|
|
19515
|
+
}
|
|
19516
|
+
return db;
|
|
19535
19517
|
}
|
|
19536
|
-
|
|
19537
|
-
|
|
19538
|
-
|
|
19539
|
-
return;
|
|
19540
|
-
}).then(async () => {
|
|
19541
|
-
await operation();
|
|
19542
|
-
});
|
|
19543
|
-
heapHolder.sessionLocks.set(sessionId, run);
|
|
19544
|
-
run.finally(() => {
|
|
19545
|
-
if (heapHolder.sessionLocks.get(sessionId) === run) {
|
|
19546
|
-
heapHolder.sessionLocks.delete(sessionId);
|
|
19547
|
-
}
|
|
19548
|
-
}).catch(() => {
|
|
19518
|
+
var refusedEmptyIdentityDirectories = new Set;
|
|
19519
|
+
function logEmptyIdentityRefusalOnce(directory) {
|
|
19520
|
+
if (refusedEmptyIdentityDirectories.has(directory))
|
|
19549
19521
|
return;
|
|
19550
|
-
|
|
19551
|
-
|
|
19522
|
+
refusedEmptyIdentityDirectories.add(directory);
|
|
19523
|
+
log2(`[dreamer] not registering ${directory}: it has no project identity, so no project-scoped work runs for it`);
|
|
19552
19524
|
}
|
|
19553
|
-
|
|
19554
|
-
|
|
19555
|
-
|
|
19525
|
+
var registeredProjects = new Map;
|
|
19526
|
+
function stopDreamScheduleTimerIfIdle() {
|
|
19527
|
+
if (registeredProjects.size !== 0)
|
|
19556
19528
|
return;
|
|
19529
|
+
const stopped = activeTimer !== null || startupTickTimer !== null;
|
|
19530
|
+
if (activeTimer) {
|
|
19531
|
+
clearInterval(activeTimer);
|
|
19532
|
+
activeTimer = null;
|
|
19557
19533
|
}
|
|
19558
|
-
|
|
19559
|
-
|
|
19534
|
+
if (startupTickTimer) {
|
|
19535
|
+
clearTimeout(startupTickTimer);
|
|
19536
|
+
startupTickTimer = null;
|
|
19537
|
+
}
|
|
19538
|
+
startupJitters.clear();
|
|
19539
|
+
nextStartupJitterSlot = 0;
|
|
19540
|
+
if (stopped)
|
|
19541
|
+
log2("[dreamer] stopped dream schedule timer (no projects left)");
|
|
19560
19542
|
}
|
|
19561
|
-
function
|
|
19562
|
-
if (
|
|
19543
|
+
async function startDreamScheduleTimer(args) {
|
|
19544
|
+
if (!isUsableProjectIdentity(args.projectIdentity)) {
|
|
19545
|
+
logEmptyIdentityRefusalOnce(args.directory);
|
|
19563
19546
|
return;
|
|
19564
|
-
heapHolder.stalledSessions.add(sessionId);
|
|
19565
|
-
sessionLog2(sessionId, `message FTS reconciliation stalled: ${reason}; will retry`);
|
|
19566
|
-
}
|
|
19567
|
-
async function reconcileSessionIndex(db, sessionId, readMessages) {
|
|
19568
|
-
if (!await sourceReady(readMessages, sessionId)) {
|
|
19569
|
-
logReconciliationStall(sessionId, "raw message reader is not ready");
|
|
19570
|
-
return false;
|
|
19571
19547
|
}
|
|
19572
|
-
|
|
19573
|
-
|
|
19548
|
+
beginBootQuietPeriod();
|
|
19549
|
+
const db = openTimerDatabaseOrNull("schedule timer registration");
|
|
19550
|
+
if (!db)
|
|
19551
|
+
return;
|
|
19552
|
+
const dreamingEnabled = Boolean(args.dreamerConfig && args.dreamerConfig.disable !== true);
|
|
19553
|
+
const embeddingSweepEnabled = args.memoryEnabled === true;
|
|
19554
|
+
const commitIndexingEnabled = args.gitCommitIndexing?.enabled === true;
|
|
19555
|
+
const previousRegistration = registeredProjects.get(args.directory);
|
|
19556
|
+
const isNewRegistration = previousRegistration === undefined;
|
|
19557
|
+
if (previousRegistration && previousRegistration !== args) {
|
|
19558
|
+
const pendingStartup = startupTimers.get(args.directory);
|
|
19559
|
+
if (pendingStartup) {
|
|
19560
|
+
clearTimeout(pendingStartup);
|
|
19561
|
+
startupTimers.delete(args.directory);
|
|
19562
|
+
}
|
|
19563
|
+
}
|
|
19564
|
+
registeredProjects.set(args.directory, args);
|
|
19565
|
+
if (isNewRegistration) {
|
|
19566
|
+
log2(`[dreamer] registered project ${args.projectIdentity} (dreaming=${dreamingEnabled} embeddings=${embeddingSweepEnabled} commits=${commitIndexingEnabled}; total=${registeredProjects.size})`);
|
|
19567
|
+
}
|
|
19568
|
+
if (!activeTimer) {
|
|
19569
|
+
log2(`[dreamer] started independent schedule timer (every ${DREAM_TIMER_INTERVAL_MS / 60000}m)`);
|
|
19570
|
+
startupTickTimer = scheduleAfterBootQuiet(() => {
|
|
19571
|
+
startupTickTimer = null;
|
|
19572
|
+
runTick("startup");
|
|
19573
|
+
});
|
|
19574
|
+
const timer = inRootAsyncContext2(() => setInterval(() => runTick("interval"), DREAM_TIMER_INTERVAL_MS));
|
|
19575
|
+
if (typeof timer === "object" && "unref" in timer) {
|
|
19576
|
+
timer.unref();
|
|
19577
|
+
}
|
|
19578
|
+
activeTimer = timer;
|
|
19579
|
+
} else if (isNewRegistration || previousRegistration !== args) {
|
|
19580
|
+
scheduleInitialProjectRun(args, db);
|
|
19581
|
+
}
|
|
19582
|
+
return () => {
|
|
19583
|
+
if (registeredProjects.get(args.directory) !== args) {
|
|
19584
|
+
stopDreamScheduleTimerIfIdle();
|
|
19574
19585
|
return;
|
|
19575
|
-
let fallbackSnapshot = null;
|
|
19576
|
-
try {
|
|
19577
|
-
const fullReader = typeof readMessages === "function" ? readMessages : null;
|
|
19578
|
-
const revision = readMessages.getRevision?.(sessionId);
|
|
19579
|
-
const previous = heapHolder.reconciledSnapshots.get(sessionId);
|
|
19580
|
-
const finalWatermark = previous && revision === previous.revision ? previous.count : revision !== undefined && readMessages.getCountAfterRevision ? previous && revision > previous.revision ? previous.count + readMessages.getCountAfterRevision(sessionId, previous.revision, revision) : readMessages.getCountAfterRevision(sessionId, -1, revision) : readMessages.getCount ? readMessages.getCount(sessionId) : (fallbackSnapshot = fullReader?.(sessionId) ?? []).length;
|
|
19581
|
-
let cursor = getMessageIndexReconciliationStartOrdinal2(db, sessionId);
|
|
19582
|
-
let after = cursor === previous?.count ? previous.after : undefined;
|
|
19583
|
-
while (cursor < finalWatermark) {
|
|
19584
|
-
const pageEnd = Math.min(finalWatermark, cursor + RECONCILIATION_BATCH_SIZE);
|
|
19585
|
-
const messages = readMessages.readPage ? readMessages.readPage(sessionId, cursor, RECONCILIATION_BATCH_SIZE, finalWatermark, after) : (fallbackSnapshot ?? []).filter((message) => message.ordinal > cursor && message.ordinal <= pageEnd);
|
|
19586
|
-
const retainedMessages = fallbackSnapshot ?? messages;
|
|
19587
|
-
heapHolder.activeReconcilerBuffers.set(sessionId, retainedMessages);
|
|
19588
|
-
withSqliteBackgroundWriter2(() => indexMessagesAfterOrdinal2(db, sessionId, messages, cursor, pageEnd));
|
|
19589
|
-
const nextCursor = getMessageIndexReconciliationStartOrdinal2(db, sessionId);
|
|
19590
|
-
if (nextCursor <= cursor)
|
|
19591
|
-
break;
|
|
19592
|
-
cursor = nextCursor;
|
|
19593
|
-
const last = messages.at(-1);
|
|
19594
|
-
after = last?.ordinal === cursor && typeof last.createdAt === "number" ? { timeCreated: last.createdAt, id: last.id } : undefined;
|
|
19595
|
-
if (cursor < finalWatermark) {
|
|
19596
|
-
await yieldToEventLoop();
|
|
19597
|
-
}
|
|
19598
|
-
}
|
|
19599
|
-
if (isMessageIndexReconciledThrough2(db, sessionId, finalWatermark)) {
|
|
19600
|
-
heapHolder.reconciledSessions.add(sessionId);
|
|
19601
|
-
if (revision !== undefined) {
|
|
19602
|
-
if (!after && finalWatermark > 0 && readMessages.readPage) {
|
|
19603
|
-
const last = readMessages.readPage(sessionId, finalWatermark - 1, 1, finalWatermark).at(-1);
|
|
19604
|
-
if (last?.ordinal === finalWatermark && typeof last.createdAt === "number")
|
|
19605
|
-
after = { timeCreated: last.createdAt, id: last.id };
|
|
19606
|
-
}
|
|
19607
|
-
heapHolder.reconciledSnapshots.set(sessionId, {
|
|
19608
|
-
revision,
|
|
19609
|
-
count: finalWatermark,
|
|
19610
|
-
after
|
|
19611
|
-
});
|
|
19612
|
-
}
|
|
19613
|
-
} else {
|
|
19614
|
-
logReconciliationStall(sessionId, `gap after ordinal ${cursor} (store count ${finalWatermark})`);
|
|
19615
|
-
}
|
|
19616
|
-
} finally {
|
|
19617
|
-
heapHolder.activeReconcilerBuffers.delete(sessionId);
|
|
19618
19586
|
}
|
|
19619
|
-
|
|
19620
|
-
|
|
19587
|
+
registeredProjects.delete(args.directory);
|
|
19588
|
+
const startupTimer = startupTimers.get(args.directory);
|
|
19589
|
+
if (startupTimer) {
|
|
19590
|
+
clearTimeout(startupTimer);
|
|
19591
|
+
startupTimers.delete(args.directory);
|
|
19592
|
+
}
|
|
19593
|
+
log2(`[dreamer] unregistered project ${args.projectIdentity} (remaining=${registeredProjects.size})`);
|
|
19594
|
+
stopDreamScheduleTimerIfIdle();
|
|
19595
|
+
};
|
|
19621
19596
|
}
|
|
19622
|
-
|
|
19623
|
-
|
|
19624
|
-
|
|
19625
|
-
|
|
19626
|
-
|
|
19627
|
-
|
|
19628
|
-
defer(() => {
|
|
19629
|
-
let appendedDuringReconciliation = false;
|
|
19630
|
-
const previousSnapshot = heapHolder.reconciledSnapshots.get(sessionId);
|
|
19631
|
-
reconcileSessionIndex(db, sessionId, readMessages).then((ran) => {
|
|
19632
|
-
if (ran) {
|
|
19633
|
-
const snapshot = heapHolder.reconciledSnapshots.get(sessionId);
|
|
19634
|
-
appendedDuringReconciliation = snapshot !== undefined && snapshot !== previousSnapshot && readMessages.getRevision !== undefined && readMessages.getRevision(sessionId) !== snapshot.revision;
|
|
19635
|
-
return;
|
|
19636
|
-
}
|
|
19637
|
-
inRootAsyncContext2(() => setTimeout(() => {
|
|
19638
|
-
heapHolder.reconciliationScheduledSessions.delete(sessionId);
|
|
19639
|
-
scheduleReconciliation(db, sessionId, readMessages);
|
|
19640
|
-
}, PREPARE_RETRY_MS)).unref?.();
|
|
19641
|
-
}).catch((error) => {
|
|
19642
|
-
logReconciliationStall(sessionId, error instanceof Error ? error.message : String(error));
|
|
19643
|
-
if (isDatabaseLockedError(error))
|
|
19644
|
-
inRootAsyncContext2(() => setTimeout(() => scheduleReconciliation(db, sessionId, readMessages), 1000)).unref();
|
|
19645
|
-
}).finally(() => {
|
|
19646
|
-
heapHolder.reconciliationScheduledSessions.delete(sessionId);
|
|
19647
|
-
if (appendedDuringReconciliation)
|
|
19648
|
-
scheduleReconciliation(db, sessionId, readMessages);
|
|
19649
|
-
});
|
|
19650
|
-
});
|
|
19651
|
-
});
|
|
19652
|
-
}
|
|
19653
|
-
function scheduleIncrementalIndex(db, sessionId, messageId, messageSource) {
|
|
19654
|
-
const schedulingKey = `${sessionId}\x00${messageId}`;
|
|
19655
|
-
const epoch = sessionEpochs.get(sessionId) ?? 0;
|
|
19656
|
-
if (heapHolder.pendingIncrementalKeys.has(schedulingKey))
|
|
19597
|
+
var tickStages = {
|
|
19598
|
+
runMessageHistoryMaintenance,
|
|
19599
|
+
runProjectMaintenance
|
|
19600
|
+
};
|
|
19601
|
+
function runTick(origin) {
|
|
19602
|
+
if (tickInFlight)
|
|
19657
19603
|
return;
|
|
19658
|
-
|
|
19659
|
-
|
|
19660
|
-
|
|
19661
|
-
|
|
19662
|
-
|
|
19663
|
-
|
|
19664
|
-
const indexMessage = async () => {
|
|
19665
|
-
if ((sessionEpochs.get(sessionId) ?? 0) !== epoch || !heapHolder.pendingIncrementalKeys.has(schedulingKey))
|
|
19666
|
-
return;
|
|
19667
|
-
const message = typeof messageSource === "function" ? messageSource(sessionId, messageId) : messageSource;
|
|
19668
|
-
if (!message)
|
|
19669
|
-
return;
|
|
19670
|
-
const revisionKey = `${schedulingKey}\x00${getMessageIndexSourceIdentity2(message)}`;
|
|
19671
|
-
if (heapHolder.completedIncrementalKeys.has(revisionKey))
|
|
19672
|
-
return;
|
|
19673
|
-
const currentWatermark = getLastIndexedOrdinal2(db, sessionId);
|
|
19674
|
-
if (message.ordinal <= currentWatermark && isMessageIndexSourceCurrent2(db, sessionId, message)) {
|
|
19675
|
-
heapHolder.completedIncrementalKeys.add(revisionKey);
|
|
19604
|
+
tickInFlight = true;
|
|
19605
|
+
log2(`[magic-context] maintenance tick (${origin}) — projects=${registeredProjects.size}`);
|
|
19606
|
+
(async () => {
|
|
19607
|
+
try {
|
|
19608
|
+
const db = openTimerDatabaseOrNull("maintenance tick");
|
|
19609
|
+
if (!db)
|
|
19676
19610
|
return;
|
|
19611
|
+
let failure = null;
|
|
19612
|
+
const noteFailure = (stage, error) => {
|
|
19613
|
+
failure ??= { at: Date.now(), stage, message: getErrorMessage(error) };
|
|
19614
|
+
};
|
|
19615
|
+
try {
|
|
19616
|
+
await tickStages.runMessageHistoryMaintenance(db);
|
|
19617
|
+
} catch (error) {
|
|
19618
|
+
log2("[magic-context] timer-triggered message-history maintenance failed:", error);
|
|
19619
|
+
noteFailure("message-history maintenance", error);
|
|
19677
19620
|
}
|
|
19678
|
-
const
|
|
19679
|
-
|
|
19680
|
-
|
|
19681
|
-
|
|
19682
|
-
|
|
19683
|
-
|
|
19684
|
-
|
|
19621
|
+
for (const reg of registeredProjects.values()) {
|
|
19622
|
+
try {
|
|
19623
|
+
if (origin === "startup") {
|
|
19624
|
+
scheduleInitialProjectRun(reg, db);
|
|
19625
|
+
} else {
|
|
19626
|
+
await tickStages.runProjectMaintenance(reg, origin, db);
|
|
19627
|
+
}
|
|
19628
|
+
} catch (error) {
|
|
19629
|
+
log2(`[magic-context] timer-triggered maintenance failed for ${reg.projectIdentity}:`, error);
|
|
19630
|
+
noteFailure(`project ${reg.projectIdentity}`, error);
|
|
19631
|
+
}
|
|
19685
19632
|
}
|
|
19686
|
-
|
|
19687
|
-
|
|
19688
|
-
sourceReady(messageSource, sessionId).then((ready) => {
|
|
19689
|
-
if ((sessionEpochs.get(sessionId) ?? 0) !== epoch)
|
|
19690
|
-
return;
|
|
19691
|
-
if (ready)
|
|
19692
|
-
return runWithSessionLock(sessionId, indexMessage);
|
|
19693
|
-
setTimeout(() => scheduleIncrementalIndex(db, sessionId, messageId, messageSource), PREPARE_RETRY_MS).unref?.();
|
|
19694
|
-
}).catch((error) => {
|
|
19695
|
-
if ((sessionEpochs.get(sessionId) ?? 0) !== epoch)
|
|
19633
|
+
persistTickOutcome(db, failure);
|
|
19634
|
+
if (origin === "startup")
|
|
19696
19635
|
return;
|
|
19697
|
-
|
|
19698
|
-
|
|
19699
|
-
|
|
19700
|
-
|
|
19701
|
-
|
|
19702
|
-
if ((sessionEpochs.get(sessionId) ?? 0) === epoch)
|
|
19703
|
-
heapHolder.pendingIncrementalKeys.delete(schedulingKey);
|
|
19704
|
-
});
|
|
19705
|
-
}, INCREMENTAL_DEBOUNCE_MS));
|
|
19706
|
-
heapHolder.incrementalTimers.set(schedulingKey, timer);
|
|
19707
|
-
}
|
|
19708
|
-
async function indexSingleMessageOffThread(db, sessionId, message) {
|
|
19709
|
-
const path = getSqliteDatabasePath2(db);
|
|
19710
|
-
if (!path)
|
|
19711
|
-
return withSqliteBackgroundWriter2(() => indexSingleMessage2(db, sessionId, message));
|
|
19712
|
-
const cancellation = new Int32Array(new SharedArrayBuffer(4));
|
|
19713
|
-
const active = indexCancellations.get(sessionId) ?? new Set;
|
|
19714
|
-
active.add(cancellation);
|
|
19715
|
-
indexCancellations.set(sessionId, active);
|
|
19716
|
-
const result = await runSessionWarmupWorker({
|
|
19717
|
-
path,
|
|
19718
|
-
sessionId,
|
|
19719
|
-
indexMessage: message,
|
|
19720
|
-
harness: getHarness2(),
|
|
19721
|
-
indexCancellation: cancellation.buffer,
|
|
19722
|
-
...indexCoordinationInput2()
|
|
19723
|
-
}, {}).finally(() => {
|
|
19724
|
-
active.delete(cancellation);
|
|
19725
|
-
if (active.size === 0)
|
|
19726
|
-
indexCancellations.delete(sessionId);
|
|
19727
|
-
});
|
|
19728
|
-
if (!("inserted" in result))
|
|
19729
|
-
throw new Error("failure" in result ? result.detail : "unexpected indexing worker reply");
|
|
19730
|
-
if (result.measurements) {
|
|
19731
|
-
const sessions = workerMeasurements.get(db) ?? new Map;
|
|
19732
|
-
sessions.set(sessionId, result.measurements);
|
|
19733
|
-
workerMeasurements.set(db, sessions);
|
|
19734
|
-
if (result.measurements.skipped)
|
|
19735
|
-
log2(`[magic-context] incremental FTS skipped oversized source ${message.id} (${result.measurements.sourceBytes} bytes); identity and progress persisted`);
|
|
19736
|
-
}
|
|
19737
|
-
return result.inserted;
|
|
19738
|
-
}
|
|
19739
|
-
async function sweepOrphanedOpenCodeMessageIndexesOffThread(db, openSource, options = {}) {
|
|
19740
|
-
if (getHarness2() === "opencode" || getHarness2() === "opencode2")
|
|
19741
|
-
options = {
|
|
19742
|
-
...options,
|
|
19743
|
-
fenceLockNames: options.fenceLockNames ?? takeLkgRecoveryLockSweepPage()
|
|
19744
|
-
};
|
|
19745
|
-
if (getHarness2() === "opencode2")
|
|
19746
|
-
openSource = () => new Database2(gaDatabasePath(getDataDir(), process.env.OPENCODE_CHANNEL ?? "latest"), {
|
|
19747
|
-
readonly: true,
|
|
19748
|
-
fileMustExist: true
|
|
19749
|
-
});
|
|
19750
|
-
const path = getSqliteDatabasePath2(db);
|
|
19751
|
-
if (!path || getHarness2() !== "opencode" && getHarness2() !== "opencode2")
|
|
19752
|
-
return sweepOrphanedOpenCodeMessageIndexesCooperatively2(db, openSource, options);
|
|
19753
|
-
let source = null;
|
|
19754
|
-
let sourcePath = null;
|
|
19755
|
-
try {
|
|
19756
|
-
source = openSource();
|
|
19757
|
-
if (source)
|
|
19758
|
-
sourcePath = getSqliteDatabasePath2(source);
|
|
19759
|
-
} catch {} finally {
|
|
19760
|
-
if (sourcePath)
|
|
19761
|
-
source?.close();
|
|
19762
|
-
}
|
|
19763
|
-
if (!sourcePath)
|
|
19764
|
-
return sweepOrphanedOpenCodeMessageIndexesCooperatively2(db, () => source, options);
|
|
19765
|
-
const reply = await runSessionWarmupWorker({
|
|
19766
|
-
path,
|
|
19767
|
-
sessionId: "",
|
|
19768
|
-
harness: getHarness2(),
|
|
19769
|
-
orphanSweep: { sourcePath, options }
|
|
19770
|
-
}, {});
|
|
19771
|
-
if (!("orphanSweep" in reply))
|
|
19772
|
-
throw new Error("message history orphan sweep worker failed");
|
|
19773
|
-
return reply.orphanSweep;
|
|
19774
|
-
}
|
|
19775
|
-
function clearSessionTracking(sessionId) {
|
|
19776
|
-
sessionEpochs.set(sessionId, (sessionEpochs.get(sessionId) ?? 0) + 1);
|
|
19777
|
-
for (const cancellation of indexCancellations.get(sessionId) ?? [])
|
|
19778
|
-
Atomics.store(cancellation, 0, 1);
|
|
19779
|
-
heapHolder.clearAndReindexScheduledSessions.delete(sessionId);
|
|
19780
|
-
heapHolder.reconciledSessions.delete(sessionId);
|
|
19781
|
-
heapHolder.reconciledSnapshots.delete(sessionId);
|
|
19782
|
-
heapHolder.stalledSessions.delete(sessionId);
|
|
19783
|
-
heapHolder.reconciliationScheduledSessions.delete(sessionId);
|
|
19784
|
-
heapHolder.sessionLocks.delete(sessionId);
|
|
19785
|
-
heapHolder.activeReconcilerBuffers.delete(sessionId);
|
|
19786
|
-
const prefix = `${sessionId}\x00`;
|
|
19787
|
-
for (const [key, timer] of heapHolder.incrementalTimers) {
|
|
19788
|
-
if (key.startsWith(prefix)) {
|
|
19789
|
-
clearTimeout(timer);
|
|
19790
|
-
heapHolder.incrementalTimers.delete(key);
|
|
19791
|
-
}
|
|
19792
|
-
}
|
|
19793
|
-
for (const key of heapHolder.pendingIncrementalKeys) {
|
|
19794
|
-
if (key.startsWith(prefix)) {
|
|
19795
|
-
heapHolder.pendingIncrementalKeys.delete(key);
|
|
19636
|
+
runSqliteOptimize(db);
|
|
19637
|
+
} catch (error) {
|
|
19638
|
+
log2("[magic-context] timer-triggered maintenance check failed:", error);
|
|
19639
|
+
} finally {
|
|
19640
|
+
tickInFlight = false;
|
|
19796
19641
|
}
|
|
19797
|
-
}
|
|
19798
|
-
for (const key of heapHolder.completedIncrementalKeys) {
|
|
19799
|
-
if (key.startsWith(prefix))
|
|
19800
|
-
heapHolder.completedIncrementalKeys.delete(key);
|
|
19801
|
-
}
|
|
19642
|
+
})();
|
|
19802
19643
|
}
|
|
19803
|
-
|
|
19804
|
-
// ../plugin/src/plugin/dream-timer.ts
|
|
19805
|
-
var DREAM_TIMER_INTERVAL_MS = 15 * 60 * 1000;
|
|
19806
|
-
var GIT_COMMIT_BACKLOG_DRAIN_MAX_MS = 5 * 60 * 1000;
|
|
19807
|
-
var BOOT_PROJECT_JITTER_SLOT_MS = 1000;
|
|
19808
|
-
var activeTimer = null;
|
|
19809
|
-
var tickInFlight = false;
|
|
19810
|
-
var startupTickTimer = null;
|
|
19811
|
-
var startupTimers = new Map;
|
|
19812
|
-
var startupJitters = new Map;
|
|
19813
|
-
var nextStartupJitterSlot = 0;
|
|
19814
|
-
function directoryStillExists(directory) {
|
|
19644
|
+
function persistTickOutcome(db, failure) {
|
|
19815
19645
|
try {
|
|
19816
|
-
|
|
19817
|
-
|
|
19818
|
-
|
|
19646
|
+
if (failure)
|
|
19647
|
+
recordDreamerTickFailure(db, failure);
|
|
19648
|
+
else
|
|
19649
|
+
clearDreamerTickFailure(db);
|
|
19650
|
+
} catch (error) {
|
|
19651
|
+
log2("[dreamer] could not persist the outcome of this tick:", error);
|
|
19819
19652
|
}
|
|
19820
19653
|
}
|
|
19821
|
-
function
|
|
19822
|
-
|
|
19654
|
+
async function runMessageHistoryMaintenance(db) {
|
|
19655
|
+
const prunedIdentities = await pruneMissingProjectIdentities();
|
|
19656
|
+
if (prunedIdentities > 0)
|
|
19657
|
+
log2(`[magic-context] pruned ${prunedIdentities} missing project identities`);
|
|
19823
19658
|
try {
|
|
19824
|
-
|
|
19659
|
+
await drainStaleLkgSlots(db);
|
|
19825
19660
|
} catch (error) {
|
|
19826
|
-
log2(
|
|
19827
|
-
return null;
|
|
19661
|
+
log2("[magic-context] LKG pruning deferred:", error);
|
|
19828
19662
|
}
|
|
19829
|
-
|
|
19830
|
-
|
|
19831
|
-
|
|
19663
|
+
const cleanup = retryPendingSessionCleanups(db);
|
|
19664
|
+
if (cleanup.cleared > 0 || cleanup.failedSessionIds.length > 0) {
|
|
19665
|
+
log2(`[message-index] pending session cleanup: cleared=${cleanup.cleared} failed=${cleanup.failedSessionIds.length}`);
|
|
19666
|
+
}
|
|
19667
|
+
for (const registration of registeredProjects.values()) {
|
|
19668
|
+
const moduleClient = registration.sessionCleanupModuleClient;
|
|
19669
|
+
if (!moduleClient)
|
|
19670
|
+
continue;
|
|
19671
|
+
const rustCleanup = await retryPendingRustSessionCleanupsForProject(db, registration.projectIdentity, async (sessionId) => {
|
|
19672
|
+
try {
|
|
19673
|
+
await moduleClient.deleteSession(sessionId, registration.directory);
|
|
19674
|
+
} finally {
|
|
19675
|
+
moduleClient.closeSession?.(sessionId);
|
|
19676
|
+
}
|
|
19677
|
+
});
|
|
19678
|
+
if (rustCleanup.cleared > 0 || rustCleanup.failedSessionIds.length > 0) {
|
|
19679
|
+
log2(`[message-index] pending Rust session cleanup: cleared=${rustCleanup.cleared} failed=${rustCleanup.failedSessionIds.length}`);
|
|
19680
|
+
}
|
|
19832
19681
|
}
|
|
19833
|
-
return db;
|
|
19834
19682
|
}
|
|
19835
|
-
|
|
19836
|
-
|
|
19837
|
-
if (
|
|
19838
|
-
return;
|
|
19839
|
-
|
|
19840
|
-
|
|
19683
|
+
function startupJitterMs(directory) {
|
|
19684
|
+
const existing = startupJitters.get(directory);
|
|
19685
|
+
if (existing !== undefined)
|
|
19686
|
+
return existing;
|
|
19687
|
+
const slot = nextStartupJitterSlot;
|
|
19688
|
+
nextStartupJitterSlot += 1;
|
|
19689
|
+
const hash = [...directory].reduce((value, character) => value * 33 + character.charCodeAt(0) >>> 0, 5381);
|
|
19690
|
+
const jitter = slot * BOOT_PROJECT_JITTER_SLOT_MS + hash % BOOT_PROJECT_JITTER_SLOT_MS;
|
|
19691
|
+
startupJitters.set(directory, jitter);
|
|
19692
|
+
return jitter;
|
|
19841
19693
|
}
|
|
19842
|
-
|
|
19843
|
-
|
|
19844
|
-
if (registeredProjects.size !== 0)
|
|
19845
|
-
return;
|
|
19846
|
-
const stopped = activeTimer !== null || startupTickTimer !== null;
|
|
19847
|
-
if (activeTimer) {
|
|
19848
|
-
clearInterval(activeTimer);
|
|
19849
|
-
activeTimer = null;
|
|
19850
|
-
}
|
|
19851
|
-
if (startupTickTimer) {
|
|
19852
|
-
clearTimeout(startupTickTimer);
|
|
19853
|
-
startupTickTimer = null;
|
|
19854
|
-
}
|
|
19855
|
-
startupJitters.clear();
|
|
19856
|
-
nextStartupJitterSlot = 0;
|
|
19857
|
-
if (stopped)
|
|
19858
|
-
log2("[dreamer] stopped dream schedule timer (no projects left)");
|
|
19859
|
-
}
|
|
19860
|
-
async function startDreamScheduleTimer(args) {
|
|
19861
|
-
if (!isUsableProjectIdentity(args.projectIdentity)) {
|
|
19862
|
-
logEmptyIdentityRefusalOnce(args.directory);
|
|
19863
|
-
return;
|
|
19864
|
-
}
|
|
19865
|
-
beginBootQuietPeriod();
|
|
19866
|
-
const db = openTimerDatabaseOrNull("schedule timer registration");
|
|
19867
|
-
if (!db)
|
|
19868
|
-
return;
|
|
19869
|
-
const dreamingEnabled = Boolean(args.dreamerConfig && args.dreamerConfig.disable !== true);
|
|
19870
|
-
const embeddingSweepEnabled = args.memoryEnabled === true;
|
|
19871
|
-
const commitIndexingEnabled = args.gitCommitIndexing?.enabled === true;
|
|
19872
|
-
const previousRegistration = registeredProjects.get(args.directory);
|
|
19873
|
-
const isNewRegistration = previousRegistration === undefined;
|
|
19874
|
-
if (previousRegistration && previousRegistration !== args) {
|
|
19875
|
-
const pendingStartup = startupTimers.get(args.directory);
|
|
19876
|
-
if (pendingStartup) {
|
|
19877
|
-
clearTimeout(pendingStartup);
|
|
19878
|
-
startupTimers.delete(args.directory);
|
|
19879
|
-
}
|
|
19880
|
-
}
|
|
19881
|
-
registeredProjects.set(args.directory, args);
|
|
19882
|
-
if (isNewRegistration) {
|
|
19883
|
-
log2(`[dreamer] registered project ${args.projectIdentity} (dreaming=${dreamingEnabled} embeddings=${embeddingSweepEnabled} commits=${commitIndexingEnabled}; total=${registeredProjects.size})`);
|
|
19884
|
-
}
|
|
19885
|
-
if (!activeTimer) {
|
|
19886
|
-
log2(`[dreamer] started independent schedule timer (every ${DREAM_TIMER_INTERVAL_MS / 60000}m)`);
|
|
19887
|
-
startupTickTimer = scheduleAfterBootQuiet(() => {
|
|
19888
|
-
startupTickTimer = null;
|
|
19889
|
-
runTick("startup");
|
|
19890
|
-
});
|
|
19891
|
-
const timer = inRootAsyncContext2(() => setInterval(() => runTick("interval"), DREAM_TIMER_INTERVAL_MS));
|
|
19892
|
-
if (typeof timer === "object" && "unref" in timer) {
|
|
19893
|
-
timer.unref();
|
|
19894
|
-
}
|
|
19895
|
-
activeTimer = timer;
|
|
19896
|
-
} else if (isNewRegistration || previousRegistration !== args) {
|
|
19897
|
-
scheduleInitialProjectRun(args, db);
|
|
19898
|
-
}
|
|
19899
|
-
return () => {
|
|
19900
|
-
if (registeredProjects.get(args.directory) !== args) {
|
|
19901
|
-
stopDreamScheduleTimerIfIdle();
|
|
19902
|
-
return;
|
|
19903
|
-
}
|
|
19904
|
-
registeredProjects.delete(args.directory);
|
|
19905
|
-
const startupTimer = startupTimers.get(args.directory);
|
|
19906
|
-
if (startupTimer) {
|
|
19907
|
-
clearTimeout(startupTimer);
|
|
19908
|
-
startupTimers.delete(args.directory);
|
|
19909
|
-
}
|
|
19910
|
-
log2(`[dreamer] unregistered project ${args.projectIdentity} (remaining=${registeredProjects.size})`);
|
|
19911
|
-
stopDreamScheduleTimerIfIdle();
|
|
19912
|
-
};
|
|
19913
|
-
}
|
|
19914
|
-
var tickStages = {
|
|
19915
|
-
runMessageHistoryMaintenance,
|
|
19916
|
-
runProjectMaintenance
|
|
19917
|
-
};
|
|
19918
|
-
function runTick(origin) {
|
|
19919
|
-
if (tickInFlight)
|
|
19920
|
-
return;
|
|
19921
|
-
tickInFlight = true;
|
|
19922
|
-
log2(`[magic-context] maintenance tick (${origin}) — projects=${registeredProjects.size}`);
|
|
19923
|
-
(async () => {
|
|
19924
|
-
try {
|
|
19925
|
-
const db = openTimerDatabaseOrNull("maintenance tick");
|
|
19926
|
-
if (!db)
|
|
19927
|
-
return;
|
|
19928
|
-
let failure = null;
|
|
19929
|
-
const noteFailure = (stage, error) => {
|
|
19930
|
-
failure ??= { at: Date.now(), stage, message: getErrorMessage(error) };
|
|
19931
|
-
};
|
|
19932
|
-
try {
|
|
19933
|
-
await tickStages.runMessageHistoryMaintenance(db);
|
|
19934
|
-
} catch (error) {
|
|
19935
|
-
log2("[magic-context] timer-triggered message-history maintenance failed:", error);
|
|
19936
|
-
noteFailure("message-history maintenance", error);
|
|
19937
|
-
}
|
|
19938
|
-
for (const reg of registeredProjects.values()) {
|
|
19939
|
-
try {
|
|
19940
|
-
if (origin === "startup") {
|
|
19941
|
-
scheduleInitialProjectRun(reg, db);
|
|
19942
|
-
} else {
|
|
19943
|
-
await tickStages.runProjectMaintenance(reg, origin, db);
|
|
19944
|
-
}
|
|
19945
|
-
} catch (error) {
|
|
19946
|
-
log2(`[magic-context] timer-triggered maintenance failed for ${reg.projectIdentity}:`, error);
|
|
19947
|
-
noteFailure(`project ${reg.projectIdentity}`, error);
|
|
19948
|
-
}
|
|
19949
|
-
}
|
|
19950
|
-
persistTickOutcome(db, failure);
|
|
19951
|
-
if (origin === "startup")
|
|
19952
|
-
return;
|
|
19953
|
-
runSqliteOptimize(db);
|
|
19954
|
-
} catch (error) {
|
|
19955
|
-
log2("[magic-context] timer-triggered maintenance check failed:", error);
|
|
19956
|
-
} finally {
|
|
19957
|
-
tickInFlight = false;
|
|
19958
|
-
}
|
|
19959
|
-
})();
|
|
19960
|
-
}
|
|
19961
|
-
function persistTickOutcome(db, failure) {
|
|
19962
|
-
try {
|
|
19963
|
-
if (failure)
|
|
19964
|
-
recordDreamerTickFailure(db, failure);
|
|
19965
|
-
else
|
|
19966
|
-
clearDreamerTickFailure(db);
|
|
19967
|
-
} catch (error) {
|
|
19968
|
-
log2("[dreamer] could not persist the outcome of this tick:", error);
|
|
19969
|
-
}
|
|
19970
|
-
}
|
|
19971
|
-
async function runMessageHistoryMaintenance(db) {
|
|
19972
|
-
const prunedIdentities = await pruneMissingProjectIdentities();
|
|
19973
|
-
if (prunedIdentities > 0)
|
|
19974
|
-
log2(`[magic-context] pruned ${prunedIdentities} missing project identities`);
|
|
19975
|
-
try {
|
|
19976
|
-
await drainStaleLkgSlots(db);
|
|
19977
|
-
} catch (error) {
|
|
19978
|
-
log2("[magic-context] LKG pruning deferred:", error);
|
|
19979
|
-
}
|
|
19980
|
-
const cleanup = retryPendingSessionCleanups(db);
|
|
19981
|
-
if (cleanup.cleared > 0 || cleanup.failedSessionIds.length > 0) {
|
|
19982
|
-
log2(`[message-index] pending session cleanup: cleared=${cleanup.cleared} failed=${cleanup.failedSessionIds.length}`);
|
|
19983
|
-
}
|
|
19984
|
-
for (const registration of registeredProjects.values()) {
|
|
19985
|
-
const moduleClient = registration.sessionCleanupModuleClient;
|
|
19986
|
-
if (!moduleClient)
|
|
19987
|
-
continue;
|
|
19988
|
-
const rustCleanup = await retryPendingRustSessionCleanupsForProject(db, registration.projectIdentity, async (sessionId) => {
|
|
19989
|
-
try {
|
|
19990
|
-
await moduleClient.deleteSession(sessionId, registration.directory);
|
|
19991
|
-
} finally {
|
|
19992
|
-
moduleClient.closeSession?.(sessionId);
|
|
19993
|
-
}
|
|
19994
|
-
});
|
|
19995
|
-
if (rustCleanup.cleared > 0 || rustCleanup.failedSessionIds.length > 0) {
|
|
19996
|
-
log2(`[message-index] pending Rust session cleanup: cleared=${rustCleanup.cleared} failed=${rustCleanup.failedSessionIds.length}`);
|
|
19997
|
-
}
|
|
19998
|
-
}
|
|
19999
|
-
const sweep = await sweepOrphanedOpenCodeMessageIndexesOffThread(db, openOpenCodeDb);
|
|
20000
|
-
if (sweep.reason)
|
|
20001
|
-
log2(`[message-index] orphan sweep parked: ${sweep.reason}`);
|
|
20002
|
-
if (sweep.deleted > 0) {
|
|
20003
|
-
log2(`[message-index] orphan sweep: scanned=${sweep.scanned} deleted=${sweep.deleted} cursor=${sweep.cursor || "<complete>"}`);
|
|
20004
|
-
}
|
|
20005
|
-
}
|
|
20006
|
-
function startupJitterMs(directory) {
|
|
20007
|
-
const existing = startupJitters.get(directory);
|
|
20008
|
-
if (existing !== undefined)
|
|
20009
|
-
return existing;
|
|
20010
|
-
const slot = nextStartupJitterSlot;
|
|
20011
|
-
nextStartupJitterSlot += 1;
|
|
20012
|
-
const hash = [...directory].reduce((value, character) => value * 33 + character.charCodeAt(0) >>> 0, 5381);
|
|
20013
|
-
const jitter = slot * BOOT_PROJECT_JITTER_SLOT_MS + hash % BOOT_PROJECT_JITTER_SLOT_MS;
|
|
20014
|
-
startupJitters.set(directory, jitter);
|
|
20015
|
-
return jitter;
|
|
20016
|
-
}
|
|
20017
|
-
function scheduleInitialProjectRun(reg, db) {
|
|
20018
|
-
if (startupTimers.has(reg.directory))
|
|
19694
|
+
function scheduleInitialProjectRun(reg, db) {
|
|
19695
|
+
if (startupTimers.has(reg.directory))
|
|
20019
19696
|
return;
|
|
20020
19697
|
const timer = scheduleAfterBootQuiet(() => {
|
|
20021
19698
|
startupTimers.delete(reg.directory);
|
|
@@ -23796,144 +23473,428 @@ ${statusText}`,
|
|
|
23796
23473
|
level: "info",
|
|
23797
23474
|
rpcDisplay: "dialog"
|
|
23798
23475
|
});
|
|
23799
|
-
}
|
|
23476
|
+
}
|
|
23477
|
+
});
|
|
23478
|
+
}
|
|
23479
|
+
function maybeAutoEmbedPiSession(deps, sessionId, projectDir, projectIdentity) {
|
|
23480
|
+
if (!isUsableProjectIdentity(projectIdentity))
|
|
23481
|
+
return;
|
|
23482
|
+
maybeAutoEmbedSessionBackfill({
|
|
23483
|
+
db: deps.db,
|
|
23484
|
+
projectIdentity,
|
|
23485
|
+
ensureRegistered: () => ensureProjectRegisteredFromPiDirectory(projectDir, deps.db, {
|
|
23486
|
+
preflightLocalPooling: true
|
|
23487
|
+
})
|
|
23488
|
+
}, sessionId);
|
|
23489
|
+
}
|
|
23490
|
+
|
|
23491
|
+
// ../plugin/src/hooks/magic-context/execute-flush.ts
|
|
23492
|
+
function executeFlush(db, sessionId) {
|
|
23493
|
+
try {
|
|
23494
|
+
const pendingOps = getPendingOps(db, sessionId);
|
|
23495
|
+
if (pendingOps.length === 0) {
|
|
23496
|
+
return "No pending operations to flush.";
|
|
23497
|
+
}
|
|
23498
|
+
let dropped = 0;
|
|
23499
|
+
db.transaction(() => {
|
|
23500
|
+
for (const op of pendingOps) {
|
|
23501
|
+
updateTagStatus2(db, sessionId, op.tagId, "dropped");
|
|
23502
|
+
removePendingOp(db, sessionId, op.tagId);
|
|
23503
|
+
dropped++;
|
|
23504
|
+
}
|
|
23505
|
+
}).immediate();
|
|
23506
|
+
const parts = [];
|
|
23507
|
+
if (dropped > 0)
|
|
23508
|
+
parts.push(`${dropped} dropped`);
|
|
23509
|
+
return `Flushed: ${parts.join(", ")}. Changes take effect on next message.`;
|
|
23510
|
+
} catch (error) {
|
|
23511
|
+
sessionLog2(sessionId, "ctx-flush failed:", error);
|
|
23512
|
+
return `Error: Failed to flush context operations. ${getErrorMessage(error)}`;
|
|
23513
|
+
}
|
|
23514
|
+
}
|
|
23515
|
+
|
|
23516
|
+
// src/compaction-off-pi.ts
|
|
23517
|
+
var COMPACTION_OFF_COMMAND_UNAVAILABLE = `Unavailable: magic-context is in compaction-off mode (${COMPACTION_ENABLED_PATH}=false).`;
|
|
23518
|
+
var NO_TRANSITION = {
|
|
23519
|
+
recordToWrite: null,
|
|
23520
|
+
invalidatedBaseline: false,
|
|
23521
|
+
clearDeferredMarkerState: false,
|
|
23522
|
+
historianCatchUpSignaled: false,
|
|
23523
|
+
notice: null
|
|
23524
|
+
};
|
|
23525
|
+
var PI_COMPACTION_OFF_NOTICE = "Magic Context is in compaction-off mode. Native Pi compaction now owns this session's context window; the first turn after disabling may trigger one native compaction cycle on long sessions.";
|
|
23526
|
+
var PI_COMPACTION_ON_NOTICE = "Magic Context context-window management resumed. Run /ctx-wrapup to digest history accumulated while compaction was off.";
|
|
23527
|
+
function reconcilePiCompactionMode(args) {
|
|
23528
|
+
const stored = getCompactionModeRecord(args.db, args.sessionId);
|
|
23529
|
+
if (stored === "on_notice_pending") {
|
|
23530
|
+
return {
|
|
23531
|
+
...NO_TRANSITION,
|
|
23532
|
+
recordToWrite: "on",
|
|
23533
|
+
notice: PI_COMPACTION_ON_NOTICE
|
|
23534
|
+
};
|
|
23535
|
+
}
|
|
23536
|
+
if (stored === "off_notice_pending") {
|
|
23537
|
+
return {
|
|
23538
|
+
...NO_TRANSITION,
|
|
23539
|
+
recordToWrite: "off",
|
|
23540
|
+
notice: PI_COMPACTION_OFF_NOTICE
|
|
23541
|
+
};
|
|
23542
|
+
}
|
|
23543
|
+
if (!args.compactionOff) {
|
|
23544
|
+
if (stored === null || resolveCompactionModeRecord(stored) === "on") {
|
|
23545
|
+
return stored === null ? { ...NO_TRANSITION, recordToWrite: "on" } : NO_TRANSITION;
|
|
23546
|
+
}
|
|
23547
|
+
clearCachedM0M1(args.db, args.sessionId);
|
|
23548
|
+
if (!args.historianRunnable) {
|
|
23549
|
+
return {
|
|
23550
|
+
...NO_TRANSITION,
|
|
23551
|
+
recordToWrite: "on",
|
|
23552
|
+
invalidatedBaseline: true
|
|
23553
|
+
};
|
|
23554
|
+
}
|
|
23555
|
+
updateSessionMeta(args.db, args.sessionId, {
|
|
23556
|
+
compartmentInProgress: true
|
|
23557
|
+
});
|
|
23558
|
+
setCompactionModeRecord(args.db, args.sessionId, "on_notice_pending");
|
|
23559
|
+
return {
|
|
23560
|
+
recordToWrite: "on",
|
|
23561
|
+
invalidatedBaseline: true,
|
|
23562
|
+
clearDeferredMarkerState: false,
|
|
23563
|
+
historianCatchUpSignaled: true,
|
|
23564
|
+
notice: PI_COMPACTION_ON_NOTICE
|
|
23565
|
+
};
|
|
23566
|
+
}
|
|
23567
|
+
if (resolveCompactionModeRecord(stored) === "off")
|
|
23568
|
+
return NO_TRANSITION;
|
|
23569
|
+
let clearedSomething = false;
|
|
23570
|
+
if (getPendingPiCompactionMarkerState(args.db, args.sessionId) !== null) {
|
|
23571
|
+
setPendingPiCompactionMarkerState(args.db, args.sessionId, null, "compaction-disabled");
|
|
23572
|
+
clearedSomething = true;
|
|
23573
|
+
}
|
|
23574
|
+
if (getOverflowState(args.db, args.sessionId).needsEmergencyRecovery) {
|
|
23575
|
+
clearEmergencyRecovery(args.db, args.sessionId);
|
|
23576
|
+
clearedSomething = true;
|
|
23577
|
+
}
|
|
23578
|
+
const channel2 = getChannel2NudgeState(args.db, args.sessionId);
|
|
23579
|
+
if (channel2 === "pending" || channel2 === "claimed") {
|
|
23580
|
+
setChannel2NudgeState(args.db, args.sessionId, "");
|
|
23581
|
+
clearedSomething = true;
|
|
23582
|
+
}
|
|
23583
|
+
if (getPendingOps(args.db, args.sessionId).length > 0) {
|
|
23584
|
+
clearPendingOps(args.db, args.sessionId);
|
|
23585
|
+
clearedSomething = true;
|
|
23586
|
+
}
|
|
23587
|
+
const meta = getOrCreateSessionMeta(args.db, args.sessionId);
|
|
23588
|
+
if (meta.compartmentInProgress) {
|
|
23589
|
+
updateSessionMeta(args.db, args.sessionId, {
|
|
23590
|
+
compartmentInProgress: false
|
|
23591
|
+
});
|
|
23592
|
+
}
|
|
23593
|
+
clearCachedM0M1(args.db, args.sessionId);
|
|
23594
|
+
sessionLog2(args.sessionId, `Pi compaction-off transition: clearedPendingMarker=${clearedSomething}`);
|
|
23595
|
+
const notice = clearedSomething ? PI_COMPACTION_OFF_NOTICE : null;
|
|
23596
|
+
if (notice) {
|
|
23597
|
+
setCompactionModeRecord(args.db, args.sessionId, "off_notice_pending");
|
|
23598
|
+
}
|
|
23599
|
+
return {
|
|
23600
|
+
recordToWrite: "off",
|
|
23601
|
+
invalidatedBaseline: true,
|
|
23602
|
+
clearDeferredMarkerState: true,
|
|
23603
|
+
historianCatchUpSignaled: false,
|
|
23604
|
+
notice
|
|
23605
|
+
};
|
|
23606
|
+
}
|
|
23607
|
+
function commitPiCompactionModeRecord(db, sessionId, record) {
|
|
23608
|
+
setCompactionModeRecord(db, sessionId, record);
|
|
23609
|
+
}
|
|
23610
|
+
|
|
23611
|
+
// src/context-handler.ts
|
|
23612
|
+
import * as crypto3 from "node:crypto";
|
|
23613
|
+
|
|
23614
|
+
// ../plugin/src/features/magic-context/message-index-async.ts
|
|
23615
|
+
var indexCancellations = new Map;
|
|
23616
|
+
var sessionEpochs = new Map;
|
|
23617
|
+
var workerMeasurements = new WeakMap;
|
|
23618
|
+
function isDatabaseLockedError(error) {
|
|
23619
|
+
if (!error || typeof error !== "object")
|
|
23620
|
+
return false;
|
|
23621
|
+
const e = error;
|
|
23622
|
+
if (typeof e.code === "string") {
|
|
23623
|
+
if (e.code === "SQLITE_BUSY" || e.code === "SQLITE_LOCKED")
|
|
23624
|
+
return true;
|
|
23625
|
+
}
|
|
23626
|
+
if (typeof e.message === "string") {
|
|
23627
|
+
if (/database is locked/i.test(e.message))
|
|
23628
|
+
return true;
|
|
23629
|
+
if (/sqlite_(busy|locked)/i.test(e.message))
|
|
23630
|
+
return true;
|
|
23631
|
+
}
|
|
23632
|
+
return false;
|
|
23633
|
+
}
|
|
23634
|
+
var INCREMENTAL_DEBOUNCE_MS = 100;
|
|
23635
|
+
var RECONCILIATION_BATCH_SIZE = 100;
|
|
23636
|
+
|
|
23637
|
+
class MagicContextMessageIndexHeapHolder {
|
|
23638
|
+
reconciledSessions = new Set;
|
|
23639
|
+
reconciledSnapshots = new Map;
|
|
23640
|
+
stalledSessions = new Set;
|
|
23641
|
+
reconciliationScheduledSessions = new Set;
|
|
23642
|
+
clearAndReindexScheduledSessions = new Map;
|
|
23643
|
+
sessionLocks = new Map;
|
|
23644
|
+
incrementalTimers = new Map;
|
|
23645
|
+
pendingIncrementalKeys = new Set;
|
|
23646
|
+
completedIncrementalKeys = new Set;
|
|
23647
|
+
activeReconcilerBuffers = new Map;
|
|
23648
|
+
}
|
|
23649
|
+
var heapHolder = new MagicContextMessageIndexHeapHolder;
|
|
23650
|
+
var PREPARE_RETRY_MS = 60000;
|
|
23651
|
+
async function sourceReady(source, sessionId) {
|
|
23652
|
+
const prepare = source?.prepare;
|
|
23653
|
+
return typeof prepare === "function" ? prepare(sessionId) : true;
|
|
23654
|
+
}
|
|
23655
|
+
function defer(fn) {
|
|
23656
|
+
const immediate = globalThis.setImmediate;
|
|
23657
|
+
if (typeof immediate === "function") {
|
|
23658
|
+
inRootAsyncContext2(() => immediate(fn));
|
|
23659
|
+
return;
|
|
23660
|
+
}
|
|
23661
|
+
inRootAsyncContext2(() => setTimeout(fn, 0));
|
|
23662
|
+
}
|
|
23663
|
+
function yieldToEventLoop() {
|
|
23664
|
+
return new Promise((resolve) => defer(resolve));
|
|
23665
|
+
}
|
|
23666
|
+
function runWithSessionLock(sessionId, operation) {
|
|
23667
|
+
const previous = heapHolder.sessionLocks.get(sessionId) ?? Promise.resolve();
|
|
23668
|
+
const run = previous.catch(() => {
|
|
23669
|
+
return;
|
|
23670
|
+
}).then(async () => {
|
|
23671
|
+
await operation();
|
|
23672
|
+
});
|
|
23673
|
+
heapHolder.sessionLocks.set(sessionId, run);
|
|
23674
|
+
run.finally(() => {
|
|
23675
|
+
if (heapHolder.sessionLocks.get(sessionId) === run) {
|
|
23676
|
+
heapHolder.sessionLocks.delete(sessionId);
|
|
23677
|
+
}
|
|
23678
|
+
}).catch(() => {
|
|
23679
|
+
return;
|
|
23680
|
+
});
|
|
23681
|
+
return run;
|
|
23682
|
+
}
|
|
23683
|
+
function logIndexingError(sessionId, action, error) {
|
|
23684
|
+
if (isDatabaseLockedError(error)) {
|
|
23685
|
+
sessionLog2(sessionId, `message FTS async ${action} skipped (database busy; will retry on next reconciliation)`);
|
|
23686
|
+
return;
|
|
23687
|
+
}
|
|
23688
|
+
sessionLog2(sessionId, `message FTS async ${action} failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
23689
|
+
log2(`[message-index-async] ${action} failed for ${sessionId}:`, error);
|
|
23690
|
+
}
|
|
23691
|
+
function logReconciliationStall(sessionId, reason) {
|
|
23692
|
+
if (heapHolder.stalledSessions.has(sessionId))
|
|
23693
|
+
return;
|
|
23694
|
+
heapHolder.stalledSessions.add(sessionId);
|
|
23695
|
+
sessionLog2(sessionId, `message FTS reconciliation stalled: ${reason}; will retry`);
|
|
23696
|
+
}
|
|
23697
|
+
async function reconcileSessionIndex(db, sessionId, readMessages) {
|
|
23698
|
+
if (!await sourceReady(readMessages, sessionId)) {
|
|
23699
|
+
logReconciliationStall(sessionId, "raw message reader is not ready");
|
|
23700
|
+
return false;
|
|
23701
|
+
}
|
|
23702
|
+
await runWithSessionLock(sessionId, async () => {
|
|
23703
|
+
if (!readMessages.getRevision && heapHolder.reconciledSessions.has(sessionId))
|
|
23704
|
+
return;
|
|
23705
|
+
let fallbackSnapshot = null;
|
|
23706
|
+
try {
|
|
23707
|
+
const fullReader = typeof readMessages === "function" ? readMessages : null;
|
|
23708
|
+
const revision = readMessages.getRevision?.(sessionId);
|
|
23709
|
+
const previous = heapHolder.reconciledSnapshots.get(sessionId);
|
|
23710
|
+
const finalWatermark = previous && revision === previous.revision ? previous.count : revision !== undefined && readMessages.getCountAfterRevision ? previous && revision > previous.revision ? previous.count + readMessages.getCountAfterRevision(sessionId, previous.revision, revision) : readMessages.getCountAfterRevision(sessionId, -1, revision) : readMessages.getCount ? readMessages.getCount(sessionId) : (fallbackSnapshot = fullReader?.(sessionId) ?? []).length;
|
|
23711
|
+
let cursor = getMessageIndexReconciliationStartOrdinal(db, sessionId);
|
|
23712
|
+
let after = cursor === previous?.count ? previous.after : undefined;
|
|
23713
|
+
while (cursor < finalWatermark) {
|
|
23714
|
+
const pageEnd = Math.min(finalWatermark, cursor + RECONCILIATION_BATCH_SIZE);
|
|
23715
|
+
const messages = readMessages.readPage ? readMessages.readPage(sessionId, cursor, RECONCILIATION_BATCH_SIZE, finalWatermark, after) : (fallbackSnapshot ?? []).filter((message) => message.ordinal > cursor && message.ordinal <= pageEnd);
|
|
23716
|
+
const retainedMessages = fallbackSnapshot ?? messages;
|
|
23717
|
+
heapHolder.activeReconcilerBuffers.set(sessionId, retainedMessages);
|
|
23718
|
+
withSqliteBackgroundWriter2(() => indexMessagesAfterOrdinal(db, sessionId, messages, cursor, pageEnd));
|
|
23719
|
+
const nextCursor = getMessageIndexReconciliationStartOrdinal(db, sessionId);
|
|
23720
|
+
if (nextCursor <= cursor)
|
|
23721
|
+
break;
|
|
23722
|
+
cursor = nextCursor;
|
|
23723
|
+
const last = messages.at(-1);
|
|
23724
|
+
after = last?.ordinal === cursor && typeof last.createdAt === "number" ? { timeCreated: last.createdAt, id: last.id } : undefined;
|
|
23725
|
+
if (cursor < finalWatermark) {
|
|
23726
|
+
await yieldToEventLoop();
|
|
23727
|
+
}
|
|
23728
|
+
}
|
|
23729
|
+
if (isMessageIndexReconciledThrough(db, sessionId, finalWatermark)) {
|
|
23730
|
+
heapHolder.reconciledSessions.add(sessionId);
|
|
23731
|
+
if (revision !== undefined) {
|
|
23732
|
+
if (!after && finalWatermark > 0 && readMessages.readPage) {
|
|
23733
|
+
const last = readMessages.readPage(sessionId, finalWatermark - 1, 1, finalWatermark).at(-1);
|
|
23734
|
+
if (last?.ordinal === finalWatermark && typeof last.createdAt === "number")
|
|
23735
|
+
after = { timeCreated: last.createdAt, id: last.id };
|
|
23736
|
+
}
|
|
23737
|
+
heapHolder.reconciledSnapshots.set(sessionId, {
|
|
23738
|
+
revision,
|
|
23739
|
+
count: finalWatermark,
|
|
23740
|
+
after
|
|
23741
|
+
});
|
|
23742
|
+
}
|
|
23743
|
+
} else {
|
|
23744
|
+
logReconciliationStall(sessionId, `gap after ordinal ${cursor} (store count ${finalWatermark})`);
|
|
23745
|
+
}
|
|
23746
|
+
} finally {
|
|
23747
|
+
heapHolder.activeReconcilerBuffers.delete(sessionId);
|
|
23748
|
+
}
|
|
23749
|
+
});
|
|
23750
|
+
return true;
|
|
23751
|
+
}
|
|
23752
|
+
function scheduleReconciliation(db, sessionId, readMessages) {
|
|
23753
|
+
if (!readMessages.getRevision && heapHolder.reconciledSessions.has(sessionId) || heapHolder.reconciliationScheduledSessions.has(sessionId)) {
|
|
23754
|
+
return;
|
|
23755
|
+
}
|
|
23756
|
+
heapHolder.reconciliationScheduledSessions.add(sessionId);
|
|
23757
|
+
scheduleAfterBootQuiet(() => {
|
|
23758
|
+
defer(() => {
|
|
23759
|
+
let appendedDuringReconciliation = false;
|
|
23760
|
+
const previousSnapshot = heapHolder.reconciledSnapshots.get(sessionId);
|
|
23761
|
+
reconcileSessionIndex(db, sessionId, readMessages).then((ran) => {
|
|
23762
|
+
if (ran) {
|
|
23763
|
+
const snapshot = heapHolder.reconciledSnapshots.get(sessionId);
|
|
23764
|
+
appendedDuringReconciliation = snapshot !== undefined && snapshot !== previousSnapshot && readMessages.getRevision !== undefined && readMessages.getRevision(sessionId) !== snapshot.revision;
|
|
23765
|
+
return;
|
|
23766
|
+
}
|
|
23767
|
+
inRootAsyncContext2(() => setTimeout(() => {
|
|
23768
|
+
heapHolder.reconciliationScheduledSessions.delete(sessionId);
|
|
23769
|
+
scheduleReconciliation(db, sessionId, readMessages);
|
|
23770
|
+
}, PREPARE_RETRY_MS)).unref?.();
|
|
23771
|
+
}).catch((error) => {
|
|
23772
|
+
logReconciliationStall(sessionId, error instanceof Error ? error.message : String(error));
|
|
23773
|
+
if (isDatabaseLockedError(error))
|
|
23774
|
+
inRootAsyncContext2(() => setTimeout(() => scheduleReconciliation(db, sessionId, readMessages), 1000)).unref();
|
|
23775
|
+
}).finally(() => {
|
|
23776
|
+
heapHolder.reconciliationScheduledSessions.delete(sessionId);
|
|
23777
|
+
if (appendedDuringReconciliation)
|
|
23778
|
+
scheduleReconciliation(db, sessionId, readMessages);
|
|
23779
|
+
});
|
|
23780
|
+
});
|
|
23800
23781
|
});
|
|
23801
23782
|
}
|
|
23802
|
-
function
|
|
23803
|
-
|
|
23783
|
+
function scheduleIncrementalIndex(db, sessionId, messageId, messageSource) {
|
|
23784
|
+
const schedulingKey = `${sessionId}\x00${messageId}`;
|
|
23785
|
+
const epoch = sessionEpochs.get(sessionId) ?? 0;
|
|
23786
|
+
if (heapHolder.pendingIncrementalKeys.has(schedulingKey))
|
|
23804
23787
|
return;
|
|
23805
|
-
|
|
23806
|
-
|
|
23807
|
-
|
|
23808
|
-
|
|
23809
|
-
|
|
23810
|
-
|
|
23811
|
-
|
|
23812
|
-
|
|
23813
|
-
|
|
23814
|
-
|
|
23815
|
-
|
|
23816
|
-
|
|
23817
|
-
|
|
23818
|
-
|
|
23819
|
-
|
|
23820
|
-
|
|
23821
|
-
|
|
23822
|
-
|
|
23823
|
-
|
|
23824
|
-
updateTagStatus2(db, sessionId, op.tagId, "dropped");
|
|
23825
|
-
removePendingOp(db, sessionId, op.tagId);
|
|
23826
|
-
dropped++;
|
|
23788
|
+
const activeTimer = heapHolder.incrementalTimers.get(schedulingKey);
|
|
23789
|
+
if (activeTimer)
|
|
23790
|
+
clearTimeout(activeTimer);
|
|
23791
|
+
const timer = inRootAsyncContext2(() => setTimeout(() => {
|
|
23792
|
+
heapHolder.incrementalTimers.delete(schedulingKey);
|
|
23793
|
+
heapHolder.pendingIncrementalKeys.add(schedulingKey);
|
|
23794
|
+
const indexMessage = async () => {
|
|
23795
|
+
if ((sessionEpochs.get(sessionId) ?? 0) !== epoch || !heapHolder.pendingIncrementalKeys.has(schedulingKey))
|
|
23796
|
+
return;
|
|
23797
|
+
const message = typeof messageSource === "function" ? messageSource(sessionId, messageId) : messageSource;
|
|
23798
|
+
if (!message)
|
|
23799
|
+
return;
|
|
23800
|
+
const revisionKey = `${schedulingKey}\x00${getMessageIndexSourceIdentity(message)}`;
|
|
23801
|
+
if (heapHolder.completedIncrementalKeys.has(revisionKey))
|
|
23802
|
+
return;
|
|
23803
|
+
const currentWatermark = getLastIndexedOrdinal(db, sessionId);
|
|
23804
|
+
if (message.ordinal <= currentWatermark && isMessageIndexSourceCurrent(db, sessionId, message)) {
|
|
23805
|
+
heapHolder.completedIncrementalKeys.add(revisionKey);
|
|
23806
|
+
return;
|
|
23827
23807
|
}
|
|
23828
|
-
|
|
23829
|
-
|
|
23830
|
-
|
|
23831
|
-
|
|
23832
|
-
|
|
23833
|
-
|
|
23834
|
-
|
|
23835
|
-
|
|
23836
|
-
|
|
23837
|
-
}
|
|
23838
|
-
|
|
23839
|
-
// src/compaction-off-pi.ts
|
|
23840
|
-
var COMPACTION_OFF_COMMAND_UNAVAILABLE = `Unavailable: magic-context is in compaction-off mode (${COMPACTION_ENABLED_PATH}=false).`;
|
|
23841
|
-
var NO_TRANSITION = {
|
|
23842
|
-
recordToWrite: null,
|
|
23843
|
-
invalidatedBaseline: false,
|
|
23844
|
-
clearDeferredMarkerState: false,
|
|
23845
|
-
historianCatchUpSignaled: false,
|
|
23846
|
-
notice: null
|
|
23847
|
-
};
|
|
23848
|
-
var PI_COMPACTION_OFF_NOTICE = "Magic Context is in compaction-off mode. Native Pi compaction now owns this session's context window; the first turn after disabling may trigger one native compaction cycle on long sessions.";
|
|
23849
|
-
var PI_COMPACTION_ON_NOTICE = "Magic Context context-window management resumed. Run /ctx-wrapup to digest history accumulated while compaction was off.";
|
|
23850
|
-
function reconcilePiCompactionMode(args) {
|
|
23851
|
-
const stored = getCompactionModeRecord(args.db, args.sessionId);
|
|
23852
|
-
if (stored === "on_notice_pending") {
|
|
23853
|
-
return {
|
|
23854
|
-
...NO_TRANSITION,
|
|
23855
|
-
recordToWrite: "on",
|
|
23856
|
-
notice: PI_COMPACTION_ON_NOTICE
|
|
23857
|
-
};
|
|
23858
|
-
}
|
|
23859
|
-
if (stored === "off_notice_pending") {
|
|
23860
|
-
return {
|
|
23861
|
-
...NO_TRANSITION,
|
|
23862
|
-
recordToWrite: "off",
|
|
23863
|
-
notice: PI_COMPACTION_OFF_NOTICE
|
|
23808
|
+
const wasReconciled = heapHolder.reconciledSessions.delete(sessionId);
|
|
23809
|
+
await indexSingleMessageOffThread(db, sessionId, message);
|
|
23810
|
+
if ((sessionEpochs.get(sessionId) ?? 0) !== epoch || !heapHolder.pendingIncrementalKeys.has(schedulingKey))
|
|
23811
|
+
return;
|
|
23812
|
+
const finalWatermark = getLastIndexedOrdinal(db, sessionId);
|
|
23813
|
+
if (wasReconciled && isMessageIndexReconciledThrough(db, sessionId, finalWatermark)) {
|
|
23814
|
+
heapHolder.reconciledSessions.add(sessionId);
|
|
23815
|
+
}
|
|
23816
|
+
heapHolder.completedIncrementalKeys.add(revisionKey);
|
|
23864
23817
|
};
|
|
23865
|
-
|
|
23866
|
-
|
|
23867
|
-
|
|
23868
|
-
|
|
23869
|
-
|
|
23870
|
-
|
|
23871
|
-
|
|
23872
|
-
|
|
23873
|
-
|
|
23874
|
-
|
|
23875
|
-
|
|
23876
|
-
|
|
23877
|
-
|
|
23878
|
-
|
|
23879
|
-
|
|
23818
|
+
sourceReady(messageSource, sessionId).then((ready) => {
|
|
23819
|
+
if ((sessionEpochs.get(sessionId) ?? 0) !== epoch)
|
|
23820
|
+
return;
|
|
23821
|
+
if (ready)
|
|
23822
|
+
return runWithSessionLock(sessionId, indexMessage);
|
|
23823
|
+
setTimeout(() => scheduleIncrementalIndex(db, sessionId, messageId, messageSource), PREPARE_RETRY_MS).unref?.();
|
|
23824
|
+
}).catch((error) => {
|
|
23825
|
+
if ((sessionEpochs.get(sessionId) ?? 0) !== epoch)
|
|
23826
|
+
return;
|
|
23827
|
+
heapHolder.reconciledSessions.delete(sessionId);
|
|
23828
|
+
logIndexingError(sessionId, `incremental index for ${messageId}`, error);
|
|
23829
|
+
if (isDatabaseLockedError(error))
|
|
23830
|
+
setTimeout(() => scheduleIncrementalIndex(db, sessionId, messageId, messageSource), 1000).unref();
|
|
23831
|
+
}).finally(() => {
|
|
23832
|
+
if ((sessionEpochs.get(sessionId) ?? 0) === epoch)
|
|
23833
|
+
heapHolder.pendingIncrementalKeys.delete(schedulingKey);
|
|
23880
23834
|
});
|
|
23881
|
-
|
|
23882
|
-
|
|
23883
|
-
|
|
23884
|
-
|
|
23885
|
-
|
|
23886
|
-
|
|
23887
|
-
|
|
23888
|
-
|
|
23889
|
-
|
|
23890
|
-
|
|
23891
|
-
|
|
23892
|
-
|
|
23893
|
-
|
|
23894
|
-
|
|
23895
|
-
|
|
23896
|
-
|
|
23897
|
-
|
|
23898
|
-
|
|
23899
|
-
|
|
23900
|
-
|
|
23901
|
-
|
|
23902
|
-
|
|
23903
|
-
|
|
23904
|
-
|
|
23835
|
+
}, INCREMENTAL_DEBOUNCE_MS));
|
|
23836
|
+
heapHolder.incrementalTimers.set(schedulingKey, timer);
|
|
23837
|
+
}
|
|
23838
|
+
async function indexSingleMessageOffThread(db, sessionId, message) {
|
|
23839
|
+
const path = getSqliteDatabasePath2(db);
|
|
23840
|
+
if (!path)
|
|
23841
|
+
return withSqliteBackgroundWriter2(() => indexSingleMessage(db, sessionId, message));
|
|
23842
|
+
const cancellation = new Int32Array(new SharedArrayBuffer(4));
|
|
23843
|
+
const active = indexCancellations.get(sessionId) ?? new Set;
|
|
23844
|
+
active.add(cancellation);
|
|
23845
|
+
indexCancellations.set(sessionId, active);
|
|
23846
|
+
const result = await runSessionWarmupWorker({
|
|
23847
|
+
path,
|
|
23848
|
+
sessionId,
|
|
23849
|
+
indexMessage: message,
|
|
23850
|
+
harness: getHarness2(),
|
|
23851
|
+
indexCancellation: cancellation.buffer,
|
|
23852
|
+
...indexCoordinationInput2()
|
|
23853
|
+
}, {}).finally(() => {
|
|
23854
|
+
active.delete(cancellation);
|
|
23855
|
+
if (active.size === 0)
|
|
23856
|
+
indexCancellations.delete(sessionId);
|
|
23857
|
+
});
|
|
23858
|
+
if (!("inserted" in result))
|
|
23859
|
+
throw new Error("failure" in result ? result.detail : "unexpected indexing worker reply");
|
|
23860
|
+
if (result.measurements) {
|
|
23861
|
+
const sessions = workerMeasurements.get(db) ?? new Map;
|
|
23862
|
+
sessions.set(sessionId, result.measurements);
|
|
23863
|
+
workerMeasurements.set(db, sessions);
|
|
23864
|
+
if (result.measurements.skipped)
|
|
23865
|
+
log2(`[magic-context] incremental FTS skipped oversized source ${message.id} (${result.measurements.sourceBytes} bytes); identity and progress persisted`);
|
|
23905
23866
|
}
|
|
23906
|
-
|
|
23907
|
-
|
|
23908
|
-
|
|
23867
|
+
return result.inserted;
|
|
23868
|
+
}
|
|
23869
|
+
function clearSessionTracking(sessionId) {
|
|
23870
|
+
sessionEpochs.set(sessionId, (sessionEpochs.get(sessionId) ?? 0) + 1);
|
|
23871
|
+
for (const cancellation of indexCancellations.get(sessionId) ?? [])
|
|
23872
|
+
Atomics.store(cancellation, 0, 1);
|
|
23873
|
+
heapHolder.clearAndReindexScheduledSessions.delete(sessionId);
|
|
23874
|
+
heapHolder.reconciledSessions.delete(sessionId);
|
|
23875
|
+
heapHolder.reconciledSnapshots.delete(sessionId);
|
|
23876
|
+
heapHolder.stalledSessions.delete(sessionId);
|
|
23877
|
+
heapHolder.reconciliationScheduledSessions.delete(sessionId);
|
|
23878
|
+
heapHolder.sessionLocks.delete(sessionId);
|
|
23879
|
+
heapHolder.activeReconcilerBuffers.delete(sessionId);
|
|
23880
|
+
const prefix = `${sessionId}\x00`;
|
|
23881
|
+
for (const [key, timer] of heapHolder.incrementalTimers) {
|
|
23882
|
+
if (key.startsWith(prefix)) {
|
|
23883
|
+
clearTimeout(timer);
|
|
23884
|
+
heapHolder.incrementalTimers.delete(key);
|
|
23885
|
+
}
|
|
23909
23886
|
}
|
|
23910
|
-
const
|
|
23911
|
-
|
|
23912
|
-
|
|
23913
|
-
|
|
23914
|
-
});
|
|
23887
|
+
for (const key of heapHolder.pendingIncrementalKeys) {
|
|
23888
|
+
if (key.startsWith(prefix)) {
|
|
23889
|
+
heapHolder.pendingIncrementalKeys.delete(key);
|
|
23890
|
+
}
|
|
23915
23891
|
}
|
|
23916
|
-
|
|
23917
|
-
|
|
23918
|
-
|
|
23919
|
-
if (notice) {
|
|
23920
|
-
setCompactionModeRecord(args.db, args.sessionId, "off_notice_pending");
|
|
23892
|
+
for (const key of heapHolder.completedIncrementalKeys) {
|
|
23893
|
+
if (key.startsWith(prefix))
|
|
23894
|
+
heapHolder.completedIncrementalKeys.delete(key);
|
|
23921
23895
|
}
|
|
23922
|
-
return {
|
|
23923
|
-
recordToWrite: "off",
|
|
23924
|
-
invalidatedBaseline: true,
|
|
23925
|
-
clearDeferredMarkerState: true,
|
|
23926
|
-
historianCatchUpSignaled: false,
|
|
23927
|
-
notice
|
|
23928
|
-
};
|
|
23929
|
-
}
|
|
23930
|
-
function commitPiCompactionModeRecord(db, sessionId, record) {
|
|
23931
|
-
setCompactionModeRecord(db, sessionId, record);
|
|
23932
23896
|
}
|
|
23933
23897
|
|
|
23934
|
-
// src/context-handler.ts
|
|
23935
|
-
import * as crypto3 from "node:crypto";
|
|
23936
|
-
|
|
23937
23898
|
// ../plugin/src/hooks/magic-context/event-resolvers.ts
|
|
23938
23899
|
var DEFAULT_CONTEXT_LIMIT = 200000;
|
|
23939
23900
|
var clampWarnSeen = new Set;
|
|
@@ -26380,6 +26341,60 @@ function isHistorianDrainBudgetSpent(args) {
|
|
|
26380
26341
|
return true;
|
|
26381
26342
|
}
|
|
26382
26343
|
|
|
26344
|
+
// ../plugin/src/hooks/magic-context/historian-history-wait.ts
|
|
26345
|
+
var historyWaits = new Map;
|
|
26346
|
+
function newestMessageId(snapshot) {
|
|
26347
|
+
try {
|
|
26348
|
+
const count = getRawSessionMessageOrdinalCount2(snapshot.sessionId);
|
|
26349
|
+
const ids = readRawSessionMessageIdOrdinalsForRange2(snapshot.sessionId, count, count);
|
|
26350
|
+
const newest = ids.keys().next().value;
|
|
26351
|
+
if (newest)
|
|
26352
|
+
return newest;
|
|
26353
|
+
} catch {}
|
|
26354
|
+
return snapshot.rawLastMessageIdAtTrigger;
|
|
26355
|
+
}
|
|
26356
|
+
function recoveryWindow(snapshot, percentage) {
|
|
26357
|
+
return percentage >= escalationBands(snapshot.executeThresholdPercentage).forceMaterializationPercentage || snapshot.emergencyTailScale !== undefined;
|
|
26358
|
+
}
|
|
26359
|
+
function clearHistorianHistoryWait(sessionId) {
|
|
26360
|
+
historyWaits.delete(sessionId);
|
|
26361
|
+
}
|
|
26362
|
+
function recordHistorianNoProgress(snapshot, emergency = false) {
|
|
26363
|
+
const newest = newestMessageId(snapshot);
|
|
26364
|
+
if (!newest)
|
|
26365
|
+
return;
|
|
26366
|
+
const previous = historyWaits.get(snapshot.sessionId);
|
|
26367
|
+
const sameHistory = previous?.newestMessageId === newest;
|
|
26368
|
+
historyWaits.set(snapshot.sessionId, {
|
|
26369
|
+
newestMessageId: newest,
|
|
26370
|
+
recoveryAttempted: sameHistory && previous.recoveryAttempted || emergency || recoveryWindow(snapshot, snapshot.usagePercentage),
|
|
26371
|
+
floorLiftAttempted: sameHistory && previous.floorLiftAttempted || snapshot.liftLivePromptFloor === true
|
|
26372
|
+
});
|
|
26373
|
+
}
|
|
26374
|
+
function shouldWaitForNewHistorianHistory(snapshot, options = {}) {
|
|
26375
|
+
const wait = historyWaits.get(snapshot.sessionId);
|
|
26376
|
+
if (!wait)
|
|
26377
|
+
return false;
|
|
26378
|
+
const newest = newestMessageId(snapshot);
|
|
26379
|
+
if (newest && newest !== wait.newestMessageId) {
|
|
26380
|
+
try {
|
|
26381
|
+
const order = compareRawSessionMessageOrder2(snapshot.sessionId, newest, wait.newestMessageId);
|
|
26382
|
+
if (order === null || order > 0)
|
|
26383
|
+
return false;
|
|
26384
|
+
} catch {}
|
|
26385
|
+
}
|
|
26386
|
+
const recovery = options.emergency || recoveryWindow(snapshot, options.usagePercentage ?? snapshot.usagePercentage);
|
|
26387
|
+
const floorLift = snapshot.liftLivePromptFloor === true;
|
|
26388
|
+
if (recovery && !wait.recoveryAttempted || floorLift && !wait.floorLiftAttempted) {
|
|
26389
|
+
if (options.claim) {
|
|
26390
|
+
wait.recoveryAttempted ||= recovery;
|
|
26391
|
+
wait.floorLiftAttempted ||= floorLift;
|
|
26392
|
+
}
|
|
26393
|
+
return false;
|
|
26394
|
+
}
|
|
26395
|
+
return true;
|
|
26396
|
+
}
|
|
26397
|
+
|
|
26383
26398
|
// ../plugin/src/hooks/magic-context/compartment-trigger.ts
|
|
26384
26399
|
var PROACTIVE_TRIGGER_OFFSET_PERCENTAGE = 2;
|
|
26385
26400
|
var POST_DROP_TARGET_RATIO = 0.75;
|
|
@@ -26638,6 +26653,13 @@ function checkCompartmentTrigger(db, sessionId, sessionMeta, usage, _previousPer
|
|
|
26638
26653
|
}
|
|
26639
26654
|
return { shouldFire: false };
|
|
26640
26655
|
}
|
|
26656
|
+
if (tailInfo.boundarySnapshot && shouldWaitForNewHistorianHistory(tailInfo.boundarySnapshot, {
|
|
26657
|
+
usagePercentage: usage.percentage,
|
|
26658
|
+
emergency: getOverflowState(db, sessionId).needsEmergencyRecovery
|
|
26659
|
+
})) {
|
|
26660
|
+
sessionLog2(sessionId, "compartment trigger: waiting for new history after no forward progress");
|
|
26661
|
+
return { shouldFire: false };
|
|
26662
|
+
}
|
|
26641
26663
|
const canClearReasoning = reasoningProjection?.canClearReasoning ?? modelAcceptsEmptyContent(reasoningProjection?.providerID);
|
|
26642
26664
|
const projectedPostDropPercentage = estimateProjectedPostDropPercentage(db, sessionId, usage, preloadedActiveTags ?? getActiveTagsBySession2(db, sessionId), keepReasoningTokens, sessionMeta.clearedReasoningThroughTag, canClearReasoning, reasoningProjection?.budgetCutoff, eligiblePendingDropTagIds);
|
|
26643
26665
|
const relativePostDropTarget = executeThresholdPercentage * POST_DROP_TARGET_RATIO;
|
|
@@ -30030,6 +30052,14 @@ async function runPiHistorian(deps) {
|
|
|
30030
30052
|
return;
|
|
30031
30053
|
}
|
|
30032
30054
|
const protectedTailStart = boundarySnapshot.protectedTailStart;
|
|
30055
|
+
if (!forceDrainQuota && shouldWaitForNewHistorianHistory(boundarySnapshot, {
|
|
30056
|
+
emergency: getOverflowState(db, sessionId).needsEmergencyRecovery
|
|
30057
|
+
})) {
|
|
30058
|
+
telemetry.status = "noop";
|
|
30059
|
+
telemetry.failureReason = "waiting for new history after no forward progress";
|
|
30060
|
+
sessionLog2(sessionId, "historian no-op: waiting for new history after no forward progress");
|
|
30061
|
+
return;
|
|
30062
|
+
}
|
|
30033
30063
|
const eligibleEndOrdinal = Math.min(boundarySnapshot.eligibleEndOrdinal, protectedTailStart);
|
|
30034
30064
|
if (protectedTailStart <= offset || eligibleEndOrdinal <= offset) {
|
|
30035
30065
|
telemetry.status = "noop";
|
|
@@ -30230,6 +30260,12 @@ ${chunkText}`,
|
|
|
30230
30260
|
};
|
|
30231
30261
|
retainDrainReservationForRetryThrottle = true;
|
|
30232
30262
|
const historianEditorSystemPrompt = withContentLanguageDirective(HISTORIAN_EDITOR_SYSTEM_PROMPT, deps.language, { preserveUserQuotes: true });
|
|
30263
|
+
if (!forceDrainQuota) {
|
|
30264
|
+
shouldWaitForNewHistorianHistory(boundarySnapshot, {
|
|
30265
|
+
emergency: getOverflowState(db, sessionId).needsEmergencyRecovery,
|
|
30266
|
+
claim: true
|
|
30267
|
+
});
|
|
30268
|
+
}
|
|
30233
30269
|
const firstResult = await runHistorianSubagentWithTransientRetries({
|
|
30234
30270
|
runner,
|
|
30235
30271
|
sessionId,
|
|
@@ -30393,6 +30429,7 @@ ${chunkText}`,
|
|
|
30393
30429
|
const failCount = incrementHistorianFailure(db, sessionId, errorMsg);
|
|
30394
30430
|
await notify(buildHistorianFailureNotice(failCount, errorMsg));
|
|
30395
30431
|
}
|
|
30432
|
+
recordHistorianNoProgress(boundarySnapshot, inEmergency);
|
|
30396
30433
|
rollbackDrainReservation();
|
|
30397
30434
|
return;
|
|
30398
30435
|
}
|
|
@@ -30488,6 +30525,7 @@ ${chunkText}`,
|
|
|
30488
30525
|
}
|
|
30489
30526
|
db.exec("COMMIT");
|
|
30490
30527
|
published = true;
|
|
30528
|
+
clearHistorianHistoryWait(sessionId);
|
|
30491
30529
|
logSlowWriteTransaction("pi_historian_publish", transactionStartedAt);
|
|
30492
30530
|
} finally {
|
|
30493
30531
|
if (!published) {
|