@cortexkit/pi-magic-context 0.49.19 → 0.49.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/auto-search-worker.js +48 -9
- package/dist/chunk-embedding-write-worker.js +29 -22
- package/dist/{compaction-marker-candidates-wjzqbspm.js → compaction-marker-candidates-p1kqkqhq.js} +3 -3
- package/dist/compaction-marker-candidates-worker.js +2 -2
- package/dist/compartment-chunk-embedding-y01m95c2.js +64 -0
- package/dist/embedding-backfill-2vh6m3bg.js +24 -0
- package/dist/embedding-worker-process.js +2 -2
- package/dist/embedding-worker.js +6 -6
- package/dist/historian-calibration-extension.js +1 -1
- package/dist/{index-e1rza8gp.js → index-29nw7n1r.js} +1 -1
- package/dist/{index-rrwv9y2k.js → index-2qtx0q6h.js} +214 -5
- package/dist/{index-msqrs4nf.js → index-3c3ztwgc.js} +5 -5
- package/dist/index-54wmg4rj.js +82 -0
- package/dist/{index-yygsh681.js → index-579m1751.js} +3 -3
- package/dist/{index-rc1dez8m.js → index-5j6vwg4z.js} +154 -13
- package/dist/{index-qpq6w67c.js → index-6wdwv3a6.js} +2 -2
- package/dist/{index-vktahpdj.js → index-at6py2k3.js} +9 -3
- package/dist/{index-bxmf4dhr.js → index-ccanzg8p.js} +2367 -1243
- package/dist/{index-sqt2haap.js → index-dbgg6hsf.js} +28 -14
- package/dist/{index-7bqrfyny.js → index-fgs78jyd.js} +12 -15
- package/dist/{index-58mj0dzc.js → index-ftvm2acc.js} +181 -46
- package/dist/{index-wveb51es.js → index-gqjxy2p7.js} +74 -9
- package/dist/{index-d8gc0112.js → index-gwxaz0z5.js} +4 -6
- package/dist/{index-d4dt19md.js → index-hqw0rt9k.js} +1 -1
- package/dist/{index-g7p75667.js → index-hxp6q5f8.js} +2 -2
- package/dist/{index-svgkv4g2.js → index-j3pz3a1b.js} +2 -2
- package/dist/{index-f18ec69t.js → index-jn44h81p.js} +1 -1
- package/dist/{index-1kcfwk9c.js → index-my0dn413.js} +1 -1
- package/dist/{index-apgwdby5.js → index-n4wyxcgr.js} +1 -1
- package/dist/{index-rhprywn6.js → index-n8e5e9fp.js} +28 -7
- package/dist/{index-z2rkv0mb.js → index-pe5gz5c4.js} +5 -5
- package/dist/{index-dtp1efvf.js → index-st4gwt64.js} +2 -2
- package/dist/{index-xnch62s6.js → index-vx6f7wff.js} +314 -127
- package/dist/{index-jrap43gc.js → index-w1avzkdt.js} +1 -1
- package/dist/{index-vmcgew7x.js → index-w1twp2kp.js} +78 -58
- package/dist/{index-ek9zvs4f.js → index-w6r3ggnp.js} +57 -27
- package/dist/{index-ym6nh2gk.js → index-wgmhk91m.js} +113 -20
- package/dist/{index-2mmy81qt.js → index-wqpjhx1p.js} +501 -20
- package/dist/index.js +1178 -417
- package/dist/{logger-dygt96rn.js → logger-shetp43v.js} +1 -1
- package/dist/{message-fts-rowid-map-cvpy2dmb.js → message-fts-rowid-map-59y476bk.js} +6 -5
- package/dist/{message-index-rc2x41k4.js → message-index-3snxf11x.js} +15 -14
- package/dist/{message-index-bounded-66saeq5j.js → message-index-bounded-4ngt600v.js} +15 -14
- package/dist/{message-time-backfill-ssshbdy8.js → message-time-backfill-e3r8ddh1.js} +3 -3
- package/dist/migration-worker.js +12 -11
- package/dist/{project-embedding-registry-vf1ncptn.js → project-embedding-registry-hfyzs654.js} +14 -13
- package/dist/raw-ordinal-warmup-worker.js +136 -101
- package/dist/{read-session-chunk-h5bzj91r.js → read-session-chunk-5dmc77q2.js} +10 -9
- package/dist/{read-session-raw-9s3xn7fc.js → read-session-raw-yxv2fv34.js} +4 -3
- package/dist/{safe-notification-target-2rkvkmw8.js → safe-notification-target-zcpchea6.js} +2 -2
- package/dist/search-archive-3daj5ttc.js +21 -0
- package/dist/{search-wh7b9zht.js → search-jhfh3tsj.js} +23 -22
- package/dist/{sqlite-fsp0sf3r.js → sqlite-v89bj0yy.js} +7 -1
- package/dist/{storage-memory-yxxc5v4n.js → storage-memory-0ndn65fm.js} +2 -2
- package/dist/{storage-tags-1kets0q8.js → storage-tags-r48vxthe.js} +7 -5
- package/dist/subagent-entry.js +28 -26
- package/dist/tag-snapshot-worker-pi.js +22 -14
- package/package.json +5 -2
- package/dist/embedding-backfill-vwag34t0.js +0 -23
- package/dist/search-archive-4817pqev.js +0 -20
|
@@ -5,21 +5,23 @@ import {
|
|
|
5
5
|
importPluginModule,
|
|
6
6
|
writeForwardedLogLine2,
|
|
7
7
|
log2
|
|
8
|
-
} from "./index-
|
|
8
|
+
} from "./index-fgs78jyd.js";
|
|
9
9
|
import {
|
|
10
10
|
inRootAsyncContext2,
|
|
11
11
|
registerSlowWriteReporter2,
|
|
12
12
|
registerSqliteDiagnosticSink2,
|
|
13
13
|
detectSqliteRuntime2,
|
|
14
14
|
Database2,
|
|
15
|
+
createSqliteTurnDatabase2,
|
|
16
|
+
hasSqliteTurnDeadlineRouting2,
|
|
15
17
|
withSqliteBackgroundWriter2,
|
|
16
18
|
withoutSqliteTransformPass2,
|
|
17
19
|
isTransientSqliteError2,
|
|
18
20
|
withPrivilegedWriter2
|
|
19
|
-
} from "./index-
|
|
21
|
+
} from "./index-2qtx0q6h.js";
|
|
20
22
|
import {
|
|
21
23
|
scheduleAfterBootQuiet
|
|
22
|
-
} from "./index-
|
|
24
|
+
} from "./index-hxp6q5f8.js";
|
|
23
25
|
import {
|
|
24
26
|
getHarness2,
|
|
25
27
|
harnessOwnsOpenCodeStore2
|
|
@@ -34,13 +36,13 @@ import {
|
|
|
34
36
|
parseRpcPortFile,
|
|
35
37
|
inspectProcessesAsync,
|
|
36
38
|
inspectWindowsProcessesSync
|
|
37
|
-
} from "./index-
|
|
39
|
+
} from "./index-n4wyxcgr.js";
|
|
38
40
|
import {
|
|
39
41
|
configureContextDatabasePragmas2
|
|
40
42
|
} from "./index-fapenpch.js";
|
|
41
43
|
import {
|
|
42
44
|
startMessageFtsRowidMapBackfill2
|
|
43
|
-
} from "./index-
|
|
45
|
+
} from "./index-579m1751.js";
|
|
44
46
|
import {
|
|
45
47
|
LKG_SLOT_CHUNKS_DDL,
|
|
46
48
|
SESSION_REPLAY_DECISIONS_DDL,
|
|
@@ -50,13 +52,13 @@ import {
|
|
|
50
52
|
hasPendingMigrations,
|
|
51
53
|
getMainThreadMigrationBodyCount,
|
|
52
54
|
runMigrations
|
|
53
|
-
} from "./index-
|
|
55
|
+
} from "./index-3c3ztwgc.js";
|
|
54
56
|
import {
|
|
55
57
|
closeQuietly,
|
|
56
58
|
ensureColumn,
|
|
57
59
|
installV95PerfSchema,
|
|
58
60
|
installTagIdentityRevisionTrigger
|
|
59
|
-
} from "./index-
|
|
61
|
+
} from "./index-w1avzkdt.js";
|
|
60
62
|
import {
|
|
61
63
|
resolveOpenCodeDbPath,
|
|
62
64
|
hasV1MessageTables,
|
|
@@ -65,7 +67,7 @@ import {
|
|
|
65
67
|
} from "./index-8976mezv.js";
|
|
66
68
|
import {
|
|
67
69
|
logSlowWriteTransaction
|
|
68
|
-
} from "./index-
|
|
70
|
+
} from "./index-hqw0rt9k.js";
|
|
69
71
|
import {
|
|
70
72
|
shouldEnforcePrivateStoragePermissions,
|
|
71
73
|
ensureStorageDirectorySync,
|
|
@@ -1305,8 +1307,8 @@ function finishDatabaseOpen(db, dbPath, explicitDbPath, latestSupportedVersion)
|
|
|
1305
1307
|
state.running = startMessageFtsRowidMapBackfill2(db, signal).then(async () => {
|
|
1306
1308
|
if (signal.aborted || !readsOpenCodeStore)
|
|
1307
1309
|
return;
|
|
1308
|
-
const { readRawSessionMessagePage, readRawSessionMessages } = await importPluginModule(() => import("./read-session-chunk-
|
|
1309
|
-
const { startMessageTimeBackfill } = await importPluginModule(() => import("./message-time-backfill-
|
|
1310
|
+
const { readRawSessionMessagePage, readRawSessionMessages } = await importPluginModule(() => import("./read-session-chunk-5dmc77q2.js"));
|
|
1311
|
+
const { startMessageTimeBackfill } = await importPluginModule(() => import("./message-time-backfill-e3r8ddh1.js"));
|
|
1310
1312
|
if (signal.aborted)
|
|
1311
1313
|
return;
|
|
1312
1314
|
await startMessageTimeBackfill(db, Object.assign(readRawSessionMessages, {
|
|
@@ -2611,6 +2613,10 @@ function openDatabase(dbPathOrOptions) {
|
|
|
2611
2613
|
return null;
|
|
2612
2614
|
}
|
|
2613
2615
|
if (existing) {
|
|
2616
|
+
if (options?.nativeTurnDeadlines && !hasSqliteTurnDeadlineRouting2(existing)) {
|
|
2617
|
+
log2("[magic-context] storage unavailable: native turn deadlines must be selected on the first open; restart the host");
|
|
2618
|
+
return null;
|
|
2619
|
+
}
|
|
2614
2620
|
if (!enforceSchemaFence(existing, dbPath, latestSupportedVersion)) {
|
|
2615
2621
|
return null;
|
|
2616
2622
|
}
|
|
@@ -2635,7 +2641,7 @@ function openDatabase(dbPathOrOptions) {
|
|
|
2635
2641
|
throw error;
|
|
2636
2642
|
}
|
|
2637
2643
|
}
|
|
2638
|
-
db = new Database2(dbPath);
|
|
2644
|
+
db = options?.nativeTurnDeadlines ? createSqliteTurnDatabase2(dbPath) : new Database2(dbPath);
|
|
2639
2645
|
installBootBusyTimeout(db, dbPath, busyTimeoutMs, options?.onBootBusyTimeout);
|
|
2640
2646
|
if (!enforceSchemaFence(db, dbPath, latestSupportedVersion)) {
|
|
2641
2647
|
closeQuietly(db);
|
|
@@ -2682,6 +2688,10 @@ async function openDatabaseAsync(dbPathOrOptions) {
|
|
|
2682
2688
|
lastMigrationOnOpenRefusal = null;
|
|
2683
2689
|
const existing = databases.get(dbPath);
|
|
2684
2690
|
if (existing) {
|
|
2691
|
+
if (options?.nativeTurnDeadlines && !hasSqliteTurnDeadlineRouting2(existing)) {
|
|
2692
|
+
log2("[magic-context] storage unavailable: native turn deadlines must be selected on the first open; restart the host");
|
|
2693
|
+
return null;
|
|
2694
|
+
}
|
|
2685
2695
|
const startedAt = performance.now();
|
|
2686
2696
|
const accepted = enforceSchemaFence(existing, dbPath, latestSupportedVersion);
|
|
2687
2697
|
if (accepted) {
|
|
@@ -2697,8 +2707,12 @@ async function openDatabaseAsync(dbPathOrOptions) {
|
|
|
2697
2707
|
return accepted ? existing : null;
|
|
2698
2708
|
}
|
|
2699
2709
|
const pending = pendingAsyncOpens.get(dbPath);
|
|
2700
|
-
if (pending)
|
|
2701
|
-
|
|
2710
|
+
if (pending) {
|
|
2711
|
+
if (!options?.nativeTurnDeadlines)
|
|
2712
|
+
return pending;
|
|
2713
|
+
const opened = await pending;
|
|
2714
|
+
return opened && hasSqliteTurnDeadlineRouting2(opened) ? opened : null;
|
|
2715
|
+
}
|
|
2702
2716
|
if (!isFileBackedPath(dbPath)) {
|
|
2703
2717
|
throw new Error("async migration requires a file-backed database; use the explicit synchronous opener for in-memory or URI test databases");
|
|
2704
2718
|
}
|
|
@@ -2724,7 +2738,7 @@ async function openDatabaseAsync(dbPathOrOptions) {
|
|
|
2724
2738
|
throw error;
|
|
2725
2739
|
}
|
|
2726
2740
|
}
|
|
2727
|
-
db = new Database2(dbPath);
|
|
2741
|
+
db = options?.nativeTurnDeadlines ? createSqliteTurnDatabase2(dbPath) : new Database2(dbPath);
|
|
2728
2742
|
installBootBusyTimeout(db, dbPath, busyTimeoutMs, options?.onBootBusyTimeout);
|
|
2729
2743
|
openMs = performance.now() - openStartedAt;
|
|
2730
2744
|
guardStartedAt = performance.now();
|
|
@@ -486,6 +486,7 @@ async function importPluginModule(load) {
|
|
|
486
486
|
|
|
487
487
|
// ../plugin/src/shared/logger.ts
|
|
488
488
|
var isTestEnv = false;
|
|
489
|
+
var syncLogWrites = process.env.MAGIC_CONTEXT_LOG_SYNC === "1";
|
|
489
490
|
var buffer = [];
|
|
490
491
|
var flushTimer = null;
|
|
491
492
|
var FLUSH_INTERVAL_MS = 500;
|
|
@@ -630,6 +631,15 @@ function scheduleFlush() {
|
|
|
630
631
|
flush();
|
|
631
632
|
}, FLUSH_INTERVAL_MS);
|
|
632
633
|
}
|
|
634
|
+
function appendLine(line) {
|
|
635
|
+
buffer.push(line);
|
|
636
|
+
bufferedBytes += Buffer.byteLength(line);
|
|
637
|
+
boundBuffer();
|
|
638
|
+
if (syncLogWrites || buffer.length >= BUFFER_SIZE_LIMIT)
|
|
639
|
+
flush();
|
|
640
|
+
else
|
|
641
|
+
scheduleFlush();
|
|
642
|
+
}
|
|
633
643
|
var lineForwarder = null;
|
|
634
644
|
function setLogLineForwarder2(forward) {
|
|
635
645
|
lineForwarder = forward;
|
|
@@ -638,13 +648,7 @@ function writeForwardedLogLine2(line) {
|
|
|
638
648
|
if (isTestEnv)
|
|
639
649
|
return;
|
|
640
650
|
try {
|
|
641
|
-
|
|
642
|
-
bufferedBytes += Buffer.byteLength(line);
|
|
643
|
-
boundBuffer();
|
|
644
|
-
if (buffer.length >= BUFFER_SIZE_LIMIT)
|
|
645
|
-
flush();
|
|
646
|
-
else
|
|
647
|
-
scheduleFlush();
|
|
651
|
+
appendLine(line);
|
|
648
652
|
} catch {}
|
|
649
653
|
}
|
|
650
654
|
function log2(message, data) {
|
|
@@ -669,14 +673,7 @@ ${data.stack}` : ""}`)}` : ` ${JSON.stringify(sanitizeConfigValue(data))}`;
|
|
|
669
673
|
lineForwarder(line);
|
|
670
674
|
return;
|
|
671
675
|
}
|
|
672
|
-
|
|
673
|
-
bufferedBytes += Buffer.byteLength(line);
|
|
674
|
-
boundBuffer();
|
|
675
|
-
if (buffer.length >= BUFFER_SIZE_LIMIT) {
|
|
676
|
-
flush();
|
|
677
|
-
} else {
|
|
678
|
-
scheduleFlush();
|
|
679
|
-
}
|
|
676
|
+
appendLine(line);
|
|
680
677
|
} catch {}
|
|
681
678
|
}
|
|
682
679
|
function sessionLog2(sessionId, message, data) {
|
|
@@ -4,7 +4,7 @@ import {
|
|
|
4
4
|
import {
|
|
5
5
|
inRootAsyncContext2,
|
|
6
6
|
isTransientSqliteError2
|
|
7
|
-
} from "./index-
|
|
7
|
+
} from "./index-2qtx0q6h.js";
|
|
8
8
|
import {
|
|
9
9
|
PROMOTABLE_CATEGORIES,
|
|
10
10
|
CATEGORY_DEFAULT_TTL
|
|
@@ -14,7 +14,7 @@ import {
|
|
|
14
14
|
sanitizeDiagnosticText,
|
|
15
15
|
log2,
|
|
16
16
|
sessionLog2
|
|
17
|
-
} from "./index-
|
|
17
|
+
} from "./index-fgs78jyd.js";
|
|
18
18
|
import {
|
|
19
19
|
resolveWorkspaceShareCategories,
|
|
20
20
|
resolveWorkspaceIdentitySet,
|
|
@@ -22,17 +22,19 @@ import {
|
|
|
22
22
|
resolveStoredPathWorkspaceIdentity,
|
|
23
23
|
sourceNameForMemory,
|
|
24
24
|
managedAuthorityNoteRow
|
|
25
|
-
} from "./index-
|
|
25
|
+
} from "./index-3c3ztwgc.js";
|
|
26
26
|
import {
|
|
27
27
|
isKnownAutocommit
|
|
28
|
-
} from "./index-
|
|
28
|
+
} from "./index-w1avzkdt.js";
|
|
29
29
|
import {
|
|
30
30
|
SubcClient
|
|
31
|
-
} from "./index-
|
|
31
|
+
} from "./index-gwxaz0z5.js";
|
|
32
32
|
import {
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
33
|
+
buildCanonicalChunkTextFromFts2,
|
|
34
|
+
loadCompartmentChunkEmbeddingsForSearch2,
|
|
35
|
+
peekCompartmentChunkSearchPool2,
|
|
36
|
+
warmCompartmentChunkSearchPool2
|
|
37
|
+
} from "./index-vx6f7wff.js";
|
|
36
38
|
import {
|
|
37
39
|
recordEmbeddingMeasurement,
|
|
38
40
|
saveCommitEmbedding,
|
|
@@ -50,7 +52,7 @@ import {
|
|
|
50
52
|
getProjectEmbeddingSnapshot2,
|
|
51
53
|
embedTextForProject2,
|
|
52
54
|
embedItemsForProject2
|
|
53
|
-
} from "./index-
|
|
55
|
+
} from "./index-w1twp2kp.js";
|
|
54
56
|
import {
|
|
55
57
|
saveEmbeddingIfHashMatches,
|
|
56
58
|
getProjectEmbeddings2
|
|
@@ -60,7 +62,7 @@ import {
|
|
|
60
62
|
isEmbeddingEnabled,
|
|
61
63
|
embedText,
|
|
62
64
|
ensureMemoryEmbeddings2
|
|
63
|
-
} from "./index-
|
|
65
|
+
} from "./index-pe5gz5c4.js";
|
|
64
66
|
import {
|
|
65
67
|
getMemorySelectColumns2,
|
|
66
68
|
isMemoryRow2,
|
|
@@ -74,13 +76,13 @@ import {
|
|
|
74
76
|
getMemoryById2,
|
|
75
77
|
updateMemorySeenCount2,
|
|
76
78
|
updateMemoryRetrievalCount2
|
|
77
|
-
} from "./index-
|
|
79
|
+
} from "./index-29nw7n1r.js";
|
|
78
80
|
import {
|
|
79
81
|
cleanSearchText,
|
|
80
82
|
keywordRelevance,
|
|
81
83
|
matchedSearchSnippet,
|
|
82
84
|
getIndexedMessageCorpusSize2
|
|
83
|
-
} from "./index-
|
|
85
|
+
} from "./index-wgmhk91m.js";
|
|
84
86
|
import {
|
|
85
87
|
MESSAGE_FTS_SESSION_FILTER_SQL,
|
|
86
88
|
withMessageFtsSessionFilter
|
|
@@ -1968,6 +1970,7 @@ function getDefaultSubcConnectionFile() {
|
|
|
1968
1970
|
var RERANK_MODEL = "gte-reranker-modernbert-base-f32";
|
|
1969
1971
|
var RERANK_LANE_LIMIT = 25;
|
|
1970
1972
|
var DEADLINE_MS = 1500;
|
|
1973
|
+
var DOCUMENT_BYTES = 1024;
|
|
1971
1974
|
function isProseQuery(query) {
|
|
1972
1975
|
const text = query.trim();
|
|
1973
1976
|
if (!text || /[\n;]|^["'`]|["'`]$/.test(text))
|
|
@@ -2047,7 +2050,7 @@ function candidateText(location, title, body) {
|
|
|
2047
2050
|
${truncateUtf8(title.replaceAll(`
|
|
2048
2051
|
`, " "), 120)}
|
|
2049
2052
|
`;
|
|
2050
|
-
return header + truncateUtf8(body,
|
|
2053
|
+
return header + truncateUtf8(body, DOCUMENT_BYTES - Buffer.byteLength(header));
|
|
2051
2054
|
}
|
|
2052
2055
|
|
|
2053
2056
|
class RerankFailure extends Error {
|
|
@@ -2117,7 +2120,79 @@ async function connect(connectionFile) {
|
|
|
2117
2120
|
}
|
|
2118
2121
|
return bounded(connection);
|
|
2119
2122
|
}
|
|
2120
|
-
async function
|
|
2123
|
+
async function remoteScores(config, query, documents, signal) {
|
|
2124
|
+
const controller = new AbortController;
|
|
2125
|
+
let timer;
|
|
2126
|
+
let onAbort;
|
|
2127
|
+
try {
|
|
2128
|
+
const cancelled = new Promise((_, reject) => {
|
|
2129
|
+
const cancel = (reason) => {
|
|
2130
|
+
reject(new RerankFailure(reason));
|
|
2131
|
+
controller.abort();
|
|
2132
|
+
};
|
|
2133
|
+
onAbort = () => cancel("aborted");
|
|
2134
|
+
if (signal?.aborted)
|
|
2135
|
+
onAbort();
|
|
2136
|
+
else
|
|
2137
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
2138
|
+
timer = setTimeout(() => cancel("timeout"), Math.max(50, Math.min(15000, config.timeout_ms ?? DEADLINE_MS)));
|
|
2139
|
+
});
|
|
2140
|
+
const request = async () => {
|
|
2141
|
+
const tei = config.endpoint?.startsWith("tei+");
|
|
2142
|
+
const endpoint = (tei ? config.endpoint?.slice(4) : config.endpoint)?.replace(/\/+$/, "");
|
|
2143
|
+
if (!endpoint || !/^https?:\/\//.test(endpoint))
|
|
2144
|
+
throw new RerankFailure("invalid_endpoint");
|
|
2145
|
+
const response = await fetch(`${endpoint}/rerank`, {
|
|
2146
|
+
method: "POST",
|
|
2147
|
+
headers: {
|
|
2148
|
+
"Content-Type": "application/json",
|
|
2149
|
+
...config.api_key ? { Authorization: `Bearer ${config.api_key}` } : {}
|
|
2150
|
+
},
|
|
2151
|
+
body: JSON.stringify(tei ? { query, texts: documents } : {
|
|
2152
|
+
model: config.model,
|
|
2153
|
+
query,
|
|
2154
|
+
documents,
|
|
2155
|
+
top_n: documents.length
|
|
2156
|
+
}),
|
|
2157
|
+
signal: controller.signal,
|
|
2158
|
+
redirect: "error"
|
|
2159
|
+
});
|
|
2160
|
+
if (!response.ok)
|
|
2161
|
+
throw new RerankFailure(`http_${response.status}`);
|
|
2162
|
+
const body = await response.json();
|
|
2163
|
+
const rows = tei ? body : body?.results;
|
|
2164
|
+
if (!Array.isArray(rows) || rows.length !== documents.length)
|
|
2165
|
+
throw new RerankFailure("invalid_reply");
|
|
2166
|
+
const scores = new Array(documents.length);
|
|
2167
|
+
const seen = new Set;
|
|
2168
|
+
for (const row of rows) {
|
|
2169
|
+
const index = row?.index;
|
|
2170
|
+
const score = tei ? row?.score : row?.relevance_score;
|
|
2171
|
+
if (!Number.isInteger(index) || index < 0 || index >= documents.length || seen.has(index) || typeof score !== "number" || !Number.isFinite(score))
|
|
2172
|
+
throw new RerankFailure("invalid_reply");
|
|
2173
|
+
seen.add(index);
|
|
2174
|
+
scores[index] = score;
|
|
2175
|
+
}
|
|
2176
|
+
return scores;
|
|
2177
|
+
};
|
|
2178
|
+
return await Promise.race([cancelled, request()]);
|
|
2179
|
+
} finally {
|
|
2180
|
+
clearTimeout(timer);
|
|
2181
|
+
if (onAbort)
|
|
2182
|
+
signal?.removeEventListener("abort", onAbort);
|
|
2183
|
+
}
|
|
2184
|
+
}
|
|
2185
|
+
function remoteRerankSkipped(reason, onSkipped) {
|
|
2186
|
+
const safe = /^(?:invalid_reply|invalid_endpoint|validation|timeout|aborted|candidate_validation|http_\d{3})$/.test(reason) ? reason : "remote_error";
|
|
2187
|
+
const line = `Reranking skipped: ${safe}`;
|
|
2188
|
+
logRerankFallback(safe);
|
|
2189
|
+
onSkipped?.(line);
|
|
2190
|
+
}
|
|
2191
|
+
async function prepareReranker(projectRoot, sessionId, connectionFile, injectedClient, config) {
|
|
2192
|
+
if (config?.provider === "off")
|
|
2193
|
+
return null;
|
|
2194
|
+
if (config?.provider === "remote")
|
|
2195
|
+
return { remote: config };
|
|
2121
2196
|
try {
|
|
2122
2197
|
if (injectedClient === null)
|
|
2123
2198
|
throw new RerankFailure("missing_synapse");
|
|
@@ -2134,36 +2209,43 @@ async function prepareReranker(projectRoot, sessionId, connectionFile, injectedC
|
|
|
2134
2209
|
return null;
|
|
2135
2210
|
}
|
|
2136
2211
|
}
|
|
2137
|
-
async function rerankSearchResults(session, query, baseline, pool, texts, limit) {
|
|
2212
|
+
async function rerankSearchResults(session, query, baseline, pool, texts, limit, signal, onSkipped) {
|
|
2138
2213
|
if (!pool.length)
|
|
2139
2214
|
return baseline;
|
|
2140
2215
|
try {
|
|
2141
|
-
if (pool.length > 20 || texts.length !== pool.length || texts.some((text) => Buffer.byteLength(text) >
|
|
2142
|
-
throw new RerankFailure("validation");
|
|
2143
|
-
const body = unwrap(await bounded(session.client.call("synapse", "rerank.score", {
|
|
2144
|
-
model: RERANK_MODEL,
|
|
2145
|
-
query,
|
|
2146
|
-
candidates: texts,
|
|
2147
|
-
required_fingerprint: session.lane.fingerprint,
|
|
2148
|
-
allow_equivalent: false,
|
|
2149
|
-
accept_declared: false,
|
|
2150
|
-
max_queue_ms: 100,
|
|
2151
|
-
deadline_ms: DEADLINE_MS
|
|
2152
|
-
}, {
|
|
2153
|
-
timeoutMs: DEADLINE_MS,
|
|
2154
|
-
targetKind: "management_surface",
|
|
2155
|
-
identity: session.identity
|
|
2156
|
-
})));
|
|
2157
|
-
if (body.fingerprint !== session.lane.fingerprint)
|
|
2158
|
-
throw new RerankFailure("fingerprint_mismatch");
|
|
2159
|
-
if (body.table_epoch !== session.lane.tableEpoch)
|
|
2160
|
-
throw new RerankFailure("epoch_mismatch");
|
|
2161
|
-
if (!Array.isArray(body.scores) || body.scores.length !== pool.length || body.scores.some((score) => typeof score !== "number" || !Number.isFinite(score)))
|
|
2216
|
+
if (pool.length > 20 || texts.length !== pool.length || texts.some((text) => Buffer.byteLength(text) > DOCUMENT_BYTES))
|
|
2162
2217
|
throw new RerankFailure("validation");
|
|
2163
|
-
|
|
2218
|
+
let scores;
|
|
2219
|
+
if ("remote" in session) {
|
|
2220
|
+
scores = await remoteScores(session.remote, query, texts, signal);
|
|
2221
|
+
} else {
|
|
2222
|
+
const body = unwrap(await bounded(session.client.call("synapse", "rerank.score", {
|
|
2223
|
+
model: RERANK_MODEL,
|
|
2224
|
+
query,
|
|
2225
|
+
candidates: texts,
|
|
2226
|
+
required_fingerprint: session.lane.fingerprint,
|
|
2227
|
+
allow_equivalent: false,
|
|
2228
|
+
max_queue_ms: 100,
|
|
2229
|
+
deadline_ms: DEADLINE_MS
|
|
2230
|
+
}, {
|
|
2231
|
+
timeoutMs: DEADLINE_MS,
|
|
2232
|
+
targetKind: "management_surface",
|
|
2233
|
+
identity: session.identity
|
|
2234
|
+
})));
|
|
2235
|
+
if (body.fingerprint !== session.lane.fingerprint)
|
|
2236
|
+
throw new RerankFailure("fingerprint_mismatch");
|
|
2237
|
+
if (body.table_epoch !== session.lane.tableEpoch)
|
|
2238
|
+
throw new RerankFailure("epoch_mismatch");
|
|
2239
|
+
if (!Array.isArray(body.scores) || body.scores.length !== pool.length || body.scores.some((score) => typeof score !== "number" || !Number.isFinite(score)))
|
|
2240
|
+
throw new RerankFailure("validation");
|
|
2241
|
+
scores = body.scores;
|
|
2242
|
+
}
|
|
2164
2243
|
return pool.map((hit, index) => ({ hit, index, score: scores[index] })).sort((a, b) => b.score - a.score || a.index - b.index).slice(0, limit).map((row) => row.hit);
|
|
2165
2244
|
} catch (error) {
|
|
2166
|
-
|
|
2245
|
+
if ("remote" in session)
|
|
2246
|
+
remoteRerankSkipped(error instanceof RerankFailure ? error.reason : "remote_error", onSkipped);
|
|
2247
|
+
else
|
|
2248
|
+
logRerankFallback(reasonOf(error));
|
|
2167
2249
|
return baseline;
|
|
2168
2250
|
}
|
|
2169
2251
|
}
|
|
@@ -2218,7 +2300,7 @@ Reason: ${row.reason}` : ""}`;
|
|
|
2218
2300
|
case "compartment": {
|
|
2219
2301
|
title = hit.title;
|
|
2220
2302
|
location = `compartment ${hit.compartmentId} ${hit.sessionId} @${hit.startOrdinal}-${hit.endOrdinal}`;
|
|
2221
|
-
body =
|
|
2303
|
+
body = buildCanonicalChunkTextFromFts2(db, hit.sessionId, hit.startOrdinal, hit.endOrdinal) ?? "";
|
|
2222
2304
|
break;
|
|
2223
2305
|
}
|
|
2224
2306
|
}
|
|
@@ -3157,10 +3239,42 @@ function searchNotes(args) {
|
|
|
3157
3239
|
}));
|
|
3158
3240
|
}
|
|
3159
3241
|
async function searchCompartmentChunks(args) {
|
|
3160
|
-
if (
|
|
3242
|
+
if (args.limit <= 0 || !args.modelId || args.modelId === "off") {
|
|
3243
|
+
if (args.timings)
|
|
3244
|
+
args.timings.chunkPool = "off";
|
|
3245
|
+
return [];
|
|
3246
|
+
}
|
|
3247
|
+
if (!args.queryEmbedding) {
|
|
3248
|
+
if (args.timings)
|
|
3249
|
+
args.timings.chunkPool = "no-vector";
|
|
3250
|
+
if (args.warmOnly)
|
|
3251
|
+
warmCompartmentChunkSearchPool2(args.db, args.sessionId, args.projectPath, args.modelId);
|
|
3161
3252
|
return [];
|
|
3253
|
+
}
|
|
3162
3254
|
const cutoff = args.maxOrdinal != null && args.maxOrdinal >= 0 ? args.maxOrdinal : null;
|
|
3163
|
-
|
|
3255
|
+
let rows;
|
|
3256
|
+
if (args.warmOnly) {
|
|
3257
|
+
const warm = peekCompartmentChunkSearchPool2(args.db, args.sessionId, args.projectPath, args.modelId, args.dateRange);
|
|
3258
|
+
if (warm === null) {
|
|
3259
|
+
if (args.timings)
|
|
3260
|
+
args.timings.chunkPool = "cold";
|
|
3261
|
+
warmCompartmentChunkSearchPool2(args.db, args.sessionId, args.projectPath, args.modelId);
|
|
3262
|
+
return [];
|
|
3263
|
+
}
|
|
3264
|
+
if (args.timings)
|
|
3265
|
+
args.timings.chunkPool = "warm";
|
|
3266
|
+
rows = warm;
|
|
3267
|
+
} else {
|
|
3268
|
+
try {
|
|
3269
|
+
rows = await loadCompartmentChunkEmbeddingsForSearch2(args.db, args.sessionId, args.projectPath, args.modelId, args.dateRange, args.signal);
|
|
3270
|
+
} catch (error) {
|
|
3271
|
+
if (args.signal?.aborted)
|
|
3272
|
+
return [];
|
|
3273
|
+
throw error;
|
|
3274
|
+
}
|
|
3275
|
+
if (args.timings)
|
|
3276
|
+
args.timings.chunkPool = "loaded";
|
|
3277
|
+
}
|
|
3164
3278
|
if (rows.length === 0)
|
|
3165
3279
|
return [];
|
|
3166
3280
|
const byCompartment = new Map;
|
|
@@ -3452,7 +3566,7 @@ async function unifiedSearch2(db, sessionId, projectPath, query, options = {}) {
|
|
|
3452
3566
|
return [];
|
|
3453
3567
|
}
|
|
3454
3568
|
const limit = normalizeLimit(options.limit);
|
|
3455
|
-
const reranker = options.explicitSearch && options.rerank !== false && isProseQuery(trimmedQuery) ? await prepareReranker(projectPath, sessionId, options.rerankConnectionFile, options.rerankClient) : null;
|
|
3569
|
+
const reranker = options.explicitSearch && options.rerank !== false && isProseQuery(trimmedQuery) ? await prepareReranker(projectPath, sessionId, options.rerankConnectionFile, options.rerankClient, options.rerankConfig) : null;
|
|
3456
3570
|
const lanes = reranker || options.captureRerankLanes ? emptySearchLanes() : undefined;
|
|
3457
3571
|
const dateRange = normalizeDateRange(options.from, options.to);
|
|
3458
3572
|
const tierLimit = Math.max(limit * 3, DEFAULT_UNIFIED_SEARCH_LIMIT);
|
|
@@ -3485,6 +3599,8 @@ async function unifiedSearch2(db, sessionId, projectPath, query, options = {}) {
|
|
|
3485
3599
|
if (options.signal?.aborted)
|
|
3486
3600
|
return [];
|
|
3487
3601
|
const messageProbes = options.explicitSearch ? extractLiteralProbes(trimmedQuery) : [];
|
|
3602
|
+
const timings = options.stepTimings;
|
|
3603
|
+
let stepStartedAt = performance.now();
|
|
3488
3604
|
let messageResults = [];
|
|
3489
3605
|
if (runMessages) {
|
|
3490
3606
|
const messageRequest = {
|
|
@@ -3511,8 +3627,13 @@ async function unifiedSearch2(db, sessionId, projectPath, query, options = {}) {
|
|
|
3511
3627
|
messageResults = outcome.results.filter((result) => result.messageId !== options.excludeMessageId);
|
|
3512
3628
|
if (lanes)
|
|
3513
3629
|
lanes.message_fts = (outcome.laneResults ?? outcome.results).slice(0, 25);
|
|
3630
|
+
if (timings)
|
|
3631
|
+
timings.messageMs = performance.now() - stepStartedAt;
|
|
3514
3632
|
}
|
|
3633
|
+
stepStartedAt = performance.now();
|
|
3515
3634
|
const capturedQuery = await queryEmbeddingPromise;
|
|
3635
|
+
if (timings && needsEmbedding)
|
|
3636
|
+
timings.vectorWaitMs = performance.now() - stepStartedAt;
|
|
3516
3637
|
if (options.signal?.aborted)
|
|
3517
3638
|
return [];
|
|
3518
3639
|
const embeddingSnapshot = getProjectEmbeddingSnapshot2(projectPath);
|
|
@@ -3522,6 +3643,9 @@ async function unifiedSearch2(db, sessionId, projectPath, query, options = {}) {
|
|
|
3522
3643
|
const workspace = resolveSearchWorkspaceContext(db, projectPath);
|
|
3523
3644
|
const embeddingModelId = queryContract?.modelId ?? options.embeddingModelIdOverride ?? embeddingSnapshot?.modelId;
|
|
3524
3645
|
const chunkModelId = queryContract?.chunkModelId ?? options.chunkModelIdOverride ?? embeddingSnapshot?.chunkModelId;
|
|
3646
|
+
stepStartedAt = performance.now();
|
|
3647
|
+
if (timings)
|
|
3648
|
+
timings.chunkPool = "off";
|
|
3525
3649
|
const compartmentResults = runCompartmentChunks ? await searchCompartmentChunks({
|
|
3526
3650
|
db,
|
|
3527
3651
|
sessionId,
|
|
@@ -3530,8 +3654,14 @@ async function unifiedSearch2(db, sessionId, projectPath, query, options = {}) {
|
|
|
3530
3654
|
limit: lanes ? Math.max(tierLimit, 25) : tierLimit,
|
|
3531
3655
|
maxOrdinal: options.maxMessageOrdinal,
|
|
3532
3656
|
modelId: chunkModelId && chunkModelId !== "off" ? chunkModelId : null,
|
|
3533
|
-
dateRange
|
|
3657
|
+
dateRange,
|
|
3658
|
+
warmOnly: options.chunkPoolWarmOnly,
|
|
3659
|
+
signal: options.signal,
|
|
3660
|
+
timings
|
|
3534
3661
|
}) : [];
|
|
3662
|
+
if (timings && runCompartmentChunks)
|
|
3663
|
+
timings.chunkMs = performance.now() - stepStartedAt;
|
|
3664
|
+
stepStartedAt = performance.now();
|
|
3535
3665
|
if (lanes)
|
|
3536
3666
|
lanes.chunk_vector = compartmentResults.slice(0, 25);
|
|
3537
3667
|
const messageLikeResults = mergeMessageAndCompartmentResults({
|
|
@@ -3587,6 +3717,8 @@ async function unifiedSearch2(db, sessionId, projectPath, query, options = {}) {
|
|
|
3587
3717
|
dateRange
|
|
3588
3718
|
})) : Promise.resolve([])
|
|
3589
3719
|
]);
|
|
3720
|
+
if (timings)
|
|
3721
|
+
timings.otherLanesMs = performance.now() - stepStartedAt;
|
|
3590
3722
|
if (options.diagnostics) {
|
|
3591
3723
|
options.diagnostics.suppressedVisibleMemoryIds = memoryOutcome.suppressedVisibleIds;
|
|
3592
3724
|
}
|
|
@@ -3604,11 +3736,14 @@ async function unifiedSearch2(db, sessionId, projectPath, query, options = {}) {
|
|
|
3604
3736
|
options.captureRerankLanes?.(lanes);
|
|
3605
3737
|
}
|
|
3606
3738
|
if (reranker && lanes) {
|
|
3607
|
-
const pool = fuseSearchLanes(lanes);
|
|
3739
|
+
const pool = fuseSearchLanes(lanes).slice(0, "remote" in reranker ? reranker.remote.top_n : 20);
|
|
3608
3740
|
try {
|
|
3609
|
-
results = await rerankSearchResults(reranker, trimmedQuery, results, pool, candidateTexts(db, sessionId, projectPath, pool), limit);
|
|
3741
|
+
results = await rerankSearchResults(reranker, trimmedQuery, results, pool, candidateTexts(db, sessionId, projectPath, pool), limit, options.signal, options.onRerankSkipped);
|
|
3610
3742
|
} catch {
|
|
3611
|
-
|
|
3743
|
+
if ("remote" in reranker)
|
|
3744
|
+
remoteRerankSkipped("candidate_validation", options.onRerankSkipped);
|
|
3745
|
+
else
|
|
3746
|
+
logRerankFallback("candidate_validation");
|
|
3612
3747
|
}
|
|
3613
3748
|
}
|
|
3614
3749
|
if (!options.measurementDisabled) {
|