@cortexkit/pi-magic-context 0.49.14 → 0.49.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/auto-search-worker.js +3 -3
- package/dist/chunk-embedding-write-worker.js +8 -6
- package/dist/{embedding-backfill-7kj16fh9.js → embedding-backfill-c4bdw8pp.js} +10 -8
- package/dist/embedding-worker.js +2 -2
- package/dist/historian-calibration-extension.js +15 -103
- package/dist/index-1cvyaw9a.js +58 -0
- package/dist/{index-65kty5te.js → index-4178cbfw.js} +20 -69
- package/dist/{index-75ah9413.js → index-44h64qza.js} +3 -3
- package/dist/{index-9jvktm7k.js → index-657xh9w4.js} +1 -1
- package/dist/{index-7qc73yk5.js → index-afy3dzf9.js} +3 -3
- package/dist/{index-y29y2j2w.js → index-ajfpyk0x.js} +164 -11
- package/dist/{index-1k729e4f.js → index-bxmf4dhr.js} +6 -6
- package/dist/{index-j0n09wtp.js → index-dne4ja7a.js} +7 -5
- package/dist/{index-a4pk5yv9.js → index-fhyy4ave.js} +14 -14
- package/dist/index-ja1c33qa.js +62 -0
- package/dist/{index-1mrvw9en.js → index-mx59cve4.js} +1 -1
- package/dist/{index-nwbhe6qv.js → index-n3r2d2pr.js} +5 -5
- package/dist/{index-7wagp5jp.js → index-p3evwech.js} +3 -3
- package/dist/{index-he9s4qsj.js → index-qqxqwqhp.js} +22 -2
- package/dist/{index-djxc8w8c.js → index-s1xn8qfw.js} +21 -25
- package/dist/{index-bet4sd3e.js → index-tympym62.js} +1 -1
- package/dist/{index-ccsscaap.js → index-v8d4vgam.js} +37 -16
- package/dist/index-wrr9rp9a.js +16 -0
- package/dist/{index-1vw4f2pe.js → index-x3k1530z.js} +5 -7
- package/dist/{index-0awvkeb5.js → index-xmzhrht0.js} +10 -61
- package/dist/{index-6ex901g3.js → index-y5c73caa.js} +2 -2
- package/dist/index.js +787 -271
- package/dist/message-fts-rowid-map-mbanfkt1.js +22 -0
- package/dist/{message-index-bounded-z4d3x9ty.js → message-index-bounded-xzerjxmp.js} +7 -7
- package/dist/{message-index-dw4z9efj.js → message-index-easvmz22.js} +8 -8
- package/dist/{message-time-backfill-r2ar1qvk.js → message-time-backfill-3q4kemfc.js} +4 -4
- package/dist/migration-worker.js +6 -4
- package/dist/{project-embedding-registry-52zwmtbn.js → project-embedding-registry-xwnmqmx8.js} +9 -7
- package/dist/raw-ordinal-warmup-worker.js +37 -7
- package/dist/{read-session-chunk-0fsgg2k1.js → read-session-chunk-1sb5n7zs.js} +5 -5
- package/dist/{read-session-raw-azkqjgzg.js → read-session-raw-jz5e9jz0.js} +1 -1
- package/dist/search-archive-3ma5n0wr.js +20 -0
- package/dist/{search-ddbz6v60.js → search-x3fa0561.js} +16 -15
- package/dist/{storage-tags-3kb5tvgg.js → storage-tags-jtessrec.js} +1 -1
- package/dist/subagent-entry.js +19 -17
- package/dist/tag-snapshot-worker-pi.js +15 -18
- package/package.json +3 -1
|
@@ -250,6 +250,24 @@ function subagentTemperature(model, metadata, temperature, log) {
|
|
|
250
250
|
return decision.supported === true ? temperature : undefined;
|
|
251
251
|
}
|
|
252
252
|
|
|
253
|
+
// src/historian-calibration-extension.ts
|
|
254
|
+
import { readFileSync as readFileSync2 } from "node:fs";
|
|
255
|
+
|
|
256
|
+
// src/host-system-prompt.ts
|
|
257
|
+
function readHostSystemPrompt(ctx) {
|
|
258
|
+
if (typeof ctx.getSystemPrompt !== "function")
|
|
259
|
+
return;
|
|
260
|
+
const raw = ctx.getSystemPrompt();
|
|
261
|
+
if (typeof raw === "string")
|
|
262
|
+
return raw;
|
|
263
|
+
if (Array.isArray(raw) && raw.every((part) => typeof part === "string")) {
|
|
264
|
+
return raw.join(`
|
|
265
|
+
|
|
266
|
+
`);
|
|
267
|
+
}
|
|
268
|
+
return;
|
|
269
|
+
}
|
|
270
|
+
|
|
253
271
|
// src/subagent-telemetry.ts
|
|
254
272
|
import { createHash } from "node:crypto";
|
|
255
273
|
var stepCaps = new Map(buildHiddenAgentRegistrations({
|
|
@@ -351,19 +369,154 @@ class SubagentTelemetry {
|
|
|
351
369
|
}
|
|
352
370
|
}
|
|
353
371
|
|
|
354
|
-
// src/
|
|
355
|
-
|
|
356
|
-
|
|
372
|
+
// src/historian-calibration-extension.ts
|
|
373
|
+
var HISTORIAN_TEMPERATURE_ENV2 = "MAGIC_CONTEXT_HISTORIAN_TEMPERATURE";
|
|
374
|
+
var HISTORIAN_MAX_OUTPUT_TOKENS_ENV2 = "MAGIC_CONTEXT_HISTORIAN_MAX_OUTPUT_TOKENS";
|
|
375
|
+
var SUBAGENT_OUTPUT_CEILING_ENV2 = "MAGIC_CONTEXT_SUBAGENT_OUTPUT_CEILING";
|
|
376
|
+
function finiteNumber(value) {
|
|
377
|
+
if (value === undefined || value.trim().length === 0)
|
|
357
378
|
return;
|
|
358
|
-
const
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
379
|
+
const parsed = Number(value);
|
|
380
|
+
return Number.isFinite(parsed) && parsed >= 0 ? parsed : undefined;
|
|
381
|
+
}
|
|
382
|
+
function calibrateHistorianProviderPayload2(payload, temperature, maxOutputTokens) {
|
|
383
|
+
if (typeof payload !== "object" || payload === null || Array.isArray(payload))
|
|
384
|
+
return payload;
|
|
385
|
+
if (temperature === undefined && maxOutputTokens === undefined)
|
|
386
|
+
return payload;
|
|
387
|
+
const calibrated = { ...payload };
|
|
388
|
+
const generationConfig = calibrated.generationConfig;
|
|
389
|
+
if (typeof generationConfig === "object" && generationConfig !== null && !Array.isArray(generationConfig)) {
|
|
390
|
+
calibrated.generationConfig = {
|
|
391
|
+
...generationConfig,
|
|
392
|
+
...temperature !== undefined ? { temperature } : {},
|
|
393
|
+
...maxOutputTokens !== undefined ? { maxOutputTokens } : {}
|
|
394
|
+
};
|
|
395
|
+
return calibrated;
|
|
396
|
+
}
|
|
397
|
+
const inferenceConfig = calibrated.inferenceConfig;
|
|
398
|
+
if (typeof inferenceConfig === "object" && inferenceConfig !== null && !Array.isArray(inferenceConfig)) {
|
|
399
|
+
calibrated.inferenceConfig = {
|
|
400
|
+
...inferenceConfig,
|
|
401
|
+
...temperature !== undefined ? { temperature } : {},
|
|
402
|
+
...maxOutputTokens !== undefined ? { maxTokens: maxOutputTokens } : {}
|
|
403
|
+
};
|
|
404
|
+
return calibrated;
|
|
405
|
+
}
|
|
406
|
+
if (temperature !== undefined) {
|
|
407
|
+
calibrated.temperature = temperature;
|
|
408
|
+
}
|
|
409
|
+
if (maxOutputTokens !== undefined) {
|
|
410
|
+
if ("max_output_tokens" in calibrated) {
|
|
411
|
+
calibrated.max_output_tokens = maxOutputTokens;
|
|
412
|
+
} else if ("max_completion_tokens" in calibrated) {
|
|
413
|
+
calibrated.max_completion_tokens = maxOutputTokens;
|
|
414
|
+
} else if ("max_tokens" in calibrated) {
|
|
415
|
+
calibrated.max_tokens = maxOutputTokens;
|
|
416
|
+
} else if ("maxTokens" in calibrated) {
|
|
417
|
+
calibrated.maxTokens = maxOutputTokens;
|
|
418
|
+
}
|
|
419
|
+
}
|
|
420
|
+
return calibrated;
|
|
421
|
+
}
|
|
422
|
+
function ceilProviderOutputTokens2(payload, ceiling) {
|
|
423
|
+
if (ceiling === undefined)
|
|
424
|
+
return payload;
|
|
425
|
+
if (typeof payload !== "object" || payload === null || Array.isArray(payload))
|
|
426
|
+
return payload;
|
|
427
|
+
const lowered = (options, keys) => {
|
|
428
|
+
const next = { ...options };
|
|
429
|
+
for (const key of keys) {
|
|
430
|
+
const value = next[key];
|
|
431
|
+
if (typeof value === "number" && value > ceiling)
|
|
432
|
+
next[key] = ceiling;
|
|
433
|
+
}
|
|
434
|
+
return next;
|
|
435
|
+
};
|
|
436
|
+
const record = payload;
|
|
437
|
+
const nested = (key) => {
|
|
438
|
+
const value = record[key];
|
|
439
|
+
return typeof value === "object" && value !== null && !Array.isArray(value) ? value : undefined;
|
|
440
|
+
};
|
|
441
|
+
const generationConfig = nested("generationConfig");
|
|
442
|
+
if (generationConfig)
|
|
443
|
+
return {
|
|
444
|
+
...record,
|
|
445
|
+
generationConfig: lowered(generationConfig, ["maxOutputTokens"])
|
|
446
|
+
};
|
|
447
|
+
const inferenceConfig = nested("inferenceConfig");
|
|
448
|
+
if (inferenceConfig)
|
|
449
|
+
return {
|
|
450
|
+
...record,
|
|
451
|
+
inferenceConfig: lowered(inferenceConfig, ["maxTokens"])
|
|
452
|
+
};
|
|
453
|
+
const capped = lowered(record, [
|
|
454
|
+
"max_output_tokens",
|
|
455
|
+
"max_completion_tokens",
|
|
456
|
+
"max_tokens",
|
|
457
|
+
"maxTokens"
|
|
458
|
+
]);
|
|
459
|
+
const thinking = nested("thinking");
|
|
460
|
+
if (thinking && typeof capped.max_tokens === "number" && typeof thinking.budget_tokens === "number" && thinking.budget_tokens >= capped.max_tokens) {
|
|
461
|
+
capped.thinking = {
|
|
462
|
+
...thinking,
|
|
463
|
+
budget_tokens: Math.max(1024, capped.max_tokens - 1024)
|
|
464
|
+
};
|
|
465
|
+
}
|
|
466
|
+
return capped;
|
|
467
|
+
}
|
|
468
|
+
function withoutProviderTemperature2(payload) {
|
|
469
|
+
if (!payload || typeof payload !== "object" || Array.isArray(payload))
|
|
470
|
+
return payload;
|
|
471
|
+
const clean = { ...payload };
|
|
472
|
+
delete clean.temperature;
|
|
473
|
+
for (const key of ["generationConfig", "inferenceConfig"]) {
|
|
474
|
+
const nested = clean[key];
|
|
475
|
+
if (nested && typeof nested === "object" && !Array.isArray(nested)) {
|
|
476
|
+
const options = { ...nested };
|
|
477
|
+
delete options.temperature;
|
|
478
|
+
clean[key] = options;
|
|
479
|
+
}
|
|
480
|
+
}
|
|
481
|
+
return clean;
|
|
482
|
+
}
|
|
483
|
+
function historianCalibrationExtension(pi) {
|
|
484
|
+
const intendedPath = process.env.MAGIC_CONTEXT_SUBAGENT_PROMPT_FILE;
|
|
485
|
+
if (intendedPath) {
|
|
486
|
+
const intended = readFileSync2(intendedPath, "utf8");
|
|
487
|
+
pi.on("context", (_event, ctx) => {
|
|
488
|
+
const effective = readHostSystemPrompt(ctx);
|
|
489
|
+
if (effective === undefined)
|
|
490
|
+
return;
|
|
491
|
+
process.stdout.write(`${JSON.stringify({ type: "mc_system_prompt", ...promptFingerprint(effective), containsIntended: effective.includes(intended) })}
|
|
364
492
|
`);
|
|
493
|
+
});
|
|
365
494
|
}
|
|
366
|
-
|
|
495
|
+
const provenanceOnly = process.env.MAGIC_CONTEXT_SUBAGENT_PROVENANCE_ONLY === "1";
|
|
496
|
+
const maxOutputTokens = finiteNumber(process.env[HISTORIAN_MAX_OUTPUT_TOKENS_ENV2]);
|
|
497
|
+
const outputCeiling = finiteNumber(process.env[SUBAGENT_OUTPUT_CEILING_ENV2]);
|
|
498
|
+
if (provenanceOnly) {
|
|
499
|
+
if (maxOutputTokens === undefined && outputCeiling === undefined)
|
|
500
|
+
return;
|
|
501
|
+
pi.on("before_provider_request", (event) => ceilProviderOutputTokens2(calibrateHistorianProviderPayload2(event.payload, undefined, maxOutputTokens), outputCeiling));
|
|
502
|
+
return;
|
|
503
|
+
}
|
|
504
|
+
const temperature = finiteNumber(process.env[HISTORIAN_TEMPERATURE_ENV2]);
|
|
505
|
+
if (temperature === undefined && maxOutputTokens === undefined) {
|
|
506
|
+
if (outputCeiling === undefined)
|
|
507
|
+
return;
|
|
508
|
+
pi.on("before_provider_request", (event) => ceilProviderOutputTokens2(event.payload, outputCeiling));
|
|
509
|
+
return;
|
|
510
|
+
}
|
|
511
|
+
pi.on("before_provider_request", (event, ctx) => {
|
|
512
|
+
const model = ctx.model;
|
|
513
|
+
const key = `${model?.provider ?? "unknown"}/${model?.id ?? "unknown"}`;
|
|
514
|
+
const sampling = subagentTemperature(key, model, temperature, (_message, decision) => {
|
|
515
|
+
process.stdout.write(`${JSON.stringify({ type: "mc_temperature", model: key, supported: decision.supported ?? null, source: decision.source, ...decision.entry ? { entry: decision.entry } : {} })}
|
|
516
|
+
`);
|
|
517
|
+
});
|
|
518
|
+
return ceilProviderOutputTokens2(calibrateHistorianProviderPayload2(sampling === undefined ? withoutProviderTemperature2(event.payload) : event.payload, sampling, maxOutputTokens), outputCeiling);
|
|
519
|
+
});
|
|
367
520
|
}
|
|
368
521
|
|
|
369
|
-
export { DREAMER_PRIMER_INVESTIGATOR_ALLOWED_TOOLS, buildHiddenAgentRegistrations, logTemperatureDecision,
|
|
522
|
+
export { DREAMER_PRIMER_INVESTIGATOR_ALLOWED_TOOLS, buildHiddenAgentRegistrations, logTemperatureDecision, readHostSystemPrompt, subagentStepCap, SubagentTelemetry, HISTORIAN_TEMPERATURE_ENV2, HISTORIAN_MAX_OUTPUT_TOKENS_ENV2, SUBAGENT_OUTPUT_CEILING_ENV2, calibrateHistorianProviderPayload2, ceilProviderOutputTokens2, withoutProviderTemperature2, historianCalibrationExtension };
|
|
@@ -5399,18 +5399,18 @@ var OmpEntryObjectSchema = object({
|
|
|
5399
5399
|
}).strict();
|
|
5400
5400
|
var OmpEntrySchema = union([string2(), OmpEntryObjectSchema]);
|
|
5401
5401
|
var OpenCodeHarnessBlockSchema = object({
|
|
5402
|
-
model: OcEntrySchema.optional().describe("Primary OpenCode model entry."),
|
|
5403
|
-
fallback_models: array(OcEntrySchema).optional().describe("Ordered fallback OpenCode entries.
|
|
5402
|
+
model: OcEntrySchema.optional().describe("Primary historian OpenCode model entry. When set, only this model and fallback_models are used; exhaustion reports failure, never the session model. When absent, OpenCode inherits the session model."),
|
|
5403
|
+
fallback_models: array(OcEntrySchema).optional().describe("Ordered historian fallback OpenCode entries. A configured primary never adds the session model after this list. Requires an array; legacy singleton values migrate to a one-element array."),
|
|
5404
5404
|
variant: string2().optional().describe("OpenCode reasoning variant for the primary entry when it declares none. Fallback entries declare variants per-entry.")
|
|
5405
5405
|
}).strict().describe("Strict OpenCode model-resolution block. It accepts no Pi vocabulary.");
|
|
5406
5406
|
var PiHarnessBlockSchema = object({
|
|
5407
|
-
model: PiEntrySchema.optional().describe("Primary Pi model entry."),
|
|
5408
|
-
fallback_models: array(PiEntrySchema).optional().describe("Ordered fallback Pi entries.
|
|
5407
|
+
model: PiEntrySchema.optional().describe("Primary historian Pi model entry. When set, only this model and fallback_models are used; exhaustion reports failure, never the session model. When absent, Pi does not run the historian."),
|
|
5408
|
+
fallback_models: array(PiEntrySchema).optional().describe("Ordered historian fallback Pi entries. The session model is never appended. Requires an array; legacy singleton values migrate to a one-element array."),
|
|
5409
5409
|
thinking_level: PiThinkingLevelSchema.describe("Pi thinking level for the primary entry when it declares none. Fallback entries declare thinking levels per-entry.")
|
|
5410
5410
|
}).strict().describe("Strict Pi model-resolution block. It accepts no OpenCode vocabulary.");
|
|
5411
5411
|
var OmpHarnessBlockSchema = object({
|
|
5412
|
-
model: OmpEntrySchema.optional().describe("Primary OMP model entry."),
|
|
5413
|
-
fallback_models: array(OmpEntrySchema).optional().describe("Ordered fallback OMP entries."),
|
|
5412
|
+
model: OmpEntrySchema.optional().describe("Primary historian OMP model entry. When set, only this model and fallback_models are used; exhaustion reports failure, never the session model. An absent OMP block inherits the Pi block; a selected block without a model does not run the historian."),
|
|
5413
|
+
fallback_models: array(OmpEntrySchema).optional().describe("Ordered historian fallback OMP entries. The session model is never appended."),
|
|
5414
5414
|
thinking_level: OmpThinkingLevelSchema.describe("OMP thinking level for the primary entry when it declares none. Fallback entries declare thinking levels per-entry.")
|
|
5415
5415
|
}).strict().describe("Strict OMP model-resolution block. It accepts no OpenCode vocabulary.");
|
|
5416
5416
|
var OpenCodeTaskExecutionSchema = object({
|
|
@@ -1658,9 +1658,11 @@ function planEmergencyDrop(input) {
|
|
|
1658
1658
|
const workingSpan = Math.max(ceilingTokens - fixedFloor, 0);
|
|
1659
1659
|
const target = fixedFloor + TARGET_FRACTION * workingSpan;
|
|
1660
1660
|
const reclaimTokens = Math.round(currentTotalInputTokens - target);
|
|
1661
|
-
|
|
1661
|
+
if (fixedFloor > ceilingTokens) {
|
|
1662
|
+
return noop(`fixed floor ≈${Math.round(fixedFloor)} already exceeds ceiling ${Math.round(ceilingTokens)}: tool drops cannot reach the target`);
|
|
1663
|
+
}
|
|
1662
1664
|
if (reclaimTokens <= EMERGENCY_REARM_MIN_TOKENS) {
|
|
1663
|
-
return noop(`reclaim<=min (${reclaimTokens} <= ${EMERGENCY_REARM_MIN_TOKENS})
|
|
1665
|
+
return noop(`reclaim<=min (${reclaimTokens} <= ${EMERGENCY_REARM_MIN_TOKENS})`);
|
|
1664
1666
|
}
|
|
1665
1667
|
const cutoff = input.protectedCutoff;
|
|
1666
1668
|
const windowYields = absoluteEmergency;
|
|
@@ -1715,16 +1717,16 @@ function planEmergencyDrop(input) {
|
|
|
1715
1717
|
}
|
|
1716
1718
|
}
|
|
1717
1719
|
if (selected.length === 0) {
|
|
1718
|
-
return noop(
|
|
1720
|
+
return noop("no-candidates");
|
|
1719
1721
|
}
|
|
1720
1722
|
if (!input.passAlreadyPriced && reclaimed < EMERGENCY_MIN_ACHIEVABLE_RECLAIM_TOKENS) {
|
|
1721
|
-
return noop(`achievable reclaim below minimum (${selected.length} tags, reclaim≈${Math.round(reclaimed)} < ${EMERGENCY_MIN_ACHIEVABLE_RECLAIM_TOKENS} against gap ${reclaimTokens})
|
|
1723
|
+
return noop(`achievable reclaim below minimum (${selected.length} tags, reclaim≈${Math.round(reclaimed)} < ${EMERGENCY_MIN_ACHIEVABLE_RECLAIM_TOKENS} against gap ${reclaimTokens})`);
|
|
1722
1724
|
}
|
|
1723
1725
|
return {
|
|
1724
1726
|
shouldDrop: true,
|
|
1725
1727
|
tagNumbers: selected,
|
|
1726
1728
|
reclaimTokens,
|
|
1727
|
-
reason: `tiered drop: ${selected.length} tags, reclaim≈${reclaimed}/${reclaimTokens} tokens (floor≈${fixedFloor}, ceiling=${Math.round(ceilingTokens)})
|
|
1729
|
+
reason: `tiered drop: ${selected.length} tags, reclaim≈${reclaimed}/${reclaimTokens} tokens (floor≈${fixedFloor}, ceiling=${Math.round(ceilingTokens)})`
|
|
1728
1730
|
};
|
|
1729
1731
|
}
|
|
1730
1732
|
function measureEmergencyTag(tag, target, calibration, skeleton) {
|
|
@@ -18,14 +18,14 @@ import {
|
|
|
18
18
|
withoutSqliteTransformPass2
|
|
19
19
|
} from "./index-asrxgdt4.js";
|
|
20
20
|
import {
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
} from "./index-
|
|
21
|
+
recordMessageFtsRowid2,
|
|
22
|
+
recordIndexedMessageTime2,
|
|
23
|
+
messageFtsOrdinalRangeIsMapped2,
|
|
24
|
+
deleteUnmappedMessageFtsRows2
|
|
25
|
+
} from "./index-s1xn8qfw.js";
|
|
26
26
|
import {
|
|
27
27
|
deleteChunkEmbedBackoffForSession
|
|
28
|
-
} from "./index-
|
|
28
|
+
} from "./index-afy3dzf9.js";
|
|
29
29
|
import {
|
|
30
30
|
closeQuietly,
|
|
31
31
|
TAG_TOMBSTONE_OWNER_PREFIX,
|
|
@@ -37,7 +37,7 @@ import {
|
|
|
37
37
|
} from "./index-jrap43gc.js";
|
|
38
38
|
import {
|
|
39
39
|
cleanUserText2
|
|
40
|
-
} from "./index-
|
|
40
|
+
} from "./index-44h64qza.js";
|
|
41
41
|
import {
|
|
42
42
|
logSlowWriteTransaction
|
|
43
43
|
} from "./index-d4dt19md.js";
|
|
@@ -616,7 +616,7 @@ function deleteSessionScopedRows(db, sessionIds, harness, options = {}) {
|
|
|
616
616
|
FROM message_fts_rowid_map
|
|
617
617
|
WHERE session_id IN (${placeholders})
|
|
618
618
|
)`).run(...deletableSessionIds);
|
|
619
|
-
|
|
619
|
+
deleteUnmappedMessageFtsRows2(db, deletableSessionIds);
|
|
620
620
|
continue;
|
|
621
621
|
}
|
|
622
622
|
const predicates = [`session_id IN (${placeholders})`];
|
|
@@ -887,7 +887,7 @@ function getDeleteMessageFtsMapStatement(db) {
|
|
|
887
887
|
}
|
|
888
888
|
function insertMessageFtsRow(db, sessionId, messageOrdinal, messageId, role, content, messageTimeMs) {
|
|
889
889
|
const result = getInsertMessageStatement(db).run(sessionId, messageOrdinal, messageId, role, content);
|
|
890
|
-
|
|
890
|
+
recordMessageFtsRowid2(db, sessionId, messageOrdinal, result.lastInsertRowid, messageTimeMs ?? null);
|
|
891
891
|
}
|
|
892
892
|
function normalizeSourceVersion(version) {
|
|
893
893
|
if (typeof version === "number")
|
|
@@ -987,7 +987,7 @@ function deleteIndexedMessage2(db, sessionId, messageId) {
|
|
|
987
987
|
function clearIndexedMessagesInTransaction2(db, sessionId) {
|
|
988
988
|
deleteMessageIndexSkips(db, sessionId);
|
|
989
989
|
getDeleteFtsStatement(db).run(sessionId);
|
|
990
|
-
|
|
990
|
+
deleteUnmappedMessageFtsRows2(db, [sessionId]);
|
|
991
991
|
getDeleteFtsMapStatement(db).run(sessionId);
|
|
992
992
|
getDeleteMessageSourceStatement(db).run(sessionId);
|
|
993
993
|
getDeleteIndexStatement(db).run(sessionId);
|
|
@@ -1018,10 +1018,10 @@ function indexSingleMessageInTransaction(db, sessionId, message, now, dirtyFloor
|
|
|
1018
1018
|
const dirtyFloor = getDirtyIndexFloor(db, sessionId);
|
|
1019
1019
|
if (message.ordinal <= currentWatermark) {
|
|
1020
1020
|
if (isIndexSourceCurrent(db, sessionId, source)) {
|
|
1021
|
-
|
|
1021
|
+
recordIndexedMessageTime2(db, sessionId, message.ordinal, message.createdAt);
|
|
1022
1022
|
return false;
|
|
1023
1023
|
}
|
|
1024
|
-
if (!
|
|
1024
|
+
if (!messageFtsOrdinalRangeIsMapped2(db, sessionId, message.ordinal, message.ordinal)) {
|
|
1025
1025
|
return false;
|
|
1026
1026
|
}
|
|
1027
1027
|
getDeleteMessageFtsStatement(db).run(sessionId, message.id);
|
|
@@ -1051,7 +1051,7 @@ function indexSingleMessage2(db, sessionId, message, shouldCommit = () => true,
|
|
|
1051
1051
|
const source = preparedSource ?? getMessageSourceSnapshot(message);
|
|
1052
1052
|
const currentWatermark = getLastIndexedOrdinal2(db, sessionId);
|
|
1053
1053
|
if (message.ordinal <= currentWatermark && isIndexSourceCurrent(db, sessionId, source)) {
|
|
1054
|
-
|
|
1054
|
+
recordIndexedMessageTime2(db, sessionId, message.ordinal, message.createdAt);
|
|
1055
1055
|
return false;
|
|
1056
1056
|
}
|
|
1057
1057
|
const dirtyFloorBeforeAttempt = getDirtyIndexFloor(db, sessionId);
|
|
@@ -1092,7 +1092,7 @@ function indexItemsAfterOrdinal(db, sessionId, items, finalWatermark, toSource)
|
|
|
1092
1092
|
const currentWatermark = getLastIndexedOrdinal2(db, sessionId);
|
|
1093
1093
|
const dirtyFloor = getDirtyIndexFloor(db, sessionId);
|
|
1094
1094
|
const effectiveWatermark = dirtyFloor === null ? currentWatermark : Math.min(currentWatermark, Math.max(0, dirtyFloor - 1));
|
|
1095
|
-
if (dirtyFloor !== null && dirtyFloor <= currentWatermark && !
|
|
1095
|
+
if (dirtyFloor !== null && dirtyFloor <= currentWatermark && !messageFtsOrdinalRangeIsMapped2(db, sessionId, dirtyFloor, Math.min(currentWatermark, finalWatermark))) {
|
|
1096
1096
|
db.exec("COMMIT");
|
|
1097
1097
|
committed = true;
|
|
1098
1098
|
logSlowWriteTransaction("message_index_reconcile", transactionStartedAt);
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import {
|
|
2
|
+
withMessageFtsSessionFilter
|
|
3
|
+
} from "./index-1cvyaw9a.js";
|
|
4
|
+
import {
|
|
5
|
+
estimateTokens
|
|
6
|
+
} from "./index-g4jf772p.js";
|
|
7
|
+
import {
|
|
8
|
+
readRawSessionMessageIdOrdinalsForRange2
|
|
9
|
+
} from "./index-44h64qza.js";
|
|
10
|
+
|
|
11
|
+
// ../plugin/src/tools/ctx-expand/search-archive.ts
|
|
12
|
+
var archiveReadsOnThisThread = 0;
|
|
13
|
+
function renderSearchArchive2(db, sessionId, start, end, tokenBudget, archiveOnly = false, readRawIds = () => readRawSessionMessageIdOrdinalsForRange2(sessionId, start, end)) {
|
|
14
|
+
archiveReadsOnThisThread += 1;
|
|
15
|
+
return withMessageFtsSessionFilter(db, sessionId, (sessionFirst) => {
|
|
16
|
+
let rawIds;
|
|
17
|
+
const rows = db.prepare(sessionFirst ? `SELECT f.message_ordinal AS ordinal, f.message_id AS id, f.role, f.content
|
|
18
|
+
FROM message_fts_rowid_map AS m
|
|
19
|
+
JOIN message_history_fts AS f ON f.rowid = m.fts_rowid
|
|
20
|
+
WHERE m.session_id = ?1 AND m.message_ordinal BETWEEN ?2 AND ?3
|
|
21
|
+
AND f.session_id = ?1 AND f.message_ordinal BETWEEN ?2 AND ?3
|
|
22
|
+
ORDER BY m.message_ordinal` : `SELECT message_ordinal AS ordinal, message_id AS id, role, content
|
|
23
|
+
FROM message_history_fts
|
|
24
|
+
WHERE session_id = ?1 AND message_ordinal BETWEEN ?2 AND ?3
|
|
25
|
+
ORDER BY CAST(message_ordinal AS INTEGER)`);
|
|
26
|
+
const blocks = [];
|
|
27
|
+
let tokens = 0;
|
|
28
|
+
let lastOrdinal = start - 1;
|
|
29
|
+
let truncated = false;
|
|
30
|
+
for (const row of rows.iterate(sessionId, start, end)) {
|
|
31
|
+
if (!row.content)
|
|
32
|
+
continue;
|
|
33
|
+
if (!archiveOnly) {
|
|
34
|
+
rawIds ??= readRawIds();
|
|
35
|
+
if (rawIds.get(row.id) === row.ordinal)
|
|
36
|
+
continue;
|
|
37
|
+
}
|
|
38
|
+
const block = `[${row.ordinal}] ${row.role === "user" ? "U" : "A"}: ${row.content}`;
|
|
39
|
+
const cost = estimateTokens(block);
|
|
40
|
+
if (blocks.length > 0 && tokens + cost > tokenBudget) {
|
|
41
|
+
truncated = true;
|
|
42
|
+
break;
|
|
43
|
+
}
|
|
44
|
+
blocks.push(block);
|
|
45
|
+
tokens += cost;
|
|
46
|
+
lastOrdinal = row.ordinal;
|
|
47
|
+
}
|
|
48
|
+
if (blocks.length === 0)
|
|
49
|
+
return null;
|
|
50
|
+
const text = [
|
|
51
|
+
"From the search archive: indexed text only; no surrounding messages or tool output.",
|
|
52
|
+
"",
|
|
53
|
+
...blocks
|
|
54
|
+
];
|
|
55
|
+
if (truncated) {
|
|
56
|
+
text.push("", `Truncated at message ${lastOrdinal} (budget: ~${tokenBudget} tokens). Call ctx_expand(start=${lastOrdinal + 1}, end=${end}) for more archived text.`);
|
|
57
|
+
}
|
|
58
|
+
return text.join(`
|
|
59
|
+
`);
|
|
60
|
+
});
|
|
61
|
+
}
|
|
62
|
+
export { renderSearchArchive2 };
|
|
@@ -38,20 +38,20 @@ import {
|
|
|
38
38
|
countSessionCompartmentEmbedCoverage,
|
|
39
39
|
countSessionCompartmentEmbedCoveragePolite,
|
|
40
40
|
countUnembeddedSessionCompartmentsPolite
|
|
41
|
-
} from "./index-
|
|
41
|
+
} from "./index-afy3dzf9.js";
|
|
42
42
|
import {
|
|
43
43
|
DEFAULT_LOCAL_EMBEDDING_MODEL
|
|
44
|
-
} from "./index-
|
|
44
|
+
} from "./index-bxmf4dhr.js";
|
|
45
45
|
import {
|
|
46
46
|
embeddingModelsMatch,
|
|
47
47
|
resolveEmbeddingTextPrefixes,
|
|
48
48
|
getEmbeddingProviderIdentity,
|
|
49
49
|
dominantEmbeddingFailure,
|
|
50
50
|
LocalEmbeddingProvider
|
|
51
|
-
} from "./index-
|
|
51
|
+
} from "./index-657xh9w4.js";
|
|
52
52
|
import {
|
|
53
53
|
setBootQuietPeriodForTests
|
|
54
|
-
} from "./index-
|
|
54
|
+
} from "./index-y5c73caa.js";
|
|
55
55
|
import {
|
|
56
56
|
isEmbeddingHostBusy2
|
|
57
57
|
} from "./index-9p40dgkm.js";
|
|
@@ -62,7 +62,7 @@ import {
|
|
|
62
62
|
acquireGitSweepLease,
|
|
63
63
|
renewGitSweepLease,
|
|
64
64
|
releaseGitSweepLease
|
|
65
|
-
} from "./index-
|
|
65
|
+
} from "./index-mx59cve4.js";
|
|
66
66
|
import {
|
|
67
67
|
saveEmbeddingIfHashMatches,
|
|
68
68
|
getMemoryEmbedCoverage,
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
DEFAULT_LOCAL_EMBEDDING_MODEL
|
|
3
|
-
} from "./index-
|
|
3
|
+
} from "./index-bxmf4dhr.js";
|
|
4
4
|
import {
|
|
5
5
|
log2
|
|
6
6
|
} from "./index-7bqrfyny.js";
|
|
@@ -10,12 +10,12 @@ import {
|
|
|
10
10
|
} from "./index-6dz2ed1v.js";
|
|
11
11
|
import {
|
|
12
12
|
LocalEmbeddingProvider
|
|
13
|
-
} from "./index-
|
|
13
|
+
} from "./index-657xh9w4.js";
|
|
14
14
|
import {
|
|
15
15
|
OpenAICompatibleEmbeddingProvider,
|
|
16
16
|
getProjectEmbeddingSnapshot2,
|
|
17
17
|
embedBatchForProject2
|
|
18
|
-
} from "./index-
|
|
18
|
+
} from "./index-n3r2d2pr.js";
|
|
19
19
|
import {
|
|
20
20
|
isEmbeddingHostBusy2
|
|
21
21
|
} from "./index-9p40dgkm.js";
|
|
@@ -43,6 +43,7 @@ function ensureExitHook() {
|
|
|
43
43
|
function runSessionWarmupWorker(input, options) {
|
|
44
44
|
ensureExitHook();
|
|
45
45
|
return new Promise((resolve) => {
|
|
46
|
+
const openCancellation = new Int32Array(new SharedArrayBuffer(Int32Array.BYTES_PER_ELEMENT));
|
|
46
47
|
let worker;
|
|
47
48
|
let deadline;
|
|
48
49
|
let settled = false;
|
|
@@ -50,7 +51,9 @@ function runSessionWarmupWorker(input, options) {
|
|
|
50
51
|
if (settled)
|
|
51
52
|
return;
|
|
52
53
|
settled = true;
|
|
54
|
+
Atomics.store(openCancellation, 0, 1);
|
|
53
55
|
clearTimeout(deadline);
|
|
56
|
+
options.signal?.removeEventListener("abort", abort);
|
|
54
57
|
if (worker) {
|
|
55
58
|
activeWorkers.delete(worker);
|
|
56
59
|
const stopped = worker.terminate();
|
|
@@ -69,8 +72,16 @@ function runSessionWarmupWorker(input, options) {
|
|
|
69
72
|
deadline = setTimeout(() => finish({ failure, detail: `no reply within ${ms} ms` }), ms);
|
|
70
73
|
deadline.unref?.();
|
|
71
74
|
};
|
|
75
|
+
const abort = () => finish({ failure: "stopped", detail: "stopped" });
|
|
76
|
+
if (options.signal?.aborted) {
|
|
77
|
+
abort();
|
|
78
|
+
return;
|
|
79
|
+
}
|
|
80
|
+
options.signal?.addEventListener("abort", abort, { once: true });
|
|
72
81
|
try {
|
|
73
|
-
worker = inRootAsyncContext2(() => new Worker(options.entry ?? defaultEntry(), {
|
|
82
|
+
worker = inRootAsyncContext2(() => new Worker(options.entry ?? defaultEntry(), {
|
|
83
|
+
workerData: { ...input, openCancellation: openCancellation.buffer }
|
|
84
|
+
}));
|
|
74
85
|
} catch (error) {
|
|
75
86
|
finish({ failure: "start-failed", detail: String(error) });
|
|
76
87
|
return;
|
|
@@ -93,8 +104,17 @@ function runSessionWarmupWorker(input, options) {
|
|
|
93
104
|
finish({ orphanSweep: reply.result });
|
|
94
105
|
else if (reply.kind === "exact-ordinal-ids")
|
|
95
106
|
finish({ exactOrdinalIds: reply.ids });
|
|
107
|
+
else if (reply.kind === "search-archive")
|
|
108
|
+
finish({ searchArchive: reply.text });
|
|
109
|
+
else if (reply.kind === "fts-rowid-backfill")
|
|
110
|
+
finish({ ftsRowidBackfill: true });
|
|
96
111
|
else
|
|
97
|
-
finish({
|
|
112
|
+
finish({
|
|
113
|
+
failure: "worker-error",
|
|
114
|
+
detail: reply.error,
|
|
115
|
+
code: reply.code,
|
|
116
|
+
errcode: reply.errcode
|
|
117
|
+
});
|
|
98
118
|
});
|
|
99
119
|
worker.on("error", (error) => finish({ failure: "worker-error", detail: String(error) }));
|
|
100
120
|
worker.on("exit", (code) => finish({ failure: "exited", detail: `exit code ${code}` }));
|
|
@@ -1,26 +1,15 @@
|
|
|
1
1
|
import {
|
|
2
|
+
getSqliteDatabasePath2,
|
|
2
3
|
withSqliteBackgroundWriter2,
|
|
3
4
|
withoutSqliteTransformPass2
|
|
4
5
|
} from "./index-asrxgdt4.js";
|
|
6
|
+
import {
|
|
7
|
+
runSessionWarmupWorker
|
|
8
|
+
} from "./index-qqxqwqhp.js";
|
|
5
9
|
import {
|
|
6
10
|
logSlowWriteTransaction
|
|
7
11
|
} from "./index-d4dt19md.js";
|
|
8
12
|
|
|
9
|
-
// ../plugin/src/shared/project-directory-key.ts
|
|
10
|
-
import path from "node:path";
|
|
11
|
-
function projectDirectoryKey(directory) {
|
|
12
|
-
const slashed = directory.replaceAll("\\", "/").replace(/^\/\/\?\/UNC\//i, "//").replace(/^\/\/\?\//, "");
|
|
13
|
-
if (/^[a-z]:\//i.test(slashed) || slashed.startsWith("//")) {
|
|
14
|
-
return path.win32.normalize(slashed).replaceAll("\\", "/").replace(/\/$/, "").toLowerCase();
|
|
15
|
-
}
|
|
16
|
-
return path.resolve(directory);
|
|
17
|
-
}
|
|
18
|
-
function sameProjectDirectory(a, b) {
|
|
19
|
-
if (a.length === 0 || b.length === 0)
|
|
20
|
-
return false;
|
|
21
|
-
return projectDirectoryKey(a) === projectDirectoryKey(b);
|
|
22
|
-
}
|
|
23
|
-
|
|
24
13
|
// ../plugin/src/features/magic-context/message-fts-rowid-map.ts
|
|
25
14
|
import { createHash } from "node:crypto";
|
|
26
15
|
var MESSAGE_FTS_ROWID_MAP_BACKFILL_BATCH_SIZE = 100;
|
|
@@ -56,7 +45,7 @@ function getBackfillState(db) {
|
|
|
56
45
|
completed: row?.completed === 1
|
|
57
46
|
};
|
|
58
47
|
}
|
|
59
|
-
function
|
|
48
|
+
function recordMessageFtsRowid2(db, sessionId, messageOrdinal, ftsRowid, messageTimeMs = null) {
|
|
60
49
|
const numericRowid = Number(ftsRowid);
|
|
61
50
|
if (!Number.isSafeInteger(numericRowid) || numericRowid <= 0) {
|
|
62
51
|
throw new Error(`invalid message FTS rowid: ${String(ftsRowid)}`);
|
|
@@ -64,7 +53,7 @@ function recordMessageFtsRowid(db, sessionId, messageOrdinal, ftsRowid, messageT
|
|
|
64
53
|
const normalizedTime = typeof messageTimeMs === "number" && Number.isSafeInteger(messageTimeMs) && messageTimeMs >= 0 ? messageTimeMs : null;
|
|
65
54
|
getUpsertMapStatement(db).run(sessionId, messageOrdinal, numericRowid, normalizedTime);
|
|
66
55
|
}
|
|
67
|
-
function
|
|
56
|
+
function recordIndexedMessageTime2(db, sessionId, messageOrdinal, messageTimeMs) {
|
|
68
57
|
if (typeof messageTimeMs !== "number" || !Number.isSafeInteger(messageTimeMs) || messageTimeMs < 0) {
|
|
69
58
|
return;
|
|
70
59
|
}
|
|
@@ -106,7 +95,7 @@ function backfillMessageFtsRowidMapBatch(db, batchSize = MESSAGE_FTS_ROWID_MAP_B
|
|
|
106
95
|
watermarkRowid = ftsRowid;
|
|
107
96
|
}
|
|
108
97
|
if (typeof row.sessionId === "string" && Number.isSafeInteger(messageOrdinal) && messageOrdinal >= 0 && Number.isSafeInteger(ftsRowid) && ftsRowid > 0) {
|
|
109
|
-
|
|
98
|
+
recordMessageFtsRowid2(db, row.sessionId, messageOrdinal, ftsRowid);
|
|
110
99
|
}
|
|
111
100
|
}
|
|
112
101
|
const completed = rows.length < boundedBatchSize;
|
|
@@ -122,26 +111,33 @@ function backfillMessageFtsRowidMapBatch(db, batchSize = MESSAGE_FTS_ROWID_MAP_B
|
|
|
122
111
|
logSlowWriteTransaction("message_fts_rowid_backfill", transactionStartedAt);
|
|
123
112
|
return progress;
|
|
124
113
|
}
|
|
125
|
-
async function
|
|
114
|
+
async function runMessageFtsRowidMapBackfill2(db, signal) {
|
|
126
115
|
await new Promise((resolve) => setTimeout(resolve, 0));
|
|
127
|
-
|
|
116
|
+
while (!signal?.aborted) {
|
|
128
117
|
const progress = backfillMessageFtsRowidMapBatch(db);
|
|
129
118
|
if (progress.completed)
|
|
130
119
|
return;
|
|
131
120
|
await new Promise((resolve) => setTimeout(resolve, 0));
|
|
132
121
|
}
|
|
133
122
|
}
|
|
134
|
-
function
|
|
123
|
+
function startMessageFtsRowidMapBackfill2(db, signal) {
|
|
135
124
|
const active = activeBackfills.get(db);
|
|
136
125
|
if (active)
|
|
137
126
|
return active;
|
|
138
|
-
const
|
|
127
|
+
const path = getSqliteDatabasePath2(db);
|
|
128
|
+
const run = (path ? runSessionWarmupWorker({ path, sessionId: "", ftsRowidBackfill: true }, { signal }).then((result) => {
|
|
129
|
+
if (signal?.aborted)
|
|
130
|
+
return;
|
|
131
|
+
if (!("ftsRowidBackfill" in result)) {
|
|
132
|
+
throw Object.assign(new Error(`FTS rowid backfill unavailable: ${"failure" in result ? result.detail : "unexpected reply"}`), "failure" in result ? { code: result.code, errcode: result.errcode } : {});
|
|
133
|
+
}
|
|
134
|
+
}) : withoutSqliteTransformPass2(() => withSqliteBackgroundWriter2(() => runMessageFtsRowidMapBackfill2(db, signal)))).finally(() => {
|
|
139
135
|
activeBackfills.delete(db);
|
|
140
136
|
});
|
|
141
137
|
activeBackfills.set(db, run);
|
|
142
138
|
return run;
|
|
143
139
|
}
|
|
144
|
-
function
|
|
140
|
+
function messageFtsOrdinalRangeIsMapped2(db, sessionId, startOrdinal, endOrdinal) {
|
|
145
141
|
if (endOrdinal < startOrdinal)
|
|
146
142
|
return true;
|
|
147
143
|
if (getBackfillState(db).completed)
|
|
@@ -172,7 +168,7 @@ function messageFtsOrdinalRangeIsMapped(db, sessionId, startOrdinal, endOrdinal)
|
|
|
172
168
|
const ordinalCount = endOrdinal - startOrdinal + 1;
|
|
173
169
|
return row?.sourceOrdinalCount === ordinalCount && row.expectedMapCount === row.mappedCount;
|
|
174
170
|
}
|
|
175
|
-
function
|
|
171
|
+
function deleteUnmappedMessageFtsRows2(db, sessionIds) {
|
|
176
172
|
if (sessionIds.length === 0)
|
|
177
173
|
return;
|
|
178
174
|
const state = getBackfillState(db);
|
|
@@ -183,4 +179,4 @@ function deleteUnmappedMessageFtsRows(db, sessionIds) {
|
|
|
183
179
|
WHERE rowid > ? AND session_id IN (${placeholders})`).run(state.watermarkRowid, ...sessionIds);
|
|
184
180
|
}
|
|
185
181
|
|
|
186
|
-
export {
|
|
182
|
+
export { recordMessageFtsRowid2, recordIndexedMessageTime2, runMessageFtsRowidMapBackfill2, startMessageFtsRowidMapBackfill2, messageFtsOrdinalRangeIsMapped2, deleteUnmappedMessageFtsRows2 };
|