@cortexkit/pi-magic-context 0.30.2 → 0.30.3

package/dist/index.js

@@ -70,6 +70,9 @@ function getMagicContextTempDir(harness = getHarness()) {
   return path2.join(os.tmpdir(), harness, "magic-context");
 }
 function getMagicContextLogPath(harness = getHarness()) {
+  const envPath = process.env.MAGIC_CONTEXT_LOG_PATH?.trim();
+  if (envPath)
+    return envPath;
   return path2.join(getMagicContextTempDir(harness), "magic-context.log");
 }
 function getProjectMagicContextDir(directory) {
@@ -172658,6 +172661,94 @@ init_logger();
 
 // ../plugin/src/features/magic-context/compartment-chunk-embedding.ts
 import { createHash as createHash6 } from "node:crypto";
+init_logger();
+
+// ../plugin/src/features/magic-context/recursive-text-splitter.ts
+var DEFAULT_SEPARATORS = [`
+
+`, `
+`, " ", ""];
+function splitOnSeparator(text, separator) {
+  const splits = separator ? text.split(separator) : text.split("");
+  return splits.filter((s) => s !== "");
+}
+function mergeSplits(splits, separator, chunkSize, lengthFunction) {
+  const docs = [];
+  const currentDoc = [];
+  let total = 0;
+  const joinDocs = (docsToJoin) => {
+    const joined = docsToJoin.join(separator).trim();
+    return joined === "" ? null : joined;
+  };
+  for (const d of splits) {
+    const len = lengthFunction(d);
+    if (total + len + currentDoc.length * separator.length > chunkSize) {
+      if (currentDoc.length > 0) {
+        const doc3 = joinDocs(currentDoc);
+        if (doc3 !== null)
+          docs.push(doc3);
+        while (total > 0 && currentDoc.length > 0) {
+          total -= lengthFunction(currentDoc[0]);
+          currentDoc.shift();
+        }
+      }
+    }
+    currentDoc.push(d);
+    total += len;
+  }
+  const doc2 = joinDocs(currentDoc);
+  if (doc2 !== null)
+    docs.push(doc2);
+  return docs;
+}
+function splitTextRecursive(text, separators, chunkSize, lengthFunction) {
+  const finalChunks = [];
+  let separator = separators[separators.length - 1];
+  let newSeparators;
+  for (let i = 0;i < separators.length; i += 1) {
+    const s = separators[i];
+    if (s === "") {
+      separator = s;
+      break;
+    }
+    if (text.includes(s)) {
+      separator = s;
+      newSeparators = separators.slice(i + 1);
+      break;
+    }
+  }
+  const splits = splitOnSeparator(text, separator);
+  let goodSplits = [];
+  for (const s of splits) {
+    if (lengthFunction(s) < chunkSize) {
+      goodSplits.push(s);
+    } else {
+      if (goodSplits.length) {
+        finalChunks.push(...mergeSplits(goodSplits, separator, chunkSize, lengthFunction));
+        goodSplits = [];
+      }
+      if (!newSeparators) {
+        finalChunks.push(s);
+      } else {
+        finalChunks.push(...splitTextRecursive(s, newSeparators, chunkSize, lengthFunction));
+      }
+    }
+  }
+  if (goodSplits.length) {
+    finalChunks.push(...mergeSplits(goodSplits, separator, chunkSize, lengthFunction));
+  }
+  return finalChunks;
+}
+function recursiveCharacterSplit(text, options) {
+  const chunkSize = options.chunkSize;
+  const lengthFunction = options.lengthFunction ?? ((t) => t.length);
+  const separators = options.separators ?? DEFAULT_SEPARATORS;
+  if (text.length === 0)
+    return [];
+  return splitTextRecursive(text, separators, chunkSize, lengthFunction);
+}
+
+// ../plugin/src/features/magic-context/compartment-chunk-embedding.ts
 var DEFAULT_COMPARTMENT_CHUNK_MAX_INPUT_TOKENS = 512;
 var CHUNK_WINDOW_SAFETY_RATIO = 0.9;
 var loadFtsRowsStatements = new WeakMap;
@@ -172936,6 +173027,19 @@ function chunkCanonicalText(canonicalText, startOrdinal, endOrdinal, maxInputTok
     const lineStart = range?.start ?? startOrdinal;
     const lineEnd = range?.end ?? lineStart;
     const lineTokens = estimateTokens(line);
+    if (lineTokens > effectiveMax) {
+      flush2();
+      for (const slice of splitOversizedLine(line, effectiveMax)) {
+        windows.push({
+          windowIndex: windows.length + 1,
+          startOrdinal: lineStart,
+          endOrdinal: lineEnd,
+          text: slice,
+          chunkHash: hashChunkText(slice)
+        });
+      }
+      continue;
+    }
     if (currentLines.length > 0 && currentTokens + lineTokens > effectiveMax) {
       flush2();
     }
@@ -172949,6 +173053,56 @@ function chunkCanonicalText(canonicalText, startOrdinal, endOrdinal, maxInputTok
   flush2();
   return windows;
 }
+function splitOversizedLine(line, effectiveMax) {
+  let slices = [];
+  try {
+    slices = recursiveCharacterSplit(line, {
+      chunkSize: effectiveMax,
+      lengthFunction: estimateTokens
+    });
+  } catch (error51) {
+    log("[magic-context] recursiveCharacterSplit failed; using char-budget fallback:", error51);
+    slices = [];
+  }
+  if (slices.length === 0) {
+    slices = charBudgetSplit(line, effectiveMax);
+  }
+  const safe = [];
+  const pushChecked = (slice) => {
+    if (estimateTokens(slice) > effectiveMax && slice.length > 1) {
+      safe.push(...charBudgetSplit(slice, effectiveMax));
+      return;
+    }
+    safe.push(slice);
+  };
+  for (const slice of slices) {
+    if (estimateTokens(slice) <= effectiveMax) {
+      safe.push(slice);
+    } else {
+      for (const sub of charBudgetSplit(slice, effectiveMax))
+        pushChecked(sub);
+    }
+  }
+  return safe.filter((s) => s.length > 0);
+}
+function charBudgetSplit(text, effectiveMax) {
+  const totalTokens = Math.max(1, estimateTokens(text));
+  const charsPerToken = Math.max(1, Math.floor(text.length / totalTokens));
+  const sliceChars = Math.max(1, effectiveMax * charsPerToken);
+  const out = [];
+  let pos = 0;
+  while (pos < text.length) {
+    let end = Math.min(text.length, pos + sliceChars);
+    let slice = text.slice(pos, end);
+    while (slice.length > 1 && estimateTokens(slice) > effectiveMax) {
+      end = pos + Math.max(1, Math.floor((end - pos) / 2));
+      slice = text.slice(pos, end);
+    }
+    out.push(slice);
+    pos = end;
+  }
+  return out;
+}
 function getExistingChunkHashes(db, compartmentId, modelId, projectPath) {
   const scoped = typeof projectPath === "string" && projectPath.length > 0;
   const rows = scoped ? getExistingHashStatement(db, true).all(compartmentId, modelId, projectPath) : getExistingHashStatement(db, false).all(compartmentId, modelId);
@@ -173681,6 +173835,7 @@ class OpenAICompatibleEmbeddingProvider {
     if (texts.length === 0) {
       return [];
     }
+    const requestTexts = texts.map((t) => t.trim().length === 0 ? " " : t);
     if (!await this.initialize()) {
       return Array.from({ length: texts.length }, () => null);
     }
@@ -173712,7 +173867,7 @@ class OpenAICompatibleEmbeddingProvider {
         },
         body: JSON.stringify({
           model: this.model,
-          input: texts,
+          input: requestTexts,
           ...inputTypeForRequest ? { input_type: inputTypeForRequest } : {},
           ...this.truncate ? { truncate: this.truncate } : {}
         }),
@@ -174799,6 +174954,32 @@ async function embedBatchForProject(projectIdentity, texts, signal, purpose = "p
   }
   return { vectors, modelId, generation };
 }
+async function embedTextsWindowBounded(projectIdentity, texts, signal) {
+  if (texts.length <= MAX_WINDOWS_PER_EMBED_CALL) {
+    return embedBatchForProject(projectIdentity, texts, signal);
+  }
+  const vectors = [];
+  let modelId = null;
+  let generation = null;
+  for (let start = 0;start < texts.length; start += MAX_WINDOWS_PER_EMBED_CALL) {
+    if (signal?.aborted)
+      return null;
+    const sub = texts.slice(start, start + MAX_WINDOWS_PER_EMBED_CALL);
+    const result = await embedBatchForProject(projectIdentity, sub, signal);
+    if (!result)
+      return null;
+    if (modelId === null) {
+      modelId = result.modelId;
+      generation = result.generation;
+    } else if (result.modelId !== modelId || result.generation !== generation) {
+      return null;
+    }
+    vectors.push(...result.vectors);
+  }
+  if (modelId === null || generation === null)
+    return null;
+  return { vectors, modelId, generation };
+}
 function isUnembeddedMemoryRow(row) {
   if (row === null || typeof row !== "object")
     return false;
@@ -174932,7 +175113,7 @@ async function embedCandidateChunkBatch(db, projectIdentity, modelId, candidates
       let result = null;
       const attemptStart = Date.now();
       try {
-        result = await embedBatchForProject(projectIdentity, texts, signal);
+        result = await embedTextsWindowBounded(projectIdentity, texts, signal);
       } catch (error51) {
         log("[magic-context] failed to proactively embed compartment chunks:", error51);
       }
@@ -183553,6 +183734,7 @@ async function loadDefaultPiSessionDeps() {
 }
 
 // src/dreamer/retrospective-raw-provider-pi.ts
+import { readFileSync as readFileSync9 } from "node:fs";
 import { resolve as resolve2 } from "node:path";
 var PI_CODING_AGENT_MODULE2 = "@earendil-works/pi-coding-agent";
 
@@ -183660,11 +183842,14 @@ function extractPiTextContent(content) {
 }
 async function loadDefaultPiSessionDeps2() {
   const mod = await import(PI_CODING_AGENT_MODULE2);
-  const listSessions = mod.SessionManager?.listSessions;
-  const loadEntriesFromFile = mod.loadEntriesFromFile;
-  if (typeof listSessions !== "function" || typeof loadEntriesFromFile !== "function") {
-    throw new Error("Pi session APIs unavailable: expected SessionManager.listSessions and loadEntriesFromFile");
+  const listSessions = mod.SessionManager?.listAll;
+  if (typeof listSessions !== "function") {
+    throw new Error("Pi session APIs unavailable: expected SessionManager.listAll on pi-coding-agent");
   }
+  const loadEntriesFromFile = mod.loadEntriesFromFile ?? ((filePath) => {
+    const content = readFileSync9(filePath, "utf8");
+    return mod.parseSessionEntries?.(content) ?? [];
+  });
   return {
     listSessions: listSessions.bind(mod.SessionManager),
     loadEntriesFromFile
@@ -199816,7 +200001,7 @@ function formatThresholdPercent(value) {
 // package.json
 var package_default = {
   name: "@cortexkit/pi-magic-context",
-  version: "0.30.2",
+  version: "0.30.3",
   type: "module",
   description: "Pi coding agent extension for Magic Context — cross-session memory and context management",
   main: "dist/index.js",

package/dist/subagent-entry.js

@@ -70,6 +70,9 @@ function getMagicContextTempDir(harness = getHarness()) {
   return path.join(os.tmpdir(), harness, "magic-context");
 }
 function getMagicContextLogPath(harness = getHarness()) {
+  const envPath = process.env.MAGIC_CONTEXT_LOG_PATH?.trim();
+  if (envPath)
+    return envPath;
   return path.join(getMagicContextTempDir(harness), "magic-context.log");
 }
 function getProjectMagicContextDir(directory) {
@@ -160282,6 +160285,94 @@ init_logger();
 
 // ../plugin/src/features/magic-context/compartment-chunk-embedding.ts
 import { createHash as createHash3 } from "node:crypto";
+init_logger();
+
+// ../plugin/src/features/magic-context/recursive-text-splitter.ts
+var DEFAULT_SEPARATORS = [`
+
+`, `
+`, " ", ""];
+function splitOnSeparator(text, separator) {
+  const splits = separator ? text.split(separator) : text.split("");
+  return splits.filter((s) => s !== "");
+}
+function mergeSplits(splits, separator, chunkSize, lengthFunction) {
+  const docs = [];
+  const currentDoc = [];
+  let total = 0;
+  const joinDocs = (docsToJoin) => {
+    const joined = docsToJoin.join(separator).trim();
+    return joined === "" ? null : joined;
+  };
+  for (const d of splits) {
+    const len = lengthFunction(d);
+    if (total + len + currentDoc.length * separator.length > chunkSize) {
+      if (currentDoc.length > 0) {
+        const doc3 = joinDocs(currentDoc);
+        if (doc3 !== null)
+          docs.push(doc3);
+        while (total > 0 && currentDoc.length > 0) {
+          total -= lengthFunction(currentDoc[0]);
+          currentDoc.shift();
+        }
+      }
+    }
+    currentDoc.push(d);
+    total += len;
+  }
+  const doc2 = joinDocs(currentDoc);
+  if (doc2 !== null)
+    docs.push(doc2);
+  return docs;
+}
+function splitTextRecursive(text, separators, chunkSize, lengthFunction) {
+  const finalChunks = [];
+  let separator = separators[separators.length - 1];
+  let newSeparators;
+  for (let i = 0;i < separators.length; i += 1) {
+    const s = separators[i];
+    if (s === "") {
+      separator = s;
+      break;
+    }
+    if (text.includes(s)) {
+      separator = s;
+      newSeparators = separators.slice(i + 1);
+      break;
+    }
+  }
+  const splits = splitOnSeparator(text, separator);
+  let goodSplits = [];
+  for (const s of splits) {
+    if (lengthFunction(s) < chunkSize) {
+      goodSplits.push(s);
+    } else {
+      if (goodSplits.length) {
+        finalChunks.push(...mergeSplits(goodSplits, separator, chunkSize, lengthFunction));
+        goodSplits = [];
+      }
+      if (!newSeparators) {
+        finalChunks.push(s);
+      } else {
+        finalChunks.push(...splitTextRecursive(s, newSeparators, chunkSize, lengthFunction));
+      }
+    }
+  }
+  if (goodSplits.length) {
+    finalChunks.push(...mergeSplits(goodSplits, separator, chunkSize, lengthFunction));
+  }
+  return finalChunks;
+}
+function recursiveCharacterSplit(text, options) {
+  const chunkSize = options.chunkSize;
+  const lengthFunction = options.lengthFunction ?? ((t) => t.length);
+  const separators = options.separators ?? DEFAULT_SEPARATORS;
+  if (text.length === 0)
+    return [];
+  return splitTextRecursive(text, separators, chunkSize, lengthFunction);
+}
+
+// ../plugin/src/features/magic-context/compartment-chunk-embedding.ts
 var DEFAULT_COMPARTMENT_CHUNK_MAX_INPUT_TOKENS = 512;
 var CHUNK_WINDOW_SAFETY_RATIO = 0.9;
 var loadFtsRowsStatements = new WeakMap;
@@ -160560,6 +160651,19 @@ function chunkCanonicalText(canonicalText, startOrdinal, endOrdinal, maxInputTok
     const lineStart = range?.start ?? startOrdinal;
     const lineEnd = range?.end ?? lineStart;
     const lineTokens = estimateTokens(line);
+    if (lineTokens > effectiveMax) {
+      flush2();
+      for (const slice of splitOversizedLine(line, effectiveMax)) {
+        windows.push({
+          windowIndex: windows.length + 1,
+          startOrdinal: lineStart,
+          endOrdinal: lineEnd,
+          text: slice,
+          chunkHash: hashChunkText(slice)
+        });
+      }
+      continue;
+    }
     if (currentLines.length > 0 && currentTokens + lineTokens > effectiveMax) {
       flush2();
     }
@@ -160573,6 +160677,56 @@ function chunkCanonicalText(canonicalText, startOrdinal, endOrdinal, maxInputTok
   flush2();
   return windows;
 }
+function splitOversizedLine(line, effectiveMax) {
+  let slices = [];
+  try {
+    slices = recursiveCharacterSplit(line, {
+      chunkSize: effectiveMax,
+      lengthFunction: estimateTokens
+    });
+  } catch (error51) {
+    log("[magic-context] recursiveCharacterSplit failed; using char-budget fallback:", error51);
+    slices = [];
+  }
+  if (slices.length === 0) {
+    slices = charBudgetSplit(line, effectiveMax);
+  }
+  const safe = [];
+  const pushChecked = (slice) => {
+    if (estimateTokens(slice) > effectiveMax && slice.length > 1) {
+      safe.push(...charBudgetSplit(slice, effectiveMax));
+      return;
+    }
+    safe.push(slice);
+  };
+  for (const slice of slices) {
+    if (estimateTokens(slice) <= effectiveMax) {
+      safe.push(slice);
+    } else {
+      for (const sub of charBudgetSplit(slice, effectiveMax))
+        pushChecked(sub);
+    }
+  }
+  return safe.filter((s) => s.length > 0);
+}
+function charBudgetSplit(text, effectiveMax) {
+  const totalTokens = Math.max(1, estimateTokens(text));
+  const charsPerToken = Math.max(1, Math.floor(text.length / totalTokens));
+  const sliceChars = Math.max(1, effectiveMax * charsPerToken);
+  const out = [];
+  let pos = 0;
+  while (pos < text.length) {
+    let end = Math.min(text.length, pos + sliceChars);
+    let slice = text.slice(pos, end);
+    while (slice.length > 1 && estimateTokens(slice) > effectiveMax) {
+      end = pos + Math.max(1, Math.floor((end - pos) / 2));
+      slice = text.slice(pos, end);
+    }
+    out.push(slice);
+    pos = end;
+  }
+  return out;
+}
 function getExistingChunkHashes(db, compartmentId, modelId, projectPath) {
   const scoped = typeof projectPath === "string" && projectPath.length > 0;
   const rows = scoped ? getExistingHashStatement(db, true).all(compartmentId, modelId, projectPath) : getExistingHashStatement(db, false).all(compartmentId, modelId);
@@ -161305,6 +161459,7 @@ class OpenAICompatibleEmbeddingProvider {
     if (texts.length === 0) {
       return [];
     }
+    const requestTexts = texts.map((t) => t.trim().length === 0 ? " " : t);
     if (!await this.initialize()) {
       return Array.from({ length: texts.length }, () => null);
     }
@@ -161336,7 +161491,7 @@ class OpenAICompatibleEmbeddingProvider {
         },
         body: JSON.stringify({
           model: this.model,
-          input: texts,
+          input: requestTexts,
           ...inputTypeForRequest ? { input_type: inputTypeForRequest } : {},
           ...this.truncate ? { truncate: this.truncate } : {}
         }),
@@ -162423,6 +162578,32 @@ async function embedBatchForProject(projectIdentity, texts, signal, purpose = "p
   }
   return { vectors, modelId, generation };
 }
+async function embedTextsWindowBounded(projectIdentity, texts, signal) {
+  if (texts.length <= MAX_WINDOWS_PER_EMBED_CALL) {
+    return embedBatchForProject(projectIdentity, texts, signal);
+  }
+  const vectors = [];
+  let modelId = null;
+  let generation = null;
+  for (let start = 0;start < texts.length; start += MAX_WINDOWS_PER_EMBED_CALL) {
+    if (signal?.aborted)
+      return null;
+    const sub = texts.slice(start, start + MAX_WINDOWS_PER_EMBED_CALL);
+    const result = await embedBatchForProject(projectIdentity, sub, signal);
+    if (!result)
+      return null;
+    if (modelId === null) {
+      modelId = result.modelId;
+      generation = result.generation;
+    } else if (result.modelId !== modelId || result.generation !== generation) {
+      return null;
+    }
+    vectors.push(...result.vectors);
+  }
+  if (modelId === null || generation === null)
+    return null;
+  return { vectors, modelId, generation };
+}
 function isUnembeddedMemoryRow(row) {
   if (row === null || typeof row !== "object")
     return false;
@@ -162556,7 +162737,7 @@ async function embedCandidateChunkBatch(db, projectIdentity, modelId, candidates
       let result = null;
       const attemptStart = Date.now();
       try {
-        result = await embedBatchForProject(projectIdentity, texts, signal);
+        result = await embedTextsWindowBounded(projectIdentity, texts, signal);
       } catch (error51) {
         log("[magic-context] failed to proactively embed compartment chunks:", error51);
       }

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@cortexkit/pi-magic-context",
-	"version": "0.30.2",
+	"version": "0.30.3",
 	"type": "module",
 	"description": "Pi coding agent extension for Magic Context — cross-session memory and context management",
 	"main": "dist/index.js",