@loreai/core 0.10.2 → 0.11.1

package/src/prompt.ts

@@ -184,14 +184,30 @@ EXACT NUMBERS: When two segments report different numbers for what seems like th
 
 EARLY-SESSION CONTENT: Bug fixes, code changes, and decisions from the start of a session are just as important as later work. Never drop them just because the segment is short or old. If the first segment contains a specific bug fix with file paths and root cause, it MUST survive into the reflection.
 
+ANCHORED UPDATES: If the prompt includes a <previous-meta-summary> block, treat it as the current consolidated state. Update it using the NEW observation segments — preserve still-true details, remove stale details, and merge in new facts. Keep the same section headings. Do NOT re-derive unchanged sections verbatim unless the new segments contradict them.
+
 Output ONLY an <observations> block with the consolidated observations.`;
 
 export function recursiveUser(
   distillations: Array<{ observations: string }>,
+  previousMeta?: string,
 ): string {
   const entries = distillations.map(
     (d, i) => `Segment ${i + 1}:\n${d.observations}`,
   );
+  if (previousMeta) {
+    return `Update the anchored meta-summary below using the NEW observation segments. Preserve still-true details, remove stale details, and merge in new facts. Keep the same section headings.
+
+<previous-meta-summary>
+${previousMeta}
+</previous-meta-summary>
+
+---
+
+New observation segments to merge (chronological order):
+
+${entries.join("\n\n---\n\n")}`;
+  }
   return `Observation segments to consolidate (chronological order):
 
 ${entries.join("\n\n---\n\n")}`;
@@ -388,6 +404,91 @@ export function formatDistillations(
   return sections.join("\n\n");
 }
 
+// Strict Markdown skeleton for the /compact session summary. Task-oriented
+// sections so the next agent starting from the compacted context has a clear
+// "where am I, what's next, what's blocked" briefing. Derived from upstream
+// OpenCode's SUMMARY_TEMPLATE (session/compaction.ts in #23870) with a "(none)"
+// directive added for explicit empty sections and a closing "I'm ready to
+// continue." sentinel to preserve Lore's post-compact UX.
+export const COMPACT_SUMMARY_TEMPLATE = `Output exactly this Markdown structure. Keep every section in this order, even when empty (use "(none)").
+
+---
+## Goal
+- [single-sentence task summary]
+
+## Constraints & Preferences
+- [user constraints, preferences, specs, or "(none)"]
+
+## Progress
+### Done
+- [completed work or "(none)"]
+
+### In Progress
+- [current work or "(none)"]
+
+### Blocked
+- [blockers or "(none)"]
+
+## Key Decisions
+- [decision and why, or "(none)"]
+
+## Next Steps
+- [ordered next actions or "(none)"]
+
+## Critical Context
+- [important technical facts, errors, open questions, or "(none)"]
+
+## Relevant Files
+- [file or directory path: why it matters, or "(none)"]
+---
+
+Rules:
+- Keep every section, even when empty.
+- Use terse bullets, not prose paragraphs.
+- Preserve exact file paths, commands, error strings, and identifiers when known.
+- Do not mention the summary process or that context was compacted.
+- End with "I'm ready to continue." on its own line after the closing "---".`;
+
+// Build the user-facing prompt passed to the compaction agent during /compact.
+// Lore injects pre-computed distillations as context separately; this prompt
+// just tells the model how to render its summary.
+//
+// `hasDistillations` is a boolean rather than the full array because this
+// function only cares about presence — the distillation bodies are pushed into
+// `output.context` separately by the caller. Passing the array shape would be
+// misleading dead weight.
+//
+// `previousSummary` is the prior `/compact` output text (typically from the
+// most recent assistant message with `info.summary === true`). When present,
+// the prompt asks the model to UPDATE the anchored summary in place rather
+// than re-derive from scratch — matching upstream OpenCode's behavior at
+// `compaction.ts:121-132` (`buildPrompt`). When absent, the prompt is
+// byte-identical to today's non-anchored output.
+//
+// F1b (this parameter) is OpenCode-specific: the retrieval path uses
+// `client.session.messages` to find the prior summary by `info.summary === true`.
+// See `findPreviousCompactSummary` in `packages/opencode/src/index.ts`.
+export function buildCompactPrompt(input: {
+  hasDistillations: boolean;
+  knowledge?: string;
+  previousSummary?: string;
+}): string {
+  const distillSection = input.hasDistillations
+    ? "Lore has pre-computed chunked summaries of the session history (injected above as context). Use them as the authoritative source — do NOT re-read raw conversation messages that conflict with them.\n\n"
+    : "";
+
+  const anchorBlock = input.previousSummary
+    ? `A prior compacted summary exists for this session. Update it using the conversation history above: preserve still-true details, remove stale details, and merge in new facts. Keep every section in place.\n\n<previous-summary>\n${input.previousSummary}\n</previous-summary>\n\n`
+    : "";
+
+  const knowledgeBlock = input.knowledge ? `\n${input.knowledge}\n` : "";
+
+  return `You are producing a compacted session summary for an AI coding agent. This summary will be the ONLY context available in the next part of the conversation.
+
+${distillSection}${anchorBlock}${COMPACT_SUMMARY_TEMPLATE}
+${knowledgeBlock}`;
+}
+
 // ~3 chars per token — validated as best heuristic against real API data.
 function estimateTokens(text: string): number {
   return Math.ceil(text.length / 3);

package/src/recall.ts

@@ -116,14 +116,14 @@ function searchDistillationsScored(input: {
 
   const ftsSQL = input.sessionID
     ? `SELECT d.id, d.observations, d.generation, d.created_at, d.session_id, rank
-       FROM distillations d
-       JOIN distillation_fts f ON d.rowid = f.rowid
+       FROM distillation_fts f
+       CROSS JOIN distillations d ON d.rowid = f.rowid
        WHERE distillation_fts MATCH ?
        AND d.project_id = ? AND d.session_id = ?
        ORDER BY rank LIMIT ?`
     : `SELECT d.id, d.observations, d.generation, d.created_at, d.session_id, rank
-       FROM distillations d
-       JOIN distillation_fts f ON d.rowid = f.rowid
+       FROM distillation_fts f
+       CROSS JOIN distillations d ON d.rowid = f.rowid
        WHERE distillation_fts MATCH ?
        AND d.project_id = ?
        ORDER BY rank LIMIT ?`;

package/src/temporal.ts

@@ -9,7 +9,38 @@ function estimate(text: string): number {
   return Math.ceil(text.length / 3);
 }
 
-function partsToText(parts: LorePart[]): string {
+/**
+ * Chunk-boundary terminator inserted between chunks by `partsToText`.
+ *
+ * `\x1f` is ASCII Unit Separator — a non-word control char that:
+ *   - cannot legitimately appear in normal chat or tool content (control
+ *     chars are vanishingly rare even in binary file dumps),
+ *   - is treated as a token separator by FTS5's `unicode61` tokenizer, so
+ *     it has zero effect on BM25 indexing or scoring,
+ *   - survives `sanitizeSurrogates()` (which only touches lone UTF-16
+ *     surrogates, never ASCII control chars).
+ *
+ * Placed AFTER the existing `\n` so display tools that split on `\n`
+ * still render correctly; the structural parser (in `distillation.ts`)
+ * splits on `"\n" + CHUNK_TERMINATOR` for unambiguous chunk recovery.
+ *
+ * Adopted in F3b. Pre-F3b rows are rewritten in-place by a SQL migration
+ * (see `db.ts`); after that migration runs, every `temporal_messages.content`
+ * value uses this format consistently.
+ */
+export const CHUNK_TERMINATOR = "\x1f";
+
+/**
+ * Serialize a list of message parts into a single content string for the
+ * `temporal_messages.content` column. Chunks are separated by
+ * `"\n" + CHUNK_TERMINATOR` so the structural parser can recover chunk
+ * boundaries unambiguously regardless of payload contents (including
+ * payloads that contain literal `[tool:...]` substrings — e.g. when the
+ * agent reads a file that documents this very format).
+ *
+ * Exported so tests can pin producer/consumer round-trip behavior.
+ */
+export function partsToText(parts: LorePart[]): string {
   const chunks: string[] = [];
   for (const part of parts) {
     if (isTextPart(part)) chunks.push(part.text);
@@ -21,7 +52,7 @@ function partsToText(parts: LorePart[]): string {
   // Sanitize unpaired surrogates from tool outputs and other raw text.
   // Without this, surrogates survive into the DB and later break JSON
   // serialization when included in recall tool responses.
-  return sanitizeSurrogates(chunks.join("\n"));
+  return sanitizeSurrogates(chunks.join("\n" + CHUNK_TERMINATOR));
 }
 
 function messageMetadata(info: LoreMessage, parts: LorePart[]): string {
@@ -167,12 +198,12 @@ export function search(input: {
   if (q === EMPTY_QUERY) return [];
 
   const ftsSQL = input.sessionID
-    ? `SELECT m.* FROM temporal_messages m
-       JOIN temporal_fts f ON m.rowid = f.rowid
+    ? `SELECT m.* FROM temporal_fts f
+       CROSS JOIN temporal_messages m ON m.rowid = f.rowid
        WHERE f.content MATCH ? AND m.project_id = ? AND m.session_id = ?
        ORDER BY rank LIMIT ?`
-    : `SELECT m.* FROM temporal_messages m
-       JOIN temporal_fts f ON m.rowid = f.rowid
+    : `SELECT m.* FROM temporal_fts f
+       CROSS JOIN temporal_messages m ON m.rowid = f.rowid
        WHERE f.content MATCH ? AND m.project_id = ?
        ORDER BY rank LIMIT ?`;
   const params = input.sessionID
@@ -222,12 +253,12 @@ export function searchScored(input: {
   if (q === EMPTY_QUERY) return [];
 
   const ftsSQL = input.sessionID
-    ? `SELECT m.*, rank FROM temporal_messages m
-       JOIN temporal_fts f ON m.rowid = f.rowid
+    ? `SELECT m.*, rank FROM temporal_fts f
+       CROSS JOIN temporal_messages m ON m.rowid = f.rowid
        WHERE f.content MATCH ? AND m.project_id = ? AND m.session_id = ?
        ORDER BY rank LIMIT ?`
-    : `SELECT m.*, rank FROM temporal_messages m
-       JOIN temporal_fts f ON m.rowid = f.rowid
+    : `SELECT m.*, rank FROM temporal_fts f
+       CROSS JOIN temporal_messages m ON m.rowid = f.rowid
        WHERE f.content MATCH ? AND m.project_id = ?
        ORDER BY rank LIMIT ?`;
   const params = input.sessionID

package/src/types.ts

@@ -35,6 +35,15 @@ export type LoreAssistantMessage = {
   modelID: string;
   providerID: string;
   mode: string;
+  /**
+   * Set to `true` by the OpenCode compaction agent on the assistant
+   * message that holds a `/compact` summary (see upstream
+   * `compaction.ts:435`). Lore reads this flag in F1b's
+   * `findPreviousCompactSummary` to anchor repeat `/compact`
+   * invocations to the prior summary. Always undefined for normal
+   * assistant turns.
+   */
+  summary?: boolean;
   path: { cwd: string; root: string };
   cost: number;
   tokens: {

package/src/worker-model.ts

@@ -0,0 +1,264 @@
+/**
+ * Dynamic worker model selection.
+ *
+ * Background workers (distillation, curation, query expansion) don't need
+ * frontier reasoning. This module discovers cheaper models from the same
+ * provider and validates their quality via a two-phase comparison:
+ *   Phase 1: structural checks (parsability, observation count, token bounds)
+ *   Phase 2: LLM judge (session model rates candidate output vs reference)
+ *
+ * Results are persisted in kv_meta and re-evaluated when the model landscape
+ * changes (new models, session model switch, model deprecation).
+ */
+
+import { db } from "./db";
+import { sha256 } from "#db/driver";
+import * as log from "./log";
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+/** Minimal model info needed for worker selection — provider-agnostic. */
+export type ModelInfo = {
+  id: string;
+  providerID: string;
+  cost: { input: number }; // per-token cost
+  status: string;
+  capabilities: { input: { text: boolean } };
+};
+
+/** Result of a worker model validation stored in kv_meta. */
+export type WorkerModelResult = {
+  modelID: string;
+  providerID: string;
+  fingerprint: string;
+  validatedAt: number;
+  judgeScore: number | null; // null = structural-only (no judge run yet)
+};
+
+const KV_PREFIX = "lore:worker_model:";
+
+// ---------------------------------------------------------------------------
+// Candidate selection
+// ---------------------------------------------------------------------------
+
+/**
+ * Select worker model candidates from the available models.
+ *
+ * Returns up to 2 candidates: cheapest overall + one tier below the session
+ * model. The session model itself is included (if it's the cheapest, the list
+ * has 1 entry and no comparison is needed).
+ */
+export function selectWorkerCandidates(
+  sessionModel: { id: string; providerID: string; cost: { input: number } },
+  providerModels: ModelInfo[],
+): ModelInfo[] {
+  // Filter: same provider, active, text-capable
+  const eligible = providerModels.filter(
+    (m) =>
+      m.providerID === sessionModel.providerID &&
+      m.status === "active" &&
+      m.capabilities.input.text,
+  );
+
+  if (eligible.length === 0) return [];
+
+  // Sort by cost ascending (cheapest first)
+  const sorted = [...eligible].sort((a, b) => a.cost.input - b.cost.input);
+
+  // Cheapest overall
+  const cheapest = sorted[0];
+
+  // One tier below session model: the most expensive model that's still
+  // cheaper than the session model. If session IS cheapest, this is undefined.
+  const belowSession = sorted
+    .filter((m) => m.cost.input < sessionModel.cost.input)
+    .pop(); // last = most expensive among cheaper ones
+
+  // Deduplicate
+  const candidates = new Map<string, ModelInfo>();
+  candidates.set(cheapest.id, cheapest);
+  if (belowSession && belowSession.id !== cheapest.id) {
+    candidates.set(belowSession.id, belowSession);
+  }
+
+  // If session model is the cheapest, return just it
+  if (cheapest.id === sessionModel.id || cheapest.cost.input >= sessionModel.cost.input) {
+    return [cheapest];
+  }
+
+  return [...candidates.values()];
+}
+
+// ---------------------------------------------------------------------------
+// Fingerprinting
+// ---------------------------------------------------------------------------
+
+/**
+ * Compute a fingerprint from the model landscape. Changes when:
+ * - Models are added or removed from the provider
+ * - The session model changes
+ */
+export function computeModelFingerprint(
+  providerID: string,
+  sessionModelID: string,
+  activeModelIDs: string[],
+): string {
+  const sorted = [...activeModelIDs].sort();
+  return sha256(
+    JSON.stringify({ providerID, sessionModelID, modelIDs: sorted }),
+  );
+}
+
+// ---------------------------------------------------------------------------
+// Persistence
+// ---------------------------------------------------------------------------
+
+export function getValidatedWorkerModel(
+  providerID: string,
+): WorkerModelResult | null {
+  const row = db()
+    .query("SELECT value FROM kv_meta WHERE key = ?")
+    .get(`${KV_PREFIX}${providerID}`) as { value: string } | null;
+  if (!row) return null;
+  try {
+    return JSON.parse(row.value) as WorkerModelResult;
+  } catch {
+    return null;
+  }
+}
+
+export function storeValidatedWorkerModel(result: WorkerModelResult): void {
+  const key = `${KV_PREFIX}${result.providerID}`;
+  const value = JSON.stringify(result);
+  db()
+    .query(
+      "INSERT INTO kv_meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = ?",
+    )
+    .run(key, value, value);
+}
+
+/**
+ * Check whether the stored validation is stale (fingerprint mismatch).
+ */
+export function isValidationStale(
+  stored: WorkerModelResult | null,
+  currentFingerprint: string,
+): boolean {
+  if (!stored) return true;
+  return stored.fingerprint !== currentFingerprint;
+}
+
+// ---------------------------------------------------------------------------
+// Structural validation
+// ---------------------------------------------------------------------------
+
+export type StructuralCheckResult = {
+  passed: boolean;
+  observationCount: number;
+  tokenCount: number;
+  reason?: string;
+};
+
+/**
+ * Structural quality check: does the candidate distillation output meet
+ * minimum quality thresholds relative to the reference?
+ */
+export function structuralCheck(
+  candidateObservations: string | null,
+  referenceObservations: string,
+): StructuralCheckResult {
+  if (candidateObservations == null || candidateObservations.length === 0) {
+    return { passed: false, observationCount: 0, tokenCount: 0, reason: candidateObservations === null ? "parse_failed" : "empty" };
+  }
+
+  // Count observation lines (non-empty lines starting with common markers)
+  const countObs = (text: string) =>
+    text.split("\n").filter((l) => l.trim().length > 0).length;
+
+  const refCount = countObs(referenceObservations);
+  const candCount = countObs(candidateObservations);
+  const candTokens = Math.ceil(candidateObservations.length / 3);
+
+  // Observation count within ±50% of reference
+  if (refCount > 0 && (candCount < refCount * 0.5 || candCount > refCount * 1.5)) {
+    return {
+      passed: false,
+      observationCount: candCount,
+      tokenCount: candTokens,
+      reason: `observation_count_${candCount}_vs_ref_${refCount}`,
+    };
+  }
+
+  // Not degenerate: not empty, not >3x reference size
+  const refTokens = Math.ceil(referenceObservations.length / 3);
+  if (candTokens === 0) {
+    return { passed: false, observationCount: candCount, tokenCount: candTokens, reason: "empty" };
+  }
+  if (refTokens > 0 && candTokens > refTokens * 3) {
+    return {
+      passed: false,
+      observationCount: candCount,
+      tokenCount: candTokens,
+      reason: `token_count_${candTokens}_vs_ref_${refTokens}_3x`,
+    };
+  }
+
+  return { passed: true, observationCount: candCount, tokenCount: candTokens };
+}
+
+// ---------------------------------------------------------------------------
+// Judge prompt
+// ---------------------------------------------------------------------------
+
+export const WORKER_JUDGE_SYSTEM = `You are evaluating distillation quality. You will be given a REFERENCE distillation (produced by a capable model) and a CANDIDATE distillation (produced by a cheaper model) of the same conversation segment.
+
+Rate the candidate on a scale of 1-5:
+5 = Captures all key facts and decisions, equivalent to reference
+4 = Captures most facts, minor omissions
+3 = Captures the essential facts, some detail loss acceptable
+2 = Missing important facts or technical details
+1 = Significantly incomplete or inaccurate
+
+Respond with ONLY a single digit (1-5).`;
+
+export function workerJudgeUser(
+  reference: string,
+  candidate: string,
+): string {
+  return `<reference>\n${reference}\n</reference>\n\n<candidate>\n${candidate}\n</candidate>`;
+}
+
+/** Parse the judge's score from a response. Returns null on parse failure. */
+export function parseJudgeScore(response: string): number | null {
+  const match = response.trim().match(/^([1-5])/);
+  if (!match) return null;
+  return parseInt(match[1], 10);
+}
+
+// ---------------------------------------------------------------------------
+// Effective worker model resolution
+// ---------------------------------------------------------------------------
+
+/**
+ * Resolve the effective worker model for a given provider.
+ * Priority: explicit config > validated auto-selection > session model (fallback).
+ */
+export function resolveWorkerModel(
+  providerID: string,
+  configWorkerModel?: { providerID: string; modelID: string },
+  configModel?: { providerID: string; modelID: string },
+): { providerID: string; modelID: string } | undefined {
+  // Explicit override wins
+  if (configWorkerModel) return configWorkerModel;
+
+  // Check for validated auto-selection
+  const validated = getValidatedWorkerModel(providerID);
+  if (validated) {
+    return { providerID: validated.providerID, modelID: validated.modelID };
+  }
+
+  // Fall back to the session model config (or undefined = host default)
+  return configModel;
+}