@loreai/core 0.10.2 → 0.11.0

package/src/temporal.ts

@@ -9,7 +9,38 @@ function estimate(text: string): number {
   return Math.ceil(text.length / 3);
 }
 
-function partsToText(parts: LorePart[]): string {
+/**
+ * Chunk-boundary terminator inserted between chunks by `partsToText`.
+ *
+ * `\x1f` is ASCII Unit Separator — a non-word control char that:
+ *   - cannot legitimately appear in normal chat or tool content (control
+ *     chars are vanishingly rare even in binary file dumps),
+ *   - is treated as a token separator by FTS5's `unicode61` tokenizer, so
+ *     it has zero effect on BM25 indexing or scoring,
+ *   - survives `sanitizeSurrogates()` (which only touches lone UTF-16
+ *     surrogates, never ASCII control chars).
+ *
+ * Placed AFTER the existing `\n` so display tools that split on `\n`
+ * still render correctly; the structural parser (in `distillation.ts`)
+ * splits on `"\n" + CHUNK_TERMINATOR` for unambiguous chunk recovery.
+ *
+ * Adopted in F3b. Pre-F3b rows are rewritten in-place by a SQL migration
+ * (see `db.ts`); after that migration runs, every `temporal_messages.content`
+ * value uses this format consistently.
+ */
+export const CHUNK_TERMINATOR = "\x1f";
+
+/**
+ * Serialize a list of message parts into a single content string for the
+ * `temporal_messages.content` column. Chunks are separated by
+ * `"\n" + CHUNK_TERMINATOR` so the structural parser can recover chunk
+ * boundaries unambiguously regardless of payload contents (including
+ * payloads that contain literal `[tool:...]` substrings — e.g. when the
+ * agent reads a file that documents this very format).
+ *
+ * Exported so tests can pin producer/consumer round-trip behavior.
+ */
+export function partsToText(parts: LorePart[]): string {
   const chunks: string[] = [];
   for (const part of parts) {
     if (isTextPart(part)) chunks.push(part.text);
@@ -21,7 +52,7 @@ function partsToText(parts: LorePart[]): string {
   // Sanitize unpaired surrogates from tool outputs and other raw text.
   // Without this, surrogates survive into the DB and later break JSON
   // serialization when included in recall tool responses.
-  return sanitizeSurrogates(chunks.join("\n"));
+  return sanitizeSurrogates(chunks.join("\n" + CHUNK_TERMINATOR));
 }
 
 function messageMetadata(info: LoreMessage, parts: LorePart[]): string {
@@ -167,12 +198,12 @@ export function search(input: {
   if (q === EMPTY_QUERY) return [];
 
   const ftsSQL = input.sessionID
-    ? `SELECT m.* FROM temporal_messages m
-       JOIN temporal_fts f ON m.rowid = f.rowid
+    ? `SELECT m.* FROM temporal_fts f
+       CROSS JOIN temporal_messages m ON m.rowid = f.rowid
        WHERE f.content MATCH ? AND m.project_id = ? AND m.session_id = ?
        ORDER BY rank LIMIT ?`
-    : `SELECT m.* FROM temporal_messages m
-       JOIN temporal_fts f ON m.rowid = f.rowid
+    : `SELECT m.* FROM temporal_fts f
+       CROSS JOIN temporal_messages m ON m.rowid = f.rowid
        WHERE f.content MATCH ? AND m.project_id = ?
        ORDER BY rank LIMIT ?`;
   const params = input.sessionID
@@ -222,12 +253,12 @@ export function searchScored(input: {
   if (q === EMPTY_QUERY) return [];
 
   const ftsSQL = input.sessionID
-    ? `SELECT m.*, rank FROM temporal_messages m
-       JOIN temporal_fts f ON m.rowid = f.rowid
+    ? `SELECT m.*, rank FROM temporal_fts f
+       CROSS JOIN temporal_messages m ON m.rowid = f.rowid
        WHERE f.content MATCH ? AND m.project_id = ? AND m.session_id = ?
        ORDER BY rank LIMIT ?`
-    : `SELECT m.*, rank FROM temporal_messages m
-       JOIN temporal_fts f ON m.rowid = f.rowid
+    : `SELECT m.*, rank FROM temporal_fts f
+       CROSS JOIN temporal_messages m ON m.rowid = f.rowid
        WHERE f.content MATCH ? AND m.project_id = ?
        ORDER BY rank LIMIT ?`;
   const params = input.sessionID

package/src/types.ts

@@ -35,6 +35,15 @@ export type LoreAssistantMessage = {
   modelID: string;
   providerID: string;
   mode: string;
+  /**
+   * Set to `true` by the OpenCode compaction agent on the assistant
+   * message that holds a `/compact` summary (see upstream
+   * `compaction.ts:435`). Lore reads this flag in F1b's
+   * `findPreviousCompactSummary` to anchor repeat `/compact`
+   * invocations to the prior summary. Always undefined for normal
+   * assistant turns.
+   */
+  summary?: boolean;
   path: { cwd: string; root: string };
   cost: number;
   tokens: {

package/src/worker-model.ts

@@ -0,0 +1,264 @@
+/**
+ * Dynamic worker model selection.
+ *
+ * Background workers (distillation, curation, query expansion) don't need
+ * frontier reasoning. This module discovers cheaper models from the same
+ * provider and validates their quality via a two-phase comparison:
+ *   Phase 1: structural checks (parsability, observation count, token bounds)
+ *   Phase 2: LLM judge (session model rates candidate output vs reference)
+ *
+ * Results are persisted in kv_meta and re-evaluated when the model landscape
+ * changes (new models, session model switch, model deprecation).
+ */
+
+import { db } from "./db";
+import { sha256 } from "#db/driver";
+import * as log from "./log";
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+/** Minimal model info needed for worker selection — provider-agnostic. */
+export type ModelInfo = {
+  id: string;
+  providerID: string;
+  cost: { input: number }; // per-token cost
+  status: string;
+  capabilities: { input: { text: boolean } };
+};
+
+/** Result of a worker model validation stored in kv_meta. */
+export type WorkerModelResult = {
+  modelID: string;
+  providerID: string;
+  fingerprint: string;
+  validatedAt: number;
+  judgeScore: number | null; // null = structural-only (no judge run yet)
+};
+
+const KV_PREFIX = "lore:worker_model:";
+
+// ---------------------------------------------------------------------------
+// Candidate selection
+// ---------------------------------------------------------------------------
+
+/**
+ * Select worker model candidates from the available models.
+ *
+ * Returns up to 2 candidates: cheapest overall + one tier below the session
+ * model. The session model itself is included (if it's the cheapest, the list
+ * has 1 entry and no comparison is needed).
+ */
+export function selectWorkerCandidates(
+  sessionModel: { id: string; providerID: string; cost: { input: number } },
+  providerModels: ModelInfo[],
+): ModelInfo[] {
+  // Filter: same provider, active, text-capable
+  const eligible = providerModels.filter(
+    (m) =>
+      m.providerID === sessionModel.providerID &&
+      m.status === "active" &&
+      m.capabilities.input.text,
+  );
+
+  if (eligible.length === 0) return [];
+
+  // Sort by cost ascending (cheapest first)
+  const sorted = [...eligible].sort((a, b) => a.cost.input - b.cost.input);
+
+  // Cheapest overall
+  const cheapest = sorted[0];
+
+  // One tier below session model: the most expensive model that's still
+  // cheaper than the session model. If session IS cheapest, this is undefined.
+  const belowSession = sorted
+    .filter((m) => m.cost.input < sessionModel.cost.input)
+    .pop(); // last = most expensive among cheaper ones
+
+  // Deduplicate
+  const candidates = new Map<string, ModelInfo>();
+  candidates.set(cheapest.id, cheapest);
+  if (belowSession && belowSession.id !== cheapest.id) {
+    candidates.set(belowSession.id, belowSession);
+  }
+
+  // If session model is the cheapest, return just it
+  if (cheapest.id === sessionModel.id || cheapest.cost.input >= sessionModel.cost.input) {
+    return [cheapest];
+  }
+
+  return [...candidates.values()];
+}
+
+// ---------------------------------------------------------------------------
+// Fingerprinting
+// ---------------------------------------------------------------------------
+
+/**
+ * Compute a fingerprint from the model landscape. Changes when:
+ * - Models are added or removed from the provider
+ * - The session model changes
+ */
+export function computeModelFingerprint(
+  providerID: string,
+  sessionModelID: string,
+  activeModelIDs: string[],
+): string {
+  const sorted = [...activeModelIDs].sort();
+  return sha256(
+    JSON.stringify({ providerID, sessionModelID, modelIDs: sorted }),
+  );
+}
+
+// ---------------------------------------------------------------------------
+// Persistence
+// ---------------------------------------------------------------------------
+
+export function getValidatedWorkerModel(
+  providerID: string,
+): WorkerModelResult | null {
+  const row = db()
+    .query("SELECT value FROM kv_meta WHERE key = ?")
+    .get(`${KV_PREFIX}${providerID}`) as { value: string } | null;
+  if (!row) return null;
+  try {
+    return JSON.parse(row.value) as WorkerModelResult;
+  } catch {
+    return null;
+  }
+}
+
+export function storeValidatedWorkerModel(result: WorkerModelResult): void {
+  const key = `${KV_PREFIX}${result.providerID}`;
+  const value = JSON.stringify(result);
+  db()
+    .query(
+      "INSERT INTO kv_meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = ?",
+    )
+    .run(key, value, value);
+}
+
+/**
+ * Check whether the stored validation is stale (fingerprint mismatch).
+ */
+export function isValidationStale(
+  stored: WorkerModelResult | null,
+  currentFingerprint: string,
+): boolean {
+  if (!stored) return true;
+  return stored.fingerprint !== currentFingerprint;
+}
+
+// ---------------------------------------------------------------------------
+// Structural validation
+// ---------------------------------------------------------------------------
+
+export type StructuralCheckResult = {
+  passed: boolean;
+  observationCount: number;
+  tokenCount: number;
+  reason?: string;
+};
+
+/**
+ * Structural quality check: does the candidate distillation output meet
+ * minimum quality thresholds relative to the reference?
+ */
+export function structuralCheck(
+  candidateObservations: string | null,
+  referenceObservations: string,
+): StructuralCheckResult {
+  if (candidateObservations == null || candidateObservations.length === 0) {
+    return { passed: false, observationCount: 0, tokenCount: 0, reason: candidateObservations === null ? "parse_failed" : "empty" };
+  }
+
+  // Count observation lines (non-empty lines starting with common markers)
+  const countObs = (text: string) =>
+    text.split("\n").filter((l) => l.trim().length > 0).length;
+
+  const refCount = countObs(referenceObservations);
+  const candCount = countObs(candidateObservations);
+  const candTokens = Math.ceil(candidateObservations.length / 3);
+
+  // Observation count within ±50% of reference
+  if (refCount > 0 && (candCount < refCount * 0.5 || candCount > refCount * 1.5)) {
+    return {
+      passed: false,
+      observationCount: candCount,
+      tokenCount: candTokens,
+      reason: `observation_count_${candCount}_vs_ref_${refCount}`,
+    };
+  }
+
+  // Not degenerate: not empty, not >3x reference size
+  const refTokens = Math.ceil(referenceObservations.length / 3);
+  if (candTokens === 0) {
+    return { passed: false, observationCount: candCount, tokenCount: candTokens, reason: "empty" };
+  }
+  if (refTokens > 0 && candTokens > refTokens * 3) {
+    return {
+      passed: false,
+      observationCount: candCount,
+      tokenCount: candTokens,
+      reason: `token_count_${candTokens}_vs_ref_${refTokens}_3x`,
+    };
+  }
+
+  return { passed: true, observationCount: candCount, tokenCount: candTokens };
+}
+
+// ---------------------------------------------------------------------------
+// Judge prompt
+// ---------------------------------------------------------------------------
+
+export const WORKER_JUDGE_SYSTEM = `You are evaluating distillation quality. You will be given a REFERENCE distillation (produced by a capable model) and a CANDIDATE distillation (produced by a cheaper model) of the same conversation segment.
+
+Rate the candidate on a scale of 1-5:
+5 = Captures all key facts and decisions, equivalent to reference
+4 = Captures most facts, minor omissions
+3 = Captures the essential facts, some detail loss acceptable
+2 = Missing important facts or technical details
+1 = Significantly incomplete or inaccurate
+
+Respond with ONLY a single digit (1-5).`;
+
+export function workerJudgeUser(
+  reference: string,
+  candidate: string,
+): string {
+  return `<reference>\n${reference}\n</reference>\n\n<candidate>\n${candidate}\n</candidate>`;
+}
+
+/** Parse the judge's score from a response. Returns null on parse failure. */
+export function parseJudgeScore(response: string): number | null {
+  const match = response.trim().match(/^([1-5])/);
+  if (!match) return null;
+  return parseInt(match[1], 10);
+}
+
+// ---------------------------------------------------------------------------
+// Effective worker model resolution
+// ---------------------------------------------------------------------------
+
+/**
+ * Resolve the effective worker model for a given provider.
+ * Priority: explicit config > validated auto-selection > session model (fallback).
+ */
+export function resolveWorkerModel(
+  providerID: string,
+  configWorkerModel?: { providerID: string; modelID: string },
+  configModel?: { providerID: string; modelID: string },
+): { providerID: string; modelID: string } | undefined {
+  // Explicit override wins
+  if (configWorkerModel) return configWorkerModel;
+
+  // Check for validated auto-selection
+  const validated = getValidatedWorkerModel(providerID);
+  if (validated) {
+    return { providerID: validated.providerID, modelID: validated.modelID };
+  }
+
+  // Fall back to the session model config (or undefined = host default)
+  return configModel;
+}