@equationalapplications/core-llm-wiki 7.3.0 → 7.5.0

package/dist/testing.js

@@ -817,8 +817,7 @@ var EmbeddingService = class {
     }
   }
   async tryEmbedFact(fact, ctx) {
-    const embedFn = this.options.llmProvider.embed;
-    if (typeof embedFn !== "function") return { ok: false, kind: "no_provider" };
+    if (typeof this.options.llmProvider.embed !== "function") return { ok: false, kind: "no_provider" };
     let tagsStr;
     if (Array.isArray(fact.tags)) {
       tagsStr = fact.tags.join(" ");
@@ -835,7 +834,7 @@ var EmbeddingService = class {
     const text = clip(`${fact.title} ${fact.body} ${tagsStr}`.trim(), maxEmbedChars);
     let float32Vector;
     try {
-      const vector = await embedFn(text);
+      const vector = await this.options.llmProvider.embed(text);
       if (vector.length === 0 || !vector.every((v) => typeof v === "number" && isFinite(v))) {
         console.warn(`[WikiMemory] embedFact: embed() returned an invalid vector for ${fact.id}; skipping.`);
         this.reportEmbed(ctx, fact, "embedding_failed", "invalid_vector");
@@ -2281,6 +2280,51 @@ var IngestionService = class {
   }
 };
 
+// src/utils/classifier.ts
+var isUnit = (v) => typeof v === "number" && Number.isFinite(v) && v >= 0 && v <= 1;
+function validateClassifierAnswer(response, key, question) {
+  try {
+    if (response === null || typeof response !== "object") return { ok: false, reason: "malformed" };
+    const answers = response.answers;
+    if (answers === null || typeof answers !== "object") return { ok: false, reason: "missing_answer" };
+    if (!Object.prototype.hasOwnProperty.call(answers, key)) return { ok: false, reason: "missing_answer" };
+    const raw = answers[key];
+    if (raw === null || typeof raw !== "object") return { ok: false, reason: "missing_answer" };
+    const a = raw;
+    if (a.kind !== question.kind) return { ok: false, reason: "kind_mismatch" };
+    if (question.kind === "choice") {
+      if (typeof a.choice !== "string" || !question.options.includes(a.choice)) return { ok: false, reason: "choice_not_offered" };
+      if (!isUnit(a.confidence)) return { ok: false, reason: "invalid_probability" };
+      const probs = a.probabilities;
+      if (probs === null || typeof probs !== "object" || Array.isArray(probs)) return { ok: false, reason: "invalid_probability" };
+      const probabilities = {};
+      for (const [k, v] of Object.entries(probs)) {
+        if (!isUnit(v)) return { ok: false, reason: "invalid_probability" };
+        probabilities[k] = v;
+      }
+      return { ok: true, answer: { kind: "choice", choice: a.choice, confidence: a.confidence, probabilities } };
+    }
+    if (question.kind === "binary") {
+      if (!isUnit(a.probability)) return { ok: false, reason: "invalid_probability" };
+      return { ok: true, answer: { kind: "binary", probability: a.probability } };
+    }
+    const maxScore = question.levels.length - 1;
+    if (typeof a.score !== "number" || !Number.isFinite(a.score) || a.score < 0 || a.score > maxScore) {
+      return { ok: false, reason: "score_out_of_range" };
+    }
+    if (!isUnit(a.confidence)) return { ok: false, reason: "invalid_probability" };
+    if (!Array.isArray(a.probabilities) || !a.probabilities.every(isUnit)) return { ok: false, reason: "invalid_probability" };
+    return { ok: true, answer: { kind: "score", score: a.score, confidence: a.confidence, probabilities: a.probabilities.slice() } };
+  } catch {
+    return { ok: false, reason: "malformed" };
+  }
+}
+function classifierStateForFact(fact) {
+  const parts = [fact.title, fact.body];
+  if (Array.isArray(fact.tags) && fact.tags.length > 0) parts.push(`Tags: ${fact.tags.join(", ")}`);
+  return parts.join("\n\n");
+}
+
 // src/utils/embedding.ts
 function parseEmbedding(blob, text) {
   if (blob && blob.byteLength > 0) {
@@ -3191,7 +3235,7 @@ var MaintenanceService = class {
     if (!ontologyService) {
       return { ...zeroed, remaining: 0, deferred: 0 };
     }
-    const { mode } = await ontologyService.getEffectiveState(entityId);
+    const { mode, manifest: effectiveManifest } = await ontologyService.getEffectiveState(entityId);
     if (mode === "off") {
       const counts2 = await this.entryRepo.countUntypedByEntityId(entityId, recheckCutoff);
       return { ...zeroed, remaining: 0, deferred: counts2.deferred };
@@ -3201,6 +3245,14 @@ var MaintenanceService = class {
       const counts2 = await this.entryRepo.countUntypedByEntityId(entityId, recheckCutoff);
       return { ...zeroed, remaining: counts2.eligible, deferred: counts2.deferred };
     }
+    const classifierMode = options?.classifier ?? this.options.config?.ontology?.backfillClassifier ?? "llm";
+    const classify = this.options.llmProvider.classify;
+    if (classifierMode === "auto" && typeof classify === "function") {
+      const slugs = effectiveManifest.node_types.map((n) => n.type);
+      if (slugs.length > 0 && slugs.length <= 255) {
+        return this._runClassifierBackfill(entityId, candidates, effectiveManifest, now, recheckCutoff);
+      }
+    }
     const ontologyContext = await ontologyService.buildPromptContext(entityId);
     const toPromptShape = (f) => ({ id: f.id, title: f.title, body: f.body, tags: f.tags });
     const buildPrompt = (facts) => this.promptService.buildOntologyBackfillPrompt(
@@ -3279,6 +3331,87 @@ var MaintenanceService = class {
       deferred: counts.deferred
     };
   }
+  /**
+   * Classifier-mode backfill (spec §7.3): one `choice` question per untyped
+   * fact over the effective manifest's node-type slugs. Accepted answers go
+   * through `_applyOntologyBackfillBatch` exactly like LLM classifications.
+   * No edges are proposed.
+   */
+  async _runClassifierBackfill(entityId, candidates, manifest, now, recheckCutoff) {
+    const provider = this.options.llmProvider;
+    const rawMin = this.options.config?.ontology?.classifyMinConfidence;
+    const minConfidence = typeof rawMin === "number" && Number.isFinite(rawMin) && rawMin >= 0 && rawMin <= 1 ? rawMin : 0.5;
+    const rawConcurrency = this.options.config?.chunkConcurrency ?? 1;
+    const concurrency = Number.isFinite(rawConcurrency) && rawConcurrency >= 1 ? Math.floor(rawConcurrency) : 1;
+    const question = {
+      kind: "choice",
+      options: manifest.node_types.map((n) => n.type),
+      instructions: "Choose the ontology node type that best describes this fact.\n" + manifest.node_types.map((n) => `- ${n.type}: ${n.description}`).join("\n")
+    };
+    const outcomes = await withConcurrency(
+      candidates.map((fact) => async () => {
+        let response;
+        try {
+          response = await provider.classify({ state: classifierStateForFact(fact), questions: { okf_type: question } });
+        } catch {
+          return { fact, kind: "threw" };
+        }
+        const checked = validateClassifierAnswer(response, "okf_type", question);
+        if (!checked.ok) return { fact, kind: "invalid", reason: checked.reason };
+        if (checked.answer.kind !== "choice") return { fact, kind: "invalid", reason: "kind_mismatch" };
+        if (checked.answer.confidence < minConfidence) return { fact, kind: "low_confidence" };
+        return { fact, kind: "accepted", okfType: checked.answer.choice };
+      }),
+      concurrency
+    );
+    const attempted = outcomes.filter((o) => o.kind !== "threw").map((o) => o.fact);
+    const skipped = outcomes.length - attempted.length;
+    const classifications = outcomes.filter((o) => o.kind === "accepted").map((o) => ({ id: o.fact.id, okf_type: o.okfType }));
+    const invalidCount = outcomes.filter((o) => o.kind === "invalid").length;
+    let typed = 0;
+    let failedValidation = 0;
+    let edgesAdded = 0;
+    let aborted = false;
+    if (attempted.length > 0) {
+      const applied = await this._applyOntologyBackfillBatch(
+        entityId,
+        { batch: attempted, classifications, ontologyUpdates: void 0 },
+        now
+      );
+      aborted = applied.abortedOntologyOff;
+      if (!aborted) {
+        typed = applied.typed;
+        failedValidation = invalidCount + applied.failedValidation;
+        edgesAdded = applied.edgesAdded;
+      }
+    }
+    const diagBuffer = new DiagnosticBuffer();
+    const diagBase = { entityId, operation: "ontologyBackfill", trigger: "call" };
+    for (const o of outcomes) {
+      if (o.kind === "low_confidence") {
+        diagBuffer.push({ ...diagBase, code: "classification_low_confidence", detail: { factId: o.fact.id, reason: "below_threshold" } });
+      } else if (o.kind === "invalid") {
+        diagBuffer.push({ ...diagBase, code: "classification_invalid", detail: { factId: o.fact.id, reason: o.reason } });
+      } else if (o.kind === "threw") {
+        diagBuffer.push({ ...diagBase, code: "classification_invalid", detail: { factId: o.fact.id, reason: "classify_threw" } });
+      }
+    }
+    diagBuffer.flush(this.options);
+    const counts = await this.entryRepo.countUntypedByEntityId(entityId, recheckCutoff);
+    if (aborted) {
+      return { scanned: 0, typed: 0, failedValidation: 0, edgesAdded: 0, skipped, remaining: 0, deferred: counts.deferred };
+    }
+    this.searchService.evictCache(entityId);
+    return {
+      scanned: candidates.length,
+      typed,
+      failedValidation,
+      edgesAdded,
+      skipped,
+      remaining: counts.eligible,
+      deferred: counts.deferred
+    };
+  }
   /**
    * Applies one parsed backfill batch in its own transaction. Per-batch rather
    * than one transaction for the pass, so mergeEmergentUpdates semantics and
@@ -3516,6 +3649,7 @@ function selectWithFloors(sortedRows, floors, maxResults) {
 }
 
 // src/services/RetrievalService.ts
+var EMPTY_ID_SET = /* @__PURE__ */ new Set();
 var RetrievalService = class {
   constructor(options, entryRepo, taskRepo, eventRepo, metadataRepo, searchService) {
     this.options = options;
@@ -3548,6 +3682,7 @@ var RetrievalService = class {
     }
     const rawMaxResults = options?.maxResults ?? config?.maxResults ?? config?.maxFtsResults ?? 10;
     const maxResults = Number.isFinite(rawMaxResults) ? Math.max(0, Math.trunc(rawMaxResults)) : 10;
+    const excludeDrafts = (options?.excludeDrafts ?? config?.excludeDrafts ?? false) === true;
     const trimmedQuery = query.trim();
     const sanitizedTierFloors = trimmedQuery && exposeMetadata ? validateTierFloors(
       entityIds,
@@ -3561,19 +3696,21 @@ var RetrievalService = class {
     const hybridWeight = options?.hybridWeight ?? config?.hybridWeight;
     const weight = hybridWeight !== void 0 && !Number.isNaN(hybridWeight) ? Math.max(0, Math.min(1, hybridWeight)) : void 0;
     const skipEmbed = weight === 0;
-    const embedFn = this.options.llmProvider.embed;
     let facts = [];
     let scoreByFactId;
     if (maxResults === 0) ; else if (trimmedQuery) {
       let usedEmbed = false;
       const scoredEntityIds = this._filterScoredEntities(entityIds, sanitizedTierWeights, options?.includeZeroWeightEntities);
+      const draftIds = excludeDrafts && scoredEntityIds.length > 0 ? await this.entryRepo.findDraftIdsByEntityIds(scoredEntityIds) : EMPTY_ID_SET;
+      const draftPad = draftIds.size;
+      const padLimit = (n) => n >= Number.MAX_SAFE_INTEGER ? n : n + draftPad;
       if (scoredEntityIds.length === 0) {
         usedEmbed = true;
-      } else if (!skipEmbed && embedFn) {
+      } else if (!skipEmbed && typeof this.options.llmProvider.embed === "function") {
         let rankerShouldRethrow = false;
         let pendingRankerFallbackError;
         try {
-          const queryVec = await embedFn(trimmedQuery);
+          const queryVec = await this.options.llmProvider.embed(trimmedQuery);
           if (queryVec.length === 0 || !queryVec.every((v) => typeof v === "number" && isFinite(v))) {
             throw new Error(
               "embed() returned an empty or non-finite vector. Falling back to keyword search."
@@ -3600,7 +3737,7 @@ var RetrievalService = class {
           let miniSearchScores;
           if (effectivePreFilterLimit !== void 0) {
             populateCache = false;
-            const preResults = this.searchService.searchKeyword(trimmedQuery, scoredEntityIds, Number.MAX_SAFE_INTEGER);
+            const preResults = this.searchService.searchKeyword(trimmedQuery, scoredEntityIds, Number.MAX_SAFE_INTEGER).filter((r) => !draftIds.has(r.id));
             if (preResults.length === 0) {
               candidateRows = null;
             } else {
@@ -3622,9 +3759,9 @@ var RetrievalService = class {
             }
           } else {
             if (useRanker) {
-              candidateRows = await this.entryRepo.findMetadataByEntityIds(scoredEntityIds);
+              candidateRows = this._withoutDrafts(await this.entryRepo.findMetadataByEntityIds(scoredEntityIds), draftIds);
             } else {
-              candidateRows = await this.entryRepo.findWithEmbeddingsByEntityIds(scoredEntityIds);
+              candidateRows = this._withoutDrafts(await this.entryRepo.findWithEmbeddingsByEntityIds(scoredEntityIds), draftIds);
             }
             if (weight !== void 0 && weight < 1) {
               miniSearchScores = this.searchService.getMiniSearchScores(trimmedQuery, scoredEntityIds);
@@ -3654,7 +3791,7 @@ var RetrievalService = class {
                       candidateRows: rowsForEntity,
                       weight,
                       miniSearchScores,
-                      limit: Math.max(maxResults * 2, maxResults + 50)
+                      limit: padLimit(Math.max(maxResults * 2, maxResults + 50))
                     });
                     return ranked.map((row) => ({ ...row, entity_id: scopedEntityId }));
                   })
@@ -3868,7 +4005,7 @@ var RetrievalService = class {
                   });
                 } else if (policy === "keyword") {
                   const hasActiveFloorsKeywordFallback = sanitizedTierFloors !== void 0 && Object.values(sanitizedTierFloors).some((f) => f > 0);
-                  const keywordOversampledLimit = hasActiveFloorsKeywordFallback ? Number.MAX_SAFE_INTEGER : Math.max(maxResults * 2, maxResults + 50);
+                  const keywordOversampledLimit = hasActiveFloorsKeywordFallback ? Number.MAX_SAFE_INTEGER : padLimit(Math.max(maxResults * 2, maxResults + 50));
                   const preFilteredIds = effectivePreFilterLimit !== void 0 ? new Set(candidateRows.map((r) => r.id)) : void 0;
                   const keywordResults = this.searchService.searchKeyword(trimmedQuery, scoredEntityIds, keywordOversampledLimit);
                   const topResults = preFilteredIds === void 0 ? keywordResults : keywordResults.filter((r) => preFilteredIds.has(r.id));
@@ -3909,6 +4046,7 @@ var RetrievalService = class {
                 // read() re-sorts after applying tier weights
               });
             }
+            if (draftPad > 0) scored = scored.filter((s) => !draftIds.has(s.id));
             if (scored.length > 0) {
               scored = scored.map((row) => ({
                 ...row,
@@ -3971,8 +4109,8 @@ var RetrievalService = class {
       }
       if (!usedEmbed && scoredEntityIds.length > 0) {
         const hasActiveFloors = sanitizedTierFloors !== void 0 && Object.values(sanitizedTierFloors).some((f) => f > 0);
-        const fallbackOversampledLimit = hasActiveFloors ? Number.MAX_SAFE_INTEGER : Math.max(maxResults * 2, maxResults + 50);
-        const results = this.searchService.searchKeyword(trimmedQuery, scoredEntityIds, fallbackOversampledLimit);
+        const fallbackOversampledLimit = hasActiveFloors ? Number.MAX_SAFE_INTEGER : padLimit(Math.max(maxResults * 2, maxResults + 50));
+        const results = this.searchService.searchKeyword(trimmedQuery, scoredEntityIds, fallbackOversampledLimit).filter((r) => !draftIds.has(r.id));
         const candidates = results.map((r) => ({
           id: r.id,
           entity_id: r.entity_id,
@@ -3996,7 +4134,7 @@ var RetrievalService = class {
         await this.entryRepo.trackAccess(ids, now);
       }
     } else {
-      facts = await this.entryRepo.findRecentByEntityIds(entityIds, maxResults);
+      facts = excludeDrafts ? await this.entryRepo.findRecentByEntityIds(entityIds, maxResults, void 0, { excludeDrafts: true }) : await this.entryRepo.findRecentByEntityIds(entityIds, maxResults);
     }
     const eventsLimit = Math.min(10 * entityIds.length, 100);
     const [tasks, events] = await Promise.all([
@@ -4032,6 +4170,9 @@ var RetrievalService = class {
   _tieBreakSort(items) {
     items.sort((a, b) => this._compareScoredRows(a, b));
   }
+  _withoutDrafts(rows, draftIds) {
+    return draftIds.size === 0 ? rows : rows.filter((row) => !draftIds.has(row.id));
+  }
   /**
    * Comparator for score + deterministic tie-break fields.
    * Negative return means "a ranks ahead of b" for descending score order.