@equationalapplications/core-llm-wiki 7.4.0 → 7.5.0

package/dist/index.d.mts

@@ -1,5 +1,5 @@
-import { M as MemoryBundle, F as FormatContextOptions, G as GraphNeighborhood, a as MemoryDump, b as FormattedMemoryDump, R as ReadOptions, S as SQLiteAdapter, W as WikiOptions, c as WikiMemory } from './testing-Dmh1kfkd.mjs';
-export { C as ChunkFailure, D as DegradedRecord, d as DraftPage, E as EmbedFactResult, e as EmbedFailureKind, f as EmbeddingMarkerKind, g as EntityStatus, h as ExtractedFact, i as ExtractedFactEdge, j as ExtractedFactWithOntology, k as ExtractedTask, l as GraphTraversalOptions, H as HEAL_BATCH_SIZE, m as HEAL_RECHECK_MS, n as HOOK_TIMEOUT_MARKER, o as HealResult, I as IngestDocumentResult, L as LLMProvider, O as ONTOLOGY_BACKFILL_BATCH_SIZE, p as ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, q as ONTOLOGY_BACKFILL_RECHECK_MS, r as OntologyBackfillResult, s as OntologyConfig, t as OntologyEdgeType, u as OntologyManifest, v as OntologyMode, w as OntologyNodeType, x as OntologyPromptContext, y as OntologyUpdates, P as PromptOverrides, z as PromptService, A as PrunePartialFailureError, B as ReembedResult, V as VectorRanker, J as VectorRankerFallback, K as VectorRankerRankArgs, N as VectorRankerSemanticResult, Q as WikiBusyError, T as WikiBusyOperation, U as WikiCheckpoint, X as WikiConfig, Y as WikiDiagnostic, Z as WikiDiagnosticCode, _ as WikiDiagnosticDetail, $ as WikiDiagnosticOperation, a0 as WikiDiagnosticSeverity, a1 as WikiDiagnosticTrigger, a2 as WikiDraftNotFound, a3 as WikiDuplicateHashError, a4 as WikiEdge, a5 as WikiEvent, a6 as WikiFact, a7 as WikiGraphNodeOwnershipConflict, a8 as WikiIngestEmptyError, a9 as WikiInvalidReadOptions, aa as WikiMemoryTestAccess, ab as WikiOutboxEvent, ac as WikiParseError, ad as WikiSourceRefHashCollision, ae as WikiStrictOntologyViolation, af as WikiTask, ag as WikiTransactionError, ah as validateManifest } from './testing-Dmh1kfkd.mjs';
+import { M as MemoryBundle, F as FormatContextOptions, G as GraphNeighborhood, a as MemoryDump, b as FormattedMemoryDump, R as ReadOptions, S as SQLiteAdapter, W as WikiOptions, c as WikiMemory } from './testing-ClpOUiLI.mjs';
+export { C as ChunkFailure, d as ClassifierAnswer, e as ClassifierQuestion, f as ClassifyRequest, g as ClassifyResponse, D as DegradedRecord, h as DraftPage, E as EmbedFactResult, i as EmbedFailureKind, j as EmbeddingMarkerKind, k as EntityStatus, l as ExtractedFact, m as ExtractedFactEdge, n as ExtractedFactWithOntology, o as ExtractedTask, p as GraphTraversalOptions, H as HEAL_BATCH_SIZE, q as HEAL_RECHECK_MS, r as HOOK_TIMEOUT_MARKER, s as HealResult, I as IngestDocumentResult, L as LLMProvider, O as ONTOLOGY_BACKFILL_BATCH_SIZE, t as ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, u as ONTOLOGY_BACKFILL_RECHECK_MS, v as OntologyBackfillResult, w as OntologyConfig, x as OntologyEdgeType, y as OntologyManifest, z as OntologyMode, A as OntologyNodeType, B as OntologyPromptContext, J as OntologyUpdates, P as PromptOverrides, K as PromptService, N as PrunePartialFailureError, Q as ReembedResult, V as VectorRanker, T as VectorRankerFallback, U as VectorRankerRankArgs, X as VectorRankerSemanticResult, Y as WikiBusyError, Z as WikiBusyOperation, _ as WikiCheckpoint, $ as WikiConfig, a0 as WikiDiagnostic, a1 as WikiDiagnosticCode, a2 as WikiDiagnosticDetail, a3 as WikiDiagnosticOperation, a4 as WikiDiagnosticSeverity, a5 as WikiDiagnosticTrigger, a6 as WikiDraftNotFound, a7 as WikiDuplicateHashError, a8 as WikiEdge, a9 as WikiEvent, aa as WikiFact, ab as WikiGraphNodeOwnershipConflict, ac as WikiIngestEmptyError, ad as WikiInvalidReadOptions, ae as WikiMemoryTestAccess, af as WikiOutboxEvent, ag as WikiParseError, ah as WikiSourceRefHashCollision, ai as WikiStrictOntologyViolation, aj as WikiTask, ak as WikiTransactionError, al as validateManifest } from './testing-ClpOUiLI.mjs';
 import { OkfFile } from '@equationalapplications/core-okf';
 import 'minisearch';
 

package/dist/index.d.ts

@@ -1,5 +1,5 @@
-import { M as MemoryBundle, F as FormatContextOptions, G as GraphNeighborhood, a as MemoryDump, b as FormattedMemoryDump, R as ReadOptions, S as SQLiteAdapter, W as WikiOptions, c as WikiMemory } from './testing-Dmh1kfkd.js';
-export { C as ChunkFailure, D as DegradedRecord, d as DraftPage, E as EmbedFactResult, e as EmbedFailureKind, f as EmbeddingMarkerKind, g as EntityStatus, h as ExtractedFact, i as ExtractedFactEdge, j as ExtractedFactWithOntology, k as ExtractedTask, l as GraphTraversalOptions, H as HEAL_BATCH_SIZE, m as HEAL_RECHECK_MS, n as HOOK_TIMEOUT_MARKER, o as HealResult, I as IngestDocumentResult, L as LLMProvider, O as ONTOLOGY_BACKFILL_BATCH_SIZE, p as ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, q as ONTOLOGY_BACKFILL_RECHECK_MS, r as OntologyBackfillResult, s as OntologyConfig, t as OntologyEdgeType, u as OntologyManifest, v as OntologyMode, w as OntologyNodeType, x as OntologyPromptContext, y as OntologyUpdates, P as PromptOverrides, z as PromptService, A as PrunePartialFailureError, B as ReembedResult, V as VectorRanker, J as VectorRankerFallback, K as VectorRankerRankArgs, N as VectorRankerSemanticResult, Q as WikiBusyError, T as WikiBusyOperation, U as WikiCheckpoint, X as WikiConfig, Y as WikiDiagnostic, Z as WikiDiagnosticCode, _ as WikiDiagnosticDetail, $ as WikiDiagnosticOperation, a0 as WikiDiagnosticSeverity, a1 as WikiDiagnosticTrigger, a2 as WikiDraftNotFound, a3 as WikiDuplicateHashError, a4 as WikiEdge, a5 as WikiEvent, a6 as WikiFact, a7 as WikiGraphNodeOwnershipConflict, a8 as WikiIngestEmptyError, a9 as WikiInvalidReadOptions, aa as WikiMemoryTestAccess, ab as WikiOutboxEvent, ac as WikiParseError, ad as WikiSourceRefHashCollision, ae as WikiStrictOntologyViolation, af as WikiTask, ag as WikiTransactionError, ah as validateManifest } from './testing-Dmh1kfkd.js';
+import { M as MemoryBundle, F as FormatContextOptions, G as GraphNeighborhood, a as MemoryDump, b as FormattedMemoryDump, R as ReadOptions, S as SQLiteAdapter, W as WikiOptions, c as WikiMemory } from './testing-ClpOUiLI.js';
+export { C as ChunkFailure, d as ClassifierAnswer, e as ClassifierQuestion, f as ClassifyRequest, g as ClassifyResponse, D as DegradedRecord, h as DraftPage, E as EmbedFactResult, i as EmbedFailureKind, j as EmbeddingMarkerKind, k as EntityStatus, l as ExtractedFact, m as ExtractedFactEdge, n as ExtractedFactWithOntology, o as ExtractedTask, p as GraphTraversalOptions, H as HEAL_BATCH_SIZE, q as HEAL_RECHECK_MS, r as HOOK_TIMEOUT_MARKER, s as HealResult, I as IngestDocumentResult, L as LLMProvider, O as ONTOLOGY_BACKFILL_BATCH_SIZE, t as ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, u as ONTOLOGY_BACKFILL_RECHECK_MS, v as OntologyBackfillResult, w as OntologyConfig, x as OntologyEdgeType, y as OntologyManifest, z as OntologyMode, A as OntologyNodeType, B as OntologyPromptContext, J as OntologyUpdates, P as PromptOverrides, K as PromptService, N as PrunePartialFailureError, Q as ReembedResult, V as VectorRanker, T as VectorRankerFallback, U as VectorRankerRankArgs, X as VectorRankerSemanticResult, Y as WikiBusyError, Z as WikiBusyOperation, _ as WikiCheckpoint, $ as WikiConfig, a0 as WikiDiagnostic, a1 as WikiDiagnosticCode, a2 as WikiDiagnosticDetail, a3 as WikiDiagnosticOperation, a4 as WikiDiagnosticSeverity, a5 as WikiDiagnosticTrigger, a6 as WikiDraftNotFound, a7 as WikiDuplicateHashError, a8 as WikiEdge, a9 as WikiEvent, aa as WikiFact, ab as WikiGraphNodeOwnershipConflict, ac as WikiIngestEmptyError, ad as WikiInvalidReadOptions, ae as WikiMemoryTestAccess, af as WikiOutboxEvent, ag as WikiParseError, ah as WikiSourceRefHashCollision, ai as WikiStrictOntologyViolation, aj as WikiTask, ak as WikiTransactionError, al as validateManifest } from './testing-ClpOUiLI.js';
 import { OkfFile } from '@equationalapplications/core-okf';
 import 'minisearch';
 

package/dist/index.js

@@ -5112,6 +5112,51 @@ var IngestionService = class {
   }
 };
 
+// src/utils/classifier.ts
+var isUnit = (v) => typeof v === "number" && Number.isFinite(v) && v >= 0 && v <= 1;
+function validateClassifierAnswer(response, key, question) {
+  try {
+    if (response === null || typeof response !== "object") return { ok: false, reason: "malformed" };
+    const answers = response.answers;
+    if (answers === null || typeof answers !== "object") return { ok: false, reason: "missing_answer" };
+    if (!Object.prototype.hasOwnProperty.call(answers, key)) return { ok: false, reason: "missing_answer" };
+    const raw = answers[key];
+    if (raw === null || typeof raw !== "object") return { ok: false, reason: "missing_answer" };
+    const a = raw;
+    if (a.kind !== question.kind) return { ok: false, reason: "kind_mismatch" };
+    if (question.kind === "choice") {
+      if (typeof a.choice !== "string" || !question.options.includes(a.choice)) return { ok: false, reason: "choice_not_offered" };
+      if (!isUnit(a.confidence)) return { ok: false, reason: "invalid_probability" };
+      const probs = a.probabilities;
+      if (probs === null || typeof probs !== "object" || Array.isArray(probs)) return { ok: false, reason: "invalid_probability" };
+      const probabilities = {};
+      for (const [k, v] of Object.entries(probs)) {
+        if (!isUnit(v)) return { ok: false, reason: "invalid_probability" };
+        probabilities[k] = v;
+      }
+      return { ok: true, answer: { kind: "choice", choice: a.choice, confidence: a.confidence, probabilities } };
+    }
+    if (question.kind === "binary") {
+      if (!isUnit(a.probability)) return { ok: false, reason: "invalid_probability" };
+      return { ok: true, answer: { kind: "binary", probability: a.probability } };
+    }
+    const maxScore = question.levels.length - 1;
+    if (typeof a.score !== "number" || !Number.isFinite(a.score) || a.score < 0 || a.score > maxScore) {
+      return { ok: false, reason: "score_out_of_range" };
+    }
+    if (!isUnit(a.confidence)) return { ok: false, reason: "invalid_probability" };
+    if (!Array.isArray(a.probabilities) || !a.probabilities.every(isUnit)) return { ok: false, reason: "invalid_probability" };
+    return { ok: true, answer: { kind: "score", score: a.score, confidence: a.confidence, probabilities: a.probabilities.slice() } };
+  } catch {
+    return { ok: false, reason: "malformed" };
+  }
+}
+function classifierStateForFact(fact) {
+  const parts = [fact.title, fact.body];
+  if (Array.isArray(fact.tags) && fact.tags.length > 0) parts.push(`Tags: ${fact.tags.join(", ")}`);
+  return parts.join("\n\n");
+}
+
 // src/services/BoundedLlmCall.ts
 var DEFAULT_BATCH_SIZE = 10;
 var ESTIMATED_OUTPUT_TOKENS_PER_ITEM = 150;
@@ -5994,7 +6039,7 @@ var MaintenanceService = class {
     if (!ontologyService) {
       return { ...zeroed, remaining: 0, deferred: 0 };
     }
-    const { mode } = await ontologyService.getEffectiveState(entityId);
+    const { mode, manifest: effectiveManifest } = await ontologyService.getEffectiveState(entityId);
     if (mode === "off") {
       const counts2 = await this.entryRepo.countUntypedByEntityId(entityId, recheckCutoff);
       return { ...zeroed, remaining: 0, deferred: counts2.deferred };
@@ -6004,6 +6049,14 @@ var MaintenanceService = class {
       const counts2 = await this.entryRepo.countUntypedByEntityId(entityId, recheckCutoff);
       return { ...zeroed, remaining: counts2.eligible, deferred: counts2.deferred };
     }
+    const classifierMode = options?.classifier ?? this.options.config?.ontology?.backfillClassifier ?? "llm";
+    const classify = this.options.llmProvider.classify;
+    if (classifierMode === "auto" && typeof classify === "function") {
+      const slugs = effectiveManifest.node_types.map((n) => n.type);
+      if (slugs.length > 0 && slugs.length <= 255) {
+        return this._runClassifierBackfill(entityId, candidates, effectiveManifest, now, recheckCutoff);
+      }
+    }
     const ontologyContext = await ontologyService.buildPromptContext(entityId);
     const toPromptShape = (f) => ({ id: f.id, title: f.title, body: f.body, tags: f.tags });
     const buildPrompt = (facts) => this.promptService.buildOntologyBackfillPrompt(
@@ -6082,6 +6135,87 @@ var MaintenanceService = class {
       deferred: counts.deferred
     };
   }
+  /**
+   * Classifier-mode backfill (spec §7.3): one `choice` question per untyped
+   * fact over the effective manifest's node-type slugs. Accepted answers go
+   * through `_applyOntologyBackfillBatch` exactly like LLM classifications.
+   * No edges are proposed.
+   */
+  async _runClassifierBackfill(entityId, candidates, manifest, now, recheckCutoff) {
+    const provider = this.options.llmProvider;
+    const rawMin = this.options.config?.ontology?.classifyMinConfidence;
+    const minConfidence = typeof rawMin === "number" && Number.isFinite(rawMin) && rawMin >= 0 && rawMin <= 1 ? rawMin : 0.5;
+    const rawConcurrency = this.options.config?.chunkConcurrency ?? 1;
+    const concurrency = Number.isFinite(rawConcurrency) && rawConcurrency >= 1 ? Math.floor(rawConcurrency) : 1;
+    const question = {
+      kind: "choice",
+      options: manifest.node_types.map((n) => n.type),
+      instructions: "Choose the ontology node type that best describes this fact.\n" + manifest.node_types.map((n) => `- ${n.type}: ${n.description}`).join("\n")
+    };
+    const outcomes = await withConcurrency(
+      candidates.map((fact) => async () => {
+        let response;
+        try {
+          response = await provider.classify({ state: classifierStateForFact(fact), questions: { okf_type: question } });
+        } catch {
+          return { fact, kind: "threw" };
+        }
+        const checked = validateClassifierAnswer(response, "okf_type", question);
+        if (!checked.ok) return { fact, kind: "invalid", reason: checked.reason };
+        if (checked.answer.kind !== "choice") return { fact, kind: "invalid", reason: "kind_mismatch" };
+        if (checked.answer.confidence < minConfidence) return { fact, kind: "low_confidence" };
+        return { fact, kind: "accepted", okfType: checked.answer.choice };
+      }),
+      concurrency
+    );
+    const attempted = outcomes.filter((o) => o.kind !== "threw").map((o) => o.fact);
+    const skipped = outcomes.length - attempted.length;
+    const classifications = outcomes.filter((o) => o.kind === "accepted").map((o) => ({ id: o.fact.id, okf_type: o.okfType }));
+    const invalidCount = outcomes.filter((o) => o.kind === "invalid").length;
+    let typed = 0;
+    let failedValidation = 0;
+    let edgesAdded = 0;
+    let aborted = false;
+    if (attempted.length > 0) {
+      const applied = await this._applyOntologyBackfillBatch(
+        entityId,
+        { batch: attempted, classifications, ontologyUpdates: void 0 },
+        now
+      );
+      aborted = applied.abortedOntologyOff;
+      if (!aborted) {
+        typed = applied.typed;
+        failedValidation = invalidCount + applied.failedValidation;
+        edgesAdded = applied.edgesAdded;
+      }
+    }
+    const diagBuffer = new DiagnosticBuffer();
+    const diagBase = { entityId, operation: "ontologyBackfill", trigger: "call" };
+    for (const o of outcomes) {
+      if (o.kind === "low_confidence") {
+        diagBuffer.push({ ...diagBase, code: "classification_low_confidence", detail: { factId: o.fact.id, reason: "below_threshold" } });
+      } else if (o.kind === "invalid") {
+        diagBuffer.push({ ...diagBase, code: "classification_invalid", detail: { factId: o.fact.id, reason: o.reason } });
+      } else if (o.kind === "threw") {
+        diagBuffer.push({ ...diagBase, code: "classification_invalid", detail: { factId: o.fact.id, reason: "classify_threw" } });
+      }
+    }
+    diagBuffer.flush(this.options);
+    const counts = await this.entryRepo.countUntypedByEntityId(entityId, recheckCutoff);
+    if (aborted) {
+      return { scanned: 0, typed: 0, failedValidation: 0, edgesAdded: 0, skipped, remaining: 0, deferred: counts.deferred };
+    }
+    this.searchService.evictCache(entityId);
+    return {
+      scanned: candidates.length,
+      typed,
+      failedValidation,
+      edgesAdded,
+      skipped,
+      remaining: counts.eligible,
+      deferred: counts.deferred
+    };
+  }
   /**
    * Applies one parsed backfill batch in its own transaction. Per-batch rather
    * than one transaction for the pass, so mergeEmergentUpdates semantics and
@@ -6785,8 +6919,7 @@ var EmbeddingService = class {
     }
   }
   async tryEmbedFact(fact, ctx) {
-    const embedFn = this.options.llmProvider.embed;
-    if (typeof embedFn !== "function") return { ok: false, kind: "no_provider" };
+    if (typeof this.options.llmProvider.embed !== "function") return { ok: false, kind: "no_provider" };
     let tagsStr;
     if (Array.isArray(fact.tags)) {
       tagsStr = fact.tags.join(" ");
@@ -6803,7 +6936,7 @@ var EmbeddingService = class {
     const text = clip(`${fact.title} ${fact.body} ${tagsStr}`.trim(), maxEmbedChars);
     let float32Vector;
     try {
-      const vector = await embedFn(text);
+      const vector = await this.options.llmProvider.embed(text);
       if (vector.length === 0 || !vector.every((v) => typeof v === "number" && isFinite(v))) {
         console.warn(`[WikiMemory] embedFact: embed() returned an invalid vector for ${fact.id}; skipping.`);
         this.reportEmbed(ctx, fact, "embedding_failed", "invalid_vector");
@@ -7062,7 +7195,6 @@ var RetrievalService = class {
     const hybridWeight = options?.hybridWeight ?? config?.hybridWeight;
     const weight = hybridWeight !== void 0 && !Number.isNaN(hybridWeight) ? Math.max(0, Math.min(1, hybridWeight)) : void 0;
     const skipEmbed = weight === 0;
-    const embedFn = this.options.llmProvider.embed;
     let facts = [];
     let scoreByFactId;
     if (maxResults === 0) ; else if (trimmedQuery) {
@@ -7073,11 +7205,11 @@ var RetrievalService = class {
       const padLimit = (n) => n >= Number.MAX_SAFE_INTEGER ? n : n + draftPad;
       if (scoredEntityIds.length === 0) {
         usedEmbed = true;
-      } else if (!skipEmbed && embedFn) {
+      } else if (!skipEmbed && typeof this.options.llmProvider.embed === "function") {
         let rankerShouldRethrow = false;
         let pendingRankerFallbackError;
         try {
-          const queryVec = await embedFn(trimmedQuery);
+          const queryVec = await this.options.llmProvider.embed(trimmedQuery);
           if (queryVec.length === 0 || !queryVec.every((v) => typeof v === "number" && isFinite(v))) {
             throw new Error(
               "embed() returned an empty or non-finite vector. Falling back to keyword search."