ltcai 6.7.0 → 7.1.0

package/frontend/src/features/brain/BrainHome.tsx

@@ -7,7 +7,19 @@ import { t } from "@/i18n";
 import { BrainConversation } from "./BrainConversation";
 import { buildBrainProof, buildBrainReadiness, buildMemoryFragments, currentModelName, parseKnowledgeGraph } from "./brainData";
 import { DepthEmergence } from "./DepthEmergence";
-import { DEPTHS, type BrainDepth, type MemoryFragment, type Message } from "./types";
+import {
+  DEPTHS,
+  INGESTION_STAGE_ORDER,
+  type BrainDepth,
+  type BrainProof,
+  type EmergenceEvent,
+  type IngestionPipelineStage,
+  type IngestionSourceType,
+  type IngestionState,
+  type MemoryFragment,
+  type Message,
+  type MessageProof,
+} from "./types";
 
 export function BrainHome({
   brainState,
@@ -31,8 +43,27 @@ export function BrainHome({
   const [memoryFeedback, setMemoryFeedback] = React.useState<string | null>(null);
   const [uploadingDocument, setUploadingDocument] = React.useState(false);
   const [lastRecallQuery, setLastRecallQuery] = React.useState("");
+  const [ingestionStates, setIngestionStates] = React.useState<Record<IngestionSourceType, IngestionState | null>>({
+    file: null,
+    folder: null,
+    note: null,
+    web: null,
+  });
+  const [emergenceEvents, setEmergenceEvents] = React.useState<EmergenceEvent[]>([]);
   const streamRef = React.useRef<HTMLDivElement>(null);
   const recallTimerRef = React.useRef<number | null>(null);
+  const stageTimersRef = React.useRef<Record<IngestionSourceType, number[]>>({
+    file: [],
+    folder: [],
+    note: [],
+    web: [],
+  });
+  const pendingBaselineRef = React.useRef<
+    Partial<Record<IngestionSourceType, { memories: number; entities: number; label: string }>>
+  >({});
+  // Source types whose request has resolved and are awaiting count settle to record emergence.
+  const awaitingEmergenceRef = React.useRef<Set<IngestionSourceType>>(new Set());
+  const settleTimerRef = React.useRef<number | null>(null);
 
   const memoriesQ = useQuery({ queryKey: ["memoryManager"], queryFn: latticeApi.memoryManager });
   const historyQ = useQuery({ queryKey: ["chatHistory"], queryFn: latticeApi.chatHistory });
@@ -89,9 +120,156 @@ export function BrainHome({
   React.useEffect(() => {
     return () => {
       if (recallTimerRef.current !== null) window.clearTimeout(recallTimerRef.current);
+      for (const timers of Object.values(stageTimersRef.current)) {
+        for (const timer of timers) window.clearTimeout(timer);
+      }
+    };
+  }, []);
+
+  const clearStageTimers = React.useCallback((sourceType: IngestionSourceType) => {
+    for (const timer of stageTimersRef.current[sourceType]) window.clearTimeout(timer);
+    stageTimersRef.current[sourceType] = [];
+  }, []);
+
+  const setStage = React.useCallback((sourceType: IngestionSourceType, stage: IngestionPipelineStage) => {
+    setIngestionStates((prev) => {
+      const current = prev[sourceType];
+      if (!current) return prev;
+      return {
+        ...prev,
+        [sourceType]: {
+          ...current,
+          stage,
+          completedAt: stage === "complete" || stage === "error" ? Date.now() : current.completedAt,
+        },
+      };
+    });
+  }, []);
+
+  const beginIngestion = React.useCallback(
+    (sourceType: IngestionSourceType, label: string) => {
+      clearStageTimers(sourceType);
+      pendingBaselineRef.current[sourceType] = {
+        memories: memoryFragments.length,
+        entities: knowledgeConcepts.length,
+        label,
+      };
+      setIngestionStates((prev) => ({
+        ...prev,
+        [sourceType]: {
+          sourceType,
+          label,
+          stage: "preparing",
+          startedAt: Date.now(),
+          completedAt: null,
+          newMemories: 0,
+          newEntities: 0,
+        },
+      }));
+      // Progressive disclosure of the in-flight pipeline while the request runs.
+      const interim: IngestionPipelineStage[] = ["parsing", "embedding", "indexing"];
+      interim.forEach((stage, index) => {
+        const timer = window.setTimeout(() => setStage(sourceType, stage), 420 * (index + 1));
+        stageTimersRef.current[sourceType].push(timer);
+      });
+    },
+    [clearStageTimers, knowledgeConcepts.length, memoryFragments.length, setStage],
+  );
+
+  const resolveEmergence = React.useCallback(
+    (sourceType: IngestionSourceType, memoryCount: number, entityCount: number) => {
+      clearStageTimers(sourceType);
+      const baseline = pendingBaselineRef.current[sourceType];
+      const label = baseline?.label ?? "";
+      // Snapshot deltas after invalidation lands; cap at >=0 to avoid noise.
+      const newMemories = Math.max(0, memoryCount - (baseline?.memories ?? memoryCount));
+      const newEntities = Math.max(0, entityCount - (baseline?.entities ?? entityCount));
+      delete pendingBaselineRef.current[sourceType];
+      setIngestionStates((prev) => {
+        const current = prev[sourceType];
+        if (!current) return prev;
+        return {
+          ...prev,
+          [sourceType]: {
+            ...current,
+            stage: "complete",
+            completedAt: Date.now(),
+            newMemories,
+            newEntities,
+          },
+        };
+      });
+      setEmergenceEvents((events) =>
+        [
+          {
+            id: `${sourceType}-${Date.now()}`,
+            sourceType,
+            label,
+            newMemories,
+            newEntities,
+            at: Date.now(),
+          },
+          ...events,
+        ].slice(0, 10),
+      );
+    },
+    [clearStageTimers],
+  );
+
+  // Once a request resolves we wait for the refetched counts to settle, then record the
+  // real emergence delta. A fallback timer guarantees the panel never hangs on a stale count.
+  const markAwaitingEmergence = React.useCallback(
+    (sourceType: IngestionSourceType) => {
+      awaitingEmergenceRef.current.add(sourceType);
+      if (settleTimerRef.current !== null) window.clearTimeout(settleTimerRef.current);
+      settleTimerRef.current = window.setTimeout(() => {
+        for (const pending of Array.from(awaitingEmergenceRef.current)) {
+          resolveEmergence(pending, memoryFragments.length, knowledgeConcepts.length);
+          awaitingEmergenceRef.current.delete(pending);
+        }
+      }, 1600);
+    },
+    [knowledgeConcepts.length, memoryFragments.length, resolveEmergence],
+  );
+
+  // Flush awaiting ingestions as soon as the underlying counts change post-invalidation.
+  React.useEffect(() => {
+    if (awaitingEmergenceRef.current.size === 0) return;
+    for (const pending of Array.from(awaitingEmergenceRef.current)) {
+      const baseline = pendingBaselineRef.current[pending];
+      if (!baseline) {
+        awaitingEmergenceRef.current.delete(pending);
+        continue;
+      }
+      if (memoryFragments.length !== baseline.memories || knowledgeConcepts.length !== baseline.entities) {
+        resolveEmergence(pending, memoryFragments.length, knowledgeConcepts.length);
+        awaitingEmergenceRef.current.delete(pending);
+      }
+    }
+  }, [memoryFragments.length, knowledgeConcepts.length, resolveEmergence]);
+
+  React.useEffect(() => {
+    return () => {
+      if (settleTimerRef.current !== null) window.clearTimeout(settleTimerRef.current);
     };
   }, []);
 
+  const failIngestion = React.useCallback(
+    (sourceType: IngestionSourceType, reason: string) => {
+      clearStageTimers(sourceType);
+      delete pendingBaselineRef.current[sourceType];
+      setIngestionStates((prev) => {
+        const current = prev[sourceType];
+        if (!current) return prev;
+        return {
+          ...prev,
+          [sourceType]: { ...current, stage: "error", completedAt: Date.now(), error: reason },
+        };
+      });
+    },
+    [clearStageTimers],
+  );
+
   async function send() {
     const text = draft.trim();
     if (!text || streaming) return;
@@ -134,6 +312,7 @@ export function BrainHome({
       } else {
         setMemoryFeedback(t(language, "brain.saved", { topics: knowledgeConcepts.length, memories: memoryFragments.length }));
         setLastRecallQuery(text);
+        void attachAnswerProof(text);
       }
     } finally {
       setStreaming(false);
@@ -144,17 +323,35 @@ export function BrainHome({
     }
   }
 
+  async function attachAnswerProof(query: string) {
+    const proofResult = await latticeApi.memoryBrainProof(query, 4);
+    const proof = buildBrainProof(proofResult.data, modelName);
+    const messageProof = toMessageProof(proof, query, modelName);
+    setMessages((items) => {
+      const next = [...items];
+      for (let index = next.length - 1; index >= 0; index -= 1) {
+        if (next[index].role === "assistant") {
+          next[index] = { ...next[index], proof: messageProof };
+          break;
+        }
+      }
+      return next;
+    });
+  }
+
   async function uploadDocument(file: File) {
     if (uploadingDocument) return;
 
     setUploadingDocument(true);
     setMemoryFeedback(t(language, "brain.upload.pending", { name: file.name }));
     onBrainChange("recalling", 0.86);
+    beginIngestion("file", file.name);
 
     try {
       const result = await latticeApi.uploadDocument(file);
       if (result.error) {
         setMemoryFeedback(t(language, "brain.upload.failed", { reason: result.error }));
+        failIngestion("file", result.error);
         return;
       }
 
@@ -164,11 +361,93 @@ export function BrainHome({
       void qc.invalidateQueries({ queryKey: ["memoryManager"] });
       void qc.invalidateQueries({ queryKey: ["graph"] });
       void qc.invalidateQueries({ queryKey: ["memoryBrainProof"] });
+      markAwaitingEmergence("file");
     } finally {
       setUploadingDocument(false);
     }
   }
 
+  async function connectFolder(path: string) {
+    const target = path.trim();
+    if (!target) return;
+    setMemoryFeedback(t(language, "brain.ingest.folder.pending", { path: target }));
+    onBrainChange("recalling", 0.84);
+    beginIngestion("folder", target);
+    const result = await latticeApi.connectFolder(target);
+    if (result.error || !result.ok) {
+      setMemoryFeedback(t(language, "brain.ingest.folder.failed", { reason: result.error || "unavailable" }));
+      failIngestion("folder", result.error || "unavailable");
+      return;
+    }
+    setMemoryFeedback(t(language, "brain.ingest.folder.saved", { path: target }));
+    setLastRecallQuery(target);
+    triggerBrainRecall();
+    void refreshBrainProof(target);
+    markAwaitingEmergence("folder");
+  }
+
+  async function ingestNote(note: string) {
+    const content = note.trim();
+    if (!content) return;
+    setMemoryFeedback(t(language, "brain.ingest.note.pending"));
+    onBrainChange("recalling", 0.84);
+    beginIngestion("note", content.slice(0, 80));
+    const result = await latticeApi.ingestNote(content, content.slice(0, 80));
+    if (result.error || !result.ok) {
+      setMemoryFeedback(t(language, "brain.ingest.note.failed", { reason: result.error || "unavailable" }));
+      failIngestion("note", result.error || "unavailable");
+      return;
+    }
+    setMemoryFeedback(t(language, "brain.ingest.note.saved"));
+    setLastRecallQuery(content.slice(0, 120));
+    triggerBrainRecall();
+    void refreshBrainProof(content.slice(0, 120));
+    markAwaitingEmergence("note");
+  }
+
+  async function ingestWeb(url: string) {
+    const target = url.trim();
+    if (!target) return;
+    setMemoryFeedback(t(language, "brain.ingest.web.pending", { url: target }));
+    onBrainChange("recalling", 0.84);
+    beginIngestion("web", target);
+    const result = await latticeApi.browserReadUrl(target);
+    if (result.error || !result.ok) {
+      setMemoryFeedback(t(language, "brain.ingest.web.failed", { reason: result.error || "unavailable" }));
+      failIngestion("web", result.error || "unavailable");
+      return;
+    }
+    setMemoryFeedback(t(language, "brain.ingest.web.saved", { url: target }));
+    setLastRecallQuery(target);
+    triggerBrainRecall();
+    void refreshBrainProof(target);
+    markAwaitingEmergence("web");
+  }
+
+  async function refreshBrainProof(query = lastRecallQuery) {
+    await Promise.all([
+      qc.invalidateQueries({ queryKey: ["memoryManager"] }),
+      qc.invalidateQueries({ queryKey: ["graph"] }),
+      qc.invalidateQueries({ queryKey: ["memoryBrainProof"] }),
+    ]);
+    if (query.trim()) {
+      await attachAnswerProof(query);
+    }
+  }
+
+  async function verifyModelContinuity() {
+    const lastUserMessage = [...messages].reverse().find((message: Message) => message.role === "user");
+    const query = lastRecallQuery || lastUserMessage?.content || "";
+    if (!query.trim()) {
+      setMemoryFeedback(t(language, "brain.modelDemo.needQuestion"));
+      return;
+    }
+    setMemoryFeedback(t(language, "brain.modelDemo.checking", { model: modelName }));
+    await attachAnswerProof(query);
+    setLastRecallQuery(query);
+    setMemoryFeedback(t(language, "brain.modelDemo.done", { model: modelName }));
+  }
+
   function deepen() {
     setExplorationDepth((depth) => {
       const next = Math.min(5, depth + 1) as BrainDepth;
@@ -272,6 +551,8 @@ export function BrainHome({
         messages={messages}
         starterPrompts={starterPrompts}
         memoryFeedback={memoryFeedback}
+        ingestionStates={ingestionStates}
+        emergenceEvents={emergenceEvents}
         draft={draft}
         streaming={streaming}
         imageData={imageData}
@@ -285,8 +566,26 @@ export function BrainHome({
         onDraftChange={setDraft}
         onImageDataChange={setImageData}
         onUploadDocument={(file) => void uploadDocument(file)}
+        onConnectFolder={(path) => void connectFolder(path)}
+        onIngestNote={(note) => void ingestNote(note)}
+        onIngestWeb={(url) => void ingestWeb(url)}
+        onVerifyModelContinuity={() => void verifyModelContinuity()}
         onSend={() => void send()}
       />
     </main>
   );
 }
+
+function toMessageProof(proof: BrainProof, query: string, fallbackModelName: string): MessageProof {
+  return {
+    query,
+    model: proof.modelContinuity.activeModel || fallbackModelName,
+    provenAcrossModels: proof.modelContinuity.proven && proof.claims.keepsContextAcrossModels,
+    citations: proof.recall.items.slice(0, 4).map((item) => ({
+      id: item.id,
+      source: item.source,
+      title: item.title,
+      snippet: item.snippet,
+    })),
+  };
+}

package/frontend/src/features/brain/brainData.ts

@@ -33,7 +33,9 @@ export function parseKnowledgeGraph(data: unknown): KnowledgeGraphModel {
     const label = textValue(node, ["title", "label", "name"], id.replace(/^[^:]+:/, ""));
     const summary = textValue(node, ["summary", "description", "snippet"]) || textValue(metadata, ["summary", "description", "relative_path", "filename"]);
     const importance = clamp(numberValue(node, ["importance_norm", "importance", "score"]) || 0.5, 0.08, 1);
-    return [{ id, label, type, summary, importance }];
+    const createdAt = timestampValue(node, ["created_at", "createdAt", "added_at", "addedAt", "timestamp", "updated_at", "updatedAt"])
+      ?? timestampValue(metadata, ["created_at", "createdAt", "added_at", "addedAt", "timestamp", "updated_at", "updatedAt"]);
+    return [{ id, label, type, summary, importance, ...(createdAt !== undefined ? { createdAt } : {}) }];
   }).sort((left, right) => right.importance - left.importance);
   const ids = new Set(nodes.map((node) => node.id));
   const edges = rawEdges.flatMap((edge, index): RelationshipThread[] => {
@@ -205,6 +207,28 @@ function titleValue(record: ApiRecord, keys: string[], fallback = "") {
     .replace(/\b\w/g, (character) => character.toUpperCase());
 }
 
+// Parse a created/added timestamp into unix epoch milliseconds. Accepts ISO 8601
+// strings, unix seconds, or unix milliseconds. Returns undefined when absent or
+// unparseable so the time-exploration UI can fall back gracefully.
+function timestampValue(record: ApiRecord, keys: string[]): number | undefined {
+  for (const key of keys) {
+    const value = record[key];
+    if (typeof value === "number" && Number.isFinite(value)) {
+      // Heuristic: values below ~1e12 are seconds, otherwise milliseconds.
+      return value < 1e12 ? value * 1000 : value;
+    }
+    if (typeof value === "string" && value.trim()) {
+      const numeric = Number(value);
+      if (Number.isFinite(numeric) && /^\d+$/.test(value.trim())) {
+        return numeric < 1e12 ? numeric * 1000 : numeric;
+      }
+      const parsed = Date.parse(value);
+      if (Number.isFinite(parsed)) return parsed;
+    }
+  }
+  return undefined;
+}
+
 function numberValue(record: ApiRecord, keys: string[]) {
   for (const key of keys) {
     const value = Number(record[key]);

package/frontend/src/features/brain/graphLayout.ts

@@ -1,5 +1,16 @@
 import * as React from "react";
-import type { KnowledgeConcept } from "./types";
+import type { KnowledgeConcept, RelationshipThread } from "./types";
+
+// Return the set of node ids that share a direct (1-hop) edge with `nodeId`.
+// The focused node itself is not included.
+export function computeGraphNeighbors(nodeId: string, edges: RelationshipThread[]): Set<string> {
+  const neighbors = new Set<string>();
+  for (const edge of edges) {
+    if (edge.source === nodeId) neighbors.add(edge.target);
+    else if (edge.target === nodeId) neighbors.add(edge.source);
+  }
+  return neighbors;
+}
 
 export function layoutGraphNodes(nodes: KnowledgeConcept[], radiusX: number, radiusY: number) {
   return nodes.map((node, index) => {

package/frontend/src/features/brain/types.ts

@@ -6,6 +6,19 @@ export type BrainDepth = 1 | 2 | 3 | 4 | 5;
 export type Message = {
   role: "user" | "assistant";
   content: string;
+  proof?: MessageProof;
+};
+
+export type MessageProof = {
+  query: string;
+  model: string;
+  provenAcrossModels: boolean;
+  citations: Array<{
+    id: string;
+    source: string;
+    title: string;
+    snippet: string;
+  }>;
 };
 
 export type MemoryFragment = {
@@ -20,6 +33,9 @@ export type KnowledgeConcept = {
   type: string;
   summary: string;
   importance: number;
+  // Optional unix epoch (ms) the concept was added to the graph. Enables
+  // time-based exploration; absent when the backend does not yet emit it.
+  createdAt?: number;
 };
 
 export type RelationshipThread = {
@@ -87,6 +103,44 @@ export type BrainProof = {
   };
 };
 
+export type IngestionSourceType = "file" | "folder" | "note" | "web";
+
+export type IngestionPipelineStage =
+  | "preparing"
+  | "parsing"
+  | "embedding"
+  | "indexing"
+  | "complete"
+  | "error";
+
+export type IngestionState = {
+  sourceType: IngestionSourceType;
+  label: string;
+  stage: IngestionPipelineStage;
+  startedAt: number;
+  completedAt: number | null;
+  newMemories: number;
+  newEntities: number;
+  error?: string;
+};
+
+export type EmergenceEvent = {
+  id: string;
+  sourceType: IngestionSourceType;
+  label: string;
+  newMemories: number;
+  newEntities: number;
+  at: number;
+};
+
+export const INGESTION_STAGE_ORDER: IngestionPipelineStage[] = [
+  "preparing",
+  "parsing",
+  "embedding",
+  "indexing",
+  "complete",
+];
+
 export const DEPTHS: Array<{ level: BrainDepth; labelKey: string; state: BrainState }> = [
   { level: 1, labelKey: "brain.depthLabel.1", state: "idle" },
   { level: 2, labelKey: "brain.depthLabel.2", state: "recalling" },