@remnic/core 9.60.1 → 9.61.0

package/src/storage/entity-store.ts

@@ -40,11 +40,12 @@ import {
   countEntityStructuredFacts,
   fingerprintEntityStructuredFacts,
   normalizeEntityName as normalizeEntityNameShared,
-  normalizeStructuredSectionFacts,
+  normalizeStructuredSectionFactsWithOrigins,
   parseEntityFile,
   serializeEntityFile,
   StorageManager,
 } from "../storage.js";
+import { parseOriginClass } from "../security/origin-authority.js";
 
 export interface EntityStoreDeps {
   /** Live class object of the host instance — shared static caches (see storage.ts storageManagerClass). */
@@ -204,6 +205,7 @@ export class EntityStore {
       source?: string;
       sessionKey?: string;
       principal?: string;
+      origin?: string;
       structuredSections?: EntityStructuredSection[];
     } = {},
   ): Promise<string> {
@@ -223,6 +225,7 @@ export class EntityStore {
       source?: string;
       sessionKey?: string;
       principal?: string;
+      origin?: string;
       structuredSections?: EntityStructuredSection[];
     } = {},
   ): Promise<string> {
@@ -283,26 +286,35 @@ export class EntityStore {
     const source = options.source?.trim() || undefined;
     const sessionKey = options.sessionKey?.trim() || undefined;
     const principal = options.principal?.trim() || undefined;
+    const origin = options.origin?.trim() ? parseOriginClass(options.origin) : undefined;
     const structuredSectionMap = new Map(
       (entity.structuredSections ?? []).map((section) => [section.key, {
         ...section,
         facts: [...section.facts],
+        ...(section.factOrigins ? { factOrigins: [...section.factOrigins] } : {}),
       }]),
     );
     for (const section of options.structuredSections ?? []) {
       const normalizedSection = normalizeEntityStructuredSection(type, section, this.deps.entitySchemas);
-      const normalizedFacts = normalizeStructuredSectionFacts(section.facts);
-      if (normalizedFacts.length === 0) continue;
+      const normalized = normalizeStructuredSectionFactsWithOrigins(section.facts, section.factOrigins, origin);
+      if (normalized.facts.length === 0) continue;
       const existingSection = structuredSectionMap.get(normalizedSection.key);
       if (!existingSection) {
         structuredSectionMap.set(normalizedSection.key, {
           key: normalizedSection.key,
           title: normalizedSection.title,
-          facts: normalizedFacts,
+          ...normalized,
         });
         continue;
       }
-      existingSection.facts = normalizeStructuredSectionFacts([...existingSection.facts, ...normalizedFacts]);
+      const existingOrigins = existingSection.facts.map((_, index) => existingSection.factOrigins?.[index]);
+      const incomingOrigins = normalized.facts.map((_, index) => normalized.factOrigins?.[index]);
+      const merged = normalizeStructuredSectionFactsWithOrigins(
+        [...existingSection.facts, ...normalized.facts],
+        [...existingOrigins, ...incomingOrigins],
+      );
+      existingSection.facts = merged.facts;
+      existingSection.factOrigins = merged.factOrigins;
       if (!existingSection.title.trim() && normalizedSection.title.trim()) {
         existingSection.title = normalizedSection.title;
       }
@@ -314,6 +326,7 @@ export class EntityStore {
         ...(source ? { source } : {}),
         ...(sessionKey ? { sessionKey } : {}),
         ...(principal ? { principal } : {}),
+        ...(origin ? { origin } : {}),
       };
       const alreadyPresent = entity.timeline.some((entry) =>
         entry.timestamp === nextEntry.timestamp
@@ -321,6 +334,7 @@ export class EntityStore {
         && entry.source === nextEntry.source
         && entry.sessionKey === nextEntry.sessionKey
         && entry.principal === nextEntry.principal
+        && entry.origin === nextEntry.origin
       );
       if (alreadyPresent) continue;
       entity.timeline.push(nextEntry);
@@ -762,26 +776,44 @@ export class EntityStore {
               (mergedEntity.structuredSections ?? []).map((section) => [section.key, {
                 ...section,
                 facts: [...section.facts],
+                factOrigins: section.factOrigins ? [...section.factOrigins] : undefined,
               }]),
             );
             for (const section of parsed.structuredSections ?? []) {
               const existingSection = mergedStructuredSectionMap.get(section.key);
               if (!existingSection) {
+                const facts: string[] = [];
+                const factOrigins: Array<string | undefined> = [];
+                for (const [index, fact] of section.facts.entries()) {
+                  const trimmed = fact.trim();
+                  if (!trimmed || facts.includes(trimmed)) continue;
+                  facts.push(trimmed);
+                  factOrigins.push(section.factOrigins?.[index]);
+                }
                 mergedStructuredSectionMap.set(section.key, {
                   key: section.key,
                   title: section.title,
-                  facts: [...new Set(section.facts.map((fact) => fact.trim()).filter((fact) => fact.length > 0))],
+                  facts,
+                  factOrigins: section.factOrigins ? factOrigins : undefined,
                 });
                 continue;
               }
 
-              const mergedFacts = new Set(existingSection.facts.map((fact) => fact.trim()));
-              for (const fact of section.facts) {
+              if (section.factOrigins && !existingSection.factOrigins) {
+                existingSection.factOrigins = existingSection.facts.map(() => undefined);
+              }
+              for (const [index, fact] of section.facts.entries()) {
                 const trimmed = fact.trim();
                 if (!trimmed) continue;
-                mergedFacts.add(trimmed);
+                const existingIndex = existingSection.facts.indexOf(trimmed);
+                const origin = section.factOrigins?.[index];
+                if (existingIndex === -1) {
+                  existingSection.facts.push(trimmed);
+                  existingSection.factOrigins?.push(origin);
+                } else if (existingSection.factOrigins?.[existingIndex] !== origin) {
+                  existingSection.factOrigins?.splice(existingIndex, 1, undefined);
+                }
               }
-              existingSection.facts = Array.from(mergedFacts);
               if (!existingSection.title.trim() && section.title.trim()) {
                 existingSection.title = section.title;
               }
@@ -802,9 +834,8 @@ export class EntityStore {
           }
         }
 
-        // Deduplicate timeline entries and derive facts from the timeline.
-        const timelineKeys = new Set<string>();
-        mergedEntity.timeline = mergedEntity.timeline.filter((entry) => {
+        const timelineByKey = new Map<string, (typeof mergedEntity.timeline)[number]>();
+        for (const entry of mergedEntity.timeline) {
           const key = JSON.stringify([
             entry.timestamp,
             entry.source ?? "",
@@ -812,10 +843,14 @@ export class EntityStore {
             entry.principal ?? "",
             entry.text,
           ]);
-          if (timelineKeys.has(key)) return false;
-          timelineKeys.add(key);
-          return true;
-        });
+          const existing = timelineByKey.get(key);
+          if (!existing) {
+            timelineByKey.set(key, entry);
+          } else if (existing.origin !== entry.origin) {
+            existing.origin = undefined;
+          }
+        }
+        mergedEntity.timeline = [...timelineByKey.values()];
         // Deduplicate relationships by target+label
         const relKeys = new Set<string>();
         mergedEntity.relationships = mergedEntity.relationships.filter((r) => {

package/src/storage/entity-timeline.ts

@@ -6,6 +6,7 @@
 import { createHash } from "node:crypto";
 import { matchEntitySchemaSection, normalizeEntityStructuredSection } from "../entity-schema.js";
 import type { EntityFile, EntityStructuredSection, EntityTimelineEntry, PluginConfig } from "../types.js";
+export const ENTITY_TIMELINE_METADATA_MARKER = "remnic-meta-v1";
 
 export function parseEntityTimelineBullet(bullet: string, fallbackTimestamp: string): EntityTimelineEntry | null {
   const trimmed = bullet.trim();
@@ -17,10 +18,11 @@ export function parseEntityTimelineBullet(bullet: string, fallbackTimestamp: str
     text: "",
   };
   const consumedMetadataSegments: string[] = [];
+  let metadataMarkerSeen = false;
   let literalSingleSourceSegment: string | undefined;
 
   if (!trimmed.startsWith("[")) {
-    entry.text = trimmed;
+    entry.text = unescapeEntityTimelineText(trimmed);
     return entry.text ? entry : null;
   }
 
@@ -42,6 +44,13 @@ export function parseEntityTimelineBullet(bullet: string, fallbackTimestamp: str
     if (end === -1) break;
     const rawSegment = rest.slice(0, end + 1);
     const token = rest.slice(1, end).trim();
+    const nextRest = rest.slice(end + 1).trimStart();
+    if (token === ENTITY_TIMELINE_METADATA_MARKER) {
+      metadataMarkerSeen = true;
+      consumedMetadataSegments.push(rawSegment);
+      rest = nextRest;
+      continue;
+    }
     const equalsIdx = token.indexOf("=");
     if (equalsIdx === -1) {
       if (rest === trimmed) {
@@ -53,7 +62,10 @@ export function parseEntityTimelineBullet(bullet: string, fallbackTimestamp: str
     const key = token.slice(0, equalsIdx).trim().toLowerCase();
     const value = unescapeEntityTimelineMetadataValue(token.slice(equalsIdx + 1).trim());
     if (!value) break;
-    const nextRest = rest.slice(end + 1).trimStart();
+    if (key === "remnic-origin" && !metadataMarkerSeen) {
+      entry.text = rest.trim();
+      return entry.text ? entry : null;
+    }
     switch (key) {
       case "source_meta":
         entry.source = value;
@@ -78,6 +90,9 @@ export function parseEntityTimelineBullet(bullet: string, fallbackTimestamp: str
       case "principal":
         entry.principal = value;
         break;
+      case "remnic-origin":
+        entry.origin = value;
+        break;
       default:
         entry.text = rest.trim();
         return entry.text ? entry : null;
@@ -93,8 +108,7 @@ export function parseEntityTimelineBullet(bullet: string, fallbackTimestamp: str
       text: `${literalSingleSourceSegment} ${rest}`.trim(),
     };
   }
-
-  entry.text = rest.trim();
+  entry.text = unescapeEntityTimelineText(rest.trim());
   if (!entry.text) return null;
   return entry;
 }
@@ -241,6 +255,13 @@ export function unescapeEntityTimelineMetadataValue(value: string): string {
   }
   return result;
 }
+export function escapeEntityTimelineText(value: string): string {
+  return /^\\*\[(?:remnic-origin=|remnic-meta-v1\])/i.test(value) ? `\\${value}` : value;
+}
+
+export function unescapeEntityTimelineText(value: string): string {
+  return /^\\+\[(?:remnic-origin=|remnic-meta-v1\])/i.test(value) ? value.slice(1) : value;
+}
 
 export function serializeEntityTimelineEntry(entry: EntityTimelineEntry): string {
   const tokens: string[] = [];
@@ -257,8 +278,12 @@ export function serializeEntityTimelineEntry(entry: EntityTimelineEntry): string
   if (entry.principal) {
     tokens.push(`[principal=${escapeEntityTimelineMetadataValue(entry.principal)}]`);
   }
+  if (entry.origin) {
+    tokens.push(`[${ENTITY_TIMELINE_METADATA_MARKER}]`);
+    tokens.push(`[remnic-origin=${escapeEntityTimelineMetadataValue(entry.origin)}]`);
+  }
   const serializedMetadata = tokens.length > 0 ? `${tokens.join(" ")} ` : "";
-  return `- ${serializedMetadata}${entry.text}`.trimEnd();
+  return `- ${serializedMetadata}${escapeEntityTimelineText(entry.text)}`.trimEnd();
 }
 
 export function dedupeEntityTimelineFacts(timeline: EntityTimelineEntry[]): string[] {
@@ -269,6 +294,32 @@ export function normalizeEntitySectionFact(value: string): string {
   return value.replace(/\s+/g, " ").trim();
 }
 
+export function normalizeStructuredSectionFactsWithOrigins(
+  facts: string[],
+  factOrigins?: Array<string | undefined>,
+  defaultOrigin?: string,
+): Pick<EntityStructuredSection, "facts" | "factOrigins"> {
+  const normalizedFacts: string[] = [];
+  const origins: Array<string | undefined> = [];
+  const seen = new Map<string, number>();
+  for (const [index, fact] of facts.entries()) {
+    const normalized = normalizeEntitySectionFact(fact);
+    if (!normalized) continue;
+    const existingIndex = seen.get(normalized) ?? -1;
+    const origin = factOrigins?.[index] ?? defaultOrigin;
+    if (existingIndex >= 0) {
+      if (origins[existingIndex] !== origin) origins[existingIndex] = undefined;
+      continue;
+    }
+    seen.set(normalized, normalizedFacts.length);
+    normalizedFacts.push(normalized);
+    origins.push(origin);
+  }
+  return {
+    facts: normalizedFacts,
+    ...(origins.some((origin) => origin !== undefined) ? { factOrigins: origins } : {}),
+  };
+}
 export function normalizeStructuredSectionFacts(facts: string[]): string[] {
   return [...new Set(facts.map((fact) => normalizeEntitySectionFact(fact)).filter((fact) => fact.length > 0))];
 }
@@ -304,14 +355,22 @@ export function compileEntityFacts(
   return facts;
 }
 
-function parseEntityStructuredSectionFacts(lines: string[]): string[] {
+function parseEntityStructuredSectionFacts(
+  lines: string[],
+): Pick<EntityStructuredSection, "facts" | "factOrigins"> {
   const facts: string[] = [];
+  const factOrigins: Array<string | undefined> = [];
   let currentBlock: string[] = [];
+  let currentOrigin: string | undefined;
 
   const flushCurrentBlock = (): void => {
     const normalized = normalizeEntitySectionFact(currentBlock.join(" "));
-    if (normalized.length > 0) facts.push(normalized);
+    if (normalized.length > 0) {
+      facts.push(normalized);
+      factOrigins.push(currentOrigin);
+    }
     currentBlock = [];
+    currentOrigin = undefined;
   };
 
   for (const rawLine of lines) {
@@ -322,16 +381,29 @@ function parseEntityStructuredSectionFacts(lines: string[]): string[] {
     }
     if (line.startsWith("- ")) {
       flushCurrentBlock();
-      currentBlock = [line.slice(2).trim()];
+      const bullet = line.slice(2).trim();
+      const metadataPrefix = `[${ENTITY_TIMELINE_METADATA_MARKER}] `;
+      const hasMarker = bullet.startsWith(metadataPrefix);
+      const originStart = metadataPrefix.length;
+      const originPrefix = hasMarker && bullet.startsWith("[remnic-origin=", originStart);
+      const originEndOffset = originPrefix
+        ? findEntityTimelineTokenEnd(bullet.slice(originStart))
+        : -1;
+      const originEnd = originEndOffset >= 0 ? originEndOffset + originStart : -1;
+      const token = originEnd >= 0 ? bullet.slice(originStart + 15, originEnd) : "";
+      currentBlock = [originEnd >= 0
+        ? unescapeEntityTimelineText(bullet.slice(originEnd + 1).trimStart())
+        : unescapeEntityTimelineText(hasMarker ? bullet.slice(metadataPrefix.length) : bullet)];
+      currentOrigin = originEnd >= 0
+        ? unescapeEntityTimelineMetadataValue(token)
+        : undefined;
       continue;
     }
     currentBlock.push(line);
   }
-
   flushCurrentBlock();
-  return [...new Set(facts)];
+  return normalizeStructuredSectionFactsWithOrigins(facts, factOrigins);
 }
-
 function looksLikeStructuredSectionFactList(lines: string[]): boolean {
   const firstNonBlank = lines.find((line) => line.trim().length > 0)?.trim() ?? "";
   return firstNonBlank.startsWith("- ");
@@ -355,27 +427,35 @@ export function partitionEntityStructuredSections(
       remainingExtraSections.push(section);
       continue;
     }
-    const facts = parseEntityStructuredSectionFacts(section.lines);
-    if (!matchedSection && facts.length === 0) {
+    const parsedFacts = parseEntityStructuredSectionFacts(section.lines);
+    if (!matchedSection && parsedFacts.facts.length === 0) {
       remainingExtraSections.push(section);
       continue;
     }
     const normalizedSection = matchedSection
       ? { key: matchedSection.key, title: matchedSection.title }
       : normalizeEntityStructuredSection(entityType, { key: section.title, title: section.title }, entitySchemas);
-    if (facts.length === 0) {
+    if (parsedFacts.facts.length === 0) {
       remainingExtraSections.push(section);
       continue;
     }
     const existing = structuredSectionIndex.get(normalizedSection.key);
     if (existing) {
-      existing.facts = normalizeStructuredSectionFacts([...existing.facts, ...facts]);
+      const existingOrigins = existing.facts.map((_, index) => existing.factOrigins?.[index]);
+      const incomingOrigins = parsedFacts.facts.map((_, index) => parsedFacts.factOrigins?.[index]);
+      const merged = normalizeStructuredSectionFactsWithOrigins(
+        [...existing.facts, ...parsedFacts.facts],
+        [...existingOrigins, ...incomingOrigins],
+      );
+      existing.facts = merged.facts;
+      existing.factOrigins = merged.factOrigins;
       continue;
     }
     const structuredSection: EntityStructuredSection = {
       key: normalizedSection.key,
       title: normalizedSection.title,
-      facts: normalizeStructuredSectionFacts(facts),
+      facts: parsedFacts.facts,
+      ...(parsedFacts.factOrigins ? { factOrigins: parsedFacts.factOrigins } : {}),
     };
     structuredSections.push(structuredSection);
     structuredSectionIndex.set(normalizedSection.key, structuredSection);

package/src/storage.ts

@@ -185,11 +185,15 @@ import {
   collectStructuredSectionFacts,
   compareEntityTimestamps,
   compileEntityFacts,
+  ENTITY_TIMELINE_METADATA_MARKER,
+  escapeEntityTimelineMetadataValue,
+  escapeEntityTimelineText,
   isEntitySynthesisStale,
   isEntitySynthesisTimelinePromotionBullet,
   latestEntityTimelineTimestamp,
   normalizeEntitySectionFact,
   normalizeStructuredSectionFacts,
+  normalizeStructuredSectionFactsWithOrigins,
   parseEntityTimelineBullet,
   partitionEntityStructuredSections,
   serializeEntityTimelineEntry,
@@ -220,6 +224,7 @@ export {
   fingerprintEntityStructuredFacts,
   isEntitySynthesisStale,
   normalizeStructuredSectionFacts,
+  normalizeStructuredSectionFactsWithOrigins,
 } from "./storage/entity-timeline.js";
 import {
   type ProjectedMemoryBrowseOptions,
@@ -244,7 +249,6 @@ import {
 } from "./write-envelope.js";
 // stripCitation import removed: legacy rebuild fallback was replaced by a
 // skip-with-warning strategy (Finding 1 — Uhol).  See ensureFactHashIndexAuthoritative.
-
 type SharedVersionKind = "memory-status" | "artifact-write" | "cold-write" | "memory-corpus" | "entity-mutation";
 
 type OfflineSyncDigestCacheEntry = {
@@ -255,15 +259,12 @@ type OfflineSyncDigestCacheEntry = {
   sha256: string;
   bytes: number;
 };
-
-
 export interface ReextractJobRequest {
   memoryId: string;
   model: string;
   requestedAt: string;
   source: "cli-migrate";
 }
-
 export interface MemoryLifecycleEventWriteOptions {
   at?: Date;
   actor?: string;
@@ -272,7 +273,6 @@ export interface MemoryLifecycleEventWriteOptions {
   relatedMemoryIds?: string[];
   correlationId?: string;
 }
-
 /**
  * Validate a Memory Worth counter (`mw_success` / `mw_fail`) before we persist
  * it. Rejects non-finite, non-integer, and negative values rather than silently
@@ -1420,10 +1420,12 @@ export function serializeEntityFile(entity: EntityFile, entitySchemas?: PluginCo
   const structuredSections = sortStructuredSectionsBySchema(
     entity.type,
     (entity.structuredSections ?? [])
-      .map((section) => ({
-        ...section,
-        facts: normalizeStructuredSectionFacts(section.facts),
-      }))
+      .map((section) => {
+        const normalized = normalizeStructuredSectionFactsWithOrigins(section.facts, section.factOrigins);
+        const result = { ...section, ...normalized };
+        if (normalized.factOrigins === undefined) delete result.factOrigins;
+        return result;
+      })
       .filter((section) => section.facts.length > 0),
     entitySchemas
   );
@@ -1443,7 +1445,6 @@ export function serializeEntityFile(entity: EntityFile, entitySchemas?: PluginCo
   const synthesisStructuredFactCount = entity.synthesisStructuredFactCount;
   const synthesisStructuredFactDigest = entity.synthesisStructuredFactDigest?.trim() || "";
   const synthesisVersion = entity.synthesisVersion ?? (synthesis ? 1 : 0);
-
   const lines: string[] = [
     "---",
     `created: ${created}`,
@@ -1464,14 +1465,12 @@ export function serializeEntityFile(entity: EntityFile, entitySchemas?: PluginCo
     `**Updated:** ${updated}`,
     "",
   ];
-
   if ((entity.preSectionLines ?? []).length > 0) {
     lines.push(...(entity.preSectionLines ?? []));
     if (entity.preSectionLines?.[entity.preSectionLines.length - 1] !== "") {
       lines.push("");
     }
   }
-
   lines.push("## Synthesis", "");
   if (synthesis) {
     lines.push(synthesis);
@@ -1485,7 +1484,6 @@ export function serializeEntityFile(entity: EntityFile, entitySchemas?: PluginCo
     }
     lines.push("");
   }
-
   if (legacyFacts.length > 0) {
     lines.push("## Facts", "");
     for (const fact of legacyFacts) {
@@ -1493,15 +1491,17 @@ export function serializeEntityFile(entity: EntityFile, entitySchemas?: PluginCo
     }
     lines.push("");
   }
-
   for (const section of structuredSections) {
     lines.push(`## ${section.title}`, "");
-    for (const fact of section.facts) {
-      lines.push(`- ${fact}`);
+    for (const [index, fact] of section.facts.entries()) {
+      const origin = section.factOrigins?.[index];
+      const prefix = origin
+        ? `[${ENTITY_TIMELINE_METADATA_MARKER}] [remnic-origin=${escapeEntityTimelineMetadataValue(origin)}] `
+        : "";
+      lines.push(`- ${prefix}${escapeEntityTimelineText(fact)}`);
     }
     lines.push("");
   }
-
   // Connected to (optional)
   if (entity.relationships.length > 0) {
     lines.push("## Connected to", "");
@@ -1510,7 +1510,6 @@ export function serializeEntityFile(entity: EntityFile, entitySchemas?: PluginCo
     }
     lines.push("");
   }
-
   // Activity (optional)
   if (entity.activity.length > 0) {
     lines.push("## Activity", "");
@@ -4239,6 +4238,7 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
       source?: string;
       sessionKey?: string;
       principal?: string;
+      origin?: string;
       structuredSections?: EntityStructuredSection[];
     } = {}
   ): Promise<string> {

package/src/types.ts

@@ -3374,19 +3374,19 @@ export interface EntityActivityEntry {
   date: string;
   note: string;
 }
-
 export interface EntityTimelineEntry {
   timestamp: string;
   text: string;
   source?: string;
   sessionKey?: string;
   principal?: string;
+  origin?: string;
 }
-
 export interface EntityStructuredSection {
   key: string;
   title: string;
   facts: string[];
+  factOrigins?: Array<string | undefined>;
 }
 
 export interface EntitySchemaSectionDefinition {