@chendpoc/pi-memory 0.1.12 → 0.2.0

package/src/trainer/bundleBuilder.ts

@@ -1,192 +0,0 @@
-import { createHash } from "node:crypto";
-import fs from "node:fs/promises";
-import path from "node:path";
-
-import type { ResolvedGraph, ResolvedEntity, ResolvedRelation } from "./entityResolver.js";
-import type { ExtractedEvent } from "./extractFacts.js";
-import type { BundleManifest, BundleManifestFile } from "../sidecar/bundle.js";
-
-export interface BundleData {
-  graph: ResolvedGraph;
-  events: ExtractedEvent[];
-}
-
-/** TLM-compatible entity record for the graph data file. */
-interface BundleEntity {
-  entity_id: string;
-  label: string;
-  type: string;
-  aliases: string[];
-  mention_count: number;
-  distinct_session_count: number;
-}
-
-/** TLM-compatible relation edge for the graph data file. */
-interface BundleEdge {
-  head_entity_id: string;
-  relation: string;
-  tail_entity_id: string;
-  supporting_event_ids: string[];
-  evidence: string;
-}
-
-/** TLM-compatible event record. */
-interface BundleEvent {
-  event_id: string;
-  description: string;
-  session_id: string;
-  timestamp: string;
-}
-
-interface GraphDataFile {
-  entities: BundleEntity[];
-  edges: BundleEdge[];
-  events: BundleEvent[];
-}
-
-function eventId(ev: ExtractedEvent, idx: number): string {
-  const hash = createHash("sha256")
-    .update(`${ev.sessionId}:${ev.turnIndex}:${idx}`)
-    .digest("hex")
-    .slice(0, 12);
-  return `ev_${hash}`;
-}
-
-function distinctSessions(ent: ResolvedEntity): number {
-  const s = new Set(ent.mentions.map((m) => m.sessionId));
-  return s.size;
-}
-
-function buildGraphData(
-  graph: ResolvedGraph,
-  events: ExtractedEvent[],
-): GraphDataFile {
-  const eventRecords: BundleEvent[] = events.map((ev, i) => ({
-    event_id: eventId(ev, i),
-    description: ev.description,
-    session_id: ev.sessionId,
-    timestamp: ev.timestamp,
-  }));
-
-  const eventsBySession = new Map<string, string[]>();
-  for (const ev of eventRecords) {
-    let arr = eventsBySession.get(ev.session_id);
-    if (!arr) {
-      arr = [];
-      eventsBySession.set(ev.session_id, arr);
-    }
-    arr.push(ev.event_id);
-  }
-
-  const entities: BundleEntity[] = graph.entities.map((ent) => ({
-    entity_id: ent.id,
-    label: ent.canonicalName,
-    type: ent.type,
-    aliases: ent.aliases,
-    mention_count: ent.mentions.length,
-    distinct_session_count: distinctSessions(ent),
-  }));
-
-  const edges: BundleEdge[] = graph.relations.map((rel) => {
-    const evIds = eventsBySession.get(rel.sessionId) ?? [];
-    return {
-      head_entity_id: rel.headEntityId,
-      relation: rel.relation,
-      tail_entity_id: rel.tailEntityId,
-      supporting_event_ids: evIds.slice(0, 5),
-      evidence: rel.evidence,
-    };
-  });
-
-  return { entities, edges, events: eventRecords };
-}
-
-async function writeJsonFile(
-  dir: string,
-  filename: string,
-  data: unknown,
-): Promise<BundleManifestFile> {
-  const content = JSON.stringify(data, null, 2) + "\n";
-  const filePath = path.join(dir, filename);
-  await fs.writeFile(filePath, content, "utf8");
-  const sha256 = createHash("sha256").update(content).digest("hex");
-  return {
-    path: filename,
-    size: Buffer.byteLength(content, "utf8"),
-    sha256,
-  };
-}
-
-export interface BuildBundleOptions {
-  outputDir: string;
-  bundleVersion?: string;
-}
-
-export interface BuildBundleResult {
-  bundleDir: string;
-  manifest: BundleManifest;
-  stats: {
-    entityCount: number;
-    edgeCount: number;
-    eventCount: number;
-  };
-}
-
-/**
- * Build a TLM-compatible bundle directory from resolved graph + events.
- *
- * Produces:
- *   <outputDir>/bundles/<iso-ts>/
- *     manifest.json
- *     graph.json      — entities, edges, events
- */
-export async function buildBundle(
-  data: BundleData,
-  opts: BuildBundleOptions,
-): Promise<BuildBundleResult> {
-  const bundleTs = new Date()
-    .toISOString()
-    .replace(/[:.]/g, "-");
-  const bundleVersion = opts.bundleVersion ?? "0.6.0";
-
-  const bundlesDir = path.join(opts.outputDir, "bundles");
-  const bundleDir = path.join(bundlesDir, bundleTs);
-  await fs.mkdir(bundleDir, { recursive: true, mode: 0o700 });
-
-  const graphData = buildGraphData(data.graph, data.events);
-  const files: BundleManifestFile[] = [];
-
-  const graphFile = await writeJsonFile(bundleDir, "graph.json", graphData);
-  files.push(graphFile);
-
-  const totalSize = files.reduce((sum, f) => sum + f.size, 0);
-  const integrityInput = files.map((f) => f.sha256).join(":");
-  const integritySha256 = createHash("sha256")
-    .update(integrityInput)
-    .digest("hex");
-
-  const manifest: BundleManifest = {
-    bundle_ts: bundleTs,
-    bundle_version: bundleVersion,
-    size_bytes: totalSize,
-    integrity_sha256: integritySha256,
-    files,
-  };
-
-  const manifestContent = JSON.stringify(manifest, null, 2) + "\n";
-  await fs.writeFile(
-    path.join(bundleDir, "manifest.json"),
-    manifestContent,
-    "utf8",
-  );
-
-  return {
-    bundleDir,
-    manifest,
-    stats: {
-      entityCount: graphData.entities.length,
-      edgeCount: graphData.edges.length,
-      eventCount: graphData.events.length,
-    },
-  };
-}

package/src/trainer/bundleLoader.ts

@@ -1,105 +0,0 @@
-import fs from "node:fs/promises";
-import path from "node:path";
-
-import type { ResolvedEntity, ResolvedRelation, ResolvedGraph } from "./entityResolver.js";
-import type { ExtractedEvent, EntityType } from "./extractFacts.js";
-
-interface BundleEntityRecord {
-  entity_id: string;
-  label: string;
-  type: string;
-  aliases: string[];
-  mention_count: number;
-  distinct_session_count: number;
-}
-
-interface BundleEdgeRecord {
-  head_entity_id: string;
-  relation: string;
-  tail_entity_id: string;
-  supporting_event_ids: string[];
-  evidence: string;
-}
-
-interface BundleEventRecord {
-  event_id: string;
-  description: string;
-  session_id: string;
-  timestamp: string;
-}
-
-interface GraphDataFile {
-  entities: BundleEntityRecord[];
-  edges: BundleEdgeRecord[];
-  events: BundleEventRecord[];
-}
-
-const VALID_ENTITY_TYPES = new Set<string>([
-  "Person", "Company", "Organization", "Project", "Tool",
-  "Language", "Concept", "Document", "File", "Location",
-  "Product", "Website", "Platform", "Unknown",
-]);
-
-function toEntityType(t: string): EntityType {
-  return VALID_ENTITY_TYPES.has(t) ? (t as EntityType) : "Unknown";
-}
-
-export interface ExistingBundle {
-  graph: ResolvedGraph;
-  events: ExtractedEvent[];
-}
-
-/**
- * Load the existing bundle's graph.json from `<bundleRoot>/current/graph.json`.
- * Returns null when no current bundle exists or graph.json is missing/unparseable.
- */
-export async function loadExistingBundle(
-  bundleRoot: string,
-): Promise<ExistingBundle | null> {
-  const graphPath = path.join(bundleRoot, "current", "graph.json");
-
-  let raw: string;
-  try {
-    raw = await fs.readFile(graphPath, "utf8");
-  } catch {
-    return null;
-  }
-
-  let data: GraphDataFile;
-  try {
-    data = JSON.parse(raw) as GraphDataFile;
-  } catch {
-    return null;
-  }
-
-  if (!data.entities || !data.edges) return null;
-
-  const entities: ResolvedEntity[] = (data.entities ?? []).map((e) => ({
-    id: e.entity_id,
-    canonicalName: e.label,
-    type: toEntityType(e.type),
-    aliases: e.aliases ?? [],
-    mentions: [],
-  }));
-
-  const relations: ResolvedRelation[] = (data.edges ?? []).map((e) => ({
-    headEntityId: e.head_entity_id,
-    relation: e.relation,
-    tailEntityId: e.tail_entity_id,
-    sessionId: "",
-    turnIndex: 0,
-    evidence: e.evidence ?? "",
-  }));
-
-  const events: ExtractedEvent[] = (data.events ?? []).map((e) => ({
-    description: e.description,
-    sessionId: e.session_id,
-    timestamp: e.timestamp,
-    turnIndex: 0,
-  }));
-
-  return {
-    graph: { entities, relations },
-    events,
-  };
-}

package/src/trainer/deltaMerge.ts

@@ -1,221 +0,0 @@
-import type { ResolvedGraph, ResolvedEntity, ResolvedRelation } from "./entityResolver.js";
-import type { ExtractedEvent } from "./extractFacts.js";
-import { UPDATE_SIGNAL_RE } from "./extractFacts.js";
-
-export type DeltaOp = "add" | "update" | "delete" | "skip";
-
-export interface DeltaLogEntry {
-  op: DeltaOp;
-  kind: "entity" | "edge" | "event";
-  detail: string;
-}
-
-export interface DeltaLog {
-  entries: DeltaLogEntry[];
-  added: number;
-  updated: number;
-  deleted: number;
-  skipped: number;
-}
-
-export interface MergeResult {
-  graph: ResolvedGraph;
-  events: ExtractedEvent[];
-  delta: DeltaLog;
-}
-
-function edgeKey(headId: string, relation: string, tailId: string): string {
-  return `${headId}\0${relation}\0${tailId}`;
-}
-
-function headRelKey(headId: string, relation: string): string {
-  return `${headId}\0${relation}`;
-}
-
-/**
- * Merge new extracted graph + events into an existing bundle graph,
- * producing a combined result with delta operation log.
- *
- * Operations:
- * - SKIP: entity+relation+tail already present with same value
- * - ADD: genuinely new entity/edge/event
- * - UPDATE: same head+relation but different tail, with update signal in evidence
- * - DELETE: negated relation matches existing edge
- */
-export function deltaMerge(
-  existing: { graph: ResolvedGraph; events: ExtractedEvent[] },
-  incoming: { graph: ResolvedGraph; events: ExtractedEvent[] },
-): MergeResult {
-  const delta: DeltaLog = { entries: [], added: 0, updated: 0, deleted: 0, skipped: 0 };
-
-  function log(op: DeltaOp, kind: DeltaLogEntry["kind"], detail: string): void {
-    delta.entries.push({ op, kind, detail });
-    switch (op) {
-      case "add": delta.added++; break;
-      case "update": delta.updated++; break;
-      case "delete": delta.deleted++; break;
-      case "skip": delta.skipped++; break;
-    }
-  }
-
-  // ── Entities ──
-  const entityMap = new Map<string, ResolvedEntity>();
-  for (const ent of existing.graph.entities) {
-    entityMap.set(ent.id, { ...ent, mentions: [...ent.mentions] });
-  }
-
-  for (const ent of incoming.graph.entities) {
-    const prev = entityMap.get(ent.id);
-    if (prev) {
-      const newMentions = ent.mentions.filter(
-        (m) => !prev.mentions.some(
-          (pm) => pm.sessionId === m.sessionId && pm.turnIndex === m.turnIndex,
-        ),
-      );
-      if (newMentions.length > 0) {
-        prev.mentions.push(...newMentions);
-        log("update", "entity", `${ent.canonicalName}: +${newMentions.length} mentions`);
-      } else {
-        log("skip", "entity", ent.canonicalName);
-      }
-    } else {
-      entityMap.set(ent.id, { ...ent, mentions: [...ent.mentions] });
-      log("add", "entity", ent.canonicalName);
-    }
-  }
-
-  // ── Edges: collect negations and normal edges ──
-  const existingEdgeMap = new Map<string, ResolvedRelation>();
-  const existingByHeadRel = new Map<string, ResolvedRelation[]>();
-  for (const edge of existing.graph.relations) {
-    const key = edgeKey(edge.headEntityId, edge.relation, edge.tailEntityId);
-    existingEdgeMap.set(key, edge);
-    const hrKey = headRelKey(edge.headEntityId, edge.relation);
-    let arr = existingByHeadRel.get(hrKey);
-    if (!arr) { arr = []; existingByHeadRel.set(hrKey, arr); }
-    arr.push(edge);
-  }
-
-  const negatedRelations: ResolvedRelation[] = [];
-  const normalRelations: ResolvedRelation[] = [];
-  for (const rel of incoming.graph.relations) {
-    if (rel.negated) {
-      negatedRelations.push(rel);
-    } else {
-      normalRelations.push(rel);
-    }
-  }
-
-  // Process negations → DELETE from existing
-  const deletedKeys = new Set<string>();
-  for (const neg of negatedRelations) {
-    if (neg.headEntityId) {
-      const key = edgeKey(neg.headEntityId, neg.relation, neg.tailEntityId);
-      if (existingEdgeMap.has(key)) {
-        deletedKeys.add(key);
-        const ent = entityMap.get(neg.tailEntityId);
-        log("delete", "edge", `${neg.headEntityId}:${neg.relation}:${ent?.canonicalName ?? neg.tailEntityId}`);
-      }
-    } else {
-      for (const [key, edge] of existingEdgeMap) {
-        if (edge.relation === neg.relation && edge.tailEntityId === neg.tailEntityId) {
-          deletedKeys.add(key);
-          const headEnt = entityMap.get(edge.headEntityId);
-          const tailEnt = entityMap.get(edge.tailEntityId);
-          log("delete", "edge", `${headEnt?.canonicalName ?? edge.headEntityId}:${neg.relation}:${tailEnt?.canonicalName ?? edge.tailEntityId}`);
-        }
-      }
-    }
-  }
-
-  // Process normal edges → ADD / SKIP / UPDATE
-  const mergedEdges: ResolvedRelation[] = [];
-
-  for (const [key, edge] of existingEdgeMap) {
-    if (!deletedKeys.has(key)) {
-      mergedEdges.push(edge);
-    }
-  }
-
-  for (const rel of normalRelations) {
-    const key = edgeKey(rel.headEntityId, rel.relation, rel.tailEntityId);
-
-    if (existingEdgeMap.has(key) && !deletedKeys.has(key)) {
-      const tailEnt = entityMap.get(rel.tailEntityId);
-      log("skip", "edge", `${rel.headEntityId}:${rel.relation}:${tailEnt?.canonicalName ?? rel.tailEntityId}`);
-      continue;
-    }
-
-    if (deletedKeys.has(key)) {
-      // Re-adding a previously-deleted edge is still an add
-    }
-
-    const hasUpdateSignal = UPDATE_SIGNAL_RE.test(rel.evidence);
-    if (hasUpdateSignal) {
-      const hrKey = headRelKey(rel.headEntityId, rel.relation);
-      const existingForHR = existingByHeadRel.get(hrKey);
-      if (existingForHR) {
-        for (const oldEdge of existingForHR) {
-          if (oldEdge.tailEntityId !== rel.tailEntityId) {
-            const oldKey = edgeKey(oldEdge.headEntityId, oldEdge.relation, oldEdge.tailEntityId);
-            if (!deletedKeys.has(oldKey)) {
-              deletedKeys.add(oldKey);
-              const idx = mergedEdges.findIndex(
-                (e) => e.headEntityId === oldEdge.headEntityId &&
-                       e.relation === oldEdge.relation &&
-                       e.tailEntityId === oldEdge.tailEntityId,
-              );
-              if (idx !== -1) mergedEdges.splice(idx, 1);
-              const oldTailEnt = entityMap.get(oldEdge.tailEntityId);
-              const newTailEnt = entityMap.get(rel.tailEntityId);
-              log("update", "edge",
-                `${rel.relation}: ${oldTailEnt?.canonicalName ?? oldEdge.tailEntityId} → ${newTailEnt?.canonicalName ?? rel.tailEntityId}`);
-            }
-          }
-        }
-      }
-    }
-
-    if (!mergedEdges.some(
-      (e) => e.headEntityId === rel.headEntityId &&
-             e.relation === rel.relation &&
-             e.tailEntityId === rel.tailEntityId,
-    )) {
-      mergedEdges.push(rel);
-      if (!hasUpdateSignal || !existingByHeadRel.has(headRelKey(rel.headEntityId, rel.relation))) {
-        const tailEnt = entityMap.get(rel.tailEntityId);
-        log("add", "edge", `${rel.headEntityId}:${rel.relation}:${tailEnt?.canonicalName ?? rel.tailEntityId}`);
-      }
-    }
-  }
-
-  // ── Events ──
-  const eventSigSet = new Set<string>();
-  const mergedEvents: ExtractedEvent[] = [];
-
-  for (const ev of existing.events) {
-    const sig = `${ev.sessionId}\0${ev.turnIndex}\0${ev.description.slice(0, 80)}`;
-    eventSigSet.add(sig);
-    mergedEvents.push(ev);
-  }
-
-  for (const ev of incoming.events) {
-    const sig = `${ev.sessionId}\0${ev.turnIndex}\0${ev.description.slice(0, 80)}`;
-    if (eventSigSet.has(sig)) {
-      log("skip", "event", ev.description.slice(0, 60));
-    } else {
-      eventSigSet.add(sig);
-      mergedEvents.push(ev);
-      log("add", "event", ev.description.slice(0, 60));
-    }
-  }
-
-  return {
-    graph: {
-      entities: [...entityMap.values()],
-      relations: mergedEdges,
-    },
-    events: mergedEvents,
-    delta,
-  };
-}

package/src/trainer/entityResolver.ts

@@ -1,140 +0,0 @@
-import { createHash } from "node:crypto";
-
-import type {
-  ExtractedEntity,
-  ExtractedRelation,
-  EntityMention,
-  EntityType,
-} from "./extractFacts.js";
-
-export interface ResolvedEntity {
-  id: string;
-  canonicalName: string;
-  type: EntityType;
-  aliases: string[];
-  mentions: EntityMention[];
-}
-
-export interface ResolvedRelation {
-  headEntityId: string;
-  relation: string;
-  tailEntityId: string;
-  sessionId: string;
-  turnIndex: number;
-  evidence: string;
-  /** True when the session explicitly negates this relation. */
-  negated?: boolean;
-}
-
-export interface ResolvedGraph {
-  entities: ResolvedEntity[];
-  relations: ResolvedRelation[];
-}
-
-function normalizeKey(name: string): string {
-  return name
-    .toLowerCase()
-    .replace(/[''`]/g, "'")
-    .replace(/[""]/g, '"')
-    .replace(/\s+/g, " ")
-    .replace(/[.\-_]/g, "")
-    .trim();
-}
-
-function entityHash(normalizedKey: string): string {
-  return "ent_" + createHash("sha256").update(normalizedKey).digest("hex").slice(0, 12);
-}
-
-function pickBestType(types: EntityType[]): EntityType {
-  const priority: EntityType[] = [
-    "Person", "Company", "Organization", "Project", "Tool",
-    "Language", "Product", "Platform", "Website", "Location",
-    "Document", "File", "Concept", "Unknown",
-  ];
-  for (const t of priority) {
-    if (types.includes(t)) return t;
-  }
-  return "Unknown";
-}
-
-/**
- * Cross-session entity dedup: normalize names, merge by key, assign stable IDs.
- */
-export function resolveEntities(
-  entities: ExtractedEntity[],
-  relations: ExtractedRelation[],
-): ResolvedGraph {
-  const keyMap = new Map<string, {
-    names: Set<string>;
-    types: EntityType[];
-    mentions: EntityMention[];
-  }>();
-
-  for (const ent of entities) {
-    const key = normalizeKey(ent.name);
-    if (!key) continue;
-
-    let bucket = keyMap.get(key);
-    if (!bucket) {
-      bucket = { names: new Set(), types: [], mentions: [] };
-      keyMap.set(key, bucket);
-    }
-    bucket.names.add(ent.name);
-    bucket.types.push(ent.type);
-    bucket.mentions.push(...ent.mentions);
-  }
-
-  const nameToId = new Map<string, string>();
-  const resolved: ResolvedEntity[] = [];
-
-  for (const [key, bucket] of keyMap) {
-    const id = entityHash(key);
-    const namesArr = [...bucket.names];
-    const canonical = namesArr.sort((a, b) => b.length - a.length)[0] ?? key;
-
-    for (const name of namesArr) {
-      nameToId.set(normalizeKey(name), id);
-    }
-
-    resolved.push({
-      id,
-      canonicalName: canonical,
-      type: pickBestType(bucket.types),
-      aliases: namesArr.filter((n) => n !== canonical),
-      mentions: bucket.mentions,
-    });
-  }
-
-  const resolvedRelations: ResolvedRelation[] = [];
-  for (const rel of relations) {
-    const tailId = nameToId.get(normalizeKey(rel.tailName));
-    if (!tailId) continue;
-
-    if (rel.negated) {
-      resolvedRelations.push({
-        headEntityId: rel.headName ? (nameToId.get(normalizeKey(rel.headName)) ?? "") : "",
-        relation: rel.relation,
-        tailEntityId: tailId,
-        sessionId: rel.sessionId,
-        turnIndex: rel.turnIndex,
-        evidence: rel.evidence,
-        negated: true,
-      });
-      continue;
-    }
-
-    const headId = nameToId.get(normalizeKey(rel.headName));
-    if (!headId) continue;
-
-    resolvedRelations.push({
-      headEntityId: headId,
-      relation: rel.relation,
-      tailEntityId: tailId,
-      sessionId: rel.sessionId,
-      turnIndex: rel.turnIndex,
-      evidence: rel.evidence,
-    });
-  }
-
-  return { entities: resolved, relations: resolvedRelations };
-}