@chendpoc/pi-memory 0.1.0

package/src/tools/memoryRecall.ts

@@ -0,0 +1,364 @@
+import { createFallbackQuery } from "../fallback/index.js";
+import { rerankWithLLM, type RerankOptions, type RankedResult } from "../fallback/llmRerank.js";
+import type { SessionSearchHit } from "../fallback/sessionSearch.js";
+import type { FallbackQuery, MemoryQuerier, MemoryRecallArgs, QueryIntent, QueryMode, ResponseEnvelope, ToolResult } from "../types.js";
+
+export const MEMORY_RECALL_NAME = "memory_recall";
+
+export const MEMORY_RECALL_DESCRIPTION =
+  "Read structured episodic memory from the local TLM sidecar — past sessions consolidated into long-term records.\n" +
+  "Modes:\n" +
+  "- direct_relation: one-hop predicate (e.g. \"what did X create?\"). Read groups[].via_relations.\n" +
+  "- path_query: multi-hop / possessive. relation_constraints is the ordered path; inverse hops use ^-1.\n" +
+  "- typed_neighborhood: typed target with exactly one relation. Requires candidate_type.\n\n" +
+  "Ground answers in supporting evidence internally; tell the user \"past records\" / \"以前的记录\", not raw event IDs.";
+
+export const MEMORY_RECALL_PROMPT_SNIPPET =
+  "Query local episodic memory by entity or relationship";
+
+export const MEMORY_RECALL_PROMPT_GUIDELINES = [
+  "Use memory_recall when the user asks about past sessions, people, projects, or decisions stored in long-term memory.",
+  "Use memory_recall for relationship questions (e.g. who is X to me) when implicit preflight did not already answer.",
+] as const;
+
+export const MEMORY_RECALL_PARAMETERS = {
+  type: "object",
+  properties: {
+    mode: {
+      type: "string",
+      enum: ["direct_relation", "path_query", "typed_neighborhood"],
+    },
+    anchor_mentions: { type: "array", items: { type: "string" } },
+    relation_constraints: { type: "array", items: { type: "string" } },
+    candidate_type: { type: "string" },
+    scope_filter: { type: "array", items: { type: "string" } },
+    target_slot: { type: "string", enum: ["head", "tail"] },
+    time_window: { type: "string" },
+    evidence_budget: { type: "integer" },
+    result_limit: { type: "integer" },
+  },
+  required: ["anchor_mentions"],
+} as const;
+
+export class MemoryRecallTool {
+  constructor(
+    private readonly service: MemoryQuerier,
+    private readonly fallback: FallbackQuery | null = null,
+    private readonly rerankOpts: RerankOptions | null = null,
+  ) {}
+
+  info() {
+    return {
+      name: MEMORY_RECALL_NAME,
+      description: MEMORY_RECALL_DESCRIPTION,
+      parameters: MEMORY_RECALL_PARAMETERS,
+    };
+  }
+
+  async run(argsJson: string, signal?: AbortSignal): Promise<ToolResult> {
+    const args = parseArgs(argsJson);
+    if ("error" in args) {
+      return { content: args.error, isError: true };
+    }
+
+    const intent = argsToIntent(args);
+    const validation = validateArgs(args);
+    if (validation) {
+      return { content: validation, isError: true };
+    }
+
+    return this.runIntent(intent, signal);
+  }
+
+  async runIntent(intent: QueryIntent, signal?: AbortSignal): Promise<ToolResult> {
+    if (this.service.status() !== "ready") {
+      return this.fallbackResult(intent, "service_unavailable", "fallback");
+    }
+
+    let result = await this.service.query(intent, signal);
+    if (result.transportError || result.errorClass === "unavailable") {
+      return this.fallbackResult(intent, "service_unavailable", "fallback");
+    }
+
+    if (result.errorClass === "retryable") {
+      await sleep(500, signal);
+      result = await this.service.query(intent, signal);
+      if (
+        result.transportError ||
+        result.errorClass === "unavailable" ||
+        result.errorClass === "retryable"
+      ) {
+        return this.fallbackResult(intent, "retryable_failed", "fallback_after_retry");
+      }
+    }
+
+    if (result.errorClass === "permanent") {
+      return permanentResult(result.env);
+    }
+
+    return shapeResult(result.env!);
+  }
+
+  private async fallbackResult(
+    intent: QueryIntent,
+    reason: string,
+    source: string,
+  ): Promise<ToolResult> {
+    const candidates: Record<string, unknown>[] = [];
+    const warnings: Record<string, unknown>[] = [];
+
+    if (this.fallback) {
+      const query = intent.anchor_mentions.map((s) => s.trim()).filter(Boolean).join(" ");
+      const limit = intent.result_limit && intent.result_limit > 0 ? intent.result_limit : 10;
+      try {
+        const hits = await this.fallback.sessionKeyword(query, limit);
+
+        let reranked: RankedResult[] | null = null;
+        if (this.rerankOpts && hits.length > 0) {
+          try {
+            reranked = await rerankWithLLM(
+              query,
+              hits as SessionSearchHit[],
+              this.rerankOpts,
+            );
+          } catch {
+            /* silent fallback to original order */
+          }
+        }
+
+        if (reranked) {
+          for (const r of reranked) {
+            const original = hits[r.index];
+            candidates.push({
+              value: r.summary,
+              score: r.score,
+              evidence: "llm_rerank",
+              ...(typeof original === "object" && original !== null &&
+                "session_id" in (original as Record<string, unknown>)
+                ? { scope: "session_search", session_id: (original as Record<string, unknown>).session_id }
+                : {}),
+            });
+          }
+        } else {
+          for (const h of hits) {
+            candidates.push({
+              value: typeof h === "string" ? h : JSON.stringify(h),
+              evidence: "text_search",
+              ...(typeof h === "object" &&
+              h !== null &&
+              "session_id" in (h as Record<string, unknown>)
+                ? { scope: "session_search" }
+                : {}),
+            });
+          }
+        }
+      } catch (err) {
+        warnings.push({
+          code: "fallback_session_search_failed",
+          message: err instanceof Error ? err.message : String(err),
+        });
+      }
+      try {
+        const snippet = await this.fallback.memoryFileSnippet(query);
+        if (snippet) {
+          candidates.push({
+            value: snippet,
+            evidence: "text_search",
+            scope: "memory_md",
+          });
+        }
+      } catch {
+        /* optional */
+      }
+    }
+
+    const out = {
+      source,
+      evidence_quality: "text_search",
+      bundle_version: null,
+      candidates,
+      warnings,
+      fallback_reason: reason,
+    };
+    return { content: JSON.stringify(out) };
+  }
+}
+
+function parseArgs(argsJson: string): MemoryRecallArgs | { error: string } {
+  let raw: Record<string, unknown>;
+  try {
+    raw = JSON.parse(coerceArgs(argsJson)) as Record<string, unknown>;
+  } catch (e) {
+    return { error: `invalid input: ${e instanceof Error ? e.message : e}` };
+  }
+  const anchor = raw.anchor_mentions;
+  if (!Array.isArray(anchor) || anchor.length === 0) {
+    return { error: "anchor_mentions is required and must be non-empty" };
+  }
+  return raw as unknown as MemoryRecallArgs;
+}
+
+function argsToIntent(a: MemoryRecallArgs): QueryIntent {
+  return {
+    mode: (a.mode as QueryMode) || "direct_relation",
+    anchor_mentions: a.anchor_mentions,
+    relation_constraints: a.relation_constraints,
+    candidate_type: a.candidate_type,
+    scope_filter: a.scope_filter,
+    target_slot: (a.target_slot as QueryIntent["target_slot"]) ?? "",
+    time_window: a.time_window,
+    evidence_budget: a.evidence_budget && a.evidence_budget > 0 ? a.evidence_budget : 5,
+    result_limit: a.result_limit && a.result_limit > 0 ? a.result_limit : 10,
+  };
+}
+
+function validateArgs(a: MemoryRecallArgs): string | null {
+  for (const rel of a.relation_constraints ?? []) {
+    if (isBroadRelation(rel)) {
+      return "memory_recall requires concrete relation_constraints. Broad relations like related_to are not valid for structured lookup.";
+    }
+  }
+  if (a.mode === "typed_neighborhood") {
+    if (!a.candidate_type?.trim()) {
+      return "typed_neighborhood requires candidate_type.";
+    }
+    if ((a.relation_constraints?.length ?? 0) !== 1) {
+      return "typed_neighborhood requires exactly one relation_constraints value.";
+    }
+  }
+  return null;
+}
+
+function isBroadRelation(rel: string): boolean {
+  let r = rel.trim().toLowerCase();
+  r = r.replace(/^\^/, "").replace(/\^-1$/, "");
+  return ["related_to", "relates_to", "associated_with", "about", "mentions", "other"].includes(r);
+}
+
+function coerceArgs(argsJson: string): string {
+  let raw: Record<string, unknown>;
+  try {
+    raw = JSON.parse(argsJson) as Record<string, unknown>;
+  } catch {
+    return argsJson;
+  }
+  let changed = false;
+  for (const field of ["anchor_mentions", "relation_constraints", "scope_filter"]) {
+    const v = raw[field];
+    if (typeof v === "string") {
+      try {
+        raw[field] = JSON.parse(v);
+        changed = true;
+      } catch {
+        /* keep */
+      }
+    }
+  }
+  for (const field of ["result_limit", "evidence_budget"]) {
+    const v = raw[field];
+    if (typeof v === "string") {
+      const n = Number(v);
+      if (!Number.isNaN(n)) {
+        raw[field] = n;
+        changed = true;
+      }
+    }
+  }
+  return changed ? JSON.stringify(raw) : argsJson;
+}
+
+function shapeResult(env: ResponseEnvelope): ToolResult {
+  let quality = "structured";
+  const warnings = envelopeWarnings(env);
+  if (env.reason === "degraded") {
+    quality = "structured_degraded";
+    warnings.unshift({
+      code: "bundle_degraded",
+      message: "memory bundle degraded — results may be incomplete",
+    });
+  }
+  const cands = env.candidates.map((c) => {
+    const m: Record<string, unknown> = {
+      value: c.value,
+      score: c.score,
+      evidence: c.evidence,
+      supporting_event_ids: c.supporting_event_ids,
+    };
+    if (c.scope != null) m.scope = c.scope;
+    if (c.support_count != null) m.support_count = c.support_count;
+    if (c.distinct_session_count != null) {
+      m.distinct_session_count = c.distinct_session_count;
+    }
+    return m;
+  });
+  const out = {
+    source: "memory_sidecar",
+    evidence_quality: quality,
+    bundle_version: env.bundle_version ?? null,
+    memory_block: env.memory_block ?? null,
+    candidates: cands,
+    warnings,
+    fallback_reason: null,
+  };
+  return { content: JSON.stringify(out) };
+}
+
+function permanentResult(env: ResponseEnvelope | null): ToolResult {
+  const out = {
+    source: "memory_sidecar",
+    evidence_quality: "structured",
+    bundle_version: env?.bundle_version ?? null,
+    candidates: [],
+    warnings: envelopeWarnings(env),
+    fallback_reason: null,
+  };
+  return { content: JSON.stringify(out), isError: true };
+}
+
+function envelopeWarnings(env: ResponseEnvelope | null): Record<string, unknown>[] {
+  const out: Record<string, unknown>[] = [];
+  if (!env) return out;
+  for (const w of env.warnings ?? []) {
+    out.push({ code: w.code, message: w.message });
+  }
+  if (env.error) {
+    out.push({
+      code: env.error.code,
+      message: env.error.message,
+      sub_code: env.error.details?.sub_code,
+    });
+  }
+  return out;
+}
+
+function sleep(ms: number, signal?: AbortSignal): Promise<void> {
+  return new Promise((resolve, reject) => {
+    if (signal?.aborted) {
+      reject(new Error("aborted"));
+      return;
+    }
+    const t = setTimeout(resolve, ms);
+    signal?.addEventListener("abort", () => {
+      clearTimeout(t);
+      reject(new Error("aborted"));
+    }, { once: true });
+  });
+}
+
+/** @deprecated Use createFallbackQuery — kept for backward-compatible imports. */
+export function createStubFallback(
+  memoryMdPaths: string[],
+  sessionsDir = "",
+): FallbackQuery {
+  return createFallbackQuery({ sessionsDir, memoryMdPaths });
+}
+
+export { createFallbackQuery };
+
+export function createMemoryRecallTool(
+  service: MemoryQuerier,
+  fallback: FallbackQuery | null,
+  rerankOpts?: RerankOptions | null,
+): MemoryRecallTool {
+  return new MemoryRecallTool(service, fallback, rerankOpts ?? null);
+}

package/src/trainer/bundleBuilder.ts

@@ -0,0 +1,192 @@
+import { createHash } from "node:crypto";
+import fs from "node:fs/promises";
+import path from "node:path";
+
+import type { ResolvedGraph, ResolvedEntity, ResolvedRelation } from "./entityResolver.js";
+import type { ExtractedEvent } from "./extractFacts.js";
+import type { BundleManifest, BundleManifestFile } from "../sidecar/bundle.js";
+
+export interface BundleData {
+  graph: ResolvedGraph;
+  events: ExtractedEvent[];
+}
+
+/** TLM-compatible entity record for the graph data file. */
+interface BundleEntity {
+  entity_id: string;
+  label: string;
+  type: string;
+  aliases: string[];
+  mention_count: number;
+  distinct_session_count: number;
+}
+
+/** TLM-compatible relation edge for the graph data file. */
+interface BundleEdge {
+  head_entity_id: string;
+  relation: string;
+  tail_entity_id: string;
+  supporting_event_ids: string[];
+  evidence: string;
+}
+
+/** TLM-compatible event record. */
+interface BundleEvent {
+  event_id: string;
+  description: string;
+  session_id: string;
+  timestamp: string;
+}
+
+interface GraphDataFile {
+  entities: BundleEntity[];
+  edges: BundleEdge[];
+  events: BundleEvent[];
+}
+
+function eventId(ev: ExtractedEvent, idx: number): string {
+  const hash = createHash("sha256")
+    .update(`${ev.sessionId}:${ev.turnIndex}:${idx}`)
+    .digest("hex")
+    .slice(0, 12);
+  return `ev_${hash}`;
+}
+
+function distinctSessions(ent: ResolvedEntity): number {
+  const s = new Set(ent.mentions.map((m) => m.sessionId));
+  return s.size;
+}
+
+function buildGraphData(
+  graph: ResolvedGraph,
+  events: ExtractedEvent[],
+): GraphDataFile {
+  const eventRecords: BundleEvent[] = events.map((ev, i) => ({
+    event_id: eventId(ev, i),
+    description: ev.description,
+    session_id: ev.sessionId,
+    timestamp: ev.timestamp,
+  }));
+
+  const eventsBySession = new Map<string, string[]>();
+  for (const ev of eventRecords) {
+    let arr = eventsBySession.get(ev.session_id);
+    if (!arr) {
+      arr = [];
+      eventsBySession.set(ev.session_id, arr);
+    }
+    arr.push(ev.event_id);
+  }
+
+  const entities: BundleEntity[] = graph.entities.map((ent) => ({
+    entity_id: ent.id,
+    label: ent.canonicalName,
+    type: ent.type,
+    aliases: ent.aliases,
+    mention_count: ent.mentions.length,
+    distinct_session_count: distinctSessions(ent),
+  }));
+
+  const edges: BundleEdge[] = graph.relations.map((rel) => {
+    const evIds = eventsBySession.get(rel.sessionId) ?? [];
+    return {
+      head_entity_id: rel.headEntityId,
+      relation: rel.relation,
+      tail_entity_id: rel.tailEntityId,
+      supporting_event_ids: evIds.slice(0, 5),
+      evidence: rel.evidence,
+    };
+  });
+
+  return { entities, edges, events: eventRecords };
+}
+
+async function writeJsonFile(
+  dir: string,
+  filename: string,
+  data: unknown,
+): Promise<BundleManifestFile> {
+  const content = JSON.stringify(data, null, 2) + "\n";
+  const filePath = path.join(dir, filename);
+  await fs.writeFile(filePath, content, "utf8");
+  const sha256 = createHash("sha256").update(content).digest("hex");
+  return {
+    path: filename,
+    size: Buffer.byteLength(content, "utf8"),
+    sha256,
+  };
+}
+
+export interface BuildBundleOptions {
+  outputDir: string;
+  bundleVersion?: string;
+}
+
+export interface BuildBundleResult {
+  bundleDir: string;
+  manifest: BundleManifest;
+  stats: {
+    entityCount: number;
+    edgeCount: number;
+    eventCount: number;
+  };
+}
+
+/**
+ * Build a TLM-compatible bundle directory from resolved graph + events.
+ *
+ * Produces:
+ *   <outputDir>/bundles/<iso-ts>/
+ *     manifest.json
+ *     graph.json      — entities, edges, events
+ */
+export async function buildBundle(
+  data: BundleData,
+  opts: BuildBundleOptions,
+): Promise<BuildBundleResult> {
+  const bundleTs = new Date()
+    .toISOString()
+    .replace(/[:.]/g, "-");
+  const bundleVersion = opts.bundleVersion ?? "0.6.0";
+
+  const bundlesDir = path.join(opts.outputDir, "bundles");
+  const bundleDir = path.join(bundlesDir, bundleTs);
+  await fs.mkdir(bundleDir, { recursive: true, mode: 0o700 });
+
+  const graphData = buildGraphData(data.graph, data.events);
+  const files: BundleManifestFile[] = [];
+
+  const graphFile = await writeJsonFile(bundleDir, "graph.json", graphData);
+  files.push(graphFile);
+
+  const totalSize = files.reduce((sum, f) => sum + f.size, 0);
+  const integrityInput = files.map((f) => f.sha256).join(":");
+  const integritySha256 = createHash("sha256")
+    .update(integrityInput)
+    .digest("hex");
+
+  const manifest: BundleManifest = {
+    bundle_ts: bundleTs,
+    bundle_version: bundleVersion,
+    size_bytes: totalSize,
+    integrity_sha256: integritySha256,
+    files,
+  };
+
+  const manifestContent = JSON.stringify(manifest, null, 2) + "\n";
+  await fs.writeFile(
+    path.join(bundleDir, "manifest.json"),
+    manifestContent,
+    "utf8",
+  );
+
+  return {
+    bundleDir,
+    manifest,
+    stats: {
+      entityCount: graphData.entities.length,
+      edgeCount: graphData.edges.length,
+      eventCount: graphData.events.length,
+    },
+  };
+}

package/src/trainer/bundleLoader.ts

@@ -0,0 +1,105 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+
+import type { ResolvedEntity, ResolvedRelation, ResolvedGraph } from "./entityResolver.js";
+import type { ExtractedEvent, EntityType } from "./extractFacts.js";
+
+interface BundleEntityRecord {
+  entity_id: string;
+  label: string;
+  type: string;
+  aliases: string[];
+  mention_count: number;
+  distinct_session_count: number;
+}
+
+interface BundleEdgeRecord {
+  head_entity_id: string;
+  relation: string;
+  tail_entity_id: string;
+  supporting_event_ids: string[];
+  evidence: string;
+}
+
+interface BundleEventRecord {
+  event_id: string;
+  description: string;
+  session_id: string;
+  timestamp: string;
+}
+
+interface GraphDataFile {
+  entities: BundleEntityRecord[];
+  edges: BundleEdgeRecord[];
+  events: BundleEventRecord[];
+}
+
+const VALID_ENTITY_TYPES = new Set<string>([
+  "Person", "Company", "Organization", "Project", "Tool",
+  "Language", "Concept", "Document", "File", "Location",
+  "Product", "Website", "Platform", "Unknown",
+]);
+
+function toEntityType(t: string): EntityType {
+  return VALID_ENTITY_TYPES.has(t) ? (t as EntityType) : "Unknown";
+}
+
+export interface ExistingBundle {
+  graph: ResolvedGraph;
+  events: ExtractedEvent[];
+}
+
+/**
+ * Load the existing bundle's graph.json from `<bundleRoot>/current/graph.json`.
+ * Returns null when no current bundle exists or graph.json is missing/unparseable.
+ */
+export async function loadExistingBundle(
+  bundleRoot: string,
+): Promise<ExistingBundle | null> {
+  const graphPath = path.join(bundleRoot, "current", "graph.json");
+
+  let raw: string;
+  try {
+    raw = await fs.readFile(graphPath, "utf8");
+  } catch {
+    return null;
+  }
+
+  let data: GraphDataFile;
+  try {
+    data = JSON.parse(raw) as GraphDataFile;
+  } catch {
+    return null;
+  }
+
+  if (!data.entities || !data.edges) return null;
+
+  const entities: ResolvedEntity[] = (data.entities ?? []).map((e) => ({
+    id: e.entity_id,
+    canonicalName: e.label,
+    type: toEntityType(e.type),
+    aliases: e.aliases ?? [],
+    mentions: [],
+  }));
+
+  const relations: ResolvedRelation[] = (data.edges ?? []).map((e) => ({
+    headEntityId: e.head_entity_id,
+    relation: e.relation,
+    tailEntityId: e.tail_entity_id,
+    sessionId: "",
+    turnIndex: 0,
+    evidence: e.evidence ?? "",
+  }));
+
+  const events: ExtractedEvent[] = (data.events ?? []).map((e) => ({
+    description: e.description,
+    sessionId: e.session_id,
+    timestamp: e.timestamp,
+    turnIndex: 0,
+  }));
+
+  return {
+    graph: { entities, relations },
+    events,
+  };
+}