@yun520-1/heartflow 0.15.2

package/src/storage/checkpoint/CheckpointEngine.ts

@@ -0,0 +1,259 @@
+/**
+ * CheckpointEngine — 心流检查点持久化引擎
+ * @version v0.12.50
+ * 
+ * 提供 save/load/cleanup 三个核心方法：
+ * - save()   : 将状态快照写入 ~/.hermes/heartflow/checkpoints/<timestamp>.json
+ * - load()   : 从指定 ID 或最新快照恢复状态
+ * - cleanup(): 保留最新 N 个快照，删除其余
+ * 
+ * 快照命名: checkpoint-<unix_ms>.json
+ * 元数据  : checkpoint-meta.json (索引 + 版本)
+ */
+
+'use strict';
+
+import * as fs from 'fs';
+import * as path from 'path';
+import * as os from 'os';
+
+const CHECKPOINT_DIR = path.join(os.homedir(), '.hermes', 'heartflow', 'checkpoints');
+const MAX_KEEP = 10;
+const SNAPSHOT_PREFIX = 'checkpoint-';
+const META_FILE = 'checkpoint-meta.json';
+
+interface CheckpointMeta {
+  version: string;
+  lastSavedAt: string | null;
+  totalSnapshots: number;
+  snapshots: CheckpointEntry[];
+}
+
+interface CheckpointEntry {
+  id: string;        // unix ms string
+  filename: string;
+  savedAt: string;   // ISO string
+  sizeBytes: number;
+  label?: string;
+}
+
+interface SaveResult {
+  success: boolean;
+  id?: string;
+  filename?: string;
+  savedAt?: string;
+  error?: string;
+}
+
+interface LoadResult {
+  success: boolean;
+  data?: object;
+  id?: string;
+  error?: string;
+}
+
+interface CleanupResult {
+  success: boolean;
+  deleted: string[];
+  kept: number;
+  error?: string;
+}
+
+interface ListResult {
+  success: boolean;
+  snapshots: CheckpointEntry[];
+  error?: string;
+}
+
+/**
+ * CheckpointEngine
+ * 
+ * 管理 HeartFlow 状态快照的持久化。
+ * 快照以 JSON 格式写入 ~/.hermes/heartflow/checkpoints/，
+ * 自动保留最新 10 个，cleanup() 删除更旧的。
+ */
+export class CheckpointEngine {
+  private dir: string;
+  private maxKeep: number;
+
+  constructor(opts?: { dir?: string; maxKeep?: number }) {
+    this.dir = opts?.dir ?? CHECKPOINT_DIR;
+    this.maxKeep = opts?.maxKeep ?? MAX_KEEP;
+  }
+
+  // ─── 私有工具 ─────────────────────────────────────────────────────────────
+
+  private ensureDir(): void {
+    if (!fs.existsSync(this.dir)) {
+      fs.mkdirSync(this.dir, { recursive: true });
+    }
+  }
+
+  private metaPath(): string {
+    return path.join(this.dir, META_FILE);
+  }
+
+  private snapshotPath(id: string): string {
+    return path.join(this.dir, `${SNAPSHOT_PREFIX}${id}.json`);
+  }
+
+  private loadMeta(): CheckpointMeta {
+    try {
+      if (fs.existsSync(this.metaPath())) {
+        return JSON.parse(fs.readFileSync(this.metaPath(), 'utf8'));
+      }
+    } catch (_) {
+      // ignore
+    }
+    return { version: '1.0', lastSavedAt: null, totalSnapshots: 0, snapshots: [] };
+  }
+
+  private saveMeta(meta: CheckpointMeta): void {
+    this.ensureDir();
+    fs.writeFileSync(this.metaPath(), JSON.stringify(meta, null, 2), 'utf8');
+  }
+
+  private sortByIdDesc(entries: CheckpointEntry[]): CheckpointEntry[] {
+    return entries.slice().sort((a, b) => parseInt(b.id) - parseInt(a.id));
+  }
+
+  // ─── 公开 API ─────────────────────────────────────────────────────────────
+
+  /**
+   * save — 保存一个状态快照
+   * 
+   * @param data      要持久化的状态对象
+   * @param label     可选标签，如 'pre-upgrade'
+   * @returns SaveResult
+   */
+  save(data: object, label?: string): SaveResult {
+    try {
+      this.ensureDir();
+
+      const id = Date.now().toString();
+      const savedAt = new Date().toISOString();
+      const filename = `${SNAPSHOT_PREFIX}${id}.json`;
+      const filePath = path.join(this.dir, filename);
+
+      const payload = {
+        id,
+        savedAt,
+        label: label ?? null,
+        data,
+      };
+
+      const json = JSON.stringify(payload, null, 2);
+      fs.writeFileSync(filePath, json, 'utf8');
+      const sizeBytes = Buffer.byteLength(json, 'utf8');
+
+      // 更新元数据
+      const meta = this.loadMeta();
+      meta.snapshots.push({ id, filename, savedAt, sizeBytes, label: label ?? undefined });
+      meta.lastSavedAt = savedAt;
+      meta.totalSnapshots++;
+      this.saveMeta(meta);
+
+      return { success: true, id, filename, savedAt };
+    } catch (err: any) {
+      return { success: false, error: err.message };
+    }
+  }
+
+  /**
+   * load — 恢复一个快照
+   * 
+   * @param id     快照 ID（unix ms），或传入 'latest' / undefined 加载最新
+   * @returns LoadResult
+   */
+  load(id?: string): LoadResult {
+    try {
+      const meta = this.loadMeta();
+      let targetId: string;
+
+      if (!id || id === 'latest') {
+        const sorted = this.sortByIdDesc(meta.snapshots);
+        if (sorted.length === 0) {
+          return { success: false, error: 'No snapshots found' };
+        }
+        targetId = sorted[0].id;
+      } else {
+        targetId = id;
+      }
+
+      const filePath = this.snapshotPath(targetId);
+      if (!fs.existsSync(filePath)) {
+        return { success: false, error: `Snapshot ${targetId} not found` };
+      }
+
+      const raw = JSON.parse(fs.readFileSync(filePath, 'utf8'));
+      return { success: true, data: raw.data, id: targetId };
+    } catch (err: any) {
+      return { success: false, error: err.message };
+    }
+  }
+
+  /**
+   * cleanup — 删除旧快照，保留最新 N 个
+   * 
+   * @param keep  保留数量，默认 this.maxKeep (=10)
+   * @returns CleanupResult
+   */
+  cleanup(keep?: number): CleanupResult {
+    try {
+      const count = keep ?? this.maxKeep;
+      const meta = this.loadMeta();
+
+      if (meta.snapshots.length <= count) {
+        return { success: true, deleted: [], kept: meta.snapshots.length };
+      }
+
+      const sorted = this.sortByIdDesc(meta.snapshots);
+      const keepIds = new Set(sorted.slice(0, count).map(s => s.id));
+      const toDelete = sorted.slice(count);
+
+      const deleted: string[] = [];
+      for (const snap of toDelete) {
+        const fp = this.snapshotPath(snap.id);
+        if (fs.existsSync(fp)) {
+          fs.unlinkSync(fp);
+          deleted.push(snap.id);
+        }
+      }
+
+      meta.snapshots = meta.snapshots.filter(s => keepIds.has(s.id));
+      this.saveMeta(meta);
+
+      return { success: true, deleted, kept: meta.snapshots.length };
+    } catch (err: any) {
+      return { success: false, deleted: [], kept: 0, error: err.message };
+    }
+  }
+
+  /**
+   * list — 列出所有快照（最新在前）
+   */
+  list(): ListResult {
+    try {
+      const meta = this.loadMeta();
+      return { success: true, snapshots: this.sortByIdDesc(meta.snapshots) };
+    } catch (err: any) {
+      return { success: false, snapshots: [], error: err.message };
+    }
+  }
+
+  /**
+   * getStats — 统计信息
+   */
+  getStats() {
+    const meta = this.loadMeta();
+    return {
+      dir: this.dir,
+      totalSnapshots: meta.totalSnapshots,
+      maxKeep: this.maxKeep,
+      lastSavedAt: meta.lastSavedAt,
+    };
+  }
+}
+
+export { CHECKPOINT_DIR, MAX_KEEP };
+export type { CheckpointMeta, CheckpointEntry, SaveResult, LoadResult, CleanupResult, ListResult };

package/src/storage/checkpoint.ts

@@ -0,0 +1,79 @@
+// Checkpoint Engine — 状态快照 + 恢复机制
+// Periodic snapshots of engine state for recovery
+
+export interface Checkpoint {
+  id: string;
+  timestamp: number;
+  version: string;
+  data: Record<string, unknown>;
+  metadata: Record<string, unknown>;
+}
+
+export interface CheckpointEngine {
+  boot(): void;
+  save(key: string, data: Record<string, unknown>): string;
+  load(key: string): Checkpoint | null;
+  list(): Checkpoint[];
+  prune(maxCheckpoints?: number): number;
+  getLatest(key: string): Checkpoint | null;
+  shutdown(): void;
+}
+
+export function createCheckpointEngine(version = '0.13.0'): CheckpointEngine {
+  const checkpoints = new Map<string, Checkpoint[]>();
+
+  function boot(): void {
+    console.log('[CheckpointEngine] boot — state snapshot ready');
+  }
+
+  function save(key: string, data: Record<string, unknown>): string {
+    const id = `cp_${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 6)}`;
+    const checkpoint: Checkpoint = {
+      id,
+      timestamp: Date.now(),
+      version,
+      data,
+      metadata: { key },
+    };
+    if (!checkpoints.has(key)) checkpoints.set(key, []);
+    checkpoints.get(key)!.push(checkpoint);
+    return id;
+  }
+
+  function load(key: string): Checkpoint | null {
+    const list = checkpoints.get(key);
+    if (!list || list.length === 0) return null;
+    return list[list.length - 1];
+  }
+
+  function list(): Checkpoint[] {
+    const all: Checkpoint[] = [];
+    for (const [, cps] of checkpoints) all.push(...cps);
+    return all.sort((a, b) => b.timestamp - a.timestamp);
+  }
+
+  function prune(maxCheckpoints = 10): number {
+    let removed = 0;
+    for (const [key, cps] of checkpoints) {
+      if (cps.length > maxCheckpoints) {
+        const excess = cps.splice(0, cps.length - maxCheckpoints);
+        removed += excess.length;
+      }
+      if (cps.length === 0) checkpoints.delete(key);
+    }
+    return removed;
+  }
+
+  function getLatest(key: string): Checkpoint | null {
+    const list = checkpoints.get(key);
+    if (!list || list.length === 0) return null;
+    return list[list.length - 1];
+  }
+
+  function shutdown(): void {
+    checkpoints.clear();
+    console.log('[CheckpointEngine] shutdown');
+  }
+
+  return { boot, save, load, list, prune, getLatest, shutdown };
+}

package/src/storage/index.ts

@@ -0,0 +1,112 @@
+/**
+ * HeartFlow Storage — 持久化存储统一导出
+ * 
+ * 整合 CheckpointEngine + VectorStore，提供：
+ *   - checkpoint: 状态快照持久化 (save/load/cleanup/list)
+ *   - vector: 向量存储 + 相似度检索
+ * 
+ * 使用示例:
+ *   const { createStorageEngine } = require('./storage');
+ *   const storage = createStorageEngine();
+ *   await storage.save('key', data);
+ *   const data = await storage.load('key');
+ */
+
+import { CheckpointEngine } from './checkpoint/CheckpointEngine';
+import { VectorStore, createVectorStore } from '../memory/vector-store';
+
+export { CheckpointEngine } from './checkpoint/CheckpointEngine';
+export { VectorStore, createVectorStore } from '../memory/vector-store';
+
+export interface StorageEngine {
+  // Checkpoint API
+  checkpoint: {
+    save: (data: object, label?: string) => ReturnType<CheckpointEngine['save']>;
+    load: (id?: string) => ReturnType<CheckpointEngine['load']>;
+    cleanup: (keep?: number) => ReturnType<CheckpointEngine['cleanup']>;
+    list: () => ReturnType<CheckpointEngine['list']>;
+    getStats: () => ReturnType<CheckpointEngine['getStats']>;
+  };
+  
+  // Vector API  
+  vector: {
+    upsert: (id: string, embedding: number[], metadata?: object) => VectorStore;
+    search: (query: number[], topK?: number, threshold?: number) => Array<{id: string; score: number; metadata: object}>;
+    get: (id: string) => {id: string; embedding: number[]; metadata: object} | null;
+    delete: (id: string) => boolean;
+    size: () => number;
+    save: () => boolean;
+  };
+  
+  // Convenience
+  save: (key: string, data: object) => ReturnType<CheckpointEngine['save']>;
+  load: (key: string) => ReturnType<CheckpointEngine['load']>;
+}
+
+/**
+ * 创建存储引擎实例
+ */
+export function createStorageEngine(options?: {
+  checkpointDir?: string;
+  vectorPersistPath?: string;
+  vectorDimension?: number;
+}): StorageEngine {
+  const checkpointEngine = new CheckpointEngine({
+    dir: options?.checkpointDir,
+  });
+
+  const vectorStore = createVectorStore({
+    persistPath: options?.vectorPersistPath 
+      ?? (() => {
+        const os = require('os');
+        const path = require('path');
+        return path.join(os.homedir(), '.hermes', 'heartflow', 'data', 'vector-store.json');
+      })(),
+    dimension: options?.vectorDimension ?? 384,
+  });
+
+  return {
+    checkpoint: {
+      save: (data, label) => checkpointEngine.save(data, label),
+      load: (id) => checkpointEngine.load(id),
+      cleanup: (keep) => checkpointEngine.cleanup(keep),
+      list: () => checkpointEngine.list(),
+      getStats: () => checkpointEngine.getStats(),
+    },
+
+    vector: {
+      upsert: (id, embedding, metadata = {}) => {
+        vectorStore.upsert(id, embedding, metadata);
+        return vectorStore;
+      },
+      search: (query, topK, threshold) => vectorStore.search(query, topK, threshold),
+      get: (id) => vectorStore.get(id),
+      delete: (id) => vectorStore.delete(id),
+      size: () => vectorStore.size(),
+      save: () => vectorStore.save(),
+    },
+
+    save: (key, data) => checkpointEngine.save({ key, data, savedAt: Date.now() }, key),
+    load: (key) => {
+      const result = checkpointEngine.load('latest');
+      if (result.success && result.data && (result.data as any).key === key) {
+        return { success: true, data: (result.data as any).data };
+      }
+      return { success: false, error: 'Key not found' };
+    },
+  };
+}
+
+/**
+ * 独立使用: 创建带持久化的 VectorStore
+ */
+export function createPersistentVectorStore(persistPath?: string, dimension = 384) {
+  return createVectorStore({ persistPath, dimension });
+}
+
+/**
+ * 独立使用: 创建 CheckpointEngine
+ */
+export function createCheckpointEngine(dir?: string, maxKeep = 10) {
+  return new CheckpointEngine({ dir, maxKeep });
+}

package/src/storage/vector/VectorStoreEngine.ts

@@ -0,0 +1,205 @@
+// VectorStoreEngine — 内存向量存储 + 增量异步持久化
+// Simple in-memory vector storage with cosine similarity + async incremental persist
+
+import * as fs from 'fs';
+import * as path from 'path';
+import * as os from 'os';
+
+export interface VectorEntry {
+  id: string;
+  vector: number[];
+  metadata: Record<string, unknown>;
+}
+
+export interface SearchResult {
+  id: string;
+  score: number;
+  metadata: Record<string, unknown>;
+}
+
+export interface VectorStoreEngine {
+  boot(): Promise<void>;
+  add(id: string, vector: number[], metadata?: Record<string, unknown>): void;
+  persist(): Promise<void>;
+  search(query: number[], k?: number): SearchResult[];
+  query(vector: number[], k?: number): SearchResult[];
+  get(id: string): VectorEntry | undefined;
+  remove(id: string): boolean;
+  clear(): void;
+  size(): number;
+  shutdown(): Promise<void>;
+}
+
+function cosine(a: number[], b: number[]): number {
+  if (a.length !== b.length) return 0;
+  let dot = 0, magA = 0, magB = 0;
+  for (let i = 0; i < a.length; i++) {
+    dot += a[i] * b[i];
+    magA += a[i] * a[i];
+    magB += b[i] * b[i];
+  }
+  const denom = Math.sqrt(magA) * Math.sqrt(magB);
+  return denom === 0 ? 0 : dot / denom;
+}
+
+function normalize(v: number[]): number[] {
+  const mag = Math.sqrt(v.reduce((s, x) => s + x * x, 0));
+  if (mag === 0) return v;
+  return v.map(x => x / mag);
+}
+
+export function createVectorStoreEngine(
+  storeDir = path.join(os.homedir(), '.hermes', 'heartflow', 'vectors')
+): VectorStoreEngine {
+  const entries = new Map<string, VectorEntry>();
+  let persistTimer: NodeJS.Timeout | null = null;
+  let dirty = false;
+  let bootPromise: Promise<void> | null = null;
+
+  // Ensure directory exists
+  function ensureDir(): void {
+    if (!fs.existsSync(storeDir)) {
+      fs.mkdirSync(storeDir, { recursive: true });
+    }
+  }
+
+  // Path for the entries index file
+  function indexPath(): string {
+    return path.join(storeDir, 'index.json');
+  }
+
+  // Path for individual vector file
+  function entryPath(id: string): string {
+    return path.join(storeDir, `${id}.json`);
+  }
+
+  // Load all persisted entries from disk
+  async function loadFromDisk(): Promise<void> {
+    ensureDir();
+    const idx = indexPath();
+    if (!fs.existsSync(idx)) return;
+
+    try {
+      const data = fs.readFileSync(idx, 'utf-8');
+      const ids: string[] = JSON.parse(data);
+      for (const id of ids) {
+        const ep = entryPath(id);
+        if (fs.existsSync(ep)) {
+          try {
+            const raw = fs.readFileSync(ep, 'utf-8');
+            const entry: VectorEntry = JSON.parse(raw);
+            entries.set(id, entry);
+          } catch {
+            // Skip corrupted entry files
+          }
+        }
+      }
+      console.log(`[VectorStoreEngine] loaded ${entries.size} entries from disk`);
+    } catch (e) {
+      console.warn('[VectorStoreEngine] failed to load index, starting fresh', e);
+    }
+  }
+
+  // Persist entries map to disk (index + individual files)
+  async function persistToDisk(): Promise<void> {
+    ensureDir();
+    // Write index
+    const ids = Array.from(entries.keys());
+    fs.writeFileSync(indexPath(), JSON.stringify(ids), 'utf-8');
+
+    // Write each entry individually for incremental updates
+    for (const [id, entry] of entries) {
+      fs.writeFileSync(entryPath(id), JSON.stringify(entry), 'utf-8');
+    }
+
+    dirty = false;
+    console.log(`[VectorStoreEngine] persisted ${entries.size} entries`);
+  }
+
+  function boot(): Promise<void> {
+    if (!bootPromise) {
+      bootPromise = loadFromDisk();
+    }
+    console.log('[VectorStoreEngine] boot — vector search ready');
+    return bootPromise;
+  }
+
+  function add(id: string, vector: number[], metadata: Record<string, unknown> = {}): void {
+    entries.set(id, { id, vector: normalize(vector), metadata });
+    dirty = true;
+    schedulePersist();
+  }
+
+  // Debounced persist — flush to disk after 2 seconds of inactivity
+  function schedulePersist(): void {
+    if (persistTimer) clearTimeout(persistTimer);
+    persistTimer = setTimeout(() => {
+      persistToDisk().catch(err => console.error('[VectorStoreEngine] persist error', err));
+    }, 2000);
+  }
+
+  async function persist(): Promise<void> {
+    if (persistTimer) {
+      clearTimeout(persistTimer);
+      persistTimer = null;
+    }
+    await persistToDisk();
+  }
+
+  function search(query: number[], k = 5): SearchResult[] {
+    const q = normalize(query);
+    const results: SearchResult[] = [];
+    for (const entry of entries.values()) {
+      const score = cosine(q, entry.vector);
+      results.push({ id: entry.id, score, metadata: entry.metadata });
+    }
+    return results.sort((a, b) => b.score - a.score).slice(0, k);
+  }
+
+  // Alias for semantic clarity
+  function query(vector: number[], k = 5): SearchResult[] {
+    return search(vector, k);
+  }
+
+  function get(id: string): VectorEntry | undefined {
+    return entries.get(id);
+  }
+
+  function remove(id: string): boolean {
+    const deleted = entries.delete(id);
+    if (deleted) {
+      dirty = true;
+      // Remove file from disk
+      const ep = entryPath(id);
+      if (fs.existsSync(ep)) {
+        fs.unlinkSync(ep);
+      }
+      schedulePersist();
+    }
+    return deleted;
+  }
+
+  function clear(): void {
+    entries.clear();
+    dirty = true;
+    schedulePersist();
+  }
+
+  function size(): number {
+    return entries.size;
+  }
+
+  async function shutdown(): Promise<void> {
+    if (persistTimer) {
+      clearTimeout(persistTimer);
+      persistTimer = null;
+    }
+    if (dirty) {
+      await persistToDisk();
+    }
+    entries.clear();
+    console.log('[VectorStoreEngine] shutdown');
+  }
+
+  return { boot, add, persist, search, query, get, remove, clear, size, shutdown };
+}