@alfe.ai/openclaw-memory-cloud 0.0.38 → 0.0.39

package/src/session-backfill.ts

@@ -1,220 +0,0 @@
-/**
- * Session backfill — one-time ingestion of pre-existing chat history into
- * cloud memory, run fire-and-forget at plugin startup (see index.ts).
- *
- * Reads the openclaw-chat plugin's on-disk session files directly:
- *   ~/.alfe/sessions/chat/{sessionId}.json
- * The write side is packages/openclaw-chat/src/session-store.ts — that path
- * and file shape are a read contract; keep the two in sync. The chat plugin
- * is not a package dependency because its session-store functions aren't
- * part of its public entry and the plugin isn't guaranteed to be installed
- * alongside memory. Like the session store itself, this assumes one agent
- * per machine (`alfe remove` deletes ~/.alfe, so files never outlive the
- * agent they belong to).
- *
- * No-re-sync guarantees (in concert with services/memory):
- * 1. Runs only while the server-side `sessionsBackfillSyncedAt` flag is
- *    unset; the flag survives integration uninstall/reinstall.
- * 2. Only messages with timestamp < cutoffMs are ingested — for agents that
- *    already had memory installed, everything after `bootstrapSyncedAt` was
- *    already captured live by AutoCapture.
- * 3. sessionKey = file sessionId and index = file array position, so the
- *    ingest consumer's per-session `lastProcessedIndex` high-water mark makes
- *    a crash-and-retry replay free (filtered server-side before any cost).
- *
- * Each session is sent as exactly ONE memoryIngest call (one SQS message,
- * windowed to the most recent messages that fit the caps below). Splitting a
- * session across multiple SQS messages would race the shared per-session
- * high-water mark on the standard (non-FIFO) ingest queue: a transiently
- * failed early chunk redelivered after a later chunk succeeds gets filtered
- * out and silently lost.
- */
-
-import { readdir, readFile } from "node:fs/promises";
-import { join } from "node:path";
-import { homedir } from "node:os";
-import type { MemoryCloudConfig, MemoryMessage, SessionMetadata } from "./types.js";
-
-const DEFAULT_SESSIONS_DIR = join(homedir(), ".alfe", "sessions", "chat");
-
-// Server-side zod caps on /agent/memory/ingest (services/memory
-// api-agents/ingest/post.ts) — exceeding any of these is a deterministic 400,
-// which would put the startup retry loop into a permanent failure.
-const MAX_MESSAGES_PER_SESSION = 500;
-const MAX_SESSION_KEY_CHARS = 256;
-const MAX_MESSAGE_INDEX = 1_000_000;
-const MAX_CONTENT_CHARS = 100_000;
-/** One ingest call becomes one SQS SendMessage (256KB hard limit). */
-const MAX_BATCH_BYTES = 180_000;
-
-interface BackfillApi {
-  memoryIngest(sessionKey: string, messages: MemoryMessage[], metadata?: SessionMetadata): Promise<{ queued: boolean; messageCount: number }>;
-  memoryBootstrapStatusMark(scope?: "files" | "sessions"): Promise<{ synced: true; syncedAt: string }>;
-}
-
-interface BackfillLogger {
-  info: (msg: string, ctx?: Record<string, unknown>) => void;
-  debug: (msg: string, ctx?: Record<string, unknown>) => void;
-  warn: (msg: string, ctx?: Record<string, unknown>) => void;
-}
-
-interface ParsedSession {
-  sessionId: string;
-  channel?: string;
-  userId?: string;
-  /** Recent-window messages carrying their original file-array position as `index`. */
-  messages: MemoryMessage[];
-  lastMessageMs: number;
-}
-
-export interface SessionsBackfillOptions {
-  cutoffMs: number;
-  sessionsDir?: string;
-}
-
-export async function runSessionsBackfill(
-  client: BackfillApi,
-  config: MemoryCloudConfig,
-  logger: BackfillLogger,
-  opts: SessionsBackfillOptions,
-): Promise<void> {
-  const sessionsDir = opts.sessionsDir ?? DEFAULT_SESSIONS_DIR;
-
-  let fileNames: string[];
-  try {
-    fileNames = (await readdir(sessionsDir)).filter((name) => name.endsWith(".json"));
-  } catch (err) {
-    if ((err as NodeJS.ErrnoException).code === "ENOENT") {
-      // No sessions directory — nothing to backfill, ever.
-      logger.debug("sessions backfill: no sessions directory, marking complete");
-      try { await client.memoryBootstrapStatusMark("sessions"); } catch { /* retry next start */ }
-    } else {
-      // Transient (EACCES/EMFILE/...) — don't forfeit the backfill.
-      logger.warn("sessions backfill: sessions directory unreadable; will retry on next startup", { err: String(err) });
-    }
-    return;
-  }
-
-  const sessions: ParsedSession[] = [];
-  for (const name of fileNames) {
-    const parsed = await parseSessionFile(join(sessionsDir, name), config.captureMaxChars, opts.cutoffMs);
-    if (parsed) sessions.push(parsed);
-  }
-
-  // Most recent non-empty sessions first, capped. Filter BEFORE capping:
-  // on a retrofit agent the newest sessions are often entirely post-install
-  // (emptied by the cutoff) and must not occupy the cap slots.
-  const eligible = sessions
-    .filter((s) => s.messages.length > 0)
-    .sort((a, b) => b.lastMessageMs - a.lastMessageMs)
-    .slice(0, config.backfillMaxSessions);
-
-  if (eligible.length === 0) {
-    logger.debug("sessions backfill: no eligible sessions, marking complete");
-    try { await client.memoryBootstrapStatusMark("sessions"); } catch { /* retry next start */ }
-    return;
-  }
-
-  logger.info("sessions backfill: ingesting sessions", {
-    sessionCount: eligible.length,
-    messageCount: eligible.reduce((n, s) => n + s.messages.length, 0),
-  });
-
-  for (const session of eligible) {
-    const metadata: SessionMetadata = {
-      ...(session.channel ? { channelId: session.channel } : {}),
-      ...(session.userId ? { userId: session.userId } : {}),
-    };
-    try {
-      await client.memoryIngest(session.sessionId, session.messages, metadata);
-    } catch (err) {
-      // Abort without marking — next startup replays, and the server-side
-      // lastProcessedIndex filter discards already-processed sessions free.
-      logger.warn("sessions backfill: ingest failed; will retry on next startup", {
-        sessionId: session.sessionId,
-        err: String(err),
-      });
-      return;
-    }
-  }
-
-  try {
-    await client.memoryBootstrapStatusMark("sessions");
-    logger.info("sessions backfill: complete", { sessionCount: eligible.length });
-  } catch (err) {
-    logger.warn("sessions backfill: mark failed; will retry on next startup", { err: String(err) });
-  }
-}
-
-/**
- * Tolerant parse of a single session file. Returns null for corrupt or
- * shape-mismatched files (skipped, never fatal). Message `index` is the
- * original array position in the file — stable because the store is
- * append-only — so retries resume idempotently server-side.
- */
-async function parseSessionFile(
-  filePath: string,
-  captureMaxChars: number,
-  cutoffMs: number,
-): Promise<ParsedSession | null> {
-  let raw: unknown;
-  try {
-    raw = JSON.parse(await readFile(filePath, "utf8"));
-  } catch {
-    return null;
-  }
-  if (typeof raw !== "object" || raw === null) return null;
-
-  const data = raw as Record<string, unknown>;
-  if (typeof data.sessionId !== "string" || data.sessionId.length === 0) return null;
-  if (data.sessionId.length > MAX_SESSION_KEY_CHARS) return null;
-  if (!Array.isArray(data.messages)) return null;
-
-  const maxContentChars = Math.min(captureMaxChars, MAX_CONTENT_CHARS);
-  let lastMessageMs = 0;
-  const filtered: MemoryMessage[] = [];
-  for (const [index, entry] of data.messages.entries()) {
-    if (index > MAX_MESSAGE_INDEX) break;
-    if (typeof entry !== "object" || entry === null) continue;
-    const msg = entry as Record<string, unknown>;
-    if (msg.role !== "user" && msg.role !== "assistant") continue;
-    if (typeof msg.content !== "string" || msg.content.trim().length === 0) continue;
-    if (typeof msg.timestamp !== "number") continue;
-
-    lastMessageMs = Math.max(lastMessageMs, msg.timestamp);
-    if (msg.timestamp >= cutoffMs) continue;
-
-    filtered.push({
-      role: msg.role,
-      content: msg.content.length > maxContentChars ? msg.content.slice(0, maxContentChars) : msg.content,
-      index,
-      timestamp: new Date(msg.timestamp).toISOString(),
-    });
-  }
-
-  return {
-    sessionId: data.sessionId,
-    channel: typeof data.channel === "string" ? data.channel : undefined,
-    userId: typeof data.userId === "string" ? data.userId : undefined,
-    messages: recentWindow(filtered),
-    lastMessageMs,
-  };
-}
-
-/**
- * The most recent contiguous run of messages fitting both ingest caps
- * (message count and serialized bytes). Walks backwards from the newest
- * message and stops at the first one that would overflow the byte budget.
- */
-function recentWindow(messages: MemoryMessage[]): MemoryMessage[] {
-  const window: MemoryMessage[] = [];
-  let bytes = 0;
-  for (let i = messages.length - 1; i >= 0; i--) {
-    if (window.length >= MAX_MESSAGES_PER_SESSION) break;
-    const msgBytes = Buffer.byteLength(JSON.stringify(messages[i])) + 1;
-    if (bytes + msgBytes > MAX_BATCH_BYTES) break;
-    window.unshift(messages[i]);
-    bytes += msgBytes;
-  }
-  return window;
-}

package/src/types.ts

@@ -1,93 +0,0 @@
-export interface MemoryCloudConfig {
-  autoCapture: boolean;
-  autoRecall: boolean;
-  captureMaxChars: number;
-  idleFlushSeconds: number;
-  backfillSessions: boolean;
-  backfillMaxSessions: number;
-}
-
-export interface MemoryMessage {
-  role: string;
-  content: string;
-  index: number;
-  timestamp?: string;
-}
-
-export interface SessionMetadata {
-  channelId?: string;
-  userId?: string;
-  userName?: string;
-}
-
-export interface MemorySearchResult {
-  facts: {
-    subject: string;
-    predicate: string;
-    object: string;
-    since: string;
-    confidence: number;
-  }[];
-  memories: {
-    id: string;
-    text: string;
-    topic: string;
-    subtopic: string;
-    tag: string;
-    importance: number;
-    timestamp: number;
-    score: number;
-  }[];
-}
-
-export interface MemoryContext {
-  tier: number;
-  facts: { subject: string; predicate: string; object: string; since: string }[];
-  memories: { text: string; topic: string; subtopic: string; score: number }[];
-  tokenEstimate: number;
-  formatted: string;
-}
-
-/**
- * Interface for memory API calls. Satisfied by AgentApiClient so the plugin
- * doesn't need a hard dependency on the agent-api-client package.
- */
-export interface MemoryApi {
-  memorySearch(query: string, opts?: {
-    limit?: number;
-    topic?: string;
-    subtopic?: string;
-    tag?: string;
-    includeKnowledge?: boolean;
-  }): Promise<MemorySearchResult>;
-
-  memoryStore(text: string, opts?: {
-    topic?: string;
-    subtopic?: string;
-    tag?: string;
-    importance?: number;
-  }): Promise<{ memoryId: string }>;
-
-  memoryIngest(sessionKey: string, messages: MemoryMessage[], metadata?: SessionMetadata, ingestEpoch?: number): Promise<{ queued: boolean; messageCount: number }>;
-
-  memoryLoadContext(tier?: number, topicHint?: string): Promise<{ formatted: string; [key: string]: unknown }>;
-
-  memoryLookupEntity(subject: string): Promise<{
-    subject: string;
-    triples: { tripleId: string; predicate: string; object: string; validFrom: string; validTo?: string; confidence: number }[];
-  }>;
-
-  memoryNavigate(): Promise<{
-    topics: { name: string; tripleCount: number; subtopics: string[] }[];
-    cursor: string | null;
-  }>;
-
-  memoryDelete(memoryId: string): Promise<{ deleted: boolean }>;
-
-  memoryStats(): Promise<{
-    vectorCount: number;
-    tripleCount: number;
-    storageEstimateBytes: number;
-    lastIngestionAt?: string;
-  }>;
-}

package/sst-env.d.ts

@@ -1,10 +0,0 @@
-/* This file is auto-generated by SST. Do not edit. */
-/* tslint:disable */
-/* eslint-disable */
-/* deno-fmt-ignore-file */
-/* biome-ignore-all lint: auto-generated */
-
-/// <reference path="../../sst-env.d.ts" />
-
-import "sst"
-export {}

package/tsconfig.json

@@ -1,20 +0,0 @@
-{
-  "compilerOptions": {
-    "target": "ES2022",
-    "module": "Node16",
-    "moduleResolution": "Node16",
-    "lib": ["ES2022"],
-    "outDir": "dist",
-    "rootDir": "src",
-    "strict": true,
-    "esModuleInterop": true,
-    "skipLibCheck": true,
-    "forceConsistentCasingInFileNames": true,
-    "resolveJsonModule": true,
-    "declaration": true,
-    "declarationMap": true,
-    "sourceMap": true
-  },
-  "include": ["src/**/*"],
-  "exclude": ["node_modules", "dist", "**/*.test.ts"]
-}