@ppagent/memory 0.4.4 → 0.4.6

package/dist/cli.js

@@ -51,9 +51,27 @@ function resolveConfig(config) {
       "topicCompactionSyncRatio"
     ),
     contextUsageRatio: ratio(config.contextUsageRatio ?? 0.75, "contextUsageRatio"),
-    precompressionRatio: ratio(config.precompressionRatio ?? 0.75, "precompressionRatio"),
-    compressionBatchRatio: ratio(config.compressionBatchRatio ?? 0.5, "compressionBatchRatio"),
+    precompressionRatio: ratio(config.precompressionRatio ?? 0.6, "precompressionRatio"),
+    compressionBatchRatio: ratio(config.compressionBatchRatio ?? 0.7, "compressionBatchRatio"),
     compressionBatchTokenLimit: Math.max(0, Math.floor(config.compressionBatchTokenLimit ?? 0)),
+    compressionMaxRetries: nonNegativeInt(
+      config.compressionMaxRetries ?? 2,
+      "compressionMaxRetries"
+    ),
+    compressionRetryBaseDelayMs: nonNegativeInt(
+      config.compressionRetryBaseDelayMs ?? 500,
+      "compressionRetryBaseDelayMs"
+    ),
+    backgroundCompressionMaxRetries: nonNegativeInt(
+      config.backgroundCompressionMaxRetries ?? 2,
+      "backgroundCompressionMaxRetries"
+    ),
+    backgroundCompressionRetryBaseDelayMs: nonNegativeInt(
+      config.backgroundCompressionRetryBaseDelayMs ?? 1e3,
+      "backgroundCompressionRetryBaseDelayMs"
+    ),
+    onCompressionEvent: config.onCompressionEvent ?? (() => {
+    }),
     topicSummaryMaxTokens: positiveInt(
       config.topicSummaryMaxTokens ?? config.detailMaxTokens ?? 2048,
       "topicSummaryMaxTokens"
@@ -90,6 +108,8 @@ function resolveConfig(config) {
     graphExtractConcurrency: config.graphExtractConcurrency ?? 2,
     graphBuildTimeoutMs: config.graphBuildTimeoutMs ?? 12e4,
     autoOptimizeOnInit: config.autoOptimizeOnInit ?? true,
+    autoMigrateLegacyDataOnInit: config.autoMigrateLegacyDataOnInit ?? true,
+    autoResumeCompressionOnInit: config.autoResumeCompressionOnInit ?? true,
     autoOptimizeIntervalMs: config.autoOptimizeIntervalMs ?? 6 * 36e5,
     optimizeVersionRetentionMs: config.optimizeVersionRetentionMs ?? 0,
     restoreConcurrency: config.restoreConcurrency ?? 8
@@ -101,6 +121,12 @@ function ratio(value, name) {
   }
   return value;
 }
+function nonNegativeInt(value, name) {
+  if (!Number.isFinite(value) || value < 0) {
+    throw new Error(`[MemoryConfig] ${name} \u5FC5\u987B\u662F\u975E\u8D1F\u6570`);
+  }
+  return Math.floor(value);
+}
 function positiveInt(value, name) {
   if (!Number.isFinite(value) || value <= 0) {
     throw new Error(`[MemoryConfig] ${name} \u5FC5\u987B\u662F\u6B63\u6570`);
@@ -134,6 +160,136 @@ function countTokens(text) {
   return encoder.encode(text).length;
 }
 
+// src/history-image.ts
+var OMIT_IMAGE = Symbol("omit-history-image");
+var IMAGE_PART_TYPES = /* @__PURE__ */ new Set([
+  "image",
+  "image-data",
+  "image-url",
+  "image-file-id",
+  "image-file-reference",
+  "input-image",
+  "output-image"
+]);
+function sanitizeRawHistoryFields(message) {
+  const partsResult = sanitizeValue(message.parts ?? []);
+  const payloadResult = sanitizeValue(message.payload);
+  const contextResult = sanitizeValue(message.contextPayload);
+  const metadataResult = sanitizeValue(message.metadata ?? {});
+  const payload = optionalSanitizedValue(payloadResult);
+  const contextPayload = optionalSanitizedValue(contextResult);
+  return {
+    parts: Array.isArray(partsResult.value) ? partsResult.value : [],
+    ...payload !== void 0 && { payload },
+    ...contextPayload !== void 0 && { contextPayload },
+    metadata: isRecord(metadataResult.value) ? metadataResult.value : {},
+    removedImages: partsResult.removedImages + payloadResult.removedImages + contextResult.removedImages + metadataResult.removedImages
+  };
+}
+function sanitizeStoredMessageImages(message) {
+  const parsed = {
+    content: message.content,
+    parts: parseArray(message.parts),
+    ...message.payload !== void 0 && { payload: parseValue(message.payload) },
+    ...message.contextPayload !== void 0 && {
+      contextPayload: parseValue(message.contextPayload)
+    },
+    metadata: parseObject(message.metadata)
+  };
+  const sanitized = sanitizeRawHistoryFields(parsed);
+  if (sanitized.removedImages === 0) {
+    return { message, changed: false, removedImages: 0 };
+  }
+  const parts = JSON.stringify(sanitized.parts);
+  const payload = sanitized.payload === void 0 ? void 0 : JSON.stringify(sanitized.payload);
+  const contextPayload = sanitized.contextPayload === void 0 ? void 0 : JSON.stringify(sanitized.contextPayload);
+  const metadata = JSON.stringify(sanitized.metadata);
+  const usage2 = countTokens(
+    [message.content, parts, metadata, payload ?? "", contextPayload ?? ""].join("\n")
+  );
+  return {
+    message: {
+      ...message,
+      parts,
+      payload,
+      contextPayload,
+      usage: usage2,
+      metadata
+    },
+    changed: true,
+    removedImages: sanitized.removedImages
+  };
+}
+function sanitizeValue(value) {
+  if (typeof value === "string" && /^data:image\//i.test(value.trim())) {
+    return { value: OMIT_IMAGE, removedImages: 1, changed: true };
+  }
+  if (value == null || typeof value !== "object") {
+    return { value, removedImages: 0, changed: false };
+  }
+  if (isImageNode(value)) {
+    return { value: OMIT_IMAGE, removedImages: 1, changed: true };
+  }
+  if (Array.isArray(value)) {
+    const out2 = [];
+    let removedImages2 = 0;
+    let changed2 = false;
+    for (const item of value) {
+      const sanitized = sanitizeValue(item);
+      removedImages2 += sanitized.removedImages;
+      changed2 ||= sanitized.changed;
+      if (sanitized.value !== OMIT_IMAGE) out2.push(sanitized.value);
+    }
+    return { value: changed2 ? out2 : value, removedImages: removedImages2, changed: changed2 };
+  }
+  const out = {};
+  let removedImages = 0;
+  let changed = false;
+  for (const [key, item] of Object.entries(value)) {
+    const sanitized = sanitizeValue(item);
+    removedImages += sanitized.removedImages;
+    changed ||= sanitized.changed;
+    if (sanitized.value !== OMIT_IMAGE) out[key] = sanitized.value;
+  }
+  return { value: changed ? out : value, removedImages, changed };
+}
+function isImageNode(value) {
+  if (!isRecord(value)) return false;
+  const type = normalizeType(value.type);
+  if (IMAGE_PART_TYPES.has(type)) return true;
+  const mediaType = String(value.mediaType ?? value.mimeType ?? "").trim().toLowerCase();
+  return mediaType.startsWith("image/") && (type === "file" || "data" in value || "url" in value || "fileId" in value || "providerReference" in value);
+}
+function normalizeType(value) {
+  return typeof value === "string" ? value.trim().toLowerCase().replaceAll("_", "-") : "";
+}
+function optionalSanitizedValue(result) {
+  if (result.value === OMIT_IMAGE) return void 0;
+  if (result.changed && isEmptyContainer(result.value)) return void 0;
+  return result.value;
+}
+function isEmptyContainer(value) {
+  return Array.isArray(value) ? value.length === 0 : isRecord(value) && Object.keys(value).length === 0;
+}
+function isRecord(value) {
+  return !!value && typeof value === "object" && !Array.isArray(value);
+}
+function parseValue(value) {
+  try {
+    return JSON.parse(value);
+  } catch {
+    return value;
+  }
+}
+function parseArray(value) {
+  const parsed = parseValue(value);
+  return Array.isArray(parsed) ? parsed : [];
+}
+function parseObject(value) {
+  const parsed = parseValue(value);
+  return isRecord(parsed) ? parsed : {};
+}
+
 // src/db/memory.store.ts
 var MESSAGES_TABLE = "messages";
 var TOPICS_TABLE = "topics";
@@ -596,17 +752,28 @@ var MemoryStore = class {
     const messageRows = await this.provider.query(MESSAGES_TABLE);
     report.messagesScanned = messageRows.length;
     for (const row of messageRows) {
+      const sanitized = sanitizeStoredMessageImages(rowToMessage(row)).message;
       const usage2 = countTokens(
         [
-          row.content,
-          row.parts ?? "[]",
-          row.metadata ?? "{}",
-          row.payload ?? "",
-          row.context_payload ?? ""
+          sanitized.content,
+          sanitized.parts,
+          sanitized.metadata,
+          sanitized.payload ?? "",
+          sanitized.contextPayload ?? ""
         ].map(String).join("\n")
       );
-      if (Number(row.usage) === usage2) continue;
-      await this.provider.update(MESSAGES_TABLE, { usage: usage2 }, [
+      const values = {
+        parts: sanitized.parts,
+        payload: sanitized.payload ?? null,
+        context_payload: sanitized.contextPayload ?? null,
+        metadata: sanitized.metadata,
+        usage: usage2
+      };
+      const changed = Object.entries(values).some(
+        ([key, value]) => !migrationValueEquals(row[key], value)
+      );
+      if (!changed) continue;
+      await this.provider.update(MESSAGES_TABLE, values, [
         eq("message_id", String(row.message_id))
       ]);
       report.messagesUpdated++;

package/dist/index.d.ts

@@ -2,6 +2,26 @@ type MessageType = "text" | "image" | "file";
 type FactLevel = "user" | "chat";
 type SearchMode = "fast" | "auto" | "all";
 type SearchScope = "session" | "chat" | "user" | "all";
+type CompressionRole = "user" | "assistant" | "tool" | "system";
+type MemoryCompressionKind = "raw" | "topics";
+type MemoryCompressionMode = "background" | "blocking" | "manual" | "startup";
+type MemoryCompressionPhase = "start" | "retry" | "success" | "failure";
+interface MemoryCompressionEvent {
+    sessionId: string;
+    kind: MemoryCompressionKind;
+    mode: MemoryCompressionMode;
+    phase: MemoryCompressionPhase;
+    /** 当前尝试序号，从 1 开始。 */
+    attempt: number;
+    /** 本层最多尝试次数。 */
+    maxAttempts: number;
+    inputTokens?: number;
+    selectedTokens?: number;
+    outputTokens?: number;
+    durationMs?: number;
+    retryKind?: "semantic" | "task";
+    error?: string;
+}
 /**
  * 分页参数（可选）。
  * - offset：起始偏移，从 0 开始（负数按 0 处理）。
@@ -67,6 +87,13 @@ interface RawMessage {
      * Topic 摘要或知识图谱抽取。适合保存 provider-ready 工具协议消息等大体积细节。
      */
     contextPayload?: unknown;
+    /**
+     * 同一个完整对话轮次（含 user、assistant 与 tool 链）应使用相同 id。
+     * 压缩只会在 group 边界切分；省略时按 compressionRole/talkerId 兼容推断。
+     */
+    compressionGroupId?: string;
+    /** 消息在对话轮次中的角色，用于旧数据缺少显式 group id 时推断完整边界。 */
+    compressionRole?: CompressionRole;
     usage?: number;
     metadata?: Record<string, unknown>;
     createdAt?: number;
@@ -245,6 +272,8 @@ interface MemoryRawMessage {
     parts?: ContentPart[];
     payload?: unknown;
     contextPayload?: unknown;
+    compressionGroupId?: string;
+    compressionRole?: CompressionRole;
     usage: number;
     metadata?: Record<string, unknown>;
     createdAt: number;
@@ -419,12 +448,22 @@ interface MemoryConfig {
     topicCompactionSyncRatio?: number;
     /** 模型上下文中允许历史记忆使用的比例，默认 0.75。 */
     contextUsageRatio?: number;
-    /** 原始消息达到其可用预算的此比例时后台预压缩，默认 0.75。 */
+    /** 原始消息达到其可用预算的此比例时后台预压缩，默认 0.6。 */
     precompressionRatio?: number;
-    /** 单次压缩目标占原始消息预算的比例，默认 0.5。 */
+    /** 单次压缩目标占当前未压缩消息 token 总量的比例，默认 0.7（压 7 留 3）。 */
     compressionBatchRatio?: number;
     /** 单次压缩硬上限；0 表示只受 compressionBatchRatio 控制。 */
     compressionBatchTokenLimit?: number;
+    /** 压缩模型返回非法 JSON、缺字段或空摘要时的最大重试次数，默认 2（即最多 3 次）。 */
+    compressionMaxRetries?: number;
+    /** 压缩结果级重试的指数退避基数（毫秒），默认 500；0 表示不等待。 */
+    compressionRetryBaseDelayMs?: number;
+    /** 后台压缩整任务失败后的最大重试次数，默认 2（即最多执行 3 轮任务）。 */
+    backgroundCompressionMaxRetries?: number;
+    /** 后台压缩整任务重试的指数退避基数（毫秒），默认 1000；0 表示立即重试。 */
+    backgroundCompressionRetryBaseDelayMs?: number;
+    /** 压缩生命周期事件；回调异常会被隔离，不影响记忆读写。 */
+    onCompressionEvent?: (event: MemoryCompressionEvent) => void | Promise<void>;
     /** 单条 Topic 摘要硬上限，实际长度由重要性决定，默认 2048。 */
     topicSummaryMaxTokens?: number;
     /** 未传 getHistoryWindow 模型窗口时使用，默认 256K。 */
@@ -490,6 +529,10 @@ interface MemoryConfig {
      * 不阻塞启动；嵌入式单进程使用时建议保持开启，否则 LanceDB 版本目录会无限膨胀、启动越来越慢。
      */
     autoOptimizeOnInit?: boolean;
+    /** init 后是否在后台幂等执行一次 0.4 历史数据迁移（含历史图片剥离），默认 true。 */
+    autoMigrateLegacyDataOnInit?: boolean;
+    /** init 后是否后台扫描未完成的原始窗口并恢复自动压缩，默认 true。 */
+    autoResumeCompressionOnInit?: boolean;
     /**
      * 运行期定期压实的间隔（毫秒，默认 6 小时；设为 0 或负数关闭定时任务）。
      * 长驻服务不重启时版本仍会随写入累积，靠该定时任务周期性回收。
@@ -684,13 +727,19 @@ declare class MemoryManager {
     private sessionSweepTimer?;
     private optimizeRunning;
     private optimizeTask?;
+    private startupMaintenanceTask?;
     private destroyTask?;
     private readonly hydration;
     private readonly pendingWrites;
     private readonly topicCompactions;
+    private readonly backgroundCompressions;
     private warnedDefaultModelContext;
     constructor(config: MemoryConfig);
     init(): Promise<void>;
+    private runStartupMaintenance;
+    private getPersistedRawTailTokens;
+    private migrateLegacyDataOnce;
+    private legacyMigrationMarkerPath;
     /** 后台压实的统一入口：防重入（上一轮未结束则跳过），失败仅告警不影响服务。*/
     private runBackgroundOptimize;
     /**
@@ -702,6 +751,8 @@ declare class MemoryManager {
     private isSessionBusy;
     private waitForPendingWrites;
     private queueSessionWrite;
+    private scheduleBackgroundCompression;
+    private waitForBackgroundCompressionIdle;
     updateChat(messages: RawMessage[], opts?: UpdateChatOptions): Promise<void>;
     private doUpdateChat;
     flushChat(sessionId?: string, opts?: {
@@ -1019,4 +1070,4 @@ type RelationType = (typeof DEFAULT_RELATION_TYPES)[number];
 declare const KIND_CONVERSATION = "conversation";
 declare const KIND_KNOWLEDGE = "knowledge";
 
-export { type AddDocumentOptions, type BuildGraphMode, type Chunk, type ChunkHit, type ChunkPiece, type CompressOutput, type ContentPart, DEFAULT_NODE_TYPES, DEFAULT_RELATION_TYPES, type Document, type DomainIds, type Entity, type EntityMeta, type Fact, type FactLevel, type Filter, type FilterCondition, type GetHistoryWindowOptions, KIND_CONVERSATION, KIND_KNOWLEDGE, type KnowledgeSearchOptions, type KnowledgeSearchResult, type MemoryBlockingCompressionEvent, type MemoryBlockingCompressionReason, type MemoryConfig, type MemoryContextBudget, type MemoryContextWindow, type MemoryContextWindowUsage, MemoryManager, type MemoryMigrationReport, type MemoryRawMessage, MemoryStore, type MessageType, type NodeType, type PageParams, type Paginated, type ProviderCapabilities, type ProviderKind, type RawMessage, type Relation, type RelationType, type SearchMode, type SearchOptions, type SearchResult, type SearchScope, type Session, type SessionEntry, type SessionSearchOptions, type SessionView, type StorageOptimizeResult, type StoredMessage, type Topic, type UpdateChatOptions, type UpdateEntityOptions, type UpdateSessionOptions, type VectorStoreProvider, calculateContextBudget };
+export { type AddDocumentOptions, type BuildGraphMode, type Chunk, type ChunkHit, type ChunkPiece, type CompressOutput, type CompressionRole, type ContentPart, DEFAULT_NODE_TYPES, DEFAULT_RELATION_TYPES, type Document, type DomainIds, type Entity, type EntityMeta, type Fact, type FactLevel, type Filter, type FilterCondition, type GetHistoryWindowOptions, KIND_CONVERSATION, KIND_KNOWLEDGE, type KnowledgeSearchOptions, type KnowledgeSearchResult, type MemoryBlockingCompressionEvent, type MemoryBlockingCompressionReason, type MemoryCompressionEvent, type MemoryCompressionKind, type MemoryCompressionMode, type MemoryCompressionPhase, type MemoryConfig, type MemoryContextBudget, type MemoryContextWindow, type MemoryContextWindowUsage, MemoryManager, type MemoryMigrationReport, type MemoryRawMessage, MemoryStore, type MessageType, type NodeType, type PageParams, type Paginated, type ProviderCapabilities, type ProviderKind, type RawMessage, type Relation, type RelationType, type SearchMode, type SearchOptions, type SearchResult, type SearchScope, type Session, type SessionEntry, type SessionSearchOptions, type SessionView, type StorageOptimizeResult, type StoredMessage, type Topic, type UpdateChatOptions, type UpdateEntityOptions, type UpdateSessionOptions, type VectorStoreProvider, calculateContextBudget };