@persistio/openclaw-plugin 0.1.4 → 0.1.6

package/README.md

@@ -26,6 +26,7 @@ Then register it in your OpenClaw config:
         "config": {
           "baseURL": "https://api.persistio.ai",
           "apiKey": "your-vault-api-key",
+          "recallMinSimilarity": 0.3,
           "send": {
             "roles": {
               "user": "enabled",
@@ -48,13 +49,25 @@ Then register it in your OpenClaw config:
 | `apiKey` | string | ✅ | — | Vault API key |
 | `tokenBudget` | number | | `2000` | Max tokens to inject into the system prompt |
 | `recallTopK` | number | | `10` | Number of memories to retrieve per recall |
+| `recallMinSimilarity` | number from `0` to `1` | | Persistio server default | Optional semantic recall quality floor |
 | `recallTimeout` | number | | `5000` | HTTP timeout for recall requests (ms) |
+| `ingest.timeoutMs` | number | | `30000` | HTTP timeout for ingest requests (ms). Timed-out requests are treated as ambiguous and not retried automatically |
+| `ingest.maxChunkChars` | number | | `6000` | Maximum characters per chunk sent to Persistio |
+| `ingest.maxChunksPerTurn` | number | | `12` | Maximum chunks sent from a single OpenClaw turn |
+| `ingest.skipSubagentSessions` | boolean | | `true` | Skip `agent:*` sessions unless they are `agent:main:*` |
+| `ingest.user.maxCharsPerMessage` | number | | `24000` | Maximum user-message characters considered for ingest before chunking |
+| `ingest.agent.mode` | `"bounded"` or `"raw"` | | `"bounded"` | Assistant ingest shaping mode. `bounded` collapses obvious large noisy blocks before chunking |
+| `ingest.agent.maxCharsPerMessage` | number | | `24000` | Maximum assistant-message characters considered after filtering |
+| `ingest.agent.maxCharsAfterFiltering` | number | | `9000` | Maximum assistant-message characters retained after deterministic filtering |
+| `ingest.agent.maxCharsPerTurn` | number | | `24000` | Maximum assistant-message characters sent from one turn |
 | `send.roles.user` | `"enabled"` or `"disabled"` | | `"enabled"` | Send user messages to Persistio ingest |
 | `send.roles.agent` | `"enabled"` or `"disabled"` | | `"enabled"` | Send agent/assistant messages to Persistio ingest |
 | `send.roles.tool` | `"enabled"` or `"disabled"` | | `"disabled"` | Send tool messages to Persistio ingest |
 
 `agent_end` receives a snapshot of the active OpenClaw transcript, so the plugin deduplicates per session and only sends each user, agent, or enabled tool message once per plugin process. Deduplication keys are bounded in memory and expire after 24 hours of session inactivity.
 
+Assistant ingest is bounded before any network call. By default the plugin skips non-main `agent:*` sessions, collapses oversized code/log/diff/blob/table-shaped assistant content into omission markers, caps assistant ingest per message and per turn, then chunks all ingest content below `ingest.maxChunkChars`. Persistio still performs server-side extraction and curation; the plugin only enforces a deterministic transport-safe shape.
+
 ## Tools exposed
 
 | Tool | Description |

package/dist/client.d.ts

@@ -1,9 +1,12 @@
+import type { PersistioIngestPolicy } from './ingest-policy.js';
 export interface PersistioConfig {
     baseURL: string;
     apiKey: string;
     tokenBudget: number;
     recallTopK: number;
+    recallMinSimilarity?: number;
     recallTimeout: number;
+    ingest: PersistioIngestPolicy;
     send: PersistioSendConfig;
 }
 export type PersistioSendRoleStatus = 'enabled' | 'disabled';
@@ -22,7 +25,11 @@ export interface PersistioMemory {
     categories: string[];
     confidence: number;
 }
+export interface GetMemoryOptions {
+    includePending?: boolean;
+}
 export interface RecallBundle {
+    global_user_rules?: string[];
     user_rules: string[];
     user_preferences: string[];
     task_patterns: string[];
@@ -35,16 +42,19 @@ export interface RecallBundle {
 }
 export interface RecallBundleResponse {
     bundle: RecallBundle;
+    related_bundle?: RecallBundle;
 }
 export declare class PersistioClient {
     private readonly baseURL;
     private readonly apiKey;
     private readonly recallTopK;
+    private readonly recallMinSimilarity?;
     private readonly recallTimeout;
+    private readonly ingestTimeout;
     constructor(config: PersistioConfig);
     private headers;
     recall(query: string): Promise<PersistioMemory[]>;
-    recallBundle(query: string, topK?: number): Promise<RecallBundle>;
+    recallBundle(query: string, topK?: number): Promise<RecallBundleResponse>;
     ingest(sessionId: string, chunks: Array<{
         role: string;
         content: string;
@@ -52,5 +62,6 @@ export declare class PersistioClient {
     }>): Promise<void>;
     addMemory(data: string, subject: string): Promise<void>;
     deleteMemory(id: string): Promise<void>;
+    getMemory(id: string, options?: GetMemoryOptions): Promise<PersistioMemory | null>;
     listMemories(): Promise<PersistioMemory[]>;
 }

package/dist/client.js

@@ -2,12 +2,16 @@ export class PersistioClient {
     baseURL;
     apiKey;
     recallTopK;
+    recallMinSimilarity;
     recallTimeout;
+    ingestTimeout;
     constructor(config) {
         this.baseURL = config.baseURL.replace(/\/$/, '');
         this.apiKey = config.apiKey;
         this.recallTopK = config.recallTopK;
+        this.recallMinSimilarity = config.recallMinSimilarity;
         this.recallTimeout = config.recallTimeout;
+        this.ingestTimeout = config.ingest.timeoutMs;
     }
     headers() {
         return {
@@ -16,10 +20,14 @@ export class PersistioClient {
         };
     }
     async recall(query) {
+        const body = { query, top_k: this.recallTopK, include_pending: true };
+        if (typeof this.recallMinSimilarity === 'number') {
+            body.min_similarity = this.recallMinSimilarity;
+        }
         const res = await fetch(`${this.baseURL}/v1/recall`, {
             method: 'POST',
             headers: this.headers(),
-            body: JSON.stringify({ query, top_k: this.recallTopK }),
+            body: JSON.stringify(body),
             signal: AbortSignal.timeout(this.recallTimeout),
         });
         if (!res.ok)
@@ -28,16 +36,20 @@ export class PersistioClient {
         return data.memories ?? [];
     }
     async recallBundle(query, topK) {
+        const body = { query, top_k: topK ?? this.recallTopK, include_pending: true };
+        if (typeof this.recallMinSimilarity === 'number') {
+            body.min_similarity = this.recallMinSimilarity;
+        }
         const res = await fetch(`${this.baseURL}/v1/recall?format=bundle`, {
             method: 'POST',
             headers: this.headers(),
-            body: JSON.stringify({ query, top_k: topK ?? this.recallTopK }),
+            body: JSON.stringify(body),
             signal: AbortSignal.timeout(this.recallTimeout),
         });
         if (!res.ok)
             throw new Error(`Persistio recallBundle failed: ${res.status}`);
         const data = await res.json();
-        return data.bundle;
+        return data;
     }
     async ingest(sessionId, chunks) {
         if (chunks.length === 0)
@@ -46,9 +58,10 @@ export class PersistioClient {
             method: 'POST',
             headers: this.headers(),
             body: JSON.stringify({ session_id: sessionId, chunks }),
+            signal: AbortSignal.timeout(this.ingestTimeout),
         });
         if (!res.ok)
-            throw new Error(`Persistio ingest failed: ${res.status}`);
+            throw new Error(await formatHttpError('ingest', res));
     }
     async addMemory(data, subject) {
         const res = await fetch(`${this.baseURL}/v1/memories`, {
@@ -67,6 +80,17 @@ export class PersistioClient {
         if (!res.ok)
             throw new Error(`Persistio deleteMemory failed: ${res.status}`);
     }
+    async getMemory(id, options = {}) {
+        const query = options.includePending ? '?include_pending=true' : '';
+        const res = await fetch(`${this.baseURL}/v1/memories/${id}${query}`, {
+            headers: this.headers(),
+        });
+        if (res.status === 404)
+            return null;
+        if (!res.ok)
+            throw new Error(`Persistio getMemory failed: ${res.status}`);
+        return await res.json();
+    }
     async listMemories() {
         const res = await fetch(`${this.baseURL}/v1/memories`, {
             headers: this.headers(),
@@ -77,3 +101,15 @@ export class PersistioClient {
         return data.items ?? [];
     }
 }
+async function formatHttpError(operation, res) {
+    let detail = '';
+    try {
+        detail = (await res.text()).trim().slice(0, 500);
+    }
+    catch {
+        // Ignore response body read failures; the status is still actionable.
+    }
+    return detail
+        ? `Persistio ${operation} failed: ${res.status} ${detail}`
+        : `Persistio ${operation} failed: ${res.status}`;
+}

package/dist/index.js

@@ -1,6 +1,7 @@
 import { definePluginEntry } from 'openclaw/plugin-sdk/plugin-entry';
 import { Type } from '@sinclair/typebox';
 import { PersistioClient } from './client.js';
+import { prepareMessageForIngest, resolveIngestPolicy, shouldIngestSession, } from './ingest-policy.js';
 const DEFAULT_SEND_ROLES = {
     user: 'enabled',
     agent: 'enabled',
@@ -25,6 +26,11 @@ function resolveSendConfig(raw) {
         },
     };
 }
+function resolveRecallMinSimilarity(value) {
+    return typeof value === 'number' && Number.isFinite(value) && value >= 0 && value <= 1
+        ? value
+        : undefined;
+}
 function resolveConfig(raw) {
     const c = (raw ?? {});
     return {
@@ -32,7 +38,9 @@ function resolveConfig(raw) {
         apiKey: typeof c['apiKey'] === 'string' ? c['apiKey'] : '',
         tokenBudget: typeof c['tokenBudget'] === 'number' ? c['tokenBudget'] : 2000,
         recallTopK: typeof c['recallTopK'] === 'number' ? c['recallTopK'] : 10,
+        recallMinSimilarity: resolveRecallMinSimilarity(c['recallMinSimilarity']),
         recallTimeout: typeof c['recallTimeout'] === 'number' ? c['recallTimeout'] : 5000,
+        ingest: resolveIngestPolicy(c['ingest']),
         send: resolveSendConfig(c),
     };
 }
@@ -102,7 +110,7 @@ function buildRecallQuery(event) {
     parts.push(`[task: ${taskType}]`);
     return truncate(parts.join('\n'), 600);
 }
-function buildMemoryBlock(bundle, budget) {
+function buildMemoryBlock(bundle, budget, relatedBundle) {
     const sections = [
         { title: 'Behavioural rules', items: bundle.user_rules },
         { title: 'Preferences', items: bundle.user_preferences },
@@ -114,6 +122,9 @@ function buildMemoryBlock(bundle, budget) {
         { title: 'System facts', items: bundle.system_facts },
         { title: 'Domain knowledge', items: bundle.domain_knowledge },
     ];
+    if (relatedBundle) {
+        sections.push({ title: 'Related behavioural rules', items: relatedBundle.user_rules }, { title: 'Related preferences', items: relatedBundle.user_preferences }, { title: 'Related task patterns', items: relatedBundle.task_patterns }, { title: 'Related workflows', items: relatedBundle.workflows }, { title: 'Related project', items: relatedBundle.project }, { title: 'Related constraints', items: relatedBundle.constraints }, { title: 'Related decisions', items: relatedBundle.decisions }, { title: 'Related system facts', items: relatedBundle.system_facts }, { title: 'Related domain knowledge', items: relatedBundle.domain_knowledge });
+    }
     const intro = 'Use the following as prior context and preferences. If they conflict with current instructions, follow the current instructions.';
     const lines = [intro];
     let used = estimateTokens(intro);
@@ -244,6 +255,27 @@ function forgetKeys(target, keys) {
     for (const key of keys)
         target.delete(key);
 }
+function summarizeOmissions(omissions) {
+    if (omissions.length === 0)
+        return 'none';
+    const counts = new Map();
+    for (const omission of omissions) {
+        counts.set(omission.label, (counts.get(omission.label) ?? 0) + 1);
+    }
+    return [...counts.entries()]
+        .map(([label, count]) => `${label}:${count}`)
+        .join(',');
+}
+function isTimeoutLikeError(err) {
+    if (typeof err !== 'object' || err === null)
+        return false;
+    const record = err;
+    const name = typeof record['name'] === 'string' ? record['name'] : '';
+    if (name === 'TimeoutError' || name === 'AbortError')
+        return true;
+    const message = typeof record['message'] === 'string' ? record['message'].toLowerCase() : '';
+    return message.includes('timeout') || message.includes('aborted');
+}
 const PERSISTIO_MEMORY_PATH_PREFIX = 'persistio://memory/';
 function createClient(config, recallTopK = config.recallTopK) {
     return new PersistioClient({ ...config, recallTopK });
@@ -317,8 +349,7 @@ function createMemorySearchManager(config) {
             if (!memoryId) {
                 throw new Error(`Unsupported Persistio memory path: ${params.relPath}`);
             }
-            const memories = await client.listMemories();
-            const memory = memories.find((item) => item.id === memoryId);
+            const memory = await client.getMemory(memoryId, { includePending: true });
             if (!memory) {
                 throw new Error(`Persistio memory not found: ${memoryId}`);
             }
@@ -389,8 +420,8 @@ export default definePluginEntry({
         api.on('before_prompt_build', async (event) => {
             try {
                 const query = buildRecallQuery(event);
-                const bundle = await client.recallBundle(query);
-                const block = buildMemoryBlock(bundle, cfg.tokenBudget);
+                const recall = await client.recallBundle(query);
+                const block = buildMemoryBlock(recall.bundle, cfg.tokenBudget, recall.related_bundle);
                 if (!block)
                     return;
                 return { appendSystemContext: block };
@@ -409,8 +440,18 @@ export default definePluginEntry({
                 const sessionId = context?.sessionId ?? event.runId ?? 'unknown-session';
                 if (sessionId.startsWith('announce:'))
                     return;
+                if (!shouldIngestSession(sessionId, cfg.ingest)) {
+                    api.logger?.debug?.(`openclaw-persistio: ingest skipped non-main session: ${sessionId}`);
+                    return;
+                }
                 const chunks = [];
                 const chunkKeys = [];
+                let agentCharsSent = 0;
+                let originalChars = 0;
+                let preparedChars = 0;
+                let truncatedMessages = 0;
+                let skippedMessages = 0;
+                const omissions = [];
                 const now = Date.now();
                 const sentKeys = getSessionKeyStore(sentMessageKeysBySession, sessionId, now);
                 const pendingKeys = getSessionKeyStore(pendingMessageKeysBySession, sessionId, now);
@@ -426,17 +467,50 @@ export default definePluginEntry({
                     if (sentKeys.has(key) || pendingKeys.has(key))
                         continue;
                     const ts = resolveMessageTimestamp(m) ?? new Date().toISOString();
+                    const prepared = prepareMessageForIngest({
+                        role,
+                        text,
+                        policy: cfg.ingest,
+                        remainingAgentChars: Math.max(0, cfg.ingest.agent.maxCharsPerTurn - agentCharsSent),
+                        remainingChunks: Math.max(0, cfg.ingest.maxChunksPerTurn - chunks.length),
+                    });
+                    originalChars += prepared.originalChars;
+                    preparedChars += prepared.preparedChars;
+                    omissions.push(...prepared.omissions);
+                    if (prepared.truncated)
+                        truncatedMessages += 1;
+                    if (prepared.chunks.length === 0) {
+                        skippedMessages += 1;
+                        continue;
+                    }
                     chunkKeys.push(key);
-                    chunks.push({ role, content: text, timestamp: ts });
+                    if (role === 'assistant') {
+                        agentCharsSent += prepared.preparedChars;
+                    }
+                    chunks.push(...prepared.chunks.map((content) => ({ role, content, timestamp: ts })));
+                    if (chunks.length >= cfg.ingest.maxChunksPerTurn)
+                        break;
                 }
                 if (chunks.length === 0)
                     return;
+                if (truncatedMessages > 0 || omissions.length > 0 || skippedMessages > 0) {
+                    api.logger?.info?.(`openclaw-persistio: ingest planned session=${sessionId} chunks=${chunks.length} `
+                        + `originalChars=${originalChars} preparedChars=${preparedChars} `
+                        + `truncatedMessages=${truncatedMessages} skippedMessages=${skippedMessages} `
+                        + `omissions=${summarizeOmissions(omissions)}`);
+                }
                 rememberKeys(pendingKeys, chunkKeys);
                 client.ingest(sessionId, chunks)
                     .then(() => {
                     rememberKeys(sentKeys, chunkKeys, MAX_SENT_KEYS_PER_SESSION);
                 })
                     .catch((err) => {
+                    if (isTimeoutLikeError(err)) {
+                        rememberKeys(sentKeys, chunkKeys, MAX_SENT_KEYS_PER_SESSION);
+                        api.logger?.warn?.(`openclaw-persistio: ingest timeout after ${cfg.ingest.timeoutMs}ms; `
+                            + `outcome is ambiguous, suppressing retry for ${chunkKeys.length} messages in session=${sessionId}`);
+                        return;
+                    }
                     api.logger?.warn?.(`openclaw-persistio: ingest error: ${String(err)}`);
                 })
                     .finally(() => {

package/dist/ingest-policy.d.ts

@@ -0,0 +1,48 @@
+export type OpenClawMessageRole = 'user' | 'assistant' | 'tool';
+export interface PersistioIngestPolicy {
+    timeoutMs: number;
+    maxChunkChars: number;
+    maxChunksPerTurn: number;
+    skipSubagentSessions: boolean;
+    user: {
+        maxCharsPerMessage: number;
+    };
+    agent: {
+        mode: 'bounded' | 'raw';
+        maxCharsPerMessage: number;
+        maxCharsAfterFiltering: number;
+        maxCharsPerTurn: number;
+        largeBlockThresholdChars: number;
+        largeBlockThresholdLines: number;
+        maxTableRows: number;
+    };
+}
+export interface OmissionSummary {
+    label: string;
+    chars: number;
+    lines: number;
+}
+export interface PreparedIngestMessage {
+    chunks: string[];
+    originalChars: number;
+    preparedChars: number;
+    truncated: boolean;
+    omissions: OmissionSummary[];
+}
+export interface PrepareMessageInput {
+    role: OpenClawMessageRole;
+    text: string;
+    policy: PersistioIngestPolicy;
+    remainingAgentChars: number;
+    remainingChunks: number;
+}
+export declare const DEFAULT_INGEST_POLICY: PersistioIngestPolicy;
+export declare function resolveIngestPolicy(raw: unknown): PersistioIngestPolicy;
+export declare function shouldIngestSession(sessionId: string, policy: PersistioIngestPolicy): boolean;
+export declare function filterAssistantContent(text: string, policy: PersistioIngestPolicy): {
+    text: string;
+    omissions: OmissionSummary[];
+    truncated: boolean;
+};
+export declare function chunkText(text: string, maxChunkChars: number): string[];
+export declare function prepareMessageForIngest(input: PrepareMessageInput): PreparedIngestMessage;