n8n-nodes-tembory 1.1.25 → 1.1.27

package/dist/nodes/Tembory/TemboryMemory.node.js

@@ -1205,16 +1205,18 @@ const getMemoryStore = (ctx) => {
         data.tembory.activeSummary = data.tembory.activeSummary || {};
         data.tembory.connectedModelSummaryCache = data.tembory.connectedModelSummaryCache || {};
         data.tembory.captureState = data.tembory.captureState || {};
+        data.tembory.memoryDedupe = data.tembory.memoryDedupe || {};
         return data.tembory;
     }
     catch {
-        global.__temboryMemory = global.__temboryMemory || { toolHistory: {}, recentMessages: {}, profileFacts: {}, workingMemory: {}, decisionState: {}, memoryCompression: {}, activeSummary: {}, connectedModelSummaryCache: {}, captureState: {} };
+        global.__temboryMemory = global.__temboryMemory || { toolHistory: {}, recentMessages: {}, profileFacts: {}, workingMemory: {}, decisionState: {}, memoryCompression: {}, activeSummary: {}, connectedModelSummaryCache: {}, captureState: {}, memoryDedupe: {} };
         global.__temboryMemory.workingMemory = global.__temboryMemory.workingMemory || {};
         global.__temboryMemory.decisionState = global.__temboryMemory.decisionState || {};
         global.__temboryMemory.memoryCompression = global.__temboryMemory.memoryCompression || {};
         global.__temboryMemory.activeSummary = global.__temboryMemory.activeSummary || {};
         global.__temboryMemory.connectedModelSummaryCache = global.__temboryMemory.connectedModelSummaryCache || {};
         global.__temboryMemory.captureState = global.__temboryMemory.captureState || {};
+        global.__temboryMemory.memoryDedupe = global.__temboryMemory.memoryDedupe || {};
         return global.__temboryMemory;
     }
 };
@@ -1268,6 +1270,8 @@ const mergeRemoteThreadState = (store, key, state) => {
         store.memoryCompression[key] = state.memoryCompression;
     if (state.captureState && typeof state.captureState === 'object')
         store.captureState[key] = { ...(store.captureState[key] || {}), ...state.captureState };
+    if (state.memoryDedupe && typeof state.memoryDedupe === 'object')
+        store.memoryDedupe[key] = mergeMemoryDedupeState(store.memoryDedupe[key], state.memoryDedupe);
     if (typeof state.activeSummary === 'string' && state.activeSummary)
         store.activeSummary[key] = state.activeSummary;
 };
@@ -1632,6 +1636,195 @@ const normalizeIntentText = (value = '') => String(value || '')
     .toLowerCase()
     .normalize('NFD')
     .replace(/[\u0300-\u036f]/g, '');
+const MEMORY_DEDUPE_MAX_KEYS = 600;
+const LEDGER_MEMORY_KINDS = new Set(['conversation_ledger', 'tool_ledger', 'turn_archive']);
+const normalizeDedupeText = (value = '') => normalizeIntentText(value).replace(/\s+/g, ' ').trim();
+const memoryBodyText = (body = {}) => {
+    const messages = Array.isArray(body.messages) ? body.messages : [];
+    if (messages.length)
+        return messages.map((message) => message.content || message.text || message.message || '').filter(Boolean).join('\n');
+    return body.memory || body.text || body.value || body.content || '';
+};
+const parseTurnArchiveText = (text = '') => {
+    const raw = String(text || '');
+    const start = raw.indexOf('{');
+    if (start < 0)
+        return null;
+    try {
+        const parsed = JSON.parse(raw.slice(start));
+        return parsed && parsed.marker === 'tembory_turn_archive_v1' ? parsed : null;
+    }
+    catch {
+        return null;
+    }
+};
+const compactToolForDedupe = (tool = {}, index = 0) => ({
+    id: tool.id || tool.callId || tool.call_id || undefined,
+    sequence: tool.sequence || index + 1,
+    name: tool.name || tool.tool || tool.toolName || '',
+    input: canonicalToolInput(tool.input || tool.toolInput || tool.args || ''),
+    ok: tool.ok !== false && tool.status !== 'failed',
+    result: String(tool.result || tool.output || ''),
+});
+const canonicalMemoryPayloadForDedupe = (kind = 'memory', body = {}, diagnostics = {}) => {
+    const meta = body.metadata || {};
+    const text = memoryBodyText(body);
+    if (kind === 'recent_message') {
+        const marker = parseRecentMessageMarker(text);
+        return stableStringify({
+            role: meta.role || marker?.role || 'user',
+            content: normalizeDedupeText(meta.content || marker?.content || text),
+        });
+    }
+    if (kind === 'tool_history') {
+        const marker = parseToolHistoryMarker(text);
+        const tool = marker || {
+            id: meta.id,
+            turnId: meta.turn_id || meta.turnId,
+            sequence: meta.sequence,
+            name: meta.name || diagnostics.tool,
+            input: meta.input,
+            ok: meta.ok,
+            result: meta.result,
+        };
+        return stableStringify(compactToolForDedupe(tool));
+    }
+    if (kind === 'tool_ledger') {
+        return stableStringify(dedupeToolHistory(parseToolLedgerMarker(text)).map(compactToolForDedupe));
+    }
+    if (kind === 'conversation_ledger') {
+        const messages = parseConversationLedgerMarker(text);
+        return stableStringify((messages || []).map((message) => ({
+            role: message.role || 'user',
+            content: normalizeDedupeText(message.content || ''),
+        })));
+    }
+    if (kind === 'turn_archive') {
+        const archive = parseTurnArchiveText(text);
+        if (archive) {
+            return stableStringify({
+                conversation: (archive.conversation || []).map((message) => ({
+                    role: message.role || 'user',
+                    content: normalizeDedupeText(message.content || ''),
+                })),
+                tools: (archive.tools || []).map(compactToolForDedupe),
+            });
+        }
+    }
+    return normalizeDedupeText(String(text || '')
+        .replace(/"generated_at"\s*:\s*"[^"]+"/g, '"generated_at":"<ts>"')
+        .replace(/"timestamp"\s*:\s*"[^"]+"/g, '"timestamp":"<ts>"'));
+};
+const inferMemoryKindFromText = (text = '') => {
+    const raw = String(text || '');
+    if (parseRecentMessageMarker(raw))
+        return 'recent_message';
+    if (parseToolHistoryMarker(raw))
+        return 'tool_history';
+    if (parseToolLedgerMarker(raw).length)
+        return 'tool_ledger';
+    if (parseConversationLedgerMarker(raw).length)
+        return 'conversation_ledger';
+    if (parseTurnArchiveText(raw))
+        return 'turn_archive';
+    return '';
+};
+const buildMemoryDedupeCandidate = (body = {}, diagnostics = {}) => {
+    const meta = body.metadata || {};
+    const text = memoryBodyText(body);
+    const kind = String(meta.kind || diagnostics.kind || inferMemoryKindFromText(text) || 'memory');
+    const thread = String(meta.thread_id || diagnostics.thread_id || body.thread_id || '');
+    const project = String(meta.project || diagnostics.project || body.project || '');
+    const slot = [kind, thread, project].filter(Boolean).join(':') || kind;
+    const canonical = canonicalMemoryPayloadForDedupe(kind, body, diagnostics);
+    const key = `${slot}:${stableHash(canonical)}`;
+    return cleanContextValue({
+        key,
+        slot,
+        kind,
+        content_hash: stableHash(canonical),
+        strategy: LEDGER_MEMORY_KINDS.has(kind) ? 'latest-ledger-per-content' : 'exact-canonical-content',
+    });
+};
+const mergeMemoryDedupeState = (current = {}, incoming = {}) => {
+    const merged = {
+        seen: { ...((current || {}).seen || {}), ...((incoming || {}).seen || {}) },
+        latestBySlot: { ...((current || {}).latestBySlot || {}), ...((incoming || {}).latestBySlot || {}) },
+        lastSummary: (incoming || {}).lastSummary || (current || {}).lastSummary || undefined,
+        updatedAt: (incoming || {}).updatedAt || (current || {}).updatedAt || undefined,
+    };
+    const entries = Object.entries(merged.seen);
+    if (entries.length > MEMORY_DEDUPE_MAX_KEYS) {
+        entries
+            .sort((a, b) => String(a[1] || '').localeCompare(String(b[1] || '')))
+            .slice(0, entries.length - MEMORY_DEDUPE_MAX_KEYS)
+            .forEach(([key]) => delete merged.seen[key]);
+    }
+    return cleanContextValue(merged);
+};
+const ensureMemoryDedupeState = (store, key) => {
+    store.memoryDedupe = store.memoryDedupe || {};
+    store.memoryDedupe[key] = mergeMemoryDedupeState(store.memoryDedupe[key] || {}, {});
+    store.memoryDedupe[key].seen = store.memoryDedupe[key].seen || {};
+    store.memoryDedupe[key].latestBySlot = store.memoryDedupe[key].latestBySlot || {};
+    return store.memoryDedupe[key];
+};
+const createDedupeSummary = () => ({
+    enabled: true,
+    write: { candidates: 0, persisted: 0, skipped: 0, byKind: {} },
+    load: {},
+});
+const recordDedupeWriteSummary = (summary, candidate, decision) => {
+    if (!summary || !candidate)
+        return;
+    summary.write.candidates += 1;
+    summary.write[decision.persist ? 'persisted' : 'skipped'] += 1;
+    summary.write.byKind[candidate.kind] = summary.write.byKind[candidate.kind] || { candidates: 0, persisted: 0, skipped: 0 };
+    summary.write.byKind[candidate.kind].candidates += 1;
+    summary.write.byKind[candidate.kind][decision.persist ? 'persisted' : 'skipped'] += 1;
+};
+const evaluateMemoryDedupeWrite = (state, candidate, at = nowIso()) => {
+    if (!candidate || !candidate.key)
+        return { persist: true, reason: 'no_dedupe_key' };
+    state.seen = state.seen || {};
+    state.latestBySlot = state.latestBySlot || {};
+    if (state.seen[candidate.key])
+        return { persist: false, reason: 'duplicate_memory_key' };
+    state.seen[candidate.key] = at;
+    state.latestBySlot[candidate.slot] = candidate.key;
+    state.updatedAt = at;
+    return { persist: true, reason: 'new_memory_key' };
+};
+const dedupeMemoryItemsForLoad = (items = []) => {
+    const list = Array.isArray(items) ? items : [];
+    const seen = new Set();
+    const seenSlots = new Set();
+    const out = [];
+    for (const item of [...list].reverse()) {
+        const candidate = buildMemoryDedupeCandidate({ messages: [{ role: 'system', content: memoryText(item) }], metadata: metadataOf(item) }, {});
+        const slotKey = LEDGER_MEMORY_KINDS.has(candidate.kind) ? candidate.slot : candidate.key;
+        const key = LEDGER_MEMORY_KINDS.has(candidate.kind) ? slotKey : candidate.key;
+        if (LEDGER_MEMORY_KINDS.has(candidate.kind)) {
+            if (seenSlots.has(slotKey))
+                continue;
+            seenSlots.add(slotKey);
+        }
+        else if (seen.has(key)) {
+            continue;
+        }
+        seen.add(key);
+        out.push(item);
+    }
+    out.reverse();
+    return {
+        items: out,
+        summary: cleanContextValue({
+            input: list.length,
+            kept: out.length,
+            removed: Math.max(0, list.length - out.length),
+        }),
+    };
+};
 const hasCommitIntent = (value = '') => /\b(confirm\w*|confim\w*|cnfirm\w*|cofnirm\w*|ocnfi\w*|ocnfia\w*|fechar|aprovar|finalizar|prosseguir|continuar|sim)\b/.test(normalizeIntentText(value));
 const isConversationRecallQuery = (value = '') => {
     const text = normalizeIntentText(value);
@@ -1859,21 +2052,73 @@ const deriveNextExpectedAction = (intent, operationalState = {}) => {
     return 'continue according to the agent prompt using retrieved context; call tools when the agent prompt or tool policy requires them';
 };
 const isGenericMemoryNextAction = (value = '') => /answer using retrieved context and avoid unnecessary tool calls|continue according to the agent prompt using retrieved context/i.test(String(value || ''));
+const shouldCarryPreviousGoal = (intent = '', previousGoal = '') => {
+    if (!previousGoal || isGenericMemoryNextAction(previousGoal))
+        return false;
+    return ['general_message', 'profile_update', 'unknown'].includes(intent);
+};
 const deriveWorkingMemory = ({ query = '', profileFacts = {}, recentMessages = [], toolHistory = [], operationalState = {}, previous = {} }) => {
     const intent = inferUserIntent(query, recentMessages);
-    const lastUser = [...(recentMessages || [])].reverse().find((msg) => /^(user|human)$/i.test(String(msg.role || '')));
+    const chronological = sortConversationChronological(recentMessages || []);
+    const userMessages = chronological.filter((msg) => /^(user|human)$/i.test(String(msg.role || '')));
+    const assistantMessages = chronological.filter((msg) => /^(assistant|ai|agent)$/i.test(String(msg.role || '')));
+    const normalizedQuery = normalizeIntentText(query).trim();
+    const lastUser = [...userMessages].reverse().find((msg) => /^(user|human)$/i.test(String(msg.role || '')));
+    const priorUser = [...userMessages].reverse().find((msg) => normalizeIntentText(msg.content).trim() !== normalizedQuery);
+    const lastAssistant = assistantMessages[assistantMessages.length - 1] || null;
     const activeEntities = [];
     for (const key of ['name', 'company', 'email', 'phone']) {
         if (profileFacts && profileFacts[key])
             activeEntities.push({ type: key, value: profileFacts[key] });
     }
     const lastTool = toolHistory && toolHistory.length ? toolHistory[toolHistory.length - 1] : null;
+    const toolState = (operationalState || {}).tool_state || {};
     const nextExpectedAction = deriveNextExpectedAction(intent, operationalState);
+    const carryPreviousGoal = shouldCarryPreviousGoal(intent, previous.current_goal);
+    const currentGoal = carryPreviousGoal ? previous.current_goal : nextExpectedAction;
+    const toolNames = Array.from(new Set((toolHistory || []).map((tool) => tool.name || tool.tool_name || tool.tool).filter(Boolean)));
+    const currentTurnMayNeedTool = ['affirm', 'commit_or_continue', 'tool_action_candidate', 'selection_or_slot'].includes(intent);
     return {
-        current_goal: previous.current_goal && !isGenericMemoryNextAction(previous.current_goal) ? previous.current_goal : nextExpectedAction,
+        current_goal: currentGoal,
+        current_goal_reason: carryPreviousGoal ? 'carried_from_previous_non_generic_goal' : 'derived_from_current_turn_intent',
         current_task: nextExpectedAction,
         last_user_intent: intent,
         last_user_message: lastUser ? truncate(lastUser.content, 500) : truncate(query, 500),
+        conversation_digest: cleanContextValue({
+            messages: chronological.length,
+            user_messages: userMessages.length,
+            assistant_messages: assistantMessages.length,
+            current_user: truncate(query, 220),
+            prior_user: priorUser ? { content: truncate(priorUser.content, 180), at: priorUser.at } : undefined,
+            last_assistant: lastAssistant ? { content: truncate(lastAssistant.content, 180), at: lastAssistant.at } : undefined,
+        }),
+        tool_digest: cleanContextValue({
+            total: Array.isArray(toolHistory) ? toolHistory.length : 0,
+            ok: (operationalState || {}).tool_counts?.ok,
+            failed: (operationalState || {}).tool_counts?.failed,
+            names: toolNames.slice(0, 12),
+            last_successful: toolState.last_successful_tool ? {
+                name: toolState.last_successful_tool.name,
+                at: toolState.last_successful_tool.at,
+                status: toolState.last_successful_tool.status || 'ok',
+            } : undefined,
+            failed_by_name: toolState.failed_by_name,
+        }),
+        turn_flags: cleanContextValue({
+            recall_only: intent === 'conversation_recall',
+            status_question: intent === 'operational_status_question',
+            current_turn_may_need_tool: currentTurnMayNeedTool,
+            has_prior_tools: Array.isArray(toolHistory) && toolHistory.length > 0,
+            has_profile_facts: activeEntities.length > 0,
+            carry_previous_goal: carryPreviousGoal,
+        }),
+        context_sources: cleanContextValue({
+            conversation: chronological.length > 0,
+            tool_history: Array.isArray(toolHistory) && toolHistory.length > 0,
+            operational_state: Boolean((operationalState || {}).tool_state || (operationalState || {}).tool_counts),
+            profile_facts: activeEntities.length > 0,
+        }),
+        agent_guidance: nextExpectedAction,
         active_entities: activeEntities,
         open_decisions: previous.open_decisions || [],
         last_error: lastTool && lastTool.ok === false ? { tool: lastTool.name, at: lastTool.at, result: lastTool.result } : null,
@@ -2148,9 +2393,15 @@ const cleanContextValue = (value) => {
 };
 const compactWorkingMemoryForAgent = (memory = {}) => cleanContextValue({
     current_goal: memory.current_goal,
+    current_goal_reason: memory.current_goal_reason,
     current_task: memory.current_task,
     last_user_intent: memory.last_user_intent,
     last_user_message: truncate(memory.last_user_message, 220),
+    conversation_digest: memory.conversation_digest,
+    tool_digest: memory.tool_digest,
+    turn_flags: memory.turn_flags,
+    context_sources: memory.context_sources,
+    agent_guidance: memory.agent_guidance,
     active_entities: memory.active_entities,
     open_decisions: memory.open_decisions,
     last_error: memory.last_error,
@@ -2229,6 +2480,7 @@ const compactSaveAuditForAgent = (capture = {}) => cleanContextValue({
         tool_history_after_save: capture.last_save_tool_history_after_save,
         thread_state_saved: capture.last_save_thread_state_saved,
         backend_memory_persistence: capture.last_save_backend_memory_persistence,
+        dedupe: capture.last_save_dedupe_summary,
     },
 });
 const compactEntityTimelineForAgent = (timeline = [], maxItems = 8) => pruneByLimit(timeline || [], maxItems).map((item) => cleanContextValue({
@@ -2523,6 +2775,7 @@ const summarizeMemoryMessagesForSideChannel = (messages = []) => {
             recentHighlights: Array.isArray(parsed.recentHighlights) ? parsed.recentHighlights.length : undefined,
         });
         summary.lastSave = memoryAudit.last_save || parsed.state?.last_save || undefined;
+        summary.dedupeSummary = parsed.dedupeSummary || parsed.diagnostics?.dedupeSummary || undefined;
         summary.loadedSections = cleanContextValue({
             conversation: Boolean(parsed.conversation),
             summary: Boolean(parsed.summary),
@@ -2543,10 +2796,16 @@ const summarizeMemoryMessagesForSideChannel = (messages = []) => {
         });
         summary.workingMemory = cleanContextValue({
             currentGoal: parsed.workingMemory?.current_goal,
+            currentGoalReason: parsed.workingMemory?.current_goal_reason,
             currentTask: parsed.workingMemory?.current_task,
             nextExpectedAction: parsed.workingMemory?.next_expected_action,
             lastUserIntent: parsed.workingMemory?.last_user_intent,
             lastUserMessage: parsed.workingMemory?.last_user_message ? truncate(parsed.workingMemory.last_user_message, 220) : undefined,
+            conversation: parsed.workingMemory?.conversation_digest,
+            tools: parsed.workingMemory?.tool_digest,
+            turnFlags: parsed.workingMemory?.turn_flags,
+            contextSources: parsed.workingMemory?.context_sources,
+            agentGuidance: parsed.workingMemory?.agent_guidance,
         });
         summary.decisionState = cleanContextValue({
             currentIntent: parsed.decisionState?.current_intent,
@@ -2792,6 +3051,7 @@ const buildContextMessages = ({ payloadFormat, query, userId, profileFacts, work
             profile: sectionValue('profile_facts'),
             tools: compactToolLedger,
             memoryAudit,
+            dedupeSummary: diagnostics?.dedupeSummary,
         });
         const renderCompactSection = (section) => {
             if (section.value === null || section.value === undefined)
@@ -3311,6 +3571,58 @@ class TemboryMemory {
         const output = cleanAssistantTranscriptText(rawOutput);
         const toolCalls = extractToolCalls(outputValues);
         const saveAt = nowIso();
+        const dedupeState = ensureMemoryDedupeState(store, key);
+        const dedupeSummary = createDedupeSummary();
+        const persistLegacyMemory = async (legacyBody, diagnostics = {}) => {
+            const candidate = buildMemoryDedupeCandidate(legacyBody, diagnostics);
+            const decision = evaluateMemoryDedupeWrite(dedupeState, candidate, saveAt);
+            recordDedupeWriteSummary(dedupeSummary, candidate, decision);
+            if (!decision.persist)
+                return { skipped: true, reason: decision.reason, dedupeKey: candidate.key };
+            const bodyWithDedupe = {
+                ...legacyBody,
+                metadata: cleanContextValue({
+                    ...(legacyBody.metadata || {}),
+                    memory_dedupe_key: candidate.key,
+                    memory_dedupe_slot: candidate.slot,
+                    memory_dedupe_strategy: candidate.strategy,
+                }),
+            };
+            return await safePersistLegacyMemory(this, bodyWithDedupe, {
+                ...diagnostics,
+                memory_dedupe_key: candidate.key,
+                memory_dedupe_slot: candidate.slot,
+            });
+        };
+        const finalizeDedupeSummary = async () => {
+            dedupeState.lastSummary = dedupeSummary;
+            dedupeState.updatedAt = nowIso();
+            store.memoryDedupe[key] = mergeMemoryDedupeState(store.memoryDedupe[key], dedupeState);
+            store.captureState[key] = cleanContextValue({
+                ...(store.captureState[key] || {}),
+                last_save_dedupe_summary: dedupeSummary,
+            });
+            const finalThreadStateSaved = await saveThreadState(this, key, threadId, project, {
+                kind: 'tembory.thread_state.v1',
+                threadId,
+                project: project || undefined,
+                updatedAt: nowIso(),
+                recentMessages: recentForTurn,
+                toolHistory: toolHistoryForTurn,
+                profileFacts: store.profileFacts[key] || {},
+                workingMemory: workingMemoryForTurn,
+                decisionState: decisionStateForTurn,
+                memoryCompression: compressionForTurn,
+                operationalState: operationalStateForTurn,
+                captureState: store.captureState[key],
+                memoryDedupe: store.memoryDedupe[key],
+                activeSummary: store.activeSummary[key] || '',
+            });
+            store.captureState[key] = cleanContextValue({
+                ...(store.captureState[key] || {}),
+                last_save_thread_state_saved: (store.captureState[key] || {}).last_save_thread_state_saved || finalThreadStateSaved,
+            });
+        };
         store.captureState[key] = {
             last_save_status: 'started',
             last_save_saved: false,
@@ -3324,6 +3636,7 @@ class TemboryMemory {
             last_save_tool_calls_captured: toolCalls.length,
             last_save_tool_names: toolCalls.map((tool) => tool.name).filter(Boolean).slice(0, 20),
             last_save_capture_sources: Array.from(new Set(toolCalls.map((tool) => tool.source).filter(Boolean))),
+            last_save_dedupe_summary: dedupeSummary,
         };
         const recentForTembory = [];
         const profileFromTurn = extractProfileFactsFromText(input, 'user_message', nowIso());
@@ -3424,6 +3737,7 @@ class TemboryMemory {
                 : adv.useVectorMemory === false
                     ? 'thread_state_only'
                     : 'enabled',
+            last_save_dedupe_summary: dedupeSummary,
         });
         const threadStateSaved = await saveThreadState(this, key, threadId, project, {
             kind: 'tembory.thread_state.v1',
@@ -3438,6 +3752,7 @@ class TemboryMemory {
             memoryCompression: compressionForTurn,
             operationalState: operationalStateForTurn,
             captureState: nextCaptureState,
+            memoryDedupe: dedupeState,
             activeSummary: store.activeSummary[key] || '',
         });
         store.captureState[key] = cleanContextValue({
@@ -3449,10 +3764,14 @@ class TemboryMemory {
             globalData.__dataChanged = true;
         }
         catch { }
-        if (adv.persistBackendMemories === false)
+        if (adv.persistBackendMemories === false) {
+            await finalizeDedupeSummary();
             return;
-        if (adv.useVectorMemory === false)
+        }
+        if (adv.useVectorMemory === false) {
+            await finalizeDedupeSummary();
             return;
+        }
         const connectedEmbedding = await getConnectedEmbedding(this, itemIndex);
         if (connectedEmbedding) {
             const ids = { user_id: body.user_id, agent_id: body.agent_id, run_id: body.run_id };
@@ -3465,21 +3784,32 @@ class TemboryMemory {
                 memoryCompression: compressionForTurn,
                 operationalState: operationalStateForTurn,
             });
-            await saveClientVectorMemories(this, [
-                await createClientVectorMemory(connectedEmbedding, archiveText, {
-                    kind: 'turn_archive',
-                    thread_id: threadId,
-                    project: project || undefined,
-                    source: 'n8n_connected_embedding',
-                    generated_at: nowIso(),
-                    message_count: recentForTurn.length,
-                    tool_count: toolHistoryForTurn.length,
-                    latest_tool: toolHistoryForTurn.at(-1)?.name || null,
-                }, ids),
-            ], ids);
+            const archiveMetadata = {
+                kind: 'turn_archive',
+                thread_id: threadId,
+                project: project || undefined,
+                source: 'n8n_connected_embedding',
+                generated_at: nowIso(),
+                message_count: recentForTurn.length,
+                tool_count: toolHistoryForTurn.length,
+                latest_tool: toolHistoryForTurn.at(-1)?.name || null,
+            };
+            const archiveCandidate = buildMemoryDedupeCandidate({ messages: [{ role: 'system', content: archiveText }], metadata: archiveMetadata }, { kind: 'turn_archive', user_id: body.user_id });
+            const archiveDecision = evaluateMemoryDedupeWrite(dedupeState, archiveCandidate, saveAt);
+            recordDedupeWriteSummary(dedupeSummary, archiveCandidate, archiveDecision);
+            if (archiveDecision.persist) {
+                await saveClientVectorMemories(this, [
+                    await createClientVectorMemory(connectedEmbedding, archiveText, {
+                        ...archiveMetadata,
+                        memory_dedupe_key: archiveCandidate.key,
+                        memory_dedupe_slot: archiveCandidate.slot,
+                        memory_dedupe_strategy: archiveCandidate.strategy,
+                    }, ids),
+                ], ids);
+            }
             if (adv.includeRecentMessages !== false && recentForTembory.length) {
                 for (const recent of recentForTembory) {
-                    await safePersistLegacyMemory(this, {
+                    await persistLegacyMemory({
                         messages: [{ role: 'system', content: encodeRecentMessage(recent, threadId) }],
                         infer: false,
                         user_id: body.user_id,
@@ -3497,9 +3827,10 @@ class TemboryMemory {
                     }, {
                         kind: 'recent_message',
                         user_id: body.user_id,
+                        thread_id: threadId,
                     });
                 }
-                await safePersistLegacyMemory(this, {
+                await persistLegacyMemory({
                     messages: [{ role: 'system', content: encodeConversationLedger(recentForTurn, threadId) }],
                     infer: false,
                     user_id: body.user_id,
@@ -3515,11 +3846,12 @@ class TemboryMemory {
                 }, {
                     kind: 'conversation_ledger',
                     user_id: body.user_id,
+                    thread_id: threadId,
                 });
             }
             if (adv.includeToolHistory !== false && toolCalls.length) {
                 for (const tool of toolCalls) {
-                    await safePersistLegacyMemory(this, {
+                    await persistLegacyMemory({
                         messages: [{ role: 'system', content: encodeToolCall(tool, threadId) }],
                         infer: false,
                         user_id: body.user_id,
@@ -3543,9 +3875,10 @@ class TemboryMemory {
                         kind: 'tool_history',
                         user_id: body.user_id,
                         tool: tool.name,
+                        thread_id: threadId,
                     });
                 }
-                await safePersistLegacyMemory(this, {
+                await persistLegacyMemory({
                     messages: [{ role: 'system', content: encodeToolLedger(toolHistoryForTurn, threadId) }],
                     infer: false,
                     user_id: body.user_id,
@@ -3561,18 +3894,21 @@ class TemboryMemory {
                 }, {
                     kind: 'tool_ledger',
                     user_id: body.user_id,
+                    thread_id: threadId,
                 });
             }
+            await finalizeDedupeSummary();
             return;
         }
         if (messages.length)
-            await safePersistLegacyMemory(this, body, {
+            await persistLegacyMemory(body, {
                 kind: 'conversation_messages',
                 user_id: body.user_id,
+                thread_id: threadId,
             });
         if (adv.includeRecentMessages !== false && recentForTembory.length) {
             for (const recent of recentForTembory) {
-                await safePersistLegacyMemory(this, {
+                await persistLegacyMemory({
                     messages: [{ role: 'system', content: encodeRecentMessage(recent, threadId) }],
                     infer: false,
                     user_id: body.user_id,
@@ -3589,9 +3925,10 @@ class TemboryMemory {
                 }, {
                     kind: 'recent_message',
                     user_id: body.user_id,
+                    thread_id: threadId,
                 });
             }
-            await safePersistLegacyMemory(this, {
+            await persistLegacyMemory({
                 messages: [{ role: 'system', content: encodeConversationLedger(recentForTurn, threadId) }],
                 infer: false,
                 user_id: body.user_id,
@@ -3607,11 +3944,12 @@ class TemboryMemory {
             }, {
                 kind: 'conversation_ledger',
                 user_id: body.user_id,
+                thread_id: threadId,
             });
         }
         if (adv.includeToolHistory !== false && !adv.persistToolFactsToTembory && toolCalls.length) {
             for (const tool of toolCalls) {
-                await safePersistLegacyMemory(this, {
+                await persistLegacyMemory({
                     messages: [{ role: 'system', content: encodeToolCall(tool, threadId) }],
                     infer: false,
                     user_id: body.user_id,
@@ -3635,9 +3973,10 @@ class TemboryMemory {
                     kind: 'tool_history',
                     user_id: body.user_id,
                     tool: tool.name,
+                    thread_id: threadId,
                 });
             }
-            await safePersistLegacyMemory(this, {
+            await persistLegacyMemory({
                 messages: [{ role: 'system', content: encodeToolLedger(toolHistoryForTurn, threadId) }],
                 infer: false,
                 user_id: body.user_id,
@@ -3653,11 +3992,12 @@ class TemboryMemory {
             }, {
                 kind: 'tool_ledger',
                 user_id: body.user_id,
+                thread_id: threadId,
             });
         }
         if (adv.persistToolFactsToTembory && toolCalls.length) {
             const facts = toolCalls.map((tool) => `Tool ${tool.name} input=${tool.input}${tool.result ? ` result=${tool.result}` : ''}`).join('\n');
-            await safePersistLegacyMemory(this, {
+            await persistLegacyMemory({
                 messages: [{ role: 'system', content: `Tool facts (read-only):\n${truncate(facts, 2000)}` }],
                 infer: false,
                 user_id: body.user_id,
@@ -3665,9 +4005,10 @@ class TemboryMemory {
             }, {
                 kind: 'tool_facts',
                 user_id: body.user_id,
+                thread_id: threadId,
             });
             for (const tool of toolCalls) {
-                await safePersistLegacyMemory(this, {
+                await persistLegacyMemory({
                     messages: [{ role: 'system', content: encodeToolCall(tool, threadId) }],
                     infer: false,
                     user_id: body.user_id,
@@ -3690,9 +4031,29 @@ class TemboryMemory {
                     kind: 'tool_history',
                     user_id: body.user_id,
                     tool: tool.name,
+                    thread_id: threadId,
                 });
             }
+            await persistLegacyMemory({
+                messages: [{ role: 'system', content: encodeToolLedger(toolHistoryForTurn, threadId) }],
+                infer: false,
+                user_id: body.user_id,
+                agent_id: body.agent_id,
+                run_id: body.run_id,
+                metadata: {
+                    kind: 'tool_ledger',
+                    thread_id: threadId,
+                    project: project || undefined,
+                    source: 'tembory_transcript',
+                    generated_at: nowIso(),
+                },
+            }, {
+                kind: 'tool_ledger',
+                user_id: body.user_id,
+                thread_id: threadId,
+            });
         }
+        await finalizeDedupeSummary();
     }
     async loadMemoryVariablesForItem(itemIndex, inputValues = {}) {
         var _a, _b, _c, _d, _e, _f, _g;
@@ -3744,6 +4105,11 @@ class TemboryMemory {
         catch { }
         let payload;
         let vectorMemories = [];
+        const loadDedupeSummary = {
+            enabled: true,
+            load: {},
+            lastSave: store.memoryDedupe[key]?.lastSummary,
+        };
         if (vectorMemoryEnabled && (retrievalMode === 'semantic' || retrievalMode === 'semanticV2' || retrievalMode === 'hybrid')) {
             const body = { query: String(query || '') };
             // IDs
@@ -4004,6 +4370,12 @@ class TemboryMemory {
                 connectedAi.errors.push(`persistedContext.load: ${error.message || String(error)}`);
             }
         }
+        const vectorDedupe = dedupeMemoryItemsForLoad(vectorMemories);
+        vectorMemories = vectorDedupe.items;
+        loadDedupeSummary.load.vectorMemories = vectorDedupe.summary;
+        const persistedDedupe = dedupeMemoryItemsForLoad(persistedMemoryItems);
+        persistedMemoryItems = persistedDedupe.items;
+        loadDedupeSummary.load.persistedMemories = persistedDedupe.summary;
         if (adv.includeRecentMessages !== false) {
             persistedRecentMessages = persistedMemoryItems
                 .flatMap(recentMessagesFromMemory)
@@ -4018,7 +4390,13 @@ class TemboryMemory {
                 .sort((a, b) => new Date(a.at || 0).getTime() - new Date(b.at || 0).getTime());
         }
         const storedRecentMessages = adv.includeRecentMessages === false ? [] : (store.recentMessages[key] || []).concat(persistedRecentMessages, vectorRecentMessages);
-        const allRecentMessages = dedupeRecentMessages(appendCurrentUserMessage(storedRecentMessages, query));
+        const appendedRecentMessages = appendCurrentUserMessage(storedRecentMessages, query);
+        const allRecentMessages = dedupeRecentMessages(appendedRecentMessages);
+        loadDedupeSummary.load.recentMessages = cleanContextValue({
+            input: appendedRecentMessages.length,
+            kept: allRecentMessages.length,
+            removed: Math.max(0, appendedRecentMessages.length - allRecentMessages.length),
+        });
         const recentMessages = pruneConversationMessagesPreserveAnchors(allRecentMessages, Math.max(Number(adv.recentMessagesLastN || 8), 8));
         const toolHistoryFromRecentMessages = [];
         if (adv.includeToolHistory !== false) {
@@ -4027,7 +4405,13 @@ class TemboryMemory {
         }
         const toolHistoryFromVectorMarkers = adv.includeToolHistory === false ? [] : vectorMemories.flatMap(explicitToolHistoryItemsFromMemory);
         const toolHistoryFromVectorMemories = adv.includeToolHistory === false || adv.includeToolHistorySemanticFallback !== true ? [] : vectorMemories.flatMap(toolHistoryItemsFromMemory);
-        const toolHistory = adv.includeToolHistory === false ? [] : applyToolHistoryWindow((store.toolHistory[key] || []).concat(persistedToolHistory, toolHistoryFromRecentMessages, toolHistoryFromVectorMarkers, toolHistoryFromVectorMemories), adv.toolHistoryTTLSeconds, adv.toolHistoryLastN || 15);
+        const combinedToolHistory = (store.toolHistory[key] || []).concat(persistedToolHistory, toolHistoryFromRecentMessages, toolHistoryFromVectorMarkers, toolHistoryFromVectorMemories);
+        const toolHistory = adv.includeToolHistory === false ? [] : applyToolHistoryWindow(combinedToolHistory, adv.toolHistoryTTLSeconds, adv.toolHistoryLastN || 15);
+        loadDedupeSummary.load.toolHistory = cleanContextValue({
+            input: adv.includeToolHistory === false ? 0 : combinedToolHistory.length,
+            kept: toolHistory.length,
+            removed: Math.max(0, (adv.includeToolHistory === false ? 0 : combinedToolHistory.length) - toolHistory.length),
+        });
         const highlights = adv.includeRecentHighlights === false ? [] : getRecentHighlights(recentMessages, toolHistory, adv.recentHighlightsMaxItems || 6);
         const profileFacts = adv.includeProfileFacts === false ? {} : mergeProfileFacts(store.profileFacts[key], profileFactsFromMessages(allRecentMessages), profileFactsFromMemories(vectorMemories));
         if (adv.includeProfileFacts !== false && Object.keys(profileFacts).length) {
@@ -4157,6 +4541,7 @@ class TemboryMemory {
             },
             connectedAi,
             activeSummary: summaryDiagnostics,
+            dedupeSummary: cleanContextValue(loadDedupeSummary),
         };
         const contextHealth = deriveContextHealth({
             userId: key,
@@ -4296,6 +4681,7 @@ class TemboryMemory {
                 })),
             },
             diagnostics,
+            dedupeSummary: diagnostics.dedupeSummary,
         };
         return {
             response: {
@@ -4429,6 +4815,10 @@ exports.__private = {
     deriveOperationalState,
     deriveActionLedger,
     deriveEntityTimeline,
+    buildMemoryDedupeCandidate,
+    dedupeMemoryItemsForLoad,
+    mergeMemoryDedupeState,
+    evaluateMemoryDedupeWrite,
     scoreOf,
     scoreMetaOf,
     extractProfileFactsFromText,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "n8n-nodes-tembory",
-  "version": "1.1.25",
+  "version": "1.1.27",
   "description": "Tembory node for n8n AI Agents with operational memory, tool history and decision state",
   "license": "MIT",
   "homepage": "https://tembory.com",