@haaaiawd/second-nature 0.1.39 → 0.1.41

package/runtime/dream/insight-extractor.js

@@ -1,135 +1,135 @@
-/**
- * Insight Extractor
- *
- * Core logic: extract source-grounded insight candidates from sampled evidence.
- * Rules-based (no LLM required) using keyword patterns, frequency counts, and
- * temporal clustering. Each insight carries type, summary, sourceRefs, and confidence.
- *
- * - Pattern: recurring themes across multiple evidence entries.
- * - Learning: new skills or behaviors observed over time.
- * - Observation: one-off notable events.
- * - Conflict: contradictory claims or repeated failures.
- * Test coverage: tests/unit/dream/t7-1-3-insight-extraction.test.ts
- */
-// Keywords that indicate learning/development
-const LEARNING_KEYWORDS = [
-    "learned", "learn", "figured out", "discovered", "understood",
-    "new approach", "different way", "better method", "improved",
-];
-// Keywords that indicate recurring patterns
-const PATTERN_KEYWORDS = [
-    "again", "regularly", "routine", "habit", "consistently",
-    "every", "frequently", "pattern", "trend",
-];
-// Keywords that indicate conflict or failure
-const CONFLICT_KEYWORDS = [
-    "failed", "failure", "error", "bug", "broke", "broken",
-    "conflict", "contradict", "disagree", "mismatch", "unexpected",
-    "not working", "does not work", "issue", "problem",
-];
-function countKeywordMatches(text, keywords) {
-    const lower = text.toLowerCase();
-    return keywords.reduce((count, kw) => {
-        const regex = new RegExp(kw.replace(/\s+/g, "\\s+"), "g");
-        const matches = lower.match(regex);
-        return count + (matches ? matches.length : 0);
-    }, 0);
-}
-function groupByDay(items) {
-    const groups = new Map();
-    for (const item of items) {
-        const day = item.createdAt.slice(0, 10);
-        const existing = groups.get(day) ?? [];
-        existing.push(item.summary);
-        groups.set(day, existing);
-    }
-    return groups;
-}
-function findRecurringThemes(items) {
-    const themeCounts = new Map();
-    for (const item of items) {
-        const words = item.summary.toLowerCase().split(/\s+/).filter((w) => w.length > 4);
-        for (const word of words) {
-            const entry = themeCounts.get(word) ?? { count: 0, sourceIds: [] };
-            entry.count++;
-            if (!entry.sourceIds.includes(item.id)) {
-                entry.sourceIds.push(item.id);
-            }
-            themeCounts.set(word, entry);
-        }
-    }
-    return Array.from(themeCounts.entries())
-        .filter(([, v]) => v.count >= 3 && v.sourceIds.length >= 2)
-        .map(([theme, v]) => ({ theme, count: v.count, sourceIds: v.sourceIds }))
-        .sort((a, b) => b.count - a.count)
-        .slice(0, 5);
-}
-export function extractInsights(input) {
-    const allItems = [
-        ...input.evidenceSummaries.map((e) => ({ ...e, origin: "evidence" })),
-        ...input.chronicleSummaries.map((c) => ({ ...c, origin: "chronicle" })),
-    ];
-    const insights = [];
-    const unsupportedClaims = [];
-    if (allItems.length === 0) {
-        return { insights: [], unsupportedClaims: ["no_evidence_for_insight"] };
-    }
-    // 1. Pattern insights — recurring themes
-    const themes = findRecurringThemes(allItems);
-    for (const theme of themes) {
-        insights.push({
-            id: `insight:pattern:${theme.theme}:${crypto.randomUUID()}`,
-            type: "pattern",
-            summary: `Recurring theme "${theme.theme}" observed across ${theme.sourceIds.length} entries`,
-            sourceRefs: theme.sourceIds.slice(0, 10),
-            confidence: Math.min(0.95, 0.5 + theme.count * 0.05),
-        });
-    }
-    // 2. Learning insights — entries with learning keywords
-    const learningItems = allItems.filter((item) => countKeywordMatches(item.summary, LEARNING_KEYWORDS) > 0);
-    if (learningItems.length > 0) {
-        insights.push({
-            id: `insight:learning:${crypto.randomUUID()}`,
-            type: "learning",
-            summary: `Learning observed in ${learningItems.length} entries: ${learningItems[0].summary.slice(0, 60)}...`,
-            sourceRefs: learningItems.map((i) => i.id).slice(0, 10),
-            confidence: Math.min(0.9, 0.4 + learningItems.length * 0.1),
-        });
-    }
-    // 3. Conflict insights — entries with conflict/failure keywords
-    const conflictItems = allItems.filter((item) => countKeywordMatches(item.summary, CONFLICT_KEYWORDS) > 0);
-    if (conflictItems.length >= 2) {
-        insights.push({
-            id: `insight:conflict:${crypto.randomUUID()}`,
-            type: "conflict",
-            summary: `Repeated issues observed in ${conflictItems.length} entries`,
-            sourceRefs: conflictItems.map((i) => i.id).slice(0, 10),
-            confidence: Math.min(0.85, 0.5 + conflictItems.length * 0.05),
-        });
-    }
-    // 4. Observation insights — notable one-off events (high-activity days)
-    const byDay = groupByDay(allItems);
-    let maxDay = "";
-    let maxCount = 0;
-    for (const [day, summaries] of byDay) {
-        if (summaries.length > maxCount) {
-            maxCount = summaries.length;
-            maxDay = day;
-        }
-    }
-    if (maxCount >= 3 && maxDay) {
-        const dayItems = allItems.filter((i) => i.createdAt.startsWith(maxDay));
-        insights.push({
-            id: `insight:observation:${crypto.randomUUID()}`,
-            type: "observation",
-            summary: `High activity day (${maxDay}): ${maxCount} notable events`,
-            sourceRefs: dayItems.map((i) => i.id).slice(0, 10),
-            confidence: Math.min(0.8, 0.4 + maxCount * 0.05),
-        });
-    }
-    // If no insights extracted, record unsupported claim
-    if (insights.length === 0) {
-        unsupportedClaims.push("no_insight_patterns_detected");
-    }
-    return { insights, unsupportedClaims };
-}
+/**
+ * Insight Extractor
+ *
+ * Core logic: extract source-grounded insight candidates from sampled evidence.
+ * Rules-based (no LLM required) using keyword patterns, frequency counts, and
+ * temporal clustering. Each insight carries type, summary, sourceRefs, and confidence.
+ *
+ * - Pattern: recurring themes across multiple evidence entries.
+ * - Learning: new skills or behaviors observed over time.
+ * - Observation: one-off notable events.
+ * - Conflict: contradictory claims or repeated failures.
+ * Test coverage: tests/unit/dream/t7-1-3-insight-extraction.test.ts
+ */
+// Keywords that indicate learning/development
+const LEARNING_KEYWORDS = [
+    "learned", "learn", "figured out", "discovered", "understood",
+    "new approach", "different way", "better method", "improved",
+];
+// Keywords that indicate recurring patterns
+const PATTERN_KEYWORDS = [
+    "again", "regularly", "routine", "habit", "consistently",
+    "every", "frequently", "pattern", "trend",
+];
+// Keywords that indicate conflict or failure
+const CONFLICT_KEYWORDS = [
+    "failed", "failure", "error", "bug", "broke", "broken",
+    "conflict", "contradict", "disagree", "mismatch", "unexpected",
+    "not working", "does not work", "issue", "problem",
+];
+function countKeywordMatches(text, keywords) {
+    const lower = text.toLowerCase();
+    return keywords.reduce((count, kw) => {
+        const regex = new RegExp(kw.replace(/\s+/g, "\\s+"), "g");
+        const matches = lower.match(regex);
+        return count + (matches ? matches.length : 0);
+    }, 0);
+}
+function groupByDay(items) {
+    const groups = new Map();
+    for (const item of items) {
+        const day = item.createdAt.slice(0, 10);
+        const existing = groups.get(day) ?? [];
+        existing.push(item.summary);
+        groups.set(day, existing);
+    }
+    return groups;
+}
+function findRecurringThemes(items) {
+    const themeCounts = new Map();
+    for (const item of items) {
+        const words = item.summary.toLowerCase().split(/\s+/).filter((w) => w.length > 4);
+        for (const word of words) {
+            const entry = themeCounts.get(word) ?? { count: 0, sourceIds: [] };
+            entry.count++;
+            if (!entry.sourceIds.includes(item.id)) {
+                entry.sourceIds.push(item.id);
+            }
+            themeCounts.set(word, entry);
+        }
+    }
+    return Array.from(themeCounts.entries())
+        .filter(([, v]) => v.count >= 3 && v.sourceIds.length >= 2)
+        .map(([theme, v]) => ({ theme, count: v.count, sourceIds: v.sourceIds }))
+        .sort((a, b) => b.count - a.count)
+        .slice(0, 5);
+}
+export function extractInsights(input) {
+    const allItems = [
+        ...input.evidenceSummaries.map((e) => ({ ...e, origin: "evidence" })),
+        ...input.chronicleSummaries.map((c) => ({ ...c, origin: "chronicle" })),
+    ];
+    const insights = [];
+    const unsupportedClaims = [];
+    if (allItems.length === 0) {
+        return { insights: [], unsupportedClaims: ["no_evidence_for_insight"] };
+    }
+    // 1. Pattern insights — recurring themes
+    const themes = findRecurringThemes(allItems);
+    for (const theme of themes) {
+        insights.push({
+            id: `insight:pattern:${theme.theme}:${crypto.randomUUID()}`,
+            type: "pattern",
+            summary: `Recurring theme "${theme.theme}" observed across ${theme.sourceIds.length} entries`,
+            sourceRefs: theme.sourceIds.slice(0, 10),
+            confidence: Math.min(0.95, 0.5 + theme.count * 0.05),
+        });
+    }
+    // 2. Learning insights — entries with learning keywords
+    const learningItems = allItems.filter((item) => countKeywordMatches(item.summary, LEARNING_KEYWORDS) > 0);
+    if (learningItems.length > 0) {
+        insights.push({
+            id: `insight:learning:${crypto.randomUUID()}`,
+            type: "learning",
+            summary: `Learning observed in ${learningItems.length} entries: ${learningItems[0].summary.slice(0, 60)}...`,
+            sourceRefs: learningItems.map((i) => i.id).slice(0, 10),
+            confidence: Math.min(0.9, 0.4 + learningItems.length * 0.1),
+        });
+    }
+    // 3. Conflict insights — entries with conflict/failure keywords
+    const conflictItems = allItems.filter((item) => countKeywordMatches(item.summary, CONFLICT_KEYWORDS) > 0);
+    if (conflictItems.length >= 2) {
+        insights.push({
+            id: `insight:conflict:${crypto.randomUUID()}`,
+            type: "conflict",
+            summary: `Repeated issues observed in ${conflictItems.length} entries`,
+            sourceRefs: conflictItems.map((i) => i.id).slice(0, 10),
+            confidence: Math.min(0.85, 0.5 + conflictItems.length * 0.05),
+        });
+    }
+    // 4. Observation insights — notable one-off events (high-activity days)
+    const byDay = groupByDay(allItems);
+    let maxDay = "";
+    let maxCount = 0;
+    for (const [day, summaries] of byDay) {
+        if (summaries.length > maxCount) {
+            maxCount = summaries.length;
+            maxDay = day;
+        }
+    }
+    if (maxCount >= 3 && maxDay) {
+        const dayItems = allItems.filter((i) => i.createdAt.startsWith(maxDay));
+        insights.push({
+            id: `insight:observation:${crypto.randomUUID()}`,
+            type: "observation",
+            summary: `High activity day (${maxDay}): ${maxCount} notable events`,
+            sourceRefs: dayItems.map((i) => i.id).slice(0, 10),
+            confidence: Math.min(0.8, 0.4 + maxCount * 0.05),
+        });
+    }
+    // If no insights extracted, record unsupported claim
+    if (insights.length === 0) {
+        unsupportedClaims.push("no_insight_patterns_detected");
+    }
+    return { insights, unsupportedClaims };
+}

package/runtime/dream/memory-consolidator.d.ts

@@ -1,45 +1,45 @@
-/**
- * Rules-based memory consolidation.
- *
- * Core logic: dedupe, merge, stale cleanup, and conflict marking on evidence,
- * chronicle, and existing memory entries. No LLM required.
- *
- * - Deduplicate by sourceRef id + kind; keep the most recent.
- * - Merge entries with same kind + similar summary (naive prefix match).
- * - Mark entries older than 90 days as stale (retain but flag).
- * - Mark entries with conflicting sourceRefs as conflict.
- * Test coverage: tests/integration/dream/t7-1-1-dream-pipeline.test.ts
- */
-import type { CanonicalMemoryEntry, SourceRef } from "../storage/memory-store/memory-store-lifecycle.js";
-export interface ConsolidationInput {
-    evidenceSummaries: Array<{
-        id: string;
-        summary: string;
-        sourceRefs: SourceRef[];
-        createdAt: string;
-        sensitivity?: string;
-    }>;
-    chronicleSummaries: Array<{
-        id: string;
-        summary: string;
-        sourceRefs: SourceRef[];
-        createdAt: string;
-    }>;
-    toolExperienceSummaries?: Array<{
-        id: string;
-        summary: string;
-        sourceRefs: SourceRef[];
-        createdAt: string;
-    }>;
-    existingEntries: CanonicalMemoryEntry[];
-}
-export interface ConsolidationResult {
-    entries: CanonicalMemoryEntry[];
-    conflicts: Array<{
-        entryId: string;
-        reason: string;
-    }>;
-    staleCount: number;
-    dedupeCount: number;
-}
-export declare function consolidateMemory(input: ConsolidationInput): ConsolidationResult;
+/**
+ * Rules-based memory consolidation.
+ *
+ * Core logic: dedupe, merge, stale cleanup, and conflict marking on evidence,
+ * chronicle, and existing memory entries. No LLM required.
+ *
+ * - Deduplicate by sourceRef id + kind; keep the most recent.
+ * - Merge entries with same kind + similar summary (naive prefix match).
+ * - Mark entries older than 90 days as stale (retain but flag).
+ * - Mark entries with conflicting sourceRefs as conflict.
+ * Test coverage: tests/integration/dream/t7-1-1-dream-pipeline.test.ts
+ */
+import type { CanonicalMemoryEntry, SourceRef } from "../storage/memory-store/memory-store-lifecycle.js";
+export interface ConsolidationInput {
+    evidenceSummaries: Array<{
+        id: string;
+        summary: string;
+        sourceRefs: SourceRef[];
+        createdAt: string;
+        sensitivity?: string;
+    }>;
+    chronicleSummaries: Array<{
+        id: string;
+        summary: string;
+        sourceRefs: SourceRef[];
+        createdAt: string;
+    }>;
+    toolExperienceSummaries?: Array<{
+        id: string;
+        summary: string;
+        sourceRefs: SourceRef[];
+        createdAt: string;
+    }>;
+    existingEntries: CanonicalMemoryEntry[];
+}
+export interface ConsolidationResult {
+    entries: CanonicalMemoryEntry[];
+    conflicts: Array<{
+        entryId: string;
+        reason: string;
+    }>;
+    staleCount: number;
+    dedupeCount: number;
+}
+export declare function consolidateMemory(input: ConsolidationInput): ConsolidationResult;

package/runtime/dream/memory-consolidator.js

@@ -1,140 +1,140 @@
-/**
- * Rules-based memory consolidation.
- *
- * Core logic: dedupe, merge, stale cleanup, and conflict marking on evidence,
- * chronicle, and existing memory entries. No LLM required.
- *
- * - Deduplicate by sourceRef id + kind; keep the most recent.
- * - Merge entries with same kind + similar summary (naive prefix match).
- * - Mark entries older than 90 days as stale (retain but flag).
- * - Mark entries with conflicting sourceRefs as conflict.
- * Test coverage: tests/integration/dream/t7-1-1-dream-pipeline.test.ts
- */
-const STALE_DAYS = 90;
-function isStale(createdAt) {
-    const then = new Date(createdAt).getTime();
-    const now = Date.now();
-    return now - then > STALE_DAYS * 24 * 60 * 60 * 1000;
-}
-function keyForSourceRefs(refs) {
-    return refs
-        .map((r) => `${r.sourceId}:${r.kind}`)
-        .sort()
-        .join("|");
-}
-function summariesSimilar(a, b) {
-    const norm = (s) => s
-        .toLowerCase()
-        .replace(/[^a-z0-9\s]/g, "")
-        .split(/\s+/)
-        .filter(Boolean)
-        .slice(0, 5)
-        .join(" ");
-    return norm(a) === norm(b);
-}
-export function consolidateMemory(input) {
-    const allRaw = [
-        ...input.evidenceSummaries.map((e) => ({
-            ...e,
-            origin: "evidence",
-        })),
-        ...input.chronicleSummaries.map((c) => ({
-            ...c,
-            origin: "chronicle",
-        })),
-        ...(input.toolExperienceSummaries ?? []).map((t) => ({
-            ...t,
-            origin: "tool_experience",
-        })),
-        ...input.existingEntries.map((e) => ({
-            id: e.entryId,
-            summary: e.summary,
-            sourceRefs: e.sourceRefs,
-            createdAt: e.createdAt,
-            origin: "memory",
-        })),
-    ];
-    // 1. Deduplicate by sourceRef key, keep most recent
-    const bySourceKey = new Map();
-    for (const item of allRaw) {
-        const key = keyForSourceRefs(item.sourceRefs);
-        const existing = bySourceKey.get(key);
-        if (!existing || item.createdAt > existing.createdAt) {
-            bySourceKey.set(key, item);
-        }
-    }
-    const deduped = Array.from(bySourceKey.values());
-    const dedupeCount = allRaw.length - deduped.length;
-    // 2. Merge similar summaries
-    const merged = [];
-    const used = new Set();
-    for (const item of deduped) {
-        if (used.has(item.id))
-            continue;
-        const group = [item];
-        for (const other of deduped) {
-            if (other.id === item.id || used.has(other.id))
-                continue;
-            if (summariesSimilar(item.summary, other.summary)) {
-                group.push(other);
-                used.add(other.id);
-            }
-        }
-        used.add(item.id);
-        const mergedRefs = [];
-        const seenRefKeys = new Set();
-        for (const g of group) {
-            for (const ref of g.sourceRefs) {
-                const rk = `${ref.sourceId}:${ref.kind}`;
-                if (!seenRefKeys.has(rk)) {
-                    seenRefKeys.add(rk);
-                    mergedRefs.push(ref);
-                }
-            }
-        }
-        const latestCreatedAt = group
-            .map((g) => g.createdAt)
-            .sort()
-            .at(-1);
-        merged.push({
-            entryId: `consolidated:${crypto.randomUUID()}`,
-            kind: group[0].origin,
-            summary: group[0].summary,
-            sourceRefs: mergedRefs,
-            createdAt: latestCreatedAt,
-        });
-    }
-    // 3. Stale marking
-    let staleCount = 0;
-    for (const entry of merged) {
-        if (isStale(entry.createdAt)) {
-            staleCount++;
-        }
-    }
-    // 4. Conflict detection: entries with overlapping sourceRefs but different summaries
-    const conflicts = [];
-    for (let i = 0; i < merged.length; i++) {
-        for (let j = i + 1; j < merged.length; j++) {
-            const a = merged[i];
-            const b = merged[j];
-            const aKeys = new Set(a.sourceRefs.map((r) => `${r.sourceId}:${r.kind}`));
-            const overlap = b.sourceRefs.some((r) => aKeys.has(`${r.sourceId}:${r.kind}`));
-            if (overlap && !summariesSimilar(a.summary, b.summary)) {
-                conflicts.push({
-                    entryId: a.entryId,
-                    reason: `conflict_with:${b.entryId}`,
-                });
-                conflicts.push({
-                    entryId: b.entryId,
-                    reason: `conflict_with:${a.entryId}`,
-                });
-            }
-        }
-    }
-    return {
-        entries: merged,
-        conflicts,
-        staleCount,
-        dedupeCount,
-    };
-}
+/**
+ * Rules-based memory consolidation.
+ *
+ * Core logic: dedupe, merge, stale cleanup, and conflict marking on evidence,
+ * chronicle, and existing memory entries. No LLM required.
+ *
+ * - Deduplicate by sourceRef id + kind; keep the most recent.
+ * - Merge entries with same kind + similar summary (naive prefix match).
+ * - Mark entries older than 90 days as stale (retain but flag).
+ * - Mark entries with conflicting sourceRefs as conflict.
+ * Test coverage: tests/integration/dream/t7-1-1-dream-pipeline.test.ts
+ */
+const STALE_DAYS = 90;
+function isStale(createdAt) {
+    const then = new Date(createdAt).getTime();
+    const now = Date.now();
+    return now - then > STALE_DAYS * 24 * 60 * 60 * 1000;
+}
+function keyForSourceRefs(refs) {
+    return refs
+        .map((r) => `${r.sourceId}:${r.kind}`)
+        .sort()
+        .join("|");
+}
+function summariesSimilar(a, b) {
+    const norm = (s) => s
+        .toLowerCase()
+        .replace(/[^a-z0-9\s]/g, "")
+        .split(/\s+/)
+        .filter(Boolean)
+        .slice(0, 5)
+        .join(" ");
+    return norm(a) === norm(b);
+}
+export function consolidateMemory(input) {
+    const allRaw = [
+        ...input.evidenceSummaries.map((e) => ({
+            ...e,
+            origin: "evidence",
+        })),
+        ...input.chronicleSummaries.map((c) => ({
+            ...c,
+            origin: "chronicle",
+        })),
+        ...(input.toolExperienceSummaries ?? []).map((t) => ({
+            ...t,
+            origin: "tool_experience",
+        })),
+        ...input.existingEntries.map((e) => ({
+            id: e.entryId,
+            summary: e.summary,
+            sourceRefs: e.sourceRefs,
+            createdAt: e.createdAt,
+            origin: "memory",
+        })),
+    ];
+    // 1. Deduplicate by sourceRef key, keep most recent
+    const bySourceKey = new Map();
+    for (const item of allRaw) {
+        const key = keyForSourceRefs(item.sourceRefs);
+        const existing = bySourceKey.get(key);
+        if (!existing || item.createdAt > existing.createdAt) {
+            bySourceKey.set(key, item);
+        }
+    }
+    const deduped = Array.from(bySourceKey.values());
+    const dedupeCount = allRaw.length - deduped.length;
+    // 2. Merge similar summaries
+    const merged = [];
+    const used = new Set();
+    for (const item of deduped) {
+        if (used.has(item.id))
+            continue;
+        const group = [item];
+        for (const other of deduped) {
+            if (other.id === item.id || used.has(other.id))
+                continue;
+            if (summariesSimilar(item.summary, other.summary)) {
+                group.push(other);
+                used.add(other.id);
+            }
+        }
+        used.add(item.id);
+        const mergedRefs = [];
+        const seenRefKeys = new Set();
+        for (const g of group) {
+            for (const ref of g.sourceRefs) {
+                const rk = `${ref.sourceId}:${ref.kind}`;
+                if (!seenRefKeys.has(rk)) {
+                    seenRefKeys.add(rk);
+                    mergedRefs.push(ref);
+                }
+            }
+        }
+        const latestCreatedAt = group
+            .map((g) => g.createdAt)
+            .sort()
+            .at(-1);
+        merged.push({
+            entryId: `consolidated:${crypto.randomUUID()}`,
+            kind: group[0].origin,
+            summary: group[0].summary,
+            sourceRefs: mergedRefs,
+            createdAt: latestCreatedAt,
+        });
+    }
+    // 3. Stale marking
+    let staleCount = 0;
+    for (const entry of merged) {
+        if (isStale(entry.createdAt)) {
+            staleCount++;
+        }
+    }
+    // 4. Conflict detection: entries with overlapping sourceRefs but different summaries
+    const conflicts = [];
+    for (let i = 0; i < merged.length; i++) {
+        for (let j = i + 1; j < merged.length; j++) {
+            const a = merged[i];
+            const b = merged[j];
+            const aKeys = new Set(a.sourceRefs.map((r) => `${r.sourceId}:${r.kind}`));
+            const overlap = b.sourceRefs.some((r) => aKeys.has(`${r.sourceId}:${r.kind}`));
+            if (overlap && !summariesSimilar(a.summary, b.summary)) {
+                conflicts.push({
+                    entryId: a.entryId,
+                    reason: `conflict_with:${b.entryId}`,
+                });
+                conflicts.push({
+                    entryId: b.entryId,
+                    reason: `conflict_with:${a.entryId}`,
+                });
+            }
+        }
+    }
+    return {
+        entries: merged,
+        conflicts,
+        staleCount,
+        dedupeCount,
+    };
+}