ahok-skill 1.3.1

package/src/memory/reflect.ts

@@ -0,0 +1,158 @@
+import { q, log_maint_op } from "../core/db";
+import { add_hsg_memory } from "./hsg";
+import { env } from "../core/cfg";
+import { j } from "../utils";
+
+const sim = (t1: string, t2: string): number => {
+    // Robust Jaccard overlap
+    const s1 = new Set(t1.toLowerCase().split(/\s+/).filter(x => x.length > 0));
+    const s2 = new Set(t2.toLowerCase().split(/\s+/).filter(x => x.length > 0));
+    if (s1.size === 0 || s2.size === 0) return 0;
+
+    let inter = 0;
+    for (const token of s1) {
+        if (s2.has(token)) inter++;
+    }
+    const union = new Set([...s1, ...s2]).size;
+    return union > 0 ? inter / union : 0;
+};
+
+const cluster = (mems: any[]): any[] => {
+    const cls: any[] = [];
+    const used = new Set();
+    for (const m of mems) {
+        if (
+            used.has(m.id) ||
+            m.primary_sector === "reflective" ||
+            m.metadata?.consolidated
+        )
+            continue;
+        const c = { mem: [m], n: 1 };
+        used.add(m.id);
+        for (const o of mems) {
+            if (used.has(o.id) || m.primary_sector !== o.primary_sector)
+                continue;
+            if (sim(m.content, o.content) > 0.8) {
+                c.mem.push(o);
+                c.n++;
+                used.add(o.id);
+            }
+        }
+        if (c.n >= 2) cls.push(c);
+    }
+    return cls;
+};
+
+const sal = (c: any): number => {
+    const now = Date.now();
+    const p = c.n / 10;
+    const r =
+        c.mem.reduce(
+            (s: number, m: any) =>
+                s +
+                Math.exp(-(now - new Date(m.created_at).getTime()) / 43200000),
+            0,
+        ) / c.n;
+    const e = c.mem.some(
+        (m: any) =>
+            m.sectors &&
+            Array.isArray(m.sectors) &&
+            m.sectors.includes("emotional"),
+    )
+        ? 1
+        : 0;
+    return Math.min(1, 0.6 * p + 0.3 * r + 0.1 * e);
+};
+
+const summ = (c: any): string => {
+    const sec = c.mem[0].primary_sector;
+    const n = c.n;
+    const txt = c.mem.map((m: any) => m.content.substring(0, 60)).join("; ");
+    return `${n} ${sec} pattern: ${txt.substring(0, 200)}`;
+};
+
+const mark = async (ids: string[]) => {
+    for (const id of ids) {
+        const m = await q.get_mem.get(id);
+        if (m) {
+            const meta = JSON.parse(m.meta || "{}");
+            meta.consolidated = true;
+            await q.upd_mem.run(
+                m.content,
+                m.tags,
+                JSON.stringify(meta),
+                Date.now(),
+                id,
+            );
+        }
+    }
+};
+
+const boost = async (ids: string[]) => {
+    for (const id of ids) {
+        const m = await q.get_mem.get(id);
+        if (m) await q.upd_mem.run(m.content, m.tags, m.meta, Date.now(), id);
+        await q.upd_seen.run(
+            id,
+            m.last_seen_at,
+            Math.min(1, m.salience * 1.1),
+            Date.now(),
+        );
+    }
+};
+
+export const run_reflection = async () => {
+    console.error("[REFLECT] Starting reflection job...");
+    const min = env.reflect_min || 20;
+    const mems = await q.all_mem.all(100, 0);
+    console.error(
+        `[REFLECT] Fetched ${mems.length} memories (min required: ${min})`,
+    );
+    if (mems.length < min) {
+        console.error("[REFLECT] Not enough memories, skipping");
+        return { created: 0, reason: "low" };
+    }
+    const cls = cluster(mems);
+    console.error(`[REFLECT] Clustered into ${cls.length} groups`);
+    let n = 0;
+    for (const c of cls) {
+        const txt = summ(c);
+        const s = sal(c);
+        const src = c.mem.map((m: any) => m.id);
+        const meta = {
+            type: "auto_reflect",
+            sources: src,
+            freq: c.n,
+            at: new Date().toISOString(),
+        };
+        console.error(
+            `[REFLECT] Creating reflection: ${c.n} memories, salience=${s.toFixed(3)}, sector=${c.mem[0].primary_sector}`,
+        );
+        await add_hsg_memory(txt, j(["reflect:auto"]), meta);
+        await mark(src);
+        await boost(src);
+        n++;
+    }
+    if (n > 0) await log_maint_op("reflect", n);
+    console.error(`[REFLECT] Job complete: created ${n} reflections`);
+    return { created: n, clusters: cls.length };
+};
+
+let timer: NodeJS.Timeout | null = null;
+
+export const start_reflection = () => {
+    if (!env.auto_reflect || timer) return;
+    const int = (env.reflect_interval || 10) * 60000;
+    timer = setInterval(
+        () => run_reflection().catch((e) => console.error("[REFLECT]", e)),
+        int,
+    );
+    console.error(`[REFLECT] Started: every ${env.reflect_interval || 10}m`);
+};
+
+export const stop_reflection = () => {
+    if (timer) {
+        clearInterval(timer);
+        timer = null;
+    }
+};

package/src/memory/user_summary.ts

@@ -0,0 +1,110 @@
+import { q } from "../core/db";
+import { env } from "../core/cfg";
+
+const cos = (a: number[], b: number[]): number => {
+    let d = 0,
+        ma = 0,
+        mb = 0;
+    for (let i = 0; i < a.length; i++) {
+        d += a[i] * b[i];
+        ma += a[i] * a[i];
+        mb += b[i] * b[i];
+    }
+    return d / (Math.sqrt(ma) * Math.sqrt(mb));
+};
+
+const gen_user_summary = (mems: any[]): string => {
+    if (!mems.length) return "User profile initializing... (No memories recorded yet)";
+
+    const recent = mems.slice(0, 10);
+    const projects = new Set<string>();
+    const languages = new Set<string>();
+    const files = new Set<string>();
+
+    let events = 0;
+    let saves = 0;
+
+    for (const m of mems) {
+        if (m.meta) {
+            try {
+                const meta = typeof m.meta === 'string' ? JSON.parse(m.meta) : m.meta;
+                if (meta.ide_project_name) projects.add(meta.ide_project_name);
+                if (meta.language) languages.add(meta.language);
+                if (meta.ide_file_path) files.add(meta.ide_file_path.split(/[\\/]/).pop());
+                if (meta.ide_event_type === 'save') saves++;
+            } catch (e) { /* ignore */ }
+        }
+        events++;
+    }
+
+    const project_str = projects.size > 0 ? Array.from(projects).join(", ") : "Unknown Project";
+    const lang_str = languages.size > 0 ? Array.from(languages).join(", ") : "General";
+    const recent_files = Array.from(files).slice(0, 3).join(", ");
+
+    const last_active = mems[0].created_at ? new Date(mems[0].created_at).toLocaleString() : "Recently";
+
+    return `Active in ${project_str} using ${lang_str}. Focused on ${recent_files || "various files"}. (${mems.length} memories, ${saves} saves). Last active: ${last_active}.`;
+};
+
+export const gen_user_summary_async = async (
+    user_id: string,
+): Promise<string> => {
+    const mems = await q.all_mem_by_user.all(user_id, 100, 0);
+    return gen_user_summary(mems);
+};
+
+export const update_user_summary = async (user_id: string): Promise<void> => {
+    try {
+        const summary = await gen_user_summary_async(user_id);
+        const now = Date.now();
+
+        const existing = await q.get_user.get(user_id);
+        if (!existing) {
+            await q.ins_user.run(user_id, summary, 0, now, now);
+        } else {
+            await q.upd_user_summary.run(user_id, summary, now);
+        }
+    } catch (e) {
+        console.error(`[USER_SUMMARY] Fatal error for ${user_id}:`, e);
+    }
+};
+
+export const auto_update_user_summaries = async (): Promise<{
+    updated: number;
+}> => {
+    const all_mems = await q.all_mem.all(10000, 0);
+    const user_ids = new Set(all_mems.map((m) => m.user_id).filter(Boolean));
+
+    let updated = 0;
+    for (const uid of user_ids) {
+        try {
+            await update_user_summary(uid as string);
+            updated++;
+        } catch (e) {
+            console.error(`[USER_SUMMARY] Failed for ${uid}:`, e);
+        }
+    }
+
+    return { updated };
+};
+
+let timer: NodeJS.Timeout | null = null;
+
+export const start_user_summary_reflection = () => {
+    if (timer) return;
+    const int = (env.user_summary_interval || 30) * 60000;
+    timer = setInterval(
+        () =>
+            auto_update_user_summaries().catch((e) =>
+                console.error("[USER_SUMMARY]", e),
+            ),
+        int,
+    );
+};
+
+export const stop_user_summary_reflection = () => {
+    if (timer) {
+        clearInterval(timer);
+        timer = null;
+    }
+};

package/src/migrate.ts

@@ -0,0 +1,8 @@
+import { run_migrations } from "./core/migrate";
+
+run_migrations().then(() => {
+    process.exit(0);
+}).catch((err) => {
+    console.error(err);
+    process.exit(1);
+});

package/src/ops/compress.ts

@@ -0,0 +1,296 @@
+import { createHash } from "crypto";
+
+export interface CompressionMetrics {
+    ogTok: number;
+    compTok: number;
+    ratio: number;
+    saved: number;
+    pct: number;
+    latency: number;
+    algo: string;
+    ts: number;
+}
+
+export interface CompressionResult {
+    og: string;
+    comp: string;
+    metrics: CompressionMetrics;
+    hash: string;
+}
+
+export interface CompressionStats {
+    total: number;
+    ogTok: number;
+    compTok: number;
+    saved: number;
+    avgRatio: number;
+    latency: number;
+    algos: Record<string, number>;
+    updated: number;
+}
+
+class MemoryCompressionEngine {
+    private stats: CompressionStats = {
+        total: 0,
+        ogTok: 0,
+        compTok: 0,
+        saved: 0,
+        avgRatio: 0,
+        latency: 0,
+        algos: {},
+        updated: Date.now(),
+    };
+
+    private cache = new Map<string, CompressionResult>();
+    private readonly MAX = 500;
+    private readonly MS = 0.05;
+
+    private tok(t: string): number {
+        if (!t) return 0;
+        const w = t.split(/\s+/).length;
+        const c = t.length;
+        return Math.ceil(c / 4 + w / 2);
+    }
+
+    private sem(t: string): string {
+        if (!t || t.length < 50) return t;
+        let c = t;
+        const s = c.split(/[.!?]+\s+/);
+        const u = s.filter((x, i, a) => {
+            if (i === 0) return true;
+            const n = x.toLowerCase().trim();
+            const p = a[i - 1].toLowerCase().trim();
+            return n !== p;
+        });
+        c = u.join(". ").trim();
+        const f = [
+            /\b(just|really|very|quite|rather|somewhat|somehow)\b/gi,
+            /\b(actually|basically|essentially|literally)\b/gi,
+            /\b(I think that|I believe that|It seems that|It appears that)\b/gi,
+            /\b(in order to)\b/gi,
+        ];
+        for (const p of f) c = c.replace(p, "");
+        c = c.replace(/\s+/g, " ").trim();
+        const r: [RegExp, string][] = [
+            [/\bat this point in time\b/gi, "now"],
+            [/\bdue to the fact that\b/gi, "because"],
+            [/\bin the event that\b/gi, "if"],
+            [/\bfor the purpose of\b/gi, "to"],
+            [/\bin the near future\b/gi, "soon"],
+            [/\ba number of\b/gi, "several"],
+            [/\bprior to\b/gi, "before"],
+            [/\bsubsequent to\b/gi, "after"],
+        ];
+        for (const [p, x] of r) c = c.replace(p, x);
+        return c;
+    }
+
+    private syn(t: string): string {
+        if (!t || t.length < 30) return t;
+        let c = t;
+        const ct: [RegExp, string][] = [
+            [/\bdo not\b/gi, "don't"],
+            [/\bcannot\b/gi, "can't"],
+            [/\bwill not\b/gi, "won't"],
+            [/\bshould not\b/gi, "shouldn't"],
+            [/\bwould not\b/gi, "wouldn't"],
+            [/\bit is\b/gi, "it's"],
+            [/\bthat is\b/gi, "that's"],
+            [/\bwhat is\b/gi, "what's"],
+            [/\bwho is\b/gi, "who's"],
+            [/\bthere is\b/gi, "there's"],
+            [/\bhas been\b/gi, "been"],
+            [/\bhave been\b/gi, "been"],
+        ];
+        for (const [p, x] of ct) c = c.replace(p, x);
+        c = c.replace(/\b(the|a|an)\s+(\w+),\s+(the|a|an)\s+/gi, "$2, ");
+        c = c.replace(/\s*{\s*/g, "{").replace(/\s*}\s*/g, "}");
+        c = c.replace(/\s*\(\s*/g, "(").replace(/\s*\)\s*/g, ")");
+        c = c.replace(/\s*;\s*/g, ";");
+        return c;
+    }
+
+    private agg(t: string): string {
+        if (!t) return t;
+        let c = this.sem(t);
+        c = this.syn(c);
+        c = c.replace(/[*_~`#]/g, "");
+        c = c.replace(/https?:\/\/(www\.)?([^\/\s]+)(\/[^\s]*)?/gi, "$2");
+        const a: [RegExp, string][] = [
+            [/\bJavaScript\b/gi, "JS"],
+            [/\bTypeScript\b/gi, "TS"],
+            [/\bPython\b/gi, "Py"],
+            [/\bapplication\b/gi, "app"],
+            [/\bfunction\b/gi, "fn"],
+            [/\bparameter\b/gi, "param"],
+            [/\bargument\b/gi, "arg"],
+            [/\breturn\b/gi, "ret"],
+            [/\bvariable\b/gi, "var"],
+            [/\bconstant\b/gi, "const"],
+            [/\bdatabase\b/gi, "db"],
+            [/\brepository\b/gi, "repo"],
+            [/\benvironment\b/gi, "env"],
+            [/\bconfiguration\b/gi, "config"],
+            [/\bdocumentation\b/gi, "docs"],
+        ];
+        for (const [p, x] of a) c = c.replace(p, x);
+        c = c.replace(/\n{3,}/g, "\n\n");
+        c = c
+            .split("\n")
+            .map((l) => l.trim())
+            .join("\n");
+        return c.trim();
+    }
+
+    compress(
+        t: string,
+        a: "semantic" | "syntactic" | "aggressive" = "semantic",
+    ): CompressionResult {
+        if (!t) {
+            return {
+                og: t,
+                comp: t,
+                metrics: this.empty(a),
+                hash: this.hash(t),
+            };
+        }
+
+        const k = `${a}:${this.hash(t)}`;
+        if (this.cache.has(k)) return this.cache.get(k)!;
+
+        const ot = this.tok(t);
+        let c: string;
+
+        switch (a) {
+            case "semantic":
+                c = this.sem(t);
+                break;
+            case "syntactic":
+                c = this.syn(t);
+                break;
+            case "aggressive":
+                c = this.agg(t);
+                break;
+            default:
+                c = t;
+        }
+
+        const ct = this.tok(c);
+        const sv = ot - ct;
+        const r = ct / ot;
+        const p = (sv / ot) * 100;
+        const l = sv * this.MS;
+
+        const m: CompressionMetrics = {
+            ogTok: ot,
+            compTok: ct,
+            ratio: r,
+            saved: sv,
+            pct: p,
+            latency: l,
+            algo: a,
+            ts: Date.now(),
+        };
+
+        const res: CompressionResult = {
+            og: t,
+            comp: c,
+            metrics: m,
+            hash: this.hash(t),
+        };
+        this.up(m);
+        this.store(k, res);
+        return res;
+    }
+
+    batch(
+        ts: string[],
+        a: "semantic" | "syntactic" | "aggressive" = "semantic",
+    ): CompressionResult[] {
+        return ts.map((t) => this.compress(t, a));
+    }
+
+    auto(t: string): CompressionResult {
+        if (!t || t.length < 50) return this.compress(t, "semantic");
+        const code =
+            /\b(function|const|let|var|def|class|import|export)\b/.test(t);
+        const urls = /https?:\/\//.test(t);
+        const verb = t.split(/\s+/).length > 100;
+        let a: "semantic" | "syntactic" | "aggressive";
+        if (code || urls) a = "aggressive";
+        else if (verb) a = "semantic";
+        else a = "syntactic";
+        return this.compress(t, a);
+    }
+
+    getStats(): CompressionStats {
+        return { ...this.stats };
+    }
+
+    analyze(t: string): Record<string, CompressionMetrics> {
+        const r: Record<string, CompressionMetrics> = {};
+        for (const a of ["semantic", "syntactic", "aggressive"] as const) {
+            const x = this.compress(t, a);
+            r[a] = x.metrics;
+        }
+        return r;
+    }
+
+    reset(): void {
+        this.stats = {
+            total: 0,
+            ogTok: 0,
+            compTok: 0,
+            saved: 0,
+            avgRatio: 0,
+            latency: 0,
+            algos: {},
+            updated: Date.now(),
+        };
+    }
+
+    clear(): void {
+        this.cache.clear();
+    }
+
+    private empty(a: string): CompressionMetrics {
+        return {
+            ogTok: 0,
+            compTok: 0,
+            ratio: 1,
+            saved: 0,
+            pct: 0,
+            latency: 0,
+            algo: a,
+            ts: Date.now(),
+        };
+    }
+
+    private hash(t: string): string {
+        return createHash("md5").update(t).digest("hex").substring(0, 16);
+    }
+
+    private up(m: CompressionMetrics): void {
+        this.stats.total++;
+        this.stats.ogTok += m.ogTok;
+        this.stats.compTok += m.compTok;
+        this.stats.saved += m.saved;
+        this.stats.latency += m.latency;
+        if (this.stats.ogTok > 0)
+            this.stats.avgRatio = this.stats.compTok / this.stats.ogTok;
+        if (!this.stats.algos[m.algo]) this.stats.algos[m.algo] = 0;
+        this.stats.algos[m.algo]++;
+        this.stats.updated = Date.now();
+    }
+
+    private store(k: string, r: CompressionResult): void {
+        if (this.cache.size >= this.MAX) {
+            const f = this.cache.keys().next().value;
+            if (f) this.cache.delete(f);
+        }
+        this.cache.set(k, r);
+    }
+}
+
+export const compressionEngine = new MemoryCompressionEngine();
+export { MemoryCompressionEngine };