ahok-skill 1.3.1

package/src/temporal_graph/timeline.ts

@@ -0,0 +1,319 @@
+
+
+import { all_async } from '../core/db'
+import { TemporalFact, TimelineEntry } from './types'
+
+
+export const get_subject_timeline = async (
+    subject: string,
+    predicate?: string
+): Promise<TimelineEntry[]> => {
+    const conditions = ['subject = ?']
+    const params: any[] = [subject]
+
+    if (predicate) {
+        conditions.push('predicate = ?')
+        params.push(predicate)
+    }
+
+    const sql = `
+        SELECT subject, predicate, object, confidence, valid_from, valid_to
+        FROM temporal_facts
+        WHERE ${conditions.join(' AND ')}
+        ORDER BY valid_from ASC
+    `
+
+    const rows = await all_async(sql, params)
+    const timeline: TimelineEntry[] = []
+
+    for (const row of rows) {
+        // Creation event
+        timeline.push({
+            timestamp: new Date(row.valid_from),
+            subject: row.subject,
+            predicate: row.predicate,
+            object: row.object,
+            confidence: row.confidence,
+            change_type: 'created'
+        })
+
+        // Invalidation event (if applicable)
+        if (row.valid_to) {
+            timeline.push({
+                timestamp: new Date(row.valid_to),
+                subject: row.subject,
+                predicate: row.predicate,
+                object: row.object,
+                confidence: row.confidence,
+                change_type: 'invalidated'
+            })
+        }
+    }
+
+    return timeline.sort((a, b) => a.timestamp.getTime() - b.timestamp.getTime())
+}
+
+
+export const get_predicate_timeline = async (
+    predicate: string,
+    from?: Date,
+    to?: Date
+): Promise<TimelineEntry[]> => {
+    const conditions = ['predicate = ?']
+    const params: any[] = [predicate]
+
+    if (from) {
+        conditions.push('valid_from >= ?')
+        params.push(from.getTime())
+    }
+
+    if (to) {
+        conditions.push('valid_from <= ?')
+        params.push(to.getTime())
+    }
+
+    const sql = `
+        SELECT subject, predicate, object, confidence, valid_from, valid_to
+        FROM temporal_facts
+        WHERE ${conditions.join(' AND ')}
+        ORDER BY valid_from ASC
+    `
+
+    const rows = await all_async(sql, params)
+    const timeline: TimelineEntry[] = []
+
+    for (const row of rows) {
+        timeline.push({
+            timestamp: new Date(row.valid_from),
+            subject: row.subject,
+            predicate: row.predicate,
+            object: row.object,
+            confidence: row.confidence,
+            change_type: 'created'
+        })
+
+        if (row.valid_to) {
+            timeline.push({
+                timestamp: new Date(row.valid_to),
+                subject: row.subject,
+                predicate: row.predicate,
+                object: row.object,
+                confidence: row.confidence,
+                change_type: 'invalidated'
+            })
+        }
+    }
+
+    return timeline.sort((a, b) => a.timestamp.getTime() - b.timestamp.getTime())
+}
+
+
+export const get_changes_in_window = async (
+    from: Date,
+    to: Date,
+    subject?: string
+): Promise<TimelineEntry[]> => {
+    const from_ts = from.getTime()
+    const to_ts = to.getTime()
+    const conditions: string[] = []
+    const params: any[] = []
+
+    if (subject) {
+        conditions.push('subject = ?')
+        params.push(subject)
+    }
+
+    const where = conditions.length > 0 ? `AND ${conditions.join(' AND ')}` : ''
+
+    const sql = `
+        SELECT subject, predicate, object, confidence, valid_from, valid_to
+        FROM temporal_facts
+        WHERE ((valid_from >= ? AND valid_from <= ?) OR (valid_to >= ? AND valid_to <= ?))
+        ${where}
+        ORDER BY valid_from ASC
+    `
+
+    const rows = await all_async(sql, [from_ts, to_ts, from_ts, to_ts, ...params])
+    const timeline: TimelineEntry[] = []
+
+    for (const row of rows) {
+        if (row.valid_from >= from_ts && row.valid_from <= to_ts) {
+            timeline.push({
+                timestamp: new Date(row.valid_from),
+                subject: row.subject,
+                predicate: row.predicate,
+                object: row.object,
+                confidence: row.confidence,
+                change_type: 'created'
+            })
+        }
+
+        if (row.valid_to && row.valid_to >= from_ts && row.valid_to <= to_ts) {
+            timeline.push({
+                timestamp: new Date(row.valid_to),
+                subject: row.subject,
+                predicate: row.predicate,
+                object: row.object,
+                confidence: row.confidence,
+                change_type: 'invalidated'
+            })
+        }
+    }
+
+    return timeline.sort((a, b) => a.timestamp.getTime() - b.timestamp.getTime())
+}
+
+
+export const compare_time_points = async (
+    subject: string,
+    time1: Date,
+    time2: Date
+): Promise<{
+    added: TemporalFact[]
+    removed: TemporalFact[]
+    changed: Array<{ before: TemporalFact; after: TemporalFact }>
+    unchanged: TemporalFact[]
+}> => {
+    const t1_ts = time1.getTime()
+    const t2_ts = time2.getTime()
+
+    // Get all facts for subject at both times
+    const facts_t1 = await all_async(`
+        SELECT id, subject, predicate, object, valid_from, valid_to, confidence, last_updated, metadata
+        FROM temporal_facts
+        WHERE subject = ?
+        AND valid_from <= ? AND (valid_to IS NULL OR valid_to >= ?)
+    `, [subject, t1_ts, t1_ts])
+
+    const facts_t2 = await all_async(`
+        SELECT id, subject, predicate, object, valid_from, valid_to, confidence, last_updated, metadata
+        FROM temporal_facts
+        WHERE subject = ?
+        AND valid_from <= ? AND (valid_to IS NULL OR valid_to >= ?)
+    `, [subject, t2_ts, t2_ts])
+
+    const map_t1 = new Map<string, any>()
+    const map_t2 = new Map<string, any>()
+
+    for (const f of facts_t1) {
+        map_t1.set(f.predicate, f)
+    }
+
+    for (const f of facts_t2) {
+        map_t2.set(f.predicate, f)
+    }
+
+    const added: TemporalFact[] = []
+    const removed: TemporalFact[] = []
+    const changed: Array<{ before: TemporalFact; after: TemporalFact }> = []
+    const unchanged: TemporalFact[] = []
+
+    // Find added and changed
+    for (const [pred, fact2] of map_t2) {
+        const fact1 = map_t1.get(pred)
+        if (!fact1) {
+            added.push(row_to_fact(fact2))
+        } else if (fact1.object !== fact2.object || fact1.id !== fact2.id) {
+            changed.push({
+                before: row_to_fact(fact1),
+                after: row_to_fact(fact2)
+            })
+        } else {
+            unchanged.push(row_to_fact(fact2))
+        }
+    }
+
+    // Find removed
+    for (const [pred, fact1] of map_t1) {
+        if (!map_t2.has(pred)) {
+            removed.push(row_to_fact(fact1))
+        }
+    }
+
+    return { added, removed, changed, unchanged }
+}
+
+
+export const get_change_frequency = async (
+    subject: string,
+    predicate: string,
+    window_days: number = 30
+): Promise<{
+    predicate: string
+    total_changes: number
+    avg_duration_ms: number
+    change_rate_per_day: number
+}> => {
+    const now = Date.now()
+    const window_start = now - (window_days * 86400000)
+
+    const rows = await all_async(`
+        SELECT valid_from, valid_to
+        FROM temporal_facts
+        WHERE subject = ? AND predicate = ?
+        AND valid_from >= ?
+        ORDER BY valid_from ASC
+    `, [subject, predicate, window_start])
+
+    const total_changes = rows.length
+    let total_duration = 0
+    let valid_durations = 0
+
+    for (const row of rows) {
+        if (row.valid_to) {
+            total_duration += row.valid_to - row.valid_from
+            valid_durations++
+        }
+    }
+
+    const avg_duration_ms = valid_durations > 0 ? total_duration / valid_durations : 0
+    const change_rate_per_day = total_changes / window_days
+
+    return {
+        predicate,
+        total_changes,
+        avg_duration_ms,
+        change_rate_per_day
+    }
+}
+
+
+export const get_volatile_facts = async (
+    subject?: string,
+    limit: number = 10
+): Promise<Array<{
+    subject: string
+    predicate: string
+    change_count: number
+    avg_confidence: number
+}>> => {
+    const where = subject ? 'WHERE subject = ?' : ''
+    const params = subject ? [subject] : []
+
+    const sql = `
+        SELECT subject, predicate, COUNT(*) as change_count, AVG(confidence) as avg_confidence
+        FROM temporal_facts
+        ${where}
+        GROUP BY subject, predicate
+        HAVING change_count > 1
+        ORDER BY change_count DESC, avg_confidence ASC
+        LIMIT ?
+    `
+
+    return await all_async(sql, [...params, limit])
+}
+
+// Helper function
+function row_to_fact(row: any): TemporalFact {
+    return {
+        id: row.id,
+        subject: row.subject,
+        predicate: row.predicate,
+        object: row.object,
+        valid_from: new Date(row.valid_from),
+        valid_to: row.valid_to ? new Date(row.valid_to) : null,
+        confidence: row.confidence,
+        last_updated: new Date(row.last_updated),
+        metadata: row.metadata ? JSON.parse(row.metadata) : undefined
+    }
+}

package/src/temporal_graph/types.ts

@@ -0,0 +1,41 @@
+export interface TemporalFact {
+    id: string
+    subject: string
+    predicate: string
+    object: string
+    valid_from: Date
+    valid_to: Date | null
+    confidence: number
+    last_updated: Date
+    metadata?: Record<string, any>
+}
+
+export interface TemporalEdge {
+    id: string
+    source_id: string
+    target_id: string
+    relation_type: string
+    valid_from: Date
+    valid_to: Date | null
+    weight: number
+    metadata?: Record<string, any>
+}
+
+export interface TimelineEntry {
+    timestamp: Date
+    subject: string
+    predicate: string
+    object: string
+    confidence: number
+    change_type: 'created' | 'updated' | 'invalidated'
+}
+
+export interface TemporalQuery {
+    subject?: string
+    predicate?: string
+    object?: string
+    at?: Date
+    from?: Date
+    to?: Date
+    min_confidence?: number
+}

package/src/utils/chunking.ts

@@ -0,0 +1,66 @@
+export type chunk = {
+    text: string;
+    start: number;
+    end: number;
+    tokens: number;
+};
+
+const cpt = 4;
+const est = (t: string) => Math.ceil(t.length / cpt);
+
+export const chunk_text = (txt: string, tgt = 768, ovr = 0.1): chunk[] => {
+    const tot = est(txt);
+    if (tot <= tgt)
+        return [{ text: txt, start: 0, end: txt.length, tokens: tot }];
+
+    const tch = tgt * cpt,
+        och = Math.floor(tch * ovr);
+    const paras = txt.split(/\n\n+/);
+
+    const chks: chunk[] = [];
+    let cur = "",
+        cs = 0;
+
+    for (const p of paras) {
+        const sents = p.split(/(?<=[.!?])\s+/);
+        for (const s of sents) {
+            const pot = cur + (cur ? " " : "") + s;
+            if (pot.length > tch && cur.length > 0) {
+                chks.push({
+                    text: cur,
+                    start: cs,
+                    end: cs + cur.length,
+                    tokens: est(cur),
+                });
+                const ovt = cur.slice(-och);
+                cur = ovt + " " + s;
+                cs = cs + cur.length - ovt.length - 1;
+            } else cur = pot;
+        }
+    }
+
+    if (cur.length > 0)
+        chks.push({
+            text: cur,
+            start: cs,
+            end: cs + cur.length,
+            tokens: est(cur),
+        });
+    return chks;
+};
+
+export const agg_vec = (vecs: number[][]): number[] => {
+    const n = vecs.length;
+    if (!n) throw new Error("no vecs");
+    if (n === 1) return vecs[0].slice();
+
+    const d = vecs[0].length,
+        r = new Array(d).fill(0);
+    for (const v of vecs) for (let i = 0; i < d; i++) r[i] += v[i];
+    const rc = 1 / n;
+    for (let i = 0; i < d; i++) r[i] *= rc;
+    return r;
+};
+
+export const join_chunks = (cks: chunk[]) =>
+    cks.length ? cks.map((c) => c.text).join(" ") : "";

package/src/utils/index.ts

@@ -0,0 +1,25 @@
+export const now = (): number => Date.now();
+export const rid = (): string => crypto.randomUUID();
+export const cos_sim = (a: Float32Array, b: Float32Array): number => {
+    let dot = 0,
+        na = 0,
+        nb = 0;
+    for (let i = 0; i < a.length; i++) {
+        const x = a[i],
+            y = b[i];
+        dot += x * y;
+        na += x * x;
+        nb += y * y;
+    }
+    const d = Math.sqrt(na) * Math.sqrt(nb);
+    return d ? dot / d : 0;
+};
+export const j = JSON.stringify;
+export const p = <t = any>(x: string): t => JSON.parse(x);
+export const vec_to_buf = (v: number[]): Buffer => {
+    const f32 = new Float32Array(v);
+    return Buffer.from(f32.buffer);
+};
+export const buf_to_vec = (buf: Buffer): Float32Array => {
+    return new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4);
+};

package/src/utils/keyword.ts

@@ -0,0 +1,137 @@
+import { canonical_tokens_from_text } from "./text";
+import { env } from "../core/cfg";
+
+export interface keyword_match {
+    id: string;
+    score: number;
+    matched_terms: string[];
+}
+
+export function extract_keywords(
+    text: string,
+    min_length: number = 3,
+): Set<string> {
+    const tokens = canonical_tokens_from_text(text);
+    const keywords = new Set<string>();
+
+    for (const token of tokens) {
+        if (token.length >= min_length) {
+            keywords.add(token);
+
+            if (token.length >= 3) {
+                for (let i = 0; i <= token.length - 3; i++) {
+                    keywords.add(token.slice(i, i + 3));
+                }
+            }
+        }
+    }
+
+    for (let i = 0; i < tokens.length - 1; i++) {
+        const bigram = `${tokens[i]}_${tokens[i + 1]}`;
+        if (bigram.length >= min_length) {
+            keywords.add(bigram);
+        }
+    }
+
+    for (let i = 0; i < tokens.length - 2; i++) {
+        const trigram = `${tokens[i]}_${tokens[i + 1]}_${tokens[i + 2]}`;
+        keywords.add(trigram);
+    }
+
+    return keywords;
+}
+
+export function compute_keyword_overlap(
+    query_keywords: Set<string>,
+    content_keywords: Set<string>,
+): number {
+    let matches = 0;
+    let total_weight = 0;
+
+    for (const qk of query_keywords) {
+        if (content_keywords.has(qk)) {
+            const weight = qk.includes("_") ? 2.0 : 1.0;
+            matches += weight;
+        }
+        total_weight += qk.includes("_") ? 2.0 : 1.0;
+    }
+
+    if (total_weight === 0) return 0;
+    return matches / total_weight;
+}
+
+export function exact_phrase_match(query: string, content: string): boolean {
+    const q_norm = query.toLowerCase().trim();
+    const c_norm = content.toLowerCase();
+    return c_norm.includes(q_norm);
+}
+
+export function compute_bm25_score(
+    query_terms: string[],
+    content_terms: string[],
+    corpus_size: number = 10000,
+    avg_doc_length: number = 100,
+): number {
+    const k1 = 1.5;
+    const b = 0.75;
+
+    const term_freq = new Map<string, number>();
+    for (const term of content_terms) {
+        term_freq.set(term, (term_freq.get(term) || 0) + 1);
+    }
+
+    const doc_length = content_terms.length;
+    let score = 0;
+
+    for (const q_term of query_terms) {
+        const tf = term_freq.get(q_term) || 0;
+        if (tf === 0) continue;
+
+        const idf = Math.log((corpus_size + 1) / (tf + 0.5));
+        const numerator = tf * (k1 + 1);
+        const denominator =
+            tf + k1 * (1 - b + b * (doc_length / avg_doc_length));
+
+        score += idf * (numerator / denominator);
+    }
+
+    return score;
+}
+
+export async function keyword_filter_memories(
+    query: string,
+    all_memories: Array<{ id: string; content: string }>,
+    threshold: number = 0.1,
+): Promise<Map<string, number>> {
+    const query_keywords = extract_keywords(query, env.keyword_min_length);
+    const query_terms = canonical_tokens_from_text(query);
+    const scores = new Map<string, number>();
+
+    for (const mem of all_memories) {
+        let total_score = 0;
+
+        if (exact_phrase_match(query, mem.content)) {
+            total_score += 1.0;
+        }
+
+        const content_keywords = extract_keywords(
+            mem.content,
+            env.keyword_min_length,
+        );
+        const keyword_score = compute_keyword_overlap(
+            query_keywords,
+            content_keywords,
+        );
+        total_score += keyword_score * 0.8;
+
+        const content_terms = canonical_tokens_from_text(mem.content);
+        const bm25_score = compute_bm25_score(query_terms, content_terms);
+        total_score += Math.min(1.0, bm25_score / 10) * 0.5;
+
+        if (total_score > threshold) {
+            scores.set(mem.id, total_score);
+        }
+    }
+
+    return scores;
+}

package/src/utils/text.ts

@@ -0,0 +1,115 @@
+const syn_grps = [
+    ["prefer", "like", "love", "enjoy", "favor"],
+    ["theme", "mode", "style", "layout"],
+    ["meeting", "meet", "session", "call", "sync"],
+    ["dark", "night", "black"],
+    ["light", "bright", "day"],
+    ["user", "person", "people", "customer"],
+    ["task", "todo", "job"],
+    ["note", "memo", "reminder"],
+    ["time", "schedule", "when", "date"],
+    ["project", "initiative", "plan"],
+    ["issue", "problem", "bug"],
+    ["document", "doc", "file"],
+    ["question", "query", "ask"],
+];
+const cmap = new Map<string, string>();
+const slook = new Map<string, Set<string>>();
+
+for (const grp of syn_grps) {
+    const can = grp[0];
+    const sset = new Set(grp);
+    for (const w of grp) {
+        cmap.set(w, can);
+        slook.set(can, sset);
+    }
+}
+
+const stem_rules: Array<[RegExp, string]> = [
+    [/ies$/, "y"],
+    [/ing$/, ""],
+    [/ers?$/, "er"],
+    [/ed$/, ""],
+    [/s$/, ""],
+];
+const tok_pat = /[a-z0-9]+/gi;
+
+export const tokenize = (text: string): string[] => {
+    const toks: string[] = [];
+    let m: RegExpExecArray | null;
+    while ((m = tok_pat.exec(text))) {
+        toks.push(m[0].toLowerCase());
+    }
+    return toks;
+};
+
+const stem = (tok: string): string => {
+    if (tok.length <= 3) return tok;
+    for (const [pat, rep] of stem_rules) {
+        if (pat.test(tok)) {
+            const st = tok.replace(pat, rep);
+            if (st.length >= 3) return st;
+        }
+    }
+    return tok;
+};
+
+export const canonicalize_token = (tok: string): string => {
+    if (!tok) return "";
+    const low = tok.toLowerCase();
+    if (cmap.has(low)) return cmap.get(low)!;
+    const st = stem(low);
+    return cmap.get(st) || st;
+};
+
+export const canonical_tokens_from_text = (text: string): string[] => {
+    const res: string[] = [];
+    for (const tok of tokenize(text)) {
+        const can = canonicalize_token(tok);
+        if (can && can.length > 1) {
+            res.push(can);
+        }
+    }
+    return res;
+};
+
+export const synonyms_for = (tok: string): Set<string> => {
+    const can = canonicalize_token(tok);
+    return slook.get(can) || new Set([can]);
+};
+
+export const build_search_doc = (text: string): string => {
+    const can = canonical_tokens_from_text(text);
+    const exp = new Set<string>();
+    for (const tok of can) {
+        exp.add(tok);
+        const syns = slook.get(tok);
+        if (syns) {
+            syns.forEach((s) => exp.add(s));
+        }
+    }
+    return Array.from(exp).join(" ");
+};
+
+export const build_fts_query = (text: string): string => {
+    const can = canonical_tokens_from_text(text);
+    if (!can.length) return "";
+    const uniq = Array.from(new Set(can.filter((t) => t.length > 1)));
+    return uniq.map((t) => `"${t}"`).join(" OR ");
+};
+
+export const canonical_token_set = (text: string): Set<string> => {
+    return new Set(canonical_tokens_from_text(text));
+};
+
+export const add_synonym_tokens = (toks: Iterable<string>): Set<string> => {
+    const res = new Set<string>();
+    for (const tok of toks) {
+        res.add(tok);
+        const syns = slook.get(tok);
+        if (syns) {
+            syns.forEach((s) => res.add(canonicalize_token(s)));
+        }
+    }
+    return res;
+};