ahok-skill 1.3.1

package/src/sources/web_crawler.ts

@@ -0,0 +1,161 @@
+/**
+ * web crawler source for openmemory - production grade
+ * requires: cheerio (for html parsing)
+ * no auth required for public urls
+ */
+
+import { base_source, source_config_error, source_item, source_content, source_config } from './base';
+
+export interface web_crawler_config extends source_config {
+    max_pages?: number;
+    max_depth?: number;
+    timeout?: number;
+}
+
+export class web_crawler_source extends base_source {
+    name = 'web_crawler';
+    private max_pages: number;
+    private max_depth: number;
+    private timeout: number;
+    private visited: Set<string> = new Set();
+    private crawled: source_item[] = [];
+
+    constructor(
+        user_id?: string,
+        config?: web_crawler_config
+    ) {
+        super(user_id, config);
+        this.max_pages = config?.max_pages || 50;
+        this.max_depth = config?.max_depth || 3;
+        this.timeout = config?.timeout || 30000;
+    }
+
+    async _connect(): Promise<boolean> {
+        return true; // no auth needed
+    }
+
+    async _list_items(filters: Record<string, any>): Promise<source_item[]> {
+        if (!filters.start_url) {
+            throw new source_config_error('start_url is required', this.name);
+        }
+
+        let cheerio: any;
+        try {
+            cheerio = await import('cheerio');
+        } catch {
+            throw new source_config_error('missing deps: npm install cheerio', this.name);
+        }
+
+        this.visited.clear();
+        this.crawled = [];
+
+        const base_url = new URL(filters.start_url);
+        const base_domain = base_url.hostname;
+        const to_visit: { url: string; depth: number }[] = [{ url: filters.start_url, depth: 0 }];
+        const follow_links = filters.follow_links !== false;
+
+        while (to_visit.length > 0 && this.crawled.length < this.max_pages) {
+            const { url, depth } = to_visit.shift()!;
+
+            if (this.visited.has(url) || depth > this.max_depth) continue;
+            this.visited.add(url);
+
+            try {
+                const controller = new AbortController();
+                const timeout_id = setTimeout(() => controller.abort(), this.timeout);
+
+                const resp = await fetch(url, {
+                    headers: { 'User-Agent': 'OpenMemory-Crawler/1.0 (compatible)' },
+                    signal: controller.signal
+                });
+
+                clearTimeout(timeout_id);
+
+                if (!resp.ok) continue;
+
+                const content_type = resp.headers.get('content-type') || '';
+                if (!content_type.includes('text/html')) continue;
+
+                const html = await resp.text();
+                const $ = cheerio.load(html);
+
+                const title = $('title').text() || url;
+
+                this.crawled.push({
+                    id: url,
+                    name: title.trim(),
+                    type: 'webpage',
+                    url,
+                    depth
+                });
+
+                // find and queue links
+                if (follow_links && depth < this.max_depth) {
+                    $('a[href]').each((_: any, el: any) => {
+                        try {
+                            const href = $(el).attr('href');
+                            if (!href) return;
+
+                            const full_url = new URL(href, url);
+                            if (full_url.hostname !== base_domain) return;
+
+                            const clean_url = `${full_url.protocol}//${full_url.hostname}${full_url.pathname}`;
+                            if (!this.visited.has(clean_url)) {
+                                to_visit.push({ url: clean_url, depth: depth + 1 });
+                            }
+                        } catch { }
+                    });
+                }
+            } catch (e: any) {
+                console.warn(`[web_crawler] failed to fetch ${url}: ${e.message}`);
+            }
+        }
+
+        return this.crawled;
+    }
+
+    async _fetch_item(item_id: string): Promise<source_content> {
+        let cheerio: any;
+        try {
+            cheerio = await import('cheerio');
+        } catch {
+            throw new source_config_error('missing deps: npm install cheerio', this.name);
+        }
+
+        const controller = new AbortController();
+        const timeout_id = setTimeout(() => controller.abort(), this.timeout);
+
+        const resp = await fetch(item_id, {
+            headers: { 'User-Agent': 'OpenMemory-Crawler/1.0 (compatible)' },
+            signal: controller.signal
+        });
+
+        clearTimeout(timeout_id);
+
+        if (!resp.ok) throw new Error(`http ${resp.status}: ${resp.statusText}`);
+
+        const html = await resp.text();
+        const $ = cheerio.load(html);
+
+        // remove noise
+        $('script, style, nav, footer, header, aside').remove();
+
+        const title = $('title').text() || item_id;
+
+        // get main content
+        const main = $('main').length ? $('main') : $('article').length ? $('article') : $('body');
+        let text = main.text();
+
+        // clean up whitespace
+        text = text.split('\n').map((l: string) => l.trim()).filter(Boolean).join('\n');
+
+        return {
+            id: item_id,
+            name: title.trim(),
+            type: 'webpage',
+            text,
+            data: text,
+            meta: { source: 'web_crawler', url: item_id, char_count: text.length }
+        };
+    }
+}

package/src/temporal_graph/index.ts

@@ -0,0 +1,4 @@
+export * from './types'
+export * from './store'
+export * from './query'
+export * from './timeline'

package/src/temporal_graph/query.ts

@@ -0,0 +1,299 @@
+
+
+import { get_async, all_async } from '../core/db'
+import { TemporalFact, TemporalQuery, TimelineEntry } from './types'
+
+
+export const query_facts_at_time = async (
+    subject?: string,
+    predicate?: string,
+    object?: string,
+    at: Date = new Date(),
+    min_confidence: number = 0.1
+): Promise<TemporalFact[]> => {
+    const timestamp = at.getTime()
+    const conditions: string[] = []
+    const params: any[] = []
+
+    // Build WHERE clause
+    conditions.push('(valid_from <= ? AND (valid_to IS NULL OR valid_to >= ?))')
+    params.push(timestamp, timestamp)
+
+    if (subject) {
+        conditions.push('subject = ?')
+        params.push(subject)
+    }
+
+    if (predicate) {
+        conditions.push('predicate = ?')
+        params.push(predicate)
+    }
+
+    if (object) {
+        conditions.push('object = ?')
+        params.push(object)
+    }
+
+    if (min_confidence > 0) {
+        conditions.push('confidence >= ?')
+        params.push(min_confidence)
+    }
+
+    const sql = `
+        SELECT id, subject, predicate, object, valid_from, valid_to, confidence, last_updated, metadata
+        FROM temporal_facts
+        WHERE ${conditions.join(' AND ')}
+        ORDER BY confidence DESC, valid_from DESC
+    `
+
+    const rows = await all_async(sql, params)
+    return rows.map(row => ({
+        id: row.id,
+        subject: row.subject,
+        predicate: row.predicate,
+        object: row.object,
+        valid_from: new Date(row.valid_from),
+        valid_to: row.valid_to ? new Date(row.valid_to) : null,
+        confidence: row.confidence,
+        last_updated: new Date(row.last_updated),
+        metadata: row.metadata ? JSON.parse(row.metadata) : undefined
+    }))
+}
+
+
+export const get_current_fact = async (
+    subject: string,
+    predicate: string
+): Promise<TemporalFact | null> => {
+    const now = Date.now()
+
+    const row = await get_async(`
+        SELECT id, subject, predicate, object, valid_from, valid_to, confidence, last_updated, metadata
+        FROM temporal_facts
+        WHERE subject = ? AND predicate = ? AND valid_to IS NULL
+        ORDER BY valid_from DESC
+        LIMIT 1
+    `, [subject, predicate])
+
+    if (!row) return null
+
+    return {
+        id: row.id,
+        subject: row.subject,
+        predicate: row.predicate,
+        object: row.object,
+        valid_from: new Date(row.valid_from),
+        valid_to: row.valid_to ? new Date(row.valid_to) : null,
+        confidence: row.confidence,
+        last_updated: new Date(row.last_updated),
+        metadata: row.metadata ? JSON.parse(row.metadata) : undefined
+    }
+}
+
+
+export const query_facts_in_range = async (
+    subject?: string,
+    predicate?: string,
+    from?: Date,
+    to?: Date,
+    min_confidence: number = 0.1
+): Promise<TemporalFact[]> => {
+    const conditions: string[] = []
+    const params: any[] = []
+
+    if (from && to) {
+        const from_ts = from.getTime()
+        const to_ts = to.getTime()
+        conditions.push('((valid_from <= ? AND (valid_to IS NULL OR valid_to >= ?)) OR (valid_from >= ? AND valid_from <= ?))')
+        params.push(to_ts, from_ts, from_ts, to_ts)
+    } else if (from) {
+        conditions.push('valid_from >= ?')
+        params.push(from.getTime())
+    } else if (to) {
+        conditions.push('valid_from <= ?')
+        params.push(to.getTime())
+    }
+
+    if (subject) {
+        conditions.push('subject = ?')
+        params.push(subject)
+    }
+
+    if (predicate) {
+        conditions.push('predicate = ?')
+        params.push(predicate)
+    }
+
+    if (min_confidence > 0) {
+        conditions.push('confidence >= ?')
+        params.push(min_confidence)
+    }
+
+    const where = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : ''
+    const sql = `
+        SELECT id, subject, predicate, object, valid_from, valid_to, confidence, last_updated, metadata
+        FROM temporal_facts
+        ${where}
+        ORDER BY valid_from DESC
+    `
+
+    const rows = await all_async(sql, params)
+    return rows.map(row => ({
+        id: row.id,
+        subject: row.subject,
+        predicate: row.predicate,
+        object: row.object,
+        valid_from: new Date(row.valid_from),
+        valid_to: row.valid_to ? new Date(row.valid_to) : null,
+        confidence: row.confidence,
+        last_updated: new Date(row.last_updated),
+        metadata: row.metadata ? JSON.parse(row.metadata) : undefined
+    }))
+}
+
+
+export const find_conflicting_facts = async (
+    subject: string,
+    predicate: string,
+    at?: Date
+): Promise<TemporalFact[]> => {
+    const timestamp = at ? at.getTime() : Date.now()
+
+    const rows = await all_async(`
+        SELECT id, subject, predicate, object, valid_from, valid_to, confidence, last_updated, metadata
+        FROM temporal_facts
+        WHERE subject = ? AND predicate = ?
+        AND (valid_from <= ? AND (valid_to IS NULL OR valid_to >= ?))
+        ORDER BY confidence DESC
+    `, [subject, predicate, timestamp, timestamp])
+
+    return rows.map(row => ({
+        id: row.id,
+        subject: row.subject,
+        predicate: row.predicate,
+        object: row.object,
+        valid_from: new Date(row.valid_from),
+        valid_to: row.valid_to ? new Date(row.valid_to) : null,
+        confidence: row.confidence,
+        last_updated: new Date(row.last_updated),
+        metadata: row.metadata ? JSON.parse(row.metadata) : undefined
+    }))
+}
+
+
+export const get_facts_by_subject = async (
+    subject: string,
+    at?: Date,
+    include_historical: boolean = false
+): Promise<TemporalFact[]> => {
+    let sql: string
+    let params: any[]
+
+    if (include_historical) {
+        sql = `
+            SELECT id, subject, predicate, object, valid_from, valid_to, confidence, last_updated, metadata
+            FROM temporal_facts
+            WHERE subject = ?
+            ORDER BY predicate ASC, valid_from DESC
+        `
+        params = [subject]
+    } else {
+        const timestamp = at ? at.getTime() : Date.now()
+        sql = `
+            SELECT id, subject, predicate, object, valid_from, valid_to, confidence, last_updated, metadata
+            FROM temporal_facts
+            WHERE subject = ?
+            AND (valid_from <= ? AND (valid_to IS NULL OR valid_to >= ?))
+            ORDER BY predicate ASC, confidence DESC
+        `
+        params = [subject, timestamp, timestamp]
+    }
+
+    const rows = await all_async(sql, params)
+    return rows.map(row => ({
+        id: row.id,
+        subject: row.subject,
+        predicate: row.predicate,
+        object: row.object,
+        valid_from: new Date(row.valid_from),
+        valid_to: row.valid_to ? new Date(row.valid_to) : null,
+        confidence: row.confidence,
+        last_updated: new Date(row.last_updated),
+        metadata: row.metadata ? JSON.parse(row.metadata) : undefined
+    }))
+}
+
+
+export const search_facts = async (
+    pattern: string,
+    field: 'subject' | 'predicate' | 'object' = 'subject',
+    at?: Date
+): Promise<TemporalFact[]> => {
+    const timestamp = at ? at.getTime() : Date.now()
+    const search_pattern = `%${pattern}%`
+
+    const sql = `
+        SELECT id, subject, predicate, object, valid_from, valid_to, confidence, last_updated, metadata
+        FROM temporal_facts
+        WHERE ${field} LIKE ?
+        AND (valid_from <= ? AND (valid_to IS NULL OR valid_to >= ?))
+        ORDER BY confidence DESC, valid_from DESC
+        LIMIT 100
+    `
+
+    const rows = await all_async(sql, [search_pattern, timestamp, timestamp])
+    return rows.map(row => ({
+        id: row.id,
+        subject: row.subject,
+        predicate: row.predicate,
+        object: row.object,
+        valid_from: new Date(row.valid_from),
+        valid_to: row.valid_to ? new Date(row.valid_to) : null,
+        confidence: row.confidence,
+        last_updated: new Date(row.last_updated),
+        metadata: row.metadata ? JSON.parse(row.metadata) : undefined
+    }))
+}
+
+
+export const get_related_facts = async (
+    fact_id: string,
+    relation_type?: string,
+    at?: Date
+): Promise<Array<{ fact: TemporalFact; relation: string; weight: number }>> => {
+    const timestamp = at ? at.getTime() : Date.now()
+    const conditions = ['(e.valid_from <= ? AND (e.valid_to IS NULL OR e.valid_to >= ?))']
+    const params: any[] = [timestamp, timestamp]
+
+    if (relation_type) {
+        conditions.push('e.relation_type = ?')
+        params.push(relation_type)
+    }
+
+    const sql = `
+        SELECT f.*, e.relation_type, e.weight
+        FROM temporal_edges e
+        JOIN temporal_facts f ON e.target_id = f.id
+        WHERE e.source_id = ?
+        AND ${conditions.join(' AND ')}
+        AND (f.valid_from <= ? AND (f.valid_to IS NULL OR f.valid_to >= ?))
+        ORDER BY e.weight DESC, f.confidence DESC
+    `
+
+    const rows = await all_async(sql, [fact_id, ...params, timestamp, timestamp])
+    return rows.map(row => ({
+        fact: {
+            id: row.id,
+            subject: row.subject,
+            predicate: row.predicate,
+            object: row.object,
+            valid_from: new Date(row.valid_from),
+            valid_to: row.valid_to ? new Date(row.valid_to) : null,
+            confidence: row.confidence,
+            last_updated: new Date(row.last_updated),
+            metadata: row.metadata ? JSON.parse(row.metadata) : undefined
+        },
+        relation: row.relation_type,
+        weight: row.weight
+    }))
+}

package/src/temporal_graph/store.ts

@@ -0,0 +1,156 @@
+import { run_async, get_async, all_async } from '../core/db'
+import { TemporalFact, TemporalEdge } from './types'
+import { randomUUID } from 'crypto'
+
+export const insert_fact = async (
+    subject: string,
+    predicate: string,
+    object: string,
+    valid_from: Date = new Date(),
+    confidence: number = 1.0,
+    metadata?: Record<string, any>
+): Promise<string> => {
+    const id = randomUUID()
+    const now = Date.now()
+    const valid_from_ts = valid_from.getTime()
+
+    const existing = await all_async(`
+        SELECT id, valid_from FROM temporal_facts 
+        WHERE subject = ? AND predicate = ? AND valid_to IS NULL
+        ORDER BY valid_from DESC
+    `, [subject, predicate])
+
+    for (const old of existing) {
+        if (old.valid_from < valid_from_ts) {
+            await run_async(`UPDATE temporal_facts SET valid_to = ? WHERE id = ?`, [valid_from_ts - 1, old.id])
+            console.error(`[TEMPORAL] Closed fact ${old.id} at ${new Date(valid_from_ts - 1).toISOString()}`) // Use stderr for MCP compatibility
+        }
+    }
+
+    await run_async(`
+        INSERT INTO temporal_facts (id, subject, predicate, object, valid_from, valid_to, confidence, last_updated, metadata)
+        VALUES (?, ?, ?, ?, ?, NULL, ?, ?, ?)
+    `, [id, subject, predicate, object, valid_from_ts, confidence, now, metadata ? JSON.stringify(metadata) : null])
+
+    console.error(`[TEMPORAL] Inserted fact: ${subject} ${predicate} ${object} (from ${valid_from.toISOString()}, confidence=${confidence})`) // Use stderr for MCP compatibility
+    return id
+}
+
+export const update_fact = async (id: string, confidence?: number, metadata?: Record<string, any>): Promise<void> => {
+    const updates: string[] = []
+    const params: any[] = []
+
+    if (confidence !== undefined) {
+        updates.push('confidence = ?')
+        params.push(confidence)
+    }
+
+    if (metadata !== undefined) {
+        updates.push('metadata = ?')
+        params.push(JSON.stringify(metadata))
+    }
+
+    updates.push('last_updated = ?')
+    params.push(Date.now())
+
+    params.push(id)
+
+    if (updates.length > 0) {
+        await run_async(`UPDATE temporal_facts SET ${updates.join(', ')} WHERE id = ?`, params)
+        console.error(`[TEMPORAL] Updated fact ${id}`) // Use stderr for MCP compatibility
+    }
+}
+
+export const invalidate_fact = async (id: string, valid_to: Date = new Date()): Promise<void> => {
+    await run_async(`UPDATE temporal_facts SET valid_to = ?, last_updated = ? WHERE id = ?`, [valid_to.getTime(), Date.now(), id])
+    console.error(`[TEMPORAL] Invalidated fact ${id} at ${valid_to.toISOString()}`) // Use stderr for MCP compatibility
+}
+
+export const delete_fact = async (id: string): Promise<void> => {
+    await run_async(`DELETE FROM temporal_facts WHERE id = ?`, [id])
+    console.error(`[TEMPORAL] Deleted fact ${id}`) // Use stderr for MCP compatibility
+}
+
+export const insert_edge = async (
+    source_id: string,
+    target_id: string,
+    relation_type: string,
+    valid_from: Date = new Date(),
+    weight: number = 1.0,
+    metadata?: Record<string, any>
+): Promise<string> => {
+    const id = randomUUID()
+    const valid_from_ts = valid_from.getTime()
+
+    await run_async(`
+        INSERT INTO temporal_edges (id, source_id, target_id, relation_type, valid_from, valid_to, weight, metadata)
+        VALUES (?, ?, ?, ?, ?, NULL, ?, ?)
+    `, [id, source_id, target_id, relation_type, valid_from_ts, weight, metadata ? JSON.stringify(metadata) : null])
+
+    console.log(`[TEMPORAL] Created edge: ${source_id} --[${relation_type}]--> ${target_id}`)
+    return id
+}
+
+export const invalidate_edge = async (id: string, valid_to: Date = new Date()): Promise<void> => {
+    await run_async(`UPDATE temporal_edges SET valid_to = ? WHERE id = ?`, [valid_to.getTime(), id])
+    console.log(`[TEMPORAL] Invalidated edge ${id}`)
+}
+
+export const batch_insert_facts = async (facts: Array<{
+    subject: string
+    predicate: string
+    object: string
+    valid_from?: Date
+    confidence?: number
+    metadata?: Record<string, any>
+}>): Promise<string[]> => {
+    const ids: string[] = []
+
+    await run_async('BEGIN TRANSACTION')
+    try {
+        for (const fact of facts) {
+            const id = await insert_fact(
+                fact.subject,
+                fact.predicate,
+                fact.object,
+                fact.valid_from,
+                fact.confidence,
+                fact.metadata
+            )
+            ids.push(id)
+        }
+        await run_async('COMMIT')
+        console.log(`[TEMPORAL] Batch inserted ${ids.length} facts`)
+    } catch (error) {
+        await run_async('ROLLBACK')
+        throw error
+    }
+
+    return ids
+}
+
+export const apply_confidence_decay = async (decay_rate: number = 0.01): Promise<number> => {
+    const now = Date.now()
+    const one_day = 86400000
+
+    await run_async(`
+        UPDATE temporal_facts 
+        SET confidence = MAX(0.1, confidence * (1 - ? * ((? - valid_from) / ?)))
+        WHERE valid_to IS NULL AND confidence > 0.1
+    `, [decay_rate, now, one_day])
+
+    const result = await get_async(`SELECT changes() as changes`) as any
+    const changes = result?.changes || 0
+    console.log(`[TEMPORAL] Applied confidence decay to ${changes} facts`)
+    return changes
+}
+
+export const get_active_facts_count = async (): Promise<number> => {
+    const result = await get_async(`SELECT COUNT(*) as count FROM temporal_facts WHERE valid_to IS NULL`) as any
+    return result?.count || 0
+}
+
+export const get_total_facts_count = async (): Promise<number> => {
+    const result = await get_async(`SELECT COUNT(*) as count FROM temporal_facts`) as any
+    return result?.count || 0
+}