ahok-skill 1.3.1

package/src/core/memory.ts

@@ -0,0 +1,99 @@
+
+import { add_hsg_memory, hsg_query } from "../memory/hsg";
+import { q, log_maint_op } from "./db";
+import { env } from "./cfg";
+import { j } from "../utils";
+
+export interface MemoryOptions {
+    user_id?: string;
+    tags?: string[];
+    [key: string]: any;
+}
+
+export class Memory {
+    default_user: string | null;
+
+    constructor(user_id?: string) {
+        this.default_user = user_id || null;
+    }
+
+    async add(content: string, opts?: MemoryOptions) {
+        const uid = opts?.user_id || this.default_user;
+        const tags = opts?.tags || [];
+        const meta = { ...opts };
+        delete meta.user_id;
+        delete meta.tags;
+
+        // Ensure tags is JSON string if needed by add_hsg_memory
+        // hsg.ts signature: add_hsg_memory(content, tags, meta, user_id)
+        // tags is usually stringified JSON or string? 
+        // Checked hsg.ts: interface hsg_mem { tags?: string }
+        // Let's pass JSON string for tags.
+        const tags_str = JSON.stringify(tags);
+
+        // hsg.ts add_hsg_memory returns { id, ... } or similar?
+        // Let's check hsg.ts exports. It's likely async and returns object.
+        const res = await add_hsg_memory(content, tags_str, meta, uid ?? undefined);
+        return res;
+    }
+
+    async get(id: string) {
+        return await q.get_mem.get(id);
+    }
+
+    async search(query: string, opts?: { user_id?: string, limit?: number, sectors?: string[] }) {
+        // hsg_query(qt, k, f)
+        const k = opts?.limit || 10;
+        const uid = opts?.user_id || this.default_user;
+        const f: any = {};
+        if (uid) f.user_id = uid;
+        if (opts?.sectors) f.sectors = opts.sectors;
+
+        return await hsg_query(query, k, f);
+    }
+
+    async delete_all(user_id?: string) {
+        const uid = user_id || this.default_user;
+        if (uid) {
+            // q.del_mem usually exists or we execute raw SQL
+            // But we can't easily access q here if not exported or if we want to change memory.ts minimal
+            // I'll add a wipe() method that calls q directly if q is imported
+        }
+    }
+
+    async wipe() {
+        console.log("[Memory] Wiping DB...");
+        // q is imported from db.ts
+        await q.clear_all.run();
+    }
+
+    /**
+     * get a pre-configured source connector.
+     * 
+     * usage:
+     *   const github = mem.source("github")
+     *   await github.connect({ token: "ghp_..." })
+     *   await github.ingest_all({ repo: "owner/repo" })
+     * 
+     * available sources: github, notion, google_drive, google_sheets, 
+     *                   google_slides, onedrive, web_crawler
+     */
+    source(name: string) {
+        // dynamic import to avoid circular deps
+        const sources: Record<string, any> = {
+            github: () => import("../sources/github").then(m => new m.github_source(this.default_user ?? undefined)),
+            notion: () => import("../sources/notion").then(m => new m.notion_source(this.default_user ?? undefined)),
+            google_drive: () => import("../sources/google_drive").then(m => new m.google_drive_source(this.default_user ?? undefined)),
+            google_sheets: () => import("../sources/google_sheets").then(m => new m.google_sheets_source(this.default_user ?? undefined)),
+            google_slides: () => import("../sources/google_slides").then(m => new m.google_slides_source(this.default_user ?? undefined)),
+            onedrive: () => import("../sources/onedrive").then(m => new m.onedrive_source(this.default_user ?? undefined)),
+            web_crawler: () => import("../sources/web_crawler").then(m => new m.web_crawler_source(this.default_user ?? undefined)),
+        };
+
+        if (!(name in sources)) {
+            throw new Error(`unknown source: ${name}. available: ${Object.keys(sources).join(", ")}`);
+        }
+
+        return sources[name]();
+    }
+}

package/src/core/migrate.ts

@@ -0,0 +1,302 @@
+import { env } from "./cfg";
+import sqlite3 from "sqlite3";
+import { Pool } from "pg";
+
+const is_pg = env.metadata_backend === "postgres";
+
+const log = (msg: string) => console.log(`[MIGRATE] ${msg}`);
+
+interface Migration {
+    version: string;
+    desc: string;
+    sqlite: string[];
+    postgres: string[];
+}
+
+const migrations: Migration[] = [
+    {
+        version: "1.2.0",
+        desc: "Multi-user tenant support",
+        sqlite: [
+            `ALTER TABLE memories ADD COLUMN user_id TEXT`,
+            `CREATE INDEX IF NOT EXISTS idx_memories_user ON memories(user_id)`,
+            `ALTER TABLE vectors ADD COLUMN user_id TEXT`,
+            `CREATE INDEX IF NOT EXISTS idx_vectors_user ON vectors(user_id)`,
+            `CREATE TABLE IF NOT EXISTS waypoints_new (
+        src_id TEXT, dst_id TEXT NOT NULL, user_id TEXT,
+        weight REAL NOT NULL, created_at INTEGER, updated_at INTEGER,
+        PRIMARY KEY(src_id, user_id)
+      )`,
+            `INSERT INTO waypoints_new SELECT src_id, dst_id, NULL, weight, created_at, updated_at FROM waypoints`,
+            `DROP TABLE waypoints`,
+            `ALTER TABLE waypoints_new RENAME TO waypoints`,
+            `CREATE INDEX IF NOT EXISTS idx_waypoints_src ON waypoints(src_id)`,
+            `CREATE INDEX IF NOT EXISTS idx_waypoints_dst ON waypoints(dst_id)`,
+            `CREATE INDEX IF NOT EXISTS idx_waypoints_user ON waypoints(user_id)`,
+            `CREATE TABLE IF NOT EXISTS users (
+        user_id TEXT PRIMARY KEY, summary TEXT,
+        reflection_count INTEGER DEFAULT 0,
+        created_at INTEGER, updated_at INTEGER
+      )`,
+            `CREATE TABLE IF NOT EXISTS stats (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        type TEXT NOT NULL, count INTEGER DEFAULT 1, ts INTEGER NOT NULL
+      )`,
+            `CREATE INDEX IF NOT EXISTS idx_stats_ts ON stats(ts)`,
+            `CREATE INDEX IF NOT EXISTS idx_stats_type ON stats(type)`,
+        ],
+        postgres: [
+            `ALTER TABLE {m} ADD COLUMN IF NOT EXISTS user_id TEXT`,
+            `CREATE INDEX IF NOT EXISTS openmemory_memories_user_idx ON {m}(user_id)`,
+            `ALTER TABLE {v} ADD COLUMN IF NOT EXISTS user_id TEXT`,
+            `CREATE INDEX IF NOT EXISTS openmemory_vectors_user_idx ON {v}(user_id)`,
+            `ALTER TABLE {w} ADD COLUMN IF NOT EXISTS user_id TEXT`,
+            `ALTER TABLE {w} DROP CONSTRAINT IF EXISTS waypoints_pkey`,
+            `ALTER TABLE {w} ADD PRIMARY KEY (src_id, user_id)`,
+            `CREATE INDEX IF NOT EXISTS openmemory_waypoints_user_idx ON {w}(user_id)`,
+            `CREATE TABLE IF NOT EXISTS {u} (
+        user_id TEXT PRIMARY KEY, summary TEXT,
+        reflection_count INTEGER DEFAULT 0,
+        created_at BIGINT, updated_at BIGINT
+      )`,
+        ],
+    },
+];
+
+async function get_db_version_sqlite(
+    db: sqlite3.Database,
+): Promise<string | null> {
+    return new Promise((ok, no) => {
+        db.get(
+            `SELECT name FROM sqlite_master WHERE type='table' AND name='schema_version'`,
+            (err, row: any) => {
+                if (err) return no(err);
+                if (!row) return ok(null);
+                db.get(
+                    `SELECT version FROM schema_version ORDER BY applied_at DESC LIMIT 1`,
+                    (e, v: any) => {
+                        if (e) return no(e);
+                        ok(v?.version || null);
+                    },
+                );
+            },
+        );
+    });
+}
+
+async function set_db_version_sqlite(
+    db: sqlite3.Database,
+    version: string,
+): Promise<void> {
+    return new Promise((ok, no) => {
+        db.run(
+            `CREATE TABLE IF NOT EXISTS schema_version (
+        version TEXT PRIMARY KEY, applied_at INTEGER
+      )`,
+            (err) => {
+                if (err) return no(err);
+                db.run(
+                    `INSERT OR REPLACE INTO schema_version VALUES (?, ?)`,
+                    [version, Date.now()],
+                    (e) => {
+                        if (e) return no(e);
+                        ok();
+                    },
+                );
+            },
+        );
+    });
+}
+
+async function check_column_exists_sqlite(
+    db: sqlite3.Database,
+    table: string,
+    column: string,
+): Promise<boolean> {
+    return new Promise((ok, no) => {
+        db.all(`PRAGMA table_info(${table})`, (err, rows: any[]) => {
+            if (err) return no(err);
+            ok(rows.some((r) => r.name === column));
+        });
+    });
+}
+
+async function run_sqlite_migration(
+    db: sqlite3.Database,
+    m: Migration,
+): Promise<void> {
+    log(`Running migration: ${m.version} - ${m.desc}`);
+
+    const has_user_id = await check_column_exists_sqlite(
+        db,
+        "memories",
+        "user_id",
+    );
+    if (has_user_id) {
+        log(
+            `Migration ${m.version} already applied (user_id exists), skipping`,
+        );
+        await set_db_version_sqlite(db, m.version);
+        return;
+    }
+
+    for (const sql of m.sqlite) {
+        await new Promise<void>((ok, no) => {
+            db.run(sql, (err) => {
+                if (err && !err.message.includes("duplicate column")) {
+                    log(`ERROR: ${err.message}`);
+                    return no(err);
+                }
+                ok();
+            });
+        });
+    }
+
+    await set_db_version_sqlite(db, m.version);
+    log(`Migration ${m.version} completed successfully`);
+}
+
+async function get_db_version_pg(pool: Pool): Promise<string | null> {
+    try {
+        const sc = process.env.OM_PG_SCHEMA || "public";
+        const check = await pool.query(
+            `SELECT EXISTS (
+        SELECT FROM information_schema.tables 
+        WHERE table_schema = $1 AND table_name = 'schema_version'
+      )`,
+            [sc],
+        );
+        if (!check.rows[0].exists) return null;
+
+        const ver = await pool.query(
+            `SELECT version FROM "${sc}"."schema_version" ORDER BY applied_at DESC LIMIT 1`,
+        );
+        return ver.rows[0]?.version || null;
+    } catch (e) {
+        return null;
+    }
+}
+
+async function set_db_version_pg(pool: Pool, version: string): Promise<void> {
+    const sc = process.env.OM_PG_SCHEMA || "public";
+    await pool.query(
+        `CREATE TABLE IF NOT EXISTS "${sc}"."schema_version" (
+      version TEXT PRIMARY KEY, applied_at BIGINT
+    )`,
+    );
+    await pool.query(
+        `INSERT INTO "${sc}"."schema_version" VALUES ($1, $2) 
+     ON CONFLICT (version) DO UPDATE SET applied_at = EXCLUDED.applied_at`,
+        [version, Date.now()],
+    );
+}
+
+async function check_column_exists_pg(
+    pool: Pool,
+    table: string,
+    column: string,
+): Promise<boolean> {
+    const sc = process.env.OM_PG_SCHEMA || "public";
+    const tbl = table.replace(/"/g, "").split(".").pop() || table;
+    const res = await pool.query(
+        `SELECT EXISTS (
+      SELECT FROM information_schema.columns 
+      WHERE table_schema = $1 AND table_name = $2 AND column_name = $3
+    )`,
+        [sc, tbl, column],
+    );
+    return res.rows[0].exists;
+}
+
+async function run_pg_migration(pool: Pool, m: Migration): Promise<void> {
+    log(`Running migration: ${m.version} - ${m.desc}`);
+
+    const sc = process.env.OM_PG_SCHEMA || "public";
+    const mt = process.env.OM_PG_TABLE || "openmemory_memories";
+    const has_user_id = await check_column_exists_pg(pool, mt, "user_id");
+
+    if (has_user_id) {
+        log(
+            `Migration ${m.version} already applied (user_id exists), skipping`,
+        );
+        await set_db_version_pg(pool, m.version);
+        return;
+    }
+
+    const replacements: Record<string, string> = {
+        "{m}": `"${sc}"."${mt}"`,
+        "{v}": `"${sc}"."${process.env.OM_VECTOR_TABLE || "openmemory_vectors"}"`,
+        "{w}": `"${sc}"."openmemory_waypoints"`,
+        "{u}": `"${sc}"."openmemory_users"`,
+    };
+
+    for (let sql of m.postgres) {
+        for (const [k, v] of Object.entries(replacements)) {
+            sql = sql.replace(new RegExp(k, "g"), v);
+        }
+
+        try {
+            await pool.query(sql);
+        } catch (e: any) {
+            if (
+                !e.message.includes("already exists") &&
+                !e.message.includes("duplicate")
+            ) {
+                log(`ERROR: ${e.message}`);
+                throw e;
+            }
+        }
+    }
+
+    await set_db_version_pg(pool, m.version);
+    log(`Migration ${m.version} completed successfully`);
+}
+
+export async function run_migrations() {
+    log("Checking for pending migrations...");
+
+    if (is_pg) {
+        const ssl =
+            process.env.OM_PG_SSL === "require"
+                ? { rejectUnauthorized: false }
+                : process.env.OM_PG_SSL === "disable"
+                  ? false
+                  : undefined;
+
+        const pool = new Pool({
+            host: process.env.OM_PG_HOST,
+            port: process.env.OM_PG_PORT ? +process.env.OM_PG_PORT : undefined,
+            database: process.env.OM_PG_DB || "openmemory",
+            user: process.env.OM_PG_USER,
+            password: process.env.OM_PG_PASSWORD,
+            ssl,
+        });
+
+        const current = await get_db_version_pg(pool);
+        log(`Current database version: ${current || "none"}`);
+
+        for (const m of migrations) {
+            if (!current || m.version > current) {
+                await run_pg_migration(pool, m);
+            }
+        }
+
+        await pool.end();
+    } else {
+        const db_path = process.env.OM_DB_PATH || "./data/openmemory.sqlite";
+        const db = new sqlite3.Database(db_path);
+
+        const current = await get_db_version_sqlite(db);
+        log(`Current database version: ${current || "none"}`);
+
+        for (const m of migrations) {
+            if (!current || m.version > current) {
+                await run_sqlite_migration(db, m);
+            }
+        }
+
+        await new Promise<void>((ok) => db.close(() => ok()));
+    }
+
+    log("All migrations completed");
+}

package/src/core/models.ts

@@ -0,0 +1,107 @@
+import { readFileSync, existsSync } from "fs";
+import { join } from "path";
+interface model_cfg {
+    [sector: string]: Record<string, string>;
+}
+let cfg: model_cfg | null = null;
+
+export const load_models = (): model_cfg => {
+    if (cfg) return cfg;
+    const p = join(__dirname, "../../../models.yml");
+    if (!existsSync(p)) {
+        console.error("[MODELS] models.yml not found, using defaults");
+        return get_defaults();
+    }
+    try {
+        const yml = readFileSync(p, "utf-8");
+        cfg = parse_yaml(yml);
+        console.error(
+            `[MODELS] Loaded models.yml (${Object.keys(cfg).length} sectors)`,
+        );
+        return cfg;
+    } catch (e) {
+        console.error("[MODELS] Failed to parse models.yml:", e);
+        return get_defaults();
+    }
+};
+
+const parse_yaml = (yml: string): model_cfg => {
+    const lines = yml.split("\n");
+    const obj: model_cfg = {};
+    let cur_sec: string | null = null;
+    for (const line of lines) {
+        const trim = line.trim();
+        if (!trim || trim.startsWith("#")) continue;
+        const indent = line.search(/\S/);
+        const [key, ...val_parts] = trim.split(":");
+        const val = val_parts.join(":").trim();
+        if (indent === 0 && val) {
+            continue;
+        } else if (indent === 0) {
+            cur_sec = key;
+            obj[cur_sec] = {};
+        } else if (cur_sec && val) {
+            obj[cur_sec][key] = val;
+        }
+    }
+    return obj;
+};
+
+const get_defaults = (): model_cfg => ({
+    episodic: {
+        ollama: "nomic-embed-text",
+        openai: "text-embedding-3-small",
+        gemini: "models/embedding-001",
+        aws: "amazon.titan-embed-text-v2:0",
+        local: "all-MiniLM-L6-v2",
+    },
+    semantic: {
+        ollama: "nomic-embed-text",
+        openai: "text-embedding-3-small",
+        gemini: "models/embedding-001",
+        aws: "amazon.titan-embed-text-v2:0",
+        local: "all-MiniLM-L6-v2",
+    },
+    procedural: {
+        ollama: "nomic-embed-text",
+        openai: "text-embedding-3-small",
+        gemini: "models/embedding-001",
+        aws: "amazon.titan-embed-text-v2:0",
+        local: "all-MiniLM-L6-v2",
+    },
+    emotional: {
+        ollama: "nomic-embed-text",
+        openai: "text-embedding-3-small",
+        gemini: "models/embedding-001",
+        aws: "amazon.titan-embed-text-v2:0",
+        local: "all-MiniLM-L6-v2",
+    },
+    reflective: {
+        ollama: "nomic-embed-text",
+        openai: "text-embedding-3-large",
+        gemini: "models/embedding-001",
+        aws: "amazon.titan-embed-text-v2:0",
+        local: "all-mpnet-base-v2",
+    },
+});
+
+export const get_model = (sector: string, provider: string): string => {
+    // Environment variable overrides
+    if (provider === "ollama" && process.env.OM_OLLAMA_MODEL) {
+        return process.env.OM_OLLAMA_MODEL;
+    }
+    if (provider === "openai" && process.env.OM_OPENAI_MODEL) {
+        return process.env.OM_OPENAI_MODEL;
+    }
+
+    const cfg = load_models();
+    return (
+        cfg[sector]?.[provider] ||
+        cfg.semantic?.[provider] ||
+        "nomic-embed-text"
+    );
+};
+
+export const get_provider_config = (provider: string): any => {
+    return {};
+};

package/src/core/telemetry.ts

@@ -0,0 +1,47 @@
+import os from 'node:os'
+import { env } from './cfg'
+
+const DISABLED = (process.env.OM_TELEMETRY ?? '').toLowerCase() === 'false'
+const gatherVersion = (): string => {
+    if (process.env.npm_package_version) return process.env.npm_package_version
+    try {
+        // eslint-disable-next-line @typescript-eslint/no-var-requires
+        const pkg = require('../../package.json')
+        if (pkg?.version) return pkg.version
+    } catch {
+        // ignore
+    }
+    return 'unknown'
+}
+
+export const sendTelemetry = async () => {
+    if (DISABLED) return
+    try {
+        const ramMb = Math.round(os.totalmem() / (1024 * 1024))
+        const storageMb = ramMb * 4
+        const payload = {
+            name: os.hostname(),
+            os: os.platform(),
+            embeddings: env.emb_kind || 'synthetic',
+            metadata: env.metadata_backend || 'sqlite',
+            version: gatherVersion(),
+            ram: ramMb,
+            storage: storageMb,
+            cpu: os.cpus()?.[0]?.model || 'unknown',
+        }
+        const res = await fetch('https://telemetry.spotit.dev', {
+            method: 'POST',
+            headers: { 'content-type': 'application/json' },
+            body: JSON.stringify(payload),
+            keepalive: true,
+        })
+        if (!res.ok) {
+            console.warn(``)
+        } else {
+            console.log(`[telemetry] sent`)
+        }
+    } catch {
+        // silently ignore telemetry errors
+    }
+}
+

package/src/core/types.ts

@@ -0,0 +1,130 @@
+export type add_req = {
+    content: string;
+    tags?: string[];
+    metadata?: Record<string, unknown>;
+    salience?: number;
+    decay_lambda?: number;
+    user_id?: string;
+};
+export type q_req = {
+    query: string;
+    k?: number;
+    filters?: {
+        tags?: string[];
+        min_score?: number;
+        sector?: string;
+        user_id?: string;
+        startTime?: number;
+        endTime?: number;
+    };
+    user_id?: string;
+};
+export type sector_type =
+    | "episodic"
+    | "semantic"
+    | "procedural"
+    | "emotional"
+    | "reflective";
+
+export type mem_row = {
+    id: string;
+    content: string;
+    primary_sector: string;
+    tags: string | null;
+    meta: string | null;
+    user_id: string | null;
+    created_at: number;
+    updated_at: number;
+    last_seen_at: number;
+    salience: number;
+    decay_lambda: number;
+    version: number;
+};
+
+export type rpc_err_code = -32600 | -32603;
+
+export type ingest_req = {
+    source: "file" | "link" | "connector";
+    content_type: "pdf" | "docx" | "html" | "md" | "txt" | "audio";
+    data: string;
+    metadata?: Record<string, unknown>;
+    config?: { force_root?: boolean; sec_sz?: number; lg_thresh?: number };
+    user_id?: string;
+};
+
+export type ingest_url_req = {
+    url: string;
+    metadata?: Record<string, unknown>;
+    config?: { force_root?: boolean; sec_sz?: number; lg_thresh?: number };
+    user_id?: string;
+};
+
+export type lgm_store_req = {
+    node: string;
+    content: string;
+    tags?: string[];
+    metadata?: Record<string, unknown>;
+    namespace?: string;
+    graph_id?: string;
+    reflective?: boolean;
+    user_id?: string;
+};
+
+export type lgm_retrieve_req = {
+    node: string;
+    query?: string;
+    namespace?: string;
+    graph_id?: string;
+    limit?: number;
+    include_metadata?: boolean;
+};
+
+export type lgm_context_req = {
+    graph_id?: string;
+    namespace?: string;
+    limit?: number;
+};
+
+export type lgm_reflection_req = {
+    node?: string;
+    graph_id?: string;
+    namespace?: string;
+    content?: string;
+    context_ids?: string[];
+};
+
+export type ide_event_req = {
+    event:
+    | "edit"
+    | "open"
+    | "close"
+    | "save"
+    | "refactor"
+    | "comment"
+    | "pattern_detected"
+    | "api_call"
+    | "definition"
+    | "reflection";
+    file?: string;
+    snippet?: string;
+    comment?: string;
+    metadata: {
+        project?: string;
+        lang?: string;
+        user?: string;
+        timestamp?: number;
+        [key: string]: unknown;
+    };
+    session_id?: string;
+};
+
+export type ide_context_query_req = {
+    query: string;
+    k?: number;
+    session_id?: string;
+    file_filter?: string;
+    include_patterns?: boolean;
+    include_knowledge?: boolean;
+};
+
+export type ide_session_req = { user?: string; project?: string; ide?: string };