ahok-skill 1.3.1

package/src/ops/ingest.ts

@@ -0,0 +1,286 @@
+import { add_hsg_memory } from "../memory/hsg";
+import { q, transaction } from "../core/db";
+import { rid, now, j } from "../utils";
+import { extractText, ExtractionResult } from "./extract";
+
+const LG = 8000,
+    SEC = 3000;
+
+export interface ingestion_cfg {
+    force_root?: boolean;
+    sec_sz?: number;
+    lg_thresh?: number;
+}
+export interface IngestionResult {
+    root_memory_id: string;
+    child_count: number;
+    total_tokens: number;
+    strategy: "single" | "root-child";
+    extraction: ExtractionResult["metadata"];
+}
+
+const split = (t: string, sz: number): string[] => {
+    if (t.length <= sz) return [t];
+    const secs: string[] = [];
+    const paras = t.split(/\n\n+/);
+    let cur = "";
+    for (const p of paras) {
+        if (cur.length + p.length > sz && cur.length > 0) {
+            secs.push(cur.trim());
+            cur = p;
+        } else cur += (cur ? "\n\n" : "") + p;
+    }
+    if (cur.trim()) secs.push(cur.trim());
+    return secs;
+};
+
+const mkRoot = async (
+    txt: string,
+    ex: ExtractionResult,
+    meta?: Record<string, unknown>,
+    user_id?: string | null,
+) => {
+    const sum = txt.length > 500 ? txt.slice(0, 500) + "..." : txt;
+    const cnt = `[Document: ${ex.metadata.content_type.toUpperCase()}]\n\n${sum}\n\n[Full content split across ${Math.ceil(txt.length / SEC)} sections]`;
+    const id = rid(),
+        ts = now();
+    await transaction.begin();
+    try {
+        await q.ins_mem.run(
+            id,
+            cnt,
+            "reflective",
+            j([]),
+            j({
+                ...meta,
+                ...ex.metadata,
+                is_root: true,
+                ingestion_strategy: "root-child",
+                ingested_at: ts,
+            }),
+            ts,
+            ts,
+            ts,
+            1.0,
+            0.1,
+            1,
+            user_id || "anonymous",
+            null,
+        );
+        await transaction.commit();
+        return id;
+    } catch (e) {
+        console.error("[ERROR] Root failed:", e);
+        await transaction.rollback();
+        throw e;
+    }
+};
+
+const mkChild = async (
+    txt: string,
+    idx: number,
+    tot: number,
+    rid: string,
+    meta?: Record<string, unknown>,
+    user_id?: string | null,
+) => {
+    const r = await add_hsg_memory(
+        txt,
+        j([]),
+        {
+            ...meta,
+            is_child: true,
+            section_index: idx,
+            total_sections: tot,
+            parent_id: rid,
+        },
+        user_id || undefined,
+    );
+    return r.id;
+};
+
+const link = async (
+    rid: string,
+    cid: string,
+    idx: number,
+    user_id?: string | null,
+) => {
+    const ts = now();
+    await transaction.begin();
+    try {
+        await q.ins_waypoint.run(rid, cid, user_id || "anonymous", 1.0, ts, ts);
+        await transaction.commit();
+        console.log(
+            `[INGEST] Linked: ${rid.slice(0, 8)} -> ${cid.slice(0, 8)} (section ${idx})`,
+        );
+    } catch (e) {
+        await transaction.rollback();
+        console.error(`[INGEST] Link failed for section ${idx}:`, e);
+        throw e;
+    }
+};
+
+export async function ingestDocument(
+    t: string,
+    data: string | Buffer,
+    meta?: Record<string, unknown>,
+    cfg?: ingestion_cfg,
+    user_id?: string | null,
+): Promise<IngestionResult> {
+    const th = cfg?.lg_thresh || LG,
+        sz = cfg?.sec_sz || SEC;
+    const ex = await extractText(t, data);
+    const { text, metadata: exMeta } = ex;
+    const useRC = cfg?.force_root || exMeta.estimated_tokens > th;
+
+    if (!useRC) {
+        const r = await add_hsg_memory(
+            text,
+            j([]),
+            {
+                ...meta,
+                ...exMeta,
+                ingestion_strategy: "single",
+                ingested_at: now(),
+            },
+            user_id || undefined,
+        );
+        return {
+            root_memory_id: r.id,
+            child_count: 0,
+            total_tokens: exMeta.estimated_tokens,
+            strategy: "single",
+            extraction: exMeta,
+        };
+    }
+
+    const secs = split(text, sz);
+    console.log(`[INGEST] Document: ${exMeta.estimated_tokens} tokens`);
+    console.log(`[INGEST] Splitting into ${secs.length} sections`);
+
+    let rid: string;
+    const cids: string[] = [];
+
+    try {
+        rid = await mkRoot(text, ex, meta, user_id);
+        console.log(`[INGEST] Root memory created: ${rid}`);
+        for (let i = 0; i < secs.length; i++) {
+            try {
+                const cid = await mkChild(
+                    secs[i],
+                    i,
+                    secs.length,
+                    rid,
+                    meta,
+                    user_id,
+                );
+                cids.push(cid);
+                await link(rid, cid, i, user_id);
+                console.log(
+                    `[INGEST] Section ${i + 1}/${secs.length} processed: ${cid}`,
+                );
+            } catch (e) {
+                console.error(
+                    `[INGEST] Section ${i + 1}/${secs.length} failed:`,
+                    e,
+                );
+                throw e;
+            }
+        }
+        console.log(
+            `[INGEST] Completed: ${cids.length} sections linked to ${rid}`,
+        );
+        return {
+            root_memory_id: rid,
+            child_count: secs.length,
+            total_tokens: exMeta.estimated_tokens,
+            strategy: "root-child",
+            extraction: exMeta,
+        };
+    } catch (e) {
+        console.error("[INGEST] Document ingestion failed:", e);
+        throw e;
+    }
+}
+
+export async function ingestURL(
+    url: string,
+    meta?: Record<string, unknown>,
+    cfg?: ingestion_cfg,
+    user_id?: string | null,
+): Promise<IngestionResult> {
+    const { extractURL } = await import("./extract");
+    const ex = await extractURL(url);
+    const th = cfg?.lg_thresh || LG,
+        sz = cfg?.sec_sz || SEC;
+    const useRC = cfg?.force_root || ex.metadata.estimated_tokens > th;
+
+    if (!useRC) {
+        const r = await add_hsg_memory(
+            ex.text,
+            j([]),
+            {
+                ...meta,
+                ...ex.metadata,
+                ingestion_strategy: "single",
+                ingested_at: now(),
+            },
+            user_id || undefined,
+        );
+        return {
+            root_memory_id: r.id,
+            child_count: 0,
+            total_tokens: ex.metadata.estimated_tokens,
+            strategy: "single",
+            extraction: ex.metadata,
+        };
+    }
+
+    const secs = split(ex.text, sz);
+    console.log(`[INGEST] URL: ${ex.metadata.estimated_tokens} tokens`);
+    console.log(`[INGEST] Splitting into ${secs.length} sections`);
+
+    let rid: string;
+    const cids: string[] = [];
+
+    try {
+        rid = await mkRoot(ex.text, ex, { ...meta, source_url: url }, user_id);
+        console.log(`[INGEST] Root memory for URL: ${rid}`);
+        for (let i = 0; i < secs.length; i++) {
+            try {
+                const cid = await mkChild(
+                    secs[i],
+                    i,
+                    secs.length,
+                    rid,
+                    { ...meta, source_url: url },
+                    user_id,
+                );
+                cids.push(cid);
+                await link(rid, cid, i, user_id);
+                console.log(
+                    `[INGEST] URL section ${i + 1}/${secs.length} processed: ${cid}`,
+                );
+            } catch (e) {
+                console.error(
+                    `[INGEST] URL section ${i + 1}/${secs.length} failed:`,
+                    e,
+                );
+                throw e;
+            }
+        }
+        console.log(
+            `[INGEST] URL completed: ${cids.length} sections linked to ${rid}`,
+        );
+        return {
+            root_memory_id: rid,
+            child_count: secs.length,
+            total_tokens: ex.metadata.estimated_tokens,
+            strategy: "root-child",
+            extraction: ex.metadata,
+        };
+    } catch (e) {
+        console.error("[INGEST] URL ingestion failed:", e);
+        throw e;
+    }
+}

package/src/server/index.ts

@@ -0,0 +1,159 @@
+const server = require("./server.js");
+import { env, tier } from "../core/cfg";
+import { run_decay_process, prune_weak_waypoints } from "../memory/hsg";
+import { mcp } from "../ai/mcp";
+import { routes } from "./routes";
+import {
+    authenticate_api_request,
+    log_authenticated_request,
+} from "./middleware/auth";
+import { start_reflection } from "../memory/reflect";
+import { start_user_summary_reflection } from "../memory/user_summary";
+import { sendTelemetry } from "../core/telemetry";
+import { req_tracker_mw } from "./routes/dashboard";
+
+const ASC = `   ____                   __  __                                 
+  / __ \\                 |  \\/  |                                
+ | |  | |_ __   ___ _ __ | \\  / | ___ _ __ ___   ___  _ __ _   _ 
+ | |  | | '_ \\ / _ \\ '_ \\| |\\/| |/ _ \\ '_ \` _ \\ / _ \\| '__| | | |
+ | |__| | |_) |  __/ | | | |  | |  __/ | | | | | (_) | |  | |_| |
+  \\____/| .__/ \\___|_| |_|_|  |_|\\___|_| |_| |_|\\___/|_|   \\__, |
+        | |                                                 __/ |
+        |_|                                                |___/ `;
+
+const app = server({ max_payload_size: env.max_payload_size });
+
+console.log(ASC);
+console.log(`[CONFIG] Vector Dimension: ${env.vec_dim}`);
+console.log(`[CONFIG] Cache Segments: ${env.cache_segments}`);
+console.log(`[CONFIG] Max Active Queries: ${env.max_active}`);
+
+// Warn about configuration mismatch that causes embedding incompatibility
+if (env.emb_kind !== "synthetic" && (tier === "hybrid" || tier === "fast")) {
+    console.warn(
+        `[CONFIG] ⚠️  WARNING: Embedding configuration mismatch detected!\n` +
+        `         OM_EMBEDDINGS=${env.emb_kind} but OM_TIER=${tier}\n` +
+        `         Storage will use ${env.emb_kind} embeddings, but queries will use synthetic embeddings.\n` +
+        `         This causes semantic search to fail. Set OM_TIER=deep to fix.`
+    );
+}
+
+app.use(req_tracker_mw());
+
+app.use((req: any, res: any, next: any) => {
+    const allowedOrigins = [
+        "https://ahok.io",
+        "https://www.ahok.io",
+        "http://localhost:3000",
+        "http://localhost:8080",
+    ];
+    const origin = req.headers.origin;
+    // Allow specific origins, Amplify apps, and App Runner domains
+    if (
+        allowedOrigins.includes(origin) ||
+        origin?.endsWith(".amplifyapp.com") ||
+        origin?.endsWith(".awsapprunner.com")
+    ) {
+        res.setHeader("Access-Control-Allow-Origin", origin);
+    } else {
+        // Allow any origin for API requests (backward compatibility)
+        res.setHeader("Access-Control-Allow-Origin", "*");
+    }
+    res.setHeader(
+        "Access-Control-Allow-Methods",
+        "GET,POST,PUT,PATCH,DELETE,OPTIONS",
+    );
+    res.setHeader(
+        "Access-Control-Allow-Headers",
+        "Content-Type,Authorization,x-api-key",
+    );
+    res.setHeader("Access-Control-Allow-Credentials", "true");
+    if (req.method === "OPTIONS") {
+        res.status(200).end();
+        return;
+    }
+    next();
+});
+
+app.use(authenticate_api_request);
+
+if (process.env.OM_LOG_AUTH === "true") {
+    app.use(log_authenticated_request);
+}
+
+const path = require("path");
+
+routes(app);
+
+// Serve dashboard static files
+const publicDir = path.join(__dirname, "../..", "public");
+app.use(app.serverStatic("/", publicDir));
+
+// Fallback for SPA routing
+app.get("*", (req: any, res: any, next: any) => {
+    if (req.path.startsWith("/api") || req.path.startsWith("/auth")) {
+        return next();
+    }
+    res.setHeader("Content-Type", "text/html");
+    const fs = require("fs");
+    const indexPath = path.join(publicDir, "index.html");
+    if (fs.existsSync(indexPath)) {
+        res.send(fs.readFileSync(indexPath, "utf8"));
+    } else {
+        next();
+    }
+});
+
+mcp(app);
+if (env.mode === "langgraph") {
+    console.log("[MODE] LangGraph integration enabled");
+}
+
+const decayIntervalMs = env.decay_interval_minutes * 60 * 1000;
+console.log(
+    `[DECAY] Interval: ${env.decay_interval_minutes} minutes (${decayIntervalMs / 1000}s)`,
+);
+
+setInterval(async () => {
+    console.log("[DECAY] Running HSG decay process...");
+    try {
+        const result = await run_decay_process();
+        console.log(
+            `[DECAY] Completed: ${result.decayed}/${result.processed} memories updated`,
+        );
+    } catch (error) {
+        console.error("[DECAY] Process failed:", error);
+    }
+}, decayIntervalMs);
+setInterval(
+    async () => {
+        console.log("[PRUNE] Pruning weak waypoints...");
+        try {
+            const pruned = await prune_weak_waypoints();
+            console.log(`[PRUNE] Completed: ${pruned} waypoints removed`);
+        } catch (error) {
+            console.error("[PRUNE] Failed:", error);
+        }
+    },
+    7 * 24 * 60 * 60 * 1000,
+);
+setTimeout(() => {
+    run_decay_process()
+        .then((result: any) => {
+            console.log(
+                `[INIT] Initial decay: ${result.decayed}/${result.processed} memories updated`,
+            );
+        })
+        .catch(console.error);
+}, 3000);
+
+start_reflection();
+start_user_summary_reflection();
+
+console.log(`[SERVER] Starting on port ${env.port}`);
+app.listen(env.port, () => {
+    console.log(`[SERVER] Running on http://localhost:${env.port}`);
+    sendTelemetry().catch(() => {
+        // ignore telemetry failures
+    });
+});

package/src/server/middleware/auth.ts

@@ -0,0 +1,156 @@
+import { env } from "../../core/cfg";
+import crypto from "crypto";
+
+const rate_limit_store = new Map<
+    string,
+    { count: number; reset_time: number }
+>();
+const auth_config = {
+    api_key: env.api_key,
+    api_key_header: "x-api-key",
+    rate_limit_enabled: env.rate_limit_enabled,
+    rate_limit_window_ms: env.rate_limit_window_ms,
+    rate_limit_max_requests: env.rate_limit_max_requests,
+    public_endpoints: [
+        "/health",
+        "/api/system/health",
+        "/api/system/stats",
+        "/dashboard/health",
+        "/auth",
+        "/docs",
+        "/openapi.json",
+        "/.well-known",
+        "/mcp",
+    ],
+};
+
+function is_public_endpoint(path: string): boolean {
+    return auth_config.public_endpoints.some(
+        (e) => path === e || path.startsWith(e),
+    );
+}
+
+function extract_api_key(req: any): string | null {
+    const x_api_key = req.headers[auth_config.api_key_header];
+    if (x_api_key) return x_api_key;
+    const auth_header = req.headers["authorization"];
+    if (auth_header) {
+        if (auth_header.startsWith("Bearer ")) return auth_header.slice(7);
+        if (auth_header.startsWith("ApiKey ")) return auth_header.slice(7);
+    }
+    return null;
+}
+
+function validate_api_key(provided: string, expected: string): boolean {
+    if (!provided || !expected || provided.length !== expected.length)
+        return false;
+    return crypto.timingSafeEqual(Buffer.from(provided), Buffer.from(expected));
+}
+
+function check_rate_limit(client_id: string): {
+    allowed: boolean;
+    remaining: number;
+    reset_time: number;
+} {
+    if (!auth_config.rate_limit_enabled)
+        return { allowed: true, remaining: -1, reset_time: -1 };
+    const now = Date.now();
+    const data = rate_limit_store.get(client_id);
+    if (!data || now >= data.reset_time) {
+        const new_data = {
+            count: 1,
+            reset_time: now + auth_config.rate_limit_window_ms,
+        };
+        rate_limit_store.set(client_id, new_data);
+        return {
+            allowed: true,
+            remaining: auth_config.rate_limit_max_requests - 1,
+            reset_time: new_data.reset_time,
+        };
+    }
+    data.count++;
+    rate_limit_store.set(client_id, data);
+    const remaining = auth_config.rate_limit_max_requests - data.count;
+    return {
+        allowed: data.count <= auth_config.rate_limit_max_requests,
+        remaining: Math.max(0, remaining),
+        reset_time: data.reset_time,
+    };
+}
+
+function get_client_id(req: any, api_key: string | null): string {
+    if (api_key)
+        return crypto
+            .createHash("sha256")
+            .update(api_key)
+            .digest("hex")
+            .slice(0, 16);
+    return req.ip || req.connection.remoteAddress || "unknown";
+}
+
+export async function authenticate_api_request(req: any, res: any, next: any) {
+    const path = req.path || req.url;
+    if (is_public_endpoint(path)) return next();
+
+    const provided = extract_api_key(req);
+    if (!provided) {
+        return res.status(401).json({
+            error: "authentication_required",
+            message: "API key required",
+        });
+    }
+
+    // 1. Check master key
+    if (auth_config.api_key && validate_api_key(provided, auth_config.api_key)) {
+        req.user = { user_id: "admin", is_admin: true };
+        return next();
+    }
+
+    // 2. Check database for individual API keys
+    try {
+        const { q } = require("../../core/db");
+        let user = await q.get_user_by_api_key.get(provided);
+        if (!user) {
+            // Check secondary memory keys
+            user = await q.get_user_by_memory_key.get(provided);
+        }
+
+        if (user) {
+            // Check usage vs capacity
+            if (user.memory_usage >= user.memory_capacity && req.method === "POST" && path.includes("/memories")) {
+                return res.status(403).json({
+                    error: "capacity_exceeded",
+                    message: "Memory capacity reached. Please upgrade your plan.",
+                });
+            }
+            req.user = user;
+            if (user.key_id) {
+                req.key_id = user.key_id;
+                req.key_label = user.key_label;
+            }
+            return next();
+        }
+    } catch (err) {
+        console.error("[AUTH] Database lookup failed:", err);
+    }
+
+    return res.status(403).json({ error: "invalid_api_key" });
+}
+
+export function log_authenticated_request(req: any, res: any, next: any) {
+    const key = extract_api_key(req);
+    if (key)
+        console.log(
+            `[AUTH] ${req.method} ${req.path} [${crypto.createHash("sha256").update(key).digest("hex").slice(0, 8)}...]`,
+        );
+    next();
+}
+
+setInterval(
+    () => {
+        const now = Date.now();
+        for (const [id, data] of rate_limit_store.entries())
+            if (now >= data.reset_time) rate_limit_store.delete(id);
+    },
+    5 * 60 * 1000,
+);