ahok-skill 1.3.1

package/src/server/routes/memory.ts

@@ -0,0 +1,440 @@
+import { q, vector_store } from "../../core/db";
+import { now, rid, j, p } from "../../utils";
+import {
+    add_hsg_memory,
+    hsg_query,
+    reinforce_memory,
+    update_memory,
+} from "../../memory/hsg";
+import { ingestDocument, ingestURL } from "../../ops/ingest";
+import { env } from "../../core/cfg";
+import { update_user_summary } from "../../memory/user_summary";
+import type {
+    add_req,
+    q_req,
+    ingest_req,
+    ingest_url_req,
+} from "../../core/types";
+
+// Bulk operation interfaces
+interface BulkDeleteRequest {
+    ids: string[];
+    user_id?: string;
+}
+
+interface BulkDeleteResponse {
+    deleted: number;
+    failed: string[];
+}
+
+// POST /memory/bulk-move - Move memories to different workspace
+// Requirements: 7.1, 7.2
+interface BulkMoveRequest {
+    ids: string[];
+    target_workspace_id: string | null; // null to orphan memories
+    user_id?: string;
+}
+
+interface BulkMoveResponse {
+    moved: number;
+    failed: string[];
+}
+
+export function mem(app: any) {
+    app.post("/memory/add", async (req: any, res: any) => {
+        const b = req.body as add_req & { memory_key_id?: string };
+        if (!b?.content) return res.status(400).json({ err: "content" });
+        try {
+            // Use memory_key_id from body if provided, otherwise use authenticated key's ID
+            const keyId = b.memory_key_id || req.key_id;
+            // Use user_id from body if provided, otherwise use authenticated user's ID
+            const userId = b.user_id || req.user?.user_id;
+
+            const m = await add_hsg_memory(
+                b.content,
+                j(b.tags || []),
+                b.metadata,
+                userId,
+                keyId, // Pass memory key ID
+            );
+            res.json(m);
+
+            if (b.user_id) {
+                update_user_summary(b.user_id).catch((e) =>
+                    console.error("[mem] user summary update failed:", e),
+                );
+            }
+        } catch (e: any) {
+            res.status(500).json({ err: e.message });
+        }
+    });
+
+    app.post("/memory/ingest", async (req: any, res: any) => {
+        const b = req.body as ingest_req;
+        if (!b?.content_type || !b?.data)
+            return res.status(400).json({ err: "missing" });
+        try {
+            const userId = b.user_id || req.user?.user_id;
+            const r = await ingestDocument(
+                b.content_type,
+                b.data,
+                b.metadata,
+                b.config,
+                userId,
+            );
+            res.json(r);
+        } catch (e: any) {
+            res.status(500).json({ err: "ingest_fail", msg: e.message });
+        }
+    });
+
+    app.post("/memory/ingest/url", async (req: any, res: any) => {
+        const b = req.body as ingest_url_req;
+        if (!b?.url) return res.status(400).json({ err: "no_url" });
+        try {
+            const userId = b.user_id || req.user?.user_id;
+            const r = await ingestURL(b.url, b.metadata, b.config, userId);
+            res.json(r);
+        } catch (e: any) {
+            res.status(500).json({ err: "url_fail", msg: e.message });
+        }
+    });
+
+    app.post("/memory/query", async (req: any, res: any) => {
+        const b = req.body as q_req;
+        const k = b.k || 8;
+        try {
+            let userId = b.filters?.user_id || b.user_id;
+            // Enforce user isolation if authenticated and not admin
+            if (req.user && !req.user.is_admin) {
+                userId = req.user.user_id;
+            }
+
+            const f = {
+                sectors: b.filters?.sector ? [b.filters.sector] : undefined,
+                minSalience: b.filters?.min_score,
+                user_id: userId,
+                startTime: b.filters?.startTime,
+                endTime: b.filters?.endTime,
+            };
+            const m = await hsg_query(b.query, k, f);
+            res.json({
+                query: b.query,
+                matches: m.map((x: any) => ({
+                    id: x.id,
+                    content: x.content,
+                    score: x.score,
+                    sectors: x.sectors,
+                    primary_sector: x.primary_sector,
+                    path: x.path,
+                    salience: x.salience,
+                    last_seen_at: x.last_seen_at,
+                })),
+            });
+        } catch (e: any) {
+            res.json({ query: b.query, matches: [] });
+        }
+    });
+
+    app.post("/memory/reinforce", async (req: any, res: any) => {
+        const b = req.body as { id: string; boost?: number };
+        if (!b?.id) return res.status(400).json({ err: "id" });
+        try {
+            await reinforce_memory(b.id, b.boost);
+            res.json({ ok: true });
+        } catch (e: any) {
+            res.status(404).json({ err: "nf" });
+        }
+    });
+
+    app.patch("/memory/:id", async (req: any, res: any) => {
+        const id = req.params.id;
+        const b = req.body as {
+            content?: string;
+            tags?: string[];
+            metadata?: any;
+            user_id?: string;
+        };
+        if (!id) return res.status(400).json({ err: "id" });
+        try {
+            // Check if memory exists and user has permission
+            const m = await q.get_mem.get(id);
+            if (!m) return res.status(404).json({ err: "nf" });
+
+            // Enforce user ownership
+            if (req.user && !req.user.is_admin && m.user_id !== req.user.user_id) {
+                return res.status(403).json({ err: "forbidden" });
+            }
+            // Check explicit user_id match if provided (for admin or extra validation)
+            if (b.user_id && m.user_id !== b.user_id) {
+                return res.status(403).json({ err: "forbidden" });
+            }
+
+            const r = await update_memory(id, b.content, b.tags, b.metadata);
+            res.json(r);
+        } catch (e: any) {
+            if (e.message.includes("not found")) {
+                res.status(404).json({ err: "nf" });
+            } else {
+                res.status(500).json({ err: "internal" });
+            }
+        }
+    });
+
+    app.get("/memory/all", async (req: any, res: any) => {
+        try {
+            const u = req.query.u ? parseInt(req.query.u) : 0;
+            const l = req.query.l ? parseInt(req.query.l) : 100;
+            const s = req.query.sector;
+            let user_id = req.query.user_id;
+
+            // Enforce user isolation if authenticated and not admin
+            if (req.user && !req.user.is_admin) {
+                user_id = req.user.user_id;
+            }
+
+            let r;
+            if (user_id) {
+                // Filter by user_id
+                r = await q.all_mem_by_user.all(user_id, l, u);
+            } else if (s) {
+                // Filter by sector
+                r = await q.all_mem_by_sector.all(s, l, u);
+            } else {
+                // No filter
+                r = await q.all_mem.all(l, u);
+            }
+
+            if (req.query.key_id) {
+                r = r.filter((x: any) => x.memory_key_id === req.query.key_id);
+            }
+
+            const i = r.map((x: any) => ({
+                id: x.id,
+                content: x.content,
+                tags: p(x.tags),
+                metadata: p(x.meta),
+                created_at: x.created_at,
+                updated_at: x.updated_at,
+                last_seen_at: x.last_seen_at,
+                salience: x.salience,
+                decay_lambda: x.decay_lambda,
+                primary_sector: x.primary_sector,
+                version: x.version,
+                user_id: x.user_id,
+                memory_key_id: x.memory_key_id,
+            }));
+            res.json({ items: i });
+        } catch (e: any) {
+            res.status(500).json({ err: "internal" });
+        }
+    });
+
+    app.get("/memory/:id", async (req: any, res: any) => {
+        try {
+            const id = req.params.id;
+            const user_id = req.query.user_id;
+            const m = await q.get_mem.get(id);
+            if (!m) return res.status(404).json({ err: "nf" });
+
+            // Enforce user ownership
+            if (req.user && !req.user.is_admin && m.user_id !== req.user.user_id) {
+                return res.status(403).json({ err: "forbidden" });
+            }
+            // Check explicit user_id match if provided
+            if (user_id && m.user_id !== user_id) {
+                return res.status(403).json({ err: "forbidden" });
+            }
+
+            const v = await vector_store.getVectorsById(id);
+            const sec = v.map((x: any) => x.sector);
+            res.json({
+                id: m.id,
+                content: m.content,
+                primary_sector: m.primary_sector,
+                sectors: sec,
+                tags: p(m.tags),
+                metadata: p(m.meta),
+                created_at: m.created_at,
+                updated_at: m.updated_at,
+                last_seen_at: m.last_seen_at,
+                salience: m.salience,
+                decay_lambda: m.decay_lambda,
+                version: m.version,
+                user_id: m.user_id,
+                memory_key_id: m.memory_key_id,
+            });
+        } catch (e: any) {
+            res.status(500).json({ err: "internal" });
+        }
+    });
+
+    app.delete("/memory/:id", async (req: any, res: any) => {
+        try {
+            const id = req.params.id;
+            const user_id = req.query.user_id || req.body?.user_id;
+            const m = await q.get_mem.get(id);
+            if (!m) return res.status(404).json({ err: "nf" });
+
+            // Enforce user ownership
+            if (req.user && !req.user.is_admin && m.user_id !== req.user.user_id) {
+                return res.status(403).json({ err: "forbidden" });
+            }
+            // Check explicit user_id match if provided
+            if (user_id && m.user_id !== user_id) {
+                return res.status(403).json({ err: "forbidden" });
+            }
+
+            await q.del_mem.run(id);
+            await vector_store.deleteVectors(id);
+            await q.del_waypoints.run(id, id);
+            res.json({ ok: true });
+        } catch (e: any) {
+            res.status(500).json({ err: "internal" });
+        }
+    });
+
+    // POST /memory/bulk-delete - Delete multiple memories
+    // Requirements: 6.2, 6.3
+    app.post("/memory/bulk-delete", async (req: any, res: any) => {
+        const b = req.body as BulkDeleteRequest;
+
+        // Validate request body
+        if (!b?.ids || !Array.isArray(b.ids)) {
+            return res.status(400).json({ err: "ids array required" });
+        }
+
+        if (b.ids.length === 0) {
+            return res.json({ deleted: 0, failed: [] } as BulkDeleteResponse);
+        }
+
+        const user_id = b.user_id;
+        const deleted: string[] = [];
+        const failed: string[] = [];
+
+        // Process each memory individually for ownership verification and partial failure handling
+        for (const id of b.ids) {
+            try {
+                // Check if memory exists
+                const m = await q.get_mem.get(id);
+                if (!m) {
+                    failed.push(id);
+                    continue;
+                }
+
+                // Enforce user ownership
+                if (req.user && !req.user.is_admin && m.user_id !== req.user.user_id) {
+                    failed.push(id);
+                    continue;
+                }
+                // Check user ownership if explicit user_id is provided
+                if (user_id && m.user_id !== user_id) {
+                    failed.push(id);
+                    continue;
+                }
+
+                // Delete memory, vectors, and waypoints
+                await q.del_mem.run(id);
+                await vector_store.deleteVectors(id);
+                await q.del_waypoints.run(id, id);
+                deleted.push(id);
+            } catch (e: any) {
+                // Continue with remaining memories on individual failure
+                failed.push(id);
+            }
+        }
+
+        // Return partial success response (207 if there are failures, 200 if all succeeded)
+        const response: BulkDeleteResponse = {
+            deleted: deleted.length,
+            failed: failed,
+        };
+
+        if (failed.length > 0 && deleted.length > 0) {
+            // Partial success
+            return res.status(207).json(response);
+        } else if (failed.length > 0 && deleted.length === 0) {
+            // All failed
+            return res.status(207).json(response);
+        }
+
+        // All succeeded
+        res.json(response);
+    });
+
+    // POST /memory/bulk-move - Move memories to different workspace
+    // Requirements: 7.1, 7.2
+    app.post("/memory/bulk-move", async (req: any, res: any) => {
+        const b = req.body as BulkMoveRequest;
+
+        // Validate request body
+        if (!b?.ids || !Array.isArray(b.ids)) {
+            return res.status(400).json({ err: "ids array required" });
+        }
+
+        if (b.ids.length === 0) {
+            return res.json({ moved: 0, failed: [] } as BulkMoveResponse);
+        }
+
+        // target_workspace_id can be null (to orphan memories) or a string
+        if (b.target_workspace_id !== null && typeof b.target_workspace_id !== 'string') {
+            return res.status(400).json({ err: "target_workspace_id must be string or null" });
+        }
+
+        const user_id = b.user_id;
+        const moved: string[] = [];
+        const failed: string[] = [];
+
+        // Process each memory individually for ownership verification and partial failure handling
+        for (const id of b.ids) {
+            try {
+                // Check if memory exists
+                const m = await q.get_mem.get(id);
+                if (!m) {
+                    failed.push(id);
+                    continue;
+                }
+
+                // Enforce user ownership
+                if (req.user && !req.user.is_admin && m.user_id !== req.user.user_id) {
+                    failed.push(id);
+                    continue;
+                }
+                // Check user ownership if explicit user_id is provided
+                if (user_id && m.user_id !== user_id) {
+                    failed.push(id);
+                    continue;
+                }
+
+                // Update memory_key_id for this memory
+                await q.bulk_upd_mem_key.run(
+                    b.target_workspace_id,
+                    now(),
+                    [id],
+                    m.user_id
+                );
+                moved.push(id);
+            } catch (e: any) {
+                // Continue with remaining memories on individual failure
+                failed.push(id);
+            }
+        }
+
+        // Return partial success response (207 if there are failures, 200 if all succeeded)
+        const response: BulkMoveResponse = {
+            moved: moved.length,
+            failed: failed,
+        };
+
+        if (failed.length > 0 && moved.length > 0) {
+            // Partial success
+            return res.status(207).json(response);
+        } else if (failed.length > 0 && moved.length === 0) {
+            // All failed
+            return res.status(207).json(response);
+        }
+
+        // All succeeded
+        res.json(response);
+    });
+}

package/src/server/routes/sources.ts

@@ -0,0 +1,111 @@
+/**
+ * sources webhook routes - ingest data from external sources via HTTP
+ * 
+ * POST /sources/:source/ingest
+ *   body: { creds: {...}, filters: {...}, user_id?: string }
+ * 
+ * POST /sources/webhook/:source
+ *   generic webhook endpoint for source-specific payloads
+ */
+
+import * as sources from "../../sources";
+
+export function src(app: any) {
+    // list available sources
+    app.get("/sources", async (req: any, res: any) => {
+        res.json({
+            sources: ["github", "notion", "google_drive", "google_sheets", "google_slides", "onedrive", "web_crawler"],
+            usage: {
+                ingest: "POST /sources/:source/ingest { creds: {}, filters: {}, user_id? }",
+                webhook: "POST /sources/webhook/:source (source-specific payload)"
+            }
+        });
+    });
+
+    // ingest from a source
+    app.post("/sources/:source/ingest", async (req: any, res: any) => {
+        const { source } = req.params;
+        const { creds = {}, filters = {}, user_id } = req.body || {};
+
+        const source_map: Record<string, any> = {
+            github: sources.github_source,
+            notion: sources.notion_source,
+            google_drive: sources.google_drive_source,
+            google_sheets: sources.google_sheets_source,
+            google_slides: sources.google_slides_source,
+            onedrive: sources.onedrive_source,
+            web_crawler: sources.web_crawler_source,
+        };
+
+        if (!source_map[source]) {
+            return res.status(400).json({ error: `unknown source: ${source}`, available: Object.keys(source_map) });
+        }
+
+        try {
+            const src = new source_map[source](user_id);
+            await src.connect(creds);
+            const ids = await src.ingest_all(filters);
+            res.json({ ok: true, ingested: ids.length, memory_ids: ids });
+        } catch (e: any) {
+            res.status(500).json({ error: e.message });
+        }
+    });
+
+    // webhook endpoint for github events
+    app.post("/sources/webhook/github", async (req: any, res: any) => {
+        const event_type = req.headers["x-github-event"];
+        const payload = req.body;
+
+        if (!payload) {
+            return res.status(400).json({ error: "no payload" });
+        }
+
+        try {
+            const { ingestDocument } = await import("../../ops/ingest");
+
+            // handle different github events
+            let content = "";
+            let meta: Record<string, any> = { source: "github_webhook", event: event_type };
+
+            if (event_type === "push") {
+                const commits = payload.commits || [];
+                content = commits.map((c: any) => `${c.message}\n${c.url}`).join("\n\n");
+                meta.repo = payload.repository?.full_name;
+                meta.ref = payload.ref;
+            } else if (event_type === "issues") {
+                content = `[${payload.action}] ${payload.issue?.title}\n${payload.issue?.body || ""}`;
+                meta.repo = payload.repository?.full_name;
+                meta.issue_number = payload.issue?.number;
+            } else if (event_type === "pull_request") {
+                content = `[${payload.action}] PR: ${payload.pull_request?.title}\n${payload.pull_request?.body || ""}`;
+                meta.repo = payload.repository?.full_name;
+                meta.pr_number = payload.pull_request?.number;
+            } else {
+                content = JSON.stringify(payload, null, 2);
+            }
+
+            if (content) {
+                const result = await ingestDocument("text", content, meta);
+                res.json({ ok: true, memory_id: result.root_memory_id, event: event_type });
+            } else {
+                res.json({ ok: true, skipped: true, reason: "no content" });
+            }
+        } catch (e: any) {
+            res.status(500).json({ error: e.message });
+        }
+    });
+
+    // generic webhook for notion
+    app.post("/sources/webhook/notion", async (req: any, res: any) => {
+        const payload = req.body;
+
+        try {
+            const { ingestDocument } = await import("../../ops/ingest");
+            const content = JSON.stringify(payload, null, 2);
+            const result = await ingestDocument("text", content, { source: "notion_webhook" });
+            res.json({ ok: true, memory_id: result.root_memory_id });
+        } catch (e: any) {
+            res.status(500).json({ error: e.message });
+        }
+    });
+}

package/src/server/routes/system.ts

@@ -0,0 +1,68 @@
+import { all_async } from "../../core/db";
+import { sector_configs } from "../../memory/hsg";
+import { getEmbeddingInfo } from "../../memory/embed";
+import { tier, env } from "../../core/cfg";
+
+const TIER_BENEFITS = {
+    hybrid: {
+        recall: 98,
+        qps: "700-800",
+        ram: "0.5gb/10k",
+        use: "For high accuracy",
+    },
+    fast: {
+        recall: 70,
+        qps: "700-850",
+        ram: "0.6GB/10k",
+        use: "Local apps, extensions",
+    },
+    smart: {
+        recall: 85,
+        qps: "500-600",
+        ram: "0.9GB/10k",
+        use: "Production servers",
+    },
+    deep: {
+        recall: 94,
+        qps: "350-400",
+        ram: "1.6GB/10k",
+        use: "Cloud, high-accuracy",
+    },
+};
+
+export function sys(app: any) {
+    app.get(
+        "/health",
+        async (incoming_http_request: any, outgoing_http_response: any) => {
+            outgoing_http_response.json({
+                ok: true,
+                version: "2.0-hsg-tiered",
+                embedding: getEmbeddingInfo(),
+                tier,
+                dim: env.vec_dim,
+                cache: env.cache_segments,
+                expected: TIER_BENEFITS[tier],
+            });
+        },
+    );
+
+    app.get(
+        "/sectors",
+        async (incoming_http_request: any, outgoing_http_response: any) => {
+            try {
+                const database_sector_statistics_rows = await all_async(`
+                select primary_sector as sector, count(*) as count, avg(salience) as avg_salience 
+                from memories 
+                group by primary_sector
+            `);
+                outgoing_http_response.json({
+                    sectors: Object.keys(sector_configs),
+                    configs: sector_configs,
+                    stats: database_sector_statistics_rows,
+                });
+            } catch (unexpected_error_fetching_sectors) {
+                outgoing_http_response.status(500).json({ err: "internal" });
+            }
+        },
+    );
+}