neotoma 0.8.0 → 0.9.0

package/README.md

@@ -23,6 +23,8 @@ Neotoma is a deterministic state layer for AI agents. It stores structured recor
 
 Not retrieval memory (RAG, vector search, semantic lookup). Neotoma enforces deterministic state evolution: same observations always produce the same entity state, regardless of when or in what order they are processed.
 
+The **Inspector** — Neotoma's visual control plane for browsing the entity graph, timeline, schema editor, and agent attribution — is bundled and served at `/inspector` by default when the server starts. No separate build or configuration required. Override with `NEOTOMA_INSPECTOR_DISABLE`, `NEOTOMA_PUBLIC_INSPECTOR_URL`, `NEOTOMA_INSPECTOR_STATIC_DIR`, or `NEOTOMA_INSPECTOR_BASE_PATH` (see `.env.example`).
+
 ## Architecture
 
 ```mermaid

package/dist/actions.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"actions.d.ts","sourceRoot":"","sources":["../src/actions.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAiJ9B,eAAO,MAAM,GAAG,6CAAY,CAAC;AAwZ7B;;;;;;;;;;;;GAYG;AACH,wBAAgB,cAAc,CAAC,GAAG,EAAE,OAAO,CAAC,OAAO,GAAG,OAAO,CAU5D;AA4PD;;;;;;GAMG;AACH,wBAAgB,uBAAuB,IAAI,MAAM,CAmBhD;AA2kHD,wBAAsB,qBAAqB,CAAC,MAAM,EAAE;IAClD,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;IACpC,cAAc,EAAE,MAAM,CAAC;IACvB,iBAAiB,CAAC,EAAE,OAAO,4BAA4B,EAAE,iBAAiB,CAAC;IAC3E,cAAc,EAAE,MAAM,CAAC;IACvB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,aAAa,CAAC,EAAE,KAAK,CAAC;QACpB,iBAAiB,EAAE,MAAM,CAAC;QAC1B,YAAY,EAAE,MAAM,CAAC;QACrB,YAAY,EAAE,MAAM,CAAC;KACtB,CAAC,CAAC;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;;;;;;;;;;;;;cAkfS,MAAM;gBACJ,MAAM;2BACK,MAAM;qBACZ,MAAM;mBACR,MAAM;wBACD,MAAM;uBACP,MAAM;;;;;;;;;mBA9IV,MAAM;qBACJ,MAAM;wBACH,MAAM,GAAG,IAAI;2BACV,MAAM;;wBAET,MAAM;uBACP,MAAM,EAAE;wBACP,MAAM;uBACP,MAAM;;kBA9NX,MAAM;oBACJ,MAAM;yBACD,MAAM;4BACH,MAAM;2BACP,MAAM;;+BA4NF,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI;;;2BAkFlC,MAAM;0BACP,MAAM;0BACN,MAAM;;GA8E3B;AA29DD,wBAAsB,eAAe;;;eA6EpC"}
+{"version":3,"file":"actions.d.ts","sourceRoot":"","sources":["../src/actions.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAiK9B,eAAO,MAAM,GAAG,6CAAY,CAAC;AAif7B;;;;;;;;;;;;GAYG;AACH,wBAAgB,cAAc,CAAC,GAAG,EAAE,OAAO,CAAC,OAAO,GAAG,OAAO,CAU5D;AA4PD;;;;;;GAMG;AACH,wBAAgB,uBAAuB,IAAI,MAAM,CAmBhD;AA6nHD,wBAAsB,qBAAqB,CAAC,MAAM,EAAE;IAClD,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;IACpC,cAAc,EAAE,MAAM,CAAC;IACvB,iBAAiB,CAAC,EAAE,OAAO,4BAA4B,EAAE,iBAAiB,CAAC;IAC3E,cAAc,EAAE,MAAM,CAAC;IACvB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,aAAa,CAAC,EAAE,KAAK,CAAC;QACpB,iBAAiB,EAAE,MAAM,CAAC;QAC1B,YAAY,EAAE,MAAM,CAAC;QACrB,YAAY,EAAE,MAAM,CAAC;KACtB,CAAC,CAAC;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;;;;;;;;;;;;;cAkfS,MAAM;gBACJ,MAAM;2BACK,MAAM;qBACZ,MAAM;mBACR,MAAM;wBACD,MAAM;uBACP,MAAM;;;;;;;;;mBA9IV,MAAM;qBACJ,MAAM;wBACH,MAAM,GAAG,IAAI;2BACV,MAAM;;wBAET,MAAM;uBACP,MAAM,EAAE;wBACP,MAAM;uBACP,MAAM;;kBA9NX,MAAM;oBACJ,MAAM;yBACD,MAAM;4BACH,MAAM;2BACP,MAAM;;+BA4NF,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI;;;2BAkFlC,MAAM;0BACP,MAAM;0BACN,MAAM;;GA8E3B;AA29DD,wBAAsB,eAAe;;;eA2FpC"}

package/dist/actions.js

@@ -1,4 +1,5 @@
 import express from "express";
+import cookieParser from "cookie-parser";
 import cors from "cors";
 import helmet from "helmet";
 import morgan from "morgan";
@@ -34,7 +35,9 @@ import { logger } from "./utils/logger.js";
 import { OAuthError } from "./services/mcp_oauth_errors.js";
 import { ensureLocalDevUser, ensureSandboxAauthUser, ensureSandboxPublicUser, LOCAL_DEV_USER_ID, SANDBOX_PUBLIC_USER_ID, } from "./services/local_auth.js";
 import { isSandboxMode, sandboxDestructiveGuard, sandboxHeaderMiddleware, } from "./services/sandbox_mode.js";
-import { buildLandingContext, buildRootLandingHtml, buildRootLandingJson, buildRootLandingMarkdown, buildRobotsTxt, wantsHtml as acceptWantsHtml, wantsMarkdown as acceptWantsMarkdown, } from "./services/root_landing/index.js";
+import { createSandboxSession, redeemOneTimeCode, resolveSessionFromRequest, revokeSession, purgeSessionUserData, sweepExpiredSessions, SESSION_COOKIE_NAME, } from "./services/sandbox/sessions.js";
+import { buildLandingContext, buildRootLandingHtml, buildRootLandingJson, buildRootLandingMarkdown, buildRobotsTxt, readNeotomaConfigEnvironment, wantsHtml as acceptWantsHtml, wantsMarkdown as acceptWantsMarkdown, } from "./services/root_landing/index.js";
+import { installInspectorMount } from "./services/inspector_mount.js";
 import { getSandboxTermsResponse } from "./services/sandbox/terms.js";
 import { resolveSandboxReportTransport } from "./services/sandbox/transport.js";
 import { getSqliteDb } from "./repositories/sqlite/sqlite_client.js";
@@ -49,6 +52,7 @@ import { readOpenApiActionsFile, readOpenApiFile } from "./shared/openapi_file.j
 import { buildSmitheryServerCard } from "./mcp_server_card.js";
 import { listRecentRecordActivity, parseRecordActivityTypesQuery, } from "./services/recent_record_activity.js";
 import { listRecentConversations } from "./services/recent_conversations.js";
+import { listConversationTurns, getConversationTurn, } from "./services/conversation_turn.js";
 import { getAgent, listAgentRecords, listAgents } from "./services/agents_directory.js";
 export const app = express();
 // Trust proxy headers (required for express-rate-limit when X-Forwarded-For is present)
@@ -102,6 +106,7 @@ app.use(express.json({
     },
 }));
 app.use(morgan("dev"));
+app.use(cookieParser());
 app.use(unknownFieldsGuard);
 // Sandbox-mode response header. Stamped on every response so clients can
 // detect public-sandbox deployments (sandbox.neotoma.io) without an extra
@@ -110,37 +115,125 @@ if (isSandboxMode()) {
     app.use(sandboxHeaderMiddleware);
     logger.info("[Sandbox] NEOTOMA_SANDBOX_MODE=1 — bearer bypass to SANDBOX_PUBLIC_USER_ID, destructive routes gated, weekly reset expected");
 }
-// Inspector SPA mount. When NEOTOMA_INSPECTOR_STATIC_DIR is set, serve the
-// pre-built Inspector bundle at NEOTOMA_INSPECTOR_BASE_PATH (default /app).
-// Deliberately registered before all auth / rate-limit middleware so the SPA
-// shell + assets are reachable without a bearer — the API calls it makes still
-// flow through the normal auth stack below.
-const inspectorStaticDir = (process.env.NEOTOMA_INSPECTOR_STATIC_DIR || "").trim();
-const inspectorBasePath = ((process.env.NEOTOMA_INSPECTOR_BASE_PATH || "/app").trim() || "/app").replace(/\/$/, "");
-if (inspectorStaticDir) {
-    try {
-        const indexHtmlPath = path.resolve(inspectorStaticDir, "index.html");
-        // express.static with fallthrough so 404s on unknown files fall into the
-        // SPA history handler below rather than short-circuiting.
-        app.use(inspectorBasePath, express.static(inspectorStaticDir, {
-            index: false,
-            fallthrough: true,
-            maxAge: "1h",
-        }));
-        app.get([`${inspectorBasePath}`, `${inspectorBasePath}/*`], (req, res, next) => {
-            // Only respond if the request was headed for the SPA (accepts html).
-            if (req.method !== "GET")
-                return next();
-            res.sendFile(indexHtmlPath, (err) => {
-                if (err)
-                    next(err);
+// Inspector SPA mount. Deliberately registered before all auth / rate-limit
+// middleware so the SPA shell + assets are reachable without a bearer — the
+// API calls the Inspector makes still flow through the normal auth stack below.
+installInspectorMount(app, process.env, logger);
+// ── Sandbox session endpoints ───────────────────────────────────────────
+// Registered before general auth so the session handshake works for
+// unauthenticated visitors. Routes exist only when NEOTOMA_SANDBOX_MODE=1.
+if (isSandboxMode()) {
+    const sessionRateLimit = rateLimit({
+        windowMs: 60 * 60 * 1000,
+        limit: 10,
+        standardHeaders: true,
+        legacyHeaders: false,
+        keyGenerator: (req) => `ip:${ipKeyGenerator(req.ip || "")}`,
+        validate: { trustProxy: false },
+    });
+    app.post("/sandbox/session/new", sessionRateLimit, (req, res) => {
+        try {
+            const packId = typeof req.body?.pack_id === "string" ? req.body.pack_id : "generic";
+            const session = createSandboxSession(packId);
+            res.cookie(SESSION_COOKIE_NAME, session.bearerToken, {
+                httpOnly: true,
+                sameSite: "lax",
+                path: "/",
+                expires: new Date(session.expiresAt),
+            });
+            res.json({
+                one_time_code: session.oneTimeCode,
+                expires_at: session.expiresAt,
+                pack_id: session.packId,
+            });
+        }
+        catch (err) {
+            logger.error(`[Sandbox] session/new failed: ${err.message}`);
+            res.status(500).json({ error_code: "SESSION_CREATE_FAILED", message: err.message });
+        }
+    });
+    app.post("/sandbox/session/redeem", (req, res) => {
+        try {
+            const code = typeof req.body?.code === "string" ? req.body.code : "";
+            if (!code) {
+                res.status(400).json({ error_code: "MISSING_CODE", message: "code is required" });
+                return;
+            }
+            const result = redeemOneTimeCode(code);
+            if (!result) {
+                res.status(404).json({ error_code: "INVALID_CODE", message: "Code expired or already redeemed" });
+                return;
+            }
+            res.cookie(SESSION_COOKIE_NAME, result.bearerToken, {
+                httpOnly: true,
+                sameSite: "lax",
+                path: "/",
+                expires: new Date(result.expiresAt),
             });
+            res.json({
+                bearer_token: result.bearerToken,
+                user_id: result.userId,
+                expires_at: result.expiresAt,
+                pack_id: result.packId,
+            });
+        }
+        catch (err) {
+            logger.error(`[Sandbox] session/redeem failed: ${err.message}`);
+            res.status(500).json({ error_code: "SESSION_REDEEM_FAILED", message: err.message });
+        }
+    });
+    app.get("/sandbox/session", (req, res) => {
+        const session = resolveSessionFromRequest(req);
+        if (!session) {
+            res.status(401).json({ error_code: "NO_SESSION", message: "No active sandbox session" });
+            return;
+        }
+        res.json({
+            user_id: session.userId,
+            pack_id: session.packId,
+            created_at: session.createdAt,
+            expires_at: session.expiresAt,
         });
-        logger.info(`[Inspector] Serving SPA from ${inspectorStaticDir} at ${inspectorBasePath}`);
-    }
-    catch (err) {
-        logger.warn(`[Inspector] Failed to mount SPA: ${err.message}`);
-    }
+    });
+    app.post("/sandbox/session/reset", (req, res) => {
+        const session = resolveSessionFromRequest(req);
+        if (!session) {
+            res.status(401).json({ error_code: "NO_SESSION", message: "No active sandbox session" });
+            return;
+        }
+        const packId = typeof req.body?.pack_id === "string" ? req.body.pack_id : undefined;
+        purgeSessionUserData(session.userId);
+        const newSession = createSandboxSession(packId ?? session.packId);
+        res.cookie(SESSION_COOKIE_NAME, newSession.bearerToken, {
+            httpOnly: true,
+            sameSite: "lax",
+            path: "/",
+            expires: new Date(newSession.expiresAt),
+        });
+        res.json({
+            user_id: newSession.userId,
+            pack_id: newSession.packId,
+            expires_at: newSession.expiresAt,
+        });
+    });
+    app.delete("/sandbox/session", (req, res) => {
+        const session = resolveSessionFromRequest(req);
+        if (!session) {
+            res.status(401).json({ error_code: "NO_SESSION", message: "No active sandbox session" });
+            return;
+        }
+        revokeSession(session.userId);
+        purgeSessionUserData(session.userId);
+        res.clearCookie(SESSION_COOKIE_NAME, { path: "/" });
+        res.json({ ok: true });
+    });
+    setInterval(() => {
+        const purged = sweepExpiredSessions();
+        if (purged > 0) {
+            logger.info(`[Sandbox] Swept ${purged} expired session(s)`);
+        }
+    }, 15 * 60 * 1000);
+    logger.info("[Sandbox] Session endpoints registered");
 }
 // Rate limiters for OAuth endpoints
 // validate.trustProxy: false — we use trust proxy behind one proxy; skip strict IP check
@@ -399,6 +492,7 @@ app.get("/server-info", (_req, res) => {
         httpPort,
         apiBase: config.apiBase,
         mcpUrl,
+        neotoma_env: readNeotomaConfigEnvironment(),
     });
 });
 // ============================================================================
@@ -1803,6 +1897,16 @@ app.use(async (req, res, next) => {
         logger.info(`[Auth] ${req.method} ${req.path} auth_method=local_no_bearer user_id=${devUser.id}`);
         return next();
     }
+    // Sandbox ephemeral session: resolve from cookie or Bearer token before
+    // falling back to the shared public user. Expired/revoked sessions 401.
+    if (isSandboxMode()) {
+        const sessionInfo = resolveSessionFromRequest(req);
+        if (sessionInfo) {
+            req.authenticatedUserId = sessionInfo.userId;
+            logger.info(`[Auth] ${req.method} ${req.path} auth_method=sandbox_session user_id=${sessionInfo.userId}`);
+            return next();
+        }
+    }
     // Sandbox mode: public deployment at sandbox.neotoma.io where anonymous
     // callers are attributed to SANDBOX_PUBLIC_USER_ID without a Bearer. AAuth
     // still runs (earlier in the chain via aauthVerify) so agents exercising the
@@ -3187,6 +3291,43 @@ app.get("/recent_conversations", async (req, res) => {
         return handleApiError(req, res, error, "Failed to list recent conversations", "DB_QUERY_FAILED", "APIError:recent_conversations");
     }
 });
+// GET /turns — Inspector: paginated conversation_turn index
+app.get("/turns", async (req, res) => {
+    try {
+        const userId = await getAuthenticatedUserId(req, req.query.user_id);
+        const limit = parseInt(String(req.query.limit ?? "25"), 10) || 25;
+        const offset = parseInt(String(req.query.offset ?? "0"), 10) || 0;
+        const harness = typeof req.query.harness === "string" ? req.query.harness.trim() || null : null;
+        const status = typeof req.query.status === "string" ? req.query.status.trim() || null : null;
+        const activity_after = typeof req.query.activity_after === "string" ? req.query.activity_after.trim() || null : null;
+        const activity_before = typeof req.query.activity_before === "string" ? req.query.activity_before.trim() || null : null;
+        const result = listConversationTurns(userId, limit, offset, {
+            harness,
+            status,
+            activity_after,
+            activity_before,
+        });
+        return res.json(result);
+    }
+    catch (error) {
+        return handleApiError(req, res, error, "Failed to list turns", "DB_QUERY_FAILED", "APIError:turns");
+    }
+});
+// GET /turns/:turn_key — Inspector: conversation_turn detail
+app.get("/turns/:turn_key", async (req, res) => {
+    try {
+        const userId = await getAuthenticatedUserId(req, req.query.user_id);
+        const turnKey = decodeURIComponent(req.params.turn_key);
+        const result = getConversationTurn(userId, turnKey);
+        if (!result) {
+            return res.status(404).json({ error: "Turn not found", code: "NOT_FOUND" });
+        }
+        return res.json(result);
+    }
+    catch (error) {
+        return handleApiError(req, res, error, "Failed to get turn", "DB_QUERY_FAILED", "APIError:turn_detail");
+    }
+});
 // GET /api/sources - Get source list (FU-301)
 app.get("/sources", async (req, res) => {
     try {
@@ -5665,6 +5806,16 @@ export async function startHTTPServer() {
     }
     // Initialize encryption service
     await initServerKeys();
+    // Seed `neotoma_feedback` schema unconditionally so local-only installs
+    // (no AGENT_SITE_BASE_URL) can mirror feedback into the entity graph.
+    try {
+        const { seedNeotomaFeedbackSchema } = await import("./services/feedback/seed_schema.js");
+        await seedNeotomaFeedbackSchema();
+        logger.info("[Feedback] neotoma_feedback schema seeded");
+    }
+    catch (err) {
+        logger.warn(`[Feedback] failed to seed neotoma_feedback schema: ${err.message}`);
+    }
     // Sandbox mode: ensure the `sandbox_abuse_report` entity type is registered
     // before any report comes in so forwarded records can attach cleanly to the
     // entity graph. Non-sandbox deployments still benefit from having the schema