@realtimex/folio 0.1.2

package/api/src/routes/ingestions.ts

@@ -0,0 +1,275 @@
+import { Router } from "express";
+import multer from "multer";
+import os from "os";
+import path from "path";
+import fs from "fs/promises";
+import crypto from "crypto";
+import { asyncHandler } from "../middleware/errorHandler.js";
+import { optionalAuth } from "../middleware/auth.js";
+import { IngestionService } from "../services/IngestionService.js";
+
+const router = Router();
+const upload = multer({ storage: multer.memoryStorage(), limits: { fileSize: 20 * 1024 * 1024 } });
+
+router.use(optionalAuth);
+
+// GET /api/ingestions — list ingestions (paginated, searchable)
+router.get(
+    "/",
+    asyncHandler(async (req, res) => {
+        if (!req.supabase || !req.user) {
+            res.status(401).json({ success: false, error: "Authentication required" });
+            return;
+        }
+        const page = Math.max(1, parseInt(req.query["page"] as string) || 1);
+        const pageSize = Math.min(100, Math.max(1, parseInt(req.query["pageSize"] as string) || 20));
+        const query = (req.query["q"] as string | undefined)?.trim() || undefined;
+        const { ingestions, total } = await IngestionService.list(req.supabase, req.user.id, { page, pageSize, query });
+        res.json({ success: true, ingestions, total, page, pageSize });
+    })
+);
+
+// GET /api/ingestions/:id — get single ingestion
+router.get(
+    "/:id",
+    asyncHandler(async (req, res) => {
+        if (!req.supabase || !req.user) {
+            res.status(401).json({ success: false, error: "Authentication required" });
+            return;
+        }
+        const ingestion = await IngestionService.get(req.params["id"] as string, req.supabase, req.user.id);
+        if (!ingestion) {
+            res.status(404).json({ success: false, error: "Not found" });
+            return;
+        }
+        res.json({ success: true, ingestion });
+    })
+);
+
+// POST /api/ingestions/upload — manual file upload
+router.post(
+    "/upload",
+    upload.single("file"),
+    asyncHandler(async (req, res) => {
+        if (!req.supabase || !req.user) {
+            res.status(401).json({ success: false, error: "Authentication required" });
+            return;
+        }
+        const file = req.file;
+        if (!file) {
+            res.status(400).json({ success: false, error: "No file uploaded" });
+            return;
+        }
+
+        // Fetch user configured Storage Path (Dropzone)
+        const { data: settings } = await req.supabase
+            .from("user_settings")
+            .select("storage_path")
+            .eq("user_id", req.user.id)
+            .maybeSingle();
+
+        const dropzoneDir = settings?.storage_path || path.join(os.homedir(), ".realtimex", "folio", "dropzone");
+        await fs.mkdir(dropzoneDir, { recursive: true });
+
+        // Compute SHA-256 hash before writing — used for deduplication
+        const fileHash = crypto.createHash("sha256").update(file.buffer).digest("hex");
+
+        // Save physical file
+        const safeFilename = `${Date.now()}-${file.originalname.replace(/[^a-zA-Z0-9.-]/g, "_")}`;
+        const filePath = path.join(dropzoneDir, safeFilename);
+        await fs.writeFile(filePath, file.buffer);
+
+        // Extract text (mock extraction for pdfs, works for text files)
+        const content = file.buffer.toString("utf-8").replace(/\0/g, "").slice(0, 50_000);
+
+        const ingestion = await IngestionService.ingest({
+            supabase: req.supabase,
+            userId: req.user.id,
+            filename: file.originalname,
+            mimeType: file.mimetype,
+            fileSize: file.size,
+            source: "upload",
+            filePath,
+            content,
+            fileHash,
+        });
+
+        res.status(201).json({ success: true, ingestion });
+    })
+);
+
+// POST /api/ingestions/:id/rerun — re-run processing
+router.post(
+    "/:id/rerun",
+    asyncHandler(async (req, res) => {
+        if (!req.supabase || !req.user) {
+            res.status(401).json({ success: false, error: "Authentication required" });
+            return;
+        }
+        const matched = await IngestionService.rerun(req.params["id"] as string, req.supabase, req.user.id);
+        res.json({ success: true, matched });
+    })
+);
+
+// POST /api/ingestions/:id/match — manually assign a policy and optionally learn from it
+router.post(
+    "/:id/match",
+    asyncHandler(async (req, res) => {
+        if (!req.supabase || !req.user) {
+            res.status(401).json({ success: false, error: "Authentication required" });
+            return;
+        }
+
+        const policyId = typeof req.body?.policy_id === "string" ? req.body.policy_id.trim() : "";
+        if (!policyId) {
+            res.status(400).json({ success: false, error: "policy_id is required" });
+            return;
+        }
+
+        const learn = req.body?.learn !== false;
+        const rerun = req.body?.rerun !== false;
+        const allowSideEffects = req.body?.allow_side_effects === true;
+        try {
+            const ingestion = await IngestionService.matchToPolicy(
+                req.params["id"] as string,
+                policyId,
+                req.supabase,
+                req.user.id,
+                {
+                    learn,
+                    rerun,
+                    allowSideEffects,
+                }
+            );
+            res.json({ success: true, ingestion });
+        } catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            if (/not found/i.test(message)) {
+                res.status(404).json({ success: false, error: message });
+                return;
+            }
+            if (/required|processing|disabled|cannot manually match|side-effect/i.test(message)) {
+                res.status(400).json({ success: false, error: message });
+                return;
+            }
+            throw error;
+        }
+    })
+);
+
+// POST /api/ingestions/:id/refine-policy — suggest a refinement draft for a target policy
+router.post(
+    "/:id/refine-policy",
+    asyncHandler(async (req, res) => {
+        if (!req.supabase || !req.user) {
+            res.status(401).json({ success: false, error: "Authentication required" });
+            return;
+        }
+
+        const policyId = typeof req.body?.policy_id === "string" ? req.body.policy_id.trim() : "";
+        if (!policyId) {
+            res.status(400).json({ success: false, error: "policy_id is required" });
+            return;
+        }
+
+        try {
+            const suggestion = await IngestionService.suggestPolicyRefinement(
+                req.params["id"] as string,
+                policyId,
+                req.supabase,
+                req.user.id,
+                {
+                    provider: typeof req.body?.provider === "string" ? req.body.provider : undefined,
+                    model: typeof req.body?.model === "string" ? req.body.model : undefined,
+                }
+            );
+            res.json({ success: true, suggestion });
+        } catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            if (/not found/i.test(message)) {
+                res.status(404).json({ success: false, error: message });
+                return;
+            }
+            if (/required|disabled/i.test(message)) {
+                res.status(400).json({ success: false, error: message });
+                return;
+            }
+            throw error;
+        }
+    })
+);
+
+// POST /api/ingestions/:id/summarize — generate (or return cached) prose summary
+router.post(
+    "/:id/summarize",
+    asyncHandler(async (req, res) => {
+        if (!req.supabase || !req.user) {
+            res.status(401).json({ success: false, error: "Authentication required" });
+            return;
+        }
+        const { data: settingsRow } = await req.supabase
+            .from("user_settings")
+            .select("llm_provider, llm_model")
+            .eq("user_id", req.user.id)
+            .maybeSingle();
+
+        const llmSettings = {
+            llm_provider: settingsRow?.llm_provider ?? undefined,
+            llm_model: settingsRow?.llm_model ?? undefined,
+        };
+
+        const summary = await IngestionService.summarize(
+            req.params["id"] as string,
+            req.supabase,
+            req.user.id,
+            llmSettings
+        );
+        res.json({ success: true, summary });
+    })
+);
+
+// PATCH /api/ingestions/:id/tags — replace tags array (human edits)
+router.patch(
+    "/:id/tags",
+    asyncHandler(async (req, res) => {
+        if (!req.supabase || !req.user) {
+            res.status(401).json({ success: false, error: "Authentication required" });
+            return;
+        }
+        const tags: unknown = req.body?.tags;
+        if (!Array.isArray(tags) || tags.some((t) => typeof t !== "string")) {
+            res.status(400).json({ success: false, error: "tags must be an array of strings" });
+            return;
+        }
+        const normalized = (tags as string[]).map((t) => t.toLowerCase().trim()).filter(Boolean);
+        const { error } = await req.supabase
+            .from("ingestions")
+            .update({ tags: normalized })
+            .eq("id", req.params["id"] as string)
+            .eq("user_id", req.user.id);
+        if (error) {
+            res.status(500).json({ success: false, error: error.message });
+            return;
+        }
+        res.json({ success: true, tags: normalized });
+    })
+);
+
+// DELETE /api/ingestions/:id — delete
+router.delete(
+    "/:id",
+    asyncHandler(async (req, res) => {
+        if (!req.supabase || !req.user) {
+            res.status(401).json({ success: false, error: "Authentication required" });
+            return;
+        }
+        const deleted = await IngestionService.delete(req.params["id"] as string, req.supabase, req.user.id);
+        if (!deleted) {
+            res.status(404).json({ success: false, error: "Not found" });
+            return;
+        }
+        res.json({ success: true });
+    })
+);
+
+export default router;

package/api/src/routes/migrate.ts

@@ -0,0 +1,112 @@
+import { spawn } from "node:child_process";
+import { join } from "node:path";
+
+import { Router } from "express";
+
+import { config } from "../config/index.js";
+import { asyncHandler } from "../middleware/errorHandler.js";
+import { schemas, validateBody } from "../middleware/validation.js";
+import { createLogger } from "../utils/logger.js";
+
+const router = Router();
+const logger = createLogger("MigrateRoutes");
+
+router.post(
+  "/",
+  validateBody(schemas.migrate),
+  asyncHandler(async (req, res) => {
+    const { projectRef, accessToken, anonKey } = req.body;
+
+    logger.info("Starting migration", { projectRef });
+
+    res.setHeader("Content-Type", "text/event-stream");
+    res.setHeader("Cache-Control", "no-cache");
+    res.setHeader("Connection", "keep-alive");
+    // @ts-ignore express typings may not include flushHeaders depending on version
+    res.flushHeaders?.();
+
+    const sendEvent = (type: string, data: string) => {
+      if (res.writableEnded) {
+        return;
+      }
+      res.write(`data: ${JSON.stringify({ type, data })}\n\n`);
+    };
+
+    const scriptPath = join(config.scriptsDir, "migrate.sh");
+
+    const child = spawn("bash", [scriptPath], {
+      env: {
+        ...process.env,
+        SUPABASE_PROJECT_ID: projectRef,
+        SUPABASE_ACCESS_TOKEN: accessToken,
+        SUPABASE_ANON_KEY: anonKey || "",
+        SKIP_FUNCTIONS: process.env.SKIP_FUNCTIONS || "0"
+      },
+      cwd: config.rootDir
+    });
+
+    let clientDisconnected = false;
+    const stopChild = () => {
+      clientDisconnected = true;
+      if (!child.killed) {
+        child.kill("SIGTERM");
+      }
+    };
+
+    res.on("close", stopChild);
+    req.on("aborted", stopChild);
+
+    child.stdout.on("data", (chunk) => {
+      const lines = String(chunk)
+        .split("\n")
+        .filter((line) => line.trim().length > 0);
+      for (const line of lines) {
+        sendEvent("stdout", line);
+      }
+    });
+
+    child.stderr.on("data", (chunk) => {
+      const lines = String(chunk)
+        .split("\n")
+        .filter((line) => line.trim().length > 0);
+      for (const line of lines) {
+        sendEvent("stderr", line);
+      }
+    });
+
+    child.on("error", (error) => {
+      if (clientDisconnected) {
+        logger.info("Migration process ended after client disconnect", { projectRef });
+        return;
+      }
+      sendEvent("error", `Failed to run migration: ${error.message}`);
+      sendEvent("done", "failed");
+      if (!res.writableEnded) {
+        res.end();
+      }
+    });
+
+    child.on("close", (code, signal) => {
+      if (clientDisconnected) {
+        logger.info("Migration process stopped after client disconnect", { projectRef, code, signal });
+        return;
+      }
+
+      if (code === 0) {
+        sendEvent("info", "Migration completed successfully.");
+        sendEvent("done", "success");
+      } else {
+        const codeLabel = code === null ? "null" : String(code);
+        const signalSuffix = signal ? ` (signal: ${signal})` : "";
+        sendEvent("error", `Migration failed with code ${codeLabel}${signalSuffix}`);
+        sendEvent("done", "failed");
+      }
+
+      if (!res.writableEnded) {
+        res.end();
+      }
+    });
+  })
+);
+
+export default router;

package/api/src/routes/policies.ts

@@ -0,0 +1,121 @@
+import { Router } from "express";
+import { asyncHandler } from "../middleware/errorHandler.js";
+import { optionalAuth } from "../middleware/auth.js";
+import { PolicyLoader } from "../services/PolicyLoader.js";
+import { PolicyEngine } from "../services/PolicyEngine.js";
+import { PolicyLearningService } from "../services/PolicyLearningService.js";
+
+const router = Router();
+
+// Use optionalAuth on all routes — sets req.supabase + req.user if a valid
+// bearer token is present. Falls through without error if no token.
+router.use(optionalAuth);
+
+// GET /api/policies — list all loaded policies
+router.get(
+    "/",
+    asyncHandler(async (req, res) => {
+        const policies = await PolicyLoader.load(false, req.supabase);
+        if (!req.supabase || !req.user || policies.length === 0) {
+            res.json({ success: true, policies });
+            return;
+        }
+
+        const stats = await PolicyLearningService.getPolicyLearningStats({
+            supabase: req.supabase,
+            userId: req.user.id,
+            policyIds: policies.map((policy) => policy.metadata.id),
+        });
+
+        const enrichedPolicies = policies.map((policy) => {
+            const stat = stats[policy.metadata.id];
+            return {
+                ...policy,
+                metadata: {
+                    ...policy.metadata,
+                    learning_examples: stat?.samples ?? 0,
+                    learning_last_at: stat?.lastSampleAt ?? null,
+                },
+            };
+        });
+
+        res.json({ success: true, policies: enrichedPolicies });
+    })
+);
+
+// POST /api/policies — save a new policy
+router.post(
+    "/",
+    asyncHandler(async (req, res) => {
+        const policy = req.body;
+        if (!PolicyLoader.validate(policy)) {
+            res.status(400).json({ success: false, error: "Invalid policy schema" });
+            return;
+        }
+        const filePath = await PolicyLoader.save(policy, req.supabase, req.user?.id);
+        res.status(201).json({ success: true, filePath });
+    })
+);
+
+// DELETE /api/policies/:id — delete a policy by ID
+router.delete(
+    "/:id",
+    asyncHandler(async (req, res) => {
+        const deleted = await PolicyLoader.delete(req.params["id"] as string, req.supabase, req.user?.id);
+        if (!deleted) {
+            res.status(404).json({ success: false, error: "Policy not found" });
+            return;
+        }
+        res.json({ success: true });
+    })
+);
+
+// PATCH /api/policies/:id — partial update (enabled toggle, metadata fields)
+router.patch(
+    "/:id",
+    asyncHandler(async (req, res) => {
+        const { enabled, name, description, tags, priority } = req.body;
+        await PolicyLoader.patch(
+            req.params["id"] as string,
+            { enabled, name, description, tags, priority },
+            req.supabase,
+            req.user?.id
+        );
+        res.json({ success: true });
+    })
+);
+
+// POST /api/policies/reload — force cache invalidation
+router.post(
+    "/reload",
+    asyncHandler(async (req, res) => {
+        PolicyLoader.invalidateCache();
+        const policies = await PolicyLoader.load(true, req.supabase);
+        res.json({ success: true, count: policies.length });
+    })
+);
+
+// POST /api/policies/synthesize — NL → Policy via LLM
+router.post(
+    "/synthesize",
+    asyncHandler(async (req, res) => {
+        const { description, provider, model } = req.body;
+        if (!description || typeof description !== "string") {
+            res.status(400).json({ success: false, error: "description is required" });
+            return;
+        }
+        const result = await PolicyEngine.synthesizeFromNL(description, {
+            provider,
+            model,
+            userId: req.user?.id,
+            supabase: req.supabase,
+        });
+        if (!result.policy) {
+            res.status(503).json({ success: false, error: result.error ?? "Synthesis failed. SDK may be unavailable." });
+            return;
+        }
+        res.json({ success: true, policy: result.policy, warning: result.error });
+    })
+);
+
+export default router;

package/api/src/routes/processing.ts

@@ -0,0 +1,90 @@
+import { Router } from "express";
+
+import { config } from "../config/index.js";
+import { authMiddleware } from "../middleware/auth.js";
+import { asyncHandler } from "../middleware/errorHandler.js";
+import { validateBody, schemas } from "../middleware/validation.js";
+import { SDKService } from "../services/SDKService.js";
+
+const router = Router();
+
+router.post(
+  "/dispatch",
+  authMiddleware,
+  validateBody(schemas.dispatchProcessing),
+  asyncHandler(async (req, res) => {
+    if (!req.user || !req.supabase) {
+      res.status(401).json({
+        success: false,
+        error: {
+          code: "AUTH_REQUIRED",
+          message: "Authentication required"
+        }
+      });
+      return;
+    }
+
+    const sdkAvailable = await SDKService.isAvailable();
+    const defaultProvider = await SDKService.getDefaultChatProvider();
+
+    const { source_type, payload } = req.body;
+
+    // Foundation-only dev path: avoid FK coupling to auth.users when auth is disabled.
+    if (config.security.disableAuth && !config.isProduction) {
+      res.status(202).json({
+        success: true,
+        job: {
+          id: `dev-${Date.now()}`,
+          user_id: req.user.id,
+          status: "queued",
+          source_type,
+          payload,
+          runtime_key: sdkAvailable ? `${defaultProvider.provider}:${defaultProvider.model}` : null,
+          error_message: null,
+          created_at: new Date().toISOString(),
+          updated_at: new Date().toISOString()
+        },
+        runtime: {
+          sdkAvailable,
+          provider: defaultProvider,
+          mode: "dev_stub"
+        }
+      });
+      return;
+    }
+
+    const { data, error } = await req.supabase
+      .from("processing_jobs")
+      .insert({
+        user_id: req.user.id,
+        status: "queued",
+        source_type,
+        payload,
+        runtime_key: sdkAvailable ? `${defaultProvider.provider}:${defaultProvider.model}` : null
+      })
+      .select("*")
+      .single();
+
+    if (error) {
+      res.status(500).json({
+        success: false,
+        error: {
+          code: "DB_INSERT_FAILED",
+          message: error.message
+        }
+      });
+      return;
+    }
+
+    res.status(202).json({
+      success: true,
+      job: data,
+      runtime: {
+        sdkAvailable,
+        provider: defaultProvider
+      }
+    });
+  })
+);
+
+export default router;

package/api/src/routes/rules.ts

@@ -0,0 +1,11 @@
+import { Router } from "express";
+import { asyncHandler } from "../middleware/errorHandler.js";
+
+const router = Router();
+
+// GET /api/rules
+router.get("/", asyncHandler(async (req, res) => {
+    res.json({ rules: [] });
+}));
+
+export default router;