@realtimex/folio 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +20 -0
- package/README.md +63 -0
- package/api/server.ts +130 -0
- package/api/src/config/index.ts +96 -0
- package/api/src/middleware/auth.ts +128 -0
- package/api/src/middleware/errorHandler.ts +88 -0
- package/api/src/middleware/index.ts +4 -0
- package/api/src/middleware/rateLimit.ts +71 -0
- package/api/src/middleware/validation.ts +58 -0
- package/api/src/routes/accounts.ts +142 -0
- package/api/src/routes/baseline-config.ts +124 -0
- package/api/src/routes/chat.ts +154 -0
- package/api/src/routes/health.ts +61 -0
- package/api/src/routes/index.ts +35 -0
- package/api/src/routes/ingestions.ts +275 -0
- package/api/src/routes/migrate.ts +112 -0
- package/api/src/routes/policies.ts +121 -0
- package/api/src/routes/processing.ts +90 -0
- package/api/src/routes/rules.ts +11 -0
- package/api/src/routes/sdk.ts +100 -0
- package/api/src/routes/settings.ts +80 -0
- package/api/src/routes/setup.ts +389 -0
- package/api/src/routes/stats.ts +81 -0
- package/api/src/routes/tts.ts +190 -0
- package/api/src/services/BaselineConfigService.ts +208 -0
- package/api/src/services/ChatService.ts +204 -0
- package/api/src/services/GoogleDriveService.ts +331 -0
- package/api/src/services/GoogleSheetsService.ts +1107 -0
- package/api/src/services/IngestionService.ts +1187 -0
- package/api/src/services/ModelCapabilityService.ts +248 -0
- package/api/src/services/PolicyEngine.ts +1625 -0
- package/api/src/services/PolicyLearningService.ts +527 -0
- package/api/src/services/PolicyLoader.ts +249 -0
- package/api/src/services/RAGService.ts +391 -0
- package/api/src/services/SDKService.ts +249 -0
- package/api/src/services/supabase.ts +113 -0
- package/api/src/utils/Actuator.ts +284 -0
- package/api/src/utils/actions/ActionHandler.ts +34 -0
- package/api/src/utils/actions/AppendToGSheetAction.ts +260 -0
- package/api/src/utils/actions/AutoRenameAction.ts +58 -0
- package/api/src/utils/actions/CopyAction.ts +120 -0
- package/api/src/utils/actions/CopyToGDriveAction.ts +64 -0
- package/api/src/utils/actions/LogCsvAction.ts +48 -0
- package/api/src/utils/actions/NotifyAction.ts +39 -0
- package/api/src/utils/actions/RenameAction.ts +57 -0
- package/api/src/utils/actions/WebhookAction.ts +58 -0
- package/api/src/utils/actions/utils.ts +293 -0
- package/api/src/utils/llmResponse.ts +61 -0
- package/api/src/utils/logger.ts +67 -0
- package/bin/folio-deploy.js +12 -0
- package/bin/folio-setup.js +45 -0
- package/bin/folio.js +65 -0
- package/dist/api/server.js +106 -0
- package/dist/api/src/config/index.js +81 -0
- package/dist/api/src/middleware/auth.js +93 -0
- package/dist/api/src/middleware/errorHandler.js +73 -0
- package/dist/api/src/middleware/index.js +4 -0
- package/dist/api/src/middleware/rateLimit.js +43 -0
- package/dist/api/src/middleware/validation.js +54 -0
- package/dist/api/src/routes/accounts.js +110 -0
- package/dist/api/src/routes/baseline-config.js +91 -0
- package/dist/api/src/routes/chat.js +114 -0
- package/dist/api/src/routes/health.js +52 -0
- package/dist/api/src/routes/index.js +31 -0
- package/dist/api/src/routes/ingestions.js +207 -0
- package/dist/api/src/routes/migrate.js +91 -0
- package/dist/api/src/routes/policies.js +86 -0
- package/dist/api/src/routes/processing.js +75 -0
- package/dist/api/src/routes/rules.js +8 -0
- package/dist/api/src/routes/sdk.js +80 -0
- package/dist/api/src/routes/settings.js +68 -0
- package/dist/api/src/routes/setup.js +315 -0
- package/dist/api/src/routes/stats.js +62 -0
- package/dist/api/src/routes/tts.js +178 -0
- package/dist/api/src/services/BaselineConfigService.js +168 -0
- package/dist/api/src/services/ChatService.js +166 -0
- package/dist/api/src/services/GoogleDriveService.js +280 -0
- package/dist/api/src/services/GoogleSheetsService.js +795 -0
- package/dist/api/src/services/IngestionService.js +990 -0
- package/dist/api/src/services/ModelCapabilityService.js +179 -0
- package/dist/api/src/services/PolicyEngine.js +1353 -0
- package/dist/api/src/services/PolicyLearningService.js +397 -0
- package/dist/api/src/services/PolicyLoader.js +159 -0
- package/dist/api/src/services/RAGService.js +295 -0
- package/dist/api/src/services/SDKService.js +212 -0
- package/dist/api/src/services/supabase.js +72 -0
- package/dist/api/src/utils/Actuator.js +225 -0
- package/dist/api/src/utils/actions/ActionHandler.js +1 -0
- package/dist/api/src/utils/actions/AppendToGSheetAction.js +191 -0
- package/dist/api/src/utils/actions/AutoRenameAction.js +49 -0
- package/dist/api/src/utils/actions/CopyAction.js +112 -0
- package/dist/api/src/utils/actions/CopyToGDriveAction.js +55 -0
- package/dist/api/src/utils/actions/LogCsvAction.js +42 -0
- package/dist/api/src/utils/actions/NotifyAction.js +32 -0
- package/dist/api/src/utils/actions/RenameAction.js +51 -0
- package/dist/api/src/utils/actions/WebhookAction.js +51 -0
- package/dist/api/src/utils/actions/utils.js +237 -0
- package/dist/api/src/utils/llmResponse.js +63 -0
- package/dist/api/src/utils/logger.js +51 -0
- package/dist/assets/index-DzN8-j-e.css +1 -0
- package/dist/assets/index-Uy-ai3Dh.js +113 -0
- package/dist/favicon.svg +31 -0
- package/dist/folio-logo.svg +46 -0
- package/dist/index.html +14 -0
- package/docs-dev/FPE-spec.md +196 -0
- package/docs-dev/folio-prd.md +47 -0
- package/docs-dev/foundation-checklist.md +30 -0
- package/docs-dev/hybrid-routing-architecture.md +205 -0
- package/docs-dev/ingestion-engine.md +69 -0
- package/docs-dev/port-from-email-automator.md +32 -0
- package/docs-dev/tech-spec.md +98 -0
- package/index.html +13 -0
- package/package.json +101 -0
- package/public/favicon.svg +31 -0
- package/public/folio-logo.svg +46 -0
- package/scripts/dev-task.mjs +51 -0
- package/scripts/get-latest-migration-timestamp.mjs +34 -0
- package/scripts/migrate.sh +91 -0
- package/supabase/.temp/cli-latest +1 -0
- package/supabase/.temp/gotrue-version +1 -0
- package/supabase/.temp/pooler-url +1 -0
- package/supabase/.temp/postgres-version +1 -0
- package/supabase/.temp/project-ref +1 -0
- package/supabase/.temp/rest-version +1 -0
- package/supabase/.temp/storage-migration +1 -0
- package/supabase/.temp/storage-version +1 -0
- package/supabase/config.toml +64 -0
- package/supabase/functions/_shared/auth.ts +35 -0
- package/supabase/functions/_shared/cors.ts +12 -0
- package/supabase/functions/_shared/supabaseAdmin.ts +17 -0
- package/supabase/functions/api-v1-settings/index.ts +66 -0
- package/supabase/functions/setup/index.ts +91 -0
- package/supabase/migrations/20260223000000_initial_foundation.sql +136 -0
- package/supabase/migrations/20260223000001_add_migration_rpc.sql +10 -0
- package/supabase/migrations/20260224000002_add_init_state_view.sql +20 -0
- package/supabase/migrations/20260224000003_port_user_creation_parity.sql +139 -0
- package/supabase/migrations/20260224000004_add_avatars_storage.sql +26 -0
- package/supabase/migrations/20260224000005_add_tts_and_embed_settings.sql +24 -0
- package/supabase/migrations/20260224000006_add_policies_table.sql +48 -0
- package/supabase/migrations/20260224000007_fix_migration_rpc.sql +9 -0
- package/supabase/migrations/20260224000008_add_ingestions_table.sql +42 -0
- package/supabase/migrations/20260225000000_setup_compatible_mode.sql +119 -0
- package/supabase/migrations/20260225000001_restore_ingestions.sql +49 -0
- package/supabase/migrations/20260225000002_add_ingestion_trace.sql +2 -0
- package/supabase/migrations/20260225000003_add_baseline_configs.sql +35 -0
- package/supabase/migrations/20260226000000_add_processing_events.sql +26 -0
- package/supabase/migrations/20260226000001_add_ingestion_file_hash.sql +10 -0
- package/supabase/migrations/20260226000002_add_dynamic_rag.sql +150 -0
- package/supabase/migrations/20260226000003_add_ingestion_summary.sql +4 -0
- package/supabase/migrations/20260226000004_add_ingestion_tags.sql +7 -0
- package/supabase/migrations/20260226000005_add_chat_tables.sql +60 -0
- package/supabase/migrations/20260227000000_harden_chat_messages_rls.sql +25 -0
- package/supabase/migrations/20260228000000_add_vision_model_capabilities.sql +8 -0
- package/supabase/migrations/20260228000001_add_policy_match_feedback.sql +51 -0
- package/supabase/migrations/29991231235959_test_migration.sql +0 -0
- package/supabase/templates/confirmation.html +76 -0
- package/supabase/templates/email-change.html +76 -0
- package/supabase/templates/invite.html +72 -0
- package/supabase/templates/magic-link.html +68 -0
- package/supabase/templates/recovery.html +82 -0
- package/tsconfig.api.json +16 -0
- package/tsconfig.json +25 -0
- package/vite.config.ts +146 -0
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
import { Router } from "express";
|
|
2
|
+
import { asyncHandler } from "../middleware/errorHandler.js";
|
|
3
|
+
import { optionalAuth } from "../middleware/auth.js";
|
|
4
|
+
import { ChatService } from "../services/ChatService.js";
|
|
5
|
+
const router = Router();
|
|
6
|
+
// All chat routes require authentication
|
|
7
|
+
router.use(optionalAuth);
|
|
8
|
+
// GET /api/chat/sessions
|
|
9
|
+
router.get("/sessions", asyncHandler(async (req, res) => {
|
|
10
|
+
if (!req.supabase || !req.user) {
|
|
11
|
+
res.status(401).json({ success: false, error: "Authentication required" });
|
|
12
|
+
return;
|
|
13
|
+
}
|
|
14
|
+
const { data: sessions, error } = await req.supabase
|
|
15
|
+
.from("chat_sessions")
|
|
16
|
+
.select("*")
|
|
17
|
+
.eq("user_id", req.user.id)
|
|
18
|
+
.order("updated_at", { ascending: false });
|
|
19
|
+
if (error) {
|
|
20
|
+
res.status(500).json({ success: false, error: error.message });
|
|
21
|
+
return;
|
|
22
|
+
}
|
|
23
|
+
res.json({ success: true, sessions });
|
|
24
|
+
}));
|
|
25
|
+
// POST /api/chat/sessions
|
|
26
|
+
router.post("/sessions", asyncHandler(async (req, res) => {
|
|
27
|
+
if (!req.supabase || !req.user) {
|
|
28
|
+
res.status(401).json({ success: false, error: "Authentication required" });
|
|
29
|
+
return;
|
|
30
|
+
}
|
|
31
|
+
const { data: session, error } = await req.supabase
|
|
32
|
+
.from("chat_sessions")
|
|
33
|
+
.insert({ user_id: req.user.id, title: "New Conversation" })
|
|
34
|
+
.select("*")
|
|
35
|
+
.single();
|
|
36
|
+
if (error) {
|
|
37
|
+
res.status(500).json({ success: false, error: error.message });
|
|
38
|
+
return;
|
|
39
|
+
}
|
|
40
|
+
res.status(201).json({ success: true, session });
|
|
41
|
+
}));
|
|
42
|
+
// GET /api/chat/sessions/:id/messages
|
|
43
|
+
router.get("/sessions/:id/messages", asyncHandler(async (req, res) => {
|
|
44
|
+
if (!req.supabase || !req.user) {
|
|
45
|
+
res.status(401).json({ success: false, error: "Authentication required" });
|
|
46
|
+
return;
|
|
47
|
+
}
|
|
48
|
+
// Verify session belongs to user
|
|
49
|
+
const { data: sessionData } = await req.supabase
|
|
50
|
+
.from("chat_sessions")
|
|
51
|
+
.select("id")
|
|
52
|
+
.eq("id", req.params["id"])
|
|
53
|
+
.eq("user_id", req.user.id)
|
|
54
|
+
.maybeSingle();
|
|
55
|
+
if (!sessionData) {
|
|
56
|
+
res.status(404).json({ success: false, error: "Session not found" });
|
|
57
|
+
return;
|
|
58
|
+
}
|
|
59
|
+
const { data: messages, error } = await req.supabase
|
|
60
|
+
.from("chat_messages")
|
|
61
|
+
.select("*")
|
|
62
|
+
.eq("session_id", req.params["id"])
|
|
63
|
+
.eq("user_id", req.user.id)
|
|
64
|
+
.order("created_at", { ascending: true });
|
|
65
|
+
if (error) {
|
|
66
|
+
res.status(500).json({ success: false, error: error.message });
|
|
67
|
+
return;
|
|
68
|
+
}
|
|
69
|
+
res.json({ success: true, messages });
|
|
70
|
+
}));
|
|
71
|
+
// POST /api/chat/message
|
|
72
|
+
router.post("/message", asyncHandler(async (req, res) => {
|
|
73
|
+
if (!req.supabase || !req.user) {
|
|
74
|
+
res.status(401).json({ success: false, error: "Authentication required" });
|
|
75
|
+
return;
|
|
76
|
+
}
|
|
77
|
+
const { sessionId, content } = (req.body ?? {});
|
|
78
|
+
const normalizedSessionId = typeof sessionId === "string" ? sessionId : "";
|
|
79
|
+
const trimmedContent = typeof content === "string" ? content.trim() : "";
|
|
80
|
+
if (!normalizedSessionId || !trimmedContent) {
|
|
81
|
+
res.status(400).json({ success: false, error: "Missing sessionId or content" });
|
|
82
|
+
return;
|
|
83
|
+
}
|
|
84
|
+
// Verify session belongs to user
|
|
85
|
+
const { data: sessionData } = await req.supabase
|
|
86
|
+
.from("chat_sessions")
|
|
87
|
+
.select("id")
|
|
88
|
+
.eq("id", normalizedSessionId)
|
|
89
|
+
.eq("user_id", req.user.id)
|
|
90
|
+
.maybeSingle();
|
|
91
|
+
if (!sessionData) {
|
|
92
|
+
res.status(404).json({ success: false, error: "Session not found" });
|
|
93
|
+
return;
|
|
94
|
+
}
|
|
95
|
+
// Dynamically name session if it's the first message
|
|
96
|
+
const { count } = await req.supabase
|
|
97
|
+
.from("chat_messages")
|
|
98
|
+
.select("*", { count: 'exact', head: true })
|
|
99
|
+
.eq("session_id", normalizedSessionId)
|
|
100
|
+
.eq("user_id", req.user.id);
|
|
101
|
+
if (count === 0 && trimmedContent.length > 3) {
|
|
102
|
+
const title = trimmedContent.substring(0, 30) + (trimmedContent.length > 30 ? "..." : "");
|
|
103
|
+
await req.supabase.from("chat_sessions").update({ title }).eq("id", normalizedSessionId);
|
|
104
|
+
}
|
|
105
|
+
try {
|
|
106
|
+
const aiMessage = await ChatService.handleMessage(normalizedSessionId, req.user.id, trimmedContent, req.supabase);
|
|
107
|
+
res.json({ success: true, message: aiMessage });
|
|
108
|
+
}
|
|
109
|
+
catch (error) {
|
|
110
|
+
const message = error instanceof Error ? error.message : "Failed to process message";
|
|
111
|
+
res.status(500).json({ success: false, error: message });
|
|
112
|
+
}
|
|
113
|
+
}));
|
|
114
|
+
export default router;
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import { readFileSync } from "node:fs";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import { createClient } from "@supabase/supabase-js";
|
|
4
|
+
import { Router } from "express";
|
|
5
|
+
import { config } from "../config/index.js";
|
|
6
|
+
import { SDKService } from "../services/SDKService.js";
|
|
7
|
+
import { getServerSupabase, getSupabaseConfigFromHeaders } from "../services/supabase.js";
|
|
8
|
+
const router = Router();
|
|
9
|
+
let version = "0.0.0";
|
|
10
|
+
try {
|
|
11
|
+
const pkgPath = join(config.packageRoot, "package.json");
|
|
12
|
+
version = JSON.parse(readFileSync(pkgPath, "utf8")).version;
|
|
13
|
+
}
|
|
14
|
+
catch {
|
|
15
|
+
// fallback stays at default
|
|
16
|
+
}
|
|
17
|
+
router.get("/", async (req, res) => {
|
|
18
|
+
let supabase = getServerSupabase();
|
|
19
|
+
if (!supabase) {
|
|
20
|
+
const headerConfig = getSupabaseConfigFromHeaders(req.headers);
|
|
21
|
+
if (headerConfig) {
|
|
22
|
+
supabase = createClient(headerConfig.url, headerConfig.anonKey, {
|
|
23
|
+
auth: {
|
|
24
|
+
autoRefreshToken: false,
|
|
25
|
+
persistSession: false
|
|
26
|
+
}
|
|
27
|
+
});
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
let databaseStatus = "not_configured";
|
|
31
|
+
if (supabase) {
|
|
32
|
+
try {
|
|
33
|
+
const { error } = await supabase.from("user_settings").select("id").limit(1);
|
|
34
|
+
databaseStatus = error ? "error" : "connected";
|
|
35
|
+
}
|
|
36
|
+
catch {
|
|
37
|
+
databaseStatus = "error";
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
const sdkAvailable = await SDKService.isAvailable();
|
|
41
|
+
res.json({
|
|
42
|
+
status: "healthy",
|
|
43
|
+
timestamp: new Date().toISOString(),
|
|
44
|
+
version,
|
|
45
|
+
environment: config.nodeEnv,
|
|
46
|
+
services: {
|
|
47
|
+
database: databaseStatus,
|
|
48
|
+
realtimeXSdk: sdkAvailable ? "available" : "unavailable"
|
|
49
|
+
}
|
|
50
|
+
});
|
|
51
|
+
});
|
|
52
|
+
export default router;
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { Router } from "express";
|
|
2
|
+
import healthRoutes from "./health.js";
|
|
3
|
+
import migrateRoutes from "./migrate.js";
|
|
4
|
+
import processingRoutes from "./processing.js";
|
|
5
|
+
import setupRoutes from "./setup.js";
|
|
6
|
+
import ttsRoutes from "./tts.js";
|
|
7
|
+
import sdkRoutes from "./sdk.js";
|
|
8
|
+
import policiesRoutes from "./policies.js";
|
|
9
|
+
import ingestionsRoutes from "./ingestions.js";
|
|
10
|
+
import baselineConfigRoutes from "./baseline-config.js";
|
|
11
|
+
import accountsRoutes from "./accounts.js";
|
|
12
|
+
import settingsRoutes from "./settings.js";
|
|
13
|
+
import rulesRoutes from "./rules.js";
|
|
14
|
+
import chatRoutes from "./chat.js";
|
|
15
|
+
import statsRoutes from "./stats.js";
|
|
16
|
+
const router = Router();
|
|
17
|
+
router.use("/health", healthRoutes);
|
|
18
|
+
router.use("/migrate", migrateRoutes);
|
|
19
|
+
router.use("/setup", setupRoutes);
|
|
20
|
+
router.use("/processing", processingRoutes);
|
|
21
|
+
router.use("/tts", ttsRoutes);
|
|
22
|
+
router.use("/sdk", sdkRoutes);
|
|
23
|
+
router.use("/policies", policiesRoutes);
|
|
24
|
+
router.use("/ingestions", ingestionsRoutes);
|
|
25
|
+
router.use("/baseline-config", baselineConfigRoutes);
|
|
26
|
+
router.use("/accounts", accountsRoutes);
|
|
27
|
+
router.use("/settings", settingsRoutes);
|
|
28
|
+
router.use("/rules", rulesRoutes);
|
|
29
|
+
router.use("/chat", chatRoutes);
|
|
30
|
+
router.use("/stats", statsRoutes);
|
|
31
|
+
export default router;
|
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
import { Router } from "express";
|
|
2
|
+
import multer from "multer";
|
|
3
|
+
import os from "os";
|
|
4
|
+
import path from "path";
|
|
5
|
+
import fs from "fs/promises";
|
|
6
|
+
import crypto from "crypto";
|
|
7
|
+
import { asyncHandler } from "../middleware/errorHandler.js";
|
|
8
|
+
import { optionalAuth } from "../middleware/auth.js";
|
|
9
|
+
import { IngestionService } from "../services/IngestionService.js";
|
|
10
|
+
const router = Router();
|
|
11
|
+
const upload = multer({ storage: multer.memoryStorage(), limits: { fileSize: 20 * 1024 * 1024 } });
|
|
12
|
+
router.use(optionalAuth);
|
|
13
|
+
// GET /api/ingestions — list ingestions (paginated, searchable)
|
|
14
|
+
router.get("/", asyncHandler(async (req, res) => {
|
|
15
|
+
if (!req.supabase || !req.user) {
|
|
16
|
+
res.status(401).json({ success: false, error: "Authentication required" });
|
|
17
|
+
return;
|
|
18
|
+
}
|
|
19
|
+
const page = Math.max(1, parseInt(req.query["page"]) || 1);
|
|
20
|
+
const pageSize = Math.min(100, Math.max(1, parseInt(req.query["pageSize"]) || 20));
|
|
21
|
+
const query = req.query["q"]?.trim() || undefined;
|
|
22
|
+
const { ingestions, total } = await IngestionService.list(req.supabase, req.user.id, { page, pageSize, query });
|
|
23
|
+
res.json({ success: true, ingestions, total, page, pageSize });
|
|
24
|
+
}));
|
|
25
|
+
// GET /api/ingestions/:id — get single ingestion
|
|
26
|
+
router.get("/:id", asyncHandler(async (req, res) => {
|
|
27
|
+
if (!req.supabase || !req.user) {
|
|
28
|
+
res.status(401).json({ success: false, error: "Authentication required" });
|
|
29
|
+
return;
|
|
30
|
+
}
|
|
31
|
+
const ingestion = await IngestionService.get(req.params["id"], req.supabase, req.user.id);
|
|
32
|
+
if (!ingestion) {
|
|
33
|
+
res.status(404).json({ success: false, error: "Not found" });
|
|
34
|
+
return;
|
|
35
|
+
}
|
|
36
|
+
res.json({ success: true, ingestion });
|
|
37
|
+
}));
|
|
38
|
+
// POST /api/ingestions/upload — manual file upload
|
|
39
|
+
router.post("/upload", upload.single("file"), asyncHandler(async (req, res) => {
|
|
40
|
+
if (!req.supabase || !req.user) {
|
|
41
|
+
res.status(401).json({ success: false, error: "Authentication required" });
|
|
42
|
+
return;
|
|
43
|
+
}
|
|
44
|
+
const file = req.file;
|
|
45
|
+
if (!file) {
|
|
46
|
+
res.status(400).json({ success: false, error: "No file uploaded" });
|
|
47
|
+
return;
|
|
48
|
+
}
|
|
49
|
+
// Fetch user configured Storage Path (Dropzone)
|
|
50
|
+
const { data: settings } = await req.supabase
|
|
51
|
+
.from("user_settings")
|
|
52
|
+
.select("storage_path")
|
|
53
|
+
.eq("user_id", req.user.id)
|
|
54
|
+
.maybeSingle();
|
|
55
|
+
const dropzoneDir = settings?.storage_path || path.join(os.homedir(), ".realtimex", "folio", "dropzone");
|
|
56
|
+
await fs.mkdir(dropzoneDir, { recursive: true });
|
|
57
|
+
// Compute SHA-256 hash before writing — used for deduplication
|
|
58
|
+
const fileHash = crypto.createHash("sha256").update(file.buffer).digest("hex");
|
|
59
|
+
// Save physical file
|
|
60
|
+
const safeFilename = `${Date.now()}-${file.originalname.replace(/[^a-zA-Z0-9.-]/g, "_")}`;
|
|
61
|
+
const filePath = path.join(dropzoneDir, safeFilename);
|
|
62
|
+
await fs.writeFile(filePath, file.buffer);
|
|
63
|
+
// Extract text (mock extraction for pdfs, works for text files)
|
|
64
|
+
const content = file.buffer.toString("utf-8").replace(/\0/g, "").slice(0, 50_000);
|
|
65
|
+
const ingestion = await IngestionService.ingest({
|
|
66
|
+
supabase: req.supabase,
|
|
67
|
+
userId: req.user.id,
|
|
68
|
+
filename: file.originalname,
|
|
69
|
+
mimeType: file.mimetype,
|
|
70
|
+
fileSize: file.size,
|
|
71
|
+
source: "upload",
|
|
72
|
+
filePath,
|
|
73
|
+
content,
|
|
74
|
+
fileHash,
|
|
75
|
+
});
|
|
76
|
+
res.status(201).json({ success: true, ingestion });
|
|
77
|
+
}));
|
|
78
|
+
// POST /api/ingestions/:id/rerun — re-run processing
|
|
79
|
+
router.post("/:id/rerun", asyncHandler(async (req, res) => {
|
|
80
|
+
if (!req.supabase || !req.user) {
|
|
81
|
+
res.status(401).json({ success: false, error: "Authentication required" });
|
|
82
|
+
return;
|
|
83
|
+
}
|
|
84
|
+
const matched = await IngestionService.rerun(req.params["id"], req.supabase, req.user.id);
|
|
85
|
+
res.json({ success: true, matched });
|
|
86
|
+
}));
|
|
87
|
+
// POST /api/ingestions/:id/match — manually assign a policy and optionally learn from it
|
|
88
|
+
router.post("/:id/match", asyncHandler(async (req, res) => {
|
|
89
|
+
if (!req.supabase || !req.user) {
|
|
90
|
+
res.status(401).json({ success: false, error: "Authentication required" });
|
|
91
|
+
return;
|
|
92
|
+
}
|
|
93
|
+
const policyId = typeof req.body?.policy_id === "string" ? req.body.policy_id.trim() : "";
|
|
94
|
+
if (!policyId) {
|
|
95
|
+
res.status(400).json({ success: false, error: "policy_id is required" });
|
|
96
|
+
return;
|
|
97
|
+
}
|
|
98
|
+
const learn = req.body?.learn !== false;
|
|
99
|
+
const rerun = req.body?.rerun !== false;
|
|
100
|
+
const allowSideEffects = req.body?.allow_side_effects === true;
|
|
101
|
+
try {
|
|
102
|
+
const ingestion = await IngestionService.matchToPolicy(req.params["id"], policyId, req.supabase, req.user.id, {
|
|
103
|
+
learn,
|
|
104
|
+
rerun,
|
|
105
|
+
allowSideEffects,
|
|
106
|
+
});
|
|
107
|
+
res.json({ success: true, ingestion });
|
|
108
|
+
}
|
|
109
|
+
catch (error) {
|
|
110
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
111
|
+
if (/not found/i.test(message)) {
|
|
112
|
+
res.status(404).json({ success: false, error: message });
|
|
113
|
+
return;
|
|
114
|
+
}
|
|
115
|
+
if (/required|processing|disabled|cannot manually match|side-effect/i.test(message)) {
|
|
116
|
+
res.status(400).json({ success: false, error: message });
|
|
117
|
+
return;
|
|
118
|
+
}
|
|
119
|
+
throw error;
|
|
120
|
+
}
|
|
121
|
+
}));
|
|
122
|
+
// POST /api/ingestions/:id/refine-policy — suggest a refinement draft for a target policy
|
|
123
|
+
router.post("/:id/refine-policy", asyncHandler(async (req, res) => {
|
|
124
|
+
if (!req.supabase || !req.user) {
|
|
125
|
+
res.status(401).json({ success: false, error: "Authentication required" });
|
|
126
|
+
return;
|
|
127
|
+
}
|
|
128
|
+
const policyId = typeof req.body?.policy_id === "string" ? req.body.policy_id.trim() : "";
|
|
129
|
+
if (!policyId) {
|
|
130
|
+
res.status(400).json({ success: false, error: "policy_id is required" });
|
|
131
|
+
return;
|
|
132
|
+
}
|
|
133
|
+
try {
|
|
134
|
+
const suggestion = await IngestionService.suggestPolicyRefinement(req.params["id"], policyId, req.supabase, req.user.id, {
|
|
135
|
+
provider: typeof req.body?.provider === "string" ? req.body.provider : undefined,
|
|
136
|
+
model: typeof req.body?.model === "string" ? req.body.model : undefined,
|
|
137
|
+
});
|
|
138
|
+
res.json({ success: true, suggestion });
|
|
139
|
+
}
|
|
140
|
+
catch (error) {
|
|
141
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
142
|
+
if (/not found/i.test(message)) {
|
|
143
|
+
res.status(404).json({ success: false, error: message });
|
|
144
|
+
return;
|
|
145
|
+
}
|
|
146
|
+
if (/required|disabled/i.test(message)) {
|
|
147
|
+
res.status(400).json({ success: false, error: message });
|
|
148
|
+
return;
|
|
149
|
+
}
|
|
150
|
+
throw error;
|
|
151
|
+
}
|
|
152
|
+
}));
|
|
153
|
+
// POST /api/ingestions/:id/summarize — generate (or return cached) prose summary
|
|
154
|
+
router.post("/:id/summarize", asyncHandler(async (req, res) => {
|
|
155
|
+
if (!req.supabase || !req.user) {
|
|
156
|
+
res.status(401).json({ success: false, error: "Authentication required" });
|
|
157
|
+
return;
|
|
158
|
+
}
|
|
159
|
+
const { data: settingsRow } = await req.supabase
|
|
160
|
+
.from("user_settings")
|
|
161
|
+
.select("llm_provider, llm_model")
|
|
162
|
+
.eq("user_id", req.user.id)
|
|
163
|
+
.maybeSingle();
|
|
164
|
+
const llmSettings = {
|
|
165
|
+
llm_provider: settingsRow?.llm_provider ?? undefined,
|
|
166
|
+
llm_model: settingsRow?.llm_model ?? undefined,
|
|
167
|
+
};
|
|
168
|
+
const summary = await IngestionService.summarize(req.params["id"], req.supabase, req.user.id, llmSettings);
|
|
169
|
+
res.json({ success: true, summary });
|
|
170
|
+
}));
|
|
171
|
+
// PATCH /api/ingestions/:id/tags — replace tags array (human edits)
|
|
172
|
+
router.patch("/:id/tags", asyncHandler(async (req, res) => {
|
|
173
|
+
if (!req.supabase || !req.user) {
|
|
174
|
+
res.status(401).json({ success: false, error: "Authentication required" });
|
|
175
|
+
return;
|
|
176
|
+
}
|
|
177
|
+
const tags = req.body?.tags;
|
|
178
|
+
if (!Array.isArray(tags) || tags.some((t) => typeof t !== "string")) {
|
|
179
|
+
res.status(400).json({ success: false, error: "tags must be an array of strings" });
|
|
180
|
+
return;
|
|
181
|
+
}
|
|
182
|
+
const normalized = tags.map((t) => t.toLowerCase().trim()).filter(Boolean);
|
|
183
|
+
const { error } = await req.supabase
|
|
184
|
+
.from("ingestions")
|
|
185
|
+
.update({ tags: normalized })
|
|
186
|
+
.eq("id", req.params["id"])
|
|
187
|
+
.eq("user_id", req.user.id);
|
|
188
|
+
if (error) {
|
|
189
|
+
res.status(500).json({ success: false, error: error.message });
|
|
190
|
+
return;
|
|
191
|
+
}
|
|
192
|
+
res.json({ success: true, tags: normalized });
|
|
193
|
+
}));
|
|
194
|
+
// DELETE /api/ingestions/:id — delete
|
|
195
|
+
router.delete("/:id", asyncHandler(async (req, res) => {
|
|
196
|
+
if (!req.supabase || !req.user) {
|
|
197
|
+
res.status(401).json({ success: false, error: "Authentication required" });
|
|
198
|
+
return;
|
|
199
|
+
}
|
|
200
|
+
const deleted = await IngestionService.delete(req.params["id"], req.supabase, req.user.id);
|
|
201
|
+
if (!deleted) {
|
|
202
|
+
res.status(404).json({ success: false, error: "Not found" });
|
|
203
|
+
return;
|
|
204
|
+
}
|
|
205
|
+
res.json({ success: true });
|
|
206
|
+
}));
|
|
207
|
+
export default router;
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
import { spawn } from "node:child_process";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import { Router } from "express";
|
|
4
|
+
import { config } from "../config/index.js";
|
|
5
|
+
import { asyncHandler } from "../middleware/errorHandler.js";
|
|
6
|
+
import { schemas, validateBody } from "../middleware/validation.js";
|
|
7
|
+
import { createLogger } from "../utils/logger.js";
|
|
8
|
+
const router = Router();
|
|
9
|
+
const logger = createLogger("MigrateRoutes");
|
|
10
|
+
router.post("/", validateBody(schemas.migrate), asyncHandler(async (req, res) => {
|
|
11
|
+
const { projectRef, accessToken, anonKey } = req.body;
|
|
12
|
+
logger.info("Starting migration", { projectRef });
|
|
13
|
+
res.setHeader("Content-Type", "text/event-stream");
|
|
14
|
+
res.setHeader("Cache-Control", "no-cache");
|
|
15
|
+
res.setHeader("Connection", "keep-alive");
|
|
16
|
+
// @ts-ignore express typings may not include flushHeaders depending on version
|
|
17
|
+
res.flushHeaders?.();
|
|
18
|
+
const sendEvent = (type, data) => {
|
|
19
|
+
if (res.writableEnded) {
|
|
20
|
+
return;
|
|
21
|
+
}
|
|
22
|
+
res.write(`data: ${JSON.stringify({ type, data })}\n\n`);
|
|
23
|
+
};
|
|
24
|
+
const scriptPath = join(config.scriptsDir, "migrate.sh");
|
|
25
|
+
const child = spawn("bash", [scriptPath], {
|
|
26
|
+
env: {
|
|
27
|
+
...process.env,
|
|
28
|
+
SUPABASE_PROJECT_ID: projectRef,
|
|
29
|
+
SUPABASE_ACCESS_TOKEN: accessToken,
|
|
30
|
+
SUPABASE_ANON_KEY: anonKey || "",
|
|
31
|
+
SKIP_FUNCTIONS: process.env.SKIP_FUNCTIONS || "0"
|
|
32
|
+
},
|
|
33
|
+
cwd: config.rootDir
|
|
34
|
+
});
|
|
35
|
+
let clientDisconnected = false;
|
|
36
|
+
const stopChild = () => {
|
|
37
|
+
clientDisconnected = true;
|
|
38
|
+
if (!child.killed) {
|
|
39
|
+
child.kill("SIGTERM");
|
|
40
|
+
}
|
|
41
|
+
};
|
|
42
|
+
res.on("close", stopChild);
|
|
43
|
+
req.on("aborted", stopChild);
|
|
44
|
+
child.stdout.on("data", (chunk) => {
|
|
45
|
+
const lines = String(chunk)
|
|
46
|
+
.split("\n")
|
|
47
|
+
.filter((line) => line.trim().length > 0);
|
|
48
|
+
for (const line of lines) {
|
|
49
|
+
sendEvent("stdout", line);
|
|
50
|
+
}
|
|
51
|
+
});
|
|
52
|
+
child.stderr.on("data", (chunk) => {
|
|
53
|
+
const lines = String(chunk)
|
|
54
|
+
.split("\n")
|
|
55
|
+
.filter((line) => line.trim().length > 0);
|
|
56
|
+
for (const line of lines) {
|
|
57
|
+
sendEvent("stderr", line);
|
|
58
|
+
}
|
|
59
|
+
});
|
|
60
|
+
child.on("error", (error) => {
|
|
61
|
+
if (clientDisconnected) {
|
|
62
|
+
logger.info("Migration process ended after client disconnect", { projectRef });
|
|
63
|
+
return;
|
|
64
|
+
}
|
|
65
|
+
sendEvent("error", `Failed to run migration: ${error.message}`);
|
|
66
|
+
sendEvent("done", "failed");
|
|
67
|
+
if (!res.writableEnded) {
|
|
68
|
+
res.end();
|
|
69
|
+
}
|
|
70
|
+
});
|
|
71
|
+
child.on("close", (code, signal) => {
|
|
72
|
+
if (clientDisconnected) {
|
|
73
|
+
logger.info("Migration process stopped after client disconnect", { projectRef, code, signal });
|
|
74
|
+
return;
|
|
75
|
+
}
|
|
76
|
+
if (code === 0) {
|
|
77
|
+
sendEvent("info", "Migration completed successfully.");
|
|
78
|
+
sendEvent("done", "success");
|
|
79
|
+
}
|
|
80
|
+
else {
|
|
81
|
+
const codeLabel = code === null ? "null" : String(code);
|
|
82
|
+
const signalSuffix = signal ? ` (signal: ${signal})` : "";
|
|
83
|
+
sendEvent("error", `Migration failed with code ${codeLabel}${signalSuffix}`);
|
|
84
|
+
sendEvent("done", "failed");
|
|
85
|
+
}
|
|
86
|
+
if (!res.writableEnded) {
|
|
87
|
+
res.end();
|
|
88
|
+
}
|
|
89
|
+
});
|
|
90
|
+
}));
|
|
91
|
+
export default router;
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import { Router } from "express";
|
|
2
|
+
import { asyncHandler } from "../middleware/errorHandler.js";
|
|
3
|
+
import { optionalAuth } from "../middleware/auth.js";
|
|
4
|
+
import { PolicyLoader } from "../services/PolicyLoader.js";
|
|
5
|
+
import { PolicyEngine } from "../services/PolicyEngine.js";
|
|
6
|
+
import { PolicyLearningService } from "../services/PolicyLearningService.js";
|
|
7
|
+
const router = Router();
|
|
8
|
+
// Use optionalAuth on all routes — sets req.supabase + req.user if a valid
|
|
9
|
+
// bearer token is present. Falls through without error if no token.
|
|
10
|
+
router.use(optionalAuth);
|
|
11
|
+
// GET /api/policies — list all loaded policies
|
|
12
|
+
router.get("/", asyncHandler(async (req, res) => {
|
|
13
|
+
const policies = await PolicyLoader.load(false, req.supabase);
|
|
14
|
+
if (!req.supabase || !req.user || policies.length === 0) {
|
|
15
|
+
res.json({ success: true, policies });
|
|
16
|
+
return;
|
|
17
|
+
}
|
|
18
|
+
const stats = await PolicyLearningService.getPolicyLearningStats({
|
|
19
|
+
supabase: req.supabase,
|
|
20
|
+
userId: req.user.id,
|
|
21
|
+
policyIds: policies.map((policy) => policy.metadata.id),
|
|
22
|
+
});
|
|
23
|
+
const enrichedPolicies = policies.map((policy) => {
|
|
24
|
+
const stat = stats[policy.metadata.id];
|
|
25
|
+
return {
|
|
26
|
+
...policy,
|
|
27
|
+
metadata: {
|
|
28
|
+
...policy.metadata,
|
|
29
|
+
learning_examples: stat?.samples ?? 0,
|
|
30
|
+
learning_last_at: stat?.lastSampleAt ?? null,
|
|
31
|
+
},
|
|
32
|
+
};
|
|
33
|
+
});
|
|
34
|
+
res.json({ success: true, policies: enrichedPolicies });
|
|
35
|
+
}));
|
|
36
|
+
// POST /api/policies — save a new policy
|
|
37
|
+
router.post("/", asyncHandler(async (req, res) => {
|
|
38
|
+
const policy = req.body;
|
|
39
|
+
if (!PolicyLoader.validate(policy)) {
|
|
40
|
+
res.status(400).json({ success: false, error: "Invalid policy schema" });
|
|
41
|
+
return;
|
|
42
|
+
}
|
|
43
|
+
const filePath = await PolicyLoader.save(policy, req.supabase, req.user?.id);
|
|
44
|
+
res.status(201).json({ success: true, filePath });
|
|
45
|
+
}));
|
|
46
|
+
// DELETE /api/policies/:id — delete a policy by ID
|
|
47
|
+
router.delete("/:id", asyncHandler(async (req, res) => {
|
|
48
|
+
const deleted = await PolicyLoader.delete(req.params["id"], req.supabase, req.user?.id);
|
|
49
|
+
if (!deleted) {
|
|
50
|
+
res.status(404).json({ success: false, error: "Policy not found" });
|
|
51
|
+
return;
|
|
52
|
+
}
|
|
53
|
+
res.json({ success: true });
|
|
54
|
+
}));
|
|
55
|
+
// PATCH /api/policies/:id — partial update (enabled toggle, metadata fields)
|
|
56
|
+
router.patch("/:id", asyncHandler(async (req, res) => {
|
|
57
|
+
const { enabled, name, description, tags, priority } = req.body;
|
|
58
|
+
await PolicyLoader.patch(req.params["id"], { enabled, name, description, tags, priority }, req.supabase, req.user?.id);
|
|
59
|
+
res.json({ success: true });
|
|
60
|
+
}));
|
|
61
|
+
// POST /api/policies/reload — force cache invalidation
|
|
62
|
+
router.post("/reload", asyncHandler(async (req, res) => {
|
|
63
|
+
PolicyLoader.invalidateCache();
|
|
64
|
+
const policies = await PolicyLoader.load(true, req.supabase);
|
|
65
|
+
res.json({ success: true, count: policies.length });
|
|
66
|
+
}));
|
|
67
|
+
// POST /api/policies/synthesize — NL → Policy via LLM
|
|
68
|
+
router.post("/synthesize", asyncHandler(async (req, res) => {
|
|
69
|
+
const { description, provider, model } = req.body;
|
|
70
|
+
if (!description || typeof description !== "string") {
|
|
71
|
+
res.status(400).json({ success: false, error: "description is required" });
|
|
72
|
+
return;
|
|
73
|
+
}
|
|
74
|
+
const result = await PolicyEngine.synthesizeFromNL(description, {
|
|
75
|
+
provider,
|
|
76
|
+
model,
|
|
77
|
+
userId: req.user?.id,
|
|
78
|
+
supabase: req.supabase,
|
|
79
|
+
});
|
|
80
|
+
if (!result.policy) {
|
|
81
|
+
res.status(503).json({ success: false, error: result.error ?? "Synthesis failed. SDK may be unavailable." });
|
|
82
|
+
return;
|
|
83
|
+
}
|
|
84
|
+
res.json({ success: true, policy: result.policy, warning: result.error });
|
|
85
|
+
}));
|
|
86
|
+
export default router;
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import { Router } from "express";
|
|
2
|
+
import { config } from "../config/index.js";
|
|
3
|
+
import { authMiddleware } from "../middleware/auth.js";
|
|
4
|
+
import { asyncHandler } from "../middleware/errorHandler.js";
|
|
5
|
+
import { validateBody, schemas } from "../middleware/validation.js";
|
|
6
|
+
import { SDKService } from "../services/SDKService.js";
|
|
7
|
+
const router = Router();
|
|
8
|
+
router.post("/dispatch", authMiddleware, validateBody(schemas.dispatchProcessing), asyncHandler(async (req, res) => {
|
|
9
|
+
if (!req.user || !req.supabase) {
|
|
10
|
+
res.status(401).json({
|
|
11
|
+
success: false,
|
|
12
|
+
error: {
|
|
13
|
+
code: "AUTH_REQUIRED",
|
|
14
|
+
message: "Authentication required"
|
|
15
|
+
}
|
|
16
|
+
});
|
|
17
|
+
return;
|
|
18
|
+
}
|
|
19
|
+
const sdkAvailable = await SDKService.isAvailable();
|
|
20
|
+
const defaultProvider = await SDKService.getDefaultChatProvider();
|
|
21
|
+
const { source_type, payload } = req.body;
|
|
22
|
+
// Foundation-only dev path: avoid FK coupling to auth.users when auth is disabled.
|
|
23
|
+
if (config.security.disableAuth && !config.isProduction) {
|
|
24
|
+
res.status(202).json({
|
|
25
|
+
success: true,
|
|
26
|
+
job: {
|
|
27
|
+
id: `dev-${Date.now()}`,
|
|
28
|
+
user_id: req.user.id,
|
|
29
|
+
status: "queued",
|
|
30
|
+
source_type,
|
|
31
|
+
payload,
|
|
32
|
+
runtime_key: sdkAvailable ? `${defaultProvider.provider}:${defaultProvider.model}` : null,
|
|
33
|
+
error_message: null,
|
|
34
|
+
created_at: new Date().toISOString(),
|
|
35
|
+
updated_at: new Date().toISOString()
|
|
36
|
+
},
|
|
37
|
+
runtime: {
|
|
38
|
+
sdkAvailable,
|
|
39
|
+
provider: defaultProvider,
|
|
40
|
+
mode: "dev_stub"
|
|
41
|
+
}
|
|
42
|
+
});
|
|
43
|
+
return;
|
|
44
|
+
}
|
|
45
|
+
const { data, error } = await req.supabase
|
|
46
|
+
.from("processing_jobs")
|
|
47
|
+
.insert({
|
|
48
|
+
user_id: req.user.id,
|
|
49
|
+
status: "queued",
|
|
50
|
+
source_type,
|
|
51
|
+
payload,
|
|
52
|
+
runtime_key: sdkAvailable ? `${defaultProvider.provider}:${defaultProvider.model}` : null
|
|
53
|
+
})
|
|
54
|
+
.select("*")
|
|
55
|
+
.single();
|
|
56
|
+
if (error) {
|
|
57
|
+
res.status(500).json({
|
|
58
|
+
success: false,
|
|
59
|
+
error: {
|
|
60
|
+
code: "DB_INSERT_FAILED",
|
|
61
|
+
message: error.message
|
|
62
|
+
}
|
|
63
|
+
});
|
|
64
|
+
return;
|
|
65
|
+
}
|
|
66
|
+
res.status(202).json({
|
|
67
|
+
success: true,
|
|
68
|
+
job: data,
|
|
69
|
+
runtime: {
|
|
70
|
+
sdkAvailable,
|
|
71
|
+
provider: defaultProvider
|
|
72
|
+
}
|
|
73
|
+
});
|
|
74
|
+
}));
|
|
75
|
+
export default router;
|