@realtimex/folio 0.1.2

package/api/src/utils/actions/WebhookAction.ts

@@ -0,0 +1,58 @@
+import { ActionHandler, ActionContext, ActionResult } from "./ActionHandler.js";
+import { pickString, interpolate } from "./utils.js";
+import { Actuator } from "../Actuator.js";
+
+export class WebhookAction implements ActionHandler {
+    async execute(context: ActionContext): Promise<ActionResult> {
+        const { action, variables, data, userId, ingestionId, supabase } = context;
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        const webhookUrlTemplate = pickString(action as any, "url");
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        const webhookPayloadTemplate = pickString(action as any, "payload");
+
+        if (!webhookUrlTemplate || !webhookPayloadTemplate) {
+            return {
+                success: false,
+                logs: [],
+                trace: [{ timestamp: new Date().toISOString(), step: "Webhook failed: missing url or payload" }],
+                error: "Webhook action requires 'url' and 'payload' configs"
+            };
+        }
+
+        const url = interpolate(webhookUrlTemplate, variables, data);
+        const payloadStr = interpolate(webhookPayloadTemplate, variables, data);
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        let payload: any;
+
+        try {
+            payload = JSON.parse(payloadStr);
+        // eslint-disable-next-line @typescript-eslint/no-unused-vars
+        } catch (e) {
+            return {
+                success: false,
+                logs: [],
+                trace: [{ timestamp: new Date().toISOString(), step: "Webhook failed: invalid JSON payload" }],
+                error: "Webhook payload must be valid JSON"
+            };
+        }
+
+        await fetch(url, {
+            method: "POST",
+            headers: { "Content-Type": "application/json" },
+            body: JSON.stringify(payload),
+        });
+
+        const trace = [{
+            timestamp: new Date().toISOString(),
+            step: `Webhook payload sent to ${url}`,
+            details: { url, payload }
+        }];
+        Actuator.logEvent(ingestionId, userId, "action", "Action Execution", { action: "webhook", url }, supabase);
+
+        return {
+            success: true,
+            logs: [`Logged via webhook`],
+            trace
+        };
+    }
+}

package/api/src/utils/actions/utils.ts

@@ -0,0 +1,293 @@
+import type { ExtractField } from "../../services/PolicyLoader.js";
+import { createLogger } from "../logger.js";
+
+const logger = createLogger("ActionUtils");
+
+export type ExtractedData = Record<string, unknown>;
+
+export type ActionInput = {
+    type: string;
+    config?: Record<string, unknown>;
+    pattern?: string;
+    destination?: string;
+    path?: string;
+    columns?: string[] | string;
+    message?: string;
+    url?: string;
+    payload?: string;
+};
+
+// ─── Variable Interpolation ────────────────────────────────────────────────
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+    return value !== null && typeof value === "object" && !Array.isArray(value);
+}
+
+function maybeParseJson(value: unknown): unknown {
+    if (typeof value !== "string") return undefined;
+    const trimmed = value.trim();
+    if (!trimmed) return undefined;
+    if (!((trimmed.startsWith("{") && trimmed.endsWith("}")) || (trimmed.startsWith("[") && trimmed.endsWith("]")))) {
+        return undefined;
+    }
+    try {
+        return JSON.parse(trimmed);
+    } catch {
+        return undefined;
+    }
+}
+
+function tokenizePath(path: string): Array<string | number> {
+    const tokens: Array<string | number> = [];
+    let i = 0;
+
+    while (i < path.length) {
+        const ch = path[i];
+        if (ch === ".") {
+            i += 1;
+            continue;
+        }
+
+        if (ch === "[") {
+            const end = path.indexOf("]", i + 1);
+            if (end < 0) break;
+            const raw = path.slice(i + 1, end).trim();
+            if (/^\d+$/.test(raw)) {
+                tokens.push(Number(raw));
+            } else {
+                const unquoted = raw.replace(/^["']|["']$/g, "");
+                if (unquoted) tokens.push(unquoted);
+            }
+            i = end + 1;
+            continue;
+        }
+
+        let j = i;
+        while (j < path.length && path[j] !== "." && path[j] !== "[") {
+            j += 1;
+        }
+        const token = path.slice(i, j).trim();
+        if (token) tokens.push(token);
+        i = j;
+    }
+
+    return tokens;
+}
+
+function toTemplateString(value: unknown): string | undefined {
+    if (value == null) return undefined;
+    if (typeof value === "string") return value;
+    if (typeof value === "number" || typeof value === "boolean") return String(value);
+    try {
+        return JSON.stringify(value);
+    } catch {
+        return String(value);
+    }
+}
+
+export function getNestedVariable(
+    keyPath: string,
+    vars: Record<string, string>,
+    data?: ExtractedData
+): string | undefined {
+    const key = keyPath.trim();
+    if (!key) return undefined;
+
+    if (vars[key] !== undefined) {
+        return vars[key];
+    }
+
+    const root: Record<string, unknown> = {
+        ...(data ?? {}),
+        ...vars,
+    };
+
+    const tokens = tokenizePath(key);
+    if (tokens.length === 0) return undefined;
+
+    let current: unknown = root;
+
+    for (let i = 0; i < tokens.length; i += 1) {
+        if (typeof current === "string") {
+            const parsed = maybeParseJson(current);
+            if (parsed !== undefined) {
+                current = parsed;
+            }
+        }
+
+        const token = tokens[i];
+        if (Array.isArray(current)) {
+            if (typeof token !== "number") return undefined;
+            current = current[token];
+            continue;
+        }
+
+        if (!isRecord(current)) {
+            return undefined;
+        }
+
+        if (typeof token === "number") {
+            current = current[String(token)];
+            continue;
+        }
+
+        const lookupToken =
+            i === 0 && token === "enrichment" && current["_enrichment"] !== undefined
+                ? "_enrichment"
+                : token;
+        current = current[lookupToken];
+    }
+
+    return toTemplateString(current);
+}
+
+export function interpolate(
+    template: string,
+    vars: Record<string, string>,
+    data?: ExtractedData
+): string {
+    return template.replace(/\{([^{}]+)\}/g, (match, rawKey: string) => {
+        const resolved = getNestedVariable(rawKey, vars, data);
+        return resolved ?? match;
+    });
+}
+
+/**
+ * Derive computed variables from extracted data using transformer definitions.
+ */
+export function deriveVariables(
+    data: ExtractedData,
+    fields: ExtractField[]
+): Record<string, string> {
+    const vars: Record<string, string> = {};
+
+    // Populate raw extracted values as strings
+    for (const [k, v] of Object.entries(data)) {
+        const serialized = toTemplateString(v);
+        if (serialized !== undefined) vars[k] = serialized;
+    }
+
+    // Run transformers
+    for (const field of fields) {
+        if (!field.transformers) continue;
+        const rawValue = vars[field.key];
+        if (!rawValue) continue;
+
+        for (const t of field.transformers) {
+            try {
+                if (t.name === "get_year") {
+                    vars[t.as] = new Date(rawValue).getFullYear().toString();
+                } else if (t.name === "get_month_name") {
+                    vars[t.as] = new Date(rawValue).toLocaleString("en-US", { month: "long" });
+                } else if (t.name === "get_month") {
+                    vars[t.as] = String(new Date(rawValue).getMonth() + 1).padStart(2, "0");
+                }
+            } catch {
+                logger.warn(`Transformer '${t.name}' failed for key '${field.key}'`);
+            }
+        }
+    }
+
+    return vars;
+}
+
+export function pickString(action: ActionInput, key: string): string | undefined {
+    const value = action.config?.[key];
+    if (typeof value === "string" && value.trim().length > 0) {
+        return value;
+    }
+
+    const legacyValue = (action as Record<string, unknown>)[key];
+    if (typeof legacyValue === "string" && legacyValue.trim().length > 0) {
+        return legacyValue;
+    }
+
+    return undefined;
+}
+
+// ─── Filename Helpers ──────────────────────────────────────────────────────
+
+/**
+ * Build a filename stem from extracted metadata when suggested_filename is unavailable.
+ * Format: YYYY-MM-DD_Issuer_DocType  (any missing parts are simply omitted)
+ */
+export function deriveNameFromVariables(variables: Record<string, string>): string | null {
+    const parts: string[] = [];
+
+    if (variables.date) {
+        const d = new Date(variables.date);
+        if (!isNaN(d.getTime())) {
+            const yyyy = d.getFullYear();
+            const MM = String(d.getMonth() + 1).padStart(2, "0");
+            const dd = String(d.getDate()).padStart(2, "0");
+            parts.push(`${yyyy}-${MM}-${dd}`);
+        }
+    }
+
+    if (variables.issuer) {
+        parts.push(variables.issuer.replace(/[^a-zA-Z0-9]+/g, "-").replace(/^-|-$/g, ""));
+    }
+
+    if (variables.document_type) {
+        parts.push(variables.document_type.replace(/\s+/g, "-").replace(/[^a-zA-Z0-9-]+/g, ""));
+    }
+
+    if (variables.amount || variables.total_amount) {
+        const raw = (variables.amount ?? variables.total_amount).replace(/[^0-9.$€£]/g, "");
+        if (raw) parts.push(raw);
+    }
+
+    return parts.length > 0 ? parts.join("_") : null;
+}
+
+/**
+ * Resolve a final filename given the action's `filename` config field.
+ *
+ * Modes:
+ *   undefined / "" / "original" → keep original stem + ext
+ *   "auto"                       → AI suggested_filename → derived → originalStem, then + ext
+ *   any other string             → treat as a {variable} interpolation pattern
+ */
+export function resolveFilename(
+    filenameConfig: string | undefined,
+    variables: Record<string, string>,
+    originalStem: string,
+    ext: string,
+    data?: ExtractedData
+): string {
+    if (!filenameConfig || filenameConfig === "original") {
+        return originalStem + ext;
+    }
+
+    if (filenameConfig === "auto") {
+        const smart =
+            deriveNameFromVariables(variables) ||
+            variables.suggested_filename?.trim() ||
+            originalStem;
+        return smart.endsWith(ext) ? smart : smart + ext;
+    }
+
+    // Custom interpolation pattern
+    const interpolated = interpolate(filenameConfig, variables, data);
+    return interpolated.endsWith(ext) ? interpolated : interpolated + ext;
+}
+
+export function pickColumns(action: ActionInput, fallback: string[]): string[] {
+    const value = action.config?.columns;
+    if (Array.isArray(value)) {
+        return value.map((item) => String(item).trim()).filter(Boolean);
+    }
+    if (typeof value === "string") {
+        return value.split(",").map((item) => item.trim()).filter(Boolean);
+    }
+
+    const legacyColumns = action.columns;
+    if (Array.isArray(legacyColumns)) {
+        return legacyColumns.map((item) => String(item).trim()).filter(Boolean);
+    }
+    if (typeof legacyColumns === "string") {
+        return legacyColumns.split(",").map((item) => item.trim()).filter(Boolean);
+    }
+
+    return fallback;
+}

package/api/src/utils/llmResponse.ts

@@ -0,0 +1,61 @@
+export function normalizeLlmContent(content: unknown): string {
+    if (typeof content === "string") return content;
+    if (typeof content === "number" || typeof content === "boolean") return String(content);
+
+    if (Array.isArray(content)) {
+        return content
+            .map((part) => {
+                if (part && typeof part === "object") {
+                    const obj = part as Record<string, unknown>;
+                    if (typeof obj.text === "string") return obj.text;
+                    if (typeof obj.content === "string") return obj.content;
+                }
+                return normalizeLlmContent(part);
+            })
+            .filter(Boolean)
+            .join("\n");
+    }
+
+    if (content && typeof content === "object") {
+        const obj = content as Record<string, unknown>;
+        if (typeof obj.text === "string") return obj.text;
+        if (typeof obj.content === "string") return obj.content;
+        if ("content" in obj) return normalizeLlmContent(obj.content);
+        try {
+            return JSON.stringify(obj);
+        } catch {
+            return String(obj);
+        }
+    }
+
+    return "";
+}
+
+/**
+ * Robustly extracts the text payload from various SDK adapter response shapes.
+ */
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export function extractLlmResponse(result: any): string {
+    if (!result) return "";
+
+    const candidates = [
+        result.response?.content,
+        result.message?.content,
+        result.content,
+        result.text,
+        result.choices?.[0]?.message?.content,
+        result.result,
+        result.output,
+    ];
+
+    for (const candidate of candidates) {
+        const normalized = normalizeLlmContent(candidate);
+        if (normalized) return normalized;
+    }
+
+    return "";
+}
+
+export function previewLlmText(raw: string, maxChars = 240): string {
+    return raw.replace(/\s+/g, " ").trim().slice(0, maxChars);
+}

package/api/src/utils/logger.ts

@@ -0,0 +1,67 @@
+import type { SupabaseClient } from "@supabase/supabase-js";
+
+interface LogData {
+  [key: string]: unknown;
+}
+
+class LoggerCore {
+  private static persistence:
+    | {
+        supabase: SupabaseClient;
+        userId: string;
+      }
+    | null = null;
+
+  static setPersistence(supabase: SupabaseClient, userId: string) {
+    this.persistence = { supabase, userId };
+  }
+
+  static clearPersistence() {
+    this.persistence = null;
+  }
+
+  static async persist(level: string, scope: string, message: string, data?: LogData) {
+    if (!this.persistence) {
+      return;
+    }
+
+    try {
+      await this.persistence.supabase.from("system_logs").insert({
+        user_id: this.persistence.userId,
+        level,
+        scope,
+        message,
+        metadata: data || {}
+      });
+    } catch {
+      // persistence is best-effort and should never crash request flow
+    }
+  }
+}
+
+export const Logger = LoggerCore;
+
+export function createLogger(scope: string) {
+  function write(level: "debug" | "info" | "warn" | "error", message: string, data?: LogData) {
+    const line = `[${scope}] ${message}`;
+
+    if (level === "error") {
+      console.error(line, data || "");
+    } else if (level === "warn") {
+      console.warn(line, data || "");
+    } else if (level === "info") {
+      console.info(line, data || "");
+    } else {
+      console.debug(line, data || "");
+    }
+
+    void Logger.persist(level, scope, message, data);
+  }
+
+  return {
+    debug: (message: string, data?: LogData) => write("debug", message, data),
+    info: (message: string, data?: LogData) => write("info", message, data),
+    warn: (message: string, data?: LogData) => write("warn", message, data),
+    error: (message: string, data?: LogData) => write("error", message, data)
+  };
+}

package/bin/folio-deploy.js

@@ -0,0 +1,12 @@
+#!/usr/bin/env node
+
+import { spawn } from "node:child_process";
+
+const child = spawn("bash", ["./scripts/migrate.sh"], {
+  stdio: "inherit",
+  env: process.env
+});
+
+child.on("close", (code) => {
+  process.exit(code || 0);
+});

package/bin/folio-setup.js

@@ -0,0 +1,45 @@
+#!/usr/bin/env node
+
+import { existsSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import readline from "node:readline";
+
+const rl = readline.createInterface({
+  input: process.stdin,
+  output: process.stdout
+});
+
+function question(prompt) {
+  return new Promise((resolve) => rl.question(prompt, resolve));
+}
+
+async function setup() {
+  const envPath = join(process.cwd(), ".env");
+
+  if (existsSync(envPath)) {
+    const overwrite = await question("⚠️  .env exists. Overwrite? (y/N): ");
+    if (overwrite.toLowerCase() !== "y") {
+      console.log("Setup cancelled.");
+      rl.close();
+      return;
+    }
+  }
+
+  const supabaseUrl = await question("Supabase URL: ");
+  const supabaseAnonKey = await question("Supabase anon key: ");
+  const port = (await question("API port [3006]: ")) || "3006";
+
+  const env = `VITE_SUPABASE_URL=${supabaseUrl}\nVITE_SUPABASE_ANON_KEY=${supabaseAnonKey}\nVITE_API_URL=http://localhost:${port}\nPORT=${port}\nDISABLE_AUTH=true\nJWT_SECRET=dev-secret-change-in-production\n`;
+
+  writeFileSync(envPath, env);
+
+  console.log("✅ .env written");
+  console.log("Next:\n  1) npm install\n  2) npm run migrate\n  3) npm run dev:api\n  4) npm run dev");
+  rl.close();
+}
+
+setup().catch((error) => {
+  console.error("❌ Setup failed:", error.message);
+  rl.close();
+  process.exit(1);
+});

package/bin/folio.js

@@ -0,0 +1,65 @@
+#!/usr/bin/env node
+
+import { spawn } from "node:child_process";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { existsSync } from "node:fs";
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+
+const args = process.argv.slice(2);
+
+let port = "5176";
+const portIndex = args.indexOf("--port");
+if (portIndex !== -1 && args[portIndex + 1]) {
+  port = args[portIndex + 1];
+}
+
+const noUi = args.includes("--no-ui");
+
+console.log("🚀 Folio starting...");
+console.log(`📡 Port: ${port}`);
+if (noUi) console.log("🖥️  Mode: No-UI");
+
+const distServerPath = join(__dirname, "..", "dist", "api", "server.js");
+const sourceServerPath = join(__dirname, "..", "api", "server.ts");
+const distPath = join(__dirname, "..", "dist");
+
+let execPath = process.execPath;
+let execArgs = [];
+
+if (existsSync(distServerPath)) {
+  execArgs = [distServerPath, ...args];
+} else if (existsSync(sourceServerPath)) {
+  execPath = "npx";
+  execArgs = ["tsx", sourceServerPath, ...args];
+  console.log("🧪 Compiled server not found. Falling back to source (tsx).");
+  if (!existsSync(join(distPath, "index.html")) && !noUi) {
+    console.log("⚠️  dist/index.html not found. UI will not be served in single-port mode until you run `npm run build`.");
+  }
+} else {
+  console.error("❌ Could not find Folio server entry point.");
+  process.exit(1);
+}
+
+const child = spawn(execPath, execArgs, {
+  stdio: "inherit",
+  env: {
+    ...process.env,
+    PORT: port,
+    ELECTRON_STATIC_PATH: distPath
+  }
+});
+
+child.on("error", (error) => {
+  console.error("❌ Failed to start Folio:", error.message);
+  process.exit(1);
+});
+
+child.on("close", (code) => {
+  process.exit(code || 0);
+});
+
+process.on("SIGINT", () => child.kill("SIGINT"));
+process.on("SIGTERM", () => child.kill("SIGTERM"));