@realtimex/folio 0.1.2

package/api/src/utils/actions/AppendToGSheetAction.ts

@@ -0,0 +1,260 @@
+import { ActionHandler, ActionContext, ActionResult } from "./ActionHandler.js";
+import { GoogleSheetsService } from "../../services/GoogleSheetsService.js";
+import { pickString, pickColumns, interpolate } from "./utils.js";
+import { Actuator } from "../Actuator.js";
+
+const HEADER_ALIASES: Record<string, string[]> = {
+    amount: ["total_amount", "amount", "amount_due"],
+    total: ["total_amount", "amount", "amount_due"],
+    total_amount: ["amount", "amount_due"],
+    vendor: ["issuer", "merchant", "store_name", "seller"],
+    merchant: ["issuer", "vendor", "store_name", "seller"],
+    supplier: ["issuer", "vendor", "merchant"],
+    store: ["issuer", "vendor", "merchant", "store_name"],
+    document: ["document_type"],
+    type: ["document_type"],
+    category: ["document_type"],
+    issued_on: ["date"],
+    invoice_date: ["date"],
+    receipt_date: ["date"],
+    image_link: ["file_url", "drive_file_url", "file_link", "document_link"],
+    file_link: ["file_url", "drive_file_url", "image_link", "document_link"],
+    document_link: ["file_url", "drive_file_url", "file_link", "image_link"],
+    link: ["file_url", "drive_file_url", "file_link", "document_link", "image_link"],
+};
+
+type HeaderDropdown = {
+    strict: boolean;
+    allowedValues: string[];
+};
+
+function normalizeKey(value: string): string {
+    return value.toLowerCase().replace(/[^a-z0-9]+/g, "_").replace(/^_+|_+$/g, "");
+}
+
+function buildNormalizedVariableLookup(variables: Record<string, string>): Record<string, string> {
+    const lookup: Record<string, string> = {};
+    for (const [key, value] of Object.entries(variables)) {
+        const normalized = normalizeKey(key);
+        if (normalized && !(normalized in lookup)) {
+            lookup[normalized] = value;
+        }
+    }
+    return lookup;
+}
+
+function resolveHeaderValue(
+    header: string,
+    variables: Record<string, string>,
+    normalizedVariables: Record<string, string>
+): string {
+    const trimmed = header.trim();
+    if (!trimmed) return "";
+
+    if (variables[trimmed] !== undefined) {
+        return variables[trimmed];
+    }
+
+    const normalizedHeader = normalizeKey(trimmed);
+    if (!normalizedHeader) return "";
+
+    if (normalizedVariables[normalizedHeader] !== undefined) {
+        return normalizedVariables[normalizedHeader];
+    }
+
+    const aliases = HEADER_ALIASES[normalizedHeader] ?? [];
+    for (const alias of aliases) {
+        if (normalizedVariables[alias] !== undefined) {
+            return normalizedVariables[alias];
+        }
+    }
+
+    return "";
+}
+
+function isDropdownValueAllowed(value: string, allowedValues: string[]): boolean {
+    const candidate = value.trim().toLowerCase();
+    if (!candidate) return true;
+    return allowedValues.some((allowed) => allowed.trim().toLowerCase() === candidate);
+}
+
+function applyStrictDropdownGuards(
+    headers: string[],
+    values: string[],
+    headerDropdowns: Array<HeaderDropdown | null>
+): { values: string[]; skippedColumns: string[] } {
+    const nextValues = [...values];
+    const skippedColumns: string[] = [];
+    const maxColumns = Math.min(headers.length, nextValues.length);
+
+    for (let index = 0; index < maxColumns; index += 1) {
+        const dropdown = headerDropdowns[index];
+        if (!dropdown || dropdown.strict !== true || dropdown.allowedValues.length === 0) {
+            continue;
+        }
+
+        const currentValue = nextValues[index] ?? "";
+        if (!currentValue.trim()) continue;
+        if (isDropdownValueAllowed(currentValue, dropdown.allowedValues)) continue;
+
+        nextValues[index] = "";
+        skippedColumns.push(headers[index] || `column_${index + 1}`);
+    }
+
+    return { values: nextValues, skippedColumns };
+}
+
+export class AppendToGSheetAction implements ActionHandler {
+    async execute(context: ActionContext): Promise<ActionResult> {
+        const { ingestionId, userId, supabase } = context;
+        const result: ActionResult = {
+            success: true,
+            logs: [],
+            trace: [],
+        };
+
+        const spreadsheetReference =
+            pickString(context.action, "spreadsheet_id") ??
+            pickString(context.action, "spreadsheet_url");
+
+        if (!spreadsheetReference) {
+            result.success = false;
+            result.error = "Missing required Action configuration: 'spreadsheet_id'";
+            return result;
+        }
+
+        const configuredRange = pickString(context.action, "range");
+        const columnTemplates = pickColumns(context.action, []);
+        let rangeToAppend = configuredRange;
+        let values: string[] = [];
+        let usedDynamicMapping = false;
+
+        if (columnTemplates.length > 0) {
+            values = columnTemplates.map((template) => interpolate(template, context.variables, context.data));
+
+            const templateResult = await GoogleSheetsService.resolveTemplate(
+                context.userId,
+                spreadsheetReference,
+                configuredRange,
+                context.supabase
+            );
+            if (templateResult.success && (templateResult.headers?.length ?? 0) > 0) {
+                const dropdownGuard = applyStrictDropdownGuards(
+                    templateResult.headers ?? [],
+                    values,
+                    (templateResult.headerDropdowns ?? []) as Array<HeaderDropdown | null>
+                );
+                values = dropdownGuard.values;
+                if (dropdownGuard.skippedColumns.length > 0) {
+                    result.logs.push(
+                        `Left ${dropdownGuard.skippedColumns.length} strict dropdown column(s) blank for human selection.`
+                    );
+                    result.trace.push({
+                        timestamp: new Date().toISOString(),
+                        step: "Adjusted strict dropdown columns",
+                        details: {
+                            skippedColumns: dropdownGuard.skippedColumns,
+                        },
+                    });
+                }
+            }
+        } else {
+            const templateResult = await GoogleSheetsService.resolveTemplate(
+                context.userId,
+                spreadsheetReference,
+                configuredRange,
+                context.supabase
+            );
+
+            if (!templateResult.success) {
+                result.success = false;
+                result.error = templateResult.error || "Failed to read Google Sheet template headers";
+                if (templateResult.errorDetails) {
+                    result.errorDetails = templateResult.errorDetails;
+                }
+                return result;
+            }
+
+            const headers = templateResult.headers ?? [];
+            if (headers.length === 0) {
+                result.success = false;
+                result.error = "Google Sheet template has no header row. Add column names in row 1.";
+                return result;
+            }
+
+            rangeToAppend = templateResult.range;
+            const normalizedVariables = buildNormalizedVariableLookup(context.variables);
+            values = headers.map((header) => resolveHeaderValue(header, context.variables, normalizedVariables));
+            const dropdownGuard = applyStrictDropdownGuards(
+                headers,
+                values,
+                (templateResult.headerDropdowns ?? []) as Array<HeaderDropdown | null>
+            );
+            values = dropdownGuard.values;
+            if (dropdownGuard.skippedColumns.length > 0) {
+                result.logs.push(
+                    `Left ${dropdownGuard.skippedColumns.length} strict dropdown column(s) blank for human selection.`
+                );
+                result.trace.push({
+                    timestamp: new Date().toISOString(),
+                    step: "Adjusted strict dropdown columns",
+                    details: {
+                        skippedColumns: dropdownGuard.skippedColumns,
+                    },
+                });
+            }
+            usedDynamicMapping = true;
+
+            result.trace.push({
+                timestamp: new Date().toISOString(),
+                step: "Resolved Google Sheet template",
+                details: {
+                    spreadsheetId: templateResult.spreadsheetId,
+                    range: templateResult.range,
+                    headersCount: headers.length,
+                },
+            });
+
+            if (values.every((value) => value.trim().length === 0)) {
+                result.success = false;
+                result.error = "Unable to map extracted fields to Google Sheet headers. Provide explicit columns mapping or align header names with extracted keys.";
+                return result;
+            }
+        }
+
+        result.trace.push({
+            timestamp: new Date().toISOString(),
+            step: "Appending to Google Sheet",
+            details: { spreadsheetReference, range: rangeToAppend || "Sheet1", columnsCount: values.length, dynamicMapping: usedDynamicMapping },
+        });
+
+        const appendResult = await GoogleSheetsService.appendRow(
+            context.userId,
+            spreadsheetReference,
+            rangeToAppend,
+            values,
+            context.supabase
+        );
+
+        if (!appendResult.success) {
+            result.success = false;
+            result.error = appendResult.error;
+            if (appendResult.errorDetails) {
+                result.errorDetails = appendResult.errorDetails;
+            }
+            return result;
+        }
+
+        Actuator.logEvent(ingestionId, userId, "action", "Action Execution", {
+            action: "append_to_google_sheet",
+            spreadsheetId: appendResult.spreadsheetId ?? spreadsheetReference,
+            range: appendResult.range ?? rangeToAppend ?? "Sheet1",
+            columnsCount: values.length,
+            dynamicMapping: usedDynamicMapping,
+        }, supabase);
+
+        result.logs.push(`Appended ${values.length} columns to Google Sheet ${appendResult.spreadsheetId ?? spreadsheetReference} at ${appendResult.range ?? rangeToAppend ?? "Sheet1"}`);
+
+        return result;
+    }
+}

package/api/src/utils/actions/AutoRenameAction.ts

@@ -0,0 +1,58 @@
+import fs from "node:fs";
+import path from "node:path";
+import { ActionHandler, ActionContext, ActionResult } from "./ActionHandler.js";
+import { resolveFilename } from "./utils.js";
+import { Actuator } from "../Actuator.js";
+import { getServiceRoleSupabase } from "../../services/supabase.js";
+import { createLogger } from "../logger.js";
+
+const logger = createLogger("AutoRenameAction");
+
+export class AutoRenameAction implements ActionHandler {
+    async execute(context: ActionContext): Promise<ActionResult> {
+        const { file, variables, data, userId, ingestionId, supabase } = context;
+
+        const ext = path.extname(file.path);
+        const dir = path.dirname(file.path);
+        const stem = file.name.slice(0, file.name.length - ext.length);
+
+        logger.info("AutoRename variables", {
+            suggested_filename: variables.suggested_filename ?? "(missing)",
+            date: variables.date ?? "(missing)",
+            issuer: variables.issuer ?? "(missing)",
+            document_type: variables.document_type ?? "(missing)",
+        });
+
+        const newName = resolveFilename("auto", variables, stem, ext, data);
+        const newPath = path.join(dir, newName);
+
+        await new Promise<void>((resolve, reject) => {
+            fs.rename(file.path, newPath, (err) => {
+                if (err) reject(err);
+                else resolve();
+            });
+        });
+
+        const trace = [{
+            timestamp: new Date().toISOString(),
+            step: `Auto-Renamed file to ${newName}`,
+            details: { original: file.name, new: newName }
+        }];
+
+        logger.info(`AutoRename: '${file.name}' → '${newName}'`);
+        Actuator.logEvent(ingestionId, userId, "action", "Action Execution", { action: "auto_rename", original: file.name, new: newName }, supabase);
+
+        // Update DB so re-runs don't break
+        const db = supabase ?? getServiceRoleSupabase();
+        if (db) {
+            await db.from("ingestions").update({ storage_path: newPath, filename: newName }).eq("id", ingestionId);
+        }
+
+        return {
+            success: true,
+            newFileState: { path: newPath, name: newName },
+            logs: [`Auto-Renamed to '${newName}'`],
+            trace
+        };
+    }
+}

package/api/src/utils/actions/CopyAction.ts

@@ -0,0 +1,120 @@
+import fs from "node:fs";
+import path from "node:path";
+import { ActionHandler, ActionContext, ActionResult } from "./ActionHandler.js";
+import { pickString, interpolate, resolveFilename } from "./utils.js";
+import { Actuator } from "../Actuator.js";
+import { GoogleDriveService } from "../../services/GoogleDriveService.js";
+
+export class CopyAction implements ActionHandler {
+    async execute(context: ActionContext): Promise<ActionResult> {
+        const { action, file, variables, data, userId, ingestionId, supabase } = context;
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        const destination = pickString(action as any, "destination");
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        const pattern = pickString(action as any, "pattern");
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        const filenameConfig = pickString(action as any, "filename");
+
+        if (!destination) {
+            return {
+                success: false,
+                logs: [],
+                trace: [{ timestamp: new Date().toISOString(), step: "Copy failed: missing destination" }],
+                error: "Copy action requires a 'destination' config"
+            };
+        }
+
+        const destDir = interpolate(destination, variables, data);
+
+        // Support legacy gdrive:// destinations before copy_to_gdrive existed.
+        if (destDir.startsWith("gdrive://")) {
+            const folderPath = destDir.slice("gdrive://".length) || undefined;
+            let gdriveFileName: string | undefined;
+            if (filenameConfig) {
+                const ext = path.extname(file.name);
+                const stem = file.name.slice(0, file.name.length - ext.length);
+                gdriveFileName = resolveFilename(filenameConfig, variables, stem, ext, data);
+            }
+            const uploadResult = await GoogleDriveService.uploadFile(userId, file.path, folderPath, supabase, gdriveFileName);
+            if (!uploadResult.success) {
+                return {
+                    success: false,
+                    logs: [],
+                    trace: [{ timestamp: new Date().toISOString(), step: "Copy to Google Drive failed", details: { error: uploadResult.error } }],
+                    error: uploadResult.error || "Failed to upload to Google Drive"
+                };
+            }
+
+            const trace = [{
+                timestamp: new Date().toISOString(),
+                step: "Copied file to Google Drive",
+                details: { original: file.path, driveFileId: uploadResult.fileId, destinationFolderId: folderPath }
+            }];
+            Actuator.logEvent(ingestionId, userId, "action", "Action Execution", { action: "copy_to_gdrive", destinationFolderId: folderPath ?? null, fileId: uploadResult.fileId }, supabase);
+
+            const fileId = uploadResult.fileId as string;
+            const fileUrl = `https://drive.google.com/file/d/${fileId}/view`;
+            const outputs: Record<string, unknown> = {
+                provider: "google_drive",
+                file_id: fileId,
+                file_url: fileUrl,
+                drive_file_id: fileId,
+                drive_file_url: fileUrl,
+                destination_folder_id: folderPath ?? null,
+                uploaded_file_name: gdriveFileName ?? file.name,
+            };
+            if (/\.(jpg|jpeg|png|webp|gif|bmp|tiff|tif|heic)$/i.test(file.name)) {
+                outputs.image_link = fileUrl;
+            }
+
+            return {
+                success: true,
+                logs: [`Copied to Google Drive (ID: ${uploadResult.fileId})`],
+                trace,
+                outputs,
+            };
+        }
+
+        fs.mkdirSync(destDir, { recursive: true });
+
+        const ext = path.extname(file.name);
+        const stem = file.name.slice(0, file.name.length - ext.length);
+        let newName: string;
+        if (filenameConfig) {
+            newName = resolveFilename(filenameConfig, variables, stem, ext, data);
+        } else if (pattern) {
+            // Backward-compat: treat pattern as a filename template
+            newName = interpolate(pattern, variables, data);
+            if (!newName.endsWith(ext)) newName += ext;
+        } else {
+            newName = file.name;
+        }
+
+        const newPath = path.join(destDir, newName);
+        await new Promise<void>((resolve, reject) => {
+            fs.copyFile(file.path, newPath, (err) => {
+                if (err) reject(err);
+                else resolve();
+            });
+        });
+
+        const trace = [{
+            timestamp: new Date().toISOString(),
+            step: `Copied file to ${newPath}`,
+            details: { original: file.path, copy: newPath }
+        }];
+        Actuator.logEvent(ingestionId, userId, "action", "Action Execution", { action: "copy", destination: destDir, newName }, supabase);
+
+        return {
+            success: true,
+            logs: [`Copied to '${newPath}'`],
+            trace,
+            outputs: {
+                provider: "local",
+                destination: destDir,
+                copied_path: newPath,
+                copied_name: newName,
+            },
+        };
+    }
+}

package/api/src/utils/actions/CopyToGDriveAction.ts

@@ -0,0 +1,64 @@
+import path from "node:path";
+import { ActionHandler, ActionContext, ActionResult } from "./ActionHandler.js";
+import { pickString, interpolate, resolveFilename } from "./utils.js";
+import { Actuator } from "../Actuator.js";
+import { GoogleDriveService } from "../../services/GoogleDriveService.js";
+
+export class CopyToGDriveAction implements ActionHandler {
+    async execute(context: ActionContext): Promise<ActionResult> {
+        const { action, file, variables, data, userId, ingestionId, supabase } = context;
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        const destination = pickString(action as any, "destination");
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        const filenameConfig = pickString(action as any, "filename");
+
+        const destDirId = destination ? interpolate(destination, variables, data) : undefined;
+
+        let resolvedFileName: string | undefined;
+        if (filenameConfig) {
+            const ext = path.extname(file.name);
+            const stem = file.name.slice(0, file.name.length - ext.length);
+            resolvedFileName = resolveFilename(filenameConfig, variables, stem, ext, data);
+        }
+
+        const uploadResult = await GoogleDriveService.uploadFile(userId, file.path, destDirId, supabase, resolvedFileName);
+
+        if (!uploadResult.success) {
+            return {
+                success: false,
+                logs: [],
+                trace: [{ timestamp: new Date().toISOString(), step: "Copy to Google Drive failed", details: { error: uploadResult.error } }],
+                error: uploadResult.error || "Failed to upload to Google Drive"
+            };
+        }
+
+        const trace = [{
+            timestamp: new Date().toISOString(),
+            step: `Copied file to Google Drive`,
+            details: { original: file.path, driveFileId: uploadResult.fileId, destinationFolderId: destDirId }
+        }];
+        Actuator.logEvent(ingestionId, userId, "action", "Action Execution", { action: "copy_to_gdrive", destinationFolderId: destDirId ?? null, fileId: uploadResult.fileId }, supabase);
+
+        const fileId = uploadResult.fileId as string;
+        const fileUrl = `https://drive.google.com/file/d/${fileId}/view`;
+        const outputs: Record<string, unknown> = {
+            provider: "google_drive",
+            file_id: fileId,
+            file_url: fileUrl,
+            drive_file_id: fileId,
+            drive_file_url: fileUrl,
+            destination_folder_id: destDirId ?? null,
+            uploaded_file_name: resolvedFileName ?? file.name,
+        };
+        if (/\.(jpg|jpeg|png|webp|gif|bmp|tiff|tif|heic)$/i.test(file.name)) {
+            outputs.image_link = fileUrl;
+        }
+
+        return {
+            success: true,
+            logs: [`Copied to Google Drive (ID: ${uploadResult.fileId})`],
+            trace,
+            outputs,
+        };
+    }
+}

package/api/src/utils/actions/LogCsvAction.ts

@@ -0,0 +1,48 @@
+import fs from "node:fs";
+import path from "node:path";
+import { ActionHandler, ActionContext, ActionResult } from "./ActionHandler.js";
+import { pickString, pickColumns, interpolate, getNestedVariable } from "./utils.js";
+import { Actuator } from "../Actuator.js";
+
+export class LogCsvAction implements ActionHandler {
+    async execute(context: ActionContext): Promise<ActionResult> {
+        const { action, variables, data, userId, ingestionId, supabase } = context;
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        const csvPathTemplate = pickString(action as any, "path");
+
+        if (!csvPathTemplate) {
+            return {
+                success: false,
+                logs: [],
+                trace: [{ timestamp: new Date().toISOString(), step: "Log CSV failed: missing path" }],
+                error: "Log CSV action requires a 'path' config"
+            };
+        }
+
+        const csvPath = interpolate(csvPathTemplate, variables, data);
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        const cols = pickColumns(action as any, Object.keys(data));
+        const row = cols.map((c) => getNestedVariable(c, variables, data) ?? "").join(",") + "\n";
+        const header = cols.join(",") + "\n";
+
+        if (!fs.existsSync(csvPath)) {
+            fs.mkdirSync(path.dirname(csvPath), { recursive: true });
+            fs.writeFileSync(csvPath, header + row, "utf-8");
+        } else {
+            fs.appendFileSync(csvPath, row, "utf-8");
+        }
+
+        const trace = [{
+            timestamp: new Date().toISOString(),
+            step: "Executed log_csv action",
+            details: { csvPath, cols }
+        }];
+        Actuator.logEvent(ingestionId, userId, "action", "Action Execution", { action: "log_csv", csvPath, cols }, supabase);
+
+        return {
+            success: true,
+            logs: [`Logged CSV → ${csvPath}`],
+            trace
+        };
+    }
+}

package/api/src/utils/actions/NotifyAction.ts

@@ -0,0 +1,39 @@
+import { ActionHandler, ActionContext, ActionResult } from "./ActionHandler.js";
+import { pickString, interpolate } from "./utils.js";
+import { Actuator } from "../Actuator.js";
+import { createLogger } from "../logger.js";
+
+const logger = createLogger("NotifyAction");
+
+export class NotifyAction implements ActionHandler {
+    async execute(context: ActionContext): Promise<ActionResult> {
+        const { action, variables, data, userId, ingestionId, supabase } = context;
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        const messageTemplate = pickString(action as any, "message");
+
+        if (!messageTemplate) {
+            return {
+                success: false,
+                logs: [],
+                trace: [{ timestamp: new Date().toISOString(), step: "Notify failed: missing message" }],
+                error: "Notify action requires a 'message' config"
+            };
+        }
+
+        const msg = interpolate(messageTemplate, variables, data);
+        logger.info(`[NOTIFY] ${msg}`);
+
+        const trace = [{
+            timestamp: new Date().toISOString(),
+            step: "Executed notify action",
+            details: { message: msg }
+        }];
+        Actuator.logEvent(ingestionId, userId, "action", "Action Execution", { action: "notify", message: msg }, supabase);
+
+        return {
+            success: true,
+            logs: [`Notified: ${msg}`],
+            trace
+        };
+    }
+}

package/api/src/utils/actions/RenameAction.ts

@@ -0,0 +1,57 @@
+import fs from "node:fs";
+import path from "node:path";
+import { ActionHandler, ActionContext, ActionResult } from "./ActionHandler.js";
+import { pickString, interpolate } from "./utils.js";
+import { Actuator } from "../Actuator.js";
+import { getServiceRoleSupabase } from "../../services/supabase.js";
+
+export class RenameAction implements ActionHandler {
+    async execute(context: ActionContext): Promise<ActionResult> {
+        const { action, file, variables, data, userId, ingestionId, supabase } = context;
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        const pattern = pickString(action as any, "pattern");
+
+        if (!pattern) {
+            return {
+                success: false,
+                logs: [],
+                trace: [{ timestamp: new Date().toISOString(), step: "Rename failed: missing pattern" }],
+                error: "Rename action requires a 'pattern' config"
+            };
+        }
+
+        const ext = path.extname(file.path);
+        const dir = path.dirname(file.path);
+        let newName = interpolate(pattern, variables, data);
+        if (!newName.endsWith(ext)) newName += ext;
+        const newPath = path.join(dir, newName);
+
+        await new Promise<void>((resolve, reject) => {
+            fs.rename(file.path, newPath, (err) => {
+                if (err) reject(err);
+                else resolve();
+            });
+        });
+
+        const trace = [{
+            timestamp: new Date().toISOString(),
+            step: `Renamed file to ${newName}`,
+            details: { original: file.name, new: newName }
+        }];
+
+        Actuator.logEvent(ingestionId, userId, "action", "Action Execution", { action: "rename", original: file.name, new: newName }, supabase);
+
+        // Update DB so re-runs don't break
+        const db = supabase ?? getServiceRoleSupabase();
+        if (db) {
+            await db.from("ingestions").update({ storage_path: newPath, filename: newName }).eq("id", ingestionId);
+        }
+
+        return {
+            success: true,
+            newFileState: { path: newPath, name: newName },
+            logs: [`Renamed to '${newName}'`],
+            trace
+        };
+    }
+}