@realtimex/folio 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +20 -0
- package/README.md +63 -0
- package/api/server.ts +130 -0
- package/api/src/config/index.ts +96 -0
- package/api/src/middleware/auth.ts +128 -0
- package/api/src/middleware/errorHandler.ts +88 -0
- package/api/src/middleware/index.ts +4 -0
- package/api/src/middleware/rateLimit.ts +71 -0
- package/api/src/middleware/validation.ts +58 -0
- package/api/src/routes/accounts.ts +142 -0
- package/api/src/routes/baseline-config.ts +124 -0
- package/api/src/routes/chat.ts +154 -0
- package/api/src/routes/health.ts +61 -0
- package/api/src/routes/index.ts +35 -0
- package/api/src/routes/ingestions.ts +275 -0
- package/api/src/routes/migrate.ts +112 -0
- package/api/src/routes/policies.ts +121 -0
- package/api/src/routes/processing.ts +90 -0
- package/api/src/routes/rules.ts +11 -0
- package/api/src/routes/sdk.ts +100 -0
- package/api/src/routes/settings.ts +80 -0
- package/api/src/routes/setup.ts +389 -0
- package/api/src/routes/stats.ts +81 -0
- package/api/src/routes/tts.ts +190 -0
- package/api/src/services/BaselineConfigService.ts +208 -0
- package/api/src/services/ChatService.ts +204 -0
- package/api/src/services/GoogleDriveService.ts +331 -0
- package/api/src/services/GoogleSheetsService.ts +1107 -0
- package/api/src/services/IngestionService.ts +1187 -0
- package/api/src/services/ModelCapabilityService.ts +248 -0
- package/api/src/services/PolicyEngine.ts +1625 -0
- package/api/src/services/PolicyLearningService.ts +527 -0
- package/api/src/services/PolicyLoader.ts +249 -0
- package/api/src/services/RAGService.ts +391 -0
- package/api/src/services/SDKService.ts +249 -0
- package/api/src/services/supabase.ts +113 -0
- package/api/src/utils/Actuator.ts +284 -0
- package/api/src/utils/actions/ActionHandler.ts +34 -0
- package/api/src/utils/actions/AppendToGSheetAction.ts +260 -0
- package/api/src/utils/actions/AutoRenameAction.ts +58 -0
- package/api/src/utils/actions/CopyAction.ts +120 -0
- package/api/src/utils/actions/CopyToGDriveAction.ts +64 -0
- package/api/src/utils/actions/LogCsvAction.ts +48 -0
- package/api/src/utils/actions/NotifyAction.ts +39 -0
- package/api/src/utils/actions/RenameAction.ts +57 -0
- package/api/src/utils/actions/WebhookAction.ts +58 -0
- package/api/src/utils/actions/utils.ts +293 -0
- package/api/src/utils/llmResponse.ts +61 -0
- package/api/src/utils/logger.ts +67 -0
- package/bin/folio-deploy.js +12 -0
- package/bin/folio-setup.js +45 -0
- package/bin/folio.js +65 -0
- package/dist/api/server.js +106 -0
- package/dist/api/src/config/index.js +81 -0
- package/dist/api/src/middleware/auth.js +93 -0
- package/dist/api/src/middleware/errorHandler.js +73 -0
- package/dist/api/src/middleware/index.js +4 -0
- package/dist/api/src/middleware/rateLimit.js +43 -0
- package/dist/api/src/middleware/validation.js +54 -0
- package/dist/api/src/routes/accounts.js +110 -0
- package/dist/api/src/routes/baseline-config.js +91 -0
- package/dist/api/src/routes/chat.js +114 -0
- package/dist/api/src/routes/health.js +52 -0
- package/dist/api/src/routes/index.js +31 -0
- package/dist/api/src/routes/ingestions.js +207 -0
- package/dist/api/src/routes/migrate.js +91 -0
- package/dist/api/src/routes/policies.js +86 -0
- package/dist/api/src/routes/processing.js +75 -0
- package/dist/api/src/routes/rules.js +8 -0
- package/dist/api/src/routes/sdk.js +80 -0
- package/dist/api/src/routes/settings.js +68 -0
- package/dist/api/src/routes/setup.js +315 -0
- package/dist/api/src/routes/stats.js +62 -0
- package/dist/api/src/routes/tts.js +178 -0
- package/dist/api/src/services/BaselineConfigService.js +168 -0
- package/dist/api/src/services/ChatService.js +166 -0
- package/dist/api/src/services/GoogleDriveService.js +280 -0
- package/dist/api/src/services/GoogleSheetsService.js +795 -0
- package/dist/api/src/services/IngestionService.js +990 -0
- package/dist/api/src/services/ModelCapabilityService.js +179 -0
- package/dist/api/src/services/PolicyEngine.js +1353 -0
- package/dist/api/src/services/PolicyLearningService.js +397 -0
- package/dist/api/src/services/PolicyLoader.js +159 -0
- package/dist/api/src/services/RAGService.js +295 -0
- package/dist/api/src/services/SDKService.js +212 -0
- package/dist/api/src/services/supabase.js +72 -0
- package/dist/api/src/utils/Actuator.js +225 -0
- package/dist/api/src/utils/actions/ActionHandler.js +1 -0
- package/dist/api/src/utils/actions/AppendToGSheetAction.js +191 -0
- package/dist/api/src/utils/actions/AutoRenameAction.js +49 -0
- package/dist/api/src/utils/actions/CopyAction.js +112 -0
- package/dist/api/src/utils/actions/CopyToGDriveAction.js +55 -0
- package/dist/api/src/utils/actions/LogCsvAction.js +42 -0
- package/dist/api/src/utils/actions/NotifyAction.js +32 -0
- package/dist/api/src/utils/actions/RenameAction.js +51 -0
- package/dist/api/src/utils/actions/WebhookAction.js +51 -0
- package/dist/api/src/utils/actions/utils.js +237 -0
- package/dist/api/src/utils/llmResponse.js +63 -0
- package/dist/api/src/utils/logger.js +51 -0
- package/dist/assets/index-DzN8-j-e.css +1 -0
- package/dist/assets/index-Uy-ai3Dh.js +113 -0
- package/dist/favicon.svg +31 -0
- package/dist/folio-logo.svg +46 -0
- package/dist/index.html +14 -0
- package/docs-dev/FPE-spec.md +196 -0
- package/docs-dev/folio-prd.md +47 -0
- package/docs-dev/foundation-checklist.md +30 -0
- package/docs-dev/hybrid-routing-architecture.md +205 -0
- package/docs-dev/ingestion-engine.md +69 -0
- package/docs-dev/port-from-email-automator.md +32 -0
- package/docs-dev/tech-spec.md +98 -0
- package/index.html +13 -0
- package/package.json +101 -0
- package/public/favicon.svg +31 -0
- package/public/folio-logo.svg +46 -0
- package/scripts/dev-task.mjs +51 -0
- package/scripts/get-latest-migration-timestamp.mjs +34 -0
- package/scripts/migrate.sh +91 -0
- package/supabase/.temp/cli-latest +1 -0
- package/supabase/.temp/gotrue-version +1 -0
- package/supabase/.temp/pooler-url +1 -0
- package/supabase/.temp/postgres-version +1 -0
- package/supabase/.temp/project-ref +1 -0
- package/supabase/.temp/rest-version +1 -0
- package/supabase/.temp/storage-migration +1 -0
- package/supabase/.temp/storage-version +1 -0
- package/supabase/config.toml +64 -0
- package/supabase/functions/_shared/auth.ts +35 -0
- package/supabase/functions/_shared/cors.ts +12 -0
- package/supabase/functions/_shared/supabaseAdmin.ts +17 -0
- package/supabase/functions/api-v1-settings/index.ts +66 -0
- package/supabase/functions/setup/index.ts +91 -0
- package/supabase/migrations/20260223000000_initial_foundation.sql +136 -0
- package/supabase/migrations/20260223000001_add_migration_rpc.sql +10 -0
- package/supabase/migrations/20260224000002_add_init_state_view.sql +20 -0
- package/supabase/migrations/20260224000003_port_user_creation_parity.sql +139 -0
- package/supabase/migrations/20260224000004_add_avatars_storage.sql +26 -0
- package/supabase/migrations/20260224000005_add_tts_and_embed_settings.sql +24 -0
- package/supabase/migrations/20260224000006_add_policies_table.sql +48 -0
- package/supabase/migrations/20260224000007_fix_migration_rpc.sql +9 -0
- package/supabase/migrations/20260224000008_add_ingestions_table.sql +42 -0
- package/supabase/migrations/20260225000000_setup_compatible_mode.sql +119 -0
- package/supabase/migrations/20260225000001_restore_ingestions.sql +49 -0
- package/supabase/migrations/20260225000002_add_ingestion_trace.sql +2 -0
- package/supabase/migrations/20260225000003_add_baseline_configs.sql +35 -0
- package/supabase/migrations/20260226000000_add_processing_events.sql +26 -0
- package/supabase/migrations/20260226000001_add_ingestion_file_hash.sql +10 -0
- package/supabase/migrations/20260226000002_add_dynamic_rag.sql +150 -0
- package/supabase/migrations/20260226000003_add_ingestion_summary.sql +4 -0
- package/supabase/migrations/20260226000004_add_ingestion_tags.sql +7 -0
- package/supabase/migrations/20260226000005_add_chat_tables.sql +60 -0
- package/supabase/migrations/20260227000000_harden_chat_messages_rls.sql +25 -0
- package/supabase/migrations/20260228000000_add_vision_model_capabilities.sql +8 -0
- package/supabase/migrations/20260228000001_add_policy_match_feedback.sql +51 -0
- package/supabase/migrations/29991231235959_test_migration.sql +0 -0
- package/supabase/templates/confirmation.html +76 -0
- package/supabase/templates/email-change.html +76 -0
- package/supabase/templates/invite.html +72 -0
- package/supabase/templates/magic-link.html +68 -0
- package/supabase/templates/recovery.html +82 -0
- package/tsconfig.api.json +16 -0
- package/tsconfig.json +25 -0
- package/vite.config.ts +146 -0
|
@@ -0,0 +1,225 @@
|
|
|
1
|
+
import { createLogger } from "./logger.js";
|
|
2
|
+
import { getServiceRoleSupabase } from "../services/supabase.js";
|
|
3
|
+
import { deriveVariables } from "./actions/utils.js";
|
|
4
|
+
import { RenameAction } from "./actions/RenameAction.js";
|
|
5
|
+
import { AutoRenameAction } from "./actions/AutoRenameAction.js";
|
|
6
|
+
import { CopyAction } from "./actions/CopyAction.js";
|
|
7
|
+
import { CopyToGDriveAction } from "./actions/CopyToGDriveAction.js";
|
|
8
|
+
import { AppendToGSheetAction } from "./actions/AppendToGSheetAction.js";
|
|
9
|
+
import { LogCsvAction } from "./actions/LogCsvAction.js";
|
|
10
|
+
import { NotifyAction } from "./actions/NotifyAction.js";
|
|
11
|
+
import { WebhookAction } from "./actions/WebhookAction.js";
|
|
12
|
+
const logger = createLogger("Actuator");
|
|
13
|
+
let warnedMissingServiceRole = false;
|
|
14
|
+
function isRecord(value) {
|
|
15
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
16
|
+
}
|
|
17
|
+
function toVariableString(value) {
|
|
18
|
+
if (value == null)
|
|
19
|
+
return undefined;
|
|
20
|
+
if (typeof value === "string")
|
|
21
|
+
return value;
|
|
22
|
+
if (typeof value === "number" || typeof value === "boolean")
|
|
23
|
+
return String(value);
|
|
24
|
+
try {
|
|
25
|
+
return JSON.stringify(value);
|
|
26
|
+
}
|
|
27
|
+
catch {
|
|
28
|
+
return undefined;
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
function ensureRecord(container, key) {
|
|
32
|
+
const existing = container[key];
|
|
33
|
+
if (isRecord(existing)) {
|
|
34
|
+
return existing;
|
|
35
|
+
}
|
|
36
|
+
const next = {};
|
|
37
|
+
container[key] = next;
|
|
38
|
+
return next;
|
|
39
|
+
}
|
|
40
|
+
export class Actuator {
|
|
41
|
+
static handlers = new Map([
|
|
42
|
+
["rename", new RenameAction()],
|
|
43
|
+
["auto_rename", new AutoRenameAction()],
|
|
44
|
+
["copy", new CopyAction()],
|
|
45
|
+
["copy_to_gdrive", new CopyToGDriveAction()],
|
|
46
|
+
["append_to_google_sheet", new AppendToGSheetAction()],
|
|
47
|
+
["log_csv", new LogCsvAction()],
|
|
48
|
+
["notify", new NotifyAction()],
|
|
49
|
+
["webhook", new WebhookAction()],
|
|
50
|
+
]);
|
|
51
|
+
static registerAction(type, handler) {
|
|
52
|
+
this.handlers.set(type, handler);
|
|
53
|
+
}
|
|
54
|
+
static mergeActionOutputs(runtimeData, runtimeVariables, actionType, actionIndex, outputs) {
|
|
55
|
+
const outputKeys = Object.keys(outputs);
|
|
56
|
+
if (outputKeys.length === 0)
|
|
57
|
+
return outputKeys;
|
|
58
|
+
const actionsByType = ensureRecord(runtimeData, "_actions");
|
|
59
|
+
const previousByType = actionsByType[actionType];
|
|
60
|
+
const mergedByType = {
|
|
61
|
+
...(isRecord(previousByType) ? previousByType : {}),
|
|
62
|
+
...outputs,
|
|
63
|
+
};
|
|
64
|
+
actionsByType[actionType] = mergedByType;
|
|
65
|
+
const history = Array.isArray(runtimeData._action_history) ? [...runtimeData._action_history] : [];
|
|
66
|
+
const historyEntry = {
|
|
67
|
+
index: actionIndex,
|
|
68
|
+
type: actionType,
|
|
69
|
+
...outputs,
|
|
70
|
+
};
|
|
71
|
+
history.push(historyEntry);
|
|
72
|
+
runtimeData._action_history = history;
|
|
73
|
+
runtimeData._last = historyEntry;
|
|
74
|
+
runtimeData._last_action_type = actionType;
|
|
75
|
+
runtimeVariables._last_action_type = actionType;
|
|
76
|
+
for (const key of outputKeys) {
|
|
77
|
+
const value = outputs[key];
|
|
78
|
+
const serialized = toVariableString(value);
|
|
79
|
+
if (runtimeData[key] === undefined) {
|
|
80
|
+
runtimeData[key] = value;
|
|
81
|
+
}
|
|
82
|
+
if (serialized === undefined)
|
|
83
|
+
continue;
|
|
84
|
+
runtimeVariables[`${actionType}.${key}`] = serialized;
|
|
85
|
+
runtimeVariables[`_actions.${actionType}.${key}`] = serialized;
|
|
86
|
+
runtimeVariables[`_last.${key}`] = serialized;
|
|
87
|
+
if (runtimeVariables[key] === undefined) {
|
|
88
|
+
runtimeVariables[key] = serialized;
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
return outputKeys;
|
|
92
|
+
}
|
|
93
|
+
static async logEvent(ingestionId, userId, eventType, state,
|
|
94
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
95
|
+
details, supabaseClient) {
|
|
96
|
+
// Fire and forget, don't await blocking execution
|
|
97
|
+
const supabase = supabaseClient ?? getServiceRoleSupabase();
|
|
98
|
+
if (!supabase) {
|
|
99
|
+
if (!warnedMissingServiceRole) {
|
|
100
|
+
logger.warn("Service-role Supabase client unavailable; skipping processing_events stream writes.");
|
|
101
|
+
warnedMissingServiceRole = true;
|
|
102
|
+
}
|
|
103
|
+
return;
|
|
104
|
+
}
|
|
105
|
+
void supabase.from("processing_events").insert({
|
|
106
|
+
ingestion_id: ingestionId ?? null,
|
|
107
|
+
user_id: userId ?? null,
|
|
108
|
+
event_type: eventType,
|
|
109
|
+
agent_state: state,
|
|
110
|
+
details,
|
|
111
|
+
}).then(({ error }) => {
|
|
112
|
+
if (error)
|
|
113
|
+
logger.warn("Failed to stream actuator log", { error });
|
|
114
|
+
});
|
|
115
|
+
}
|
|
116
|
+
/**
|
|
117
|
+
* Executes an array of abstract action definitions.
|
|
118
|
+
*/
|
|
119
|
+
static async execute(ingestionId, userId, actions, data, file, fields = [], supabase) {
|
|
120
|
+
const result = {
|
|
121
|
+
success: true,
|
|
122
|
+
actionsExecuted: [],
|
|
123
|
+
errors: [],
|
|
124
|
+
trace: [],
|
|
125
|
+
};
|
|
126
|
+
result.trace.push({ timestamp: new Date().toISOString(), step: "Initializing Actuator", details: { actionsCount: actions.length } });
|
|
127
|
+
Actuator.logEvent(ingestionId, userId, "info", "Actuator Initialized", { actionsCount: actions.length }, supabase);
|
|
128
|
+
// Convert the 13-digit Dropzone timestamp into a human-readable local time
|
|
129
|
+
// (e.g. "1772148849046-bill.pdf" -> "2026-02-26_15-34-09_bill.pdf")
|
|
130
|
+
// Note: only `currentFile.name` is normalized; `file.path` (disk path) is unchanged.
|
|
131
|
+
const tsMatch = file.name.match(/^(\d{13})-(.*)$/);
|
|
132
|
+
let currentFile = { ...file };
|
|
133
|
+
if (tsMatch) {
|
|
134
|
+
const date = new Date(parseInt(tsMatch[1], 10));
|
|
135
|
+
const yyyy = date.getFullYear();
|
|
136
|
+
const MM = String(date.getMonth() + 1).padStart(2, "0");
|
|
137
|
+
const dd = String(date.getDate()).padStart(2, "0");
|
|
138
|
+
const HH = String(date.getHours()).padStart(2, "0");
|
|
139
|
+
const mm = String(date.getMinutes()).padStart(2, "0");
|
|
140
|
+
const ss = String(date.getSeconds()).padStart(2, "0");
|
|
141
|
+
currentFile.name = `${yyyy}-${MM}-${dd}_${HH}-${mm}-${ss}_${tsMatch[2]}`;
|
|
142
|
+
}
|
|
143
|
+
const runtimeData = { ...data };
|
|
144
|
+
const runtimeVariables = deriveVariables(runtimeData, fields);
|
|
145
|
+
runtimeData.current_file_name = currentFile.name;
|
|
146
|
+
runtimeData.current_file_path = currentFile.path;
|
|
147
|
+
runtimeVariables.current_file_name = currentFile.name;
|
|
148
|
+
runtimeVariables.current_file_path = currentFile.path;
|
|
149
|
+
for (const [actionIndex, action] of actions.entries()) {
|
|
150
|
+
const handler = this.handlers.get(action.type);
|
|
151
|
+
if (!handler) {
|
|
152
|
+
const msg = `Action failed: Unsupported action type '${action.type}'`;
|
|
153
|
+
logger.error(msg);
|
|
154
|
+
result.errors.push(msg);
|
|
155
|
+
result.success = false;
|
|
156
|
+
result.trace.push({ timestamp: new Date().toISOString(), step: `Unsupported action type`, details: { type: action.type } });
|
|
157
|
+
Actuator.logEvent(ingestionId, userId, "error", "Action Execution", { action: action.type, error: msg }, supabase);
|
|
158
|
+
continue;
|
|
159
|
+
}
|
|
160
|
+
try {
|
|
161
|
+
runtimeData.current_file_name = currentFile.name;
|
|
162
|
+
runtimeData.current_file_path = currentFile.path;
|
|
163
|
+
runtimeVariables.current_file_name = currentFile.name;
|
|
164
|
+
runtimeVariables.current_file_path = currentFile.path;
|
|
165
|
+
const context = {
|
|
166
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
167
|
+
action: action,
|
|
168
|
+
data: runtimeData,
|
|
169
|
+
file: currentFile,
|
|
170
|
+
variables: runtimeVariables,
|
|
171
|
+
userId,
|
|
172
|
+
ingestionId,
|
|
173
|
+
supabase,
|
|
174
|
+
};
|
|
175
|
+
const handlerResult = await handler.execute(context);
|
|
176
|
+
result.trace.push(...handlerResult.trace);
|
|
177
|
+
if (handlerResult.success) {
|
|
178
|
+
if (handlerResult.newFileState) {
|
|
179
|
+
currentFile = handlerResult.newFileState;
|
|
180
|
+
runtimeData.current_file_name = currentFile.name;
|
|
181
|
+
runtimeData.current_file_path = currentFile.path;
|
|
182
|
+
runtimeVariables.current_file_name = currentFile.name;
|
|
183
|
+
runtimeVariables.current_file_path = currentFile.path;
|
|
184
|
+
}
|
|
185
|
+
if (handlerResult.outputs && Object.keys(handlerResult.outputs).length > 0) {
|
|
186
|
+
const outputKeys = this.mergeActionOutputs(runtimeData, runtimeVariables, action.type, actionIndex, handlerResult.outputs);
|
|
187
|
+
result.trace.push({
|
|
188
|
+
timestamp: new Date().toISOString(),
|
|
189
|
+
step: "Captured action outputs",
|
|
190
|
+
details: {
|
|
191
|
+
action: action.type,
|
|
192
|
+
outputKeys,
|
|
193
|
+
},
|
|
194
|
+
});
|
|
195
|
+
}
|
|
196
|
+
result.actionsExecuted.push(...handlerResult.logs);
|
|
197
|
+
}
|
|
198
|
+
else {
|
|
199
|
+
const msg = `Action failed (${action.type}): ${handlerResult.error}`;
|
|
200
|
+
logger.error(msg);
|
|
201
|
+
result.errors.push(msg);
|
|
202
|
+
result.success = false;
|
|
203
|
+
const eventDetails = {
|
|
204
|
+
action: action.type,
|
|
205
|
+
error: handlerResult.error,
|
|
206
|
+
};
|
|
207
|
+
if (handlerResult.errorDetails && Object.keys(handlerResult.errorDetails).length > 0) {
|
|
208
|
+
Object.assign(eventDetails, handlerResult.errorDetails);
|
|
209
|
+
}
|
|
210
|
+
Actuator.logEvent(ingestionId, userId, "error", "Action Execution", eventDetails, supabase);
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
catch (err) {
|
|
214
|
+
const errMsg = err instanceof Error ? err.message : String(err);
|
|
215
|
+
const msg = `Action failed (${action.type}): ${errMsg}`;
|
|
216
|
+
logger.error(msg);
|
|
217
|
+
result.errors.push(msg);
|
|
218
|
+
result.success = false;
|
|
219
|
+
result.trace.push({ timestamp: new Date().toISOString(), step: `Action execution error`, details: { type: action.type, error: errMsg } });
|
|
220
|
+
Actuator.logEvent(ingestionId, userId, "error", "Action Execution", { action: action.type, error: errMsg }, supabase);
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
return result;
|
|
224
|
+
}
|
|
225
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
import { GoogleSheetsService } from "../../services/GoogleSheetsService.js";
|
|
2
|
+
import { pickString, pickColumns, interpolate } from "./utils.js";
|
|
3
|
+
import { Actuator } from "../Actuator.js";
|
|
4
|
+
const HEADER_ALIASES = {
|
|
5
|
+
amount: ["total_amount", "amount", "amount_due"],
|
|
6
|
+
total: ["total_amount", "amount", "amount_due"],
|
|
7
|
+
total_amount: ["amount", "amount_due"],
|
|
8
|
+
vendor: ["issuer", "merchant", "store_name", "seller"],
|
|
9
|
+
merchant: ["issuer", "vendor", "store_name", "seller"],
|
|
10
|
+
supplier: ["issuer", "vendor", "merchant"],
|
|
11
|
+
store: ["issuer", "vendor", "merchant", "store_name"],
|
|
12
|
+
document: ["document_type"],
|
|
13
|
+
type: ["document_type"],
|
|
14
|
+
category: ["document_type"],
|
|
15
|
+
issued_on: ["date"],
|
|
16
|
+
invoice_date: ["date"],
|
|
17
|
+
receipt_date: ["date"],
|
|
18
|
+
image_link: ["file_url", "drive_file_url", "file_link", "document_link"],
|
|
19
|
+
file_link: ["file_url", "drive_file_url", "image_link", "document_link"],
|
|
20
|
+
document_link: ["file_url", "drive_file_url", "file_link", "image_link"],
|
|
21
|
+
link: ["file_url", "drive_file_url", "file_link", "document_link", "image_link"],
|
|
22
|
+
};
|
|
23
|
+
function normalizeKey(value) {
|
|
24
|
+
return value.toLowerCase().replace(/[^a-z0-9]+/g, "_").replace(/^_+|_+$/g, "");
|
|
25
|
+
}
|
|
26
|
+
function buildNormalizedVariableLookup(variables) {
|
|
27
|
+
const lookup = {};
|
|
28
|
+
for (const [key, value] of Object.entries(variables)) {
|
|
29
|
+
const normalized = normalizeKey(key);
|
|
30
|
+
if (normalized && !(normalized in lookup)) {
|
|
31
|
+
lookup[normalized] = value;
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
return lookup;
|
|
35
|
+
}
|
|
36
|
+
function resolveHeaderValue(header, variables, normalizedVariables) {
|
|
37
|
+
const trimmed = header.trim();
|
|
38
|
+
if (!trimmed)
|
|
39
|
+
return "";
|
|
40
|
+
if (variables[trimmed] !== undefined) {
|
|
41
|
+
return variables[trimmed];
|
|
42
|
+
}
|
|
43
|
+
const normalizedHeader = normalizeKey(trimmed);
|
|
44
|
+
if (!normalizedHeader)
|
|
45
|
+
return "";
|
|
46
|
+
if (normalizedVariables[normalizedHeader] !== undefined) {
|
|
47
|
+
return normalizedVariables[normalizedHeader];
|
|
48
|
+
}
|
|
49
|
+
const aliases = HEADER_ALIASES[normalizedHeader] ?? [];
|
|
50
|
+
for (const alias of aliases) {
|
|
51
|
+
if (normalizedVariables[alias] !== undefined) {
|
|
52
|
+
return normalizedVariables[alias];
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
return "";
|
|
56
|
+
}
|
|
57
|
+
function isDropdownValueAllowed(value, allowedValues) {
|
|
58
|
+
const candidate = value.trim().toLowerCase();
|
|
59
|
+
if (!candidate)
|
|
60
|
+
return true;
|
|
61
|
+
return allowedValues.some((allowed) => allowed.trim().toLowerCase() === candidate);
|
|
62
|
+
}
|
|
63
|
+
function applyStrictDropdownGuards(headers, values, headerDropdowns) {
|
|
64
|
+
const nextValues = [...values];
|
|
65
|
+
const skippedColumns = [];
|
|
66
|
+
const maxColumns = Math.min(headers.length, nextValues.length);
|
|
67
|
+
for (let index = 0; index < maxColumns; index += 1) {
|
|
68
|
+
const dropdown = headerDropdowns[index];
|
|
69
|
+
if (!dropdown || dropdown.strict !== true || dropdown.allowedValues.length === 0) {
|
|
70
|
+
continue;
|
|
71
|
+
}
|
|
72
|
+
const currentValue = nextValues[index] ?? "";
|
|
73
|
+
if (!currentValue.trim())
|
|
74
|
+
continue;
|
|
75
|
+
if (isDropdownValueAllowed(currentValue, dropdown.allowedValues))
|
|
76
|
+
continue;
|
|
77
|
+
nextValues[index] = "";
|
|
78
|
+
skippedColumns.push(headers[index] || `column_${index + 1}`);
|
|
79
|
+
}
|
|
80
|
+
return { values: nextValues, skippedColumns };
|
|
81
|
+
}
|
|
82
|
+
export class AppendToGSheetAction {
|
|
83
|
+
async execute(context) {
|
|
84
|
+
const { ingestionId, userId, supabase } = context;
|
|
85
|
+
const result = {
|
|
86
|
+
success: true,
|
|
87
|
+
logs: [],
|
|
88
|
+
trace: [],
|
|
89
|
+
};
|
|
90
|
+
const spreadsheetReference = pickString(context.action, "spreadsheet_id") ??
|
|
91
|
+
pickString(context.action, "spreadsheet_url");
|
|
92
|
+
if (!spreadsheetReference) {
|
|
93
|
+
result.success = false;
|
|
94
|
+
result.error = "Missing required Action configuration: 'spreadsheet_id'";
|
|
95
|
+
return result;
|
|
96
|
+
}
|
|
97
|
+
const configuredRange = pickString(context.action, "range");
|
|
98
|
+
const columnTemplates = pickColumns(context.action, []);
|
|
99
|
+
let rangeToAppend = configuredRange;
|
|
100
|
+
let values = [];
|
|
101
|
+
let usedDynamicMapping = false;
|
|
102
|
+
if (columnTemplates.length > 0) {
|
|
103
|
+
values = columnTemplates.map((template) => interpolate(template, context.variables, context.data));
|
|
104
|
+
const templateResult = await GoogleSheetsService.resolveTemplate(context.userId, spreadsheetReference, configuredRange, context.supabase);
|
|
105
|
+
if (templateResult.success && (templateResult.headers?.length ?? 0) > 0) {
|
|
106
|
+
const dropdownGuard = applyStrictDropdownGuards(templateResult.headers ?? [], values, (templateResult.headerDropdowns ?? []));
|
|
107
|
+
values = dropdownGuard.values;
|
|
108
|
+
if (dropdownGuard.skippedColumns.length > 0) {
|
|
109
|
+
result.logs.push(`Left ${dropdownGuard.skippedColumns.length} strict dropdown column(s) blank for human selection.`);
|
|
110
|
+
result.trace.push({
|
|
111
|
+
timestamp: new Date().toISOString(),
|
|
112
|
+
step: "Adjusted strict dropdown columns",
|
|
113
|
+
details: {
|
|
114
|
+
skippedColumns: dropdownGuard.skippedColumns,
|
|
115
|
+
},
|
|
116
|
+
});
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
else {
|
|
121
|
+
const templateResult = await GoogleSheetsService.resolveTemplate(context.userId, spreadsheetReference, configuredRange, context.supabase);
|
|
122
|
+
if (!templateResult.success) {
|
|
123
|
+
result.success = false;
|
|
124
|
+
result.error = templateResult.error || "Failed to read Google Sheet template headers";
|
|
125
|
+
if (templateResult.errorDetails) {
|
|
126
|
+
result.errorDetails = templateResult.errorDetails;
|
|
127
|
+
}
|
|
128
|
+
return result;
|
|
129
|
+
}
|
|
130
|
+
const headers = templateResult.headers ?? [];
|
|
131
|
+
if (headers.length === 0) {
|
|
132
|
+
result.success = false;
|
|
133
|
+
result.error = "Google Sheet template has no header row. Add column names in row 1.";
|
|
134
|
+
return result;
|
|
135
|
+
}
|
|
136
|
+
rangeToAppend = templateResult.range;
|
|
137
|
+
const normalizedVariables = buildNormalizedVariableLookup(context.variables);
|
|
138
|
+
values = headers.map((header) => resolveHeaderValue(header, context.variables, normalizedVariables));
|
|
139
|
+
const dropdownGuard = applyStrictDropdownGuards(headers, values, (templateResult.headerDropdowns ?? []));
|
|
140
|
+
values = dropdownGuard.values;
|
|
141
|
+
if (dropdownGuard.skippedColumns.length > 0) {
|
|
142
|
+
result.logs.push(`Left ${dropdownGuard.skippedColumns.length} strict dropdown column(s) blank for human selection.`);
|
|
143
|
+
result.trace.push({
|
|
144
|
+
timestamp: new Date().toISOString(),
|
|
145
|
+
step: "Adjusted strict dropdown columns",
|
|
146
|
+
details: {
|
|
147
|
+
skippedColumns: dropdownGuard.skippedColumns,
|
|
148
|
+
},
|
|
149
|
+
});
|
|
150
|
+
}
|
|
151
|
+
usedDynamicMapping = true;
|
|
152
|
+
result.trace.push({
|
|
153
|
+
timestamp: new Date().toISOString(),
|
|
154
|
+
step: "Resolved Google Sheet template",
|
|
155
|
+
details: {
|
|
156
|
+
spreadsheetId: templateResult.spreadsheetId,
|
|
157
|
+
range: templateResult.range,
|
|
158
|
+
headersCount: headers.length,
|
|
159
|
+
},
|
|
160
|
+
});
|
|
161
|
+
if (values.every((value) => value.trim().length === 0)) {
|
|
162
|
+
result.success = false;
|
|
163
|
+
result.error = "Unable to map extracted fields to Google Sheet headers. Provide explicit columns mapping or align header names with extracted keys.";
|
|
164
|
+
return result;
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
result.trace.push({
|
|
168
|
+
timestamp: new Date().toISOString(),
|
|
169
|
+
step: "Appending to Google Sheet",
|
|
170
|
+
details: { spreadsheetReference, range: rangeToAppend || "Sheet1", columnsCount: values.length, dynamicMapping: usedDynamicMapping },
|
|
171
|
+
});
|
|
172
|
+
const appendResult = await GoogleSheetsService.appendRow(context.userId, spreadsheetReference, rangeToAppend, values, context.supabase);
|
|
173
|
+
if (!appendResult.success) {
|
|
174
|
+
result.success = false;
|
|
175
|
+
result.error = appendResult.error;
|
|
176
|
+
if (appendResult.errorDetails) {
|
|
177
|
+
result.errorDetails = appendResult.errorDetails;
|
|
178
|
+
}
|
|
179
|
+
return result;
|
|
180
|
+
}
|
|
181
|
+
Actuator.logEvent(ingestionId, userId, "action", "Action Execution", {
|
|
182
|
+
action: "append_to_google_sheet",
|
|
183
|
+
spreadsheetId: appendResult.spreadsheetId ?? spreadsheetReference,
|
|
184
|
+
range: appendResult.range ?? rangeToAppend ?? "Sheet1",
|
|
185
|
+
columnsCount: values.length,
|
|
186
|
+
dynamicMapping: usedDynamicMapping,
|
|
187
|
+
}, supabase);
|
|
188
|
+
result.logs.push(`Appended ${values.length} columns to Google Sheet ${appendResult.spreadsheetId ?? spreadsheetReference} at ${appendResult.range ?? rangeToAppend ?? "Sheet1"}`);
|
|
189
|
+
return result;
|
|
190
|
+
}
|
|
191
|
+
}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import fs from "node:fs";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { resolveFilename } from "./utils.js";
|
|
4
|
+
import { Actuator } from "../Actuator.js";
|
|
5
|
+
import { getServiceRoleSupabase } from "../../services/supabase.js";
|
|
6
|
+
import { createLogger } from "../logger.js";
|
|
7
|
+
const logger = createLogger("AutoRenameAction");
|
|
8
|
+
export class AutoRenameAction {
|
|
9
|
+
async execute(context) {
|
|
10
|
+
const { file, variables, data, userId, ingestionId, supabase } = context;
|
|
11
|
+
const ext = path.extname(file.path);
|
|
12
|
+
const dir = path.dirname(file.path);
|
|
13
|
+
const stem = file.name.slice(0, file.name.length - ext.length);
|
|
14
|
+
logger.info("AutoRename variables", {
|
|
15
|
+
suggested_filename: variables.suggested_filename ?? "(missing)",
|
|
16
|
+
date: variables.date ?? "(missing)",
|
|
17
|
+
issuer: variables.issuer ?? "(missing)",
|
|
18
|
+
document_type: variables.document_type ?? "(missing)",
|
|
19
|
+
});
|
|
20
|
+
const newName = resolveFilename("auto", variables, stem, ext, data);
|
|
21
|
+
const newPath = path.join(dir, newName);
|
|
22
|
+
await new Promise((resolve, reject) => {
|
|
23
|
+
fs.rename(file.path, newPath, (err) => {
|
|
24
|
+
if (err)
|
|
25
|
+
reject(err);
|
|
26
|
+
else
|
|
27
|
+
resolve();
|
|
28
|
+
});
|
|
29
|
+
});
|
|
30
|
+
const trace = [{
|
|
31
|
+
timestamp: new Date().toISOString(),
|
|
32
|
+
step: `Auto-Renamed file to ${newName}`,
|
|
33
|
+
details: { original: file.name, new: newName }
|
|
34
|
+
}];
|
|
35
|
+
logger.info(`AutoRename: '${file.name}' → '${newName}'`);
|
|
36
|
+
Actuator.logEvent(ingestionId, userId, "action", "Action Execution", { action: "auto_rename", original: file.name, new: newName }, supabase);
|
|
37
|
+
// Update DB so re-runs don't break
|
|
38
|
+
const db = supabase ?? getServiceRoleSupabase();
|
|
39
|
+
if (db) {
|
|
40
|
+
await db.from("ingestions").update({ storage_path: newPath, filename: newName }).eq("id", ingestionId);
|
|
41
|
+
}
|
|
42
|
+
return {
|
|
43
|
+
success: true,
|
|
44
|
+
newFileState: { path: newPath, name: newName },
|
|
45
|
+
logs: [`Auto-Renamed to '${newName}'`],
|
|
46
|
+
trace
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
}
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
import fs from "node:fs";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { pickString, interpolate, resolveFilename } from "./utils.js";
|
|
4
|
+
import { Actuator } from "../Actuator.js";
|
|
5
|
+
import { GoogleDriveService } from "../../services/GoogleDriveService.js";
|
|
6
|
+
export class CopyAction {
|
|
7
|
+
async execute(context) {
|
|
8
|
+
const { action, file, variables, data, userId, ingestionId, supabase } = context;
|
|
9
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
10
|
+
const destination = pickString(action, "destination");
|
|
11
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
12
|
+
const pattern = pickString(action, "pattern");
|
|
13
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
14
|
+
const filenameConfig = pickString(action, "filename");
|
|
15
|
+
if (!destination) {
|
|
16
|
+
return {
|
|
17
|
+
success: false,
|
|
18
|
+
logs: [],
|
|
19
|
+
trace: [{ timestamp: new Date().toISOString(), step: "Copy failed: missing destination" }],
|
|
20
|
+
error: "Copy action requires a 'destination' config"
|
|
21
|
+
};
|
|
22
|
+
}
|
|
23
|
+
const destDir = interpolate(destination, variables, data);
|
|
24
|
+
// Support legacy gdrive:// destinations before copy_to_gdrive existed.
|
|
25
|
+
if (destDir.startsWith("gdrive://")) {
|
|
26
|
+
const folderPath = destDir.slice("gdrive://".length) || undefined;
|
|
27
|
+
let gdriveFileName;
|
|
28
|
+
if (filenameConfig) {
|
|
29
|
+
const ext = path.extname(file.name);
|
|
30
|
+
const stem = file.name.slice(0, file.name.length - ext.length);
|
|
31
|
+
gdriveFileName = resolveFilename(filenameConfig, variables, stem, ext, data);
|
|
32
|
+
}
|
|
33
|
+
const uploadResult = await GoogleDriveService.uploadFile(userId, file.path, folderPath, supabase, gdriveFileName);
|
|
34
|
+
if (!uploadResult.success) {
|
|
35
|
+
return {
|
|
36
|
+
success: false,
|
|
37
|
+
logs: [],
|
|
38
|
+
trace: [{ timestamp: new Date().toISOString(), step: "Copy to Google Drive failed", details: { error: uploadResult.error } }],
|
|
39
|
+
error: uploadResult.error || "Failed to upload to Google Drive"
|
|
40
|
+
};
|
|
41
|
+
}
|
|
42
|
+
const trace = [{
|
|
43
|
+
timestamp: new Date().toISOString(),
|
|
44
|
+
step: "Copied file to Google Drive",
|
|
45
|
+
details: { original: file.path, driveFileId: uploadResult.fileId, destinationFolderId: folderPath }
|
|
46
|
+
}];
|
|
47
|
+
Actuator.logEvent(ingestionId, userId, "action", "Action Execution", { action: "copy_to_gdrive", destinationFolderId: folderPath ?? null, fileId: uploadResult.fileId }, supabase);
|
|
48
|
+
const fileId = uploadResult.fileId;
|
|
49
|
+
const fileUrl = `https://drive.google.com/file/d/${fileId}/view`;
|
|
50
|
+
const outputs = {
|
|
51
|
+
provider: "google_drive",
|
|
52
|
+
file_id: fileId,
|
|
53
|
+
file_url: fileUrl,
|
|
54
|
+
drive_file_id: fileId,
|
|
55
|
+
drive_file_url: fileUrl,
|
|
56
|
+
destination_folder_id: folderPath ?? null,
|
|
57
|
+
uploaded_file_name: gdriveFileName ?? file.name,
|
|
58
|
+
};
|
|
59
|
+
if (/\.(jpg|jpeg|png|webp|gif|bmp|tiff|tif|heic)$/i.test(file.name)) {
|
|
60
|
+
outputs.image_link = fileUrl;
|
|
61
|
+
}
|
|
62
|
+
return {
|
|
63
|
+
success: true,
|
|
64
|
+
logs: [`Copied to Google Drive (ID: ${uploadResult.fileId})`],
|
|
65
|
+
trace,
|
|
66
|
+
outputs,
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
fs.mkdirSync(destDir, { recursive: true });
|
|
70
|
+
const ext = path.extname(file.name);
|
|
71
|
+
const stem = file.name.slice(0, file.name.length - ext.length);
|
|
72
|
+
let newName;
|
|
73
|
+
if (filenameConfig) {
|
|
74
|
+
newName = resolveFilename(filenameConfig, variables, stem, ext, data);
|
|
75
|
+
}
|
|
76
|
+
else if (pattern) {
|
|
77
|
+
// Backward-compat: treat pattern as a filename template
|
|
78
|
+
newName = interpolate(pattern, variables, data);
|
|
79
|
+
if (!newName.endsWith(ext))
|
|
80
|
+
newName += ext;
|
|
81
|
+
}
|
|
82
|
+
else {
|
|
83
|
+
newName = file.name;
|
|
84
|
+
}
|
|
85
|
+
const newPath = path.join(destDir, newName);
|
|
86
|
+
await new Promise((resolve, reject) => {
|
|
87
|
+
fs.copyFile(file.path, newPath, (err) => {
|
|
88
|
+
if (err)
|
|
89
|
+
reject(err);
|
|
90
|
+
else
|
|
91
|
+
resolve();
|
|
92
|
+
});
|
|
93
|
+
});
|
|
94
|
+
const trace = [{
|
|
95
|
+
timestamp: new Date().toISOString(),
|
|
96
|
+
step: `Copied file to ${newPath}`,
|
|
97
|
+
details: { original: file.path, copy: newPath }
|
|
98
|
+
}];
|
|
99
|
+
Actuator.logEvent(ingestionId, userId, "action", "Action Execution", { action: "copy", destination: destDir, newName }, supabase);
|
|
100
|
+
return {
|
|
101
|
+
success: true,
|
|
102
|
+
logs: [`Copied to '${newPath}'`],
|
|
103
|
+
trace,
|
|
104
|
+
outputs: {
|
|
105
|
+
provider: "local",
|
|
106
|
+
destination: destDir,
|
|
107
|
+
copied_path: newPath,
|
|
108
|
+
copied_name: newName,
|
|
109
|
+
},
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import path from "node:path";
|
|
2
|
+
import { pickString, interpolate, resolveFilename } from "./utils.js";
|
|
3
|
+
import { Actuator } from "../Actuator.js";
|
|
4
|
+
import { GoogleDriveService } from "../../services/GoogleDriveService.js";
|
|
5
|
+
export class CopyToGDriveAction {
|
|
6
|
+
async execute(context) {
|
|
7
|
+
const { action, file, variables, data, userId, ingestionId, supabase } = context;
|
|
8
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
9
|
+
const destination = pickString(action, "destination");
|
|
10
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
11
|
+
const filenameConfig = pickString(action, "filename");
|
|
12
|
+
const destDirId = destination ? interpolate(destination, variables, data) : undefined;
|
|
13
|
+
let resolvedFileName;
|
|
14
|
+
if (filenameConfig) {
|
|
15
|
+
const ext = path.extname(file.name);
|
|
16
|
+
const stem = file.name.slice(0, file.name.length - ext.length);
|
|
17
|
+
resolvedFileName = resolveFilename(filenameConfig, variables, stem, ext, data);
|
|
18
|
+
}
|
|
19
|
+
const uploadResult = await GoogleDriveService.uploadFile(userId, file.path, destDirId, supabase, resolvedFileName);
|
|
20
|
+
if (!uploadResult.success) {
|
|
21
|
+
return {
|
|
22
|
+
success: false,
|
|
23
|
+
logs: [],
|
|
24
|
+
trace: [{ timestamp: new Date().toISOString(), step: "Copy to Google Drive failed", details: { error: uploadResult.error } }],
|
|
25
|
+
error: uploadResult.error || "Failed to upload to Google Drive"
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
const trace = [{
|
|
29
|
+
timestamp: new Date().toISOString(),
|
|
30
|
+
step: `Copied file to Google Drive`,
|
|
31
|
+
details: { original: file.path, driveFileId: uploadResult.fileId, destinationFolderId: destDirId }
|
|
32
|
+
}];
|
|
33
|
+
Actuator.logEvent(ingestionId, userId, "action", "Action Execution", { action: "copy_to_gdrive", destinationFolderId: destDirId ?? null, fileId: uploadResult.fileId }, supabase);
|
|
34
|
+
const fileId = uploadResult.fileId;
|
|
35
|
+
const fileUrl = `https://drive.google.com/file/d/${fileId}/view`;
|
|
36
|
+
const outputs = {
|
|
37
|
+
provider: "google_drive",
|
|
38
|
+
file_id: fileId,
|
|
39
|
+
file_url: fileUrl,
|
|
40
|
+
drive_file_id: fileId,
|
|
41
|
+
drive_file_url: fileUrl,
|
|
42
|
+
destination_folder_id: destDirId ?? null,
|
|
43
|
+
uploaded_file_name: resolvedFileName ?? file.name,
|
|
44
|
+
};
|
|
45
|
+
if (/\.(jpg|jpeg|png|webp|gif|bmp|tiff|tif|heic)$/i.test(file.name)) {
|
|
46
|
+
outputs.image_link = fileUrl;
|
|
47
|
+
}
|
|
48
|
+
return {
|
|
49
|
+
success: true,
|
|
50
|
+
logs: [`Copied to Google Drive (ID: ${uploadResult.fileId})`],
|
|
51
|
+
trace,
|
|
52
|
+
outputs,
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
}
|