@realtimex/folio 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (163) hide show
  1. package/.env.example +20 -0
  2. package/README.md +63 -0
  3. package/api/server.ts +130 -0
  4. package/api/src/config/index.ts +96 -0
  5. package/api/src/middleware/auth.ts +128 -0
  6. package/api/src/middleware/errorHandler.ts +88 -0
  7. package/api/src/middleware/index.ts +4 -0
  8. package/api/src/middleware/rateLimit.ts +71 -0
  9. package/api/src/middleware/validation.ts +58 -0
  10. package/api/src/routes/accounts.ts +142 -0
  11. package/api/src/routes/baseline-config.ts +124 -0
  12. package/api/src/routes/chat.ts +154 -0
  13. package/api/src/routes/health.ts +61 -0
  14. package/api/src/routes/index.ts +35 -0
  15. package/api/src/routes/ingestions.ts +275 -0
  16. package/api/src/routes/migrate.ts +112 -0
  17. package/api/src/routes/policies.ts +121 -0
  18. package/api/src/routes/processing.ts +90 -0
  19. package/api/src/routes/rules.ts +11 -0
  20. package/api/src/routes/sdk.ts +100 -0
  21. package/api/src/routes/settings.ts +80 -0
  22. package/api/src/routes/setup.ts +389 -0
  23. package/api/src/routes/stats.ts +81 -0
  24. package/api/src/routes/tts.ts +190 -0
  25. package/api/src/services/BaselineConfigService.ts +208 -0
  26. package/api/src/services/ChatService.ts +204 -0
  27. package/api/src/services/GoogleDriveService.ts +331 -0
  28. package/api/src/services/GoogleSheetsService.ts +1107 -0
  29. package/api/src/services/IngestionService.ts +1187 -0
  30. package/api/src/services/ModelCapabilityService.ts +248 -0
  31. package/api/src/services/PolicyEngine.ts +1625 -0
  32. package/api/src/services/PolicyLearningService.ts +527 -0
  33. package/api/src/services/PolicyLoader.ts +249 -0
  34. package/api/src/services/RAGService.ts +391 -0
  35. package/api/src/services/SDKService.ts +249 -0
  36. package/api/src/services/supabase.ts +113 -0
  37. package/api/src/utils/Actuator.ts +284 -0
  38. package/api/src/utils/actions/ActionHandler.ts +34 -0
  39. package/api/src/utils/actions/AppendToGSheetAction.ts +260 -0
  40. package/api/src/utils/actions/AutoRenameAction.ts +58 -0
  41. package/api/src/utils/actions/CopyAction.ts +120 -0
  42. package/api/src/utils/actions/CopyToGDriveAction.ts +64 -0
  43. package/api/src/utils/actions/LogCsvAction.ts +48 -0
  44. package/api/src/utils/actions/NotifyAction.ts +39 -0
  45. package/api/src/utils/actions/RenameAction.ts +57 -0
  46. package/api/src/utils/actions/WebhookAction.ts +58 -0
  47. package/api/src/utils/actions/utils.ts +293 -0
  48. package/api/src/utils/llmResponse.ts +61 -0
  49. package/api/src/utils/logger.ts +67 -0
  50. package/bin/folio-deploy.js +12 -0
  51. package/bin/folio-setup.js +45 -0
  52. package/bin/folio.js +65 -0
  53. package/dist/api/server.js +106 -0
  54. package/dist/api/src/config/index.js +81 -0
  55. package/dist/api/src/middleware/auth.js +93 -0
  56. package/dist/api/src/middleware/errorHandler.js +73 -0
  57. package/dist/api/src/middleware/index.js +4 -0
  58. package/dist/api/src/middleware/rateLimit.js +43 -0
  59. package/dist/api/src/middleware/validation.js +54 -0
  60. package/dist/api/src/routes/accounts.js +110 -0
  61. package/dist/api/src/routes/baseline-config.js +91 -0
  62. package/dist/api/src/routes/chat.js +114 -0
  63. package/dist/api/src/routes/health.js +52 -0
  64. package/dist/api/src/routes/index.js +31 -0
  65. package/dist/api/src/routes/ingestions.js +207 -0
  66. package/dist/api/src/routes/migrate.js +91 -0
  67. package/dist/api/src/routes/policies.js +86 -0
  68. package/dist/api/src/routes/processing.js +75 -0
  69. package/dist/api/src/routes/rules.js +8 -0
  70. package/dist/api/src/routes/sdk.js +80 -0
  71. package/dist/api/src/routes/settings.js +68 -0
  72. package/dist/api/src/routes/setup.js +315 -0
  73. package/dist/api/src/routes/stats.js +62 -0
  74. package/dist/api/src/routes/tts.js +178 -0
  75. package/dist/api/src/services/BaselineConfigService.js +168 -0
  76. package/dist/api/src/services/ChatService.js +166 -0
  77. package/dist/api/src/services/GoogleDriveService.js +280 -0
  78. package/dist/api/src/services/GoogleSheetsService.js +795 -0
  79. package/dist/api/src/services/IngestionService.js +990 -0
  80. package/dist/api/src/services/ModelCapabilityService.js +179 -0
  81. package/dist/api/src/services/PolicyEngine.js +1353 -0
  82. package/dist/api/src/services/PolicyLearningService.js +397 -0
  83. package/dist/api/src/services/PolicyLoader.js +159 -0
  84. package/dist/api/src/services/RAGService.js +295 -0
  85. package/dist/api/src/services/SDKService.js +212 -0
  86. package/dist/api/src/services/supabase.js +72 -0
  87. package/dist/api/src/utils/Actuator.js +225 -0
  88. package/dist/api/src/utils/actions/ActionHandler.js +1 -0
  89. package/dist/api/src/utils/actions/AppendToGSheetAction.js +191 -0
  90. package/dist/api/src/utils/actions/AutoRenameAction.js +49 -0
  91. package/dist/api/src/utils/actions/CopyAction.js +112 -0
  92. package/dist/api/src/utils/actions/CopyToGDriveAction.js +55 -0
  93. package/dist/api/src/utils/actions/LogCsvAction.js +42 -0
  94. package/dist/api/src/utils/actions/NotifyAction.js +32 -0
  95. package/dist/api/src/utils/actions/RenameAction.js +51 -0
  96. package/dist/api/src/utils/actions/WebhookAction.js +51 -0
  97. package/dist/api/src/utils/actions/utils.js +237 -0
  98. package/dist/api/src/utils/llmResponse.js +63 -0
  99. package/dist/api/src/utils/logger.js +51 -0
  100. package/dist/assets/index-DzN8-j-e.css +1 -0
  101. package/dist/assets/index-Uy-ai3Dh.js +113 -0
  102. package/dist/favicon.svg +31 -0
  103. package/dist/folio-logo.svg +46 -0
  104. package/dist/index.html +14 -0
  105. package/docs-dev/FPE-spec.md +196 -0
  106. package/docs-dev/folio-prd.md +47 -0
  107. package/docs-dev/foundation-checklist.md +30 -0
  108. package/docs-dev/hybrid-routing-architecture.md +205 -0
  109. package/docs-dev/ingestion-engine.md +69 -0
  110. package/docs-dev/port-from-email-automator.md +32 -0
  111. package/docs-dev/tech-spec.md +98 -0
  112. package/index.html +13 -0
  113. package/package.json +101 -0
  114. package/public/favicon.svg +31 -0
  115. package/public/folio-logo.svg +46 -0
  116. package/scripts/dev-task.mjs +51 -0
  117. package/scripts/get-latest-migration-timestamp.mjs +34 -0
  118. package/scripts/migrate.sh +91 -0
  119. package/supabase/.temp/cli-latest +1 -0
  120. package/supabase/.temp/gotrue-version +1 -0
  121. package/supabase/.temp/pooler-url +1 -0
  122. package/supabase/.temp/postgres-version +1 -0
  123. package/supabase/.temp/project-ref +1 -0
  124. package/supabase/.temp/rest-version +1 -0
  125. package/supabase/.temp/storage-migration +1 -0
  126. package/supabase/.temp/storage-version +1 -0
  127. package/supabase/config.toml +64 -0
  128. package/supabase/functions/_shared/auth.ts +35 -0
  129. package/supabase/functions/_shared/cors.ts +12 -0
  130. package/supabase/functions/_shared/supabaseAdmin.ts +17 -0
  131. package/supabase/functions/api-v1-settings/index.ts +66 -0
  132. package/supabase/functions/setup/index.ts +91 -0
  133. package/supabase/migrations/20260223000000_initial_foundation.sql +136 -0
  134. package/supabase/migrations/20260223000001_add_migration_rpc.sql +10 -0
  135. package/supabase/migrations/20260224000002_add_init_state_view.sql +20 -0
  136. package/supabase/migrations/20260224000003_port_user_creation_parity.sql +139 -0
  137. package/supabase/migrations/20260224000004_add_avatars_storage.sql +26 -0
  138. package/supabase/migrations/20260224000005_add_tts_and_embed_settings.sql +24 -0
  139. package/supabase/migrations/20260224000006_add_policies_table.sql +48 -0
  140. package/supabase/migrations/20260224000007_fix_migration_rpc.sql +9 -0
  141. package/supabase/migrations/20260224000008_add_ingestions_table.sql +42 -0
  142. package/supabase/migrations/20260225000000_setup_compatible_mode.sql +119 -0
  143. package/supabase/migrations/20260225000001_restore_ingestions.sql +49 -0
  144. package/supabase/migrations/20260225000002_add_ingestion_trace.sql +2 -0
  145. package/supabase/migrations/20260225000003_add_baseline_configs.sql +35 -0
  146. package/supabase/migrations/20260226000000_add_processing_events.sql +26 -0
  147. package/supabase/migrations/20260226000001_add_ingestion_file_hash.sql +10 -0
  148. package/supabase/migrations/20260226000002_add_dynamic_rag.sql +150 -0
  149. package/supabase/migrations/20260226000003_add_ingestion_summary.sql +4 -0
  150. package/supabase/migrations/20260226000004_add_ingestion_tags.sql +7 -0
  151. package/supabase/migrations/20260226000005_add_chat_tables.sql +60 -0
  152. package/supabase/migrations/20260227000000_harden_chat_messages_rls.sql +25 -0
  153. package/supabase/migrations/20260228000000_add_vision_model_capabilities.sql +8 -0
  154. package/supabase/migrations/20260228000001_add_policy_match_feedback.sql +51 -0
  155. package/supabase/migrations/29991231235959_test_migration.sql +0 -0
  156. package/supabase/templates/confirmation.html +76 -0
  157. package/supabase/templates/email-change.html +76 -0
  158. package/supabase/templates/invite.html +72 -0
  159. package/supabase/templates/magic-link.html +68 -0
  160. package/supabase/templates/recovery.html +82 -0
  161. package/tsconfig.api.json +16 -0
  162. package/tsconfig.json +25 -0
  163. package/vite.config.ts +146 -0
@@ -0,0 +1,280 @@
1
+ import { getServiceRoleSupabase } from "./supabase.js";
2
+ import { createLogger } from "../utils/logger.js";
3
+ import fs from "node:fs";
4
+ import path from "node:path";
5
+ const logger = createLogger("GoogleDriveService");
6
+ function parseCredentials(value) {
7
+ if (!value || typeof value !== "object") {
8
+ return {};
9
+ }
10
+ const record = value;
11
+ return {
12
+ access_token: typeof record.access_token === "string" ? record.access_token : undefined,
13
+ refresh_token: typeof record.refresh_token === "string" ? record.refresh_token : undefined,
14
+ expires_at: typeof record.expires_at === "number" ? record.expires_at : undefined,
15
+ client_id: typeof record.client_id === "string" ? record.client_id : undefined,
16
+ client_secret: typeof record.client_secret === "string" ? record.client_secret : undefined,
17
+ };
18
+ }
19
+ export class GoogleDriveService {
20
+ /**
21
+ * Walks a slash-separated sub-path under a root Drive folder, creating any
22
+ * missing folders along the way, and returns the final folder ID.
23
+ *
24
+ * e.g. resolveFolderPath(token, "rootId", ["Utilities", "PGE", "Energy Statements"])
25
+ * → ID of "Energy Statements" folder (created if absent)
26
+ */
27
+ static async resolveFolderPath(token, rootFolderId, subSegments) {
28
+ let parentId = rootFolderId;
29
+ for (const segment of subSegments) {
30
+ const name = segment.trim();
31
+ if (!name)
32
+ continue;
33
+ // Search for an existing folder with this name under the current parent.
34
+ const q = `name='${name.replace(/'/g, "\\'")}' and mimeType='application/vnd.google-apps.folder' and '${parentId}' in parents and trashed=false`;
35
+ const searchResp = await fetch(`https://www.googleapis.com/drive/v3/files?q=${encodeURIComponent(q)}&fields=files(id)&spaces=drive`, { headers: { Authorization: `Bearer ${token}` } });
36
+ if (!searchResp.ok) {
37
+ throw new Error(`Drive folder search failed: HTTP ${searchResp.status}`);
38
+ }
39
+ const { files } = await searchResp.json();
40
+ if (files && files.length > 0) {
41
+ parentId = files[0].id;
42
+ }
43
+ else {
44
+ // Folder doesn't exist — create it.
45
+ const createResp = await fetch("https://www.googleapis.com/drive/v3/files", {
46
+ method: "POST",
47
+ headers: {
48
+ Authorization: `Bearer ${token}`,
49
+ "Content-Type": "application/json",
50
+ },
51
+ body: JSON.stringify({
52
+ name,
53
+ mimeType: "application/vnd.google-apps.folder",
54
+ parents: [parentId],
55
+ }),
56
+ });
57
+ if (!createResp.ok) {
58
+ throw new Error(`Drive folder creation failed for '${name}': HTTP ${createResp.status}`);
59
+ }
60
+ const { id } = await createResp.json();
61
+ if (!id)
62
+ throw new Error(`Folder '${name}' was created but Drive returned no ID.`);
63
+ logger.info(`Created Drive folder '${name}' (${id}) under parent ${parentId}`);
64
+ parentId = id;
65
+ }
66
+ }
67
+ return parentId;
68
+ }
69
+ /**
70
+ * Uploads a local file to a user's connected Google Drive.
71
+ * Handles automatic token refreshment if the access_token has expired.
72
+ *
73
+ * `folderPath` can be:
74
+ * - undefined / empty → upload to My Drive root
75
+ * - a bare folder ID → upload directly into that folder
76
+ * - "rootId/Sub/Path" → resolve (and auto-create) the subfolder hierarchy,
77
+ * then upload into the deepest folder
78
+ */
79
+ static async uploadFile(userId, localFilePath, folderId, supabaseClient, fileName) {
80
+ logger.info(`Initiating Google Drive upload for user ${userId}: ${localFilePath}`);
81
+ const supabase = supabaseClient ?? getServiceRoleSupabase();
82
+ if (!supabase) {
83
+ return { success: false, error: "System error: Supabase client unavailable." };
84
+ }
85
+ let fileStat;
86
+ try {
87
+ fileStat = await fs.promises.stat(localFilePath);
88
+ if (!fileStat.isFile()) {
89
+ return { success: false, error: "Source path is not a file." };
90
+ }
91
+ }
92
+ catch {
93
+ return { success: false, error: "Source file not found." };
94
+ }
95
+ // 1. Fetch Integration
96
+ const { data: integration, error } = await supabase
97
+ .from("integrations")
98
+ .select("*")
99
+ .eq("user_id", userId)
100
+ .eq("provider", "google_drive")
101
+ .single();
102
+ if (error || !integration) {
103
+ return { success: false, error: "Google Drive is not securely connected for this user." };
104
+ }
105
+ const credentials = parseCredentials(integration.credentials);
106
+ let accessToken = credentials.access_token;
107
+ const clientId = credentials.client_id;
108
+ const clientSecret = credentials.client_secret;
109
+ const refreshToken = credentials.refresh_token;
110
+ if (!accessToken) {
111
+ return { success: false, error: "Google Drive credentials are incomplete. Please reconnect the drive." };
112
+ }
113
+ // 2. Proactively refresh the token if it is expired or expires within 60 s.
114
+ // This must happen before any Drive API calls (folder resolution, upload initiation).
115
+ const tokenIsStale = credentials.expires_at && Date.now() >= credentials.expires_at - 60_000;
116
+ if (tokenIsStale && refreshToken && clientId && clientSecret) {
117
+ logger.info("Google Drive token is stale, refreshing before Drive API calls...");
118
+ try {
119
+ const tokenResp = await fetch("https://oauth2.googleapis.com/token", {
120
+ method: "POST",
121
+ headers: { "Content-Type": "application/x-www-form-urlencoded" },
122
+ body: new URLSearchParams({
123
+ client_id: clientId,
124
+ client_secret: clientSecret,
125
+ refresh_token: refreshToken,
126
+ grant_type: "refresh_token",
127
+ }),
128
+ });
129
+ if (!tokenResp.ok) {
130
+ return { success: false, error: "Google Drive authentication expired and could not be refreshed. Please reconnect the drive." };
131
+ }
132
+ const tokenData = await tokenResp.json();
133
+ if (!tokenData?.access_token) {
134
+ return { success: false, error: "Google OAuth token response was invalid." };
135
+ }
136
+ accessToken = tokenData.access_token;
137
+ const updatedCredentials = {
138
+ ...credentials,
139
+ access_token: accessToken,
140
+ ...(typeof tokenData.refresh_token === "string" && tokenData.refresh_token
141
+ ? { refresh_token: tokenData.refresh_token }
142
+ : {}),
143
+ ...(typeof tokenData.expires_in === "number"
144
+ ? { expires_at: Date.now() + tokenData.expires_in * 1000 }
145
+ : {}),
146
+ };
147
+ await supabase.from("integrations").update({
148
+ credentials: updatedCredentials,
149
+ updated_at: new Date().toISOString(),
150
+ }).eq("id", integration.id);
151
+ }
152
+ catch (refreshErr) {
153
+ logger.error("Proactive token refresh failed", { error: refreshErr });
154
+ return { success: false, error: "Failed to communicate with Google Auth server." };
155
+ }
156
+ }
157
+ // 3. Resolve subfolder path (auto-creates missing folders).
158
+ // folderPath may be "rootId", "rootId/Sub/Folder", or undefined.
159
+ // Drive folder IDs are stable across token refreshes so we resolve once.
160
+ let resolvedFolderId;
161
+ if (folderId && folderId.trim().length > 0) {
162
+ const parts = folderId.trim().split("/");
163
+ const rootId = parts[0];
164
+ const subSegments = parts.slice(1);
165
+ try {
166
+ resolvedFolderId = subSegments.length > 0
167
+ ? await GoogleDriveService.resolveFolderPath(accessToken, rootId, subSegments)
168
+ : rootId;
169
+ }
170
+ catch (pathErr) {
171
+ const msg = pathErr instanceof Error ? pathErr.message : String(pathErr);
172
+ return { success: false, error: `Failed to resolve Drive folder path: ${msg}` };
173
+ }
174
+ }
175
+ // 4. Perform upload (optimistic; catches 401 as a last-resort fallback).
176
+ const performUpload = async (token) => {
177
+ const resolvedFileName = fileName ?? path.basename(localFilePath);
178
+ const mimeType = "application/octet-stream";
179
+ const metadata = { name: resolvedFileName };
180
+ if (resolvedFolderId) {
181
+ metadata.parents = [resolvedFolderId];
182
+ }
183
+ // Resumable upload avoids buffering/base64 encoding the full file in memory.
184
+ const startResponse = await fetch("https://www.googleapis.com/upload/drive/v3/files?uploadType=resumable", {
185
+ method: "POST",
186
+ headers: {
187
+ Authorization: `Bearer ${token}`,
188
+ "Content-Type": "application/json; charset=UTF-8",
189
+ "X-Upload-Content-Type": mimeType,
190
+ "X-Upload-Content-Length": String(fileStat.size),
191
+ },
192
+ body: JSON.stringify(metadata),
193
+ });
194
+ if (!startResponse.ok) {
195
+ return startResponse;
196
+ }
197
+ const uploadUrl = startResponse.headers.get("location");
198
+ if (!uploadUrl) {
199
+ throw new Error("Google Drive did not return an upload URL.");
200
+ }
201
+ const contentRange = fileStat.size === 0
202
+ ? `bytes */${fileStat.size}`
203
+ : `bytes 0-${fileStat.size - 1}/${fileStat.size}`;
204
+ // duplex: "half" is required by Node.js fetch when the request body is a stream.
205
+ const uploadRequest = {
206
+ method: "PUT",
207
+ headers: {
208
+ Authorization: `Bearer ${token}`,
209
+ "Content-Type": mimeType,
210
+ "Content-Length": String(fileStat.size),
211
+ "Content-Range": contentRange,
212
+ },
213
+ body: fs.createReadStream(localFilePath),
214
+ duplex: "half",
215
+ };
216
+ return fetch(uploadUrl, uploadRequest);
217
+ };
218
+ let response = await performUpload(accessToken);
219
+ // 5. Handle Token Expiry (last-resort retry if upload still returns 401)
220
+ if (response.status === 401 && refreshToken && clientId && clientSecret) {
221
+ logger.info("Google Drive token expired, attempting refresh...");
222
+ try {
223
+ const tokenResp = await fetch("https://oauth2.googleapis.com/token", {
224
+ method: "POST",
225
+ headers: { "Content-Type": "application/x-www-form-urlencoded" },
226
+ body: new URLSearchParams({
227
+ client_id: clientId,
228
+ client_secret: clientSecret,
229
+ refresh_token: refreshToken,
230
+ grant_type: "refresh_token"
231
+ })
232
+ });
233
+ if (tokenResp.ok) {
234
+ const tokenData = await tokenResp.json();
235
+ if (!tokenData?.access_token) {
236
+ return { success: false, error: "Google OAuth token response was invalid." };
237
+ }
238
+ const refreshedAccessToken = tokenData.access_token;
239
+ accessToken = refreshedAccessToken;
240
+ const updatedCredentials = {
241
+ ...credentials,
242
+ access_token: accessToken,
243
+ };
244
+ if (typeof tokenData.refresh_token === "string" && tokenData.refresh_token) {
245
+ updatedCredentials.refresh_token = tokenData.refresh_token;
246
+ }
247
+ if (typeof tokenData.expires_in === "number") {
248
+ updatedCredentials.expires_at = Date.now() + tokenData.expires_in * 1000;
249
+ }
250
+ // Transparently save refreshed credentials.
251
+ await supabase.from("integrations").update({
252
+ credentials: updatedCredentials,
253
+ updated_at: new Date().toISOString()
254
+ }).eq("id", integration.id);
255
+ // Retry Upload
256
+ logger.info("Retrying Google Drive upload with fresh token...");
257
+ response = await performUpload(refreshedAccessToken);
258
+ }
259
+ else {
260
+ return { success: false, error: "Google Drive authentication expired and could not be refreshed. Please reconnect the drive." };
261
+ }
262
+ }
263
+ catch (authErr) {
264
+ logger.error("Failed to refresh Google Drive token", { error: authErr });
265
+ return { success: false, error: "Failed to communicate with Google Auth server." };
266
+ }
267
+ }
268
+ if (!response.ok) {
269
+ const errorBody = await response.text();
270
+ logger.error("Google Drive API rejected upload", { status: response.status, body: errorBody });
271
+ return { success: false, error: `Upload failed: HTTP ${response.status}.` };
272
+ }
273
+ const responseData = await response.json();
274
+ if (!responseData.id) {
275
+ return { success: false, error: "Google Drive upload succeeded but no file ID was returned." };
276
+ }
277
+ logger.info(`Successfully uploaded file to Google Drive: ${responseData.id}`);
278
+ return { success: true, fileId: responseData.id };
279
+ }
280
+ }