@realtimex/folio 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (163) hide show
  1. package/.env.example +20 -0
  2. package/README.md +63 -0
  3. package/api/server.ts +130 -0
  4. package/api/src/config/index.ts +96 -0
  5. package/api/src/middleware/auth.ts +128 -0
  6. package/api/src/middleware/errorHandler.ts +88 -0
  7. package/api/src/middleware/index.ts +4 -0
  8. package/api/src/middleware/rateLimit.ts +71 -0
  9. package/api/src/middleware/validation.ts +58 -0
  10. package/api/src/routes/accounts.ts +142 -0
  11. package/api/src/routes/baseline-config.ts +124 -0
  12. package/api/src/routes/chat.ts +154 -0
  13. package/api/src/routes/health.ts +61 -0
  14. package/api/src/routes/index.ts +35 -0
  15. package/api/src/routes/ingestions.ts +275 -0
  16. package/api/src/routes/migrate.ts +112 -0
  17. package/api/src/routes/policies.ts +121 -0
  18. package/api/src/routes/processing.ts +90 -0
  19. package/api/src/routes/rules.ts +11 -0
  20. package/api/src/routes/sdk.ts +100 -0
  21. package/api/src/routes/settings.ts +80 -0
  22. package/api/src/routes/setup.ts +389 -0
  23. package/api/src/routes/stats.ts +81 -0
  24. package/api/src/routes/tts.ts +190 -0
  25. package/api/src/services/BaselineConfigService.ts +208 -0
  26. package/api/src/services/ChatService.ts +204 -0
  27. package/api/src/services/GoogleDriveService.ts +331 -0
  28. package/api/src/services/GoogleSheetsService.ts +1107 -0
  29. package/api/src/services/IngestionService.ts +1187 -0
  30. package/api/src/services/ModelCapabilityService.ts +248 -0
  31. package/api/src/services/PolicyEngine.ts +1625 -0
  32. package/api/src/services/PolicyLearningService.ts +527 -0
  33. package/api/src/services/PolicyLoader.ts +249 -0
  34. package/api/src/services/RAGService.ts +391 -0
  35. package/api/src/services/SDKService.ts +249 -0
  36. package/api/src/services/supabase.ts +113 -0
  37. package/api/src/utils/Actuator.ts +284 -0
  38. package/api/src/utils/actions/ActionHandler.ts +34 -0
  39. package/api/src/utils/actions/AppendToGSheetAction.ts +260 -0
  40. package/api/src/utils/actions/AutoRenameAction.ts +58 -0
  41. package/api/src/utils/actions/CopyAction.ts +120 -0
  42. package/api/src/utils/actions/CopyToGDriveAction.ts +64 -0
  43. package/api/src/utils/actions/LogCsvAction.ts +48 -0
  44. package/api/src/utils/actions/NotifyAction.ts +39 -0
  45. package/api/src/utils/actions/RenameAction.ts +57 -0
  46. package/api/src/utils/actions/WebhookAction.ts +58 -0
  47. package/api/src/utils/actions/utils.ts +293 -0
  48. package/api/src/utils/llmResponse.ts +61 -0
  49. package/api/src/utils/logger.ts +67 -0
  50. package/bin/folio-deploy.js +12 -0
  51. package/bin/folio-setup.js +45 -0
  52. package/bin/folio.js +65 -0
  53. package/dist/api/server.js +106 -0
  54. package/dist/api/src/config/index.js +81 -0
  55. package/dist/api/src/middleware/auth.js +93 -0
  56. package/dist/api/src/middleware/errorHandler.js +73 -0
  57. package/dist/api/src/middleware/index.js +4 -0
  58. package/dist/api/src/middleware/rateLimit.js +43 -0
  59. package/dist/api/src/middleware/validation.js +54 -0
  60. package/dist/api/src/routes/accounts.js +110 -0
  61. package/dist/api/src/routes/baseline-config.js +91 -0
  62. package/dist/api/src/routes/chat.js +114 -0
  63. package/dist/api/src/routes/health.js +52 -0
  64. package/dist/api/src/routes/index.js +31 -0
  65. package/dist/api/src/routes/ingestions.js +207 -0
  66. package/dist/api/src/routes/migrate.js +91 -0
  67. package/dist/api/src/routes/policies.js +86 -0
  68. package/dist/api/src/routes/processing.js +75 -0
  69. package/dist/api/src/routes/rules.js +8 -0
  70. package/dist/api/src/routes/sdk.js +80 -0
  71. package/dist/api/src/routes/settings.js +68 -0
  72. package/dist/api/src/routes/setup.js +315 -0
  73. package/dist/api/src/routes/stats.js +62 -0
  74. package/dist/api/src/routes/tts.js +178 -0
  75. package/dist/api/src/services/BaselineConfigService.js +168 -0
  76. package/dist/api/src/services/ChatService.js +166 -0
  77. package/dist/api/src/services/GoogleDriveService.js +280 -0
  78. package/dist/api/src/services/GoogleSheetsService.js +795 -0
  79. package/dist/api/src/services/IngestionService.js +990 -0
  80. package/dist/api/src/services/ModelCapabilityService.js +179 -0
  81. package/dist/api/src/services/PolicyEngine.js +1353 -0
  82. package/dist/api/src/services/PolicyLearningService.js +397 -0
  83. package/dist/api/src/services/PolicyLoader.js +159 -0
  84. package/dist/api/src/services/RAGService.js +295 -0
  85. package/dist/api/src/services/SDKService.js +212 -0
  86. package/dist/api/src/services/supabase.js +72 -0
  87. package/dist/api/src/utils/Actuator.js +225 -0
  88. package/dist/api/src/utils/actions/ActionHandler.js +1 -0
  89. package/dist/api/src/utils/actions/AppendToGSheetAction.js +191 -0
  90. package/dist/api/src/utils/actions/AutoRenameAction.js +49 -0
  91. package/dist/api/src/utils/actions/CopyAction.js +112 -0
  92. package/dist/api/src/utils/actions/CopyToGDriveAction.js +55 -0
  93. package/dist/api/src/utils/actions/LogCsvAction.js +42 -0
  94. package/dist/api/src/utils/actions/NotifyAction.js +32 -0
  95. package/dist/api/src/utils/actions/RenameAction.js +51 -0
  96. package/dist/api/src/utils/actions/WebhookAction.js +51 -0
  97. package/dist/api/src/utils/actions/utils.js +237 -0
  98. package/dist/api/src/utils/llmResponse.js +63 -0
  99. package/dist/api/src/utils/logger.js +51 -0
  100. package/dist/assets/index-DzN8-j-e.css +1 -0
  101. package/dist/assets/index-Uy-ai3Dh.js +113 -0
  102. package/dist/favicon.svg +31 -0
  103. package/dist/folio-logo.svg +46 -0
  104. package/dist/index.html +14 -0
  105. package/docs-dev/FPE-spec.md +196 -0
  106. package/docs-dev/folio-prd.md +47 -0
  107. package/docs-dev/foundation-checklist.md +30 -0
  108. package/docs-dev/hybrid-routing-architecture.md +205 -0
  109. package/docs-dev/ingestion-engine.md +69 -0
  110. package/docs-dev/port-from-email-automator.md +32 -0
  111. package/docs-dev/tech-spec.md +98 -0
  112. package/index.html +13 -0
  113. package/package.json +101 -0
  114. package/public/favicon.svg +31 -0
  115. package/public/folio-logo.svg +46 -0
  116. package/scripts/dev-task.mjs +51 -0
  117. package/scripts/get-latest-migration-timestamp.mjs +34 -0
  118. package/scripts/migrate.sh +91 -0
  119. package/supabase/.temp/cli-latest +1 -0
  120. package/supabase/.temp/gotrue-version +1 -0
  121. package/supabase/.temp/pooler-url +1 -0
  122. package/supabase/.temp/postgres-version +1 -0
  123. package/supabase/.temp/project-ref +1 -0
  124. package/supabase/.temp/rest-version +1 -0
  125. package/supabase/.temp/storage-migration +1 -0
  126. package/supabase/.temp/storage-version +1 -0
  127. package/supabase/config.toml +64 -0
  128. package/supabase/functions/_shared/auth.ts +35 -0
  129. package/supabase/functions/_shared/cors.ts +12 -0
  130. package/supabase/functions/_shared/supabaseAdmin.ts +17 -0
  131. package/supabase/functions/api-v1-settings/index.ts +66 -0
  132. package/supabase/functions/setup/index.ts +91 -0
  133. package/supabase/migrations/20260223000000_initial_foundation.sql +136 -0
  134. package/supabase/migrations/20260223000001_add_migration_rpc.sql +10 -0
  135. package/supabase/migrations/20260224000002_add_init_state_view.sql +20 -0
  136. package/supabase/migrations/20260224000003_port_user_creation_parity.sql +139 -0
  137. package/supabase/migrations/20260224000004_add_avatars_storage.sql +26 -0
  138. package/supabase/migrations/20260224000005_add_tts_and_embed_settings.sql +24 -0
  139. package/supabase/migrations/20260224000006_add_policies_table.sql +48 -0
  140. package/supabase/migrations/20260224000007_fix_migration_rpc.sql +9 -0
  141. package/supabase/migrations/20260224000008_add_ingestions_table.sql +42 -0
  142. package/supabase/migrations/20260225000000_setup_compatible_mode.sql +119 -0
  143. package/supabase/migrations/20260225000001_restore_ingestions.sql +49 -0
  144. package/supabase/migrations/20260225000002_add_ingestion_trace.sql +2 -0
  145. package/supabase/migrations/20260225000003_add_baseline_configs.sql +35 -0
  146. package/supabase/migrations/20260226000000_add_processing_events.sql +26 -0
  147. package/supabase/migrations/20260226000001_add_ingestion_file_hash.sql +10 -0
  148. package/supabase/migrations/20260226000002_add_dynamic_rag.sql +150 -0
  149. package/supabase/migrations/20260226000003_add_ingestion_summary.sql +4 -0
  150. package/supabase/migrations/20260226000004_add_ingestion_tags.sql +7 -0
  151. package/supabase/migrations/20260226000005_add_chat_tables.sql +60 -0
  152. package/supabase/migrations/20260227000000_harden_chat_messages_rls.sql +25 -0
  153. package/supabase/migrations/20260228000000_add_vision_model_capabilities.sql +8 -0
  154. package/supabase/migrations/20260228000001_add_policy_match_feedback.sql +51 -0
  155. package/supabase/migrations/29991231235959_test_migration.sql +0 -0
  156. package/supabase/templates/confirmation.html +76 -0
  157. package/supabase/templates/email-change.html +76 -0
  158. package/supabase/templates/invite.html +72 -0
  159. package/supabase/templates/magic-link.html +68 -0
  160. package/supabase/templates/recovery.html +82 -0
  161. package/tsconfig.api.json +16 -0
  162. package/tsconfig.json +25 -0
  163. package/vite.config.ts +146 -0
@@ -0,0 +1,119 @@
1
+ -- Keep old ingestions table from previous migration step for Hybrid Routing
2
+ -- (Previously this dropped public.ingestions, but we now retain it as Folio's primary UI table).
3
+ -- Create rtx_activities table as defined in Compatible Mode docs
4
+ -- Added user_id for multi-tenant isolation
5
+ CREATE TABLE public.rtx_activities (
6
+ id uuid NOT NULL DEFAULT gen_random_uuid (),
7
+ user_id uuid NOT NULL REFERENCES auth.users(id) ON DELETE CASCADE,
8
+ raw_data jsonb NULL, -- Your input data
9
+ old_data jsonb NULL, -- Previous data (for updates)
10
+ locked_by text NULL, -- Machine ID holding the lock
11
+ locked_at timestamp with time zone NULL,
12
+ status text NULL DEFAULT 'pending'::text,
13
+ completed_at timestamp with time zone NULL,
14
+ error_message text NULL,
15
+ attempted_by text[] NULL DEFAULT '{}'::text[],
16
+ retry_count integer NULL DEFAULT 0,
17
+ result jsonb NULL,
18
+ created_at timestamp with time zone NULL DEFAULT now(),
19
+ CONSTRAINT rtx_activities_pkey PRIMARY KEY (id)
20
+ );
21
+
22
+ -- Index for status queries
23
+ CREATE INDEX idx_rtx_activities_status ON public.rtx_activities (status);
24
+ -- Index for user list queries
25
+ CREATE INDEX idx_rtx_activities_user ON public.rtx_activities (user_id, created_at DESC);
26
+
27
+ -- Ensure all columns are included in Realtime events
28
+ ALTER TABLE public.rtx_activities REPLICA IDENTITY FULL;
29
+
30
+ -- RLS
31
+ ALTER TABLE public.rtx_activities ENABLE ROW LEVEL SECURITY;
32
+
33
+ CREATE POLICY "Users can manage their own activities"
34
+ ON public.rtx_activities FOR ALL
35
+ USING (auth.uid() = user_id)
36
+ WITH CHECK (auth.uid() = user_id);
37
+
38
+ -- Database Functions (RPC)
39
+
40
+ -- Claim Task
41
+ CREATE OR REPLACE FUNCTION rtx_fn_claim_task(target_task_id UUID, machine_id TEXT)
42
+ RETURNS BOOLEAN AS $$
43
+ DECLARE
44
+ updated_rows INT;
45
+ BEGIN
46
+ UPDATE public.rtx_activities
47
+ SET status = 'claimed', locked_by = machine_id, locked_at = now()
48
+ WHERE id = target_task_id
49
+ AND (status = 'pending' OR status = 'failed'
50
+ OR ((status = 'claimed' OR status = 'processing')
51
+ AND locked_at < now() - INTERVAL '5 minutes'))
52
+ AND NOT (machine_id = ANY(attempted_by));
53
+
54
+ GET DIAGNOSTICS updated_rows = ROW_COUNT;
55
+ RETURN updated_rows > 0;
56
+ END;
57
+ $$ LANGUAGE plpgsql;
58
+
59
+ -- Complete Task
60
+ CREATE OR REPLACE FUNCTION rtx_fn_complete_task(target_task_id UUID, result_data JSONB)
61
+ RETURNS BOOLEAN AS $$
62
+ DECLARE
63
+ updated_rows INT;
64
+ BEGIN
65
+ UPDATE public.rtx_activities
66
+ SET status = 'completed', result = result_data, completed_at = now()
67
+ WHERE id = target_task_id AND (status = 'claimed' OR status = 'processing');
68
+
69
+ GET DIAGNOSTICS updated_rows = ROW_COUNT;
70
+ RETURN updated_rows > 0;
71
+ END;
72
+ $$ LANGUAGE plpgsql;
73
+
74
+ -- Fail Task
75
+ CREATE OR REPLACE FUNCTION rtx_fn_fail_task(target_task_id UUID, machine_id TEXT, error_msg TEXT)
76
+ RETURNS BOOLEAN AS $$
77
+ DECLARE
78
+ updated_rows INT;
79
+ BEGIN
80
+ UPDATE public.rtx_activities
81
+ SET
82
+ status = 'failed',
83
+ error_message = error_msg,
84
+ attempted_by = array_append(attempted_by, machine_id),
85
+ retry_count = retry_count + 1,
86
+ locked_by = NULL,
87
+ locked_at = NULL
88
+ WHERE id = target_task_id;
89
+
90
+ GET DIAGNOSTICS updated_rows = ROW_COUNT;
91
+ RETURN updated_rows > 0;
92
+ END;
93
+ $$ LANGUAGE plpgsql;
94
+
95
+ -- Enable pg_cron extension
96
+ CREATE EXTENSION IF NOT EXISTS pg_cron;
97
+
98
+ -- Create cleanup function
99
+ CREATE OR REPLACE FUNCTION public.rtx_fn_unlock_stale_locks()
100
+ RETURNS void AS $$
101
+ BEGIN
102
+ UPDATE public.rtx_activities
103
+ SET status = 'pending', locked_by = NULL, locked_at = NULL
104
+ WHERE (status = 'claimed' OR status = 'processing')
105
+ AND locked_at < now() - INTERVAL '5 minutes';
106
+ END;
107
+ $$ LANGUAGE plpgsql;
108
+
109
+ -- Avoid conflicts if it exists already, just drop it and recreate
110
+ DO $$
111
+ BEGIN
112
+ PERFORM cron.unschedule('scavenge-stale-locks');
113
+ EXCEPTION WHEN OTHERS THEN
114
+ -- ignore
115
+ END $$;
116
+ SELECT cron.schedule('scavenge-stale-locks', '* * * * *', 'SELECT public.rtx_fn_unlock_stale_locks();');
117
+
118
+ -- Enable Realtime
119
+ ALTER PUBLICATION supabase_realtime ADD TABLE public.rtx_activities;
@@ -0,0 +1,49 @@
1
+ -- Ingestion log table: tracks every document processed through Folio's Policy Engine
2
+ -- (Restoring this table for the Hybrid Routing Architecture, since the previous
3
+ -- compatible_mode migration erroneously dropped it).
4
+ CREATE TABLE IF NOT EXISTS ingestions (
5
+ id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
6
+ user_id uuid NOT NULL REFERENCES auth.users(id) ON DELETE CASCADE,
7
+ source text NOT NULL DEFAULT 'upload', -- 'upload' | 'dropzone' | 'email' | 'url'
8
+ filename text NOT NULL,
9
+ mime_type text,
10
+ file_size bigint,
11
+ status text NOT NULL DEFAULT 'pending', -- 'pending' | 'processing' | 'matched' | 'no_match' | 'error'
12
+ policy_id text, -- matched policy id (nullable)
13
+ policy_name text, -- denormalised for display
14
+ extracted jsonb DEFAULT '{}'::jsonb, -- key/value pairs extracted by FPE
15
+ actions_taken jsonb DEFAULT '[]'::jsonb, -- list of actions executed
16
+ error_message text,
17
+ storage_path text, -- file path pointer for Hybrid Routing or Supabase Storage ID
18
+ created_at timestamptz NOT NULL DEFAULT now(),
19
+ updated_at timestamptz NOT NULL DEFAULT now()
20
+ );
21
+
22
+ -- Index for per-user list queries
23
+ CREATE INDEX IF NOT EXISTS ingestions_user_id_idx ON ingestions(user_id, created_at DESC);
24
+
25
+ -- RLS
26
+ ALTER TABLE ingestions ENABLE ROW LEVEL SECURITY;
27
+
28
+ DO $$ BEGIN
29
+ CREATE POLICY "Users can manage their own ingestions"
30
+ ON ingestions FOR ALL
31
+ USING (auth.uid() = user_id)
32
+ WITH CHECK (auth.uid() = user_id);
33
+ EXCEPTION
34
+ WHEN duplicate_object THEN null;
35
+ END $$;
36
+
37
+ -- updated_at trigger
38
+ CREATE OR REPLACE FUNCTION update_ingestions_updated_at()
39
+ RETURNS TRIGGER AS $$
40
+ BEGIN
41
+ NEW.updated_at = now();
42
+ RETURN NEW;
43
+ END;
44
+ $$ LANGUAGE plpgsql;
45
+
46
+ DROP TRIGGER IF EXISTS ingestions_updated_at ON ingestions;
47
+ CREATE TRIGGER ingestions_updated_at
48
+ BEFORE UPDATE ON ingestions
49
+ FOR EACH ROW EXECUTE FUNCTION update_ingestions_updated_at();
@@ -0,0 +1,2 @@
1
+ -- Add trace column to ingestions table for full AI Transparency
2
+ ALTER TABLE public.ingestions ADD COLUMN IF NOT EXISTS trace JSONB DEFAULT '[]'::jsonb;
@@ -0,0 +1,35 @@
1
+ -- Versioned baseline extraction configuration per user.
2
+ -- Each row is immutable once referenced by an ingestion record,
3
+ -- enabling full auditability of which prompt config produced each extraction.
4
+
5
+ CREATE TABLE public.baseline_configs (
6
+ id uuid NOT NULL DEFAULT gen_random_uuid(),
7
+ user_id uuid NOT NULL REFERENCES auth.users(id) ON DELETE CASCADE,
8
+ version integer NOT NULL,
9
+ context text NULL, -- free-text injected into the extraction system prompt
10
+ fields jsonb NOT NULL DEFAULT '[]', -- array of BaselineField definitions
11
+ is_active boolean NOT NULL DEFAULT false,
12
+ created_at timestamptz NOT NULL DEFAULT now(),
13
+
14
+ CONSTRAINT baseline_configs_pkey PRIMARY KEY (id),
15
+ CONSTRAINT baseline_configs_user_version_key UNIQUE (user_id, version)
16
+ );
17
+
18
+ -- Fast lookup of a user's active config
19
+ CREATE INDEX idx_baseline_configs_user_active
20
+ ON public.baseline_configs (user_id, is_active);
21
+
22
+ -- RLS
23
+ ALTER TABLE public.baseline_configs ENABLE ROW LEVEL SECURITY;
24
+
25
+ CREATE POLICY "Users manage own baseline configs"
26
+ ON public.baseline_configs FOR ALL
27
+ USING (auth.uid() = user_id)
28
+ WITH CHECK (auth.uid() = user_id);
29
+
30
+ -- Record which config version produced each ingestion's extraction.
31
+ -- NULL means the ingestion was processed before this feature existed
32
+ -- or that the default built-in fields were used.
33
+ ALTER TABLE public.ingestions
34
+ ADD COLUMN baseline_config_id uuid NULL
35
+ REFERENCES public.baseline_configs(id) ON DELETE SET NULL;
@@ -0,0 +1,26 @@
1
+ -- Create processing_events table for granular logging
2
+ CREATE TABLE IF NOT EXISTS processing_events (
3
+ id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
4
+ ingestion_id UUID REFERENCES ingestions(id) ON DELETE CASCADE,
5
+ user_id UUID REFERENCES auth.users(id) ON DELETE CASCADE,
6
+ event_type TEXT NOT NULL CHECK (event_type IN ('info', 'analysis', 'action', 'error')),
7
+ agent_state TEXT, -- e.g., 'Triage', 'Baseline Extraction', 'Policy Matching', 'Action Execution'
8
+ details JSONB, -- Stores LLM inputs/outputs, reasoning, confidence
9
+ created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
10
+ );
11
+
12
+ -- Enable RLS
13
+ ALTER TABLE processing_events ENABLE ROW LEVEL SECURITY;
14
+
15
+ -- RLS Policy: Users can access their own processing events
16
+ CREATE POLICY "Users can access their own processing events" ON processing_events
17
+ FOR ALL USING (auth.uid() = user_id);
18
+
19
+ -- Enable Realtime for this table (standard Supabase publication)
20
+ DO $$
21
+ BEGIN
22
+ IF NOT EXISTS (SELECT 1 FROM pg_publication_tables WHERE pubname = 'supabase_realtime' AND schemaname = 'public' AND tablename = 'processing_events') THEN
23
+ ALTER PUBLICATION supabase_realtime ADD TABLE processing_events;
24
+ END IF;
25
+ END
26
+ $$;
@@ -0,0 +1,10 @@
1
+ -- Add file_hash column to ingestions for duplicate detection.
2
+ -- SHA-256 hex digest (64 chars) of the raw file bytes.
3
+ -- Nullable so existing rows are unaffected.
4
+ ALTER TABLE ingestions ADD COLUMN IF NOT EXISTS file_hash TEXT;
5
+
6
+ -- Partial index: only non-null hashes, scoped per user — makes the
7
+ -- duplicate lookup (user_id + file_hash) fast without indexing NULLs.
8
+ CREATE INDEX IF NOT EXISTS idx_ingestions_user_file_hash
9
+ ON ingestions (user_id, file_hash)
10
+ WHERE file_hash IS NOT NULL;
@@ -0,0 +1,150 @@
1
+ -- Enable pgvector extension for vector similarity search
2
+ CREATE EXTENSION IF NOT EXISTS vector;
3
+
4
+ -- Document chunks table for Semantic Search (RAG)
5
+ CREATE TABLE IF NOT EXISTS document_chunks (
6
+ id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
7
+ user_id UUID NOT NULL REFERENCES auth.users(id) ON DELETE CASCADE,
8
+ ingestion_id UUID NOT NULL REFERENCES ingestions(id) ON DELETE CASCADE,
9
+
10
+ -- Content
11
+ content TEXT NOT NULL,
12
+ content_hash TEXT NOT NULL, -- To detect duplicates
13
+ embedding_provider TEXT NOT NULL,
14
+ embedding_model TEXT NOT NULL,
15
+
16
+ -- Unconstrained Vector Embedding
17
+ -- By not defining the dimension (e.g. vector(1536)), we allow multiple
18
+ -- embedding models to coexist in the same table seamlessly.
19
+ embedding vector NOT NULL,
20
+ vector_dim INTEGER NOT NULL,
21
+
22
+ created_at TIMESTAMPTZ DEFAULT NOW(),
23
+
24
+ -- Prevent duplicate chunks for the same document + model identity.
25
+ -- This allows hot-swapping models while keeping each model's vector space isolated.
26
+ UNIQUE(ingestion_id, content_hash, embedding_provider, embedding_model)
27
+ );
28
+
29
+ -- Enable RLS
30
+ ALTER TABLE document_chunks ENABLE ROW LEVEL SECURITY;
31
+
32
+ CREATE POLICY "Users can manage their own document chunks"
33
+ ON document_chunks
34
+ FOR ALL
35
+ USING (user_id = auth.uid())
36
+ WITH CHECK (user_id = auth.uid());
37
+
38
+ -- Advanced Partial Indexes for HNSW
39
+ -- We create partial indexes for the most common dimensions to maintain
40
+ -- sub-millisecond search performance while keeping the 'embedding' column unconstrained.
41
+ CREATE INDEX IF NOT EXISTS document_chunks_embedding_384_idx
42
+ ON document_chunks USING hnsw ((embedding::vector(384)) vector_cosine_ops)
43
+ WHERE (vector_dim = 384);
44
+
45
+ CREATE INDEX IF NOT EXISTS document_chunks_embedding_768_idx
46
+ ON document_chunks USING hnsw ((embedding::vector(768)) vector_cosine_ops)
47
+ WHERE (vector_dim = 768);
48
+
49
+ CREATE INDEX IF NOT EXISTS document_chunks_embedding_1536_idx
50
+ ON document_chunks USING hnsw ((embedding::vector(1536)) vector_cosine_ops)
51
+ WHERE (vector_dim = 1536);
52
+
53
+ -- Index for fast deletion and lookup by ingestion_id
54
+ CREATE INDEX IF NOT EXISTS document_chunks_ingestion_id_idx
55
+ ON document_chunks(ingestion_id);
56
+
57
+ CREATE INDEX IF NOT EXISTS document_chunks_user_id_idx
58
+ ON document_chunks(user_id);
59
+
60
+ CREATE INDEX IF NOT EXISTS document_chunks_model_scope_idx
61
+ ON document_chunks(user_id, embedding_provider, embedding_model, vector_dim);
62
+
63
+ -- Dynamic Semantic Search Function
64
+ CREATE OR REPLACE FUNCTION search_documents(
65
+ p_user_id UUID,
66
+ p_embedding_provider TEXT,
67
+ p_embedding_model TEXT,
68
+ query_embedding vector,
69
+ match_threshold float DEFAULT 0.7,
70
+ match_count int DEFAULT 5,
71
+ query_dim int DEFAULT 1536
72
+ )
73
+ RETURNS TABLE (
74
+ id UUID,
75
+ ingestion_id UUID,
76
+ content TEXT,
77
+ similarity float
78
+ )
79
+ LANGUAGE plpgsql
80
+ AS $$
81
+ BEGIN
82
+ IF query_dim = 384 THEN
83
+ RETURN QUERY
84
+ SELECT
85
+ dc.id,
86
+ dc.ingestion_id,
87
+ dc.content,
88
+ 1 - (dc.embedding::vector(384) <=> query_embedding::vector(384)) AS similarity
89
+ FROM document_chunks dc
90
+ WHERE dc.user_id = p_user_id
91
+ AND dc.embedding_provider = p_embedding_provider
92
+ AND dc.embedding_model = p_embedding_model
93
+ AND dc.vector_dim = 384
94
+ AND 1 - (dc.embedding::vector(384) <=> query_embedding::vector(384)) > match_threshold
95
+ ORDER BY dc.embedding::vector(384) <=> query_embedding::vector(384)
96
+ LIMIT match_count;
97
+ ELSIF query_dim = 768 THEN
98
+ RETURN QUERY
99
+ SELECT
100
+ dc.id,
101
+ dc.ingestion_id,
102
+ dc.content,
103
+ 1 - (dc.embedding::vector(768) <=> query_embedding::vector(768)) AS similarity
104
+ FROM document_chunks dc
105
+ WHERE dc.user_id = p_user_id
106
+ AND dc.embedding_provider = p_embedding_provider
107
+ AND dc.embedding_model = p_embedding_model
108
+ AND dc.vector_dim = 768
109
+ AND 1 - (dc.embedding::vector(768) <=> query_embedding::vector(768)) > match_threshold
110
+ ORDER BY dc.embedding::vector(768) <=> query_embedding::vector(768)
111
+ LIMIT match_count;
112
+ ELSIF query_dim = 1536 THEN
113
+ RETURN QUERY
114
+ SELECT
115
+ dc.id,
116
+ dc.ingestion_id,
117
+ dc.content,
118
+ 1 - (dc.embedding::vector(1536) <=> query_embedding::vector(1536)) AS similarity
119
+ FROM document_chunks dc
120
+ WHERE dc.user_id = p_user_id
121
+ AND dc.embedding_provider = p_embedding_provider
122
+ AND dc.embedding_model = p_embedding_model
123
+ AND dc.vector_dim = 1536
124
+ AND 1 - (dc.embedding::vector(1536) <=> query_embedding::vector(1536)) > match_threshold
125
+ ORDER BY dc.embedding::vector(1536) <=> query_embedding::vector(1536)
126
+ LIMIT match_count;
127
+ ELSE
128
+ -- Fallback to unconstrained exact nearest neighbor scan for unindexed dimensions
129
+ RETURN QUERY
130
+ SELECT
131
+ dc.id,
132
+ dc.ingestion_id,
133
+ dc.content,
134
+ 1 - (dc.embedding <=> query_embedding) AS similarity
135
+ FROM document_chunks dc
136
+ WHERE dc.user_id = p_user_id
137
+ AND dc.embedding_provider = p_embedding_provider
138
+ AND dc.embedding_model = p_embedding_model
139
+ AND dc.vector_dim = query_dim
140
+ AND 1 - (dc.embedding <=> query_embedding) > match_threshold
141
+ ORDER BY dc.embedding <=> query_embedding
142
+ LIMIT match_count;
143
+ END IF;
144
+ END;
145
+ $$;
146
+
147
+ -- Comments
148
+ COMMENT ON TABLE document_chunks IS 'Stores semantic text chunks from parsed documents for RAG.';
149
+ COMMENT ON COLUMN document_chunks.embedding IS 'Unconstrained vector to support dynamic embedding models.';
150
+ COMMENT ON FUNCTION search_documents IS 'Performs dynamic cosine similarity search against unconstrained vectors.';
@@ -0,0 +1,4 @@
1
+ -- Add AI-generated prose summary column to ingestions.
2
+ -- Populated on first modal open, cleared when a re-run resets extraction.
3
+ ALTER TABLE public.ingestions
4
+ ADD COLUMN IF NOT EXISTS summary text NULL;
@@ -0,0 +1,7 @@
1
+ -- Add LLM-generated + human-curated tags to ingestions.
2
+ -- GIN index enables efficient array-containment queries (e.g. @> '{tax-deductible}').
3
+ ALTER TABLE public.ingestions
4
+ ADD COLUMN IF NOT EXISTS tags text[] NOT NULL DEFAULT '{}';
5
+
6
+ CREATE INDEX IF NOT EXISTS ingestions_tags_gin_idx
7
+ ON public.ingestions USING gin(tags);
@@ -0,0 +1,60 @@
1
+ -- Chat Sessions Table
2
+ CREATE TABLE IF NOT EXISTS chat_sessions (
3
+ id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
4
+ user_id UUID NOT NULL REFERENCES auth.users(id) ON DELETE CASCADE,
5
+ title TEXT NOT NULL DEFAULT 'New Conversation',
6
+ created_at TIMESTAMPTZ DEFAULT NOW(),
7
+ updated_at TIMESTAMPTZ DEFAULT NOW()
8
+ );
9
+
10
+ -- Chat Messages Table
11
+ CREATE TABLE IF NOT EXISTS chat_messages (
12
+ id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
13
+ session_id UUID NOT NULL REFERENCES chat_sessions(id) ON DELETE CASCADE,
14
+ user_id UUID NOT NULL REFERENCES auth.users(id) ON DELETE CASCADE,
15
+
16
+ role TEXT NOT NULL CHECK (role IN ('user', 'assistant', 'system')),
17
+ content TEXT NOT NULL,
18
+ context_sources JSONB DEFAULT '[]'::jsonb, -- Store the RAG chunks used to answer this message
19
+
20
+ created_at TIMESTAMPTZ DEFAULT NOW()
21
+ );
22
+
23
+ -- Enable RLS
24
+ ALTER TABLE chat_sessions ENABLE ROW LEVEL SECURITY;
25
+ ALTER TABLE chat_messages ENABLE ROW LEVEL SECURITY;
26
+
27
+ -- Policies for chat_sessions
28
+ CREATE POLICY "Users can manage their own chat sessions"
29
+ ON chat_sessions
30
+ FOR ALL
31
+ USING (user_id = auth.uid())
32
+ WITH CHECK (user_id = auth.uid());
33
+
34
+ -- Policies for chat_messages
35
+ CREATE POLICY "Users can manage their own chat messages"
36
+ ON chat_messages
37
+ FOR ALL
38
+ USING (user_id = auth.uid())
39
+ WITH CHECK (user_id = auth.uid());
40
+
41
+ -- Indexes for performance
42
+ CREATE INDEX IF NOT EXISTS chat_sessions_user_id_idx ON chat_sessions(user_id);
43
+ CREATE INDEX IF NOT EXISTS chat_messages_session_id_idx ON chat_messages(session_id);
44
+ CREATE INDEX IF NOT EXISTS chat_messages_user_id_idx ON chat_messages(user_id);
45
+
46
+ -- Trigger to update session updated_at
47
+ CREATE OR REPLACE FUNCTION update_chat_session_timestamp()
48
+ RETURNS TRIGGER AS $$
49
+ BEGIN
50
+ UPDATE chat_sessions
51
+ SET updated_at = NOW()
52
+ WHERE id = NEW.session_id;
53
+ RETURN NEW;
54
+ END;
55
+ $$ LANGUAGE plpgsql;
56
+
57
+ CREATE TRIGGER trigger_update_chat_session_timestamp
58
+ AFTER INSERT ON chat_messages
59
+ FOR EACH ROW
60
+ EXECUTE FUNCTION update_chat_session_timestamp();
@@ -0,0 +1,25 @@
1
+ -- Tighten chat message authorization to require ownership of both the row user_id
2
+ -- and the referenced session. This prevents cross-session inserts/selects.
3
+ DROP POLICY IF EXISTS "Users can manage their own chat messages" ON chat_messages;
4
+
5
+ CREATE POLICY "Users can manage their own chat messages"
6
+ ON chat_messages
7
+ FOR ALL
8
+ USING (
9
+ user_id = auth.uid()
10
+ AND EXISTS (
11
+ SELECT 1
12
+ FROM chat_sessions cs
13
+ WHERE cs.id = chat_messages.session_id
14
+ AND cs.user_id = auth.uid()
15
+ )
16
+ )
17
+ WITH CHECK (
18
+ user_id = auth.uid()
19
+ AND EXISTS (
20
+ SELECT 1
21
+ FROM chat_sessions cs
22
+ WHERE cs.id = chat_messages.session_id
23
+ AND cs.user_id = auth.uid()
24
+ )
25
+ );
@@ -0,0 +1,8 @@
1
+ -- Migration: Persist learned vision capability state per user model
2
+ -- Created: 2026-02-28
3
+
4
+ ALTER TABLE public.user_settings
5
+ ADD COLUMN IF NOT EXISTS vision_model_capabilities JSONB NOT NULL DEFAULT '{}'::jsonb;
6
+
7
+ COMMENT ON COLUMN public.user_settings.vision_model_capabilities IS
8
+ 'Learned VLM capability map keyed by provider:model with state/TTL metadata.';
@@ -0,0 +1,51 @@
1
+ -- Stores explicit user feedback when they manually map an ingestion to a policy.
2
+ -- Used to improve future policy matching via lightweight similarity scoring.
3
+ CREATE TABLE IF NOT EXISTS public.policy_match_feedback (
4
+ id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
5
+ user_id UUID NOT NULL REFERENCES auth.users(id) ON DELETE CASCADE,
6
+ ingestion_id UUID NOT NULL REFERENCES public.ingestions(id) ON DELETE CASCADE,
7
+ policy_id TEXT NOT NULL,
8
+ policy_name TEXT,
9
+ feedback_type TEXT NOT NULL DEFAULT 'manual_match' CHECK (feedback_type IN ('manual_match')),
10
+ features JSONB NOT NULL DEFAULT '{}'::jsonb,
11
+ created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
12
+ updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
13
+ UNIQUE(user_id, ingestion_id, policy_id)
14
+ );
15
+
16
+ CREATE INDEX IF NOT EXISTS policy_match_feedback_user_policy_idx
17
+ ON public.policy_match_feedback(user_id, policy_id, created_at DESC);
18
+
19
+ CREATE INDEX IF NOT EXISTS policy_match_feedback_user_created_idx
20
+ ON public.policy_match_feedback(user_id, created_at DESC);
21
+
22
+ ALTER TABLE public.policy_match_feedback ENABLE ROW LEVEL SECURITY;
23
+
24
+ CREATE POLICY "Users can read own policy match feedback"
25
+ ON public.policy_match_feedback FOR SELECT
26
+ USING (auth.uid() = user_id);
27
+
28
+ CREATE POLICY "Users can insert own policy match feedback"
29
+ ON public.policy_match_feedback FOR INSERT
30
+ WITH CHECK (auth.uid() = user_id);
31
+
32
+ CREATE POLICY "Users can update own policy match feedback"
33
+ ON public.policy_match_feedback FOR UPDATE
34
+ USING (auth.uid() = user_id);
35
+
36
+ CREATE POLICY "Users can delete own policy match feedback"
37
+ ON public.policy_match_feedback FOR DELETE
38
+ USING (auth.uid() = user_id);
39
+
40
+ CREATE OR REPLACE FUNCTION update_policy_match_feedback_updated_at()
41
+ RETURNS TRIGGER AS $$
42
+ BEGIN
43
+ NEW.updated_at = NOW();
44
+ RETURN NEW;
45
+ END;
46
+ $$ LANGUAGE plpgsql;
47
+
48
+ DROP TRIGGER IF EXISTS policy_match_feedback_updated_at ON public.policy_match_feedback;
49
+ CREATE TRIGGER policy_match_feedback_updated_at
50
+ BEFORE UPDATE ON public.policy_match_feedback
51
+ FOR EACH ROW EXECUTE FUNCTION update_policy_match_feedback_updated_at();
@@ -0,0 +1,76 @@
1
+ <!doctype html>
2
+ <html>
3
+
4
+ <head>
5
+ <style>
6
+ body {
7
+ font-family: Arial, sans-serif;
8
+ background-color: #f4f4f4;
9
+ margin: 0;
10
+ padding: 0;
11
+ }
12
+
13
+ .container {
14
+ width: 100%;
15
+ max-width: 600px;
16
+ margin: 0 auto;
17
+ background-color: #ffffff;
18
+ padding: 20px;
19
+ border-radius: 8px;
20
+ box-shadow: 0 0 10px rgba(0, 0, 0, 0.1);
21
+ }
22
+
23
+ .header {
24
+ text-align: center;
25
+ padding: 10px 0;
26
+ }
27
+
28
+ .header h2 {
29
+ margin: 0;
30
+ color: #333333;
31
+ }
32
+
33
+ .content {
34
+ padding: 20px;
35
+ line-height: 1.6;
36
+ color: #555555;
37
+ }
38
+
39
+ .content p {
40
+ margin: 0 0 10px;
41
+ }
42
+
43
+ .button {
44
+ display: block;
45
+ width: 200px;
46
+ margin: 20px auto;
47
+ padding: 10px;
48
+ text-align: center;
49
+ background-color: #007bff;
50
+ color: #ffffff !important;
51
+ text-decoration: none;
52
+ border-radius: 5px;
53
+ }
54
+ </style>
55
+ </head>
56
+
57
+ <body>
58
+ <div class="container">
59
+ <div class="header">
60
+ <h2>Confirm your email</h2>
61
+ </div>
62
+ <div class="content">
63
+ <p>Hello,</p>
64
+ <p>
65
+ Thanks for signing up for Folio! Please confirm your email
66
+ address by clicking the button below.
67
+ </p>
68
+ <p>
69
+ <a href="{{ .ConfirmationURL }}auth-callback.html" class="button">Confirm my email</a>
70
+ </p>
71
+ <p>The Folio team</p>
72
+ </div>
73
+ </div>
74
+ </body>
75
+
76
+ </html>