@realtimex/folio 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +20 -0
- package/README.md +63 -0
- package/api/server.ts +130 -0
- package/api/src/config/index.ts +96 -0
- package/api/src/middleware/auth.ts +128 -0
- package/api/src/middleware/errorHandler.ts +88 -0
- package/api/src/middleware/index.ts +4 -0
- package/api/src/middleware/rateLimit.ts +71 -0
- package/api/src/middleware/validation.ts +58 -0
- package/api/src/routes/accounts.ts +142 -0
- package/api/src/routes/baseline-config.ts +124 -0
- package/api/src/routes/chat.ts +154 -0
- package/api/src/routes/health.ts +61 -0
- package/api/src/routes/index.ts +35 -0
- package/api/src/routes/ingestions.ts +275 -0
- package/api/src/routes/migrate.ts +112 -0
- package/api/src/routes/policies.ts +121 -0
- package/api/src/routes/processing.ts +90 -0
- package/api/src/routes/rules.ts +11 -0
- package/api/src/routes/sdk.ts +100 -0
- package/api/src/routes/settings.ts +80 -0
- package/api/src/routes/setup.ts +389 -0
- package/api/src/routes/stats.ts +81 -0
- package/api/src/routes/tts.ts +190 -0
- package/api/src/services/BaselineConfigService.ts +208 -0
- package/api/src/services/ChatService.ts +204 -0
- package/api/src/services/GoogleDriveService.ts +331 -0
- package/api/src/services/GoogleSheetsService.ts +1107 -0
- package/api/src/services/IngestionService.ts +1187 -0
- package/api/src/services/ModelCapabilityService.ts +248 -0
- package/api/src/services/PolicyEngine.ts +1625 -0
- package/api/src/services/PolicyLearningService.ts +527 -0
- package/api/src/services/PolicyLoader.ts +249 -0
- package/api/src/services/RAGService.ts +391 -0
- package/api/src/services/SDKService.ts +249 -0
- package/api/src/services/supabase.ts +113 -0
- package/api/src/utils/Actuator.ts +284 -0
- package/api/src/utils/actions/ActionHandler.ts +34 -0
- package/api/src/utils/actions/AppendToGSheetAction.ts +260 -0
- package/api/src/utils/actions/AutoRenameAction.ts +58 -0
- package/api/src/utils/actions/CopyAction.ts +120 -0
- package/api/src/utils/actions/CopyToGDriveAction.ts +64 -0
- package/api/src/utils/actions/LogCsvAction.ts +48 -0
- package/api/src/utils/actions/NotifyAction.ts +39 -0
- package/api/src/utils/actions/RenameAction.ts +57 -0
- package/api/src/utils/actions/WebhookAction.ts +58 -0
- package/api/src/utils/actions/utils.ts +293 -0
- package/api/src/utils/llmResponse.ts +61 -0
- package/api/src/utils/logger.ts +67 -0
- package/bin/folio-deploy.js +12 -0
- package/bin/folio-setup.js +45 -0
- package/bin/folio.js +65 -0
- package/dist/api/server.js +106 -0
- package/dist/api/src/config/index.js +81 -0
- package/dist/api/src/middleware/auth.js +93 -0
- package/dist/api/src/middleware/errorHandler.js +73 -0
- package/dist/api/src/middleware/index.js +4 -0
- package/dist/api/src/middleware/rateLimit.js +43 -0
- package/dist/api/src/middleware/validation.js +54 -0
- package/dist/api/src/routes/accounts.js +110 -0
- package/dist/api/src/routes/baseline-config.js +91 -0
- package/dist/api/src/routes/chat.js +114 -0
- package/dist/api/src/routes/health.js +52 -0
- package/dist/api/src/routes/index.js +31 -0
- package/dist/api/src/routes/ingestions.js +207 -0
- package/dist/api/src/routes/migrate.js +91 -0
- package/dist/api/src/routes/policies.js +86 -0
- package/dist/api/src/routes/processing.js +75 -0
- package/dist/api/src/routes/rules.js +8 -0
- package/dist/api/src/routes/sdk.js +80 -0
- package/dist/api/src/routes/settings.js +68 -0
- package/dist/api/src/routes/setup.js +315 -0
- package/dist/api/src/routes/stats.js +62 -0
- package/dist/api/src/routes/tts.js +178 -0
- package/dist/api/src/services/BaselineConfigService.js +168 -0
- package/dist/api/src/services/ChatService.js +166 -0
- package/dist/api/src/services/GoogleDriveService.js +280 -0
- package/dist/api/src/services/GoogleSheetsService.js +795 -0
- package/dist/api/src/services/IngestionService.js +990 -0
- package/dist/api/src/services/ModelCapabilityService.js +179 -0
- package/dist/api/src/services/PolicyEngine.js +1353 -0
- package/dist/api/src/services/PolicyLearningService.js +397 -0
- package/dist/api/src/services/PolicyLoader.js +159 -0
- package/dist/api/src/services/RAGService.js +295 -0
- package/dist/api/src/services/SDKService.js +212 -0
- package/dist/api/src/services/supabase.js +72 -0
- package/dist/api/src/utils/Actuator.js +225 -0
- package/dist/api/src/utils/actions/ActionHandler.js +1 -0
- package/dist/api/src/utils/actions/AppendToGSheetAction.js +191 -0
- package/dist/api/src/utils/actions/AutoRenameAction.js +49 -0
- package/dist/api/src/utils/actions/CopyAction.js +112 -0
- package/dist/api/src/utils/actions/CopyToGDriveAction.js +55 -0
- package/dist/api/src/utils/actions/LogCsvAction.js +42 -0
- package/dist/api/src/utils/actions/NotifyAction.js +32 -0
- package/dist/api/src/utils/actions/RenameAction.js +51 -0
- package/dist/api/src/utils/actions/WebhookAction.js +51 -0
- package/dist/api/src/utils/actions/utils.js +237 -0
- package/dist/api/src/utils/llmResponse.js +63 -0
- package/dist/api/src/utils/logger.js +51 -0
- package/dist/assets/index-DzN8-j-e.css +1 -0
- package/dist/assets/index-Uy-ai3Dh.js +113 -0
- package/dist/favicon.svg +31 -0
- package/dist/folio-logo.svg +46 -0
- package/dist/index.html +14 -0
- package/docs-dev/FPE-spec.md +196 -0
- package/docs-dev/folio-prd.md +47 -0
- package/docs-dev/foundation-checklist.md +30 -0
- package/docs-dev/hybrid-routing-architecture.md +205 -0
- package/docs-dev/ingestion-engine.md +69 -0
- package/docs-dev/port-from-email-automator.md +32 -0
- package/docs-dev/tech-spec.md +98 -0
- package/index.html +13 -0
- package/package.json +101 -0
- package/public/favicon.svg +31 -0
- package/public/folio-logo.svg +46 -0
- package/scripts/dev-task.mjs +51 -0
- package/scripts/get-latest-migration-timestamp.mjs +34 -0
- package/scripts/migrate.sh +91 -0
- package/supabase/.temp/cli-latest +1 -0
- package/supabase/.temp/gotrue-version +1 -0
- package/supabase/.temp/pooler-url +1 -0
- package/supabase/.temp/postgres-version +1 -0
- package/supabase/.temp/project-ref +1 -0
- package/supabase/.temp/rest-version +1 -0
- package/supabase/.temp/storage-migration +1 -0
- package/supabase/.temp/storage-version +1 -0
- package/supabase/config.toml +64 -0
- package/supabase/functions/_shared/auth.ts +35 -0
- package/supabase/functions/_shared/cors.ts +12 -0
- package/supabase/functions/_shared/supabaseAdmin.ts +17 -0
- package/supabase/functions/api-v1-settings/index.ts +66 -0
- package/supabase/functions/setup/index.ts +91 -0
- package/supabase/migrations/20260223000000_initial_foundation.sql +136 -0
- package/supabase/migrations/20260223000001_add_migration_rpc.sql +10 -0
- package/supabase/migrations/20260224000002_add_init_state_view.sql +20 -0
- package/supabase/migrations/20260224000003_port_user_creation_parity.sql +139 -0
- package/supabase/migrations/20260224000004_add_avatars_storage.sql +26 -0
- package/supabase/migrations/20260224000005_add_tts_and_embed_settings.sql +24 -0
- package/supabase/migrations/20260224000006_add_policies_table.sql +48 -0
- package/supabase/migrations/20260224000007_fix_migration_rpc.sql +9 -0
- package/supabase/migrations/20260224000008_add_ingestions_table.sql +42 -0
- package/supabase/migrations/20260225000000_setup_compatible_mode.sql +119 -0
- package/supabase/migrations/20260225000001_restore_ingestions.sql +49 -0
- package/supabase/migrations/20260225000002_add_ingestion_trace.sql +2 -0
- package/supabase/migrations/20260225000003_add_baseline_configs.sql +35 -0
- package/supabase/migrations/20260226000000_add_processing_events.sql +26 -0
- package/supabase/migrations/20260226000001_add_ingestion_file_hash.sql +10 -0
- package/supabase/migrations/20260226000002_add_dynamic_rag.sql +150 -0
- package/supabase/migrations/20260226000003_add_ingestion_summary.sql +4 -0
- package/supabase/migrations/20260226000004_add_ingestion_tags.sql +7 -0
- package/supabase/migrations/20260226000005_add_chat_tables.sql +60 -0
- package/supabase/migrations/20260227000000_harden_chat_messages_rls.sql +25 -0
- package/supabase/migrations/20260228000000_add_vision_model_capabilities.sql +8 -0
- package/supabase/migrations/20260228000001_add_policy_match_feedback.sql +51 -0
- package/supabase/migrations/29991231235959_test_migration.sql +0 -0
- package/supabase/templates/confirmation.html +76 -0
- package/supabase/templates/email-change.html +76 -0
- package/supabase/templates/invite.html +72 -0
- package/supabase/templates/magic-link.html +68 -0
- package/supabase/templates/recovery.html +82 -0
- package/tsconfig.api.json +16 -0
- package/tsconfig.json +25 -0
- package/vite.config.ts +146 -0
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
-- Keep old ingestions table from previous migration step for Hybrid Routing
|
|
2
|
+
-- (Previously this dropped public.ingestions, but we now retain it as Folio's primary UI table).
|
|
3
|
+
-- Create rtx_activities table as defined in Compatible Mode docs
|
|
4
|
+
-- Added user_id for multi-tenant isolation
|
|
5
|
+
CREATE TABLE public.rtx_activities (
|
|
6
|
+
id uuid NOT NULL DEFAULT gen_random_uuid (),
|
|
7
|
+
user_id uuid NOT NULL REFERENCES auth.users(id) ON DELETE CASCADE,
|
|
8
|
+
raw_data jsonb NULL, -- Your input data
|
|
9
|
+
old_data jsonb NULL, -- Previous data (for updates)
|
|
10
|
+
locked_by text NULL, -- Machine ID holding the lock
|
|
11
|
+
locked_at timestamp with time zone NULL,
|
|
12
|
+
status text NULL DEFAULT 'pending'::text,
|
|
13
|
+
completed_at timestamp with time zone NULL,
|
|
14
|
+
error_message text NULL,
|
|
15
|
+
attempted_by text[] NULL DEFAULT '{}'::text[],
|
|
16
|
+
retry_count integer NULL DEFAULT 0,
|
|
17
|
+
result jsonb NULL,
|
|
18
|
+
created_at timestamp with time zone NULL DEFAULT now(),
|
|
19
|
+
CONSTRAINT rtx_activities_pkey PRIMARY KEY (id)
|
|
20
|
+
);
|
|
21
|
+
|
|
22
|
+
-- Index for status queries
|
|
23
|
+
CREATE INDEX idx_rtx_activities_status ON public.rtx_activities (status);
|
|
24
|
+
-- Index for user list queries
|
|
25
|
+
CREATE INDEX idx_rtx_activities_user ON public.rtx_activities (user_id, created_at DESC);
|
|
26
|
+
|
|
27
|
+
-- Ensure all columns are included in Realtime events
|
|
28
|
+
ALTER TABLE public.rtx_activities REPLICA IDENTITY FULL;
|
|
29
|
+
|
|
30
|
+
-- RLS
|
|
31
|
+
ALTER TABLE public.rtx_activities ENABLE ROW LEVEL SECURITY;
|
|
32
|
+
|
|
33
|
+
CREATE POLICY "Users can manage their own activities"
|
|
34
|
+
ON public.rtx_activities FOR ALL
|
|
35
|
+
USING (auth.uid() = user_id)
|
|
36
|
+
WITH CHECK (auth.uid() = user_id);
|
|
37
|
+
|
|
38
|
+
-- Database Functions (RPC)
|
|
39
|
+
|
|
40
|
+
-- Claim Task
|
|
41
|
+
CREATE OR REPLACE FUNCTION rtx_fn_claim_task(target_task_id UUID, machine_id TEXT)
|
|
42
|
+
RETURNS BOOLEAN AS $$
|
|
43
|
+
DECLARE
|
|
44
|
+
updated_rows INT;
|
|
45
|
+
BEGIN
|
|
46
|
+
UPDATE public.rtx_activities
|
|
47
|
+
SET status = 'claimed', locked_by = machine_id, locked_at = now()
|
|
48
|
+
WHERE id = target_task_id
|
|
49
|
+
AND (status = 'pending' OR status = 'failed'
|
|
50
|
+
OR ((status = 'claimed' OR status = 'processing')
|
|
51
|
+
AND locked_at < now() - INTERVAL '5 minutes'))
|
|
52
|
+
AND NOT (machine_id = ANY(attempted_by));
|
|
53
|
+
|
|
54
|
+
GET DIAGNOSTICS updated_rows = ROW_COUNT;
|
|
55
|
+
RETURN updated_rows > 0;
|
|
56
|
+
END;
|
|
57
|
+
$$ LANGUAGE plpgsql;
|
|
58
|
+
|
|
59
|
+
-- Complete Task
|
|
60
|
+
CREATE OR REPLACE FUNCTION rtx_fn_complete_task(target_task_id UUID, result_data JSONB)
|
|
61
|
+
RETURNS BOOLEAN AS $$
|
|
62
|
+
DECLARE
|
|
63
|
+
updated_rows INT;
|
|
64
|
+
BEGIN
|
|
65
|
+
UPDATE public.rtx_activities
|
|
66
|
+
SET status = 'completed', result = result_data, completed_at = now()
|
|
67
|
+
WHERE id = target_task_id AND (status = 'claimed' OR status = 'processing');
|
|
68
|
+
|
|
69
|
+
GET DIAGNOSTICS updated_rows = ROW_COUNT;
|
|
70
|
+
RETURN updated_rows > 0;
|
|
71
|
+
END;
|
|
72
|
+
$$ LANGUAGE plpgsql;
|
|
73
|
+
|
|
74
|
+
-- Fail Task
|
|
75
|
+
CREATE OR REPLACE FUNCTION rtx_fn_fail_task(target_task_id UUID, machine_id TEXT, error_msg TEXT)
|
|
76
|
+
RETURNS BOOLEAN AS $$
|
|
77
|
+
DECLARE
|
|
78
|
+
updated_rows INT;
|
|
79
|
+
BEGIN
|
|
80
|
+
UPDATE public.rtx_activities
|
|
81
|
+
SET
|
|
82
|
+
status = 'failed',
|
|
83
|
+
error_message = error_msg,
|
|
84
|
+
attempted_by = array_append(attempted_by, machine_id),
|
|
85
|
+
retry_count = retry_count + 1,
|
|
86
|
+
locked_by = NULL,
|
|
87
|
+
locked_at = NULL
|
|
88
|
+
WHERE id = target_task_id;
|
|
89
|
+
|
|
90
|
+
GET DIAGNOSTICS updated_rows = ROW_COUNT;
|
|
91
|
+
RETURN updated_rows > 0;
|
|
92
|
+
END;
|
|
93
|
+
$$ LANGUAGE plpgsql;
|
|
94
|
+
|
|
95
|
+
-- Enable pg_cron extension
|
|
96
|
+
CREATE EXTENSION IF NOT EXISTS pg_cron;
|
|
97
|
+
|
|
98
|
+
-- Create cleanup function
|
|
99
|
+
CREATE OR REPLACE FUNCTION public.rtx_fn_unlock_stale_locks()
|
|
100
|
+
RETURNS void AS $$
|
|
101
|
+
BEGIN
|
|
102
|
+
UPDATE public.rtx_activities
|
|
103
|
+
SET status = 'pending', locked_by = NULL, locked_at = NULL
|
|
104
|
+
WHERE (status = 'claimed' OR status = 'processing')
|
|
105
|
+
AND locked_at < now() - INTERVAL '5 minutes';
|
|
106
|
+
END;
|
|
107
|
+
$$ LANGUAGE plpgsql;
|
|
108
|
+
|
|
109
|
+
-- Avoid conflicts if it exists already, just drop it and recreate
|
|
110
|
+
DO $$
|
|
111
|
+
BEGIN
|
|
112
|
+
PERFORM cron.unschedule('scavenge-stale-locks');
|
|
113
|
+
EXCEPTION WHEN OTHERS THEN
|
|
114
|
+
-- ignore
|
|
115
|
+
END $$;
|
|
116
|
+
SELECT cron.schedule('scavenge-stale-locks', '* * * * *', 'SELECT public.rtx_fn_unlock_stale_locks();');
|
|
117
|
+
|
|
118
|
+
-- Enable Realtime
|
|
119
|
+
ALTER PUBLICATION supabase_realtime ADD TABLE public.rtx_activities;
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
-- Ingestion log table: tracks every document processed through Folio's Policy Engine
|
|
2
|
+
-- (Restoring this table for the Hybrid Routing Architecture, since the previous
|
|
3
|
+
-- compatible_mode migration erroneously dropped it).
|
|
4
|
+
CREATE TABLE IF NOT EXISTS ingestions (
|
|
5
|
+
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
6
|
+
user_id uuid NOT NULL REFERENCES auth.users(id) ON DELETE CASCADE,
|
|
7
|
+
source text NOT NULL DEFAULT 'upload', -- 'upload' | 'dropzone' | 'email' | 'url'
|
|
8
|
+
filename text NOT NULL,
|
|
9
|
+
mime_type text,
|
|
10
|
+
file_size bigint,
|
|
11
|
+
status text NOT NULL DEFAULT 'pending', -- 'pending' | 'processing' | 'matched' | 'no_match' | 'error'
|
|
12
|
+
policy_id text, -- matched policy id (nullable)
|
|
13
|
+
policy_name text, -- denormalised for display
|
|
14
|
+
extracted jsonb DEFAULT '{}'::jsonb, -- key/value pairs extracted by FPE
|
|
15
|
+
actions_taken jsonb DEFAULT '[]'::jsonb, -- list of actions executed
|
|
16
|
+
error_message text,
|
|
17
|
+
storage_path text, -- file path pointer for Hybrid Routing or Supabase Storage ID
|
|
18
|
+
created_at timestamptz NOT NULL DEFAULT now(),
|
|
19
|
+
updated_at timestamptz NOT NULL DEFAULT now()
|
|
20
|
+
);
|
|
21
|
+
|
|
22
|
+
-- Index for per-user list queries
|
|
23
|
+
CREATE INDEX IF NOT EXISTS ingestions_user_id_idx ON ingestions(user_id, created_at DESC);
|
|
24
|
+
|
|
25
|
+
-- RLS
|
|
26
|
+
ALTER TABLE ingestions ENABLE ROW LEVEL SECURITY;
|
|
27
|
+
|
|
28
|
+
DO $$ BEGIN
|
|
29
|
+
CREATE POLICY "Users can manage their own ingestions"
|
|
30
|
+
ON ingestions FOR ALL
|
|
31
|
+
USING (auth.uid() = user_id)
|
|
32
|
+
WITH CHECK (auth.uid() = user_id);
|
|
33
|
+
EXCEPTION
|
|
34
|
+
WHEN duplicate_object THEN null;
|
|
35
|
+
END $$;
|
|
36
|
+
|
|
37
|
+
-- updated_at trigger
|
|
38
|
+
CREATE OR REPLACE FUNCTION update_ingestions_updated_at()
|
|
39
|
+
RETURNS TRIGGER AS $$
|
|
40
|
+
BEGIN
|
|
41
|
+
NEW.updated_at = now();
|
|
42
|
+
RETURN NEW;
|
|
43
|
+
END;
|
|
44
|
+
$$ LANGUAGE plpgsql;
|
|
45
|
+
|
|
46
|
+
DROP TRIGGER IF EXISTS ingestions_updated_at ON ingestions;
|
|
47
|
+
CREATE TRIGGER ingestions_updated_at
|
|
48
|
+
BEFORE UPDATE ON ingestions
|
|
49
|
+
FOR EACH ROW EXECUTE FUNCTION update_ingestions_updated_at();
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
-- Versioned baseline extraction configuration per user.
|
|
2
|
+
-- Each row is immutable once referenced by an ingestion record,
|
|
3
|
+
-- enabling full auditability of which prompt config produced each extraction.
|
|
4
|
+
|
|
5
|
+
CREATE TABLE public.baseline_configs (
|
|
6
|
+
id uuid NOT NULL DEFAULT gen_random_uuid(),
|
|
7
|
+
user_id uuid NOT NULL REFERENCES auth.users(id) ON DELETE CASCADE,
|
|
8
|
+
version integer NOT NULL,
|
|
9
|
+
context text NULL, -- free-text injected into the extraction system prompt
|
|
10
|
+
fields jsonb NOT NULL DEFAULT '[]', -- array of BaselineField definitions
|
|
11
|
+
is_active boolean NOT NULL DEFAULT false,
|
|
12
|
+
created_at timestamptz NOT NULL DEFAULT now(),
|
|
13
|
+
|
|
14
|
+
CONSTRAINT baseline_configs_pkey PRIMARY KEY (id),
|
|
15
|
+
CONSTRAINT baseline_configs_user_version_key UNIQUE (user_id, version)
|
|
16
|
+
);
|
|
17
|
+
|
|
18
|
+
-- Fast lookup of a user's active config
|
|
19
|
+
CREATE INDEX idx_baseline_configs_user_active
|
|
20
|
+
ON public.baseline_configs (user_id, is_active);
|
|
21
|
+
|
|
22
|
+
-- RLS
|
|
23
|
+
ALTER TABLE public.baseline_configs ENABLE ROW LEVEL SECURITY;
|
|
24
|
+
|
|
25
|
+
CREATE POLICY "Users manage own baseline configs"
|
|
26
|
+
ON public.baseline_configs FOR ALL
|
|
27
|
+
USING (auth.uid() = user_id)
|
|
28
|
+
WITH CHECK (auth.uid() = user_id);
|
|
29
|
+
|
|
30
|
+
-- Record which config version produced each ingestion's extraction.
|
|
31
|
+
-- NULL means the ingestion was processed before this feature existed
|
|
32
|
+
-- or that the default built-in fields were used.
|
|
33
|
+
ALTER TABLE public.ingestions
|
|
34
|
+
ADD COLUMN baseline_config_id uuid NULL
|
|
35
|
+
REFERENCES public.baseline_configs(id) ON DELETE SET NULL;
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
-- Create processing_events table for granular logging
|
|
2
|
+
CREATE TABLE IF NOT EXISTS processing_events (
|
|
3
|
+
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
4
|
+
ingestion_id UUID REFERENCES ingestions(id) ON DELETE CASCADE,
|
|
5
|
+
user_id UUID REFERENCES auth.users(id) ON DELETE CASCADE,
|
|
6
|
+
event_type TEXT NOT NULL CHECK (event_type IN ('info', 'analysis', 'action', 'error')),
|
|
7
|
+
agent_state TEXT, -- e.g., 'Triage', 'Baseline Extraction', 'Policy Matching', 'Action Execution'
|
|
8
|
+
details JSONB, -- Stores LLM inputs/outputs, reasoning, confidence
|
|
9
|
+
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
|
|
10
|
+
);
|
|
11
|
+
|
|
12
|
+
-- Enable RLS
|
|
13
|
+
ALTER TABLE processing_events ENABLE ROW LEVEL SECURITY;
|
|
14
|
+
|
|
15
|
+
-- RLS Policy: Users can access their own processing events
|
|
16
|
+
CREATE POLICY "Users can access their own processing events" ON processing_events
|
|
17
|
+
FOR ALL USING (auth.uid() = user_id);
|
|
18
|
+
|
|
19
|
+
-- Enable Realtime for this table (standard Supabase publication)
|
|
20
|
+
DO $$
|
|
21
|
+
BEGIN
|
|
22
|
+
IF NOT EXISTS (SELECT 1 FROM pg_publication_tables WHERE pubname = 'supabase_realtime' AND schemaname = 'public' AND tablename = 'processing_events') THEN
|
|
23
|
+
ALTER PUBLICATION supabase_realtime ADD TABLE processing_events;
|
|
24
|
+
END IF;
|
|
25
|
+
END
|
|
26
|
+
$$;
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
-- Add file_hash column to ingestions for duplicate detection.
|
|
2
|
+
-- SHA-256 hex digest (64 chars) of the raw file bytes.
|
|
3
|
+
-- Nullable so existing rows are unaffected.
|
|
4
|
+
ALTER TABLE ingestions ADD COLUMN IF NOT EXISTS file_hash TEXT;
|
|
5
|
+
|
|
6
|
+
-- Partial index: only non-null hashes, scoped per user — makes the
|
|
7
|
+
-- duplicate lookup (user_id + file_hash) fast without indexing NULLs.
|
|
8
|
+
CREATE INDEX IF NOT EXISTS idx_ingestions_user_file_hash
|
|
9
|
+
ON ingestions (user_id, file_hash)
|
|
10
|
+
WHERE file_hash IS NOT NULL;
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
-- Enable pgvector extension for vector similarity search
|
|
2
|
+
CREATE EXTENSION IF NOT EXISTS vector;
|
|
3
|
+
|
|
4
|
+
-- Document chunks table for Semantic Search (RAG)
|
|
5
|
+
CREATE TABLE IF NOT EXISTS document_chunks (
|
|
6
|
+
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
7
|
+
user_id UUID NOT NULL REFERENCES auth.users(id) ON DELETE CASCADE,
|
|
8
|
+
ingestion_id UUID NOT NULL REFERENCES ingestions(id) ON DELETE CASCADE,
|
|
9
|
+
|
|
10
|
+
-- Content
|
|
11
|
+
content TEXT NOT NULL,
|
|
12
|
+
content_hash TEXT NOT NULL, -- To detect duplicates
|
|
13
|
+
embedding_provider TEXT NOT NULL,
|
|
14
|
+
embedding_model TEXT NOT NULL,
|
|
15
|
+
|
|
16
|
+
-- Unconstrained Vector Embedding
|
|
17
|
+
-- By not defining the dimension (e.g. vector(1536)), we allow multiple
|
|
18
|
+
-- embedding models to coexist in the same table seamlessly.
|
|
19
|
+
embedding vector NOT NULL,
|
|
20
|
+
vector_dim INTEGER NOT NULL,
|
|
21
|
+
|
|
22
|
+
created_at TIMESTAMPTZ DEFAULT NOW(),
|
|
23
|
+
|
|
24
|
+
-- Prevent duplicate chunks for the same document + model identity.
|
|
25
|
+
-- This allows hot-swapping models while keeping each model's vector space isolated.
|
|
26
|
+
UNIQUE(ingestion_id, content_hash, embedding_provider, embedding_model)
|
|
27
|
+
);
|
|
28
|
+
|
|
29
|
+
-- Enable RLS
|
|
30
|
+
ALTER TABLE document_chunks ENABLE ROW LEVEL SECURITY;
|
|
31
|
+
|
|
32
|
+
CREATE POLICY "Users can manage their own document chunks"
|
|
33
|
+
ON document_chunks
|
|
34
|
+
FOR ALL
|
|
35
|
+
USING (user_id = auth.uid())
|
|
36
|
+
WITH CHECK (user_id = auth.uid());
|
|
37
|
+
|
|
38
|
+
-- Advanced Partial Indexes for HNSW
|
|
39
|
+
-- We create partial indexes for the most common dimensions to maintain
|
|
40
|
+
-- sub-millisecond search performance while keeping the 'embedding' column unconstrained.
|
|
41
|
+
CREATE INDEX IF NOT EXISTS document_chunks_embedding_384_idx
|
|
42
|
+
ON document_chunks USING hnsw ((embedding::vector(384)) vector_cosine_ops)
|
|
43
|
+
WHERE (vector_dim = 384);
|
|
44
|
+
|
|
45
|
+
CREATE INDEX IF NOT EXISTS document_chunks_embedding_768_idx
|
|
46
|
+
ON document_chunks USING hnsw ((embedding::vector(768)) vector_cosine_ops)
|
|
47
|
+
WHERE (vector_dim = 768);
|
|
48
|
+
|
|
49
|
+
CREATE INDEX IF NOT EXISTS document_chunks_embedding_1536_idx
|
|
50
|
+
ON document_chunks USING hnsw ((embedding::vector(1536)) vector_cosine_ops)
|
|
51
|
+
WHERE (vector_dim = 1536);
|
|
52
|
+
|
|
53
|
+
-- Index for fast deletion and lookup by ingestion_id
|
|
54
|
+
CREATE INDEX IF NOT EXISTS document_chunks_ingestion_id_idx
|
|
55
|
+
ON document_chunks(ingestion_id);
|
|
56
|
+
|
|
57
|
+
CREATE INDEX IF NOT EXISTS document_chunks_user_id_idx
|
|
58
|
+
ON document_chunks(user_id);
|
|
59
|
+
|
|
60
|
+
CREATE INDEX IF NOT EXISTS document_chunks_model_scope_idx
|
|
61
|
+
ON document_chunks(user_id, embedding_provider, embedding_model, vector_dim);
|
|
62
|
+
|
|
63
|
+
-- Dynamic Semantic Search Function
|
|
64
|
+
CREATE OR REPLACE FUNCTION search_documents(
|
|
65
|
+
p_user_id UUID,
|
|
66
|
+
p_embedding_provider TEXT,
|
|
67
|
+
p_embedding_model TEXT,
|
|
68
|
+
query_embedding vector,
|
|
69
|
+
match_threshold float DEFAULT 0.7,
|
|
70
|
+
match_count int DEFAULT 5,
|
|
71
|
+
query_dim int DEFAULT 1536
|
|
72
|
+
)
|
|
73
|
+
RETURNS TABLE (
|
|
74
|
+
id UUID,
|
|
75
|
+
ingestion_id UUID,
|
|
76
|
+
content TEXT,
|
|
77
|
+
similarity float
|
|
78
|
+
)
|
|
79
|
+
LANGUAGE plpgsql
|
|
80
|
+
AS $$
|
|
81
|
+
BEGIN
|
|
82
|
+
IF query_dim = 384 THEN
|
|
83
|
+
RETURN QUERY
|
|
84
|
+
SELECT
|
|
85
|
+
dc.id,
|
|
86
|
+
dc.ingestion_id,
|
|
87
|
+
dc.content,
|
|
88
|
+
1 - (dc.embedding::vector(384) <=> query_embedding::vector(384)) AS similarity
|
|
89
|
+
FROM document_chunks dc
|
|
90
|
+
WHERE dc.user_id = p_user_id
|
|
91
|
+
AND dc.embedding_provider = p_embedding_provider
|
|
92
|
+
AND dc.embedding_model = p_embedding_model
|
|
93
|
+
AND dc.vector_dim = 384
|
|
94
|
+
AND 1 - (dc.embedding::vector(384) <=> query_embedding::vector(384)) > match_threshold
|
|
95
|
+
ORDER BY dc.embedding::vector(384) <=> query_embedding::vector(384)
|
|
96
|
+
LIMIT match_count;
|
|
97
|
+
ELSIF query_dim = 768 THEN
|
|
98
|
+
RETURN QUERY
|
|
99
|
+
SELECT
|
|
100
|
+
dc.id,
|
|
101
|
+
dc.ingestion_id,
|
|
102
|
+
dc.content,
|
|
103
|
+
1 - (dc.embedding::vector(768) <=> query_embedding::vector(768)) AS similarity
|
|
104
|
+
FROM document_chunks dc
|
|
105
|
+
WHERE dc.user_id = p_user_id
|
|
106
|
+
AND dc.embedding_provider = p_embedding_provider
|
|
107
|
+
AND dc.embedding_model = p_embedding_model
|
|
108
|
+
AND dc.vector_dim = 768
|
|
109
|
+
AND 1 - (dc.embedding::vector(768) <=> query_embedding::vector(768)) > match_threshold
|
|
110
|
+
ORDER BY dc.embedding::vector(768) <=> query_embedding::vector(768)
|
|
111
|
+
LIMIT match_count;
|
|
112
|
+
ELSIF query_dim = 1536 THEN
|
|
113
|
+
RETURN QUERY
|
|
114
|
+
SELECT
|
|
115
|
+
dc.id,
|
|
116
|
+
dc.ingestion_id,
|
|
117
|
+
dc.content,
|
|
118
|
+
1 - (dc.embedding::vector(1536) <=> query_embedding::vector(1536)) AS similarity
|
|
119
|
+
FROM document_chunks dc
|
|
120
|
+
WHERE dc.user_id = p_user_id
|
|
121
|
+
AND dc.embedding_provider = p_embedding_provider
|
|
122
|
+
AND dc.embedding_model = p_embedding_model
|
|
123
|
+
AND dc.vector_dim = 1536
|
|
124
|
+
AND 1 - (dc.embedding::vector(1536) <=> query_embedding::vector(1536)) > match_threshold
|
|
125
|
+
ORDER BY dc.embedding::vector(1536) <=> query_embedding::vector(1536)
|
|
126
|
+
LIMIT match_count;
|
|
127
|
+
ELSE
|
|
128
|
+
-- Fallback to unconstrained exact nearest neighbor scan for unindexed dimensions
|
|
129
|
+
RETURN QUERY
|
|
130
|
+
SELECT
|
|
131
|
+
dc.id,
|
|
132
|
+
dc.ingestion_id,
|
|
133
|
+
dc.content,
|
|
134
|
+
1 - (dc.embedding <=> query_embedding) AS similarity
|
|
135
|
+
FROM document_chunks dc
|
|
136
|
+
WHERE dc.user_id = p_user_id
|
|
137
|
+
AND dc.embedding_provider = p_embedding_provider
|
|
138
|
+
AND dc.embedding_model = p_embedding_model
|
|
139
|
+
AND dc.vector_dim = query_dim
|
|
140
|
+
AND 1 - (dc.embedding <=> query_embedding) > match_threshold
|
|
141
|
+
ORDER BY dc.embedding <=> query_embedding
|
|
142
|
+
LIMIT match_count;
|
|
143
|
+
END IF;
|
|
144
|
+
END;
|
|
145
|
+
$$;
|
|
146
|
+
|
|
147
|
+
-- Comments
|
|
148
|
+
COMMENT ON TABLE document_chunks IS 'Stores semantic text chunks from parsed documents for RAG.';
|
|
149
|
+
COMMENT ON COLUMN document_chunks.embedding IS 'Unconstrained vector to support dynamic embedding models.';
|
|
150
|
+
COMMENT ON FUNCTION search_documents IS 'Performs dynamic cosine similarity search against unconstrained vectors.';
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
-- Add LLM-generated + human-curated tags to ingestions.
|
|
2
|
+
-- GIN index enables efficient array-containment queries (e.g. @> '{tax-deductible}').
|
|
3
|
+
ALTER TABLE public.ingestions
|
|
4
|
+
ADD COLUMN IF NOT EXISTS tags text[] NOT NULL DEFAULT '{}';
|
|
5
|
+
|
|
6
|
+
CREATE INDEX IF NOT EXISTS ingestions_tags_gin_idx
|
|
7
|
+
ON public.ingestions USING gin(tags);
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
-- Chat Sessions Table
|
|
2
|
+
CREATE TABLE IF NOT EXISTS chat_sessions (
|
|
3
|
+
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
4
|
+
user_id UUID NOT NULL REFERENCES auth.users(id) ON DELETE CASCADE,
|
|
5
|
+
title TEXT NOT NULL DEFAULT 'New Conversation',
|
|
6
|
+
created_at TIMESTAMPTZ DEFAULT NOW(),
|
|
7
|
+
updated_at TIMESTAMPTZ DEFAULT NOW()
|
|
8
|
+
);
|
|
9
|
+
|
|
10
|
+
-- Chat Messages Table
|
|
11
|
+
CREATE TABLE IF NOT EXISTS chat_messages (
|
|
12
|
+
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
13
|
+
session_id UUID NOT NULL REFERENCES chat_sessions(id) ON DELETE CASCADE,
|
|
14
|
+
user_id UUID NOT NULL REFERENCES auth.users(id) ON DELETE CASCADE,
|
|
15
|
+
|
|
16
|
+
role TEXT NOT NULL CHECK (role IN ('user', 'assistant', 'system')),
|
|
17
|
+
content TEXT NOT NULL,
|
|
18
|
+
context_sources JSONB DEFAULT '[]'::jsonb, -- Store the RAG chunks used to answer this message
|
|
19
|
+
|
|
20
|
+
created_at TIMESTAMPTZ DEFAULT NOW()
|
|
21
|
+
);
|
|
22
|
+
|
|
23
|
+
-- Enable RLS
|
|
24
|
+
ALTER TABLE chat_sessions ENABLE ROW LEVEL SECURITY;
|
|
25
|
+
ALTER TABLE chat_messages ENABLE ROW LEVEL SECURITY;
|
|
26
|
+
|
|
27
|
+
-- Policies for chat_sessions
|
|
28
|
+
CREATE POLICY "Users can manage their own chat sessions"
|
|
29
|
+
ON chat_sessions
|
|
30
|
+
FOR ALL
|
|
31
|
+
USING (user_id = auth.uid())
|
|
32
|
+
WITH CHECK (user_id = auth.uid());
|
|
33
|
+
|
|
34
|
+
-- Policies for chat_messages
|
|
35
|
+
CREATE POLICY "Users can manage their own chat messages"
|
|
36
|
+
ON chat_messages
|
|
37
|
+
FOR ALL
|
|
38
|
+
USING (user_id = auth.uid())
|
|
39
|
+
WITH CHECK (user_id = auth.uid());
|
|
40
|
+
|
|
41
|
+
-- Indexes for performance
|
|
42
|
+
CREATE INDEX IF NOT EXISTS chat_sessions_user_id_idx ON chat_sessions(user_id);
|
|
43
|
+
CREATE INDEX IF NOT EXISTS chat_messages_session_id_idx ON chat_messages(session_id);
|
|
44
|
+
CREATE INDEX IF NOT EXISTS chat_messages_user_id_idx ON chat_messages(user_id);
|
|
45
|
+
|
|
46
|
+
-- Trigger to update session updated_at
|
|
47
|
+
CREATE OR REPLACE FUNCTION update_chat_session_timestamp()
|
|
48
|
+
RETURNS TRIGGER AS $$
|
|
49
|
+
BEGIN
|
|
50
|
+
UPDATE chat_sessions
|
|
51
|
+
SET updated_at = NOW()
|
|
52
|
+
WHERE id = NEW.session_id;
|
|
53
|
+
RETURN NEW;
|
|
54
|
+
END;
|
|
55
|
+
$$ LANGUAGE plpgsql;
|
|
56
|
+
|
|
57
|
+
CREATE TRIGGER trigger_update_chat_session_timestamp
|
|
58
|
+
AFTER INSERT ON chat_messages
|
|
59
|
+
FOR EACH ROW
|
|
60
|
+
EXECUTE FUNCTION update_chat_session_timestamp();
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
-- Tighten chat message authorization to require ownership of both the row user_id
|
|
2
|
+
-- and the referenced session. This prevents cross-session inserts/selects.
|
|
3
|
+
DROP POLICY IF EXISTS "Users can manage their own chat messages" ON chat_messages;
|
|
4
|
+
|
|
5
|
+
CREATE POLICY "Users can manage their own chat messages"
|
|
6
|
+
ON chat_messages
|
|
7
|
+
FOR ALL
|
|
8
|
+
USING (
|
|
9
|
+
user_id = auth.uid()
|
|
10
|
+
AND EXISTS (
|
|
11
|
+
SELECT 1
|
|
12
|
+
FROM chat_sessions cs
|
|
13
|
+
WHERE cs.id = chat_messages.session_id
|
|
14
|
+
AND cs.user_id = auth.uid()
|
|
15
|
+
)
|
|
16
|
+
)
|
|
17
|
+
WITH CHECK (
|
|
18
|
+
user_id = auth.uid()
|
|
19
|
+
AND EXISTS (
|
|
20
|
+
SELECT 1
|
|
21
|
+
FROM chat_sessions cs
|
|
22
|
+
WHERE cs.id = chat_messages.session_id
|
|
23
|
+
AND cs.user_id = auth.uid()
|
|
24
|
+
)
|
|
25
|
+
);
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
-- Migration: Persist learned vision capability state per user model
|
|
2
|
+
-- Created: 2026-02-28
|
|
3
|
+
|
|
4
|
+
ALTER TABLE public.user_settings
|
|
5
|
+
ADD COLUMN IF NOT EXISTS vision_model_capabilities JSONB NOT NULL DEFAULT '{}'::jsonb;
|
|
6
|
+
|
|
7
|
+
COMMENT ON COLUMN public.user_settings.vision_model_capabilities IS
|
|
8
|
+
'Learned VLM capability map keyed by provider:model with state/TTL metadata.';
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
-- Stores explicit user feedback when they manually map an ingestion to a policy.
|
|
2
|
+
-- Used to improve future policy matching via lightweight similarity scoring.
|
|
3
|
+
CREATE TABLE IF NOT EXISTS public.policy_match_feedback (
|
|
4
|
+
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
5
|
+
user_id UUID NOT NULL REFERENCES auth.users(id) ON DELETE CASCADE,
|
|
6
|
+
ingestion_id UUID NOT NULL REFERENCES public.ingestions(id) ON DELETE CASCADE,
|
|
7
|
+
policy_id TEXT NOT NULL,
|
|
8
|
+
policy_name TEXT,
|
|
9
|
+
feedback_type TEXT NOT NULL DEFAULT 'manual_match' CHECK (feedback_type IN ('manual_match')),
|
|
10
|
+
features JSONB NOT NULL DEFAULT '{}'::jsonb,
|
|
11
|
+
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
|
12
|
+
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
|
13
|
+
UNIQUE(user_id, ingestion_id, policy_id)
|
|
14
|
+
);
|
|
15
|
+
|
|
16
|
+
CREATE INDEX IF NOT EXISTS policy_match_feedback_user_policy_idx
|
|
17
|
+
ON public.policy_match_feedback(user_id, policy_id, created_at DESC);
|
|
18
|
+
|
|
19
|
+
CREATE INDEX IF NOT EXISTS policy_match_feedback_user_created_idx
|
|
20
|
+
ON public.policy_match_feedback(user_id, created_at DESC);
|
|
21
|
+
|
|
22
|
+
ALTER TABLE public.policy_match_feedback ENABLE ROW LEVEL SECURITY;
|
|
23
|
+
|
|
24
|
+
CREATE POLICY "Users can read own policy match feedback"
|
|
25
|
+
ON public.policy_match_feedback FOR SELECT
|
|
26
|
+
USING (auth.uid() = user_id);
|
|
27
|
+
|
|
28
|
+
CREATE POLICY "Users can insert own policy match feedback"
|
|
29
|
+
ON public.policy_match_feedback FOR INSERT
|
|
30
|
+
WITH CHECK (auth.uid() = user_id);
|
|
31
|
+
|
|
32
|
+
CREATE POLICY "Users can update own policy match feedback"
|
|
33
|
+
ON public.policy_match_feedback FOR UPDATE
|
|
34
|
+
USING (auth.uid() = user_id);
|
|
35
|
+
|
|
36
|
+
CREATE POLICY "Users can delete own policy match feedback"
|
|
37
|
+
ON public.policy_match_feedback FOR DELETE
|
|
38
|
+
USING (auth.uid() = user_id);
|
|
39
|
+
|
|
40
|
+
CREATE OR REPLACE FUNCTION update_policy_match_feedback_updated_at()
|
|
41
|
+
RETURNS TRIGGER AS $$
|
|
42
|
+
BEGIN
|
|
43
|
+
NEW.updated_at = NOW();
|
|
44
|
+
RETURN NEW;
|
|
45
|
+
END;
|
|
46
|
+
$$ LANGUAGE plpgsql;
|
|
47
|
+
|
|
48
|
+
DROP TRIGGER IF EXISTS policy_match_feedback_updated_at ON public.policy_match_feedback;
|
|
49
|
+
CREATE TRIGGER policy_match_feedback_updated_at
|
|
50
|
+
BEFORE UPDATE ON public.policy_match_feedback
|
|
51
|
+
FOR EACH ROW EXECUTE FUNCTION update_policy_match_feedback_updated_at();
|
|
File without changes
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
<!doctype html>
|
|
2
|
+
<html>
|
|
3
|
+
|
|
4
|
+
<head>
|
|
5
|
+
<style>
|
|
6
|
+
body {
|
|
7
|
+
font-family: Arial, sans-serif;
|
|
8
|
+
background-color: #f4f4f4;
|
|
9
|
+
margin: 0;
|
|
10
|
+
padding: 0;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
.container {
|
|
14
|
+
width: 100%;
|
|
15
|
+
max-width: 600px;
|
|
16
|
+
margin: 0 auto;
|
|
17
|
+
background-color: #ffffff;
|
|
18
|
+
padding: 20px;
|
|
19
|
+
border-radius: 8px;
|
|
20
|
+
box-shadow: 0 0 10px rgba(0, 0, 0, 0.1);
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
.header {
|
|
24
|
+
text-align: center;
|
|
25
|
+
padding: 10px 0;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
.header h2 {
|
|
29
|
+
margin: 0;
|
|
30
|
+
color: #333333;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
.content {
|
|
34
|
+
padding: 20px;
|
|
35
|
+
line-height: 1.6;
|
|
36
|
+
color: #555555;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
.content p {
|
|
40
|
+
margin: 0 0 10px;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
.button {
|
|
44
|
+
display: block;
|
|
45
|
+
width: 200px;
|
|
46
|
+
margin: 20px auto;
|
|
47
|
+
padding: 10px;
|
|
48
|
+
text-align: center;
|
|
49
|
+
background-color: #007bff;
|
|
50
|
+
color: #ffffff !important;
|
|
51
|
+
text-decoration: none;
|
|
52
|
+
border-radius: 5px;
|
|
53
|
+
}
|
|
54
|
+
</style>
|
|
55
|
+
</head>
|
|
56
|
+
|
|
57
|
+
<body>
|
|
58
|
+
<div class="container">
|
|
59
|
+
<div class="header">
|
|
60
|
+
<h2>Confirm your email</h2>
|
|
61
|
+
</div>
|
|
62
|
+
<div class="content">
|
|
63
|
+
<p>Hello,</p>
|
|
64
|
+
<p>
|
|
65
|
+
Thanks for signing up for Folio! Please confirm your email
|
|
66
|
+
address by clicking the button below.
|
|
67
|
+
</p>
|
|
68
|
+
<p>
|
|
69
|
+
<a href="{{ .ConfirmationURL }}auth-callback.html" class="button">Confirm my email</a>
|
|
70
|
+
</p>
|
|
71
|
+
<p>The Folio team</p>
|
|
72
|
+
</div>
|
|
73
|
+
</div>
|
|
74
|
+
</body>
|
|
75
|
+
|
|
76
|
+
</html>
|