@arela/uploader 0.2.12 → 0.3.0

package/supabase/migrations/001_create_initial_schema.sql

@@ -0,0 +1,366 @@
+-- Migration: Create Initial Schema for ARELA-UPLOADER
+-- Date: 2025-11-14
+-- Description: Creates all necessary tables for file uploading, watch monitoring, and session tracking
+
+-- ============================================================================
+-- Main Uploader Table
+-- Stores metadata about files being uploaded
+-- ============================================================================
+CREATE TABLE IF NOT EXISTS uploader (
+  id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+  
+  -- File Identification
+  filename VARCHAR(255) NOT NULL,
+  original_path VARCHAR(2048) NOT NULL UNIQUE,
+  file_extension VARCHAR(10),
+  size BIGINT,
+  
+  -- Document Detection
+  document_type VARCHAR(100),
+  num_pedimento VARCHAR(50),
+  is_like_simplificado BOOLEAN DEFAULT FALSE,
+  year INTEGER,
+  
+  -- Upload Status
+  status VARCHAR(50) DEFAULT 'stats',
+  -- Possible values: 'stats', 'fs-stats', 'detected', 'not-detected', 
+  -- 'detection-error', 'unsupported', 'ready', 'uploading', 'uploaded', 'failed'
+  
+  -- Upload Details
+  arela_path VARCHAR(2048),
+  rfc VARCHAR(13),
+  message TEXT,
+  
+  -- Timestamps
+  created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+  modified_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+  uploaded_at TIMESTAMP,
+  
+  -- Metadata
+  metadata JSONB
+);
+
+-- Indexes for uploader table
+CREATE INDEX IF NOT EXISTS idx_uploader_status ON uploader(status);
+CREATE INDEX IF NOT EXISTS idx_uploader_rfc ON uploader(rfc);
+CREATE INDEX IF NOT EXISTS idx_uploader_year ON uploader(year);
+CREATE INDEX IF NOT EXISTS idx_uploader_document_type ON uploader(document_type);
+CREATE INDEX IF NOT EXISTS idx_uploader_original_path ON uploader(original_path);
+CREATE INDEX IF NOT EXISTS idx_uploader_created_at ON uploader(created_at);
+CREATE INDEX IF NOT EXISTS idx_uploader_num_pedimento ON uploader(num_pedimento);
+
+
+-- ============================================================================
+-- Watch Sessions Table
+-- Tracks individual watch monitoring sessions
+-- ============================================================================
+CREATE TABLE IF NOT EXISTS watch_sessions (
+  id VARCHAR(36) PRIMARY KEY,
+  
+  -- Session Metadata
+  user_id VARCHAR(255),
+  start_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  end_time TIMESTAMP,
+  
+  -- Session Status
+  status VARCHAR(20) DEFAULT 'active',
+  -- Possible values: 'active', 'paused', 'completed', 'error', 'interrupted'
+  
+  -- Statistics
+  total_uploads INTEGER DEFAULT 0,
+  total_retries INTEGER DEFAULT 0,
+  total_files_processed INTEGER DEFAULT 0,
+  total_errors INTEGER DEFAULT 0,
+  
+  -- Timestamps
+  created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+  updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+  
+  -- Metadata
+  metadata JSONB
+);
+
+-- Indexes for watch_sessions table
+CREATE INDEX IF NOT EXISTS idx_watch_sessions_status ON watch_sessions(status);
+CREATE INDEX IF NOT EXISTS idx_watch_sessions_start_time ON watch_sessions(start_time);
+CREATE INDEX IF NOT EXISTS idx_watch_sessions_user_id ON watch_sessions(user_id);
+
+
+-- ============================================================================
+-- Watch Uploads Table
+-- Records individual upload events in watch mode
+-- ============================================================================
+CREATE TABLE IF NOT EXISTS watch_uploads (
+  id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+  
+  -- Session Reference
+  session_id VARCHAR(36) NOT NULL,
+  
+  -- Upload Details
+  timestamp TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  strategy VARCHAR(20) NOT NULL,
+  -- Possible values: 'individual', 'batch', 'full-structure'
+  
+  -- Statistics
+  file_count INTEGER NOT NULL DEFAULT 0,
+  success_count INTEGER NOT NULL DEFAULT 0,
+  failure_count INTEGER NOT NULL DEFAULT 0,
+  retry_count INTEGER NOT NULL DEFAULT 0,
+  duration_ms INTEGER NOT NULL DEFAULT 0,
+  
+  -- Status
+  status VARCHAR(20) DEFAULT 'completed',
+  -- Possible values: 'pending', 'in-progress', 'completed', 'failed', 'partial'
+  
+  -- File References
+  file_paths TEXT ARRAY,
+  
+  -- Timestamps
+  created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+  
+  -- Metadata
+  metadata JSONB,
+  
+  -- Constraints
+  CONSTRAINT fk_watch_uploads_session 
+    FOREIGN KEY (session_id) REFERENCES watch_sessions(id) ON DELETE CASCADE
+);
+
+-- Indexes for watch_uploads table
+CREATE INDEX IF NOT EXISTS idx_watch_uploads_session_id ON watch_uploads(session_id);
+CREATE INDEX IF NOT EXISTS idx_watch_uploads_timestamp ON watch_uploads(timestamp);
+CREATE INDEX IF NOT EXISTS idx_watch_uploads_strategy ON watch_uploads(strategy);
+CREATE INDEX IF NOT EXISTS idx_watch_uploads_status ON watch_uploads(status);
+
+
+-- ============================================================================
+-- Watch Events Table
+-- Records detailed events during upload operations (retries, errors, warnings)
+-- ============================================================================
+CREATE TABLE IF NOT EXISTS watch_events (
+  id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+  
+  -- Session and Upload References
+  upload_event_id UUID,
+  session_id VARCHAR(36) NOT NULL,
+  
+  -- Event Details
+  timestamp TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  type VARCHAR(20) NOT NULL,
+  -- Possible values: 'retry', 'error', 'warning', 'info', 'success', 'timeout'
+  
+  -- Retry Information
+  attempt_number INTEGER,
+  backoff_ms INTEGER DEFAULT 0,
+  
+  -- Error Details
+  error_message VARCHAR(500),
+  error_code VARCHAR(50),
+  stack_trace TEXT,
+  
+  -- File Information
+  file_path VARCHAR(2048),
+  file_name VARCHAR(255),
+  
+  -- Timestamps
+  created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+  
+  -- Metadata
+  metadata JSONB,
+  
+  -- Constraints
+  CONSTRAINT fk_watch_events_upload 
+    FOREIGN KEY (upload_event_id) REFERENCES watch_uploads(id) ON DELETE CASCADE,
+  CONSTRAINT fk_watch_events_session 
+    FOREIGN KEY (session_id) REFERENCES watch_sessions(id) ON DELETE CASCADE
+);
+
+-- Indexes for watch_events table
+CREATE INDEX IF NOT EXISTS idx_watch_events_upload_id ON watch_events(upload_event_id);
+CREATE INDEX IF NOT EXISTS idx_watch_events_session_id ON watch_events(session_id);
+CREATE INDEX IF NOT EXISTS idx_watch_events_timestamp ON watch_events(timestamp);
+CREATE INDEX IF NOT EXISTS idx_watch_events_type ON watch_events(type);
+
+
+-- ============================================================================
+-- Upload Logs Table
+-- Maintains detailed logs of all upload operations for debugging and analytics
+-- ============================================================================
+CREATE TABLE IF NOT EXISTS upload_logs (
+  id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+  
+  -- Session Reference
+  session_id VARCHAR(36),
+  
+  -- Upload Reference
+  upload_event_id UUID,
+  
+  -- File Information
+  file_path VARCHAR(2048),
+  file_name VARCHAR(255),
+  file_size BIGINT,
+  
+  -- Log Details
+  log_level VARCHAR(20),
+  -- Possible values: 'debug', 'info', 'warn', 'error', 'fatal'
+  
+  message TEXT NOT NULL,
+  details TEXT,
+  
+  -- Timestamps
+  timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+  created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+  
+  -- Metadata
+  metadata JSONB
+);
+
+-- Indexes for upload_logs table
+CREATE INDEX IF NOT EXISTS idx_upload_logs_session_id ON upload_logs(session_id);
+CREATE INDEX IF NOT EXISTS idx_upload_logs_upload_event_id ON upload_logs(upload_event_id);
+CREATE INDEX IF NOT EXISTS idx_upload_logs_timestamp ON upload_logs(timestamp);
+CREATE INDEX IF NOT EXISTS idx_upload_logs_log_level ON upload_logs(log_level);
+
+
+-- ============================================================================
+-- Performance Metrics Table
+-- Tracks performance metrics for optimization and monitoring
+-- ============================================================================
+CREATE TABLE IF NOT EXISTS performance_metrics (
+  id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+  
+  -- Session Reference
+  session_id VARCHAR(36),
+  
+  -- Metric Information
+  metric_name VARCHAR(100) NOT NULL,
+  metric_value NUMERIC NOT NULL,
+  metric_unit VARCHAR(50),
+  
+  -- Context
+  operation VARCHAR(100),
+  file_path VARCHAR(2048),
+  
+  -- Timestamps
+  recorded_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+  created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+  
+  -- Metadata
+  metadata JSONB
+);
+
+-- Indexes for performance_metrics table
+CREATE INDEX IF NOT EXISTS idx_performance_metrics_session_id ON performance_metrics(session_id);
+CREATE INDEX IF NOT EXISTS idx_performance_metrics_metric_name ON performance_metrics(metric_name);
+CREATE INDEX IF NOT EXISTS idx_performance_metrics_recorded_at ON performance_metrics(recorded_at);
+
+
+-- ============================================================================
+-- Error Tracking Table
+-- Centralized tracking of all errors for debugging and monitoring
+-- ============================================================================
+CREATE TABLE IF NOT EXISTS error_tracking (
+  id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+  
+  -- Error Information
+  error_code VARCHAR(50),
+  error_type VARCHAR(100) NOT NULL,
+  error_message TEXT NOT NULL,
+  stack_trace TEXT,
+  
+  -- Context
+  session_id VARCHAR(36),
+  file_path VARCHAR(2048),
+  operation VARCHAR(100),
+  
+  -- Severity
+  severity VARCHAR(20) DEFAULT 'error',
+  -- Possible values: 'warning', 'error', 'critical'
+  
+  -- Status
+  resolved BOOLEAN DEFAULT FALSE,
+  resolution_notes TEXT,
+  
+  -- Timestamps
+  occurred_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+  created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+  resolved_at TIMESTAMP,
+  
+  -- Metadata
+  metadata JSONB
+);
+
+-- Indexes for error_tracking table
+CREATE INDEX IF NOT EXISTS idx_error_tracking_session_id ON error_tracking(session_id);
+CREATE INDEX IF NOT EXISTS idx_error_tracking_error_type ON error_tracking(error_type);
+CREATE INDEX IF NOT EXISTS idx_error_tracking_severity ON error_tracking(severity);
+CREATE INDEX IF NOT EXISTS idx_error_tracking_occurred_at ON error_tracking(occurred_at);
+CREATE INDEX IF NOT EXISTS idx_error_tracking_resolved ON error_tracking(resolved);
+
+
+-- ============================================================================
+-- Enable RLS (Row Level Security) - Optional but recommended for multi-tenant apps
+-- ============================================================================
+-- ALTER TABLE uploader ENABLE ROW LEVEL SECURITY;
+-- ALTER TABLE watch_sessions ENABLE ROW LEVEL SECURITY;
+-- ALTER TABLE watch_uploads ENABLE ROW LEVEL SECURITY;
+-- ALTER TABLE watch_events ENABLE ROW LEVEL SECURITY;
+-- ALTER TABLE upload_logs ENABLE ROW LEVEL SECURITY;
+-- ALTER TABLE performance_metrics ENABLE ROW LEVEL SECURITY;
+-- ALTER TABLE error_tracking ENABLE ROW LEVEL SECURITY;
+
+
+-- ============================================================================
+-- Cleanup Scripts (Optional)
+-- ============================================================================
+-- These functions can be used for automated cleanup of old data
+
+CREATE OR REPLACE FUNCTION cleanup_old_sessions(days_to_keep INTEGER DEFAULT 30)
+RETURNS void AS $$
+BEGIN
+  DELETE FROM watch_sessions 
+  WHERE status = 'completed' 
+    AND end_time < CURRENT_TIMESTAMP - (days_to_keep || ' days')::INTERVAL;
+END;
+$$ LANGUAGE plpgsql;
+
+
+CREATE OR REPLACE FUNCTION cleanup_old_logs(days_to_keep INTEGER DEFAULT 30)
+RETURNS void AS $$
+BEGIN
+  DELETE FROM upload_logs 
+  WHERE created_at < CURRENT_TIMESTAMP - (days_to_keep || ' days')::INTERVAL;
+END;
+$$ LANGUAGE plpgsql;
+
+
+CREATE OR REPLACE FUNCTION update_modified_timestamp()
+RETURNS TRIGGER AS $$
+BEGIN
+  NEW.updated_at = CURRENT_TIMESTAMP;
+  RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+
+-- Apply trigger to tables that have updated_at
+CREATE TRIGGER update_uploader_timestamp
+BEFORE UPDATE ON uploader
+FOR EACH ROW
+EXECUTE FUNCTION update_modified_timestamp();
+
+CREATE TRIGGER update_watch_sessions_timestamp
+BEFORE UPDATE ON watch_sessions
+FOR EACH ROW
+EXECUTE FUNCTION update_modified_timestamp();
+
+
+-- ============================================================================
+-- Grant Permissions (Optional - adjust based on your security requirements)
+-- ============================================================================
+-- GRANT SELECT, INSERT, UPDATE ON uploader TO public;
+-- GRANT SELECT, INSERT, UPDATE ON watch_sessions TO public;
+-- GRANT SELECT, INSERT, UPDATE ON watch_uploads TO public;
+-- GRANT SELECT, INSERT ON watch_events TO public;
+-- GRANT SELECT, INSERT ON upload_logs TO public;
+-- GRANT SELECT, INSERT ON performance_metrics TO public;
+-- GRANT SELECT, INSERT ON error_tracking TO public;

package/supabase/migrations/002_align_with_arela_api_schema.sql

@@ -0,0 +1,145 @@
+-- Migration: Align arela-uploader schema with arela-api (BaseEntity + Queue Monitor)
+-- Date: 2025-11-16
+-- Description: Add missing columns, rename timestamps to match BaseEntity, create proper indexes
+
+-- ============================================================================
+-- Step 1: Ensure columns exist (idempotent - won't fail if they already exist)
+-- ============================================================================
+
+-- Convert created_at to TIMESTAMP WITH TIME ZONE (match BaseEntity)
+DO $$ 
+BEGIN
+  IF EXISTS(SELECT 1 FROM information_schema.columns 
+    WHERE table_name='uploader' AND column_name='created_at') 
+  THEN
+    ALTER TABLE uploader ALTER COLUMN created_at TYPE TIMESTAMP WITH TIME ZONE USING created_at AT TIME ZONE 'UTC';
+    ALTER TABLE uploader ALTER COLUMN created_at SET DEFAULT CURRENT_TIMESTAMP;
+  END IF;
+END $$;
+
+-- Add updated_at if it doesn't exist (replaces modified_at from old schema)
+ALTER TABLE uploader ADD COLUMN IF NOT EXISTS updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP;
+
+-- Add soft delete column (from BaseEntity)
+ALTER TABLE uploader ADD COLUMN IF NOT EXISTS deleted_at TIMESTAMP WITH TIME ZONE;
+
+-- Add audit columns (from BaseEntity)
+ALTER TABLE uploader ADD COLUMN IF NOT EXISTS created_by VARCHAR(255);
+ALTER TABLE uploader ADD COLUMN IF NOT EXISTS updated_by VARCHAR(255);
+ALTER TABLE uploader ADD COLUMN IF NOT EXISTS deleted_by VARCHAR(255);
+
+-- Add seq (auto-increment for display order - from BaseEntity)
+-- Note: seq will be auto-populated by TypeORM's @Generated('increment')
+ALTER TABLE uploader ADD COLUMN IF NOT EXISTS seq INTEGER;
+
+-- Add queue/processing related columns
+ALTER TABLE uploader ADD COLUMN IF NOT EXISTS processing_status VARCHAR(50) NOT NULL DEFAULT 'PENDING';
+ALTER TABLE uploader ADD COLUMN IF NOT EXISTS depends_on_path TEXT;
+ALTER TABLE uploader ADD COLUMN IF NOT EXISTS pedimento_detected_at TIMESTAMP;
+ALTER TABLE uploader ADD COLUMN IF NOT EXISTS upload_attempts INTEGER NOT NULL DEFAULT 0;
+ALTER TABLE uploader ADD COLUMN IF NOT EXISTS last_error TEXT;
+ALTER TABLE uploader ADD COLUMN IF NOT EXISTS file_order INTEGER;
+
+-- Fix data types to match schema (PostgreSQL)
+DO $$
+BEGIN
+  ALTER TABLE uploader ALTER COLUMN filename TYPE TEXT;
+  EXCEPTION WHEN OTHERS THEN NULL;
+END $$;
+
+DO $$
+BEGIN
+  ALTER TABLE uploader ALTER COLUMN original_path TYPE TEXT;
+  EXCEPTION WHEN OTHERS THEN NULL;
+END $$;
+
+DO $$
+BEGIN
+  ALTER TABLE uploader ALTER COLUMN arela_path TYPE TEXT;
+  EXCEPTION WHEN OTHERS THEN NULL;
+END $$;
+
+DO $$
+BEGIN
+  ALTER TABLE uploader ALTER COLUMN file_extension TYPE VARCHAR(30);
+  EXCEPTION WHEN OTHERS THEN NULL;
+END $$;
+
+DO $$
+BEGIN
+  ALTER TABLE uploader ALTER COLUMN rfc TYPE VARCHAR(30);
+  EXCEPTION WHEN OTHERS THEN NULL;
+END $$;
+
+DO $$
+BEGIN
+  ALTER TABLE uploader ALTER COLUMN document_type TYPE VARCHAR(256);
+  EXCEPTION WHEN OTHERS THEN NULL;
+END $$;
+
+DO $$
+BEGIN
+  ALTER TABLE uploader ALTER COLUMN size TYPE NUMERIC;
+  EXCEPTION WHEN OTHERS THEN NULL;
+END $$;
+
+-- ============================================================================
+-- Step 2: Create/Update Indexes (match arela-api schema exactly)
+-- ============================================================================
+
+-- Unique index on original_path
+CREATE UNIQUE INDEX IF NOT EXISTS "IDX_b9e1e6885fffe886a09d5b94be" ON uploader(original_path);
+
+-- Composite indexes for common query patterns
+CREATE INDEX IF NOT EXISTS "IDX_75541ccf793dc13fbaf3fb43be" ON uploader(status, file_extension);
+CREATE INDEX IF NOT EXISTS "IDX_151b34d21e0a7d8f9e6c7d2f67" ON uploader(file_extension);
+CREATE INDEX IF NOT EXISTS "IDX_cb4f73612a275f1b561044fa5d" ON uploader(num_pedimento);
+CREATE INDEX IF NOT EXISTS "IDX_778b0d7e1fa6ef549b63adf1cd" ON uploader(status);
+CREATE INDEX IF NOT EXISTS "IDX_f82df9c75e54eee33bca87c977" ON uploader(rfc);
+CREATE INDEX IF NOT EXISTS "IDX_33560ea61a522a949bcfa87844" ON uploader(rfc, arela_path);
+CREATE INDEX IF NOT EXISTS "IDX_d896431e5a737baf88e328029a" ON uploader(rfc, num_pedimento);
+CREATE INDEX IF NOT EXISTS "IDX_9b857c099b53cb010e0c11f4ce" ON uploader(arela_path);
+CREATE INDEX IF NOT EXISTS "IDX_743fb9d6f62296e58856ef65b8" ON uploader(year);
+
+-- Processing status indexes (for arela-api queue monitor)
+CREATE INDEX IF NOT EXISTS "IDX_322a218380879cac432274bdf6" ON uploader(processing_status, created_at);
+CREATE INDEX IF NOT EXISTS "IDX_46dfb9f259182be2bda72b36c6" ON uploader(created_at);
+CREATE INDEX IF NOT EXISTS "IDX_ebf662fa6711846fb0c2bb1e6b" ON uploader(processing_status, depends_on_path);
+CREATE INDEX IF NOT EXISTS "IDX_9df55a168a5430fb2ad2abac78" ON uploader(depends_on_path);
+CREATE INDEX IF NOT EXISTS "IDX_093aff5c97826ac35184ccb095" ON uploader(processing_status);
+
+-- ============================================================================
+-- Step 3: Remove unused tables that were creating confusion
+-- ============================================================================
+
+-- These tables were created but never used in either application
+DROP TABLE IF EXISTS error_tracking CASCADE;
+DROP TABLE IF EXISTS upload_logs CASCADE;
+DROP TABLE IF EXISTS performance_metrics CASCADE;
+DROP TABLE IF EXISTS watch_sessions CASCADE;
+
+-- ============================================================================
+-- Step 4: Documentation / Schema Comments
+-- ============================================================================
+
+COMMENT ON TABLE uploader IS 'File upload tracking table shared between arela-uploader and arela-api';
+
+COMMENT ON COLUMN uploader.status IS 'Internal arela-uploader status (detection lifecycle): stats, detected, detection-error, unsupported, etc';
+
+COMMENT ON COLUMN uploader.processing_status IS 'API processing status (queue/upload lifecycle): PENDING, READY_TO_UPLOAD, PROCESSING, UPLOADED, FAILED';
+
+COMMENT ON COLUMN uploader.upload_attempts IS 'Number of upload attempts made by arela-api';
+
+COMMENT ON COLUMN uploader.last_error IS 'Last error message from upload attempt';
+
+COMMENT ON COLUMN uploader.depends_on_path IS 'File path this record depends on (e.g., pedimento path for related files)';
+
+COMMENT ON COLUMN uploader.pedimento_detected_at IS 'Timestamp when pedimento simplificado was detected';
+
+COMMENT ON COLUMN uploader.created_by IS 'User who created the record (audit)';
+
+COMMENT ON COLUMN uploader.updated_by IS 'User who last updated the record (audit)';
+
+COMMENT ON COLUMN uploader.deleted_by IS 'User who soft-deleted the record (audit)';
+
+COMMENT ON COLUMN uploader.file_order IS 'Order for display/processing (when null, uses created_at order)';

package/commands.md

@@ -1,6 +0,0 @@
-node src/index.js --stats-only
-node src/index.js --detect-pdfs
-node src/index.js --propagate-arela-path
-node src/index.js --upload-by-rfc --folder-structure palco
-
-UPLOAD_RFCS="RFC1|RFC2" node src/index.js --upload-by-rfc --folder-structure target-folder