optimal-cli 0.1.0

package/dist/lib/returnpro/upload-netsuite.js

@@ -0,0 +1,566 @@
+import { readFileSync } from 'fs';
+import { basename, extname } from 'path';
+import ExcelJS from 'exceljs';
+import { getSupabase } from '../supabase.js';
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+const CHUNK_SIZE = 500;
+/**
+ * Month names used to detect multi-sheet (monthly tab) workbooks.
+ * Matches the fiscal calendar used in dashboard-returnpro (Apr=start).
+ */
+const MONTH_NAMES = new Set([
+    'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun',
+    'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec',
+]);
+/**
+ * NetSuite "Data Entry" sheet meta columns — everything else is an account code.
+ */
+const META_COLUMNS = new Set([
+    'Master Program',
+    'Program ID',
+    'Date (Upload)',
+    'Date (Solution7)',
+]);
+// ---------------------------------------------------------------------------
+// Excel serial date conversion
+// ---------------------------------------------------------------------------
+/**
+ * Convert an Excel serial date number (e.g. 46023) to ISO YYYY-MM-DD.
+ * Excel epoch is 1899-12-30 (accounting for the Lotus 1-2-3 leap-year bug).
+ */
+function excelSerialToIso(serial) {
+    const msPerDay = 86400000;
+    const excelEpoch = new Date(Date.UTC(1899, 11, 30));
+    const date = new Date(excelEpoch.getTime() + serial * msPerDay);
+    return date.toISOString().slice(0, 10);
+}
+/**
+ * Normalise a date value from an XLSM cell.
+ * Accepts:
+ *   - JS Date objects (ExcelJS parses dates for typed cells)
+ *   - Excel serial numbers (numeric)
+ *   - ISO string / "Mon YYYY" partial strings
+ * Returns ISO YYYY-MM-DD, or null if unparseable.
+ */
+function normaliseDate(raw) {
+    if (raw == null || raw === '')
+        return null;
+    // ExcelJS may return a JS Date for date-typed cells
+    if (raw instanceof Date) {
+        if (isNaN(raw.getTime()))
+            return null;
+        return raw.toISOString().slice(0, 10);
+    }
+    if (typeof raw === 'number') {
+        // Excel serial — must be positive and plausible (> 1 = 1900-01-01)
+        if (raw > 1)
+            return excelSerialToIso(raw);
+        return null;
+    }
+    if (typeof raw === 'string') {
+        const s = raw.trim();
+        // Already ISO
+        if (/^\d{4}-\d{2}-\d{2}$/.test(s))
+            return s;
+        // Try JS Date parse for other formats
+        const d = new Date(s);
+        if (!isNaN(d.getTime()))
+            return d.toISOString().slice(0, 10);
+    }
+    return null;
+}
+// ---------------------------------------------------------------------------
+// Sheet detection
+// ---------------------------------------------------------------------------
+/**
+ * Returns true if the workbook has ≥3 sheets whose names match abbreviated
+ * month names (Jan, Feb, …, Dec). This is the same heuristic used in
+ * dashboard-returnpro's WesImporter.tsx → hasMonthlySheets().
+ */
+function hasMonthlySheets(sheetNames) {
+    const found = sheetNames.filter(name => MONTH_NAMES.has(name.trim()));
+    return found.length >= 3;
+}
+// ---------------------------------------------------------------------------
+// CSV parsing
+// ---------------------------------------------------------------------------
+/**
+ * Parse a single CSV line, respecting quoted fields.
+ */
+function parseCsvLine(line) {
+    const fields = [];
+    let current = '';
+    let inQuotes = false;
+    for (let i = 0; i < line.length; i++) {
+        const ch = line[i];
+        if (ch === '"') {
+            if (!inQuotes) {
+                inQuotes = true;
+                continue;
+            }
+            if (line[i + 1] === '"') {
+                current += '"';
+                i++;
+                continue;
+            }
+            inQuotes = false;
+            continue;
+        }
+        if (ch === ',' && !inQuotes) {
+            fields.push(current.trim());
+            current = '';
+            continue;
+        }
+        current += ch;
+    }
+    fields.push(current.trim());
+    return fields;
+}
+/**
+ * Parse a staging CSV file (long format).
+ * Expected headers: location, master_program, program_id, date, account_code, amount, mode
+ */
+function parseStagingCsv(content) {
+    const EXPECTED = ['location', 'master_program', 'program_id', 'date', 'account_code', 'amount', 'mode'];
+    const errors = [];
+    const lines = content.replace(/\r\n/g, '\n').replace(/\r/g, '\n').split('\n');
+    const nonEmpty = lines.filter(l => l.trim().length > 0);
+    if (nonEmpty.length < 2) {
+        return { rows: [], errors: ['CSV has no data rows'] };
+    }
+    const headers = parseCsvLine(nonEmpty[0]).map(h => h.toLowerCase());
+    const headersMatch = headers.length === EXPECTED.length &&
+        headers.every((h, i) => h === EXPECTED[i]);
+    if (!headersMatch) {
+        errors.push(`Header mismatch. Expected: [${EXPECTED.join(', ')}]. Got: [${headers.join(', ')}]`);
+        return { rows: [], errors };
+    }
+    const rows = [];
+    for (let i = 1; i < nonEmpty.length; i++) {
+        const values = parseCsvLine(nonEmpty[i]);
+        if (values.length !== EXPECTED.length) {
+            errors.push(`Line ${i + 1}: expected ${EXPECTED.length} columns, got ${values.length}`);
+            continue;
+        }
+        const [location, master_program, program_id, date, account_code, amount, mode] = values;
+        rows.push({ location, master_program, program_id, date, account_code, amount, mode });
+    }
+    return { rows, errors };
+}
+// ---------------------------------------------------------------------------
+// XLSM parsing — single "Data Entry" sheet (wide → long pivot)
+// ---------------------------------------------------------------------------
+async function parseDataEntrySheet(workbook, sheetName, masterProgramMap, warnings) {
+    const sheet = workbook.getWorksheet(sheetName);
+    if (!sheet) {
+        warnings.push(`Sheet "${sheetName}" not found`);
+        return [];
+    }
+    // Collect all rows as plain objects
+    const jsonRows = [];
+    let headers = [];
+    let headerRowIndex = -1;
+    sheet.eachRow((row, rowNumber) => {
+        if (headerRowIndex === -1) {
+            // First row is headers
+            const values = row.values;
+            // ExcelJS row.values[0] is undefined (1-indexed), slice from 1
+            const rawHeaders = values.slice(1).map(v => (v != null ? String(v).trim() : ''));
+            if (rawHeaders.some(h => h === 'Master Program')) {
+                headers = rawHeaders;
+                headerRowIndex = rowNumber;
+            }
+            return;
+        }
+        const obj = {};
+        const vals = row.values;
+        for (let i = 0; i < headers.length; i++) {
+            const cellVal = vals[i + 1];
+            // ExcelJS rich text
+            if (cellVal && typeof cellVal === 'object' && 'richText' in cellVal) {
+                obj[headers[i]] = cellVal.richText
+                    .map(r => r.text)
+                    .join('');
+            }
+            else {
+                obj[headers[i]] = cellVal ?? '';
+            }
+        }
+        jsonRows.push(obj);
+    });
+    if (headers.length === 0) {
+        warnings.push(`Sheet "${sheetName}": could not find header row`);
+        return [];
+    }
+    // Identify account code columns (all non-meta columns)
+    const accountCodes = headers.filter(h => h && !META_COLUMNS.has(h));
+    const rows = [];
+    for (const row of jsonRows) {
+        const masterProgram = String(row['Master Program'] ?? '').trim();
+        const programId = String(row['Program ID'] ?? '').trim();
+        const rawDate = row['Date (Upload)'];
+        if (!masterProgram)
+            continue;
+        const isoDate = normaliseDate(rawDate);
+        if (!isoDate) {
+            warnings.push(`Skipped row: unparseable date "${rawDate}" for program "${masterProgram}"`);
+            continue;
+        }
+        // Resolve location (client name) from master program map
+        const location = masterProgramMap.get(masterProgram);
+        if (!location) {
+            warnings.push(`No client mapping for master program: "${masterProgram}"`);
+            continue;
+        }
+        for (const accountCode of accountCodes) {
+            const rawAmount = row[accountCode];
+            if (rawAmount === null || rawAmount === undefined || rawAmount === '' || rawAmount === 0)
+                continue;
+            const numericAmount = typeof rawAmount === 'number' ? rawAmount : parseFloat(String(rawAmount));
+            if (isNaN(numericAmount) || numericAmount === 0)
+                continue;
+            rows.push({
+                location,
+                master_program: masterProgram,
+                program_id: programId,
+                date: isoDate,
+                account_code: accountCode,
+                amount: String(numericAmount),
+                mode: 'Actual',
+            });
+        }
+    }
+    return rows;
+}
+// ---------------------------------------------------------------------------
+// XLSM parsing — multi-sheet (one tab per month)
+// ---------------------------------------------------------------------------
+async function parseMultiSheetWorkbook(workbook, masterProgramMap, filterMonths, warnings) {
+    const allRows = [];
+    const monthSheets = workbook.worksheets
+        .map(ws => ws.name.trim())
+        .filter(name => MONTH_NAMES.has(name));
+    const sheetsToProcess = filterMonths
+        ? monthSheets.filter(s => {
+            // filterMonths may be YYYY-MM strings; we match the abbreviated month name part
+            return filterMonths.some(m => {
+                const parts = m.split('-');
+                const monthNum = parts[1] ? parseInt(parts[1], 10) : 0;
+                const abbr = ['', 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun',
+                    'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'][monthNum] ?? '';
+                return abbr === s;
+            });
+        })
+        : monthSheets;
+    for (const sheetName of sheetsToProcess) {
+        const sheetRows = await parseDataEntrySheet(workbook, sheetName, masterProgramMap, warnings);
+        allRows.push(...sheetRows);
+    }
+    return allRows;
+}
+// ---------------------------------------------------------------------------
+// Dimension lookup helpers (FK resolution against ReturnPro Supabase)
+// ---------------------------------------------------------------------------
+function chunkArray(arr, size) {
+    const out = [];
+    for (let i = 0; i < arr.length; i += size)
+        out.push(arr.slice(i, i + size));
+    return out;
+}
+async function buildMasterProgramToClientMap() {
+    const sb = getSupabase('returnpro');
+    const map = new Map();
+    let from = 0;
+    const PAGE = 1000;
+    while (true) {
+        const { data, error } = await sb
+            .from('dim_master_program')
+            .select('master_name,dim_client(client_name)')
+            .range(from, from + PAGE - 1);
+        if (error)
+            throw new Error(`Fetch dim_master_program failed: ${error.message}`);
+        if (!data || data.length === 0)
+            break;
+        for (const row of data) {
+            map.set(row.master_name, row.dim_client?.client_name ?? '- None -');
+        }
+        if (data.length < PAGE)
+            break;
+        from += PAGE;
+    }
+    return map;
+}
+async function buildAccountLookupMap(codes) {
+    const sb = getSupabase('returnpro');
+    const unique = Array.from(new Set(codes.filter(Boolean)));
+    const accountMap = new Map();
+    const signMultiplierMap = new Map();
+    for (const batch of chunkArray(unique, 100)) {
+        const { data, error } = await sb
+            .from('dim_account')
+            .select('account_id,account_code,sign_multiplier')
+            .in('account_code', batch);
+        if (error)
+            continue;
+        for (const row of (data ?? [])) {
+            accountMap.set(row.account_code, row.account_id);
+            signMultiplierMap.set(row.account_code, row.sign_multiplier ?? 1);
+        }
+    }
+    return { accountMap, signMultiplierMap };
+}
+async function buildClientLookupMap(locations) {
+    const sb = getSupabase('returnpro');
+    const unique = Array.from(new Set(locations.filter(Boolean)));
+    const map = new Map();
+    for (const batch of chunkArray(unique, 100)) {
+        const { data, error } = await sb
+            .from('dim_client')
+            .select('client_id,client_name')
+            .in('client_name', batch);
+        if (error)
+            continue;
+        for (const row of (data ?? [])) {
+            map.set(row.client_name, row.client_id);
+        }
+    }
+    return map;
+}
+async function buildMasterProgramIdMap(programs) {
+    const sb = getSupabase('returnpro');
+    const map = new Map();
+    // Group by client_id
+    const byClient = new Map();
+    for (const p of programs) {
+        if (!p.master_program || !p.client_id)
+            continue;
+        if (!byClient.has(p.client_id))
+            byClient.set(p.client_id, []);
+        byClient.get(p.client_id).push(p.master_program);
+    }
+    for (const [clientId, names] of byClient.entries()) {
+        for (const batch of chunkArray(Array.from(new Set(names)), 100)) {
+            const { data, error } = await sb
+                .from('dim_master_program')
+                .select('master_program_id,master_name,client_id')
+                .in('master_name', batch)
+                .eq('client_id', clientId);
+            if (error)
+                continue;
+            for (const row of (data ?? [])) {
+                map.set(`${row.client_id}|${row.master_name}`, row.master_program_id);
+            }
+        }
+    }
+    return map;
+}
+async function buildProgramIdMap(codes) {
+    const sb = getSupabase('returnpro');
+    const unique = Array.from(new Set(codes.filter(Boolean)));
+    const map = new Map();
+    for (const batch of chunkArray(unique, 100)) {
+        const { data, error } = await sb
+            .from('dim_program_id')
+            .select('program_id_key,program_code')
+            .in('program_code', batch);
+        if (error)
+            continue;
+        for (const row of (data ?? [])) {
+            map.set(row.program_code, row.program_id_key);
+        }
+    }
+    return map;
+}
+// ---------------------------------------------------------------------------
+// Insert helpers
+// ---------------------------------------------------------------------------
+async function insertBatch(rows) {
+    const sb = getSupabase('returnpro');
+    let inserted = 0;
+    for (const batch of chunkArray(rows, CHUNK_SIZE)) {
+        const { data, error } = await sb
+            .from('stg_financials_raw')
+            .insert(batch)
+            .select('raw_id');
+        if (error)
+            throw new Error(`Insert batch failed: ${error.message}`);
+        inserted += data?.length ?? batch.length;
+    }
+    return inserted;
+}
+// ---------------------------------------------------------------------------
+// Main export
+// ---------------------------------------------------------------------------
+/**
+ * Upload a NetSuite XLSM or staging CSV to stg_financials_raw.
+ *
+ * Supports two XLSM formats:
+ *   1. Single "Data Entry" sheet (wide format — account codes as columns)
+ *   2. Multi-sheet (≥3 month-named tabs, each a "Data Entry"-style sheet)
+ *
+ * For CSVs, expects the staging long format:
+ *   location, master_program, program_id, date, account_code, amount, mode
+ *
+ * FK columns (account_id, client_id, master_program_id, program_id_key) are
+ * resolved from dim_* tables and stamped on each row before insert.
+ *
+ * `stg_financials_raw.amount` is stored as TEXT per DB schema.
+ *
+ * @param filePath  Absolute path to .xlsm, .xlsx, or .csv file
+ * @param userId    User ID to associate with this upload (stored for audit)
+ * @param options   Optional: `months` array of YYYY-MM strings to filter which
+ *                  month tabs to process (multi-sheet XLSM only)
+ */
+export async function processNetSuiteUpload(filePath, userId, options) {
+    const fileName = basename(filePath);
+    const ext = extname(filePath).toLowerCase();
+    const loadedAt = new Date().toISOString();
+    const warnings = [];
+    if (!userId) {
+        return {
+            fileName, loadedAt, inserted: 0, monthsCovered: [], warnings,
+            error: 'userId is required',
+        };
+    }
+    let parsedRows = [];
+    try {
+        // ------------------------------------------------------------------
+        // 1. Parse the file
+        // ------------------------------------------------------------------
+        if (ext === '.csv') {
+            const content = readFileSync(filePath, 'utf-8');
+            const { rows, errors } = parseStagingCsv(content);
+            if (errors.length > 0 && rows.length === 0) {
+                return { fileName, loadedAt, inserted: 0, monthsCovered: [], warnings, error: errors.join('; ') };
+            }
+            warnings.push(...errors);
+            parsedRows = rows;
+        }
+        else if (ext === '.xlsm' || ext === '.xlsx') {
+            // Load workbook with ExcelJS
+            const workbook = new ExcelJS.Workbook();
+            const buffer = readFileSync(filePath);
+            // ExcelJS reads XLSM via xlsx extension (it is a zip-based format)
+            // Convert Node Buffer → ArrayBuffer so ExcelJS types are satisfied
+            const arrayBuffer = buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength);
+            await workbook.xlsx.load(arrayBuffer);
+            const sheetNames = workbook.worksheets.map(ws => ws.name);
+            const isMultiSheet = hasMonthlySheets(sheetNames);
+            // Fetch master program → client name map once (needed for both paths)
+            const masterProgramClientMap = await buildMasterProgramToClientMap();
+            if (isMultiSheet) {
+                parsedRows = await parseMultiSheetWorkbook(workbook, masterProgramClientMap, options?.months, warnings);
+            }
+            else {
+                // Try "Data Entry" sheet first, then first sheet
+                const sheetName = sheetNames.find(n => n === 'Data Entry') ?? sheetNames[0];
+                if (!sheetName) {
+                    return { fileName, loadedAt, inserted: 0, monthsCovered: [], warnings, error: 'Workbook has no sheets' };
+                }
+                parsedRows = await parseDataEntrySheet(workbook, sheetName, masterProgramClientMap, warnings);
+            }
+        }
+        else {
+            return {
+                fileName, loadedAt, inserted: 0, monthsCovered: [], warnings,
+                error: `Unsupported file extension: ${ext}. Expected .xlsm, .xlsx, or .csv`,
+            };
+        }
+        if (parsedRows.length === 0) {
+            return {
+                fileName, loadedAt, inserted: 0, monthsCovered: [], warnings,
+                error: 'No rows parsed from file',
+            };
+        }
+        // ------------------------------------------------------------------
+        // 2. Filter to requested months if provided (CSV path doesn't filter above)
+        // ------------------------------------------------------------------
+        if (options?.months && options.months.length > 0) {
+            const monthSet = new Set(options.months);
+            parsedRows = parsedRows.filter(r => {
+                const m = r.date ? r.date.substring(0, 7) : null;
+                return m ? monthSet.has(m) : false;
+            });
+        }
+        // ------------------------------------------------------------------
+        // 3. Resolve FK dimension lookups
+        // ------------------------------------------------------------------
+        const accountCodes = parsedRows.map(r => r.account_code).filter(Boolean);
+        const locations = parsedRows.map(r => r.location).filter(Boolean);
+        const programCodes = parsedRows.map(r => r.program_id).filter(Boolean);
+        const [{ accountMap, signMultiplierMap }, clientMap, programIdMap,] = await Promise.all([
+            buildAccountLookupMap(accountCodes),
+            buildClientLookupMap(locations),
+            buildProgramIdMap(programCodes),
+        ]);
+        // Master program lookup requires client_id — do after clientMap is ready
+        const masterProgramInputs = [];
+        for (const row of parsedRows) {
+            const clientId = clientMap.get(row.location);
+            if (clientId && row.master_program) {
+                masterProgramInputs.push({ master_program: row.master_program, client_id: clientId });
+            }
+        }
+        const masterProgramIdMap = await buildMasterProgramIdMap(masterProgramInputs);
+        // ------------------------------------------------------------------
+        // 4. Stamp rows with FK columns and apply sign convention
+        // ------------------------------------------------------------------
+        let signFlippedCount = 0;
+        const stamped = parsedRows.map(row => {
+            const accountId = row.account_code ? accountMap.get(row.account_code) ?? null : null;
+            const clientId = row.location ? clientMap.get(row.location) ?? null : null;
+            const masterProgramId = row.master_program && clientId
+                ? masterProgramIdMap.get(`${clientId}|${row.master_program}`) ?? null
+                : null;
+            const programIdKey = row.program_id ? programIdMap.get(row.program_id) ?? null : null;
+            // Revenue accounts (sign_multiplier = -1) have their sign flipped
+            const signMultiplier = row.account_code ? signMultiplierMap.get(row.account_code) ?? 1 : 1;
+            let amount = row.amount;
+            if (signMultiplier === -1) {
+                const num = parseFloat(row.amount);
+                if (!isNaN(num)) {
+                    amount = String(num * -1);
+                    signFlippedCount++;
+                }
+            }
+            return {
+                source_file_name: fileName,
+                loaded_at: loadedAt,
+                location: row.location,
+                master_program: row.master_program,
+                program_code: row.program_id,
+                date: row.date,
+                account_code: row.account_code,
+                amount, // TEXT — stored as string
+                mode: row.mode,
+                account_id: accountId,
+                client_id: clientId,
+                master_program_id: masterProgramId,
+                program_id_key: programIdKey,
+            };
+        });
+        if (signFlippedCount > 0) {
+            warnings.push(`Sign convention applied: ${signFlippedCount} revenue account rows flipped`);
+        }
+        // ------------------------------------------------------------------
+        // 5. Insert into stg_financials_raw
+        // ------------------------------------------------------------------
+        const inserted = await insertBatch(stamped);
+        // ------------------------------------------------------------------
+        // 6. Collect distinct months covered
+        // ------------------------------------------------------------------
+        const monthSet = new Set();
+        for (const row of stamped) {
+            if (row.date)
+                monthSet.add(row.date.substring(0, 7));
+        }
+        const monthsCovered = [...monthSet].sort();
+        return { fileName, loadedAt, inserted, monthsCovered, warnings };
+    }
+    catch (err) {
+        const error = err instanceof Error ? err.message : String(err);
+        return { fileName, loadedAt, inserted: 0, monthsCovered: [], warnings, error };
+    }
+}

package/dist/lib/returnpro/upload-r1.d.ts

@@ -0,0 +1,48 @@
+/**
+ * A single row extracted from an R1 XLSX file before aggregation.
+ * Columns match the canonical R1 export format documented in dashboard-returnpro.
+ */
+export interface R1Row {
+    programCode: string;
+    masterProgram: string;
+    trgid: string;
+    locationId: string;
+    avgRetail: number | null;
+}
+/**
+ * Return value of processR1Upload.
+ */
+export interface R1UploadResult {
+    /** Source file name (basename) */
+    sourceFileName: string;
+    /** YYYY-MM-DD date inserted as the `date` column (always the 1st of the given monthYear) */
+    date: string;
+    /** Total raw rows read from the XLSX (excluding header) */
+    totalRowsRead: number;
+    /** Number of rows skipped due to missing ProgramName, TRGID, or Master Program Name */
+    rowsSkipped: number;
+    /** Number of distinct (masterProgram, programCode, location) groups aggregated */
+    programGroupsFound: number;
+    /** Number of rows actually inserted into stg_financials_raw */
+    rowsInserted: number;
+    /** Any non-fatal warnings encountered during processing */
+    warnings: string[];
+}
+/**
+ * Parse an R1 XLSX file, aggregate financial data by program, and insert
+ * into the ReturnPro `stg_financials_raw` staging table.
+ *
+ * Flow:
+ *   1. Read and parse the XLSX (first sheet, required columns: ProgramName,
+ *      Master Program Name, TRGID).
+ *   2. Aggregate rows into (masterProgram, programCode, location) groups,
+ *      counting distinct TRGIDs per group.
+ *   3. Look up dim_master_program and dim_program_id to resolve FK columns.
+ *   4. Insert into stg_financials_raw in batches of 500.
+ *
+ * @param filePath   Absolute path to the R1 XLSX file on disk.
+ * @param userId     The user_id to stamp on each inserted row.
+ * @param monthYear  Target month in "YYYY-MM" format (e.g. "2025-10").
+ *                   Stored as the `date` column as "YYYY-MM-01".
+ */
+export declare function processR1Upload(filePath: string, userId: string, monthYear: string): Promise<R1UploadResult>;