datly 0.0.1 → 0.0.2

package/src/core/utils.js

@@ -0,0 +1,306 @@
+class Utils {
+    detectOutliers(column, method = 'iqr') {
+        const sortedData = [...column].sort((a, b) => a - b);
+        const outliers = [];
+        const indices = [];
+
+        switch (method) {
+            case 'iqr':
+                const q1 = this.quantile(sortedData, 0.25);
+                const q3 = this.quantile(sortedData, 0.75);
+                const iqr = q3 - q1;
+                const lowerBound = q1 - 1.5 * iqr;
+                const upperBound = q3 + 1.5 * iqr;
+
+                column.forEach((value, index) => {
+                    if (value < lowerBound || value > upperBound) {
+                        outliers.push(value);
+                        indices.push(index);
+                    }
+                });
+                break;
+
+            case 'zscore':
+                const mean = this.mean(column);
+                const std = this.standardDeviation(column);
+
+                column.forEach((value, index) => {
+                    const zscore = Math.abs((value - mean) / std);
+                    if (zscore > 3) {
+                        outliers.push(value);
+                        indices.push(index);
+                    }
+                });
+                break;
+
+            case 'modified_zscore':
+                const median = this.median(column);
+                const deviations = column.map(x => Math.abs(x - median));
+                const mad = this.median(deviations);
+
+                column.forEach((value, index) => {
+                    const modifiedZScore = 0.6745 * (value - median) / mad;
+                    if (Math.abs(modifiedZScore) > 3.5) {
+                        outliers.push(value);
+                        indices.push(index);
+                    }
+                });
+                break;
+
+            default:
+                throw new Error(`Unknown outlier detection method: ${method}`);
+        }
+
+        return {
+            outliers,
+            indices,
+            count: outliers.length,
+            percentage: (outliers.length / column.length) * 100
+        };
+    }
+
+    frequencyTable(column) {
+        const frequencies = {};
+        const total = column.length;
+
+        column.forEach(value => {
+            const key = value === null || value === undefined ? 'null' : String(value);
+            frequencies[key] = (frequencies[key] || 0) + 1;
+        });
+
+        const result = Object.entries(frequencies).map(([value, count]) => ({
+            value: value === 'null' ? null : value,
+            frequency: count,
+            relativeFrequency: count / total,
+            percentage: (count / total) * 100
+        }));
+
+        return result.sort((a, b) => b.frequency - a.frequency);
+    }
+
+    groupBy(dataset, column, aggregation) {
+        const groups = {};
+
+        dataset.data.forEach(row => {
+            const key = row[column];
+            if (!groups[key]) {
+                groups[key] = [];
+            }
+            groups[key].push(row);
+        });
+
+        const result = {};
+        Object.entries(groups).forEach(([key, rows]) => {
+            result[key] = {
+                count: rows.length,
+                data: rows
+            };
+
+            if (aggregation && typeof aggregation === 'object') {
+                Object.entries(aggregation).forEach(([targetCol, func]) => {
+                    const values = rows.map(row => row[targetCol]).filter(v =>
+                        typeof v === 'number' && !isNaN(v)
+                    );
+
+                    if (values.length > 0) {
+                        result[key][`${func}_${targetCol}`] = this.applyAggregation(values, func);
+                    }
+                });
+            }
+        });
+
+        return result;
+    }
+
+    applyAggregation(values, func) {
+        switch (func) {
+            case 'mean': return this.mean(values);
+            case 'median': return this.median(values);
+            case 'sum': return values.reduce((a, b) => a + b, 0);
+            case 'min': return Math.min(...values);
+            case 'max': return Math.max(...values);
+            case 'std': return this.standardDeviation(values);
+            case 'var': return this.variance(values);
+            case 'count': return values.length;
+            default: throw new Error(`Unknown aggregation function: ${func}`);
+        }
+    }
+
+    sample(dataset, size, method = 'random') {
+        if (size >= dataset.length) {
+            return { ...dataset };
+        }
+
+        let sampledData;
+
+        switch (method) {
+            case 'random':
+                const indices = this.randomSample(dataset.length, size);
+                sampledData = indices.map(i => dataset.data[i]);
+                break;
+
+            case 'systematic':
+                const interval = Math.floor(dataset.length / size);
+                sampledData = [];
+                for (let i = 0; i < size; i++) {
+                    sampledData.push(dataset.data[i * interval]);
+                }
+                break;
+
+            case 'first':
+                sampledData = dataset.data.slice(0, size);
+                break;
+
+            case 'last':
+                sampledData = dataset.data.slice(-size);
+                break;
+
+            default:
+                throw new Error(`Unknown sampling method: ${method}`);
+        }
+
+        return {
+            ...dataset,
+            data: sampledData,
+            length: sampledData.length
+        };
+    }
+
+    randomSample(populationSize, sampleSize) {
+        const indices = Array.from({ length: populationSize }, (_, i) => i);
+        const sample = [];
+
+        for (let i = 0; i < sampleSize; i++) {
+            const randomIndex = Math.floor(Math.random() * indices.length);
+            sample.push(indices.splice(randomIndex, 1)[0]);
+        }
+
+        return sample;
+    }
+
+    bootstrap(sample, statistic, iterations = 1000) {
+        const bootstrapStats = [];
+
+        for (let i = 0; i < iterations; i++) {
+            const bootstrapSample = [];
+            for (let j = 0; j < sample.length; j++) {
+                const randomIndex = Math.floor(Math.random() * sample.length);
+                bootstrapSample.push(sample[randomIndex]);
+            }
+
+            const stat = this.applyStatistic(bootstrapSample, statistic);
+            bootstrapStats.push(stat);
+        }
+
+        return {
+            bootstrapStats: bootstrapStats.sort((a, b) => a - b),
+            mean: this.mean(bootstrapStats),
+            standardError: this.standardDeviation(bootstrapStats),
+            confidenceInterval: {
+                lower: this.quantile(bootstrapStats, 0.025),
+                upper: this.quantile(bootstrapStats, 0.975)
+            }
+        };
+    }
+
+    applyStatistic(sample, statistic) {
+        switch (statistic) {
+            case 'mean': return this.mean(sample);
+            case 'median': return this.median(sample);
+            case 'std': return this.standardDeviation(sample);
+            case 'var': return this.variance(sample);
+            default:
+                if (typeof statistic === 'function') {
+                    return statistic(sample);
+                }
+                throw new Error(`Unknown statistic: ${statistic}`);
+        }
+    }
+
+    contingencyTable(col1, col2) {
+        const uniqueCol1 = [...new Set(col1)];
+        const uniqueCol2 = [...new Set(col2)];
+
+        const table = {};
+        const totals = { row: {}, col: {}, grand: 0 };
+
+        uniqueCol1.forEach(val1 => {
+            table[val1] = {};
+            totals.row[val1] = 0;
+        });
+
+        uniqueCol2.forEach(val2 => {
+            totals.col[val2] = 0;
+        });
+
+        for (let i = 0; i < col1.length; i++) {
+            const val1 = col1[i];
+            const val2 = col2[i];
+
+            if (!table[val1][val2]) {
+                table[val1][val2] = 0;
+            }
+
+            table[val1][val2]++;
+            totals.row[val1]++;
+            totals.col[val2]++;
+            totals.grand++;
+        }
+
+        uniqueCol1.forEach(val1 => {
+            uniqueCol2.forEach(val2 => {
+                if (!table[val1][val2]) {
+                    table[val1][val2] = 0;
+                }
+            });
+        });
+
+        return { table, totals, rows: uniqueCol1, columns: uniqueCol2 };
+    }
+
+    mean(arr) {
+        return arr.reduce((sum, val) => sum + val, 0) / arr.length;
+    }
+
+    median(arr) {
+        const sorted = [...arr].sort((a, b) => a - b);
+        const mid = Math.floor(sorted.length / 2);
+        return sorted.length % 2 === 0 ?
+            (sorted[mid - 1] + sorted[mid]) / 2 :
+            sorted[mid];
+    }
+
+    quantile(arr, q) {
+        const sorted = [...arr].sort((a, b) => a - b);
+        const index = (sorted.length - 1) * q;
+        const lower = Math.floor(index);
+        const upper = Math.ceil(index);
+        const weight = index % 1;
+
+        if (lower === upper) {
+            return sorted[lower];
+        }
+
+        return sorted[lower] * (1 - weight) + sorted[upper] * weight;
+    }
+
+    standardDeviation(arr) {
+        return Math.sqrt(this.variance(arr));
+    }
+
+    variance(arr) {
+        const mean = this.mean(arr);
+        return arr.reduce((sum, val) => sum + Math.pow(val - mean, 2), 0) / (arr.length - 1);
+    }
+
+    round(value, decimals = 4) {
+        return Math.round(value * Math.pow(10, decimals)) / Math.pow(10, decimals);
+    }
+
+    formatNumber(value, decimals = 4) {
+        if (typeof value !== 'number') return value;
+        return this.round(value, decimals);
+    }
+}
+
+export default Utils;

package/src/core/validator.js

@@ -0,0 +1,205 @@
+class Validator {
+    validateData(dataset) {
+        const errors = [];
+        const warnings = [];
+
+        if (!dataset || typeof dataset !== 'object') {
+            errors.push('Dataset must be an object');
+            return { valid: false, errors, warnings };
+        }
+
+        if (!dataset.data || !Array.isArray(dataset.data)) {
+            errors.push('Dataset must contain a data array');
+        }
+
+        if (!dataset.headers || !Array.isArray(dataset.headers)) {
+            errors.push('Dataset must contain a headers array');
+        }
+
+        if (dataset.data && dataset.data.length === 0) {
+            warnings.push('Dataset is empty');
+        }
+
+        if (dataset.data && dataset.headers) {
+            const headerSet = new Set(dataset.headers);
+            if (headerSet.size !== dataset.headers.length) {
+                errors.push('Duplicate column headers found');
+            }
+
+            dataset.data.forEach((row, index) => {
+                const rowKeys = Object.keys(row);
+                const missingHeaders = dataset.headers.filter(h => !rowKeys.includes(h));
+                const extraKeys = rowKeys.filter(k => !dataset.headers.includes(k));
+
+                if (missingHeaders.length > 0) {
+                    warnings.push(`Row ${index}: Missing columns: ${missingHeaders.join(', ')}`);
+                }
+
+                if (extraKeys.length > 0) {
+                    warnings.push(`Row ${index}: Extra columns: ${extraKeys.join(', ')}`);
+                }
+            });
+        }
+
+        return {
+            valid: errors.length === 0,
+            errors,
+            warnings
+        };
+    }
+
+    validateNumericColumn(column) {
+        if (!Array.isArray(column)) {
+            throw new Error('Column must be an array');
+        }
+
+        const numericValues = column.filter(val =>
+            typeof val === 'number' && !isNaN(val) && isFinite(val)
+        );
+
+        if (numericValues.length === 0) {
+            throw new Error('Column contains no valid numeric values');
+        }
+
+        return {
+            valid: true,
+            validCount: numericValues.length,
+            invalidCount: column.length - numericValues.length,
+            cleanData: numericValues
+        };
+    }
+
+    validateSampleSize(sample, minSize = 2) {
+        if (!Array.isArray(sample)) {
+            throw new Error('Sample must be an array');
+        }
+
+        if (sample.length < minSize) {
+            throw new Error(`Sample size (${sample.length}) must be at least ${minSize}`);
+        }
+
+        return true;
+    }
+
+    validateConfidenceLevel(confidence) {
+        if (typeof confidence !== 'number' || confidence <= 0 || confidence >= 1) {
+            throw new Error('Confidence level must be a number between 0 and 1');
+        }
+        return true;
+    }
+
+    validateCorrelationInputs(col1, col2) {
+        this.validateNumericColumn(col1);
+        this.validateNumericColumn(col2);
+
+        if (col1.length !== col2.length) {
+            throw new Error('Columns must have the same length');
+        }
+
+        if (col1.length < 3) {
+            throw new Error('Need at least 3 paired observations for correlation');
+        }
+
+        return true;
+    }
+
+    validateRegressionInputs(x, y) {
+        this.validateNumericColumn(x);
+        this.validateNumericColumn(y);
+
+        if (x.length !== y.length) {
+            throw new Error('X and Y arrays must have the same length');
+        }
+
+        if (x.length < 3) {
+            throw new Error('Need at least 3 data points for regression');
+        }
+
+        const xVariance = this.calculateVariance(x);
+        if (xVariance === 0) {
+            throw new Error('X values must have non-zero variance');
+        }
+
+        return true;
+    }
+
+    calculateVariance(arr) {
+        const mean = arr.reduce((sum, val) => sum + val, 0) / arr.length;
+        return arr.reduce((sum, val) => sum + Math.pow(val - mean, 2), 0) / (arr.length - 1);
+    }
+
+    validateGroupsForANOVA(groups) {
+        if (!Array.isArray(groups) || groups.length < 2) {
+            throw new Error('ANOVA requires at least 2 groups');
+        }
+
+        groups.forEach((group, index) => {
+            if (!Array.isArray(group)) {
+                throw new Error(`Group ${index} must be an array`);
+            }
+
+            this.validateSampleSize(group, 2);
+            this.validateNumericColumn(group);
+        });
+
+        return true;
+    }
+
+    validateContingencyTable(col1, col2) {
+        if (!Array.isArray(col1) || !Array.isArray(col2)) {
+            throw new Error('Both columns must be arrays');
+        }
+
+        if (col1.length !== col2.length) {
+            throw new Error('Columns must have the same length');
+        }
+
+        if (col1.length < 5) {
+            throw new Error('Need at least 5 observations for chi-square test');
+        }
+
+        return true;
+    }
+
+    isInteger(value) {
+        return typeof value === 'number' && Number.isInteger(value);
+    }
+
+    isPositive(value) {
+        return typeof value === 'number' && value > 0;
+    }
+
+    isInRange(value, min, max) {
+        return typeof value === 'number' && value >= min && value <= max;
+    }
+
+    hasMinimumObservations(data, minimum) {
+        return Array.isArray(data) && data.length >= minimum;
+    }
+
+    checkForConstantValues(column) {
+        const uniqueValues = new Set(column);
+        return uniqueValues.size === 1;
+    }
+
+    validateHypothesisTestInputs(sample1, sample2, testType) {
+        this.validateSampleSize(sample1, 2);
+
+        if (testType === 'two-sample' || testType === 'paired') {
+            this.validateSampleSize(sample2, 2);
+
+            if (testType === 'paired' && sample1.length !== sample2.length) {
+                throw new Error('Paired samples must have the same length');
+            }
+        }
+
+        this.validateNumericColumn(sample1);
+        if (sample2) {
+            this.validateNumericColumn(sample2);
+        }
+
+        return true;
+    }
+}
+
+export default Validator;