outliertree 0.2.1 → 0.3.0

data/vendor/outliertree/src/clusters.cpp

@@ -121,8 +121,8 @@ bool define_numerical_cluster(double *restrict x, size_t *restrict ix_arr, size_
     bool has_low_values  = false;
     bool has_high_values = false;
     long double running_mean = 0;
-    long double mean_prev    = 0;
     long double running_ssq  = 0;
+    long double mean_prev    = 0;
     double xval;
     double mean;
     double sd;
@@ -134,6 +134,14 @@ bool define_numerical_cluster(double *restrict x, size_t *restrict ix_arr, size_
     size_t end_normals   = 0;
     double min_gap = z_outlier - z_norm;
 
+    double curr_gap, next_gap, eps, lim_by_orig;
+
+    /* Note: there is no good reason and no theory behind these numbers.
+       TODO: find a better way of setting this */
+    double min_gap_orig_scale = log(sqrtl((long double)(end - st + 1))) / 2.;
+    min_gap_orig_scale = std::fmax(1.1, min_gap_orig_scale);
+    min_gap_orig_scale = std::fmin(2.5, min_gap_orig_scale);
+
     /* TODO: here it's not necessary to sort the whole data, only top/bottom N */
 
     /* sort the data */
@@ -141,6 +149,7 @@ bool define_numerical_cluster(double *restrict x, size_t *restrict ix_arr, size_
 
     /* calculate statistics with tails and previous outliers excluded */
     cnt = end_non_tail - st_non_tail + 1;
+    mean_prev = x[ ix_arr[st_non_tail] ];
     for (size_t row = st_non_tail; row <= end_non_tail; row++) {
         xval = x[ ix_arr[row] ];
         running_mean += (xval - running_mean) / (long double)(row - st_non_tail + 1);
@@ -157,10 +166,16 @@ bool define_numerical_cluster(double *restrict x, size_t *restrict ix_arr, size_
     if ((!isinf(left_tail) || !isinf(right_tail)) && !is_log_transf && !is_exp_transf) {
         sd *= 0.5;
     }
+    sd = std::fmax(sd, 1e-15);
+    while (std::numeric_limits<double>::epsilon() > sd*std::fmin(min_gap, z_norm))
+        sd *= 4;
     cluster.cluster_mean = mean;
     cluster.cluster_sd = sd;
     cnt = end - st + 1;
 
+    /* TODO: review how to better set this limit */
+    tail_size = std::min(tail_size, log2ceil(end - st + 1));
+
     /* see if the minimum and/or maximum values qualify for outliers */
     if (-z_score(x[ix_arr[st]],  mean, sd) >= z_outlier && x[ix_arr[st]]  > left_tail)  has_low_values  = true;
     if ( z_score(x[ix_arr[end]], mean, sd) >= z_outlier && x[ix_arr[end]] < right_tail) has_high_values = true;
@@ -170,6 +185,22 @@ bool define_numerical_cluster(double *restrict x, size_t *restrict ix_arr, size_
         for (size_t row = st; row < st + tail_size; row++) {
 
             if (( z_score(x[ix_arr[row + 1]], mean, sd) - z_score(x[ix_arr[row]], mean, sd) ) >= min_gap) {
+                
+                /* if the variable was transformed, check that the gap is still wide in the original scale */
+                if (is_exp_transf || is_log_transf) {
+                    curr_gap = orig_x[ix_arr[row + 1]] - orig_x[ix_arr[row]];
+                    next_gap = 0;
+                    for (size_t rr = row + 1; rr < end; rr++) {
+                        if (orig_x[ix_arr[rr+1]] > orig_x[ix_arr[rr]]) {
+                            next_gap = orig_x[ix_arr[rr+1]] - orig_x[ix_arr[rr]];
+                            break;
+                        }
+                    }
+
+                    if (next_gap > 0 && curr_gap/next_gap < min_gap_orig_scale)
+                        continue;
+                }
+
                 st_normals = row + 1;
                 if (is_exp_transf) {
                     cluster.lower_lim = log(x[ix_arr[row + 1]] - min_gap * sd) * orig_sd + orig_mean;
@@ -180,6 +211,12 @@ bool define_numerical_cluster(double *restrict x, size_t *restrict ix_arr, size_
                 }
                 cluster.display_lim_low = orig_x[ix_arr[row + 1]];
                 cluster.perc_above = (long double)(end - st_normals + 1) / (long double)(end - st + 1);
+
+                eps = 1e-15;
+                while (cluster.display_lim_low <= cluster.lower_lim) {
+                    cluster.lower_lim -= eps;
+                    eps *= 4;
+                }
                 break;
             }
             if (z_score(x[ix_arr[row]], mean, sd) > -z_outlier) break;
@@ -233,6 +270,25 @@ bool define_numerical_cluster(double *restrict x, size_t *restrict ix_arr, size_
             cluster.lower_lim = exp(x[ix_arr[st]] - min_gap * sd) + log_minval;
         }
 
+        if (cluster.lower_lim > -HUGE_VAL) {
+            eps = 1e-15;
+            while (cluster.lower_lim >= orig_x[ix_arr[st]]) {
+                cluster.lower_lim -= eps;
+                eps *= 4.;
+            }
+        }
+
+        if (is_exp_transf || is_log_transf) {
+            for (size_t row = st; row < end; row++) {
+                if (orig_x[ix_arr[row+1]] > orig_x[ix_arr[row]]) {
+                    curr_gap = orig_x[ix_arr[row+1]] - orig_x[ix_arr[row]];
+                    lim_by_orig = orig_x[ix_arr[st]] - min_gap_orig_scale * curr_gap;
+                    cluster.lower_lim = std::fmin(cluster.lower_lim, lim_by_orig);
+                    break;
+                }
+            }
+        }
+
         cluster.display_lim_low = orig_x[ix_arr[st]];
 
     }
@@ -241,6 +297,22 @@ bool define_numerical_cluster(double *restrict x, size_t *restrict ix_arr, size_
         for (size_t row = end; row > (end - tail_size); row--) {
 
             if (( z_score(x[ix_arr[row]], mean, sd) - z_score(x[ix_arr[row - 1]], mean, sd) ) >= min_gap) {
+                
+                /* if the variable was transformed, check that the gap is still wide in the original scale */
+                if (is_exp_transf || is_log_transf) {
+                    curr_gap = orig_x[ix_arr[row]] - orig_x[ix_arr[row - 1]];
+                    next_gap = 0;
+                    for (size_t rr = row-1; rr > st; rr--) {
+                        if (orig_x[ix_arr[rr]] > orig_x[ix_arr[rr-1]]) {
+                            next_gap = orig_x[ix_arr[rr]] - orig_x[ix_arr[rr-1]];
+                            break;
+                        }
+                    }
+
+                    if (next_gap > 0 && curr_gap/next_gap < min_gap_orig_scale)
+                        continue;
+                }
+
                 end_normals = row - 1;
                 if (is_exp_transf) {
                     cluster.upper_lim = log(x[ix_arr[row - 1]] + min_gap * sd) * orig_sd + orig_mean;
@@ -251,6 +323,12 @@ bool define_numerical_cluster(double *restrict x, size_t *restrict ix_arr, size_
                 }
                 cluster.display_lim_high = orig_x[ix_arr[row - 1]];
                 cluster.perc_below = (long double)(end_normals - st + 1) / (long double)(end - st + 1);
+
+                eps = 1e-15;
+                while (cluster.display_lim_high >= cluster.upper_lim) {
+                    cluster.upper_lim += eps;
+                    eps *= 4;
+                }
                 break;
             }
             if (z_score(x[ix_arr[row]], mean, sd) < z_outlier) break;
@@ -305,6 +383,25 @@ bool define_numerical_cluster(double *restrict x, size_t *restrict ix_arr, size_
             cluster.upper_lim = exp(x[ix_arr[end]] + min_gap * sd) + log_minval;
         }
 
+        if (cluster.upper_lim < HUGE_VAL) {
+            eps = 1e-15;
+            while (cluster.upper_lim <= orig_x[ix_arr[end]]) {
+                cluster.upper_lim += eps;
+                eps *= 4.;
+            }
+        }
+
+        if (is_exp_transf || is_log_transf) {
+            for (size_t row = end; row < st; row--) {
+                if (orig_x[ix_arr[row]] > orig_x[ix_arr[row-1]]) {
+                    curr_gap = orig_x[ix_arr[row]] - orig_x[ix_arr[row-1]];
+                    lim_by_orig = orig_x[ix_arr[end]] + min_gap_orig_scale * curr_gap;
+                    cluster.upper_lim = std::fmax(cluster.upper_lim, lim_by_orig);
+                    break;
+                }
+            }
+        }
+
         cluster.display_lim_high = orig_x[ix_arr[end]];
     }
 
@@ -313,8 +410,8 @@ bool define_numerical_cluster(double *restrict x, size_t *restrict ix_arr, size_
         size_t st_disp  = has_low_values?  st_normals  : st;
         size_t end_disp = has_high_values? end_normals : end;
         running_mean = 0;
-        mean_prev    = 0;
         running_ssq  = 0;
+        mean_prev    = orig_x[ix_arr[st_disp]];
         for (size_t row = st_disp; row <= end_disp; row++) {
             xval = orig_x[ix_arr[row]];
             running_mean += (xval - running_mean) / (long double)(row - st_disp + 1);
@@ -372,7 +469,7 @@ bool define_numerical_cluster(double *restrict x, size_t *restrict ix_arr, size_
 void define_categ_cluster_no_cond(int *restrict x, size_t *restrict ix_arr, size_t st, size_t end, size_t ncateg,
                                   double *restrict outlier_scores, size_t *restrict outlier_clusters, size_t *restrict outlier_trees,
                                   size_t *restrict outlier_depth, Cluster &cluster,
-                                  size_t *restrict categ_counts, char *restrict is_outlier, double perc_next_most_comm)
+                                  size_t *restrict categ_counts, signed char *restrict is_outlier, double perc_next_most_comm)
 {
     size_t cnt_common = end - st + 1;
     cluster.cluster_size = cnt_common;
@@ -474,7 +571,7 @@ bool define_categ_cluster(int *restrict x, size_t *restrict ix_arr, size_t st, s
                           double max_perc_outliers, double z_norm, double z_outlier,
                           long double *restrict perc_threshold, long double *restrict prop_prior,
                           size_t *restrict buffer_categ_counts, long double *restrict buffer_categ_pct,
-                          size_t *restrict buffer_categ_ix, char *restrict buffer_outliers,
+                          size_t *restrict buffer_categ_ix, signed char *restrict buffer_outliers,
                           bool *restrict drop_cluster)
 {
     bool found_outliers, new_is_outlier;
@@ -567,7 +664,7 @@ bool define_categ_cluster(int *restrict x, size_t *restrict ix_arr, size_t st, s
 
             cluster.perc_in_subset = (long double) buffer_categ_counts[cluster.categ_maj] / tot_dbl;
             for (size_t cat = 0; cat < ncateg; cat++) {
-                if (cat == cluster.categ_maj)
+                if ((int)cat == cluster.categ_maj)
                     continue;
                 if (cluster.subset_common[cat] != 0) {
                     cluster.score_categ[cat] = (long double)(tot - buffer_categ_counts[cluster.categ_maj] + 1)
@@ -626,10 +723,10 @@ void simplify_when_equal_cond(std::vector<Cluster> &clusters, int ncat_ord[])
                 } else {
 
                     size_subset_excl = std::accumulate(clusters[clust].split_subset.begin(), clusters[clust].split_subset.end(), (size_t)0,
-                                                       [](const size_t a, const char b){return a + ((b < 0)? 1 : 0);});
+                                                       [](const size_t a, const signed char b){return a + ((b < 0)? 1 : 0);});
                     if (size_subset_excl > 0) continue;
                     size_subset = std::accumulate(clusters[clust].split_subset.begin(), clusters[clust].split_subset.end(), (size_t)0,
-                                                  [](const size_t a, const char b){return a + ((b > 0)? 1 : 0);});
+                                                  [](const size_t a, const signed char b){return a + ((b > 0)? 1 : 0);});
                     if (size_subset == 1) {
 
                         do {col_equal++;} while (clusters[clust].split_subset[col_equal] <= 0);
@@ -681,6 +778,7 @@ void simplify_when_equal_cond(std::vector<Cluster> &clusters, int ncat_ord[])
                 break;
             }
 
+            default: {}
         }
 
     }
@@ -711,7 +809,7 @@ void simplify_when_equal_cond(std::vector<ClusterTree> &trees, int ncat_ord[])
             case Categorical:
             {
                 size_subset_excl = std::accumulate(trees[tree].split_subset.begin(), trees[tree].split_subset.end(), (size_t)0,
-                                                   [](const size_t a, const char b){return a + ((b < 0)? 1 : 0);});
+                                                   [](const size_t a, const signed char b){return a + ((b < 0)? 1 : 0);});
                 if (size_subset_excl > 0) continue;
 
                 col_equal = -1;
@@ -747,6 +845,8 @@ void simplify_when_equal_cond(std::vector<ClusterTree> &trees, int ncat_ord[])
                                 trees[tree].split_this_branch = Equal;
                                 break;
                             }
+
+                            default: {}
                         }
                     }
 
@@ -755,7 +855,7 @@ void simplify_when_equal_cond(std::vector<ClusterTree> &trees, int ncat_ord[])
                 else {
 
                     size_subset = std::accumulate(trees[tree].split_subset.begin(), trees[tree].split_subset.end(), (size_t)0,
-                                                  [](const size_t a, const char b){return a + ((b > 0)? 1 : 0);});
+                                                  [](const size_t a, const signed char b){return a + ((b > 0)? 1 : 0);});
                     if (size_subset == 1) {
 
                         do {col_equal++;} while (trees[tree].split_subset[col_equal] <= 0);
@@ -778,6 +878,8 @@ void simplify_when_equal_cond(std::vector<ClusterTree> &trees, int ncat_ord[])
                                     trees[tree].split_this_branch = Equal;
                                     break;
                                 }
+
+                                default: {}
                             }
                         }
 
@@ -801,6 +903,8 @@ void simplify_when_equal_cond(std::vector<ClusterTree> &trees, int ncat_ord[])
                                     trees[tree].split_this_branch = Equal;
                                     break;
                                 }
+
+                                default: {}
                             }
                         }
 
@@ -846,6 +950,7 @@ void simplify_when_equal_cond(std::vector<ClusterTree> &trees, int ncat_ord[])
                 break;
             }
 
+            default: {}
         }
 
     }