@trebco/treb 29.8.3 → 30.1.0

package/treb-charts/src/chart-utils.ts

@@ -1,10 +1,11 @@
 
 import { type UnionValue, ValueType, type ArrayUnion, IsComplex, type CellValue } from 'treb-base-types';
 import { IsArrayUnion, IsMetadata, IsSeries, LegendStyle } from './chart-types';
-import type { SubSeries, SeriesType, BarData, ChartDataBaseType, ChartData, ScatterData2, DonutSlice, BubbleChartData } from './chart-types';
+import type { SubSeries, SeriesType, BarData, ChartDataBaseType, ChartData, ScatterData2, DonutSlice, BubbleChartData, BoxPlotData } from './chart-types';
 import { NumberFormatCache } from 'treb-format';
 import { Util } from './util';
 import type { ReferenceSeries } from './chart-types';
+import type { RangeScale } from 'treb-utils';
 
 /**
  * this file is the concrete translation from function arguments
@@ -320,8 +321,38 @@ export const TransformSeriesData = (raw_data?: UnionValue, default_x?: UnionValu
   return list;
 };
 
+const AutoFormat = (scale: RangeScale): string => {
+
+  const zep = Math.abs(scale.step) % 1;
+  if (!zep) {
+    const log10 = Math.log10(Math.abs(scale.step));
+    if (log10 >= 5) {
+      return 'Scientific';
+    }
+    return '#,##0';
+  }
+  else {
+    const log10 = Math.log10(Math.abs(zep));
+    if (log10 < -4) {
+      return 'Scientific';
+    }
+    let count = 0;
+    for (let i = 0; i < scale.count; i++) {
+      const value = ((scale.min + scale.step * i) % 1).toFixed(6).replace(/0+$/, '');
+      count = Math.max(count, value.length - 2);
+    }
+    let format = '#,##0.';
+    for (let i = 0; i < count; i++) {
+      format += '0';
+    }
+    return format;
+  }
+
+
+};
+
 /** get a unified scale, and formats */
-export const CommonData = (series: SeriesType[], y_floor?: number, y_ceiling?: number, x_floor?: number, x_ceiling?: number) => {
+export const CommonData = (series: SeriesType[], y_floor?: number, y_ceiling?: number, x_floor?: number, x_ceiling?: number, auto_number_format?: boolean) => {
 
   let x_format = '';
   let y_format = '';
@@ -402,6 +433,11 @@ export const CommonData = (series: SeriesType[], y_floor?: number, y_ceiling?: n
     }
   }
 
+  if (!y_format && auto_number_format) {
+    // y_format = default_number_format;
+    y_format = AutoFormat(y_scale);
+  }
+
   if (y_format) {
     y_labels = [];
     const format = NumberFormatCache.Get(y_format);
@@ -451,6 +487,133 @@ const ApplyLabels = (series_list: SeriesType[], pattern: string, category_labels
 
 };
 
+/**
+ * return quartiles. we use the Tukey hinge-style. See
+ * 
+ * https://en.wikipedia.org/wiki/Quartile
+ * 
+ * Specifically,
+ * 
+ * - Use the median to divide the ordered data set into two halves. The median 
+ *   becomes the second quartiles.
+ * 
+ *   - If there are an odd number of data points in the original ordered data 
+ *     set, include the median (the central value in the ordered list) in both 
+ *     halves.
+ * 
+ *   - If there are an even number of data points in the original ordered data 
+ *     set, split this data set exactly in half.
+ * 
+ * - The lower quartile value is the median of the lower half of the data. The 
+ *   upper quartile value is the median of the upper half of the data.
+ * 
+ * @param data - must be sorted with no holes
+ */
+export const BoxStats = (data: number[]) => {
+
+  const median = (data: number[]) => {
+    const n = data.length;
+    if (n % 2) {
+      return data[Math.floor(n/2)];
+    }
+    else {
+      return (data[n/2] + data[n/2 - 1])/2;
+    }
+  };
+
+  const n = data.length;
+  const quartiles: [number, number, number] = [0, median(data), 0];
+
+  if (n % 2) {
+    quartiles[0] = median(data.slice(0, Math.ceil(n/2)));
+    quartiles[2] = median(data.slice(Math.floor(n/2)));
+  }
+  else {
+    quartiles[0] = median(data.slice(0, n/2));
+    quartiles[2] = median(data.slice(n/2));
+  }
+
+  const iqr = quartiles[2] - quartiles[0];
+  const whiskers: [number, number] = [0, 0];
+
+  let sum = 0;
+  for (let i = 0; i < n; i++) {
+    sum += data[i];
+  }
+      
+  for (let i = 0; i < n; i++) {
+    const pt = data[i];
+    if (pt >= quartiles[0] - iqr * 1.5) {
+      whiskers[0] = pt;
+      break;
+    }
+  }
+
+  for (let i = n-1; i >= 0; i--) {
+    const pt = data[i];
+    if (pt <= quartiles[2] + iqr * 1.5) {
+      whiskers[1] = pt;
+      break;
+    }
+  }
+  
+  return {
+    data, 
+    quartiles, 
+    whiskers,
+    iqr,
+    n, 
+    mean: n ? sum/n : 0,
+    min: data[0], 
+    max: data[n-1],
+  };
+
+};
+
+//------------------------------------------------------------------------------
+
+export const CreateBoxPlot = (args: UnionValue[]): ChartData => {
+
+  const series: SeriesType[] = TransformSeriesData(args[0]);
+
+  const common = CommonData(series, undefined, undefined, undefined, undefined, true);
+
+  let max_n = 0;
+
+  const stats: BoxPlotData['data'] = series.map(series => {
+    const data = series.y.data.slice(0).filter((test): test is number => test !== undefined).sort((a, b) => a - b);
+    const result = BoxStats(data);
+    max_n = Math.max(max_n, result.n);
+    return result;
+  });
+
+  const title = args[1]?.toString() || undefined;
+  const x_labels: string[] = [];
+  const series_names: string[] = [];
+  const format = NumberFormatCache.Get('#,##0');
+
+  for (const [index, entry] of stats.entries()) {
+    x_labels.push(format.Format(entry.n));
+    const s = series[index];
+    series_names.push(s.label || `Series ${index + 1}`);
+  }
+
+  const chart_data: BoxPlotData = {
+    type: 'box',
+    series,
+    title,
+    max_n,
+    data: stats,
+    x_labels, 
+    series_names: series.some(test => !!test.label) ? series_names : undefined,
+    scale: common.y.scale,
+    y_labels: common.y.labels,
+  };
+  
+  return chart_data;
+
+};
+
 //------------------------------------------------------------------------------
 
 export const CreateBubbleChart = (args: UnionValue[]): ChartData => {

package/treb-charts/src/chart.ts

@@ -4,6 +4,7 @@ import type { ChartData } from './chart-types';
 import type { ExtendedUnion, UnionValue } from 'treb-base-types';
 import * as ChartUtils from './chart-utils';
 import { DefaultChartRenderer } from './default-chart-renderer';
+import type { ChartFunction } from './chart-functions';
 
 /**
  * transitioning to new structure, this should mirror the old chart 
@@ -29,7 +30,7 @@ export class Chart {
     this.renderer.Initialize(node);
   }
 
-  public Exec(func: string, union: ExtendedUnion) {
+  public Exec(func: ChartFunction, union: ExtendedUnion) {
 
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
     const args: any[] = (union?.value as any[]) || [];
@@ -69,6 +70,10 @@ export class Chart {
         this.chart_data = ChartUtils.CreateBubbleChart(args);
         break;
 
+      case 'box.plot':
+        this.chart_data = ChartUtils.CreateBoxPlot(args);
+        break;
+
       default:
         this.Clear();
         break;

package/treb-charts/src/default-chart-renderer.ts

@@ -97,6 +97,7 @@ export class DefaultChartRenderer implements ChartRendererType {
         || chart_data.type === 'bar'
         || chart_data.type === 'scatter2'
         || chart_data.type === 'bubble'
+        || chart_data.type === 'box'
         ) {
 
       // we need to measure first, then lay out the other axis, then we
@@ -116,6 +117,19 @@ export class DefaultChartRenderer implements ChartRendererType {
         });
       }
 
+      let x_metrics2: Metrics[] = [];
+      let max_x_height2 = 0;
+
+      if (chart_data.type === 'box' && chart_data.series_names?.length) {
+        x_metrics2 = chart_data.series_names.map((text) => {
+          const metrics = this.renderer.MeasureText(text, ['axis-label', 'x-axis-label', 'series-name'], true);
+          max_x_height2 = Math.max(max_x_height2, metrics.height);
+          return metrics;
+        });
+      }
+
+      const extra_padding = max_x_height && max_x_height2 ? 4 : 0;
+
       // measure & render y axis
 
       if (chart_data.y_labels && chart_data.y_labels.length) {
@@ -144,6 +158,12 @@ export class DefaultChartRenderer implements ChartRendererType {
         if (x_metrics.length) {
           area.bottom -= (max_x_height + chart_margin.bottom);
         }
+        if (x_metrics2.length) {
+          area.bottom -= (max_x_height2 + chart_margin.bottom);
+        }
+        if (extra_padding) {
+          area.bottom -= extra_padding;
+        }
 
         if (chart_data.type === 'bar') {
           this.renderer.RenderYAxisBar(area, area.left + max_width, y_labels, ['axis-label', 'y-axis-label']);
@@ -157,7 +177,7 @@ export class DefaultChartRenderer implements ChartRendererType {
 
       // now render x axis
 
-      if (x_metrics.length && chart_data.x_labels && chart_data.x_labels.length) {
+      if (x_metrics.length && chart_data.x_labels?.length) {
 
         const tick = (chart_data.type === 'histogram2');
         const offset_tick = (
@@ -193,6 +213,31 @@ export class DefaultChartRenderer implements ChartRendererType {
 
       }
 
+      if (extra_padding) {
+        area.bottom += extra_padding;
+      }
+
+      if (chart_data.type === 'box' && x_metrics2.length && chart_data.series_names?.length) {
+
+        // console.info({chart_data, x_metrics2});
+
+        if (chart_data.y_labels) {
+          // undo, temp
+          area.bottom += (max_x_height + max_x_height2 + extra_padding + chart_margin.bottom);
+        }
+
+        // render
+        this.renderer.RenderXAxis(area, 
+          true, // offset_tick,
+          chart_data.series_names, 
+          x_metrics2, 
+          ['axis-label', 'x-axis-label', 'series-name']);
+
+        // update bottom (either we unwound for labels, or we need to do it the first time)
+        area.bottom -= (max_x_height + max_x_height2 + chart_margin.bottom + chart_margin.bottom);
+
+      }
+
     }
 
     // now do type-specific rendering
@@ -204,6 +249,30 @@ export class DefaultChartRenderer implements ChartRendererType {
       this.renderer.RenderPoints(area, chart_data.x, chart_data.y, 'mc mc-correlation series-1');
       break;
 
+    case 'box':
+      this.renderer.RenderGrid(area, 
+        chart_data.scale.count, 
+        undefined, // chart_data.x_scale.count + 1, // (sigh)
+        'chart-grid', zeros);
+
+      // FIXME: override index for coloring
+
+      for (const [index, data] of chart_data.data.entries()) {
+
+        if (data.iqr > 0) {
+          this.renderer.RenderBoxAndWhisker(
+              area, 
+              data, 
+              index, 
+              chart_data.max_n,
+              chart_data.scale, 
+              chart_data.data.length, 
+              `box-plot series-${index}`);
+          }
+      }
+
+      break;
+
     case 'bubble':
 
       if (chart_data.x_scale.min <= 0 && chart_data.x_scale.max >= 0) {

package/treb-charts/src/index.ts

@@ -21,5 +21,6 @@
 
 export { Chart } from './chart';
 export { ChartFunctions } from './chart-functions';
+export type { ChartFunction } from './chart-functions';
 export { Util } from './util';
 export type { ChartRenderer } from './renderer-type';

package/treb-charts/src/renderer.ts

@@ -21,7 +21,7 @@
 
 import type { Size, Point } from './rectangle';
 import { Area } from './rectangle';
-import type { DonutSlice, LegendOptions, SeriesType} from './chart-types';
+import type { BoxPlotData, DonutSlice, LegendOptions, SeriesType} from './chart-types';
 import { LegendLayout, LegendPosition, LegendStyle } from './chart-types';
 import type { RangeScale } from 'treb-utils';
 
@@ -268,7 +268,7 @@ export class ChartRenderer {
   }
 
   public Resize(): void {
-    const bounds = this.parent.getBoundingClientRect();
+    const bounds = this.svg_node.getBoundingClientRect();
     this.svg_node.setAttribute('width', bounds.width.toString());
     this.svg_node.setAttribute('height', bounds.height.toString());
     this.size = {
@@ -1232,6 +1232,99 @@ export class ChartRenderer {
 
   }
 
+  public RenderBoxAndWhisker(
+      area: Area, 
+      data: BoxPlotData['data'][0], 
+      index: number, 
+      max_n: number, 
+      scale: RangeScale, 
+      n: number, 
+      classes?: string|string[]){
+
+    const group = SVGNode('g', {class: classes});
+    this.group.appendChild(group);
+
+    // space for each box is 1/n of the chart area. we'll allocate margin on a box basis.
+    const width = area.width / n;
+    const margin = width * .20; // ??
+    const box_default_width = Math.min(width - 2 * margin, 90);
+
+    let box_width = box_default_width;
+
+    if (max_n > 0) {
+      box_width = box_width * Math.sqrt(data.n) / Math.sqrt(max_n);
+    }
+
+    const Y = (value: number) => area.top + area.height - (value - scale.min) / (scale.max - scale.min) * area.height; // wtf? 
+
+    const center = area.left + index * width + width / 2;
+
+    const q1 = Y(data.quartiles[0]);
+    const q3 = Y(data.quartiles[2]);
+   
+    group.appendChild(SVGNode('rect', {
+      class: `iqr`,
+      x: area.left + index * width + (width - box_width) / 2,
+      y: q3,
+      width: box_width,
+      height: q1 - q3,
+    }));
+
+    group.appendChild(SVGNode('line', {
+      class: `median`,
+      x1: center - box_width * .5,
+      x2: center + box_width * .5,
+      y1: Y(data.quartiles[1]),
+      y2: Y(data.quartiles[1]),
+    }));
+
+    group.appendChild(SVGNode('line', {
+      class: `whisker`,
+      x1: center - box_width * .25,
+      x2: center + box_width * .25,
+      y1: Y(data.whiskers[0]),
+      y2: Y(data.whiskers[0]),
+    }));
+
+    group.appendChild(SVGNode('line', {
+      class: `whisker-extent`,
+      x1: center,
+      x2: center,
+      y1: Y(data.whiskers[0]) - 2,
+      y2: Y(data.quartiles[0]) + 2,
+    }));
+
+    group.appendChild(SVGNode('line', {
+      class: `whisker`,
+      x1: center - box_width * .25,
+      x2: center + box_width * .25,
+      y1: Y(data.whiskers[1]),
+      y2: Y(data.whiskers[1]),
+    }));
+
+    group.appendChild(SVGNode('line', {
+      class: `whisker-extent`,
+      x1: center,
+      x2: center,
+      y1: Y(data.whiskers[1]) + 2,
+      y2: Y(data.quartiles[2]) - 2,
+    }));
+
+    for (const point of data.data) {
+      if (point < data.whiskers[0] || point > data.whiskers[1]) {
+        group.appendChild(SVGNode('circle', {
+          class: `outlier`,
+          cx: center,
+          cy: Y(point),
+          r: 3, // default; we can override in css
+        }));
+      }
+    }
+
+
+  }
+
+
   public RenderBubbleSeries(
         area: Area,
         series: SeriesType,

package/treb-charts/style/charts.scss

@@ -78,6 +78,11 @@
   /* axis labels */
   .axis-group {
     font-size: .9em;
+
+    .series-name {
+      font-size: 1.3em;
+    }
+
   }
 
   /* default text color */
@@ -173,6 +178,42 @@
     }
   }
 
+  .box-plot {
+
+    .iqr {
+      fill: none;
+      stroke: CanvasText;
+      stroke-width: 1px;
+    }
+
+    .median {
+      stroke-width: 3px;
+      stroke: CanvasText;
+      fill: none;
+    }
+
+    .outlier {
+      stroke-width: 1px;
+      stroke: CanvasText;
+      fill: none;
+      r: 3;
+    }
+
+    .whisker {
+      stroke-width: 1px;
+      stroke: CanvasText;
+      fill: none;
+    }
+
+    .whisker-extent {
+      stroke-width: 1px;
+      stroke: CanvasText;
+      fill: none;
+      stroke-dasharray: 3 3;
+    }
+
+  }
+
   .bubble-chart {
 
     stroke-width: 3;

package/treb-embed/src/embedded-spreadsheet.ts

@@ -92,7 +92,7 @@ import type { LanguageModel, TranslatedFunctionDescriptor } from './language-mod
 import type { SelectionState } from './selection-state';
 import type { BorderToolbarMessage, ToolbarMessage } from './toolbar-message';
 
-import { Chart, ChartFunctions } from 'treb-charts';
+import { Chart, ChartFunctions, type ChartFunction } from 'treb-charts';
 import type { SetRangeOptions, ClipboardData, PasteOptions } from 'treb-grid';
 
 import type { StateLeafVertex } from 'treb-calculator';
@@ -1203,7 +1203,8 @@ export class EmbeddedSpreadsheet<USER_DATA_TYPE = unknown> {
    */
   protected CreateCalculator(model: DataModel, options: EmbeddedSpreadsheetOptions) {
     return new Calculator(model, {
-      complex_numbers: options.complex
+      complex_numbers: options.complex,
+      spill: options.spill,
     });
   }
 
@@ -2356,7 +2357,7 @@ export class EmbeddedSpreadsheet<USER_DATA_TYPE = unknown> {
             return true;
           });
 
-          const expr_name = parse_result.expression.name.toLowerCase();
+          const expr_name = parse_result.expression.name.toLowerCase() as ChartFunction;
           const result = this.calculator.CalculateExpression(parse_result.expression);
           chart.Exec(expr_name, result as ExtendedUnion); // FIXME: type?
         }
@@ -3752,6 +3753,12 @@ export class EmbeddedSpreadsheet<USER_DATA_TYPE = unknown> {
     }
 
     this.calculator.Calculate(area);
+
+    if (this.calculator.grid_expanded) {
+      // console.info("GRID EXPANDED");
+      this.grid.UpdateLayout();
+    }
+
     this.ApplyConditionalFormats(this.grid.active_sheet, false);
 
     this.grid.Update(true); // , area);
@@ -5380,7 +5387,7 @@ export class EmbeddedSpreadsheet<USER_DATA_TYPE = unknown> {
                   return true;
                 });
 
-                const expr_name = parse_result.expression.name.toLowerCase();
+                const expr_name = parse_result.expression.name.toLowerCase() as ChartFunction;
 
                 const result = this.calculator.CalculateExpression(parse_result.expression);
 

package/treb-embed/src/options.ts

@@ -321,6 +321,12 @@ export interface EmbeddedSpreadsheetOptions {
    */
   indent_buttons?: boolean;
 
+  /**
+   * enable spill arrays and spill references. this is on by default 
+   * starting in 30.1.0. set to false to disable.
+   */
+  spill?: boolean;
+
 }
 
 /**
@@ -363,4 +369,6 @@ export const DefaultOptions: EmbeddedSpreadsheetOptions = {
   spinner: false,
   complex: 'off',
 
+  spill: true,
+
 };

package/treb-embed/style/dark-theme.scss

@@ -110,6 +110,10 @@
 
   --treb-toolbar-color: rgb(221,221,221);
   
+  --treb-spill-border-color: pink;
+  --treb-spill-border-filter: drop-shadow( 3px 3px 2px rgba(255, 255, 255, .95));
+  // --treb-spill-border-dasharray: 4 2;
+  // --treb-spill-border-width: 2px;
 
 }
 

package/treb-embed/style/grid.scss

@@ -38,6 +38,21 @@
 
 .treb-main.treb-main {
 
+  .treb-spill-border {
+
+    position: absolute;
+    pointer-events: none;
+
+    rect {
+      stroke: var(--treb-spill-border-color, rgb(92, 92, 224));
+      stroke-dasharray: var(--treb-spill-border-dasharray, 0);
+      fill: none;
+      stroke-width: var(--treb-spill-border-width, 1px);
+      z-index: $z-index-spill-border;
+      filter: var(--treb-spill-border-filter, drop-shadow( 3px 3px 2px rgba(0, 0, 0, .5)));
+    }
+  }
+
 
   /* 
     * new: we need the rendering buffer to inherit font size. so 

package/treb-embed/style/z-index.scss

@@ -31,6 +31,9 @@ $z-index-corner:                     20;
 $z-index-header-tile-cover:          22;
 $z-index-cell-editor:                24;
 $z-index-scroller:                   26; /* legacy only */
+
+$z-index-spill-border:               35;
+
 $z-index-autocomplete:               39;
 $z-index-tooltip:                    39;
 $z-index-dropdown-caret:             39;

package/treb-export/src/import2.ts

@@ -1047,6 +1047,15 @@ export class Importer {
 
             break;
 
+          case ChartType.Box:
+            type = 'treb-chart';
+            func = 'Box.Plot';
+            if (series?.length) {
+              args[0] = `Group(${series.map(s => `Series(${s.title || ''},,${s.values||''})` || '').join(', ')})`;
+            }
+            args[1] = descriptor.chart.title;
+            break;
+
           case ChartType.Scatter:
             type = 'treb-chart';
             func = 'Scatter.Line';