@trebco/treb 32.5.0 → 32.6.6

package/treb-export/src/import.ts

@@ -26,7 +26,7 @@ import Base64JS from 'base64-js';
 
 import type { AnchoredChartDescription, AnchoredImageDescription, AnchoredTextBoxDescription} from './workbook';
 import { ChartType, ConditionalFormatOperators, Workbook } from './workbook';
-import type { ParseResult } from 'treb-parser';
+import type { ExpressionUnit, ParseResult, UnitCall } from 'treb-parser';
 import { DecimalMarkType, Parser } from 'treb-parser';
 import type { RangeType, AddressType, HyperlinkType } from './address-type';
 import { is_range, ShiftRange, InRange, is_address } from './address-type';
@@ -42,6 +42,7 @@ import { ColumnWidthToPixels } from './column-width';
 import type { DataValidation, AnnotationType } from 'treb-data-model';
 import { ZipWrapper } from './zip-wrapper';
 import type { ConditionalFormat } from 'treb-data-model';
+import { LookupMetadata, type MetadataFlags } from './metadata';
 
 interface SharedFormula {
   row: number;
@@ -57,6 +58,7 @@ interface CellElementType {
     r?: string;
     t?: string;
     s?: string;
+    cm?: string;
   };
   v?: string|number|{
     t$: string;
@@ -147,6 +149,7 @@ export class Importer {
     element: CellElementType,
     shared_formulae: SharedFormulaMap,
     arrays: RangeType[],
+    dynamic_arrays: RangeType[],
     merges: RangeType[],
     links: HyperlinkType[],
     // validations: Array<{ address: ICellAddress, validation: DataValidation }>,
@@ -165,6 +168,15 @@ export class Importer {
       return undefined;
     }
 
+    // new (to us) metadata
+    let metadata_flags: MetadataFlags = {};
+    if (element.a$?.cm) {
+      const cm_index = Number(element.a$?.cm);
+      if (this.workbook?.metadata) {
+        metadata_flags = LookupMetadata(this.workbook.metadata, 'cell', cm_index).flags;
+      }
+    }
+
     // console.info(element);
 
     let value: undefined | number | boolean | string;
@@ -213,6 +225,8 @@ export class Importer {
 
         if (formula) {
 
+          // console.info("F", formula);
+
           // doing it like this is sloppy (also does not work properly).
           value = '=' + formula.replace(/^_xll\./g, '');
 
@@ -221,10 +235,66 @@ export class Importer {
             value = formula;
           }
           else {
+
             const parse_result = this.parser.Parse(formula); // l10n?
             if (parse_result.expression) {
-              this.parser.Walk(parse_result.expression, (unit) => {
+
+              const TrimPrefixes = (name: string) => {
+
+                if (/^_xll\./.test(name)) {
+                  name = name.substring(5);
+                }
+                if (/^_xlfn\./.test(name)) {
+                  console.info("xlfn:", name);
+                  name = name.substring(6);
+                }
+                if (/^_xlws\./.test(name)) {
+                  console.info("xlws:", name);
+                  name = name.substring(6);
+                }
+
+                return name;
+              };
+
+              const TreeWalker = (unit: ExpressionUnit) => {
+                
                 if (unit.type === 'call') {
+
+                  // if we see _xlfn.SINGLE, translate that into 
+                  // @ + the name of the first parameter...
+                  //
+                  // this is solving the case where single appears, but it 
+                  // doesn't solve the case where it does not appear -- they 
+                  // may be using the array flag of the cell as an indicator?
+                  // but how then do they know it _should_ be an array? not 
+                  // sure, and I don't see any other indication.
+
+                  if (/^_xlfn\.single/i.test(unit.name)) {
+
+                    const first = unit.args[0];
+                    if (first.type === 'call') {
+
+                      // we could do this in place, we don't need to copy...
+                      // although it seems like a good idea. also watch out,
+                      // these SINGLEs could be nested.
+
+                      const replacement: UnitCall = JSON.parse(JSON.stringify(first));
+                      replacement.name = '@' + TrimPrefixes(replacement.name);
+
+                      for (let i = 0; i < replacement.args.length; i++) {
+                        replacement.args[i] = this.parser.Walk2(replacement.args[i], TreeWalker);
+                      }
+
+                      return replacement;
+                    }
+                    else {
+                      console.info("_xlfn.SINGLE unexpected argument", unit.args[0]);
+                    }
+                  }
+
+                  unit.name = TrimPrefixes(unit.name);
+
+                  /*
                   if (/^_xll\./.test(unit.name)) {
                     unit.name = unit.name.substring(5);
                   }
@@ -236,9 +306,15 @@ export class Importer {
                     console.info("xlws:", unit.name);
                     unit.name = unit.name.substring(6);
                   }
+                  */
+
                 }
                 return true;
-              });
+
+              };
+
+              parse_result.expression = this.parser.Walk2(parse_result.expression, TreeWalker);
+
               value = '=' + this.parser.Render(parse_result.expression, { missing: '' });
             }
           }
@@ -274,10 +350,38 @@ export class Importer {
           }
         }
 
+        //
+        // arrays and spill/dynamic arrays
+        //
+
         if (typeof element.f !== 'string' &&  element.f.a$?.t === 'array') {
           const translated = sheet.TranslateAddress(element.f.a$.ref || '');
-          if (is_range(translated)) {
-            arrays.push(ShiftRange(translated, -1, -1));
+
+          // why are we checking "is_range" here? this should be valid 
+          // even if the ref attribute is one cell, if it explicitly
+          // says t="array"
+
+          // we will need to adjust it, though? yes, because the lists 
+          // only accept ranges. note that range type has a superfluous 
+          // sheet parameter? ...
+
+          let range = translated;
+          if (!is_range(range)) {
+            range = {
+              to: { ...range },
+              from: { ...range },
+              sheet: range.sheet,
+            };
+          }
+
+          // if (is_range(translated)) 
+          {
+            if (metadata_flags['dynamic-array']) {
+              dynamic_arrays.push(ShiftRange(range, -1, -1));
+            }
+            else {
+              arrays.push(ShiftRange(range, -1, -1));
+            }
           }
         }
 
@@ -317,12 +421,14 @@ export class Importer {
     // but perhaps we should check that... although at this point we have 
     // already added the array so we need to check for root
 
-    for (const array of arrays) {
-      if (InRange(array, shifted) && (shifted.row !== array.from.row || shifted.col !== array.from.col)) {
-        calculated_type = type;
-        calculated_value = value;
-        value = undefined;
-        type = 'undefined'; // ValueType.undefined;
+    for (const set of [arrays, dynamic_arrays]) {
+      for (const array of set) {
+        if (InRange(array, shifted) && (shifted.row !== array.from.row || shifted.col !== array.from.col)) {
+          calculated_type = type;
+          calculated_value = value;
+          value = undefined;
+          type = 'undefined'; // ValueType.undefined;
+        }
       }
     }
 
@@ -369,6 +475,20 @@ export class Importer {
       }
     }
 
+    for (const range of dynamic_arrays) {
+      if (InRange(range, shifted)) {
+        result.spill = {
+          start: {
+            row: range.from.row,
+            column: range.from.col,
+          }, end: {
+            row: range.to.row,
+            column: range.to.col,
+          },
+        }
+      }
+    }
+
     for (const range of arrays) {
       if (InRange(range, shifted)) {
         result.area = {
@@ -675,6 +795,7 @@ export class Importer {
     const data: CellParseResult[] = [];
     const shared_formulae: {[index: string]: SharedFormula} = {};
     const arrays: RangeType[] = [];
+    const dynamic_arrays: RangeType[] = [];
     const merges: RangeType[] = [];
     const conditional_formats: ConditionalFormat[] = [];
     const links: HyperlinkType[] = [];
@@ -967,7 +1088,7 @@ export class Importer {
       const cells = row.c ? Array.isArray(row.c) ? row.c : [row.c] : [];
 
       for (const element of cells) {
-        const cell = this.ParseCell(sheet, element as unknown as CellElementType, shared_formulae, arrays, merges, links); // , validations);
+        const cell = this.ParseCell(sheet, element as unknown as CellElementType, shared_formulae, arrays, dynamic_arrays, merges, links); // , validations);
         if (cell) {
           data.push(cell);
         }
@@ -1254,12 +1375,21 @@ export class Importer {
 
             break;
 
+          case ChartType.Histogram:
+            type = 'treb-chart';
+            func = 'Histogram.Plot';
+            if (series?.length) {
+              // ...
+            }
+            args[1] = descriptor.chart.title;
+            break;
+
           case ChartType.Box:
             type = 'treb-chart';
             func = 'Box.Plot';
             if (series?.length) {
               args[0] = `Group(${series.map(s => `Series(${s.title || ''},,${s.values||''})`).join(', ')})`;
-              console.info("S?", {series}, args[0])
+              // console.info("S?", {series}, args[0])
             }
             args[1] = descriptor.chart.title;
             break;
@@ -1317,6 +1447,10 @@ export class Importer {
               args[1] = series[0]?.categories || '';
             }
 
+            if (descriptor.chart.type === ChartType.Column && descriptor.chart.flags?.includes('stacked')) {
+              args[3] = '"stacked"';
+            }
+
             break;
         }
 

package/treb-export/src/metadata.ts

@@ -0,0 +1,187 @@
+import { XMLUtils } from './xml-utils';
+
+/**
+ * https://learn.microsoft.com/en-us/dotnet/api/documentformat.openxml.spreadsheet.metadatarecord?view=openxml-3.0.1
+ * 
+ * ah, mixing 0-based and 1-based indexes. that's great. not at all confusing.
+ */
+export interface MetadataRecord  {
+
+  //
+  // A 1-based index to the metadata record type in metadataTypes.
+  //
+  t: number;
+
+  //
+  // A zero based index to a specific metadata record. If the corresponding 
+  // metadataType has name="XLMDX", then this is an index to a record in 
+  // mdxMetadata, otherwise this is an index to a record in the futureMetadata 
+  // section whose name matches the name of the metadataType.
+  //
+  v: number;
+
+}
+
+/** 
+ * https://learn.microsoft.com/en-us/dotnet/api/documentformat.openxml.spreadsheet.metadatatype?view=openxml-3.0.1
+ * 
+ * punting on all these attributes for now, except the ones we specifically 
+ * want.
+ */
+export interface MetadataType {
+
+  //
+  // Represents the name of this particular metadata type. This name shall be 
+  // unique amongst all other metadataTypes.
+  //
+  name: string;
+
+  //
+  // A Boolean flag indicating whether metadata is cell metadata. True when 
+  // the metadata is cell metadata, false otherwise - in the false case it 
+  // is considered to be value metadata.
+  //
+  cell_meta?: boolean;
+
+}
+
+/**
+ * the type `Future Feature Data Storage Area` is not actually defined? not 
+ * sure. maybe it's just a standard extension type and I need to look that up?
+ * 
+ * for the time being we'll use a list of flags...
+ */
+export interface MetadataFlags {
+  'dynamic-array'?: boolean;
+}
+
+/**
+ * https://learn.microsoft.com/en-us/dotnet/api/documentformat.openxml.spreadsheet.futuremetadata?view=openxml-3.0.1
+ * https://learn.microsoft.com/en-us/dotnet/api/documentformat.openxml.spreadsheet.futuremetadatablock?view=openxml-3.0.1
+ * 
+ * these are named, matching the names in "MetadataType"; we'll use those
+ * as indexes, and not store them in the object.
+ * 
+ */
+export interface FutureMetadata {
+  flags: MetadataFlags;
+}
+
+/**
+ * we're just reflecting the actual data at this point, not simplifying 
+ * (although we should simplify). will require some lookups.
+ */
+export interface Metadata {
+  cell_metadata: MetadataRecord[];
+  metadata_types: MetadataType[];
+
+  // TODO
+  // value_metadata: MetadataRecord[];
+
+  future_metadata: Record<string, FutureMetadata[]>;
+
+}
+
+export const LookupMetadata = (source: Metadata, type: 'cell'|'value', index: number): FutureMetadata => {
+
+  if (type === 'cell') {
+
+    //
+    // the docs say "zero based", but this looks to be one based -- there's
+    // only one entry when we create a doc, but the cm index in cells is "1".
+    //
+    // https://learn.microsoft.com/en-us/dotnet/api/documentformat.openxml.spreadsheet.cell?view=openxml-3.0.1&redirectedfrom=MSDN
+    //
+
+    const metadata_type = source.cell_metadata[index - 1];
+
+    if (metadata_type) {
+      const record_type = source.metadata_types[metadata_type.t - 1]; // 1-based
+      if (record_type) {
+        if (record_type.name === 'XLMDX') {
+          // ...
+        }
+        else {
+          const future_metadata_list = source.future_metadata[record_type.name];
+          if (future_metadata_list) {
+            return future_metadata_list[metadata_type.v] || {}; // 0-based
+          }
+        }
+      }
+    }
+  }
+  else {
+    console.warn('value metadata not implemented')
+  }
+
+  return {flags: {}}; // null, essentially
+
+};
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export const ParseMetadataXML = (xml: any): Metadata => {
+
+  const metadata: Metadata = {
+    metadata_types: [],
+    cell_metadata: [],
+    future_metadata: {},
+  };
+
+  const metadata_types = XMLUtils.FindAll(xml, 'metadata/metadataTypes/metadataType');
+  for (const entry of metadata_types) {
+
+    const name: string = entry.a$?.name || '';
+    const value = entry.a$?.cellMeta;
+    const cell_meta: boolean = (value === '1' || value === 'true');
+
+    metadata.metadata_types.push({ 
+      name, cell_meta 
+    });
+
+  }
+
+  const future_metadata_blocks = XMLUtils.FindAll(xml, 'metadata/futureMetadata');
+  for (const entry of future_metadata_blocks) {
+
+    const name: string = entry.a$?.name || '';
+    if (name) {
+
+      const future_metadata_list: FutureMetadata[] = [];
+
+      // `extLst` entries can be inside of `bk` entries, but aparently not
+      // required? what's the case where they are _not_ in `bk` elements?
+      
+      for (const block of XMLUtils.FindAll(entry, 'bk')) {
+
+        const future_metadata: FutureMetadata = { flags: {} };
+
+        // I guess metadata attributes are elements? we'll probably 
+        // have to look them up individually
+
+        for (const child of XMLUtils.FindAll(block, 'extLst/ext/xda:dynamicArrayProperties')) {
+          if (child?.a$.fDynamic === '1') {
+            future_metadata.flags['dynamic-array'] = true;
+          }
+        }
+
+        future_metadata_list.push(future_metadata);
+
+      }
+
+      metadata.future_metadata[name] = future_metadata_list;
+
+    }
+  }
+
+  for (const entry of XMLUtils.FindAll(xml, 'metadata/cellMetadata/bk/rc')) {
+    metadata.cell_metadata.push({
+      t: Number(entry.a$?.t || -1),
+      v: Number(entry.a$?.v || -1),
+    });
+  }
+
+  // console.info({metadata});
+
+  return metadata;
+    
+}

package/treb-export/src/workbook.ts

@@ -38,6 +38,7 @@ import type { RelationshipMap } from './relationship';
 import { ZipWrapper } from './zip-wrapper';
 import type { CellStyle, ThemeColor } from 'treb-base-types';
 import type { SerializedNamed } from 'treb-data-model';
+import { type Metadata, ParseMetadataXML } from './metadata';
 
 /**
  * @privateRemarks -- FIXME: not sure about the equal/equals thing. need to check.
@@ -53,20 +54,42 @@ export const ConditionalFormatOperators: Record<string, string> = {
   notEqual: '<>',
 };
 
+//
+// enums? really? in 2025? FIXME
+//
 export enum ChartType {
-  Unknown = 0, Column, Bar, Line, Scatter, Donut, Pie, Bubble, Box
+  Null = 0, 
+  Column, 
+  Bar, 
+  Line, 
+  Scatter, 
+  Donut, 
+  Pie, 
+  Bubble, 
+  Box, 
+  Histogram, 
+  Unknown
 }
 
 export interface ChartSeries {
   values?: string;
   categories?: string;
   bubble_size?: string;
+
+  /** special for histogram */
+  bin_count?: number;
+
   title?: string;
 }
 
+type ChartFlags =
+  'stacked'
+  ;
+
 export interface ChartDescription {
   title?: string;
   type: ChartType; 
+  flags?: ChartFlags[];
   series?: ChartSeries[];
 }
 
@@ -151,6 +174,9 @@ export class Workbook {
   /** the workbook "rels" */
   public rels: RelationshipMap = {};
 
+  /** metadata reference; new and WIP */
+  public metadata?: Metadata;
+  
   public sheets: Sheet[] = [];
 
   public active_tab = 0;
@@ -200,6 +226,13 @@ export class Workbook {
     let xml = xmlparser2.parse(data || '');
     this.shared_strings.FromXML(xml);
 
+    // new(ish) metadata
+    if (this.zip.Has('xl/metadata.xml')) {
+      data = this.zip.Get('xl/metadata.xml');
+      xml = xmlparser2.parse(data);
+      this.metadata = ParseMetadataXML(xml);
+    }
+
     // theme
     data = this.zip.Get('xl/theme/theme1.xml');
     xml = xmlparser2.parse(data);
@@ -542,7 +575,7 @@ export class Workbook {
     const xml = xmlparser2.parse(data);
 
     const result: ChartDescription = {
-      type: ChartType.Unknown
+      type: ChartType.Null
     };
 
     // console.info("RC", xml);
@@ -562,8 +595,29 @@ export class Workbook {
         }
       }
       else {
+
+        // there's a bug in FindAll -- seems to have to do with the nodes
+        // here being strings
+
+        const nodes: (string | { t$: string })[] = [];
+        const parents = XMLUtils.FindAll(title_node, 'c:tx/c:rich/a:p/a:r');
+        for (const entry of parents) {
+          if (entry['a:t']) {
+            nodes.push(entry['a:t'])
+          }
+        }
+
+        /*
+        const xx = XMLUtils.FindAll(title_node, 'c:tx/c:rich/a:p/a:r');
+        const yy = XMLUtils.FindAll(title_node, 'c:tx/c:rich/a:p/a:r/a:t');
+        console.info({xx, yy});
+
         const nodes = XMLUtils.FindAll(title_node, 'c:tx/c:rich/a:p/a:r/a:t');
-        result.title = '"' + nodes.join('') + '"';
+        */
+
+        result.title = '"' + nodes.map(node => {
+          return typeof node === 'string' ? node : (node.t$ || '');
+        }).join('') + '"';
       }
 
     }
@@ -653,6 +707,14 @@ export class Workbook {
       if (node['c:barDir']) {
         if (node['c:barDir'].a$?.val === 'col') {
           result.type = ChartType.Column;
+
+          if (node['c:grouping']?.a$?.val === 'stacked') {
+            if (!result.flags) {
+              result.flags = [];
+            }
+            result.flags.push('stacked');
+          }
+
         }
       }
       
@@ -705,9 +767,92 @@ export class Workbook {
       // box plot uses "extended chart" which is totally different... but we 
       // might need it again later? for the time being it's just inlined
 
+      // hmmm also used for histogram... histograms aren't named, they are 
+      // clustered column type with a binning element, which has some attributes
+
       const ex_series = XMLUtils.FindAll(xml, 'cx:chartSpace/cx:chart/cx:plotArea/cx:plotAreaRegion/cx:series');
       if (ex_series?.length) {
-        if (ex_series.every(test => test.a$?.layoutId === 'boxWhisker')) {
+
+        // testing seems to require looping, so let's try to merge loops
+
+        let clustered_column = true;
+        let histogram = true;
+        let box_whisker = true;
+
+        for (const series of ex_series) {
+
+          const layout = series.a$?.layoutId;
+
+          if (clustered_column && layout !== 'clusteredColumn') {
+            clustered_column = false;
+          }
+          if (box_whisker && layout !== 'boxWhisker') {
+            box_whisker = false;
+          }
+          if (clustered_column && histogram) {
+            const binning = XMLUtils.FindAll(series, `cx:layoutPr/cx:binning`);
+            if (!binning.length) {
+              histogram = false;
+            }
+          }
+
+        }
+
+        // ok that's what we know so far...
+
+        if (histogram) {
+          result.type = ChartType.Histogram;
+          result.series = [];
+
+          for (const series_entry of ex_series) {
+
+            if (series_entry.a$?.hidden === '1') {
+              continue;
+            }
+            
+            const series: ChartSeries = {};
+
+            const data = XMLUtils.FindAll(xml, 'cx:chartSpace/cx:chartData/cx:data');
+
+            // so there are multiple series, and multiple datasets,
+            // but they are all merged together? no idea how this design
+            // works
+
+            const values_list: string[] = [];
+            for (const data_series of data) {
+              values_list.push(data_series['cx:numDim']?.['cx:f'] || '');
+            }
+            series.values = values_list.join(',');
+
+            const bin_count = XMLUtils.FindAll(series_entry, `cx:layoutPr/cx:binning/cx:binCount`);
+            if (bin_count[0]) {
+              const count = Number(bin_count[0].a$?.val || 0);
+              if (count) {
+                series.bin_count = count;
+              }
+            }
+
+            result.series.push(series);
+
+          }
+
+          const title = XMLUtils.FindAll(xml, 'cx:chartSpace/cx:chart/cx:title/cx:tx/cx:txData');
+          if (title) {
+            if (title[0]?.['cx:f']) {
+              result.title = title[0]['cx:f'];
+            }
+            else if (title[0]?.['cx:v']) {
+              result.title = '"' + title[0]['cx:v'] + '"';
+            }
+          }
+
+          console.info("histogram", result);
+          return result;
+          
+        }
+
+        if (box_whisker) {
+
           result.type = ChartType.Box;
           result.series = [];
           const data = XMLUtils.FindAll(xml, 'cx:chartSpace/cx:chartData/cx:data'); // /cx:data/cx:numDim/cx:f');
@@ -758,7 +903,8 @@ export class Workbook {
     }
 
     if (!node) {
-      console.info("Chart type not handled");
+      console.info("Chart type not handled", {xml});
+      result.type = ChartType.Unknown;
     }
 
     // console.info("RX?", result);