@trebco/treb 32.4.1 → 32.6.4

package/treb-export/src/import.ts

@@ -24,17 +24,17 @@
 // import UZip from 'uzip';
 import Base64JS from 'base64-js';
 
-import type { AnchoredChartDescription, AnchoredImageDescription, AnchoredTextBoxDescription} from './workbook2';
-import { ChartType, ConditionalFormatOperators, Workbook } from './workbook2';
-import type { ParseResult } from 'treb-parser';
+import type { AnchoredChartDescription, AnchoredImageDescription, AnchoredTextBoxDescription} from './workbook';
+import { ChartType, ConditionalFormatOperators, Workbook } from './workbook';
+import type { ExpressionUnit, ParseResult, UnitCall } from 'treb-parser';
 import { DecimalMarkType, Parser } from 'treb-parser';
 import type { RangeType, AddressType, HyperlinkType } from './address-type';
 import { is_range, ShiftRange, InRange, is_address } from './address-type';
 import { type ImportedSheetData, type AnchoredAnnotation, type CellParseResult, type AnnotationLayout, type Corner as LayoutCorner, type IArea, type GradientStop, type Color, type HTMLColor, type ThemeColor, Area } from 'treb-base-types';
 import type { SerializedValueType } from 'treb-base-types';
-import type { Sheet} from './workbook-sheet2';
-import { VisibleState } from './workbook-sheet2';
-import type { CellAnchor } from './drawing2/drawing2';
+import type { Sheet} from './workbook-sheet';
+import { VisibleState } from './workbook-sheet';
+import type { CellAnchor } from './drawing/drawing';
 import { type GenericDOMElement, XMLUtils } from './xml-utils';
 
 // import { one_hundred_pixels } from './constants';
@@ -42,6 +42,7 @@ import { ColumnWidthToPixels } from './column-width';
 import type { DataValidation, AnnotationType } from 'treb-data-model';
 import { ZipWrapper } from './zip-wrapper';
 import type { ConditionalFormat } from 'treb-data-model';
+import { LookupMetadata, type MetadataFlags } from './metadata';
 
 interface SharedFormula {
   row: number;
@@ -57,6 +58,7 @@ interface CellElementType {
     r?: string;
     t?: string;
     s?: string;
+    cm?: string;
   };
   v?: string|number|{
     t$: string;
@@ -79,7 +81,26 @@ interface ConditionalFormatRule {
     priority?: string;
     operator?: string;
   };
+
   formula?: string|[number,number]|{t$: string};
+
+  dataBar?: {
+    a$?: {
+      showValue?: string;
+    },
+    color?: {
+      a$: {
+        rgb?: string;
+      }
+    }
+  }
+  
+  extLst?: {
+    ext?: {
+      'x14:id'?: string;
+    }
+  }
+
   colorScale?: {
     cfvo?: {
       a$: {
@@ -128,6 +149,7 @@ export class Importer {
     element: CellElementType,
     shared_formulae: SharedFormulaMap,
     arrays: RangeType[],
+    dynamic_arrays: RangeType[],
     merges: RangeType[],
     links: HyperlinkType[],
     // validations: Array<{ address: ICellAddress, validation: DataValidation }>,
@@ -146,6 +168,15 @@ export class Importer {
       return undefined;
     }
 
+    // new (to us) metadata
+    let metadata_flags: MetadataFlags = {};
+    if (element.a$?.cm) {
+      const cm_index = Number(element.a$?.cm);
+      if (this.workbook?.metadata) {
+        metadata_flags = LookupMetadata(this.workbook.metadata, 'cell', cm_index).flags;
+      }
+    }
+
     // console.info(element);
 
     let value: undefined | number | boolean | string;
@@ -194,6 +225,8 @@ export class Importer {
 
         if (formula) {
 
+          // console.info("F", formula);
+
           // doing it like this is sloppy (also does not work properly).
           value = '=' + formula.replace(/^_xll\./g, '');
 
@@ -202,10 +235,66 @@ export class Importer {
             value = formula;
           }
           else {
+
             const parse_result = this.parser.Parse(formula); // l10n?
             if (parse_result.expression) {
-              this.parser.Walk(parse_result.expression, (unit) => {
+
+              const TrimPrefixes = (name: string) => {
+
+                if (/^_xll\./.test(name)) {
+                  name = name.substring(5);
+                }
+                if (/^_xlfn\./.test(name)) {
+                  console.info("xlfn:", name);
+                  name = name.substring(6);
+                }
+                if (/^_xlws\./.test(name)) {
+                  console.info("xlws:", name);
+                  name = name.substring(6);
+                }
+
+                return name;
+              };
+
+              const TreeWalker = (unit: ExpressionUnit) => {
+                
                 if (unit.type === 'call') {
+
+                  // if we see _xlfn.SINGLE, translate that into 
+                  // @ + the name of the first parameter...
+                  //
+                  // this is solving the case where single appears, but it 
+                  // doesn't solve the case where it does not appear -- they 
+                  // may be using the array flag of the cell as an indicator?
+                  // but how then do they know it _should_ be an array? not 
+                  // sure, and I don't see any other indication.
+
+                  if (/^_xlfn\.single/i.test(unit.name)) {
+
+                    const first = unit.args[0];
+                    if (first.type === 'call') {
+
+                      // we could do this in place, we don't need to copy...
+                      // although it seems like a good idea. also watch out,
+                      // these SINGLEs could be nested.
+
+                      const replacement: UnitCall = JSON.parse(JSON.stringify(first));
+                      replacement.name = '@' + TrimPrefixes(replacement.name);
+
+                      for (let i = 0; i < replacement.args.length; i++) {
+                        replacement.args[i] = this.parser.Walk2(replacement.args[i], TreeWalker);
+                      }
+
+                      return replacement;
+                    }
+                    else {
+                      console.info("_xlfn.SINGLE unexpected argument", unit.args[0]);
+                    }
+                  }
+
+                  unit.name = TrimPrefixes(unit.name);
+
+                  /*
                   if (/^_xll\./.test(unit.name)) {
                     unit.name = unit.name.substring(5);
                   }
@@ -217,9 +306,15 @@ export class Importer {
                     console.info("xlws:", unit.name);
                     unit.name = unit.name.substring(6);
                   }
+                  */
+
                 }
                 return true;
-              });
+
+              };
+
+              parse_result.expression = this.parser.Walk2(parse_result.expression, TreeWalker);
+
               value = '=' + this.parser.Render(parse_result.expression, { missing: '' });
             }
           }
@@ -255,10 +350,38 @@ export class Importer {
           }
         }
 
+        //
+        // arrays and spill/dynamic arrays
+        //
+
         if (typeof element.f !== 'string' &&  element.f.a$?.t === 'array') {
           const translated = sheet.TranslateAddress(element.f.a$.ref || '');
-          if (is_range(translated)) {
-            arrays.push(ShiftRange(translated, -1, -1));
+
+          // why are we checking "is_range" here? this should be valid 
+          // even if the ref attribute is one cell, if it explicitly
+          // says t="array"
+
+          // we will need to adjust it, though? yes, because the lists 
+          // only accept ranges. note that range type has a superfluous 
+          // sheet parameter? ...
+
+          let range = translated;
+          if (!is_range(range)) {
+            range = {
+              to: { ...range },
+              from: { ...range },
+              sheet: range.sheet,
+            };
+          }
+
+          // if (is_range(translated)) 
+          {
+            if (metadata_flags['dynamic-array']) {
+              dynamic_arrays.push(ShiftRange(range, -1, -1));
+            }
+            else {
+              arrays.push(ShiftRange(range, -1, -1));
+            }
           }
         }
 
@@ -298,12 +421,14 @@ export class Importer {
     // but perhaps we should check that... although at this point we have 
     // already added the array so we need to check for root
 
-    for (const array of arrays) {
-      if (InRange(array, shifted) && (shifted.row !== array.from.row || shifted.col !== array.from.col)) {
-        calculated_type = type;
-        calculated_value = value;
-        value = undefined;
-        type = 'undefined'; // ValueType.undefined;
+    for (const set of [arrays, dynamic_arrays]) {
+      for (const array of set) {
+        if (InRange(array, shifted) && (shifted.row !== array.from.row || shifted.col !== array.from.col)) {
+          calculated_type = type;
+          calculated_value = value;
+          value = undefined;
+          type = 'undefined'; // ValueType.undefined;
+        }
       }
     }
 
@@ -350,6 +475,20 @@ export class Importer {
       }
     }
 
+    for (const range of dynamic_arrays) {
+      if (InRange(range, shifted)) {
+        result.spill = {
+          start: {
+            row: range.from.row,
+            column: range.from.col,
+          }, end: {
+            row: range.to.row,
+            column: range.to.col,
+          },
+        }
+      }
+    }
+
     for (const range of arrays) {
       if (InRange(range, shifted)) {
         result.area = {
@@ -383,7 +522,7 @@ export class Importer {
 
   }
 
-  public ParseConditionalFormat(address: RangeType|AddressType, rule: ConditionalFormatRule): ConditionalFormat|ConditionalFormat[]|undefined {
+  public ParseConditionalFormat(address: RangeType|AddressType, rule: ConditionalFormatRule, extensions?: any[]): ConditionalFormat|ConditionalFormat[]|undefined {
 
     const area = this.AddressToArea(address);
     const operators = ConditionalFormatOperators;
@@ -543,6 +682,44 @@ export class Importer {
         }
         break;
 
+      case 'dataBar':
+        {
+          const hide_values = (rule.dataBar?.a$?.showValue === '0');
+          let extension: any = undefined;
+
+          if (rule.extLst?.ext?.['x14:id']) {
+            for (const test of (extensions || [])) {
+              if (test['x14:cfRule']?.a$?.id === rule.extLst.ext['x14:id']) {
+                extension = test;
+                break;
+              }
+            }
+            if (!extension) {
+              console.info("conditional format extension not found");
+            }
+          }
+
+          if (rule.dataBar?.color?.a$?.rgb) {
+
+            let negative: Color|undefined = undefined;
+
+            if (extension?.['x14:cfRule']?.['x14:dataBar']?.['x14:negativeFillColor']?.a$?.rgb) {
+              const rgb = extension['x14:cfRule']['x14:dataBar']['x14:negativeFillColor'].a$.rgb;
+              negative = { text: '#' + rgb.toString().substring(2) };
+            }
+
+            const fill: Color = { text: '#' + rule.dataBar.color.a$.rgb.substring(2) };
+            return {
+              type: 'data-bar',
+              area,
+              fill,
+              hide_values,
+              negative,
+            };
+          }
+        }
+        break;
+
       case 'colorScale':
         if (rule.colorScale && Array.isArray(rule.colorScale.cfvo) && Array.isArray(rule.colorScale.color)) {
 
@@ -618,6 +795,7 @@ export class Importer {
     const data: CellParseResult[] = [];
     const shared_formulae: {[index: string]: SharedFormula} = {};
     const arrays: RangeType[] = [];
+    const dynamic_arrays: RangeType[] = [];
     const merges: RangeType[] = [];
     const conditional_formats: ConditionalFormat[] = [];
     const links: HyperlinkType[] = [];
@@ -665,6 +843,11 @@ export class Importer {
     // conditionals 
 
     const conditional_formatting = FindAll('worksheet/conditionalFormatting');
+
+    // we might need extensions as well? TODO
+
+    const conditional_formattings = FindAll('worksheet/extLst/ext/x14:conditionalFormattings/x14:conditionalFormatting');
+
     for (const element of conditional_formatting) {
       if (element.a$?.sqref ){
 
@@ -678,7 +861,7 @@ export class Importer {
           if (element.cfRule) {
             const rules = Array.isArray(element.cfRule) ? element.cfRule : [element.cfRule];
             for (const rule of rules) {
-              const format = this.ParseConditionalFormat(area, rule as unknown as ConditionalFormatRule);
+              const format = this.ParseConditionalFormat(area, rule as unknown as ConditionalFormatRule, conditional_formattings);
               if (format) {
                 if (Array.isArray(format)) {
                   conditional_formats.push(...format);
@@ -905,7 +1088,7 @@ export class Importer {
       const cells = row.c ? Array.isArray(row.c) ? row.c : [row.c] : [];
 
       for (const element of cells) {
-        const cell = this.ParseCell(sheet, element as unknown as CellElementType, shared_formulae, arrays, merges, links); // , validations);
+        const cell = this.ParseCell(sheet, element as unknown as CellElementType, shared_formulae, arrays, dynamic_arrays, merges, links); // , validations);
         if (cell) {
           data.push(cell);
         }
@@ -1192,12 +1375,21 @@ export class Importer {
 
             break;
 
+          case ChartType.Histogram:
+            type = 'treb-chart';
+            func = 'Histogram.Plot';
+            if (series?.length) {
+              // ...
+            }
+            args[1] = descriptor.chart.title;
+            break;
+
           case ChartType.Box:
             type = 'treb-chart';
             func = 'Box.Plot';
             if (series?.length) {
               args[0] = `Group(${series.map(s => `Series(${s.title || ''},,${s.values||''})`).join(', ')})`;
-              console.info("S?", {series}, args[0])
+              // console.info("S?", {series}, args[0])
             }
             args[1] = descriptor.chart.title;
             break;
@@ -1255,6 +1447,10 @@ export class Importer {
               args[1] = series[0]?.categories || '';
             }
 
+            if (descriptor.chart.type === ChartType.Column && descriptor.chart.flags?.includes('stacked')) {
+              args[3] = '"stacked"';
+            }
+
             break;
         }
 

package/treb-export/src/metadata.ts

@@ -0,0 +1,187 @@
+import { XMLUtils } from './xml-utils';
+
+/**
+ * https://learn.microsoft.com/en-us/dotnet/api/documentformat.openxml.spreadsheet.metadatarecord?view=openxml-3.0.1
+ * 
+ * ah, mixing 0-based and 1-based indexes. that's great. not at all confusing.
+ */
+export interface MetadataRecord  {
+
+  //
+  // A 1-based index to the metadata record type in metadataTypes.
+  //
+  t: number;
+
+  //
+  // A zero based index to a specific metadata record. If the corresponding 
+  // metadataType has name="XLMDX", then this is an index to a record in 
+  // mdxMetadata, otherwise this is an index to a record in the futureMetadata 
+  // section whose name matches the name of the metadataType.
+  //
+  v: number;
+
+}
+
+/** 
+ * https://learn.microsoft.com/en-us/dotnet/api/documentformat.openxml.spreadsheet.metadatatype?view=openxml-3.0.1
+ * 
+ * punting on all these attributes for now, except the ones we specifically 
+ * want.
+ */
+export interface MetadataType {
+
+  //
+  // Represents the name of this particular metadata type. This name shall be 
+  // unique amongst all other metadataTypes.
+  //
+  name: string;
+
+  //
+  // A Boolean flag indicating whether metadata is cell metadata. True when 
+  // the metadata is cell metadata, false otherwise - in the false case it 
+  // is considered to be value metadata.
+  //
+  cell_meta?: boolean;
+
+}
+
+/**
+ * the type `Future Feature Data Storage Area` is not actually defined? not 
+ * sure. maybe it's just a standard extension type and I need to look that up?
+ * 
+ * for the time being we'll use a list of flags...
+ */
+export interface MetadataFlags {
+  'dynamic-array'?: boolean;
+}
+
+/**
+ * https://learn.microsoft.com/en-us/dotnet/api/documentformat.openxml.spreadsheet.futuremetadata?view=openxml-3.0.1
+ * https://learn.microsoft.com/en-us/dotnet/api/documentformat.openxml.spreadsheet.futuremetadatablock?view=openxml-3.0.1
+ * 
+ * these are named, matching the names in "MetadataType"; we'll use those
+ * as indexes, and not store them in the object.
+ * 
+ */
+export interface FutureMetadata {
+  flags: MetadataFlags;
+}
+
+/**
+ * we're just reflecting the actual data at this point, not simplifying 
+ * (although we should simplify). will require some lookups.
+ */
+export interface Metadata {
+  cell_metadata: MetadataRecord[];
+  metadata_types: MetadataType[];
+
+  // TODO
+  // value_metadata: MetadataRecord[];
+
+  future_metadata: Record<string, FutureMetadata[]>;
+
+}
+
+export const LookupMetadata = (source: Metadata, type: 'cell'|'value', index: number): FutureMetadata => {
+
+  if (type === 'cell') {
+
+    //
+    // the docs say "zero based", but this looks to be one based -- there's
+    // only one entry when we create a doc, but the cm index in cells is "1".
+    //
+    // https://learn.microsoft.com/en-us/dotnet/api/documentformat.openxml.spreadsheet.cell?view=openxml-3.0.1&redirectedfrom=MSDN
+    //
+
+    const metadata_type = source.cell_metadata[index - 1];
+
+    if (metadata_type) {
+      const record_type = source.metadata_types[metadata_type.t - 1]; // 1-based
+      if (record_type) {
+        if (record_type.name === 'XLMDX') {
+          // ...
+        }
+        else {
+          const future_metadata_list = source.future_metadata[record_type.name];
+          if (future_metadata_list) {
+            return future_metadata_list[metadata_type.v] || {}; // 0-based
+          }
+        }
+      }
+    }
+  }
+  else {
+    console.warn('value metadata not implemented')
+  }
+
+  return {flags: {}}; // null, essentially
+
+};
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export const ParseMetadataXML = (xml: any): Metadata => {
+
+  const metadata: Metadata = {
+    metadata_types: [],
+    cell_metadata: [],
+    future_metadata: {},
+  };
+
+  const metadata_types = XMLUtils.FindAll(xml, 'metadata/metadataTypes/metadataType');
+  for (const entry of metadata_types) {
+
+    const name: string = entry.a$?.name || '';
+    const value = entry.a$?.cellMeta;
+    const cell_meta: boolean = (value === '1' || value === 'true');
+
+    metadata.metadata_types.push({ 
+      name, cell_meta 
+    });
+
+  }
+
+  const future_metadata_blocks = XMLUtils.FindAll(xml, 'metadata/futureMetadata');
+  for (const entry of future_metadata_blocks) {
+
+    const name: string = entry.a$?.name || '';
+    if (name) {
+
+      const future_metadata_list: FutureMetadata[] = [];
+
+      // `extLst` entries can be inside of `bk` entries, but aparently not
+      // required? what's the case where they are _not_ in `bk` elements?
+      
+      for (const block of XMLUtils.FindAll(entry, 'bk')) {
+
+        const future_metadata: FutureMetadata = { flags: {} };
+
+        // I guess metadata attributes are elements? we'll probably 
+        // have to look them up individually
+
+        for (const child of XMLUtils.FindAll(block, 'extLst/ext/xda:dynamicArrayProperties')) {
+          if (child?.a$.fDynamic === '1') {
+            future_metadata.flags['dynamic-array'] = true;
+          }
+        }
+
+        future_metadata_list.push(future_metadata);
+
+      }
+
+      metadata.future_metadata[name] = future_metadata_list;
+
+    }
+  }
+
+  for (const entry of XMLUtils.FindAll(xml, 'metadata/cellMetadata/bk/rc')) {
+    metadata.cell_metadata.push({
+      t: Number(entry.a$?.t || -1),
+      v: Number(entry.a$?.v || -1),
+    });
+  }
+
+  // console.info({metadata});
+
+  return metadata;
+    
+}

package/treb-export/src/{workbook-sheet2.ts → workbook-sheet.ts}

@@ -43,8 +43,8 @@
 
 import type { AddressType, RangeType} from './address-type';
 import { is_range } from './address-type';
-import type { SharedStrings } from './shared-strings2';
-import type { Drawing } from './drawing2/drawing2';
+import type { SharedStrings } from './shared-strings';
+import type { Drawing } from './drawing/drawing';
 import type { RelationshipMap } from './relationship';
 
 export interface SheetOptions {

package/treb-export/src/{workbook-style2.ts → workbook-style.ts}

@@ -23,7 +23,7 @@
 // import { Element, ElementTree as Tree } from 'elementtree';
 
 import { type CompositeBorderEdge, Style, type CellStyle, type PropertyKeys, type Color, IsHTMLColor, IsThemeColor, type ThemeColor, type HTMLColor, ThemeColorIndex } from 'treb-base-types';
-import { Theme } from './workbook-theme2';
+import { Theme } from './workbook-theme';
 import { NumberFormatCache } from 'treb-format';
 import { XMLUtils } from './xml-utils';
 

package/treb-export/src/{workbook-theme2.ts → workbook-theme.ts}

@@ -19,8 +19,6 @@
  * 
  */
 
-import { XMLUtils } from './xml-utils';
-
 export interface ColorSchemeElement {
   name?: string;
   value?: string;