mapshaper 0.7.8 → 0.7.9

package/www/mapshaper-gui.js

@@ -2099,7 +2099,7 @@
   }
 
   async function loadGeoParquetLib() {
-    if (!window.modules || !window.modules.hyparquet || !window.modules['hyparquet-writer']) {
+    if (!window.modules || !window.modules.hyparquet || !window.modules['hyparquet-compressors'] || !window.modules['hyparquet-writer'] || !window.modules['zstd-codec']) {
       if (!geoParquetPromise) {
         geoParquetPromise = loadScript('geoparquet.js');
       }
@@ -4113,6 +4113,116 @@
     }
   };
 
+  var FIELD_TYPE_SCAN_LIMIT = 1000;
+  var RECENT_COMMAND_LIMIT = 20;
+  var RECENT_MESSAGE_LIMIT = 10;
+
+  function getRuntimeStateContext(gui) {
+    var layers = gui.model.getLayers();
+    var active = gui.model.getActiveLayer();
+    var history = gui.session.getHistorySnapshot();
+    var messages = gui.getMessages ? gui.getMessages() : [];
+
+    return {
+      schema: 'mapshaper-runtime-context',
+      schema_version: 1,
+      generated_at: new Date().toISOString(),
+      privacy_note: 'Contains project metadata only: layer names, field names/types, counts, CRS, recent commands and recent messages. It does not include geometry or attribute records.',
+      ui: getUiContext(gui),
+      active_layer: active ? getLayerTargetId(gui, active.layer) : null,
+      default_targets: getDefaultTargets(gui),
+      layers: layers.map(function(o, i) {
+        return getLayerContext(gui, o.layer, o.dataset, i, active && active.layer);
+      }),
+      session: {
+        command_count: history.commands.length,
+        recent_commands: history.commands.slice(-RECENT_COMMAND_LIMIT),
+        unsaved_changes: gui.session.unsavedChanges()
+      },
+      messages: messages.slice(0, RECENT_MESSAGE_LIMIT)
+    };
+  }
+
+  function stringifyRuntimeStateContext(gui) {
+    return JSON.stringify(getRuntimeStateContext(gui), null, 2);
+  }
+
+  function getUiContext(gui) {
+    return {
+      interface: 'web_app',
+      console_open: gui.consoleIsOpen(),
+      panel_mode: gui.getMode(),
+      interaction_mode: gui.interaction ? gui.interaction.getMode() : null,
+      dataset_count: gui.model.getDatasets().length,
+      layer_count: gui.model.getLayers().length,
+      has_data: !gui.model.isEmpty()
+    };
+  }
+
+  function getDefaultTargets(gui) {
+    return gui.model.getDefaultTargets().map(function(target) {
+      return {
+        layers: target.layers.map(function(lyr) {
+          return getLayerTargetId(gui, lyr);
+        })
+      };
+    });
+  }
+
+  function getLayerContext(gui, lyr, dataset, i, activeLyr) {
+    var fields = getFieldContexts(lyr);
+    var context = {
+      id: i + 1,
+      target_id: getLayerTargetId(gui, lyr),
+      name: lyr.name || null,
+      active: lyr == activeLyr,
+      geometry_type: lyr.geometry_type || null,
+      feature_count: internal.getFeatureCount(lyr),
+      field_count: fields.length,
+      fields: fields
+    };
+    if (lyr.geometry_type) {
+      context.crs = getCrsContext(dataset);
+    }
+    if (dataset.info && dataset.info.input_formats) {
+      context.input_formats = dataset.info.input_formats.slice();
+    }
+    return context;
+  }
+
+  function getCrsContext(dataset) {
+    var crs = null;
+    try {
+      crs = internal.getProjInfo(dataset);
+    } catch(e) {}
+    return crs || null;
+  }
+
+  function getFieldContexts(lyr) {
+    if (!lyr.data || lyr.data.size() === 0) return [];
+    return lyr.data.getFields().map(function(name) {
+      return {
+        name: name,
+        type: getSampledFieldType(lyr.data, name)
+      };
+    });
+  }
+
+  function getSampledFieldType(table, name) {
+    var n = Math.min(table.size(), FIELD_TYPE_SCAN_LIMIT);
+    var rec, type;
+    for (var i = 0; i < n; i++) {
+      rec = table.getReadOnlyRecordAt(i);
+      type = rec ? internal.getValueType(rec[name]) : null;
+      if (type) return type;
+    }
+    return null;
+  }
+
+  function getLayerTargetId(gui, lyr) {
+    return internal.formatOptionValue(internal.getLayerTargetId(gui.model, lyr));
+  }
+
   function Console(gui) {
     var model = gui.model;
     var CURSOR = '$ ';
@@ -4476,6 +4586,10 @@
           clear();
         } else if (cmd == 'tips') {
           printExamples();
+        } else if (cmd == 'context') {
+          toLog(stringifyRuntimeStateContext(gui));
+        } else if (cmd == 'context download') {
+          saveRuntimeContext();
         } else if (cmd == 'history') {
           toLog(gui.session.toCommandLineString());
         } else if (cmd == 'layers') {
@@ -4628,12 +4742,19 @@
       printExample("See a list of all console commands", "$ help");
       printExample("Get help using a single command", "$ help innerlines");
       printExample("Get information about imported datasets", "$ info");
+      printExample("Print bot/debug runtime context as JSON", "$ context");
       printExample("Display browser session as shell commands", "$ history");
       printExample("Delete one state from a national dataset","$ filter 'STATE != \"Alaska\"'");
       printExample("Aggregate counties to states by dissolving shared edges" ,"$ dissolve 'STATE'");
       printExample("Clear the console", "$ clear");
     }
 
+    function saveRuntimeContext() {
+      var json = stringifyRuntimeStateContext(gui);
+      saveBlobToLocalFile2('mapshaper-runtime-context.json', new Blob([json], {type: 'application/json'}));
+      toLog('Saved runtime context to mapshaper-runtime-context.json', 'console-message');
+    }
+
   }
 
   // Test if map should be re-framed to show updated layer
@@ -15159,6 +15280,18 @@ GUI and setting the size and crop of SVG output.</p><div><input type="text" clas
       }
     };
 
+    gui.getMessages = function() {
+      return entries.map(function(entry) {
+        return {
+          severity: entry.severity,
+          title: entry.title,
+          body: entry.body,
+          count: entry.count,
+          time: entry.time.toISOString()
+        };
+      });
+    };
+
     function severityRank(s) {
       return s === 'error' ? 2 : s === 'warn' ? 1 : 0;
     }
@@ -15327,6 +15460,14 @@ GUI and setting the size and crop of SVG output.</p><div><input type="text" clas
       return gui.container.hasClass('console-open');
     };
 
+    gui.getRuntimeStateContext = function() {
+      return getRuntimeStateContext(gui);
+    };
+
+    gui.stringifyRuntimeStateContext = function() {
+      return stringifyRuntimeStateContext(gui);
+    };
+
     // Make this instance interactive and editable
     gui.focus = function() {
       var curr = GUI.__active;
@@ -15440,6 +15581,8 @@ GUI and setting the size and crop of SVG output.</p><div><input type="text" clas
     new LayerControl(gui);
     HeaderMenu();
     gui.console = new Console(gui);
+    window.mapshaper.getRuntimeStateContext = gui.getRuntimeStateContext;
+    window.mapshaper.stringifyRuntimeStateContext = gui.stringifyRuntimeStateContext;
 
     startEditing = function() {};
 

package/www/mapshaper.js

@@ -25683,52 +25683,71 @@ ${svg}
   }
 
   var writerPromise = null;
+  var zstdPromise = null;
   var dynamicImportModule$1 = Function('id', 'return import(id)');
 
   async function exportGeoParquet(dataset, opts, filenameOverride) {
     var writer = await loadGeoParquetWriter();
+    var compression = await getGeoParquetCompression(opts);
     var extension = opts.extension || 'parquet';
+    var files = [];
     if (opts.file) {
       extension = getFileExtension(opts.file) || extension;
     }
-    return dataset.layers.map(function(lyr) {
-      if (!lyr.geometry_type) {
-        stop$1('GeoParquet export requires a geometry layer');
-      }
+    dataset.layers.forEach(function(lyr) {
       var features = exportLayerAsGeoJSON(lyr, dataset, opts, true, null);
-      var output = buildGeoParquetColumns(features);
-      var geoMetadata = buildGeoMetadata(features, dataset);
-      var content = writer.parquetWriteBuffer({
+      var hasGeometry = features.some(function(feat) {
+        return !!feat.geometry;
+      });
+      var output = buildGeoParquetColumns(features, hasGeometry);
+      var writeOptions = {
         columnData: output.columnData,
-        kvMetadata: [{
+        codec: compression.codec,
+        compressors: compression.compressors,
+        pageSize: compression.pageSize
+      };
+      if (hasGeometry) {
+        writeOptions.kvMetadata = [{
           key: 'geo',
-          value: JSON.stringify(geoMetadata)
-        }]
-      });
-      return {
+          value: JSON.stringify(buildGeoMetadata(features, dataset))
+        }];
+      } else {
+        warn('GeoParquet export: layer has no geometry; writing attribute data only.');
+      }
+      var content = writer.parquetWriteBuffer(writeOptions);
+      files.push({
         filename: filenameOverride || (lyr.name + '.' + extension),
         content: content
-      };
+      });
     });
+    return files;
   }
 
-  function buildGeoParquetColumns(features, writer) {
+  function buildGeoParquetColumns(features, includeGeometry) {
     var geometryName = 'geometry';
     var names = getPropertyNames(features);
     var columnData = [];
-    columnData.push({
-      name: geometryName,
-      data: features.map(function(feat) {
-        return feat.geometry || null;
-      }),
-      type: 'GEOMETRY'
-    });
+    if (features.length === 0) {
+      stop$1('GeoParquet export requires at least one record');
+    }
+    if (includeGeometry) {
+      columnData.push({
+        name: geometryName,
+        data: features.map(function(feat) {
+          return feat.geometry || null;
+        }),
+        type: 'GEOMETRY'
+      });
+    }
     names.forEach(function(name) {
       var values = features.map(function(feat) {
         return feat.properties ? feat.properties[name] : null;
       });
       columnData.push(buildAttributeColumn(name, values));
     });
+    if (columnData.length === 0) {
+      stop$1('GeoParquet export requires geometry or attribute data');
+    }
     return {columnData: columnData, geometryColumn: geometryName};
   }
 
@@ -25886,6 +25905,99 @@ ${svg}
     return nodeMod.default && !nodeMod.parquetWriteBuffer ? nodeMod.default : nodeMod;
   }
 
+  async function getGeoParquetCompression(opts) {
+    var codec = normalizeGeoParquetCompression(opts.compression);
+    var level = validateGeoParquetCompressionLevel(opts.level, codec);
+    if (codec != 'ZSTD') {
+      return {codec: codec, compressors: null};
+    }
+    var zstd = await loadZstdLib();
+    if (!zstd || typeof zstd.compress != 'function') {
+      stop$1('GeoParquet ZSTD compressor is not loaded');
+    }
+    return {
+      codec: codec,
+      pageSize: getGeoParquetPageSize(level),
+      compressors: {
+        ZSTD: function(bytes) {
+          return compressZstdPage(zstd, bytes, level);
+        }
+      }
+    };
+  }
+
+  function compressZstdPage(zstd, bytes, level) {
+    var compressed;
+    try {
+      compressed = zstd.compress(bytes, level);
+    } catch (e) {
+      stop$1('Unable to apply GeoParquet ZSTD compression. Try a lower level= value.');
+    }
+    if (!compressed) {
+      stop$1('Unable to apply GeoParquet ZSTD compression. Try a lower level= value.');
+    }
+    return compressed;
+  }
+
+  function getGeoParquetPageSize(level) {
+    return level >= 10 ? 64 * 1024 : undefined;
+  }
+
+  function normalizeGeoParquetCompression(compression) {
+    var str = compression === undefined || compression === null ? 'snappy' : String(compression).toLowerCase();
+    if (str == 'snappy') return 'SNAPPY';
+    if (str == 'zstd') return 'ZSTD';
+    if (str == 'none' || str == 'uncompressed') return null;
+    stop$1('Unsupported GeoParquet compression:', compression);
+  }
+
+  function validateGeoParquetCompressionLevel(level, codec) {
+    if (level === undefined) return undefined;
+    if (codec != 'ZSTD') {
+      stop$1('The level= option only applies with compression=zstd');
+    }
+    if (level >= 1 && level <= 22 && Math.floor(level) === level) {
+      return level;
+    }
+    stop$1('GeoParquet ZSTD level= option must be an integer from 1 to 22');
+  }
+
+  async function loadZstdLib() {
+    var mod;
+    if (runningInBrowser()) {
+      mod = require$1('zstd-codec');
+    } else {
+      if (!zstdPromise) {
+        zstdPromise = dynamicImportModule$1('zstd-codec');
+      }
+      mod = await zstdPromise;
+    }
+    if (mod && mod.default && !mod.ZstdCodec) {
+      mod = mod.default;
+    }
+    if (!mod || !mod.ZstdCodec || typeof mod.ZstdCodec.run != 'function') {
+      stop$1('GeoParquet ZSTD compressor is not loaded');
+    }
+    return initZstdCodec(mod.ZstdCodec);
+  }
+
+  function initZstdCodec(codec) {
+    return new Promise(function(resolve, reject) {
+      try {
+        codec.run(function(zstd) {
+          var simple = new zstd.Simple();
+          resolve({
+            compress: function(bytes, level) {
+              return simple.compress(bytes, level);
+            }
+          });
+        });
+      } catch (e) {
+        reject(e);
+      }
+    });
+  }
+
   function getOutputFormat(dataset, opts) {
     var outFile = opts.file || null,
         inFmt = dataset.info && dataset.info.input_formats && dataset.info.input_formats[0],
@@ -25991,6 +26103,9 @@ ${svg}
   async function exportDatasets(datasets, opts) {
     var format = getOutputFormat(datasets[0], opts);
     var files;
+    if (format != 'geoparquet' && (opts.compression || opts.level !== undefined)) {
+      error('The compression= and level= options only apply to GeoParquet output');
+    }
     if (format == PACKAGE_EXT) {
       opts = utils.defaults({compact: true}, opts);
       return exportPackedDatasets(datasets, opts);
@@ -28669,6 +28784,13 @@ ${svg}
         type: 'flag',
         describe: '[CSV] export numbers with decimal commas not points'
       })
+      .option('compression', {
+        describe: '[GeoParquet] options: snappy,zstd,none (default is snappy)'
+      })
+      .option('level', {
+        describe: '[GeoParquet] zstd compression level',
+        type: 'integer'
+      })
       .option('show-all', {
         type: 'flag',
         describe: '[Snapshot] show all layers in Web UI'
@@ -33675,6 +33797,7 @@ ${svg}
   }
 
   var hyparquetPromise = null;
+  var compressorsPromise = null;
   var dynamicImportModule = Function('id', 'return import(id)');
   var mproj = null;
 
@@ -33685,8 +33808,10 @@ ${svg}
     var file = await getHyparquetFile(source, hyparquet);
     var metadata = await hyparquet.parquetMetadataAsync(file);
     var geo = parseGeoParquetMetadata(metadata);
+    var compressors = await loadHyparquetCompressors();
     var rows = await hyparquet.parquetReadObjects({
       file: file,
+      compressors: compressors,
       rowFormat: 'object'
     });
     var geometryColumn = getGeoParquetGeometryColumn(rows, geo);
@@ -33726,6 +33851,26 @@ ${svg}
     return nodeMod.default && !nodeMod.parquetReadObjects ? nodeMod.default : nodeMod;
   }
 
+  async function loadHyparquetCompressors() {
+    if (runningInBrowser()) {
+      return getHyparquetCompressors(require$1('hyparquet-compressors'));
+    }
+    if (!compressorsPromise) {
+      compressorsPromise = dynamicImportModule('hyparquet-compressors');
+    }
+    return getHyparquetCompressors(await compressorsPromise);
+  }
+
+  function getHyparquetCompressors(mod) {
+    if (mod && mod.default && !mod.compressors) {
+      mod = mod.default;
+    }
+    if (!mod || !mod.compressors) {
+      stop$1('GeoParquet compression library is not loaded');
+    }
+    return mod.compressors;
+  }
+
   function getGeoParquetSource(input) {
     if (!input) return null;
     if (Object.prototype.hasOwnProperty.call(input, 'content')) {
@@ -54267,7 +54412,7 @@ ${svg}
     });
   }
 
-  var version = "0.7.8";
+  var version = "0.7.9";
 
   // Parse command line args into commands and run them
   // Function takes an optional Node-style callback. A Promise is returned if no callback is given.