mapshaper 0.5.65 → 0.5.69

package/CHANGELOG.md

@@ -1,3 +1,17 @@
+v0.5.69
+* Added undocumented -ignore command.
+* Bug fix
+
+v0.5.68
+* Bug fix
+
+v0.5.67
+* Bug fix
+
+v0.5.66
+* Improve interactive selection.
+* Added undocumented -define command.
+
 v0.5.65
 * Web UI zoom buttons respond to variable-length clicks.
 * Bug fixes

package/mapshaper.js

@@ -1,6 +1,6 @@
 (function () {
 
-  var VERSION = "0.5.64";
+  var VERSION = "0.5.68";
 
 
   var utils = /*#__PURE__*/Object.freeze({
@@ -1167,6 +1167,10 @@
     throw new UserError(formatLogArgs(arguments));
   };
 
+  var _interrupt = function() {
+    throw new NonFatalError(formatLogArgs(arguments));
+  };
+
   var _message = function() {
     logArgs(arguments);
   };
@@ -1185,10 +1189,14 @@
   }
 
   // Handle an error caused by invalid input or misuse of API
-  function stop () {
+  function stop() {
     _stop.apply(null, utils.toArray(arguments));
   }
 
+  function interrupt() {
+    _interrupt.apply(null, utils.toArray(arguments));
+  }
+
   // Print a status message
   function message() {
     _message.apply(null, messageArgs(arguments));
@@ -1228,7 +1236,9 @@
     if (utils.isString(err)) {
       err = new UserError(err);
     }
-    if (err.name == 'UserError') {
+    if (err.name == 'NonFatalError') {
+      console.error(messageArgs([err.message]).join(' '));
+    } else if (err.name == 'UserError') {
       msg = err.message;
       if (!/Error/.test(msg)) {
         msg = "Error: " + msg;
@@ -1248,6 +1258,12 @@
     return err;
   }
 
+  function NonFatalError(msg) {
+    var err = new Error(msg);
+    err.name = 'NonFatalError';
+    return err;
+  }
+
   function formatColumns(arr, alignments) {
     var widths = arr.reduce(function(memo, line) {
       return line.map(function(str, i) {
@@ -1308,6 +1324,7 @@
     loggingEnabled: loggingEnabled,
     error: error,
     stop: stop,
+    interrupt: interrupt,
     message: message,
     setLoggingFunctions: setLoggingFunctions,
     print: print,
@@ -1315,6 +1332,7 @@
     debug: debug,
     printError: printError,
     UserError: UserError,
+    NonFatalError: NonFatalError,
     formatColumns: formatColumns,
     formatStringsAsGrid: formatStringsAsGrid,
     formatLogArgs: formatLogArgs,
@@ -9035,7 +9053,8 @@
       var i = s.indexOf('.');
       return hasDot ? i > -1 : i == -1;
     };
-    return utils.uniq(matches.filter(f));
+    var vars = utils.uniq(matches.filter(f));
+    return vars;
   }
 
   // Return array of objects with properties assigned via dot notation
@@ -9100,23 +9119,24 @@
   }
 
   function getExpressionContext(lyr, mixins, opts) {
+    var defs = getStateVar('defs');
     var env = getBaseContext();
     var ctx = {};
     var fields = lyr.data ? lyr.data.getFields() : [];
     opts = opts || {};
     addUtils(env); // mix in round(), sprintf(), etc.
-    if (lyr.data) {
+    if (fields.length > 0) {
       // default to null values, so assignments to missing data properties
       // are applied to the data record, not the global object
       nullifyUnsetProperties(fields, env);
     }
     // Add global 'defs' to the expression context
-    mixins = utils.defaults(mixins || {}, getStateVar('defs'));
+    mixins = utils.defaults(mixins || {}, defs);
+    // also add defs as 'global' object
+    env.global = defs;
     Object.keys(mixins).forEach(function(key) {
       // Catch name collisions between data fields and user-defined functions
       var d = Object.getOwnPropertyDescriptor(mixins, key);
-      if (key in env) {
-      }
       if (d.get) {
         // copy accessor function from mixins to context
         Object.defineProperty(ctx, key, {get: d.get}); // copy getter function to context
@@ -11465,13 +11485,22 @@
   var T = {
     stack: [],
     start: function() {
-      T.stack.push(+new Date());
+      T.stack.push(Date.now());
     },
     stop: function() {
-      return (+new Date() - T.stack.pop()) + 'ms';
+      return (Date.now() - T.stack.pop()) + 'ms';
     }
   };
 
+  function tick(msg) {
+    var now = Date.now();
+    var elapsed = tickTime ? ' - ' + (now - tickTime) + 'ms' : '';
+    tickTime = now;
+    console.log((msg || '') + elapsed);
+  }
+
+  var tickTime = 0;
+
   // Create a mosaic layer from a dataset (useful for debugging commands like -clean
   //    that create a mosaic as an intermediate data structure)
   // Create additional layers if the "debug" flag is present
@@ -14295,7 +14324,7 @@
       joinStr = obj.tag == 'text' || obj.tag == 'tspan' ? '' : '\n';
       svg += '>' + joinStr;
       if (obj.value) {
-        svg += obj.value;
+        svg += stringEscape(obj.value);
       }
       if (obj.children) {
         svg += obj.children.map(stringify).join(joinStr);
@@ -14318,8 +14347,13 @@
         "'": '''
       };
   function stringEscape(s) {
-    return String(s).replace(rxp, function(s) {
-      return map[s];
+    return String(s).replace(rxp, function(match, i) {
+      var entity = map[match];
+      // don't replace & with &
+      if (match == '&' && s.substr(i, entity.length) == entity) {
+        return '&';
+      }
+      return entity;
     });
   }
 
@@ -17129,7 +17163,7 @@ ${svg}
       c = buf[i];
       if (c == DQUOTE) {
         inQuotedText = !inQuotedText;
-      } else if (c == CR || c == LF) {
+      } else if ((c == CR || c == LF) && !inQuotedText) {
         if (c == CR && i + 1 < bufLen && buf[i + 1] == LF) {
           // first half of CRLF pair: advance one byte
           i++;
@@ -17223,6 +17257,7 @@ ${svg}
     getRowConverter: getRowConverter,
     parseDelimHeaderSection: parseDelimHeaderSection,
     getDelimFieldFilter: getDelimFieldFilter,
+    readLinesAsString: readLinesAsString,
     parseDelimText: parseDelimText
   });
 
@@ -18404,6 +18439,9 @@ ${svg}
         type: 'strings',
         describe: '[CSV] comma-sep. list of fields to import'
       })
+      // .option('csv-comment', {
+      //   describe: '[CSV] comment line character(s)'
+      // })
       .option('decimal-comma', {
         type: 'flag',
         describe: '[CSV] import numbers formatted like 1.000,01 or 1 000,01'
@@ -19153,6 +19191,14 @@ ${svg}
       .option('target', targetOpt)
       .option('no-replace', noReplaceOpt);
 
+    parser.command('ignore')
+      // .describe('stop processing if a condition is met')
+      .option('empty', {
+        describe: 'ignore empty files',
+        type: 'flag'
+      })
+      .option('target', targetOpt);
+
     parser.command('inlay')
       .describe('inscribe a polygon layer inside another polygon layer')
       .option('source', {
@@ -20419,7 +20465,7 @@ ${svg}
     function findStringEncoding() {
       var ldid = header.ldid,
           codepage = lookupCodePage(ldid),
-          samples = getNonAsciiSamples(50),
+          samples = getNonAsciiSamples(),
           only7bit = samples.length === 0,
           encoding, msg;
 
@@ -20464,21 +20510,25 @@ ${svg}
       return arr;
     }
 
-    // Return up to @size buffers containing text samples
+    // Return an array of buffers containing text samples
     // with at least one byte outside the 7-bit ascii range.
-    function getNonAsciiSamples(size) {
+    function getNonAsciiSamples() {
       var samples = [];
       var stringFields = header.fields.filter(function(f) {
         return f.type == 'C';
       });
+      var cols = stringFields.length;
+      // don't scan all the rows in large files (slow)
+      var rows = Math.min(header.recordCount, 10000);
+      var maxSamples = 50;
       var buf = utils.createBuffer(256);
       var index = {};
       var f, chars, sample, hash;
       // include non-ascii field names, if any
       samples = getNonAsciiHeaders();
-      for (var r=0, rows=header.recordCount; r<rows; r++) {
-        for (var c=0, cols=stringFields.length; c<cols; c++) {
-          if (samples.length >= size) break;
+      for (var r=0; r<rows; r++) {
+        for (var c=0; c<cols; c++) {
+          if (samples.length >= maxSamples) break;
           f = stringFields[c];
           bin.position(getRowOffset(r) + f.columnOffset);
           chars = readStringBytes(bin, f.size, buf);
@@ -29661,7 +29711,7 @@ ${svg}
       stop('Missing an assignment expression');
     }
     var defs = getStateVar('defs');
-    var compiled = compileFeatureExpression(opts.expression, {}, null, {});
+    var compiled = compileFeatureExpression(opts.expression, {}, null, {no_warn: true});
     var result = compiled(null, defs);
   };
 
@@ -33263,6 +33313,12 @@ ${svg}
     };
   }
 
+  cmd.ignore = function(targetLayer, dataset, opts) {
+    if (opts.empty && layerIsEmpty(targetLayer)) {
+      interrupt('Layer is empty, stopping processing');
+    }
+  };
+
   cmd.include = function(opts) {
     var content, obj, context;
     // TODO: handle web context
@@ -37688,6 +37744,9 @@ ${svg}
           outputLayers = targetDataset.layers; // kludge to allow layer naming below
         }
 
+      } else if (name == 'ignore') {
+        applyCommandToEachLayer(cmd.ignore, targetLayers, targetDataset, opts);
+
       } else if (name == 'include') {
         cmd.include(opts);
 
@@ -38058,6 +38117,7 @@ ${svg}
         cmd.options.output = outputArr;
       }
     });
+
     runParsedCommands(commands, null, callback);
   }
 
@@ -38118,10 +38178,6 @@ ${svg}
       error("Changed in v0.4: runParsedCommands() takes a Catalog object");
     }
 
-    if (!utils.isFunction(done)) {
-      error("Missing a callback function");
-    }
-
     if (!utils.isArray(commands)) {
       error("Expected an array of parsed commands");
     }
@@ -38142,18 +38198,25 @@ ${svg}
       // that it can modify the output dataset in-place instead of making a copy.
       commands[commands.length-1].options.final = true;
     }
-    commands = divideImportCommand(commands);
-    utils.reduceAsync(commands, catalog, nextCommand, done);
 
-    function nextCommand(catalog, cmd, next) {
-      setStateVar('current_command', cmd.name); // for log msgs
-      setStateVar('verbose', !!cmd.options.verbose);
-      setStateVar('debug', !!cmd.options.debug);
-      runCommand(cmd, catalog, next);
+    var groups = divideImportCommand(commands);
+    if (groups.length == 1) {
+      // run a simple sequence of commands (input files are not batched)
+      return runParsedCommands2(commands, catalog, done);
+    }
+
+    // run duplicated commands (i.e. batch mode)
+    utils.reduceAsync(groups, catalog, nextGroup, done);
+
+    function nextGroup(catalog, commands, next) {
+      runParsedCommands2(commands, catalog, function(err, catalog) {
+        err = filterError(err);
+        next(err, catalog);
+      });
     }
 
     function done(err, catalog) {
-      if (err) printError(err);
+      err = filterError(err);
       cb(err, catalog);
       setStateVar('current_command', null);
       setStateVar('verbose', false);
@@ -38161,11 +38224,31 @@ ${svg}
     }
   }
 
+  function filterError(err) {
+    if (err) printError(err);
+    if (err && err.name == 'NonFatalError') {
+      return null;
+    }
+    return err;
+  }
+
+  function runParsedCommands2(commands, catalog, cb) {
+    utils.reduceAsync(commands, catalog, nextCommand, cb);
+
+    function nextCommand(catalog, cmd, next) {
+      setStateVar('current_command', cmd.name); // for log msgs
+      setStateVar('verbose', !!cmd.options.verbose);
+      setStateVar('debug', !!cmd.options.debug);
+      runCommand(cmd, catalog, next);
+    }
+  }
+
+
   // If an initial import command indicates that several input files should be
   //   processed separately, then duplicate the sequence of commands to run
   //   once for each input file
   // @commands Array of parsed commands
-  // Returns: either original command array or array of duplicated commands.
+  // Returns: Array of one or more sequences of parsed commands
   //
   function divideImportCommand(commands) {
     var firstCmd = commands[0],
@@ -38173,23 +38256,23 @@ ${svg}
 
     if (firstCmd.name != 'i' || opts.stdin || opts.merge_files ||
       opts.combine_files || !opts.files || opts.files.length < 2) {
-      return commands;
+      return [commands];
     }
 
-    return (opts.files).reduce(function(memo, file) {
-      var importCmd = {
+    return opts.files.map(function(file) {
+      var group = [{
         name: 'i',
         options: utils.defaults({
           files:[file],
           replace: true  // kludge to replace data catalog
         }, opts)
-      };
-      memo.push(importCmd);
-      memo.push.apply(memo, commands.slice(1));
-      return memo;
-    }, []);
+      }];
+      group.push.apply(group, commands.slice(1));
+      return group;
+    });
   }
 
+
   function printStartupMessages() {
     // print heap memory message if running with a custom amount
     var rxp = /^--max-old-space-size=([0-9]+)$/;

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "mapshaper",
-	"version": "0.5.65",
+	"version": "0.5.69",
 	"description": "A tool for editing vector datasets for mapping and GIS.",
 	"keywords": [
 		"shapefile",
@@ -42,7 +42,8 @@
 	"dependencies": {
 		"commander": "^5.1.0",
 		"cookies": "^0.8.0",
-		"d3-scale-chromatic": "^2.0.0",
+		"d3-color": "2.0.0",
+		"d3-scale-chromatic": "2.0.0",
 		"delaunator": "^5.0.0",
 		"flatbush": "^3.2.1",
 		"geokdbush": "^1.1.0",

package/www/elements.css

@@ -144,7 +144,7 @@ div.tip {
 
 .dialog-btn {
 	display: inline-block;
-	margin-bottom: 4px;
+	margin-bottom: 1px;
 	margin-top: 1px;
 	font-size: 13px;
 	color: white;