mapshaper 0.5.96 → 0.5.99

package/CHANGELOG.md

@@ -1,3 +1,12 @@
+v0.5.99
+* Added support for viewing data against a Mapbox basemap (enabled on the mapshaper.org site).
+
+v0.5.98
+* Better handling of null data by the -classify command.
+
+v0.5.97
+* Better warnings and error messages.
+
 v0.5.96
 * Added colors=random option to the -classify command.
 * Added more sensible default behavior to the -classify command.

package/mapshaper.js

@@ -1,6 +1,6 @@
 (function () {
 
-  var VERSION = "0.5.95";
+  var VERSION = "0.5.96";
 
 
   var utils = /*#__PURE__*/Object.freeze({
@@ -1771,6 +1771,8 @@
     // avoid re-allocating memory
     var xx2 = new Float64Array(data.xx.buffer, 0, n-1);
     var yy2 = new Float64Array(data.yy.buffer, 0, n-1);
+    var zz2 = arcs.isFlat() ? null : new Float64Array(data.zz.buffer, 0, n-1);
+    var z = arcs.getRetainedInterval();
     var count = 0;
     var found = false;
     for (var j=0; j<nn.length; j++) {
@@ -1784,24 +1786,31 @@
     utils.copyElements(data.yy, 0, yy2, 0, i);
     utils.copyElements(data.xx, i+1, xx2, i, n-i-1);
     utils.copyElements(data.yy, i+1, yy2, i, n-i-1);
-    arcs.updateVertexData(nn, xx2, yy2, null);
+    if (zz2) {
+      utils.copyElements(data.zz, 0, zz2, 0, i);
+      utils.copyElements(data.zz, i+1, zz2, i, n-i-1);
+    }
+    arcs.updateVertexData(nn, xx2, yy2, zz2);
+    arcs.setRetainedInterval(z);
   }
 
   function insertVertex(arcs, i, p) {
-    // TODO: add extra bytes to the buffers, to reduce new memory allocation
     var data = arcs.getVertexData();
     var nn = data.nn;
     var n = data.xx.length;
     var count = 0;
     var found = false;
-    var xx2, yy2;
+    var xx2, yy2, zz2;
     // avoid re-allocating memory on each insertion
     if (data.xx.buffer.byteLength >= data.xx.length * 8 + 8) {
       xx2 = new Float64Array(data.xx.buffer, 0, n+1);
       yy2 = new Float64Array(data.yy.buffer, 0, n+1);
     } else {
-      xx2 = new Float64Array(new ArrayBuffer((n + 20) * 8), 0, n+1);
-      yy2 = new Float64Array(new ArrayBuffer((n + 20) * 8), 0, n+1);
+      xx2 = new Float64Array(new ArrayBuffer((n + 50) * 8), 0, n+1);
+      yy2 = new Float64Array(new ArrayBuffer((n + 50) * 8), 0, n+1);
+    }
+    if (!arcs.isFlat()) {
+      zz2 = new Float64Array(new ArrayBuffer((n + 1) * 8), 0, n+1);
     }
     for (var j=0; j<nn.length; j++) {
       count += nn[j];
@@ -1816,7 +1825,12 @@
     utils.copyElements(data.yy, i, yy2, i+1, n-i);
     xx2[i] = p[0];
     yy2[i] = p[1];
-    arcs.updateVertexData(nn, xx2, yy2, null);
+    if (zz2) {
+      zz2[i] = Infinity;
+      utils.copyElements(data.zz, 0, zz2, 0, i);
+      utils.copyElements(data.zz, i, zz2, i+1, n-i);
+    }
+    arcs.updateVertexData(nn, xx2, yy2, zz2);
   }
 
   var ArcUtils = /*#__PURE__*/Object.freeze({
@@ -4663,8 +4677,8 @@
     if (isLatLngCRS(P)) {
       return xy.concat();
     }
-    proj = getProjInfo(P, getCRS('wgs84'));
-    return proj(xy);
+    proj = getProjTransform(P, getCRS('wgs84'));
+    return proj(xy[0], xy[1]);
   }
 
   function getProjInfo(dataset) {
@@ -4810,6 +4824,21 @@
     return P && P.is_latlong || false;
   }
 
+  function isWGS84(P) {
+    if (!isLatLngCRS(P)) return false;
+    var proj4 = crsToProj4(P);
+    return proj4.toLowerCase().includes('84');
+  }
+
+  function isWebMercator(P) {
+    if (!P) return false;
+    var str = crsToProj4(P);
+    // e.g. +proj=merc +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +wktext +a=6378137 +b=6378137 +nadgrids=@null
+    // e.g. +proj=merc +a=6378137 +b=6378137
+    // TODO: support  https://proj.org/operations/projections/webmerc.html
+    return str.includes('+proj=merc') && str.includes('+a=6378137') && str.includes('+b=6378137');
+  }
+
   function isLatLngDataset(dataset) {
     return isLatLngCRS(getDatasetCRS(dataset));
   }
@@ -4864,6 +4893,8 @@
     getScaleFactorAtXY: getScaleFactorAtXY,
     isProjectedCRS: isProjectedCRS,
     isLatLngCRS: isLatLngCRS,
+    isWGS84: isWGS84,
+    isWebMercator: isWebMercator,
     isLatLngDataset: isLatLngDataset,
     printProjections: printProjections,
     translatePrj: translatePrj,
@@ -8357,8 +8388,8 @@
   }
 
   function getBoundsPrecisionForDisplay(bbox) {
-    var w = bbox[2] - bbox[0],
-        h = bbox[3] - bbox[1],
+    var w = Math.abs(bbox[2] - bbox[0]),
+        h = Math.abs(bbox[3] - bbox[1]),
         range = Math.min(w, h) + 1e-8,
         digits = 0;
     while (range < 2000) {
@@ -8877,12 +8908,24 @@
     arg = arg ? String(arg) : '';
     var hexStr = hexRxp.test(arg) ? arg : lookupColorName(arg);
     var rgb = null;
-    if (hexStr) rgb = parseHexColor(hexStr);
-    else if (rgbaRxp.test(arg)) rgb = parseRGBA(arg);
-    if (!testRGB(rgb)) stop("Unsupported color:", arg);
+    if (hexStr) {
+      rgb = parseHexColor(hexStr);
+    } else if (rgbaRxp.test(arg)) {
+      rgb = parseRGBA(arg);
+    }
+    if (rgb && !testRGB(rgb)) {
+      rgb = null;
+    }
     return rgb;
   }
 
+  function validateColor(arg) {
+    if (!parseColor(arg)) {
+      stop("Unsupported color:", arg);
+    }
+    return true;
+  }
+
   function testRGB(o) {
     return !!o && testChannel(o.r) && testChannel(o.g) && testChannel(o.b) &&
       testAlpha(o.a);
@@ -9028,7 +9071,7 @@
     weights = normalizeWeights(weights);
     if (!weights) return '#eee';
     var blended = colors.reduce(function(memo, col, i) {
-      var rgb = parseColor(col);
+      var rgb = validateColor(col) && parseColor(col);
       var w = +weights[i] || 0;
       memo.r += rgb.r * w;
       memo.g += rgb.g * w;
@@ -28870,7 +28913,8 @@ ${svg}
     categorical: [],
     sequential: [],
     rainbow: [],
-    diverging: []
+    diverging: [],
+    all: []
   };
   var ramps;
 
@@ -28890,6 +28934,19 @@ ${svg}
       '3182bd6baed69ecae1c6dbefe6550dfd8d3cfdae6bfdd0a231a35474c476a1d99bc7e9c0756bb19e9ac8bcbddcdadaeb636363969696bdbdbdd9d9d9');
     addCategoricalScheme('Tableau20',
       '4c78a89ecae9f58518ffbf7954a24b88d27ab79a20f2cf5b43989483bcb6e45756ff9d9879706ebab0acd67195fcbfd2b279a2d6a5c99e765fd8b5a5');
+    index.all = [].concat(index.sequential, index.rainbow, index.diverging, index.categorical);
+
+  }
+
+  function standardName(name) {
+    if (!name) return null;
+    var lcname = name.toLowerCase();
+    for (var i=0; i<index.all.length; i++) {
+      if (index.all[i].toLowerCase() == lcname) {
+        return index.all[i];
+      }
+    }
+    return null;
   }
 
   function addSchemesFromD3(type, names) {
@@ -28944,9 +29001,26 @@ ${svg}
     print ('\nMulti-hue/rainbow\n' + formatStringsAsGrid(index.rainbow));
   }
 
+  function pickRandomColorScheme(type) {
+    initSchemes();
+    var names = index[type];
+    if (!names) error('Unknown color scheme type:', type);
+    var i = Math.floor(Math.random() * names.length);
+    return names[i];
+  }
+
+  function getRandomColors(n) {
+    initSchemes();
+    var colors = getCategoricalColorScheme('Tableau20', 20);
+    utils.shuffle(colors);
+    colors = wrapColors(colors, n);
+    return colors.slice(0, n);
+  }
+
   function getCategoricalColorScheme(name, n) {
     var colors;
     initSchemes();
+    name = standardName(name);
     if (!isColorSchemeName(name)) {
       stop('Unknown color scheme name:', name);
     } else if (isCategoricalColorScheme(name)) {
@@ -28973,17 +29047,17 @@ ${svg}
 
   function isColorSchemeName(name) {
     initSchemes();
-    return index.categorical.includes(name) || index.sequential.includes(name) ||
-      index.diverging.includes(name) || index.rainbow.includes(name);
+    return index.all.includes(standardName(name));
   }
 
   function isCategoricalColorScheme(name) {
     initSchemes();
-    return index.categorical.includes(name);
+    return index.categorical.includes(standardName(name));
   }
 
   function getColorRamp(name, n, stops) {
     initSchemes();
+    name = standardName(name);
     var lib = require('d3-scale-chromatic');
     var ramps = lib['scheme' + name];
     var interpolate = lib['interpolate' + name];
@@ -29016,47 +29090,68 @@ ${svg}
     return ramp;
   }
 
+  function getNullValue(opts) {
+    var nullValue;
+    if ('null_value' in opts) {
+      nullValue = parseNullValue(opts.null_value);
+    } else if (opts.colors) {
+      nullValue = '#eee';
+    } else if (opts.values) {
+      nullValue = null;
+    } else {
+      nullValue = -1; // kludge, to match behavior of getClassValues()
+    }
+    return nullValue;
+  }
+
+  // Parse command line string arguments to the correct data type
+  function parseNullValue(val) {
+    if (utils.isString(val) && !isNaN(+val)) {
+      val = +val;
+    }
+    if (val === 'null') {
+      val = null;
+    }
+    return val;
+  }
+
   function getClassValues(method, n, opts) {
     var categorical = method == 'categorical' || method == 'non-adjacent';
-    var colorArg = opts.colors ? opts.colors[0] : null;
+    var colorArg = opts.colors && opts.colors.length == 1 ? opts.colors[0] : null;
     var colorScheme;
 
-    if (isColorSchemeName(colorArg)) {
+    if (colorArg == 'random') {
+      if (categorical) {
+        return getRandomColors(n);
+      }
+      colorScheme = pickRandomColorScheme('sequential');
+      message('Randomly selected color ramp:', colorScheme);
+    } else if (isColorSchemeName(colorArg)) {
       colorScheme = colorArg;
-    } else if (colorArg == 'random') {
-      colorScheme = categorical ? 'Tableau20' : 'BuGn'; // TODO: randomize
+    } else if (colorArg && !parseColor(colorArg)) {
+      stop('Unrecognized color scheme name:', colorArg);
     } else if (opts.colors) {
-      // validate colors
-      opts.colors.forEach(parseColor);
+      opts.colors.forEach(validateColor);
     }
 
-    if (categorical) {
-      if (colorScheme && isCategoricalColorScheme(colorScheme)) {
+    if (colorScheme) {
+      if (categorical && isCategoricalColorScheme(colorScheme)) {
         return getCategoricalColorScheme(colorScheme, n);
-      } else if (colorScheme) {
-        // assume we have a sequential ramp
+      } else {
         return getColorRamp(colorScheme, n, opts.stops);
-      } else if (opts.colors || opts.values) {
+      }
+    } else if (opts.colors || opts.values) {
+      if (categorical) {
         return getCategoricalValues(opts.colors || opts.values, n);
       } else {
-        // numerical indexes seem to make sense for non-adjacent and categorical colors
-        return getIndexes(n);
-      }
-    } else {
-      // sequential values
-      if (colorScheme) {
-        return getColorRamp(colorScheme, n, opts.stops);
-      } else if (opts.colors || opts.values) {
         return getInterpolableValues(opts.colors || opts.values, n, opts);
-      } else {
-        // TODO: rethink this
-        // return getInterpolableValues([0, 1], n, opts);
-        return getIndexes(n);
       }
+    } else {
+      // use numerical class indexes (0, 1, ...) if no values are given
+      return getIndexes(n);
     }
   }
 
-
   function getCategoricalValues(values, n) {
     if (n != values.length) {
       stop('Mismatch in number of categories and number of values');
@@ -29093,26 +29188,36 @@ ${svg}
     return values;
   }
 
-  var sequential = ['quantile', 'nice', 'equal-interval', 'hybrid'];
+  var sequential = ['quantile', 'nice', 'equal-interval', 'hybrid', 'breaks'];
   var all = ['non-adjacent', 'indexed', 'categorical'].concat(sequential);
 
-  function getClassifyMethod(opts, dataFieldType) {
+  function getClassifyMethod(opts, dataType) {
     var method;
     if (opts.method) {
       method = opts.method;
+    } else if (opts.breaks) {
+      method = 'breaks';
     } else if (opts.index_field) {
       method = 'indexed';
-    } else if (opts.categories || dataFieldType == 'string') {
+    } else if (opts.categories || dataType == 'string') {
       method = 'categorical';
-    } else  if (dataFieldType == 'number') {
+    } else  if (dataType == 'number') {
       method = 'quantile'; // TODO: validate data field
+    } else if (dataType == 'date' || dataType == 'object') {
+      stop('Data type does not support classification:', dataType);
+    } else if (dataType === null) {
+      // data field is empty
+      return null; // kludge
+    } else if (dataType === undefined) {
+      // no data field was given
+      stop('Expected a data field to classify or the non-adjacent option');
     } else {
       stop('Unable to determine which classification method to use.');
     }
     if (!all.includes(method)) {
       stop('Not a recognized classification method:', method);
     }
-    if (sequential.includes(method) && dataFieldType != 'number') {
+    if (sequential.includes(method) && dataType != 'number' && dataType !== null) {
       stop('The', method, 'method requires a numerical data field');
     }
     return method;
@@ -29125,7 +29230,7 @@ ${svg}
     var opts = optsArg || {};
     var records = lyr.data && lyr.data.getRecords();
     var valuesAreColors = !!opts.colors;
-    var dataField, dataFieldType, outputField;
+    var dataField, fieldType, outputField;
     var values, nullValue;
     var classifyByValue, classifyByRecordId;
     var numClasses, numValues;
@@ -29139,9 +29244,10 @@ ${svg}
     //
     if (opts.index_field) {
       dataField = opts.index_field;
-    } else if (opts.field) {
+      fieldType = getColumnType(opts.field, records);
+   } else if (opts.field) {
       dataField = opts.field;
-      dataFieldType = getColumnType(opts.field, records);
+      fieldType = getColumnType(opts.field, records);
     }
     if (dataField) {
       requireDataField(lyr.data, dataField);
@@ -29149,7 +29255,7 @@ ${svg}
 
     // get classification method
     //
-    method = getClassifyMethod(opts, dataFieldType);
+    method = getClassifyMethod(opts, fieldType);
 
     // validate classification method
     if (method == 'non-adjacent') {
@@ -29167,6 +29273,7 @@ ${svg}
     // get the number of classes and the number of values
     //
     // expand categories if value is '*'
+    // use all unique values if categories option is missing
     if (method == 'categorical') {
       if ((!opts.categories || opts.categories.includes('*')) && dataField) {
         opts.categories = getUniqFieldValues(records, dataField);
@@ -29208,24 +29315,16 @@ ${svg}
       message('Colors:', formatValuesForLogging(values));
     }
 
-    // get null value
-    //
-    if ('null_value' in opts) {
-      nullValue = opts.null_value;
-    } else if (valuesAreColors) {
-      nullValue = '#eee';
-    } else if (opts.values) {
-      nullValue = null;
-    } else {
-      nullValue = -1; // kludge, to match behavior of getClassValues()
-    }
-
+    nullValue = getNullValue(opts);
 
     // get a function to convert input data to class indexes
     //
-    if (method == 'non-adjacent') {
+    if (fieldType === null) {
+      // no valid data -- always return null value
+      classifyByRecordId = function() {return nullValue;};
+    } else if (method == 'non-adjacent') {
       classifyByRecordId = getNonAdjacentClassifier(lyr, dataset, values);
-    } else if (opts.index_field) {
+    } else if (method == 'indexed') {
       // data is pre-classified... just read the index from a field
       classifyByValue = getIndexedClassifier(values, nullValue, opts);
     } else if (method == 'categorical') {
@@ -31140,7 +31239,11 @@ ${svg}
     if (!opts.force && !opts.prefix) {
       // overwrite existing fields if the "force" option is set.
       // prefix also overwrites... TODO: consider changing this
-      joinFields = utils.difference(joinFields, destFields);
+      var duplicateFields = utils.intersection(joinFields, destFields);
+      if (duplicateFields.length > 0) {
+        message('Same-named fields not joined without the "force" flag:', duplicateFields);
+        joinFields = utils.difference(joinFields, duplicateFields);
+      }
     }
     return joinFields;
   }
@@ -31872,7 +31975,7 @@ ${svg}
     //   stop("Missing required colors parameter");
     // }
     if (Array.isArray(opts.colors)) {
-      opts.colors.forEach(parseColor); // validate colors
+      opts.colors.forEach(validateColor);
     }
 
     var records = lyr.data ? lyr.data.getRecords() : [];
@@ -32111,7 +32214,7 @@ ${svg}
     var name = cmdOpts.name;
     var cmdDefn = externalCommands[name];
     if (!cmdDefn) {
-      stop('Unsupported command');
+      stop('Unsupported command:', name);
     }
     var targetType = cmdDefn.target;
     var opts = parseExternalCommand(name, cmdDefn, cmdOpts._);
@@ -34243,6 +34346,7 @@ ${svg}
         // old simplification data  will not be optimal after reprojection;
         // re-using for now to avoid error in web ui
         zz = data.zz,
+        z = arcs.getRetainedInterval(),
         p;
 
     for (var i=0, n=xx.length; i<n; i++) {
@@ -34251,6 +34355,7 @@ ${svg}
       yy[i] = p[1];
     }
     arcs.updateVertexData(data.nn, xx, yy, zz);
+    arcs.setRetainedInterval(z);
   }
 
   function projectArcs2(arcs, proj) {
@@ -36891,8 +36996,12 @@ ${svg}
     // support multiline string of commands pasted into console
     str = str.split(/\n+/g).map(function(str) {
       var match = /^[a-z][\w-]*/.exec(str = str.trim());
-      if (match && parser.isCommandName(match[0])) {
-        str = '-' + str; // add hyphen prefix to bare command
+      //if (match && parser.isCommandName(match[0])) {
+      if (match) {
+        // add hyphen prefix to bare command
+         // also add hyphen to non-command strings, for a better error message
+         // ("unsupported command" instead of "The -i command cannot be run in the browser")
+        str = '-' + str;
       }
       return str;
     }).join(' ');

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "mapshaper",
-	"version": "0.5.96",
+	"version": "0.5.99",
 	"description": "A tool for editing vector datasets for mapping and GIS.",
 	"keywords": [
 		"shapefile",
@@ -35,6 +35,7 @@
 	"files": [
 		"/bin/**",
 		"/www/**",
+		"!/www/basemap.js",
 		"!/www/nacis/",
 		"/mapshaper.js",
 		"!.DS_Store"

package/www/index.html

@@ -73,7 +73,7 @@
 		<input type="text" value="label" class="clicktext" />
 	</div></div>
 	<div id="mode-buttons" class="page-header-buttons">
-		<span class="simplify-btn header-btn btn">Simplify</span><span class="separator"></span><span class="console-btn header-btn btn">Console</span><span class="separator"></span><span class="export-btn header-btn btn">Export</span>
+		<span class="basemap-btn header-btn btn">Basemap</span><span class="separator"></span><span class="simplify-btn header-btn btn">Simplify</span><span class="separator"></span><span class="console-btn header-btn btn">Console</span><span class="separator"></span><span class="export-btn header-btn btn">Export</span>
 	</div>
 	<div id="splash-buttons" class="page-header-buttons">
 		<a href="https://github.com/mbloch/mapshaper/wiki"><span id="wiki-btn" class="header-btn btn">Wiki</span></a><span class="separator"></span><a href="https://github.com/mbloch/mapshaper"><span id="github-btn" class="header-btn btn">GitHub</span></a>
@@ -148,6 +148,18 @@
 	</div>
 </div>
 
+<div class="basemap-options main-area popup-dialog">
+	<div class="info-box">
+		<h3>Basemap options</h3>
+		<p>Basemaps are only displayed if the active layer has WGS84 or Web Mercator coordinates.</p>
+		<p class="basemap-error"></p>
+		<div class="basemap-styles"></div>
+		<div>
+			<div class="close-btn btn dialog-btn">Close</div>
+		</div>
+	</div>
+</div>
+
 <div class="simplify-options main-area popup-dialog">
 	<div class="info-box">
 		<h3>Simplification menu</h3>
@@ -280,6 +292,7 @@ interface. Examples: "no-topology"
 			<div class="repair-btn text-btn colored-text">Repair</div>
 		</div>
 		<div class="map-layers"></div>
+		<div class="basemap-container"><div class="basemap"></div></div>
 	</div>
 </div>
 
@@ -287,6 +300,7 @@ interface. Examples: "no-topology"
 <div class="drop-area"></div>
 </div>
 
+<script src="basemap.js" type="text/javascript"></script>
 <script src="zip.js" type="text/javascript"></script>
 <script src="modules.js" type="text/javascript"></script>
 <script src="mapshaper.js" type="text/javascript"></script>