mapshaper 0.5.115 → 0.5.118

package/CHANGELOG.md

@@ -1,3 +1,13 @@
+v0.5.118
+* Added -comment command, as a way to add inline comments in long sequences of commands.
+* Added support for importing .cpg files (Shapefile text encoding) in the web UI.
+
+v0.5.117
+* Support reading DBF files larger than 2GB (command line program only).
+
+v0.5.116
+* Prevent 0-length DBF string fields, for interoperability with PostGIS.
+
 v0.5.115
 * Scalebar tweaks.
 

package/bin/mapshaper-gui

@@ -247,6 +247,7 @@ function expandShapefiles(files) {
     if (isUrl(f)) return; // TODO: try to load aux files from URLs
     addAuxFile(files, f, '.dbf');
     addAuxFile(files, f, '.prj');
+    addAuxFile(files, f, '.shx');
     addAuxFile(files, f, '.cpg');
   });
   return files;

package/mapshaper.js

@@ -1,6 +1,6 @@
 (function () {
 
-  var VERSION = "0.5.115";
+  var VERSION = "0.5.118";
 
 
   var utils = /*#__PURE__*/Object.freeze({
@@ -16006,7 +16006,9 @@ ${svg}
 
   function initStringField(info, arr, name, encoding) {
     var formatter = encoding == 'ascii' ? encodeValueAsAscii : getStringWriterEncoded(encoding);
-    var size = 0;
+    // Set minimum field size to 1 byte, for interoperability with PostGIS
+    // (see https://github.com/mbloch/mapshaper/issues/541)
+    var size = 1;
     var truncated = 0;
     var buffers = arr.map(function(rec) {
       var strval = convertValueToString(rec[name]);
@@ -17213,7 +17215,7 @@ ${svg}
   function guessInputFileType(file) {
     var ext = getFileExtension(file || '').toLowerCase(),
         type = null;
-    if (ext == 'dbf' || ext == 'shp' || ext == 'prj' || ext == 'shx' || ext == 'kml') {
+    if (ext == 'dbf' || ext == 'shp' || ext == 'prj' || ext == 'shx' || ext == 'kml' || ext == 'cpg') {
       type = ext;
     } else if (/json$/.test(ext)) {
       type = 'json';
@@ -17626,7 +17628,9 @@ ${svg}
         binArr, buf;
 
     this.readToBinArray = function(start, length) {
-      if (bufSize < start + length) error("Out-of-range error");
+      if (bufSize < start + length) {
+        error("Out-of-range error");
+      }
       if (!binArr) binArr = new BinArray(src);
       binArr.position(start);
       return binArr;
@@ -21084,6 +21088,10 @@ ${svg}
     parser.command('colors')
       .describe('print list of color scheme names');
 
+    parser.command('comment')
+      .describe('add a comment to the sequence of commands')
+      .flag('multi_arg');
+
     parser.command('encodings')
       .describe('print list of supported text encodings (for .dbf import)');
 
@@ -21135,6 +21143,9 @@ ${svg}
   // http://www.digitalpreservation.gov/formats/fdd/fdd000325.shtml
   // http://www.clicketyclick.dk/databases/xbase/format/index.html
   // http://www.clicketyclick.dk/databases/xbase/format/data_types.html
+  // esri docs:
+  // https://support.esri.com/en/technical-article/000007920
+  // https://desktop.arcgis.com/en/arcmap/latest/manage-data/shapefiles/geoprocessing-considerations-for-shapefile-output.htm
 
   // source: http://webhelp.esri.com/arcpad/8.0/referenceguide/index.htm#locales/task_code.htm
   var languageIds = [0x01,'437',0x02,'850',0x03,'1252',0x08,'865',0x09,'437',0x0A,'850',0x0B,'437',0x0D,'437',0x0E,'850',0x0F,'437',0x10,'850',0x11,'437',0x12,'850',0x13,'932',0x14,'850',0x15,'437',0x16,'850',0x17,'865',0x18,'437',0x19,'437',0x1A,'850',0x1B,'437',0x1C,'863',0x1D,'850',0x1F,'852',0x22,'852',0x23,'852',0x24,'860',0x25,'850',0x26,'866',0x37,'850',0x40,'852',0x4D,'936',0x4E,'949',0x4F,'950',0x50,'874',0x57,'1252',0x58,'1252',0x59,'1252',0x64,'852',0x65,'866',0x66,'865',0x67,'861',0x6A,'737',0x6B,'857',0x6C,'863',0x78,'950',0x79,'949',0x7A,'936',0x7B,'932',0x7C,'874',0x86,'737',0x87,'852',0x88,'857',0xC8,'1250',0xC9,'1251',0xCA,'1254',0xCB,'1253',0xCC,'1257'];
@@ -21271,11 +21282,12 @@ ${svg}
   // @src is a Buffer or ArrayBuffer or filename
   //
   function DbfReader(src, encodingArg) {
-    if (utils.isString(src)) {
-      error("[DbfReader] Expected a buffer, not a string");
-    }
-    var bin = new BinArray(src);
-    var header = readHeader(bin);
+    var opts = {
+      cacheSize: 0x2000000, // 32MB
+      bufferSize: 0x400000 // 4MB
+    };
+    var dbfFile = utils.isString(src) ? new FileReader(src, opts) : new BufferReader(src);
+    var header = readHeader(dbfFile);
 
     // encoding and fields are set on first access
     var fields;
@@ -21291,7 +21303,11 @@ ${svg}
 
     this.getFields = getFieldNames;
 
-    this.getBuffer = function() {return bin.buffer();};
+    // TODO: switch to streaming output under Node.js
+    this.getBuffer = function() {
+      return dbfFile.readSync(0, dbfFile.size());
+      // return bin.buffer();
+    };
 
     this.deleteField = function(f) {
       prepareToRead();
@@ -21332,7 +21348,10 @@ ${svg}
       });
     }
 
-    function readHeader(bin) {
+    function readHeader(reader) {
+      // fetch enough bytes to accomodate any header
+      var maxHeaderLen = 32 * 256 + 1; // 255 fields * fieldRecSize + headerRecSize + terminator
+      var bin = reader.readToBinArray(0, Math.min(maxHeaderLen, reader.size()));
       bin.position(0).littleEndian();
       var header = {
         version: bin.readInt8(),
@@ -21418,30 +21437,32 @@ ${svg}
       return new Function('return {' + args.join(',') + '};');
     }
 
-    function findEofPos(bin) {
-      var pos = bin.size() - 1;
-      if (bin.peek(pos) != 0x1A) { // last byte may or may not be EOF
-        pos++;
-      }
-      return pos;
-    }
+    // function findEofPos(bin) {
+    //   var pos = bin.size() - 1;
+    //   if (bin.peek(pos) != 0x1A) { // last byte may or may not be EOF
+    //     pos++;
+    //   }
+    //   return pos;
+    // }
 
     function getRecordReader() {
       prepareToRead();
       var readers = fields.map(getFieldReader),
-          eofOffs = findEofPos(bin),
           create = getRecordConstructor(),
           values = [];
 
       return function readRow(r) {
-        var offs = getRowOffset(r),
+        var bin = dbfFile.readToBinArray(getRowOffset(r), header.recordSize),
+            rowOffs = bin.position(),
             fieldOffs, field;
+        if (bin.bytesLeft() < header.recordSize ||
+            bin.bytesLeft() == header.recordSize && bin.peek(bin.size() - 1) == 0x1A) {
+          // check for observed data error: last data byte contains EOF
+          stop('Invalid DBF file: encountered end-of-file while reading data');
+        }
         for (var c=0, cols=fields.length; c<cols; c++) {
           field = fields[c];
-          fieldOffs = offs + field.columnOffset;
-          if (fieldOffs + field.size > eofOffs) {
-            stop('Invalid DBF file: encountered end-of-file while reading data');
-          }
+          fieldOffs = rowOffs + field.columnOffset;
           bin.position(fieldOffs);
           values[c] = readers[c](bin, field.size);
         }
@@ -21499,11 +21520,12 @@ ${svg}
 
       // Show a sample of decoded text if non-ascii-range text has been found
       if (encoding && samples.length > 0) {
+        msg = "Detected DBF text encoding: " + encoding + (encoding in encodingNames ? " (" + encodingNames[encoding] + ")" : "");
+        message(msg);
         msg = decodeSamples(encoding, samples);
         msg = formatStringsAsGrid(msg.split('\n'));
         msg = "\nSample text containing non-ascii characters:" + (msg.length > 60 ? '\n' : '') + msg;
-        msg = "Detected DBF text encoding: " + encoding + (encoding in encodingNames ? " (" + encodingNames[encoding] + ")" : "") + msg;
-        message(msg);
+        verbose(msg);
       }
       return encoding;
     }
@@ -21531,14 +21553,17 @@ ${svg}
       var maxSamples = 50;
       var buf = utils.createBuffer(256);
       var index = {};
-      var f, chars, sample, hash;
+      var f, chars, sample, hash, bin, rowOffs;
       // include non-ascii field names, if any
       samples = getNonAsciiHeaders();
       for (var r=0; r<rows; r++) {
+        bin = dbfFile.readToBinArray(getRowOffset(r), header.recordSize);
+        rowOffs = bin.position();
         for (var c=0; c<cols; c++) {
           if (samples.length >= maxSamples) break;
           f = stringFields[c];
-          bin.position(getRowOffset(r) + f.columnOffset);
+          // bin.position(getRowOffset(r) + f.columnOffset);
+          bin.position(rowOffs + f.columnOffset);
           chars = readStringBytes(bin, f.size, buf);
           if (chars > 0 && bufferContainsHighBit(buf, chars)) {
             sample = utils.createBuffer(buf.slice(0, chars)); //
@@ -21554,9 +21579,9 @@ ${svg}
     }
   }
 
-  function importDbfTable(buf, o) {
+  function importDbfTable(src, o) {
     var opts = o || {};
-    return new ShapefileTable(buf, opts.encoding);
+    return new ShapefileTable(src, opts.encoding);
   }
 
   // Implements the DataTable api for DBF file data.
@@ -21565,8 +21590,8 @@ ${svg}
   // just the shapes and exporting in Shapefile format.
   // TODO: consider accepting just the filename, so buffer doesn't consume memory needlessly.
   //
-  function ShapefileTable(buf, encoding) {
-    var reader = new DbfReader(buf, encoding),
+  function ShapefileTable(src, encoding) {
+    var reader = new DbfReader(src, encoding),
         altered = false,
         table;
 
@@ -21575,15 +21600,23 @@ ${svg}
         // export DBF records on first table access
         table = new DataTable(reader.readRows());
         reader = null;
-        buf = null; // null out references to DBF data for g.c.
+        src = null; // null out references to DBF data for g.c.
       }
       return table;
     }
 
     this.exportAsDbf = function(opts) {
-      // export original dbf bytes if possible, for performance
+      // export original dbf bytes if possible
+      // (e.g. if the data attributes haven't changed)
       var useOriginal = !!reader && !altered && !opts.field_order && !opts.encoding;
-      if (useOriginal) return reader.getBuffer();
+      if (useOriginal) {
+        try {
+          // Maximum Buffer in current Node.js is 2GB
+          // We fall back to import-export if getBuffer() fails.
+          // This may produce a buffer that does not exceed the maximum size.
+          return reader.getBuffer();
+        } catch(e) {}
+      }
       return Dbf.exportRecords(getTable().getRecords(), opts.encoding, opts.field_order);
     };
 
@@ -23584,6 +23617,10 @@ ${svg}
     if (obj.prj) {
       dataset.info.prj = obj.prj.content;
     }
+    if (obj.cpg) {
+      // TODO: consider using the input encoding as the default output encoding
+      dataset.info.cpg = obj.cpg.content;
+    }
     return dataset;
   }
 
@@ -23593,7 +23630,7 @@ ${svg}
     if (input.cpg && !opts.encoding) {
       opts.encoding = input.cpg.content;
     }
-    table = importDbfTable(input.dbf.content, opts);
+    table = importDbfTable(input.dbf.content || input.dbf.filename, opts);
     return {
       info: {},
       layers: [{data: table}]
@@ -23691,18 +23728,10 @@ ${svg}
         content;
 
     cli.checkFileExists(path, cache);
-    if (fileType == 'shp' && !cached) {
-      // let ShpReader read the file (supports larger files)
+    if ((fileType == 'shp' || fileType == 'json' || fileType == 'text' || fileType == 'dbf') && !cached) {
+      // these file types are read incrementally
       content = null;
 
-    } else if (fileType == 'json' && !cached) {
-      // postpone reading of JSON files, to support incremental parsing
-      content = null;
-
-    } else if (fileType == 'text' && !cached) {
-      // content = cli.readFile(path); // read from buffer
-      content = null; // read from file, to support largest files (see mapshaper-delim-import.js)
-
     } else if (fileType && isSupportedBinaryInputType(path)) {
       content = cli.readFile(path, null, cache);
       if (utils.isString(content)) {
@@ -23759,7 +23788,11 @@ ${svg}
       obj.shx = {filename: shxPath, content: cli.readFile(shxPath, null, cache)};
     }
     if (!obj.dbf && cli.isFile(dbfPath, cache)) {
-      obj.dbf = {filename: dbfPath, content: cli.readFile(dbfPath, null, cache)};
+      // obj.dbf = {filename: dbfPath, content: cli.readFile(dbfPath, null, cache)};
+      obj.dbf = {
+        filename: dbfPath,
+        content: (cache && (dbfPath in cache)) ? cli.readFile(dbfPath, null, cache) : null
+      };
     }
     if (obj.dbf && cli.isFile(cpgPath, cache)) {
       obj.cpg = {filename: cpgPath, content: cli.readFile(cpgPath, 'utf-8', cache).trim()};
@@ -24037,6 +24070,7 @@ ${svg}
     clearStash();  // prevent errors from overwriting stash
     stashVar('current_command', cmd.name);
     stashVar('DEBUG', job.settings.DEBUG || cmd.debug);
+    stashVar('VERBOSE', job.settings.VERBOSE || cmd.verbose);
     stashVar('QUIET', job.settings.QUIET || cmd.quiet);
     stashVar('defs', job.defs);
     stashVar('input_files', job.input_files);
@@ -30463,6 +30497,9 @@ ${svg}
       targetDataset.arcs = mergedDataset.arcs;
       // dissolve clip layer shapes (to remove overlaps and other topological issues
       // that might confuse the clipping function)
+      // use a data-free copy of the clip lyr, so data records are not dissolved
+      // (this avoids triggering an unnecessary and expensive DBF read operation in some cases).
+      clipLyr = utils.defaults({data: null}, clipLyr);
       clipLyr = dissolvePolygonLayer2(clipLyr, mergedDataset, {quiet: true, silent: true});
 
     } else {
@@ -30864,6 +30901,8 @@ ${svg}
     getColorizerFunction: getColorizerFunction
   });
 
+  cmd.comment = function() {}; // no-op, so -comment doesn't trigger a parsing error
+
   function expressionUsesGeoJSON(exp) {
     return exp.includes('this.geojson');
   }
@@ -39778,7 +39817,7 @@ ${svg}
   function commandAcceptsEmptyTarget(name) {
     return name == 'graticule' || name == 'i' || name == 'help' ||
       name == 'point-grid' || name == 'shape' || name == 'rectangle' ||
-      name == 'include' || name == 'print' || name == 'if' || name == 'elif' ||
+      name == 'include' || name == 'print' || name == 'comment' || name == 'if' || name == 'elif' ||
       name == 'else' || name == 'endif';
   }
 
@@ -39886,6 +39925,9 @@ ${svg}
       } else if (name == 'colorizer') {
         outputLayers = cmd.colorizer(opts);
 
+      } else if (name == 'comment') {
+        // no-op
+
       } else if (name == 'dashlines') {
         applyCommandToEachLayer(cmd.dashlines, targetLayers, targetDataset, opts);
 

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "mapshaper",
-	"version": "0.5.115",
+	"version": "0.5.118",
 	"description": "A tool for editing vector datasets for mapping and GIS.",
 	"keywords": [
 		"shapefile",

package/www/elements.css

@@ -67,10 +67,10 @@ div.tip {
 	z-index: 500;
 	cursor: pointer;
 	text-align: center;
-	font-size: 13px;
+	font-size: 14px;
 	line-height: 1;
 	font-weight: normal;
-	color: #799FCB;
+	color: #1385B7;
 }
 
 .tip-button .tip-anchor {

package/www/index.html

@@ -130,7 +130,8 @@
 
 <div class="export-options main-area popup-dialog">
 	<div class="info-box">
-		<h3>Export menu</h3>
+		<div class="close2-btn"></div>
+		<h3>Export options</h3>
 		<div style="height:3px"></div>
 		<div class=export-layers>
 			<h4 class="menu-title">Layers</h4>
@@ -140,22 +141,23 @@
 		<div class="export-formats option-menu">
 		</div>
 
-		<div class="option-menu"><input type="text" class="advanced-options" placeholder="export options" /><div class="tip-button">?<div class="tip-anchor">
+		<div class="option-menu"><input type="text" class="advanced-options" placeholder="command line options" /><div class="tip-button">?<div class="tip-anchor">
 		<div class="tip">Enter options from the command line interface for the -o command. Examples: "bbox" "no-quantization"
 "precision=0.001"</div></div></div></div>
-		<div class="cancel-btn btn dialog-btn">Cancel</div>
+		<!-- <div class="cancel-btn btn dialog-btn">Cancel</div> -->
 		<div class="save-btn btn dialog-btn">Export</div>
 	</div>
 </div>
 
 <div class="basemap-options main-area popup-dialog">
 	<div class="info-box">
+		<div class="close2-btn"></div>
 		<h3>Basemap options</h3>
 		<p class="basemap-note">Your data will be displayed using the Mercator projection.</p>
 		<p class="basemap-warning"></p>
 		<div class="basemap-styles"></div>
 		<div>
-			<div class="close-btn btn dialog-btn">Close</div>
+			<!-- <div class="close-btn btn dialog-btn">Close</div> -->
 			<div class="clear-btn btn dialog-btn disabled">Clear</div>
 			<div class="fade-btn btn dialog-btn disabled">Fade</div>
 			<!-- <div class="hide-btn btn dialog-btn disabled">Hide</div> -->
@@ -165,7 +167,8 @@
 
 <div class="simplify-options main-area popup-dialog">
 	<div class="info-box">
-		<h3>Simplification menu</h3>
+		<div class="close2-btn"></div>
+		<h3>Simplification</h3>
 		<div class="option-menu">
 			<div><label for="import-retain-opt"><input type="checkbox" class="checkbox import-retain-opt"/>prevent shape removal</label>
 							<div class="tip-button">?<div class="tip-anchor">
@@ -208,7 +211,7 @@ a smoother appearance.</div></div></div></div>
 		</div> <!-- option menu -->
 
 		<div>
-			<div class="cancel-btn btn dialog-btn">Cancel</div>
+<!-- 			<div class="cancel-btn btn dialog-btn">Cancel</div> -->
 			<div class="submit-btn btn dialog-btn">Apply</div>
 		</div>
 	</div> <!-- .info-box -->

package/www/mapshaper-gui.js

@@ -310,7 +310,6 @@
 
   // tests if filename is a type that can be used
   GUI.isReadableFileType = function(filename) {
-    var ext = internal.getFileExtension(filename).toLowerCase();
     return !!internal.guessInputFileType(filename) || internal.couldBeDsvFile(filename) ||
       internal.isZipFile(filename);
   };
@@ -1354,7 +1353,6 @@
         }
       }
       model.updated({select: true});
-
     }
 
     function clearQueuedFiles() {
@@ -1451,9 +1449,8 @@
     }
 
 
-    // TODO: support .cpg
     function isShapefilePart(name) {
-      return /\.(shp|shx|dbf|prj)$/i.test(name);
+      return /\.(shp|shx|dbf|prj|cpg)$/i.test(name);
     }
 
     function readImportOpts() {
@@ -2131,7 +2128,7 @@
 
     // init settings menu
     new SimpleButton(menu.findChild('.submit-btn').addClass('default-btn')).on('click', onSubmit);
-    new SimpleButton(menu.findChild('.cancel-btn')).on('click', function() {
+    new SimpleButton(menu.findChild('.close2-btn')).on('click', function() {
       if (el.visible()) {
         // cancel just hides menu if slider is visible
         menu.hide();
@@ -3159,7 +3156,7 @@
     var layersArr = [];
     var toggleBtn = null; // checkbox <input> for toggling layer selection
     var exportBtn = gui.container.findChild('.export-btn');
-    new SimpleButton(menu.findChild('.cancel-btn')).on('click', gui.clearMode);
+    new SimpleButton(menu.findChild('.close2-btn')).on('click', gui.clearMode);
 
     if (!GUI.exportIsSupported()) {
       exportBtn.on('click', function() {
@@ -9764,7 +9761,7 @@
     var menu = gui.container.findChild('.basemap-options');
     // var hideBtn = new SimpleButton(menu.findChild('.hide-btn'));
     var fadeBtn = new SimpleButton(menu.findChild('.fade-btn'));
-    var closeBtn = new SimpleButton(menu.findChild('.close-btn'));
+    var closeBtn = new SimpleButton(menu.findChild('.close2-btn'));
     var clearBtn = new SimpleButton(menu.findChild('.clear-btn'));
     var list = menu.findChild('.basemap-styles');
     var container = gui.container.findChild('.basemap-container');

package/www/mapshaper.js

@@ -1,6 +1,6 @@
 (function () {
 
-  var VERSION = "0.5.115";
+  var VERSION = "0.5.118";
 
 
   var utils = /*#__PURE__*/Object.freeze({
@@ -16006,7 +16006,9 @@ ${svg}
 
   function initStringField(info, arr, name, encoding) {
     var formatter = encoding == 'ascii' ? encodeValueAsAscii : getStringWriterEncoded(encoding);
-    var size = 0;
+    // Set minimum field size to 1 byte, for interoperability with PostGIS
+    // (see https://github.com/mbloch/mapshaper/issues/541)
+    var size = 1;
     var truncated = 0;
     var buffers = arr.map(function(rec) {
       var strval = convertValueToString(rec[name]);
@@ -17213,7 +17215,7 @@ ${svg}
   function guessInputFileType(file) {
     var ext = getFileExtension(file || '').toLowerCase(),
         type = null;
-    if (ext == 'dbf' || ext == 'shp' || ext == 'prj' || ext == 'shx' || ext == 'kml') {
+    if (ext == 'dbf' || ext == 'shp' || ext == 'prj' || ext == 'shx' || ext == 'kml' || ext == 'cpg') {
       type = ext;
     } else if (/json$/.test(ext)) {
       type = 'json';
@@ -17626,7 +17628,9 @@ ${svg}
         binArr, buf;
 
     this.readToBinArray = function(start, length) {
-      if (bufSize < start + length) error("Out-of-range error");
+      if (bufSize < start + length) {
+        error("Out-of-range error");
+      }
       if (!binArr) binArr = new BinArray(src);
       binArr.position(start);
       return binArr;
@@ -21084,6 +21088,10 @@ ${svg}
     parser.command('colors')
       .describe('print list of color scheme names');
 
+    parser.command('comment')
+      .describe('add a comment to the sequence of commands')
+      .flag('multi_arg');
+
     parser.command('encodings')
       .describe('print list of supported text encodings (for .dbf import)');
 
@@ -21135,6 +21143,9 @@ ${svg}
   // http://www.digitalpreservation.gov/formats/fdd/fdd000325.shtml
   // http://www.clicketyclick.dk/databases/xbase/format/index.html
   // http://www.clicketyclick.dk/databases/xbase/format/data_types.html
+  // esri docs:
+  // https://support.esri.com/en/technical-article/000007920
+  // https://desktop.arcgis.com/en/arcmap/latest/manage-data/shapefiles/geoprocessing-considerations-for-shapefile-output.htm
 
   // source: http://webhelp.esri.com/arcpad/8.0/referenceguide/index.htm#locales/task_code.htm
   var languageIds = [0x01,'437',0x02,'850',0x03,'1252',0x08,'865',0x09,'437',0x0A,'850',0x0B,'437',0x0D,'437',0x0E,'850',0x0F,'437',0x10,'850',0x11,'437',0x12,'850',0x13,'932',0x14,'850',0x15,'437',0x16,'850',0x17,'865',0x18,'437',0x19,'437',0x1A,'850',0x1B,'437',0x1C,'863',0x1D,'850',0x1F,'852',0x22,'852',0x23,'852',0x24,'860',0x25,'850',0x26,'866',0x37,'850',0x40,'852',0x4D,'936',0x4E,'949',0x4F,'950',0x50,'874',0x57,'1252',0x58,'1252',0x59,'1252',0x64,'852',0x65,'866',0x66,'865',0x67,'861',0x6A,'737',0x6B,'857',0x6C,'863',0x78,'950',0x79,'949',0x7A,'936',0x7B,'932',0x7C,'874',0x86,'737',0x87,'852',0x88,'857',0xC8,'1250',0xC9,'1251',0xCA,'1254',0xCB,'1253',0xCC,'1257'];
@@ -21271,11 +21282,12 @@ ${svg}
   // @src is a Buffer or ArrayBuffer or filename
   //
   function DbfReader(src, encodingArg) {
-    if (utils.isString(src)) {
-      error("[DbfReader] Expected a buffer, not a string");
-    }
-    var bin = new BinArray(src);
-    var header = readHeader(bin);
+    var opts = {
+      cacheSize: 0x2000000, // 32MB
+      bufferSize: 0x400000 // 4MB
+    };
+    var dbfFile = utils.isString(src) ? new FileReader(src, opts) : new BufferReader(src);
+    var header = readHeader(dbfFile);
 
     // encoding and fields are set on first access
     var fields;
@@ -21291,7 +21303,11 @@ ${svg}
 
     this.getFields = getFieldNames;
 
-    this.getBuffer = function() {return bin.buffer();};
+    // TODO: switch to streaming output under Node.js
+    this.getBuffer = function() {
+      return dbfFile.readSync(0, dbfFile.size());
+      // return bin.buffer();
+    };
 
     this.deleteField = function(f) {
       prepareToRead();
@@ -21332,7 +21348,10 @@ ${svg}
       });
     }
 
-    function readHeader(bin) {
+    function readHeader(reader) {
+      // fetch enough bytes to accomodate any header
+      var maxHeaderLen = 32 * 256 + 1; // 255 fields * fieldRecSize + headerRecSize + terminator
+      var bin = reader.readToBinArray(0, Math.min(maxHeaderLen, reader.size()));
       bin.position(0).littleEndian();
       var header = {
         version: bin.readInt8(),
@@ -21418,30 +21437,32 @@ ${svg}
       return new Function('return {' + args.join(',') + '};');
     }
 
-    function findEofPos(bin) {
-      var pos = bin.size() - 1;
-      if (bin.peek(pos) != 0x1A) { // last byte may or may not be EOF
-        pos++;
-      }
-      return pos;
-    }
+    // function findEofPos(bin) {
+    //   var pos = bin.size() - 1;
+    //   if (bin.peek(pos) != 0x1A) { // last byte may or may not be EOF
+    //     pos++;
+    //   }
+    //   return pos;
+    // }
 
     function getRecordReader() {
       prepareToRead();
       var readers = fields.map(getFieldReader),
-          eofOffs = findEofPos(bin),
           create = getRecordConstructor(),
           values = [];
 
       return function readRow(r) {
-        var offs = getRowOffset(r),
+        var bin = dbfFile.readToBinArray(getRowOffset(r), header.recordSize),
+            rowOffs = bin.position(),
             fieldOffs, field;
+        if (bin.bytesLeft() < header.recordSize ||
+            bin.bytesLeft() == header.recordSize && bin.peek(bin.size() - 1) == 0x1A) {
+          // check for observed data error: last data byte contains EOF
+          stop('Invalid DBF file: encountered end-of-file while reading data');
+        }
         for (var c=0, cols=fields.length; c<cols; c++) {
           field = fields[c];
-          fieldOffs = offs + field.columnOffset;
-          if (fieldOffs + field.size > eofOffs) {
-            stop('Invalid DBF file: encountered end-of-file while reading data');
-          }
+          fieldOffs = rowOffs + field.columnOffset;
           bin.position(fieldOffs);
           values[c] = readers[c](bin, field.size);
         }
@@ -21499,11 +21520,12 @@ ${svg}
 
       // Show a sample of decoded text if non-ascii-range text has been found
       if (encoding && samples.length > 0) {
+        msg = "Detected DBF text encoding: " + encoding + (encoding in encodingNames ? " (" + encodingNames[encoding] + ")" : "");
+        message(msg);
         msg = decodeSamples(encoding, samples);
         msg = formatStringsAsGrid(msg.split('\n'));
         msg = "\nSample text containing non-ascii characters:" + (msg.length > 60 ? '\n' : '') + msg;
-        msg = "Detected DBF text encoding: " + encoding + (encoding in encodingNames ? " (" + encodingNames[encoding] + ")" : "") + msg;
-        message(msg);
+        verbose(msg);
       }
       return encoding;
     }
@@ -21531,14 +21553,17 @@ ${svg}
       var maxSamples = 50;
       var buf = utils.createBuffer(256);
       var index = {};
-      var f, chars, sample, hash;
+      var f, chars, sample, hash, bin, rowOffs;
       // include non-ascii field names, if any
       samples = getNonAsciiHeaders();
       for (var r=0; r<rows; r++) {
+        bin = dbfFile.readToBinArray(getRowOffset(r), header.recordSize);
+        rowOffs = bin.position();
         for (var c=0; c<cols; c++) {
           if (samples.length >= maxSamples) break;
           f = stringFields[c];
-          bin.position(getRowOffset(r) + f.columnOffset);
+          // bin.position(getRowOffset(r) + f.columnOffset);
+          bin.position(rowOffs + f.columnOffset);
           chars = readStringBytes(bin, f.size, buf);
           if (chars > 0 && bufferContainsHighBit(buf, chars)) {
             sample = utils.createBuffer(buf.slice(0, chars)); //
@@ -21554,9 +21579,9 @@ ${svg}
     }
   }
 
-  function importDbfTable(buf, o) {
+  function importDbfTable(src, o) {
     var opts = o || {};
-    return new ShapefileTable(buf, opts.encoding);
+    return new ShapefileTable(src, opts.encoding);
   }
 
   // Implements the DataTable api for DBF file data.
@@ -21565,8 +21590,8 @@ ${svg}
   // just the shapes and exporting in Shapefile format.
   // TODO: consider accepting just the filename, so buffer doesn't consume memory needlessly.
   //
-  function ShapefileTable(buf, encoding) {
-    var reader = new DbfReader(buf, encoding),
+  function ShapefileTable(src, encoding) {
+    var reader = new DbfReader(src, encoding),
         altered = false,
         table;
 
@@ -21575,15 +21600,23 @@ ${svg}
         // export DBF records on first table access
         table = new DataTable(reader.readRows());
         reader = null;
-        buf = null; // null out references to DBF data for g.c.
+        src = null; // null out references to DBF data for g.c.
       }
       return table;
     }
 
     this.exportAsDbf = function(opts) {
-      // export original dbf bytes if possible, for performance
+      // export original dbf bytes if possible
+      // (e.g. if the data attributes haven't changed)
       var useOriginal = !!reader && !altered && !opts.field_order && !opts.encoding;
-      if (useOriginal) return reader.getBuffer();
+      if (useOriginal) {
+        try {
+          // Maximum Buffer in current Node.js is 2GB
+          // We fall back to import-export if getBuffer() fails.
+          // This may produce a buffer that does not exceed the maximum size.
+          return reader.getBuffer();
+        } catch(e) {}
+      }
       return Dbf.exportRecords(getTable().getRecords(), opts.encoding, opts.field_order);
     };
 
@@ -23584,6 +23617,10 @@ ${svg}
     if (obj.prj) {
       dataset.info.prj = obj.prj.content;
     }
+    if (obj.cpg) {
+      // TODO: consider using the input encoding as the default output encoding
+      dataset.info.cpg = obj.cpg.content;
+    }
     return dataset;
   }
 
@@ -23593,7 +23630,7 @@ ${svg}
     if (input.cpg && !opts.encoding) {
       opts.encoding = input.cpg.content;
     }
-    table = importDbfTable(input.dbf.content, opts);
+    table = importDbfTable(input.dbf.content || input.dbf.filename, opts);
     return {
       info: {},
       layers: [{data: table}]
@@ -23691,18 +23728,10 @@ ${svg}
         content;
 
     cli.checkFileExists(path, cache);
-    if (fileType == 'shp' && !cached) {
-      // let ShpReader read the file (supports larger files)
+    if ((fileType == 'shp' || fileType == 'json' || fileType == 'text' || fileType == 'dbf') && !cached) {
+      // these file types are read incrementally
       content = null;
 
-    } else if (fileType == 'json' && !cached) {
-      // postpone reading of JSON files, to support incremental parsing
-      content = null;
-
-    } else if (fileType == 'text' && !cached) {
-      // content = cli.readFile(path); // read from buffer
-      content = null; // read from file, to support largest files (see mapshaper-delim-import.js)
-
     } else if (fileType && isSupportedBinaryInputType(path)) {
       content = cli.readFile(path, null, cache);
       if (utils.isString(content)) {
@@ -23759,7 +23788,11 @@ ${svg}
       obj.shx = {filename: shxPath, content: cli.readFile(shxPath, null, cache)};
     }
     if (!obj.dbf && cli.isFile(dbfPath, cache)) {
-      obj.dbf = {filename: dbfPath, content: cli.readFile(dbfPath, null, cache)};
+      // obj.dbf = {filename: dbfPath, content: cli.readFile(dbfPath, null, cache)};
+      obj.dbf = {
+        filename: dbfPath,
+        content: (cache && (dbfPath in cache)) ? cli.readFile(dbfPath, null, cache) : null
+      };
     }
     if (obj.dbf && cli.isFile(cpgPath, cache)) {
       obj.cpg = {filename: cpgPath, content: cli.readFile(cpgPath, 'utf-8', cache).trim()};
@@ -24037,6 +24070,7 @@ ${svg}
     clearStash();  // prevent errors from overwriting stash
     stashVar('current_command', cmd.name);
     stashVar('DEBUG', job.settings.DEBUG || cmd.debug);
+    stashVar('VERBOSE', job.settings.VERBOSE || cmd.verbose);
     stashVar('QUIET', job.settings.QUIET || cmd.quiet);
     stashVar('defs', job.defs);
     stashVar('input_files', job.input_files);
@@ -30463,6 +30497,9 @@ ${svg}
       targetDataset.arcs = mergedDataset.arcs;
       // dissolve clip layer shapes (to remove overlaps and other topological issues
       // that might confuse the clipping function)
+      // use a data-free copy of the clip lyr, so data records are not dissolved
+      // (this avoids triggering an unnecessary and expensive DBF read operation in some cases).
+      clipLyr = utils.defaults({data: null}, clipLyr);
       clipLyr = dissolvePolygonLayer2(clipLyr, mergedDataset, {quiet: true, silent: true});
 
     } else {
@@ -30864,6 +30901,8 @@ ${svg}
     getColorizerFunction: getColorizerFunction
   });
 
+  cmd.comment = function() {}; // no-op, so -comment doesn't trigger a parsing error
+
   function expressionUsesGeoJSON(exp) {
     return exp.includes('this.geojson');
   }
@@ -39778,7 +39817,7 @@ ${svg}
   function commandAcceptsEmptyTarget(name) {
     return name == 'graticule' || name == 'i' || name == 'help' ||
       name == 'point-grid' || name == 'shape' || name == 'rectangle' ||
-      name == 'include' || name == 'print' || name == 'if' || name == 'elif' ||
+      name == 'include' || name == 'print' || name == 'comment' || name == 'if' || name == 'elif' ||
       name == 'else' || name == 'endif';
   }
 
@@ -39886,6 +39925,9 @@ ${svg}
       } else if (name == 'colorizer') {
         outputLayers = cmd.colorizer(opts);
 
+      } else if (name == 'comment') {
+        // no-op
+
       } else if (name == 'dashlines') {
         applyCommandToEachLayer(cmd.dashlines, targetLayers, targetDataset, opts);
 

package/www/page.css

@@ -222,7 +222,7 @@ body {
 }
 
 .export-options .option-menu .advanced-options {
-	width: 175px;
+	width: 195px;
 }
 
 /* --- Main area ------------- */
@@ -456,11 +456,12 @@ body.dragover #import-options-drop-area .drop-area {
 .info-box,.popup {
 	border-radius: 9px;
 	/* border: 1px solid #ddd; */
-	box-shadow: 0 4px 6px rgba(0,0,0,0.35);
+	box-shadow: 0 3px 6px rgba(0,0,0,0.35);
 	background-color: #fff;
 }
 
 .info-box {
+	min-width: 230px;
 	word-wrap: break-word;
 	text-align: left;
 	margin-top: 12px;
@@ -472,13 +473,16 @@ body.dragover #import-options-drop-area .drop-area {
 }
 
 .info-box h3 {
-	font-size: 1.2em;
+	font-size: 19px;
+	line-height: 1.1;
 	padding: 0;
-	margin: 0 0 0.2em 0;
+	margin: 0 0 0.3em 0;
+	font-weight: normal;
 }
 
 .info-box h4 {
 	font-size: 1em;
+	font-weight: normal;
 	margin: 0 0 1px 0;
 }
 
@@ -491,9 +495,11 @@ body.dragover #import-options-drop-area .drop-area {
 	padding: 0 0 9px 0;
 }
 
-.option-menu input {
+.option-menu input[type="radio"],
+.option-menu input[type="checkbox"] 
+{
 	position: relative;
-	top: 2px;
+	top: 1px;
 	width: 12px;
 	height: 12px;
 }
@@ -720,6 +726,28 @@ body.simplify .layer-control-btn {
 
 img.close-btn {
 	opacity: 0.2;
+	position: absolute;
+	top: 8px;
+	right: 8px;
+	z-index: 2;
+}
+
+.close2-btn {
+	display: inline-block;
+	background-color: #f1f1f1;
+	background-image: url("images/close2.png");
+	background-size: cover;
+	float: right;
+	height:  18px;
+	width:  18px;
+	margin-top: 2px;
+	margin-right: -5px;
+	cursor: pointer;
+	border-radius: 4px;
+}
+
+.close2-btn:hover {
+	background-color: #ddd;
 }
 
 .pin-all img {
@@ -1016,6 +1044,7 @@ img.close-btn:hover,
 }
 
 .basemap-styles {
+	margin: 0 -2px;
 	white-space: nowrap;
 }
 
@@ -1040,19 +1069,14 @@ img.close-btn:hover,
 
 .basemap-styles > div:nth-child(even) {
 	position: relative;
-	left:  6px;
-	margin-left: 2px;
+	margin-left: 10px;
 }
 
 .basemap-style-btn img {
 	display: block;
-	left: -2px;
 	position:  relative;
 	border: 2px solid transparent;
-	/*width:  120px;
-	height:  90px;*/
-	width:  96px;
-	height:  72px;
+	width:  108px;
 }
 
 div.basemap-style-btn.active img {
@@ -1064,6 +1088,10 @@ div.basemap-style-btn.active img {
 	border: 2px solid #ead683;
 }
 
+.basemap-style-label {
+	margin-left: 2px;
+}
+
 .mapbox-improve-map {
 	display: none;
 }