mapshaper 0.5.114 → 0.5.117

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -1,3 +1,12 @@
1
+ v0.5.117
2
+ * Support reading DBF files larger than 2GB (command line program only).
3
+
4
+ v0.5.116
5
+ * Prevent 0-length DBF string fields, for interoperability with PostGIS.
6
+
7
+ v0.5.115
8
+ * Scalebar tweaks.
9
+
1
10
  v0.5.114
2
11
  * Bug fixes
3
12
 
package/mapshaper.js CHANGED
@@ -1,6 +1,6 @@
1
1
  (function () {
2
2
 
3
- var VERSION = "0.5.114";
3
+ var VERSION = "0.5.117";
4
4
 
5
5
 
6
6
  var utils = /*#__PURE__*/Object.freeze({
@@ -14942,6 +14942,11 @@
14942
14942
  var dx = pos.halign == 'right' ? frame.width - width - pos.hoffs : pos.hoffs;
14943
14943
  var dy = pos.valign == 'bottom' ? frame.height - height - pos.voffs : pos.voffs;
14944
14944
 
14945
+ if (!frame.crs) {
14946
+ message('Unable to render a scalebar: unknown CRS.');
14947
+ return [];
14948
+ }
14949
+
14945
14950
  if (labelPos == 'top') {
14946
14951
  anchorY = -labelOffs;
14947
14952
  dy += Math.round(labelOffs + fontSize * 0.8);
@@ -14982,10 +14987,11 @@
14982
14987
  }
14983
14988
 
14984
14989
  function getAutoScalebarLabel(mapWidth, metersPerPx) {
14985
- var minWidth = 75; // 100; // TODO: vary min size based on map width
14990
+ var minWidth = 70; // 100; // TODO: vary min size based on map width
14986
14991
  var minKm = metersPerPx * minWidth / 1000;
14987
- var options = ('1/8 1/5 1/4 1/2 1 1.5 2 3 4 5 8 10 12 15 20 25 30 40 50 75 ' +
14988
- '100 150 200 250 300 350 400 500 750 1,000 1,200 1,500 2,000 ' +
14992
+ // note: removed 1.5 12 and 1,200
14993
+ var options = ('1/8 1/5 1/4 1/2 1 2 3 4 5 8 10 15 20 25 30 40 50 75 ' +
14994
+ '100 150 200 250 300 350 400 500 750 1,000 1,500 2,000 ' +
14989
14995
  '2,500 3,000 4,000 5,000').split(' ');
14990
14996
  return options.reduce(function(memo, str) {
14991
14997
  if (memo) return memo;
@@ -16000,7 +16006,9 @@ ${svg}
16000
16006
 
16001
16007
  function initStringField(info, arr, name, encoding) {
16002
16008
  var formatter = encoding == 'ascii' ? encodeValueAsAscii : getStringWriterEncoded(encoding);
16003
- var size = 0;
16009
+ // Set minimum field size to 1 byte, for interoperability with PostGIS
16010
+ // (see https://github.com/mbloch/mapshaper/issues/541)
16011
+ var size = 1;
16004
16012
  var truncated = 0;
16005
16013
  var buffers = arr.map(function(rec) {
16006
16014
  var strval = convertValueToString(rec[name]);
@@ -17620,7 +17628,9 @@ ${svg}
17620
17628
  binArr, buf;
17621
17629
 
17622
17630
  this.readToBinArray = function(start, length) {
17623
- if (bufSize < start + length) error("Out-of-range error");
17631
+ if (bufSize < start + length) {
17632
+ error("Out-of-range error");
17633
+ }
17624
17634
  if (!binArr) binArr = new BinArray(src);
17625
17635
  binArr.position(start);
17626
17636
  return binArr;
@@ -21129,6 +21139,9 @@ ${svg}
21129
21139
  // http://www.digitalpreservation.gov/formats/fdd/fdd000325.shtml
21130
21140
  // http://www.clicketyclick.dk/databases/xbase/format/index.html
21131
21141
  // http://www.clicketyclick.dk/databases/xbase/format/data_types.html
21142
+ // esri docs:
21143
+ // https://support.esri.com/en/technical-article/000007920
21144
+ // https://desktop.arcgis.com/en/arcmap/latest/manage-data/shapefiles/geoprocessing-considerations-for-shapefile-output.htm
21132
21145
 
21133
21146
  // source: http://webhelp.esri.com/arcpad/8.0/referenceguide/index.htm#locales/task_code.htm
21134
21147
  var languageIds = [0x01,'437',0x02,'850',0x03,'1252',0x08,'865',0x09,'437',0x0A,'850',0x0B,'437',0x0D,'437',0x0E,'850',0x0F,'437',0x10,'850',0x11,'437',0x12,'850',0x13,'932',0x14,'850',0x15,'437',0x16,'850',0x17,'865',0x18,'437',0x19,'437',0x1A,'850',0x1B,'437',0x1C,'863',0x1D,'850',0x1F,'852',0x22,'852',0x23,'852',0x24,'860',0x25,'850',0x26,'866',0x37,'850',0x40,'852',0x4D,'936',0x4E,'949',0x4F,'950',0x50,'874',0x57,'1252',0x58,'1252',0x59,'1252',0x64,'852',0x65,'866',0x66,'865',0x67,'861',0x6A,'737',0x6B,'857',0x6C,'863',0x78,'950',0x79,'949',0x7A,'936',0x7B,'932',0x7C,'874',0x86,'737',0x87,'852',0x88,'857',0xC8,'1250',0xC9,'1251',0xCA,'1254',0xCB,'1253',0xCC,'1257'];
@@ -21265,11 +21278,12 @@ ${svg}
21265
21278
  // @src is a Buffer or ArrayBuffer or filename
21266
21279
  //
21267
21280
  function DbfReader(src, encodingArg) {
21268
- if (utils.isString(src)) {
21269
- error("[DbfReader] Expected a buffer, not a string");
21270
- }
21271
- var bin = new BinArray(src);
21272
- var header = readHeader(bin);
21281
+ var opts = {
21282
+ cacheSize: 0x2000000, // 32MB
21283
+ bufferSize: 0x400000 // 4MB
21284
+ };
21285
+ var dbfFile = utils.isString(src) ? new FileReader(src, opts) : new BufferReader(src);
21286
+ var header = readHeader(dbfFile);
21273
21287
 
21274
21288
  // encoding and fields are set on first access
21275
21289
  var fields;
@@ -21285,7 +21299,11 @@ ${svg}
21285
21299
 
21286
21300
  this.getFields = getFieldNames;
21287
21301
 
21288
- this.getBuffer = function() {return bin.buffer();};
21302
+ // TODO: switch to streaming output under Node.js
21303
+ this.getBuffer = function() {
21304
+ return dbfFile.readSync(0, dbfFile.size());
21305
+ // return bin.buffer();
21306
+ };
21289
21307
 
21290
21308
  this.deleteField = function(f) {
21291
21309
  prepareToRead();
@@ -21326,7 +21344,10 @@ ${svg}
21326
21344
  });
21327
21345
  }
21328
21346
 
21329
- function readHeader(bin) {
21347
+ function readHeader(reader) {
21348
+ // fetch enough bytes to accomodate any header
21349
+ var maxHeaderLen = 32 * 256 + 1; // 255 fields * fieldRecSize + headerRecSize + terminator
21350
+ var bin = reader.readToBinArray(0, Math.min(maxHeaderLen, reader.size()));
21330
21351
  bin.position(0).littleEndian();
21331
21352
  var header = {
21332
21353
  version: bin.readInt8(),
@@ -21412,30 +21433,32 @@ ${svg}
21412
21433
  return new Function('return {' + args.join(',') + '};');
21413
21434
  }
21414
21435
 
21415
- function findEofPos(bin) {
21416
- var pos = bin.size() - 1;
21417
- if (bin.peek(pos) != 0x1A) { // last byte may or may not be EOF
21418
- pos++;
21419
- }
21420
- return pos;
21421
- }
21436
+ // function findEofPos(bin) {
21437
+ // var pos = bin.size() - 1;
21438
+ // if (bin.peek(pos) != 0x1A) { // last byte may or may not be EOF
21439
+ // pos++;
21440
+ // }
21441
+ // return pos;
21442
+ // }
21422
21443
 
21423
21444
  function getRecordReader() {
21424
21445
  prepareToRead();
21425
21446
  var readers = fields.map(getFieldReader),
21426
- eofOffs = findEofPos(bin),
21427
21447
  create = getRecordConstructor(),
21428
21448
  values = [];
21429
21449
 
21430
21450
  return function readRow(r) {
21431
- var offs = getRowOffset(r),
21451
+ var bin = dbfFile.readToBinArray(getRowOffset(r), header.recordSize),
21452
+ rowOffs = bin.position(),
21432
21453
  fieldOffs, field;
21454
+ if (bin.bytesLeft() < header.recordSize ||
21455
+ bin.bytesLeft() == header.recordSize && bin.peek(bin.size() - 1) == 0x1A) {
21456
+ // check for observed data error: last data byte contains EOF
21457
+ stop('Invalid DBF file: encountered end-of-file while reading data');
21458
+ }
21433
21459
  for (var c=0, cols=fields.length; c<cols; c++) {
21434
21460
  field = fields[c];
21435
- fieldOffs = offs + field.columnOffset;
21436
- if (fieldOffs + field.size > eofOffs) {
21437
- stop('Invalid DBF file: encountered end-of-file while reading data');
21438
- }
21461
+ fieldOffs = rowOffs + field.columnOffset;
21439
21462
  bin.position(fieldOffs);
21440
21463
  values[c] = readers[c](bin, field.size);
21441
21464
  }
@@ -21525,14 +21548,17 @@ ${svg}
21525
21548
  var maxSamples = 50;
21526
21549
  var buf = utils.createBuffer(256);
21527
21550
  var index = {};
21528
- var f, chars, sample, hash;
21551
+ var f, chars, sample, hash, bin, rowOffs;
21529
21552
  // include non-ascii field names, if any
21530
21553
  samples = getNonAsciiHeaders();
21531
21554
  for (var r=0; r<rows; r++) {
21555
+ bin = dbfFile.readToBinArray(getRowOffset(r), header.recordSize);
21556
+ rowOffs = bin.position();
21532
21557
  for (var c=0; c<cols; c++) {
21533
21558
  if (samples.length >= maxSamples) break;
21534
21559
  f = stringFields[c];
21535
- bin.position(getRowOffset(r) + f.columnOffset);
21560
+ // bin.position(getRowOffset(r) + f.columnOffset);
21561
+ bin.position(rowOffs + f.columnOffset);
21536
21562
  chars = readStringBytes(bin, f.size, buf);
21537
21563
  if (chars > 0 && bufferContainsHighBit(buf, chars)) {
21538
21564
  sample = utils.createBuffer(buf.slice(0, chars)); //
@@ -21548,9 +21574,9 @@ ${svg}
21548
21574
  }
21549
21575
  }
21550
21576
 
21551
- function importDbfTable(buf, o) {
21577
+ function importDbfTable(src, o) {
21552
21578
  var opts = o || {};
21553
- return new ShapefileTable(buf, opts.encoding);
21579
+ return new ShapefileTable(src, opts.encoding);
21554
21580
  }
21555
21581
 
21556
21582
  // Implements the DataTable api for DBF file data.
@@ -21559,8 +21585,8 @@ ${svg}
21559
21585
  // just the shapes and exporting in Shapefile format.
21560
21586
  // TODO: consider accepting just the filename, so buffer doesn't consume memory needlessly.
21561
21587
  //
21562
- function ShapefileTable(buf, encoding) {
21563
- var reader = new DbfReader(buf, encoding),
21588
+ function ShapefileTable(src, encoding) {
21589
+ var reader = new DbfReader(src, encoding),
21564
21590
  altered = false,
21565
21591
  table;
21566
21592
 
@@ -21569,15 +21595,23 @@ ${svg}
21569
21595
  // export DBF records on first table access
21570
21596
  table = new DataTable(reader.readRows());
21571
21597
  reader = null;
21572
- buf = null; // null out references to DBF data for g.c.
21598
+ src = null; // null out references to DBF data for g.c.
21573
21599
  }
21574
21600
  return table;
21575
21601
  }
21576
21602
 
21577
21603
  this.exportAsDbf = function(opts) {
21578
- // export original dbf bytes if possible, for performance
21604
+ // export original dbf bytes if possible
21605
+ // (e.g. if the data attributes haven't changed)
21579
21606
  var useOriginal = !!reader && !altered && !opts.field_order && !opts.encoding;
21580
- if (useOriginal) return reader.getBuffer();
21607
+ if (useOriginal) {
21608
+ try {
21609
+ // Maximum Buffer in current Node.js is 2GB
21610
+ // We fall back to import-export if getBuffer() fails.
21611
+ // This may produce a buffer that does not exceed the maximum size.
21612
+ return reader.getBuffer();
21613
+ } catch(e) {}
21614
+ }
21581
21615
  return Dbf.exportRecords(getTable().getRecords(), opts.encoding, opts.field_order);
21582
21616
  };
21583
21617
 
@@ -23587,7 +23621,7 @@ ${svg}
23587
23621
  if (input.cpg && !opts.encoding) {
23588
23622
  opts.encoding = input.cpg.content;
23589
23623
  }
23590
- table = importDbfTable(input.dbf.content, opts);
23624
+ table = importDbfTable(input.dbf.content || input.dbf.filename, opts);
23591
23625
  return {
23592
23626
  info: {},
23593
23627
  layers: [{data: table}]
@@ -23685,18 +23719,10 @@ ${svg}
23685
23719
  content;
23686
23720
 
23687
23721
  cli.checkFileExists(path, cache);
23688
- if (fileType == 'shp' && !cached) {
23689
- // let ShpReader read the file (supports larger files)
23690
- content = null;
23691
-
23692
- } else if (fileType == 'json' && !cached) {
23693
- // postpone reading of JSON files, to support incremental parsing
23722
+ if ((fileType == 'shp' || fileType == 'json' || fileType == 'text' || fileType == 'dbf') && !cached) {
23723
+ // these file types are read incrementally
23694
23724
  content = null;
23695
23725
 
23696
- } else if (fileType == 'text' && !cached) {
23697
- // content = cli.readFile(path); // read from buffer
23698
- content = null; // read from file, to support largest files (see mapshaper-delim-import.js)
23699
-
23700
23726
  } else if (fileType && isSupportedBinaryInputType(path)) {
23701
23727
  content = cli.readFile(path, null, cache);
23702
23728
  if (utils.isString(content)) {
@@ -23753,7 +23779,11 @@ ${svg}
23753
23779
  obj.shx = {filename: shxPath, content: cli.readFile(shxPath, null, cache)};
23754
23780
  }
23755
23781
  if (!obj.dbf && cli.isFile(dbfPath, cache)) {
23756
- obj.dbf = {filename: dbfPath, content: cli.readFile(dbfPath, null, cache)};
23782
+ // obj.dbf = {filename: dbfPath, content: cli.readFile(dbfPath, null, cache)};
23783
+ obj.dbf = {
23784
+ filename: dbfPath,
23785
+ content: (cache && (dbfPath in cache)) ? cli.readFile(dbfPath, null, cache) : null
23786
+ };
23757
23787
  }
23758
23788
  if (obj.dbf && cli.isFile(cpgPath, cache)) {
23759
23789
  obj.cpg = {filename: cpgPath, content: cli.readFile(cpgPath, 'utf-8', cache).trim()};
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "mapshaper",
3
- "version": "0.5.114",
3
+ "version": "0.5.117",
4
4
  "description": "A tool for editing vector datasets for mapping and GIS.",
5
5
  "keywords": [
6
6
  "shapefile",
package/www/mapshaper.js CHANGED
@@ -1,6 +1,6 @@
1
1
  (function () {
2
2
 
3
- var VERSION = "0.5.114";
3
+ var VERSION = "0.5.117";
4
4
 
5
5
 
6
6
  var utils = /*#__PURE__*/Object.freeze({
@@ -14942,6 +14942,11 @@
14942
14942
  var dx = pos.halign == 'right' ? frame.width - width - pos.hoffs : pos.hoffs;
14943
14943
  var dy = pos.valign == 'bottom' ? frame.height - height - pos.voffs : pos.voffs;
14944
14944
 
14945
+ if (!frame.crs) {
14946
+ message('Unable to render a scalebar: unknown CRS.');
14947
+ return [];
14948
+ }
14949
+
14945
14950
  if (labelPos == 'top') {
14946
14951
  anchorY = -labelOffs;
14947
14952
  dy += Math.round(labelOffs + fontSize * 0.8);
@@ -14982,10 +14987,11 @@
14982
14987
  }
14983
14988
 
14984
14989
  function getAutoScalebarLabel(mapWidth, metersPerPx) {
14985
- var minWidth = 75; // 100; // TODO: vary min size based on map width
14990
+ var minWidth = 70; // 100; // TODO: vary min size based on map width
14986
14991
  var minKm = metersPerPx * minWidth / 1000;
14987
- var options = ('1/8 1/5 1/4 1/2 1 1.5 2 3 4 5 8 10 12 15 20 25 30 40 50 75 ' +
14988
- '100 150 200 250 300 350 400 500 750 1,000 1,200 1,500 2,000 ' +
14992
+ // note: removed 1.5 12 and 1,200
14993
+ var options = ('1/8 1/5 1/4 1/2 1 2 3 4 5 8 10 15 20 25 30 40 50 75 ' +
14994
+ '100 150 200 250 300 350 400 500 750 1,000 1,500 2,000 ' +
14989
14995
  '2,500 3,000 4,000 5,000').split(' ');
14990
14996
  return options.reduce(function(memo, str) {
14991
14997
  if (memo) return memo;
@@ -16000,7 +16006,9 @@ ${svg}
16000
16006
 
16001
16007
  function initStringField(info, arr, name, encoding) {
16002
16008
  var formatter = encoding == 'ascii' ? encodeValueAsAscii : getStringWriterEncoded(encoding);
16003
- var size = 0;
16009
+ // Set minimum field size to 1 byte, for interoperability with PostGIS
16010
+ // (see https://github.com/mbloch/mapshaper/issues/541)
16011
+ var size = 1;
16004
16012
  var truncated = 0;
16005
16013
  var buffers = arr.map(function(rec) {
16006
16014
  var strval = convertValueToString(rec[name]);
@@ -17620,7 +17628,9 @@ ${svg}
17620
17628
  binArr, buf;
17621
17629
 
17622
17630
  this.readToBinArray = function(start, length) {
17623
- if (bufSize < start + length) error("Out-of-range error");
17631
+ if (bufSize < start + length) {
17632
+ error("Out-of-range error");
17633
+ }
17624
17634
  if (!binArr) binArr = new BinArray(src);
17625
17635
  binArr.position(start);
17626
17636
  return binArr;
@@ -21129,6 +21139,9 @@ ${svg}
21129
21139
  // http://www.digitalpreservation.gov/formats/fdd/fdd000325.shtml
21130
21140
  // http://www.clicketyclick.dk/databases/xbase/format/index.html
21131
21141
  // http://www.clicketyclick.dk/databases/xbase/format/data_types.html
21142
+ // esri docs:
21143
+ // https://support.esri.com/en/technical-article/000007920
21144
+ // https://desktop.arcgis.com/en/arcmap/latest/manage-data/shapefiles/geoprocessing-considerations-for-shapefile-output.htm
21132
21145
 
21133
21146
  // source: http://webhelp.esri.com/arcpad/8.0/referenceguide/index.htm#locales/task_code.htm
21134
21147
  var languageIds = [0x01,'437',0x02,'850',0x03,'1252',0x08,'865',0x09,'437',0x0A,'850',0x0B,'437',0x0D,'437',0x0E,'850',0x0F,'437',0x10,'850',0x11,'437',0x12,'850',0x13,'932',0x14,'850',0x15,'437',0x16,'850',0x17,'865',0x18,'437',0x19,'437',0x1A,'850',0x1B,'437',0x1C,'863',0x1D,'850',0x1F,'852',0x22,'852',0x23,'852',0x24,'860',0x25,'850',0x26,'866',0x37,'850',0x40,'852',0x4D,'936',0x4E,'949',0x4F,'950',0x50,'874',0x57,'1252',0x58,'1252',0x59,'1252',0x64,'852',0x65,'866',0x66,'865',0x67,'861',0x6A,'737',0x6B,'857',0x6C,'863',0x78,'950',0x79,'949',0x7A,'936',0x7B,'932',0x7C,'874',0x86,'737',0x87,'852',0x88,'857',0xC8,'1250',0xC9,'1251',0xCA,'1254',0xCB,'1253',0xCC,'1257'];
@@ -21265,11 +21278,12 @@ ${svg}
21265
21278
  // @src is a Buffer or ArrayBuffer or filename
21266
21279
  //
21267
21280
  function DbfReader(src, encodingArg) {
21268
- if (utils.isString(src)) {
21269
- error("[DbfReader] Expected a buffer, not a string");
21270
- }
21271
- var bin = new BinArray(src);
21272
- var header = readHeader(bin);
21281
+ var opts = {
21282
+ cacheSize: 0x2000000, // 32MB
21283
+ bufferSize: 0x400000 // 4MB
21284
+ };
21285
+ var dbfFile = utils.isString(src) ? new FileReader(src, opts) : new BufferReader(src);
21286
+ var header = readHeader(dbfFile);
21273
21287
 
21274
21288
  // encoding and fields are set on first access
21275
21289
  var fields;
@@ -21285,7 +21299,11 @@ ${svg}
21285
21299
 
21286
21300
  this.getFields = getFieldNames;
21287
21301
 
21288
- this.getBuffer = function() {return bin.buffer();};
21302
+ // TODO: switch to streaming output under Node.js
21303
+ this.getBuffer = function() {
21304
+ return dbfFile.readSync(0, dbfFile.size());
21305
+ // return bin.buffer();
21306
+ };
21289
21307
 
21290
21308
  this.deleteField = function(f) {
21291
21309
  prepareToRead();
@@ -21326,7 +21344,10 @@ ${svg}
21326
21344
  });
21327
21345
  }
21328
21346
 
21329
- function readHeader(bin) {
21347
+ function readHeader(reader) {
21348
+ // fetch enough bytes to accomodate any header
21349
+ var maxHeaderLen = 32 * 256 + 1; // 255 fields * fieldRecSize + headerRecSize + terminator
21350
+ var bin = reader.readToBinArray(0, Math.min(maxHeaderLen, reader.size()));
21330
21351
  bin.position(0).littleEndian();
21331
21352
  var header = {
21332
21353
  version: bin.readInt8(),
@@ -21412,30 +21433,32 @@ ${svg}
21412
21433
  return new Function('return {' + args.join(',') + '};');
21413
21434
  }
21414
21435
 
21415
- function findEofPos(bin) {
21416
- var pos = bin.size() - 1;
21417
- if (bin.peek(pos) != 0x1A) { // last byte may or may not be EOF
21418
- pos++;
21419
- }
21420
- return pos;
21421
- }
21436
+ // function findEofPos(bin) {
21437
+ // var pos = bin.size() - 1;
21438
+ // if (bin.peek(pos) != 0x1A) { // last byte may or may not be EOF
21439
+ // pos++;
21440
+ // }
21441
+ // return pos;
21442
+ // }
21422
21443
 
21423
21444
  function getRecordReader() {
21424
21445
  prepareToRead();
21425
21446
  var readers = fields.map(getFieldReader),
21426
- eofOffs = findEofPos(bin),
21427
21447
  create = getRecordConstructor(),
21428
21448
  values = [];
21429
21449
 
21430
21450
  return function readRow(r) {
21431
- var offs = getRowOffset(r),
21451
+ var bin = dbfFile.readToBinArray(getRowOffset(r), header.recordSize),
21452
+ rowOffs = bin.position(),
21432
21453
  fieldOffs, field;
21454
+ if (bin.bytesLeft() < header.recordSize ||
21455
+ bin.bytesLeft() == header.recordSize && bin.peek(bin.size() - 1) == 0x1A) {
21456
+ // check for observed data error: last data byte contains EOF
21457
+ stop('Invalid DBF file: encountered end-of-file while reading data');
21458
+ }
21433
21459
  for (var c=0, cols=fields.length; c<cols; c++) {
21434
21460
  field = fields[c];
21435
- fieldOffs = offs + field.columnOffset;
21436
- if (fieldOffs + field.size > eofOffs) {
21437
- stop('Invalid DBF file: encountered end-of-file while reading data');
21438
- }
21461
+ fieldOffs = rowOffs + field.columnOffset;
21439
21462
  bin.position(fieldOffs);
21440
21463
  values[c] = readers[c](bin, field.size);
21441
21464
  }
@@ -21525,14 +21548,17 @@ ${svg}
21525
21548
  var maxSamples = 50;
21526
21549
  var buf = utils.createBuffer(256);
21527
21550
  var index = {};
21528
- var f, chars, sample, hash;
21551
+ var f, chars, sample, hash, bin, rowOffs;
21529
21552
  // include non-ascii field names, if any
21530
21553
  samples = getNonAsciiHeaders();
21531
21554
  for (var r=0; r<rows; r++) {
21555
+ bin = dbfFile.readToBinArray(getRowOffset(r), header.recordSize);
21556
+ rowOffs = bin.position();
21532
21557
  for (var c=0; c<cols; c++) {
21533
21558
  if (samples.length >= maxSamples) break;
21534
21559
  f = stringFields[c];
21535
- bin.position(getRowOffset(r) + f.columnOffset);
21560
+ // bin.position(getRowOffset(r) + f.columnOffset);
21561
+ bin.position(rowOffs + f.columnOffset);
21536
21562
  chars = readStringBytes(bin, f.size, buf);
21537
21563
  if (chars > 0 && bufferContainsHighBit(buf, chars)) {
21538
21564
  sample = utils.createBuffer(buf.slice(0, chars)); //
@@ -21548,9 +21574,9 @@ ${svg}
21548
21574
  }
21549
21575
  }
21550
21576
 
21551
- function importDbfTable(buf, o) {
21577
+ function importDbfTable(src, o) {
21552
21578
  var opts = o || {};
21553
- return new ShapefileTable(buf, opts.encoding);
21579
+ return new ShapefileTable(src, opts.encoding);
21554
21580
  }
21555
21581
 
21556
21582
  // Implements the DataTable api for DBF file data.
@@ -21559,8 +21585,8 @@ ${svg}
21559
21585
  // just the shapes and exporting in Shapefile format.
21560
21586
  // TODO: consider accepting just the filename, so buffer doesn't consume memory needlessly.
21561
21587
  //
21562
- function ShapefileTable(buf, encoding) {
21563
- var reader = new DbfReader(buf, encoding),
21588
+ function ShapefileTable(src, encoding) {
21589
+ var reader = new DbfReader(src, encoding),
21564
21590
  altered = false,
21565
21591
  table;
21566
21592
 
@@ -21569,15 +21595,23 @@ ${svg}
21569
21595
  // export DBF records on first table access
21570
21596
  table = new DataTable(reader.readRows());
21571
21597
  reader = null;
21572
- buf = null; // null out references to DBF data for g.c.
21598
+ src = null; // null out references to DBF data for g.c.
21573
21599
  }
21574
21600
  return table;
21575
21601
  }
21576
21602
 
21577
21603
  this.exportAsDbf = function(opts) {
21578
- // export original dbf bytes if possible, for performance
21604
+ // export original dbf bytes if possible
21605
+ // (e.g. if the data attributes haven't changed)
21579
21606
  var useOriginal = !!reader && !altered && !opts.field_order && !opts.encoding;
21580
- if (useOriginal) return reader.getBuffer();
21607
+ if (useOriginal) {
21608
+ try {
21609
+ // Maximum Buffer in current Node.js is 2GB
21610
+ // We fall back to import-export if getBuffer() fails.
21611
+ // This may produce a buffer that does not exceed the maximum size.
21612
+ return reader.getBuffer();
21613
+ } catch(e) {}
21614
+ }
21581
21615
  return Dbf.exportRecords(getTable().getRecords(), opts.encoding, opts.field_order);
21582
21616
  };
21583
21617
 
@@ -23587,7 +23621,7 @@ ${svg}
23587
23621
  if (input.cpg && !opts.encoding) {
23588
23622
  opts.encoding = input.cpg.content;
23589
23623
  }
23590
- table = importDbfTable(input.dbf.content, opts);
23624
+ table = importDbfTable(input.dbf.content || input.dbf.filename, opts);
23591
23625
  return {
23592
23626
  info: {},
23593
23627
  layers: [{data: table}]
@@ -23685,18 +23719,10 @@ ${svg}
23685
23719
  content;
23686
23720
 
23687
23721
  cli.checkFileExists(path, cache);
23688
- if (fileType == 'shp' && !cached) {
23689
- // let ShpReader read the file (supports larger files)
23690
- content = null;
23691
-
23692
- } else if (fileType == 'json' && !cached) {
23693
- // postpone reading of JSON files, to support incremental parsing
23722
+ if ((fileType == 'shp' || fileType == 'json' || fileType == 'text' || fileType == 'dbf') && !cached) {
23723
+ // these file types are read incrementally
23694
23724
  content = null;
23695
23725
 
23696
- } else if (fileType == 'text' && !cached) {
23697
- // content = cli.readFile(path); // read from buffer
23698
- content = null; // read from file, to support largest files (see mapshaper-delim-import.js)
23699
-
23700
23726
  } else if (fileType && isSupportedBinaryInputType(path)) {
23701
23727
  content = cli.readFile(path, null, cache);
23702
23728
  if (utils.isString(content)) {
@@ -23753,7 +23779,11 @@ ${svg}
23753
23779
  obj.shx = {filename: shxPath, content: cli.readFile(shxPath, null, cache)};
23754
23780
  }
23755
23781
  if (!obj.dbf && cli.isFile(dbfPath, cache)) {
23756
- obj.dbf = {filename: dbfPath, content: cli.readFile(dbfPath, null, cache)};
23782
+ // obj.dbf = {filename: dbfPath, content: cli.readFile(dbfPath, null, cache)};
23783
+ obj.dbf = {
23784
+ filename: dbfPath,
23785
+ content: (cache && (dbfPath in cache)) ? cli.readFile(dbfPath, null, cache) : null
23786
+ };
23757
23787
  }
23758
23788
  if (obj.dbf && cli.isFile(cpgPath, cache)) {
23759
23789
  obj.cpg = {filename: cpgPath, content: cli.readFile(cpgPath, 'utf-8', cache).trim()};