@dra2020/baseclient 1.0.170 → 1.0.172

package/dist/poly/polybin.d.ts

@@ -1,5 +1,25 @@
 import * as Util from '../util/all';
-export declare function packCollection(coder: Util.Coder, col: any): ArrayBuffer;
+/**
+ * Serialize a collection to the geopack format.
+ *
+ * The file wants one contiguous coordinate section with every feature's offset an index into it, and
+ * that used to be arranged by requiring the whole collection to already sit in ONE buffer at exactly
+ * those offsets - packCollection simply blitted that buffer across whole. Which held only as long as
+ * nothing had touched the collection since: pack a collection, add a feature, pack again, and the
+ * second pack either redid all the work or produced offsets into a buffer that no longer described it.
+ *
+ * So the coordinates are gathered feature by feature instead. Each one is given its place in the
+ * output, and its floats are copied from wherever they happen to live now - one buffer or several. The
+ * work is the same (every float is copied either way), the assumption is gone, and featurePack is free
+ * to leave already-packed features alone.
+ *
+ * THE FORMAT IS UNCHANGED. Offsets are assigned in feature order from zero, which is exactly what a
+ * single shared buffer produced, so files written before and after this are byte-identical and each
+ * reads under the other.
+ *
+ * `omit` names properties to leave out of the serialized JSON - see the note on collectOmitted.
+ */
+export declare function packCollection(coder: Util.Coder, col: any, omit?: string[]): ArrayBuffer;
 export declare function unpackCollection(coder: Util.Coder, ab: ArrayBuffer): any;
 export declare function topoToBuffer(coder: Util.Coder, topo: any): ArrayBuffer;
 export declare function topoFromBuffer(coder: Util.Coder, ab: ArrayBuffer): any;

package/dist/poly/polypack.d.ts

@@ -19,8 +19,42 @@ export declare function polyPackTopoArcs(topo: any, arcs: any, iPoly?: number):
 export declare function polyPack(coords: any, prepack?: PolyPack): PolyPack;
 export declare function polyPackCopy(coords: any): PolyPack;
 export declare function polyUnpack(prepack: any): any;
+/** Whether this feature's coordinates are already in packed form. */
+export declare function featureIsPacked(f: any): boolean;
+/**
+ * Floats this feature would ADD to a buffer if packed now: zero if it is already packed, and zero for
+ * the shapes that carry no packed coordinates at all (points, and features with no geometry).
+ */
 export declare function featurePackSize(f: any): number;
-export declare function featurePack(f: any, prepack?: PolyPack): any;
+/**
+ * Repair packed buffers that have been through JSON and come back as plain objects.
+ *
+ * A Float64Array does not survive JSON.stringify/parse - it comes back as {"0":n,"1":n,...} - so a
+ * collection that has been round-tripped through JSON while packed arrives with something that looks
+ * packed but cannot be read. This puts the numbers back into a real Float64Array.
+ *
+ * Split out of featurePack, where it used to sit inline in the middle of the packing path, because it
+ * is not packing: it is a repair for a specific way collections arrive damaged, it applies to
+ * collections that are ALREADY packed, and burying it inside meant featurePack had to return early
+ * with a different kind of value to signal it had happened. Returns whether anything was repaired.
+ */
+export declare function repairPackedBuffers(col: any): boolean;
+/**
+ * Ensure a feature or collection is fully packed, and answer with the object handed in.
+ *
+ * ALWAYS the same object, whatever was passed and whatever it had to do. The previous signature
+ * returned three different things - a PolyPack for a feature, a PolyPack for a collection it packed,
+ * and the collection itself for one that was already packed - so a caller could not use the result
+ * without knowing which case it had fallen into, and every caller but one ignored it. The one that
+ * did not assigned a PolyPack to geometry.coordinates.
+ *
+ * A collection packs its UNPACKED features into one new buffer and leaves packed ones where they are.
+ * That is what stops a second call re-packing work already done, which for a large collection means
+ * allocating and copying the whole coordinate set again for nothing. Features already packed keep
+ * whatever buffer they were in - the collection may end up spanning several - and packCollection no
+ * longer minds, because it copies feature by feature rather than assuming one buffer holds them all.
+ */
+export declare function featurePack(f: any): any;
 export declare function featureUnpack(f: any): any;
 export type SavePack = WeakMap<any, any>;
 export declare function featureUnpackTemporarily(f: any, save?: SavePack): SavePack;

package/dist/poly/topo.d.ts

@@ -5,6 +5,15 @@ import * as Q from './quad';
 export type Topo = any;
 export declare function topoFromCollection(col: any): Topo;
 export declare function topoContiguity(topo: Topo): any;
+/** Hits are geometry that would otherwise have been rebuilt from arcs; misses are the rebuilds that
+ *  were actually necessary. That distinction is the point - a raw conversion count cannot tell them
+ *  apart. Call as Poly.topoFeatureCacheStats() from the console. */
+export declare function topoFeatureCacheStats(): {
+    hits: number;
+    misses: number;
+    pct: number;
+};
+export declare function topoFeatureCacheReset(): void;
 export declare function topoToFeature(topo: Topo, geoid: string): any;
 export type TopoSpliceEntry = {
     topology: Topo;

package/lib/poly/polybin.ts

@@ -50,20 +50,109 @@ function unpackString(coder: Util.Coder, buf8: Uint8Array, buf32: Int32Array, of
   return s;
 }
 
-export function packCollection(coder: Util.Coder, col: any): ArrayBuffer
+/**
+ * Properties temporarily removed for the duration of a pack, and where they came from.
+ *
+ * WHY THIS IS WORTH HAVING. The JSON in a geopack is JSON.stringify of the collection, so anything
+ * hanging off a feature goes into the file. A Float64Array does not survive that: it serializes as
+ * {"0":n,"1":n,...}, which is both enormous and unreadable on the way back - the same damage
+ * repairPackedBuffers exists to undo for coordinates. PackedFields are exactly such arrays, and they
+ * are DERIVED, so writing them is waste even when it works. Naming them here means a caller cannot
+ * forget, and does not have to strip and rebuild them around every call.
+ */
+interface OmittedProps
 {
-  // Compute size
-  let pp = PP.featurePack(col) as PP.PolyPack;
-  let f: any = col.features.find((f: any) => { return f.geometry.packed ? f.geometry.packed.buffer : null });
-  let buffer: any = f ? f.geometry.packed.buffer : null;
-  let size = 16; // int endiness, offset to coordinates, float endiness
-  col.features.forEach((f: any) => { if (f.geometry.packed) delete f.geometry.packed.buffer; }); // reconstructed when unpacking
+  o: any;
+  name: string;
+  value: any;
+}
+
+function collectOmitted(col: any, omit?: string[]): OmittedProps[]
+{
+  let saved: OmittedProps[] = [];
+  if (! omit || omit.length === 0) return saved;
+
+  const take = (o: any) => {
+      if (! o) return;
+      omit.forEach(name => {
+          if (o[name] !== undefined) { saved.push({ o, name, value: o[name] }); delete o[name] }
+        });
+    };
+
+  take(col);
+  if (col.features) col.features.forEach((f: any) => { take(f); take(f.properties) });
+  return saved;
+}
+
+function restoreOmitted(saved: OmittedProps[]): void
+{
+  saved.forEach(s => { s.o[s.name] = s.value });
+}
+
+/** One feature's coordinates during a pack: where they are now, and where they are going. */
+interface PackSlot
+{
+  packed: any;        // the feature's geometry.packed
+  buffer: any;        // its buffer, to be put back afterwards
+  fromOffset: number; // its index within that buffer
+  toOffset: number;   // its index within the file's coordinate section
+  length: number;
+}
+
+/**
+ * Serialize a collection to the geopack format.
+ *
+ * The file wants one contiguous coordinate section with every feature's offset an index into it, and
+ * that used to be arranged by requiring the whole collection to already sit in ONE buffer at exactly
+ * those offsets - packCollection simply blitted that buffer across whole. Which held only as long as
+ * nothing had touched the collection since: pack a collection, add a feature, pack again, and the
+ * second pack either redid all the work or produced offsets into a buffer that no longer described it.
+ *
+ * So the coordinates are gathered feature by feature instead. Each one is given its place in the
+ * output, and its floats are copied from wherever they happen to live now - one buffer or several. The
+ * work is the same (every float is copied either way), the assumption is gone, and featurePack is free
+ * to leave already-packed features alone.
+ *
+ * THE FORMAT IS UNCHANGED. Offsets are assigned in feature order from zero, which is exactly what a
+ * single shared buffer produced, so files written before and after this are byte-identical and each
+ * reads under the other.
+ *
+ * `omit` names properties to leave out of the serialized JSON - see the note on collectOmitted.
+ */
+export function packCollection(coder: Util.Coder, col: any, omit?: string[]): ArrayBuffer
+{
+  // Which features arrive unpacked, so they can be left that way. Packing is how the coordinates get
+  // written; it is not something the caller asked for, and a caller that handed over an unpacked
+  // collection and got a packed one back would find its own coordinates gone. Equally, a caller whose
+  // collection was ALREADY packed must not have it unpacked - that rebuilds every coordinate array
+  // for nothing, which on a large collection is the memory this change exists to stop spending.
+  let wasUnpacked: any[] = [];
+  if (col.features) col.features.forEach((f: any) => { if (! PP.featureIsPacked(f)) wasUnpacked.push(f) });
+
+  PP.featurePack(col);
+
+  // Where every feature's coordinates are now, and where they will be in the file.
+  let slots: PackSlot[] = [];
+  let nFloats = 0;
+  col.features.forEach((f: any) => {
+      const packed = f.geometry ? f.geometry.packed : null;
+      if (! packed) return;
+      slots.push({ packed, buffer: packed.buffer, fromOffset: packed.offset, toOffset: nFloats,
+                   length: packed.length });
+      nFloats += packed.length;
+    });
+
+  // The JSON carries each feature's offset, so the offsets have to be the file's before it is built.
+  // Buffers come out because they are reconstructed on the way back in.
+  slots.forEach(s => { s.packed.offset = s.toOffset; delete s.packed.buffer });
+  const omitted = collectOmitted(col, omit);
+
+  let size = 16; // int endianness, offset to coordinates, float endianness
   let j = JSON.stringify(col);
   size += sizeOfString(coder, j);
   size += pad(size, 8);
-  let fullsize = size + pp.length * 8;  // add space for coordinates
+  let fullsize = size + nFloats * 8;
 
-  // Now pack it
   let ab = new ArrayBuffer(fullsize);
   let buf8 = new Uint8Array(ab);
   let buf32 = new Int32Array(ab);
@@ -79,14 +168,21 @@ export function packCollection(coder: Util.Coder, col: any): ArrayBuffer
   offset += pad(offset, 8);
   if (offset != size)
     throw 'Oops, packing error.';
-  let foff = offset >> 3;
-  let buf = pp.buffer as Float64Array;
-  for (let i: number = 0; i < pp.length; i++)
-    buf64[foff++] = buf[i];
-
-  // Now restore
-  col.features.forEach((f: any) => { if (f.geometry.packed) f.geometry.packed.buffer = buffer; });
-  PP.featureUnpack(col);
+
+  const foff = offset >> 3;
+  slots.forEach(s => {
+      const src = s.buffer as Float64Array;
+      if (! src) return;   // no coordinates: points, and features with no geometry
+      let to = foff + s.toOffset;
+      let from = s.fromOffset;
+      for (let i = 0; i < s.length; i++)
+        buf64[to++] = src[from++];
+    });
+
+  // Put the collection back exactly as it was found.
+  slots.forEach(s => { s.packed.offset = s.fromOffset; s.packed.buffer = s.buffer });
+  restoreOmitted(omitted);
+  wasUnpacked.forEach(f => PP.featureUnpack(f));
 
   return ab;
 }

package/lib/poly/polypack.ts

@@ -330,14 +330,58 @@ export function polyUnpack(prepack: any): any
   return coords.length > 1 ? coords : coords[0];
 }
 
+/** Whether this feature's coordinates are already in packed form. */
+export function featureIsPacked(f: any): boolean
+{
+  return !! (f && f.geometry && f.geometry.packed !== undefined);
+}
+
+/**
+ * Floats this feature would ADD to a buffer if packed now: zero if it is already packed, and zero for
+ * the shapes that carry no packed coordinates at all (points, and features with no geometry).
+ */
 export function featurePackSize(f: any): number
 {
+  if (featureIsPacked(f)) return 0;
   if (f && f.geometry && f.geometry.coordinates && f.geometry.type !== 'Point')
     return polyPackSize(f.geometry.coordinates);
   return 0;
 }
 
-export function featurePack(f: any, prepack?: PolyPack): any
+/**
+ * Repair packed buffers that have been through JSON and come back as plain objects.
+ *
+ * A Float64Array does not survive JSON.stringify/parse - it comes back as {"0":n,"1":n,...} - so a
+ * collection that has been round-tripped through JSON while packed arrives with something that looks
+ * packed but cannot be read. This puts the numbers back into a real Float64Array.
+ *
+ * Split out of featurePack, where it used to sit inline in the middle of the packing path, because it
+ * is not packing: it is a repair for a specific way collections arrive damaged, it applies to
+ * collections that are ALREADY packed, and burying it inside meant featurePack had to return early
+ * with a different kind of value to signal it had happened. Returns whether anything was repaired.
+ */
+export function repairPackedBuffers(col: any): boolean
+{
+  if (! col || ! col.features || col.features.length === 0) return false;
+
+  let ff: any = col.features[0];
+  if (! featureIsPacked(ff) || ArrayBuffer.isView(ff.geometry.packed.buffer)) return false;
+
+  // One buffer is shared by every feature of a collection, so recovering the first recovers all.
+  let o: any = ff.geometry.packed.buffer;
+  let b = new Float64Array(Util.countKeys(o));
+  for (const i in o) b[Number(i)] = o[i];
+  col.features.forEach((f: any) => { if (featureIsPacked(f)) f.geometry.packed.buffer = b });
+  return true;
+}
+
+/**
+ * Pack ONE feature's coordinates, into `prepack`'s buffer when given one. Returns the PolyPack that
+ * describes where they went, which is how the collection path below advances its offset.
+ *
+ * Internal: featurePack is the public entry and always answers with the object it was handed.
+ */
+function packOne(f: any, prepack?: PolyPack): PolyPack
 {
   if (f && f.type === 'Feature')
   {
@@ -355,45 +399,57 @@ export function featurePack(f: any, prepack?: PolyPack): any
     }
     return f.geometry.packed;
   }
+  return null;
+}
+
+/**
+ * Ensure a feature or collection is fully packed, and answer with the object handed in.
+ *
+ * ALWAYS the same object, whatever was passed and whatever it had to do. The previous signature
+ * returned three different things - a PolyPack for a feature, a PolyPack for a collection it packed,
+ * and the collection itself for one that was already packed - so a caller could not use the result
+ * without knowing which case it had fallen into, and every caller but one ignored it. The one that
+ * did not assigned a PolyPack to geometry.coordinates.
+ *
+ * A collection packs its UNPACKED features into one new buffer and leaves packed ones where they are.
+ * That is what stops a second call re-packing work already done, which for a large collection means
+ * allocating and copying the whole coordinate set again for nothing. Features already packed keep
+ * whatever buffer they were in - the collection may end up spanning several - and packCollection no
+ * longer minds, because it copies feature by feature rather than assuming one buffer holds them all.
+ */
+export function featurePack(f: any): any
+{
+  if (f && f.type === 'Feature')
+  {
+    if (! featureIsPacked(f)) packOne(f);
+    return f;
+  }
   else if (f && f.type === 'FeatureCollection' && f.features)
   {
-    // Empty?
-    if (f.features.length == 0) return f;
+    if (f.features.length === 0) return f;
 
-    // Already packed or packed incorrectly?
-    let ff: any = f.features[0];
-    if (ff && ff.geometry && ff.geometry.packed)
-    {
-      // Already packed
-      if (ArrayBuffer.isView(ff.geometry.packed.buffer))
-        return f;
-      // Improperly packed (buffer converted to object - convert back to buffer
-      let o: any = ff.geometry.packed.buffer;
-      let b = new Float64Array(Util.countKeys(o));
-      for (const i in o) b[Number(i)] = o[i];
-      f.features.forEach((ff: any) => ff.geometry.packed.buffer = b);
-      return f;
-    }
+    // Before measuring anything: a collection in this state looks packed but is not readable.
+    repairPackedBuffers(f);
 
-    // Allocate one large buffer
+    // One buffer for everything still unpacked. Sized first so it is allocated exactly once.
     let nFloats: number = 0;
     let i: number;
     for (i = 0; i < f.features.length; i++)
       nFloats += featurePackSize(f.features[i]);
-    let ab = new ArrayBuffer(nFloats * 8);
-    prepack = { offset: 0, length: nFloats, buffer: new Float64Array(ab) };
 
-    // Now pack
+    let prepack: PolyPack = { offset: 0, length: nFloats,
+                              buffer: new Float64Array(new ArrayBuffer(nFloats * 8)) };
     for (i = 0; i < f.features.length; i++)
     {
-      let postpack = featurePack(f.features[i], prepack);
+      if (featureIsPacked(f.features[i])) continue;
+      let postpack = packOne(f.features[i], prepack);
       prepack.offset += postpack ? postpack.length : 0;
     }
     if (prepack.offset != nFloats)
       throw 'oops, packing bug';
-    return prepack;
+    return f;
   }
-  return null;
+  return f;
 }
 
 export function featureUnpack(f: any): any

package/lib/poly/topo.ts

@@ -36,6 +36,20 @@ function getGEOID(f: any): string
 export function topoFromCollection(col: any): Topo
 {
   if (col == null) return null;
+
+  // The objects map below is keyed per feature, so every feature needs an id that is present, unique
+  // and a string - and the topology is meaningless without one. A collection that has no usable id
+  // would otherwise key EVERY feature to undefined and collapse the whole thing onto a single object,
+  // silently: the topology comes back with one object in it and nothing anywhere reports a problem.
+  // An imported GeoJSON file is the ordinary way to arrive here in that state, because GeoJSON puts a
+  // feature's id at the TOP level and most exporters write it there rather than in properties.
+  //
+  // geoEnsureID also NORMALIZES, which matters as much: it copies GEOID/GEOID10 into properties.id,
+  // so the key used here is properties.id whatever the collection called it. Callers that read the
+  // objects map back - topoSimplifyCollection does, by properties.id - would otherwise be looking up
+  // a key this function never used.
+  G.geoEnsureID(col);
+
   let save = PP.featureUnpackTemporarily(col);
   let prop = getGEOID(col);
   let objects: any = {};
@@ -91,9 +105,67 @@ export function topoContiguity(topo: Topo): any
   return result;
 }
 
+/**
+ * Resolved polygon geometry, keyed on the TOPOLOGY OBJECT and held at module level.
+ *
+ * The two things a topology object stands for have opposite natures, and the split matters:
+ *
+ *   ARCS are a property of the topology the object appears in. The same object shares different arcs
+ *   in different topologies - a precinct and a block do not break at the same places - which is why
+ *   arc offsets live in a per-topology WeakMap (topology.packed.objectArcs) rather than on the object.
+ *
+ *   GEOMETRY is not. The points are the same wherever the object appears. It may end up PACKED into
+ *   different buffers depending on which collection packed it, but the coordinates are identical, and
+ *   a collection packed from several buffers is fine by construction (see packCollection).
+ *
+ * So geometry can be cached once, globally, for the life of the object. Module level rather than a
+ * field on the object precisely to avoid the project-then-strip dance that packedarcs still needs
+ * around JSON.stringify in topoToBuffer - a trick that has produced real bugs by leaving values
+ * persisted where they should not be. A WeakMap is invisible to serialization and dies with the
+ * object.
+ *
+ * THE RULE THAT KEEPS THIS CORRECT. Any operation that changes an object's resolved geometry must
+ * produce a NEW object; operations that preserve geometry may share identity. Both hold today: splice
+ * renumbers and subdivides arcs, neither of which moves a point, and it now shares objects
+ * deliberately; simplification goes through topoFromCollection and so builds new objects. DRA
+ * topologies are unquantized, so there is no transform that could decode the same arcs differently.
+ */
+const builtGeometry = new WeakMap<any, any>();
+let cacheHits = 0;
+let cacheMisses = 0;
+
+/** Hits are geometry that would otherwise have been rebuilt from arcs; misses are the rebuilds that
+ *  were actually necessary. That distinction is the point - a raw conversion count cannot tell them
+ *  apart. Call as Poly.topoFeatureCacheStats() from the console. */
+export function topoFeatureCacheStats(): { hits: number, misses: number, pct: number }
+{
+  const total = cacheHits + cacheMisses;
+  return { hits: cacheHits, misses: cacheMisses, pct: total ? Math.round((cacheHits / total) * 100) : 0 };
+}
+
+export function topoFeatureCacheReset(): void { cacheHits = 0; cacheMisses = 0 }
+
 export function topoToFeature(topo: Topo, geoid: string): any
 {
-  return correctGeometry(TopoClient.feature(topo, topo.objects[geoid]));
+  const o: any = topo.objects[geoid];
+  const geometry = o == null ? undefined : builtGeometry.get(o);
+  if (geometry !== undefined)
+  {
+    cacheHits++;
+    // A fresh Feature wrapper each call, matching what TopoClient.feature returns - including sharing
+    // the properties object with the topology object, which it already did. Only the geometry, the
+    // expensive and immutable part, is reused.
+    const f: any = { type: 'Feature', properties: o.properties == null ? {} : o.properties, geometry };
+    if (o.id != null) f.id = o.id;
+    if (o.bbox != null) f.bbox = o.bbox;
+    return f;
+  }
+  cacheMisses++;
+  const built = correctGeometry(TopoClient.feature(topo, o));
+  // Cached AFTER correctGeometry, so every later reader gets the rewound rings rather than repeating
+  // the correction.
+  if (o != null && built && built.geometry) builtGeometry.set(o, built.geometry);
+  return built;
 }
 
 export type TopoSpliceEntry = { topology: Topo, filterout?: any };
@@ -260,6 +332,14 @@ export function topoSimplifyCollection(col: any, options?: SimplifyOptions): any
         let oOld: any = topo.objects[f.properties.id];
         let oNew: any = testtopo.objects[f.properties.id];
 
+        // Every feature must have found its own object. topoFromCollection guarantees it, so reaching
+        // here means the keying has diverged - and the two ways that goes wrong are both silent or
+        // worse: a collection with no ids used to put every feature on ONE object and come back
+        // simplified to a single feature, and a mismatched key dereferences undefined a few lines
+        // below. Neither is worth guessing through, so stop and leave the collection alone.
+        if (oOld === undefined || oNew === undefined)
+          throw `topoSimplifyCollection: feature id ${f.properties.id} has no topology object`;
+
         // Ignore points
         if (f.geometry && f.geometry.type === 'Point')
           keepTiny.set(f, f);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dra2020/baseclient",
-  "version": "1.0.170",
+  "version": "1.0.172",
   "description": "Utility functions for Javascript projects.",
   "main": "dist/baseclient.js",
   "types": "./dist/all/all.d.ts",
@@ -44,7 +44,7 @@
     "webpack-cli": "^5.1.4"
   },
   "dependencies": {
-    "@dra2020/topojson-client": "^3.2.17",
+    "@dra2020/topojson-client": "^3.2.18",
     "@dra2020/topojson-server": "^3.0.103",
     "@dra2020/topojson-simplify": "^3.0.102",
     "diff-match-patch": "^1.0.5",