@basemaps/lambda-tiler 6.31.0 → 6.32.0

package/src/routes/tile.wmts.ts

@@ -1,48 +1,48 @@
 import { Config, TileSetType } from '@basemaps/config';
-import { GoogleTms, ImageFormat, Nztm2000QuadTms } from '@basemaps/geo';
-import { Env, tileWmtsFromPath } from '@basemaps/shared';
-import { getImageFormat } from '@basemaps/tiler';
+import { GoogleTms, Nztm2000QuadTms, TileMatrixSet } from '@basemaps/geo';
+import { Env } from '@basemaps/shared';
 import { HttpHeader, LambdaHttpRequest, LambdaHttpResponse } from '@linzjs/lambda';
 import { createHash } from 'crypto';
-import { Router } from '../router.js';
+import { NotFound, NotModified } from '../util/response.js';
+import { Validate } from '../util/validate.js';
 import { WmtsCapabilities } from '../wmts.capability.js';
-import { NotFound, NotModified } from './response.js';
-import { TileEtag } from './tile.etag.js';
+import { Etag } from '../util/etag.js';
 
-export function getImageFormats(req: LambdaHttpRequest): ImageFormat[] | undefined {
-  const formats = [...req.query.getAll('format'), ...req.query.getAll('tileFormat')];
-  if (formats.length === 0) return;
-
-  const output: Set<ImageFormat> = new Set();
-  for (const fmt of formats) {
-    const parsed = getImageFormat(fmt);
-    if (parsed == null) continue;
-    output.add(parsed);
-  }
-  if (output.size === 0) return;
-  return [...output.values()];
+export interface WmtsCapabilitiesGet {
+  Params: {
+    tileSet?: string;
+    tileMatrix?: string;
+  };
 }
 
+export function getWmtsTileMatrix(tileMatrixParam?: string): TileMatrixSet[] | null {
+  if (tileMatrixParam == null) return [GoogleTms, Nztm2000QuadTms];
+  const tileMatrix = Validate.getTileMatrixSet(tileMatrixParam);
+  if (tileMatrix == null) return null;
+  return [tileMatrix];
+}
 /**
  * Serve a WMTS request
  *
  * /v1/tiles/:tileSet/:tileMatrixSet/WMTSCapabilities.xml
  * @example `/v1/tiles/aerial/NZTM2000Quad/WMTSCapabilities.xml`
  */
-export async function wmts(req: LambdaHttpRequest): Promise<LambdaHttpResponse> {
-  const action = Router.action(req);
-  const wmtsData = tileWmtsFromPath(action.rest);
-  if (wmtsData == null) return NotFound;
+export async function wmtsCapabilitiesGet(req: LambdaHttpRequest<WmtsCapabilitiesGet>): Promise<LambdaHttpResponse> {
+  const apiKey = Validate.apiKey(req);
+
+  const tileSetName = req.params.tileSet ?? 'aerial';
+  const tileMatrix = getWmtsTileMatrix(req.params.tileMatrix);
+  if (tileMatrix == null) return NotFound();
+
   const host = Env.get(Env.PublicUrlBase) ?? '';
 
   req.timer.start('tileset:load');
-  const tileSet = await Config.TileSet.get(Config.TileSet.id(wmtsData.name ?? 'aerial'));
+  const tileSet = await Config.TileSet.get(Config.TileSet.id(tileSetName ?? 'aerial'));
   req.timer.end('tileset:load');
-  if (tileSet == null || tileSet.type !== TileSetType.Raster) return NotFound;
+  if (tileSet == null || tileSet.type !== TileSetType.Raster) return NotFound();
 
   const provider = await Config.Provider.get(Config.Provider.id('linz'));
 
-  const tileMatrix = wmtsData.tileMatrix == null ? [GoogleTms, Nztm2000QuadTms] : [wmtsData.tileMatrix];
   req.timer.start('imagery:load');
   const imagery = await Config.getAllImagery(
     tileSet.layers,
@@ -50,27 +50,26 @@ export async function wmts(req: LambdaHttpRequest): Promise<LambdaHttpResponse>
   );
   req.timer.end('imagery:load');
 
-  const apiKey = Router.apiKey(req);
   const xml = new WmtsCapabilities({
     httpBase: host,
     provider: provider ?? undefined,
     tileSet,
     tileMatrix,
-    isIndividualLayers: wmtsData.tileMatrix == null,
+    isIndividualLayers: req.params.tileMatrix == null,
     imagery,
     apiKey,
-    formats: getImageFormats(req),
+    formats: Validate.getRequestedFormats(req),
   }).toXml();
-  if (xml == null) return NotFound;
+  if (xml == null) return NotFound();
 
   const data = Buffer.from(xml);
 
   const cacheKey = createHash('sha256').update(data).digest('base64');
-  if (TileEtag.isNotModified(req, cacheKey)) return NotModified;
+  if (Etag.isNotModified(req, cacheKey)) return NotModified();
 
   const response = new LambdaHttpResponse(200, 'ok');
   response.header(HttpHeader.ETag, cacheKey);
-  response.header(HttpHeader.CacheControl, 'max-age=0');
+  response.header(HttpHeader.CacheControl, 'no-store');
   response.buffer(data, 'text/xml');
   req.set('bytes', data.byteLength);
   return response;

package/src/routes/tile.xyz.raster.ts

@@ -0,0 +1,106 @@
+import { Config, ConfigTileSetRaster } from '@basemaps/config';
+import { Bounds, Epsg, TileMatrixSet, TileMatrixSets, VectorFormat } from '@basemaps/geo';
+import { Env, fsa } from '@basemaps/shared';
+import { Tiler } from '@basemaps/tiler';
+import { TileMakerSharp } from '@basemaps/tiler-sharp';
+import { CogTiff } from '@cogeotiff/core';
+import { HttpHeader, LambdaHttpRequest, LambdaHttpResponse } from '@linzjs/lambda';
+import pLimit from 'p-limit';
+import { Etag } from '../util/etag.js';
+import { NotFound, NotModified } from '../util/response.js';
+import { CoSources } from '../util/source.cache.js';
+import { TileXyz } from '../util/validate.js';
+
+const LoadingQueue = pLimit(Env.getNumber(Env.TiffConcurrency, 25));
+
+export function getTiffName(name: string): string {
+  const lowerName = name.toLowerCase();
+  if (lowerName.endsWith('.tif') || lowerName.endsWith('.tiff')) return name;
+  return `${name}.tiff`;
+}
+
+export const TileComposer = new TileMakerSharp(256);
+
+const DefaultResizeKernel = { in: 'lanczos3', out: 'lanczos3' } as const;
+const DefaultBackground = { r: 0, g: 0, b: 0, alpha: 0 };
+
+export const TileXyzRaster = {
+  async getTiffsForTile(req: LambdaHttpRequest, tileSet: ConfigTileSetRaster, xyz: TileXyz): Promise<string[]> {
+    const imagery = await Config.getAllImagery(tileSet.layers, [xyz.tileMatrix.projection]);
+
+    const output: string[] = [];
+    const tileBounds = xyz.tileMatrix.tileToSourceBounds(xyz.tile);
+
+    // All zoom level config is stored as Google zoom levels
+    const filterZoom = TileMatrixSet.convertZoomLevel(xyz.tile.z, xyz.tileMatrix, TileMatrixSets.get(Epsg.Google));
+    for (const layer of tileSet.layers) {
+      if (layer.maxZoom != null && filterZoom > layer.maxZoom) continue;
+      if (layer.minZoom != null && filterZoom < layer.minZoom) continue;
+
+      const imgId = layer[xyz.tileMatrix.projection.code];
+      if (imgId == null) {
+        req.log.warn({ layer: layer.name, projection: xyz.tileMatrix.projection.code }, 'Failed to lookup imagery');
+        continue;
+      }
+
+      const img = imagery.get(imgId);
+      if (img == null) {
+        req.log.warn(
+          { layer: layer.name, projection: xyz.tileMatrix.projection.code, imgId },
+          'Failed to lookup imagery',
+        );
+        continue;
+      }
+      if (!tileBounds.intersects(Bounds.fromJson(img.bounds))) continue;
+
+      for (const c of img.files) {
+        if (!tileBounds.intersects(Bounds.fromJson(c))) continue;
+        const tiffPath = fsa.join(img.uri, getTiffName(c.name));
+        output.push(tiffPath);
+      }
+    }
+    return output;
+  },
+
+  async tile(req: LambdaHttpRequest, tileSet: ConfigTileSetRaster, xyz: TileXyz): Promise<LambdaHttpResponse> {
+    if (xyz.tileType === VectorFormat.MapboxVectorTiles) return NotFound();
+
+    const tiffPaths = await this.getTiffsForTile(req, tileSet, xyz);
+    const cacheKey = Etag.key(tiffPaths);
+    if (Etag.isNotModified(req, cacheKey)) return NotModified();
+
+    const toLoad: Promise<CogTiff | null>[] = [];
+    for (const tiffPath of tiffPaths) {
+      toLoad.push(
+        LoadingQueue(() => {
+          return CoSources.getCog(tiffPath).catch((error) => {
+            req.log.warn({ error, tiff: tiffPath }, 'TiffLoadFailed');
+            return null;
+          });
+        }),
+      );
+    }
+
+    const tiffs = (await Promise.all(toLoad)).filter((f) => f != null) as CogTiff[];
+
+    const tiler = new Tiler(xyz.tileMatrix);
+    const layers = await tiler.tile(tiffs, xyz.tile.x, xyz.tile.y, xyz.tile.z);
+
+    const res = await TileComposer.compose({
+      layers,
+      format: xyz.tileType,
+      background: tileSet.background ?? DefaultBackground,
+      resizeKernel: tileSet.resizeKernel ?? DefaultResizeKernel,
+      metrics: req.timer,
+    });
+
+    req.set('layersUsed', res.layers);
+    req.set('bytes', res.buffer.byteLength);
+
+    const response = new LambdaHttpResponse(200, 'ok');
+    response.header(HttpHeader.ETag, cacheKey);
+    response.header(HttpHeader.CacheControl, 'public, max-age=604800, stale-while-revalidate=86400');
+    response.buffer(res.buffer, 'image/' + xyz.tileType);
+    return response;
+  },
+};

package/src/routes/tile.xyz.ts

@@ -1,9 +1,20 @@
-import { tileXyzFromPath } from '@basemaps/shared';
+import { Config, TileSetType } from '@basemaps/config';
 import { LambdaHttpRequest, LambdaHttpResponse } from '@linzjs/lambda';
-import { Router } from '../router.js';
-import { TileSets } from '../tile.set.cache.js';
-import { ValidateTilePath } from '../validate.js';
-import { NotFound } from './response.js';
+import { NotFound } from '../util/response.js';
+import { Validate } from '../util/validate.js';
+import { TileXyzRaster } from './tile.xyz.raster.js';
+import { tileXyzVector } from './tile.xyz.vector.js';
+
+export interface TileXyzGet {
+  Params: {
+    tileSet: string;
+    tileMatrix: string;
+    z: string;
+    x: string;
+    y: string;
+    tileType: string;
+  };
+}
 
 /**
  * Serve a tile
@@ -11,20 +22,24 @@ import { NotFound } from './response.js';
  * /v1/tiles/:tileSet/:tileMatrixSet/:z/:x/:y.:tileType
  *
  * @example
- *  Vector Tile `/v1/tiles/topographic/EPSG:3857/2/1/1.pbf`
- *  Raster Tile `/v1/tiles/aerial/EPSG:3857/6/0/38.webp`
- * @returns
+ *  Vector Tile `/v1/tiles/topographic/WebMercatorQuad/2/1/1.pbf`
+ *  Raster Tile `/v1/tiles/aerial/WebMercatorQuad/6/0/38.webp`
+ *
  */
-export async function tileXyz(req: LambdaHttpRequest): Promise<LambdaHttpResponse> {
-  const action = Router.action(req);
-  const xyzData = tileXyzFromPath(action.rest);
-  if (xyzData == null) return NotFound;
-  ValidateTilePath.validate(req, xyzData);
+export async function tileXyzGet(req: LambdaHttpRequest<TileXyzGet>): Promise<LambdaHttpResponse> {
+  const xyzData = Validate.xyz(req);
 
   req.timer.start('tileset:load');
-  const tileSet = await TileSets.get(xyzData.name, xyzData.tileMatrix);
+  const tileSet = await Config.TileSet.get(Config.TileSet.id(xyzData.tileSet));
   req.timer.end('tileset:load');
-  if (tileSet == null) return NotFound;
+  if (tileSet == null) return NotFound();
 
-  return await tileSet.tile(req, xyzData);
+  switch (tileSet.type) {
+    case TileSetType.Vector:
+      return tileXyzVector.tile(req, tileSet, xyzData);
+    case TileSetType.Raster:
+      return TileXyzRaster.tile(req, tileSet, xyzData);
+    default:
+      return new LambdaHttpResponse(400, 'Invalid tileset');
+  }
 }

package/src/routes/tile.xyz.vector.ts

@@ -0,0 +1,47 @@
+import { ConfigTileSetVector } from '@basemaps/config';
+import { GoogleTms, VectorFormat } from '@basemaps/geo';
+import { HttpHeader, LambdaHttpRequest, LambdaHttpResponse } from '@linzjs/lambda';
+import { isGzip } from '../util/cotar.serve.js';
+import { Etag } from '../util/etag.js';
+import { NotFound, NotModified } from '../util/response.js';
+import { CoSources } from '../util/source.cache.js';
+import { TileXyz } from '../util/validate.js';
+
+export const tileXyzVector = {
+  /** Serve a MVT vector tile */
+  async tile(req: LambdaHttpRequest, tileSet: ConfigTileSetVector, xyz: TileXyz): Promise<LambdaHttpResponse> {
+    if (xyz.tileType !== VectorFormat.MapboxVectorTiles) return NotFound();
+    if (xyz.tileMatrix.identifier !== GoogleTms.identifier) return NotFound();
+
+    if (tileSet.layers.length > 1) return new LambdaHttpResponse(500, 'Too many layers in tileset');
+    const [layer] = tileSet.layers;
+    const layerId = layer[3857];
+    if (layerId == null) return new LambdaHttpResponse(500, 'Layer url not found from tileset Config');
+
+    // Flip Y coordinate because MBTiles files are TMS.
+    const y = (1 << xyz.tile.z) - 1 - xyz.tile.y;
+
+    const tilePath = `tiles/${xyz.tile.z}/${xyz.tile.x}/${y}.pbf.gz`;
+    const tileId = `${layerId}#${tilePath}`;
+
+    const cacheKey = Etag.key(tileId);
+    if (Etag.isNotModified(req, cacheKey)) return NotModified();
+
+    req.timer.start('cotar:load');
+    const cotar = await CoSources.getCotar(layerId);
+    if (cotar == null) return new LambdaHttpResponse(500, 'Failed to load VectorTiles');
+    req.timer.end('cotar:load');
+
+    req.timer.start('cotar:tile');
+    const tile = await cotar.get(tilePath);
+    if (tile == null) return NotFound();
+    req.timer.end('cotar:tile');
+
+    const tileBuffer = Buffer.from(tile);
+    const response = LambdaHttpResponse.ok().buffer(tileBuffer, 'application/x-protobuf');
+    response.header(HttpHeader.ETag, cacheKey);
+    response.header(HttpHeader.CacheControl, 'public, max-age=604800, stale-while-revalidate=86400');
+    if (isGzip(tileBuffer)) response.header(HttpHeader.ContentEncoding, 'gzip');
+    return response;
+  },
+};

package/src/routes/version.ts

@@ -0,0 +1,8 @@
+import { HttpHeader, LambdaHttpResponse } from '@linzjs/lambda';
+
+export async function versionGet(): Promise<LambdaHttpResponse> {
+  const response = new LambdaHttpResponse(200, 'ok');
+  response.header(HttpHeader.CacheControl, 'no-store');
+  response.json({ version: process.env.GIT_VERSION ?? 'dev', hash: process.env.GIT_HASH });
+  return response;
+}

package/src/util/__test__/validate.test.ts

@@ -0,0 +1,74 @@
+import { GoogleTms, ImageFormat, Nztm2000QuadTms, Nztm2000Tms, VectorFormat } from '@basemaps/geo';
+import o from 'ospec';
+import { mockUrlRequest } from '../../__tests__/xyz.util.js';
+import { Validate } from '../validate.js';
+
+o.spec('GetImageFormats', () => {
+  o('should parse all formats', () => {
+    const req = mockUrlRequest('/v1/blank', 'format=png&format=jpeg');
+    const formats = Validate.getRequestedFormats(req);
+    o(formats).deepEquals([ImageFormat.Png, ImageFormat.Jpeg]);
+  });
+
+  o('should ignore bad formats', () => {
+    const req = mockUrlRequest('/v1/blank', 'format=fake&format=mvt');
+    const formats = Validate.getRequestedFormats(req);
+    o(formats).equals(null);
+  });
+
+  o('should de-dupe formats', () => {
+    const req = mockUrlRequest('/v1/blank', 'format=png&format=jpeg&format=png&format=jpeg&format=png&format=jpeg');
+    const formats = Validate.getRequestedFormats(req);
+    o(formats).deepEquals([ImageFormat.Png, ImageFormat.Jpeg]);
+  });
+
+  o('should support "tileFormat" Alias all formats', () => {
+    const req = mockUrlRequest('/v1/blank', 'tileFormat=png&format=jpeg');
+    const formats = Validate.getRequestedFormats(req);
+    o(formats).deepEquals([ImageFormat.Jpeg, ImageFormat.Png]);
+  });
+
+  o('should not duplicate "tileFormat" alias all formats', () => {
+    const req = mockUrlRequest('/v1/blank', 'tileFormat=jpeg&format=jpeg');
+    const formats = Validate.getRequestedFormats(req);
+    o(formats).deepEquals([ImageFormat.Jpeg]);
+  });
+});
+
+o.spec('getTileMatrixSet', () => {
+  o('should lookup epsg codes', () => {
+    o(Validate.getTileMatrixSet('EPSG:3857')?.identifier).equals(GoogleTms.identifier);
+    o(Validate.getTileMatrixSet('EPSG:2193')?.identifier).equals(Nztm2000Tms.identifier);
+
+    o(Validate.getTileMatrixSet('3857')?.identifier).equals(GoogleTms.identifier);
+    o(Validate.getTileMatrixSet('2193')?.identifier).equals(Nztm2000Tms.identifier);
+  });
+
+  o('should lookup by identifier', () => {
+    o(Validate.getTileMatrixSet('WebMercatorQuad')?.identifier).equals(GoogleTms.identifier);
+    o(Validate.getTileMatrixSet('NZTM2000Quad')?.identifier).equals(Nztm2000QuadTms.identifier);
+    o(Validate.getTileMatrixSet('Nztm2000')?.identifier).equals(Nztm2000Tms.identifier);
+  });
+
+  o('should be case sensitive', () => {
+    o(Validate.getTileMatrixSet('Nztm2000Quad')?.identifier).equals(undefined);
+  });
+});
+
+o.spec('getTileFormat', () => {
+  for (const ext of Object.values(ImageFormat)) {
+    o('should support image format:' + ext, () => {
+      o(Validate.getTileFormat(ext)).equals(ext);
+    });
+  }
+
+  o('should support vector format: mvt', () => {
+    o(Validate.getTileFormat('pbf')).equals(VectorFormat.MapboxVectorTiles);
+  });
+
+  for (const fmt of ['FAKE', /* 'JPEG' // TODO should this be case sensitive ,*/ 'mvt', 'json']) {
+    o('should not support format:' + fmt, () => {
+      o(Validate.getTileFormat(fmt)).equals(null);
+    });
+  }
+});

package/src/util/cotar.serve.ts

@@ -0,0 +1,46 @@
+import { HttpHeader, LambdaHttpRequest, LambdaHttpResponse } from '@linzjs/lambda';
+import { Etag } from './etag.js';
+import { NotFound, NotModified } from './response.js';
+import { CoSources } from './source.cache.js';
+
+/**
+ *  Load a cotar and look for a file inside the cotar returning the file back as a LambdaResponse
+ *
+ * This will also set two headers
+ * - Content-Encoding if the file starts with gzip magic
+ * - Content-Type from the parameter contentType
+ */
+export async function serveFromCotar(
+  req: LambdaHttpRequest,
+  cotarPath: string,
+  assetPath: string,
+  contentType: string,
+): Promise<LambdaHttpResponse> {
+  const cotar = await CoSources.getCotar(cotarPath);
+  if (cotar == null) return NotFound();
+  const fileData = await cotar.get(assetPath);
+  if (fileData == null) return NotFound();
+
+  const buf = Buffer.from(fileData);
+
+  const cacheKey = Etag.key(buf);
+  if (Etag.isNotModified(req, cacheKey)) return NotModified();
+
+  const response = LambdaHttpResponse.ok().buffer(buf, contentType);
+  response.header(HttpHeader.ETag, cacheKey);
+  response.header(HttpHeader.CacheControl, 'public, max-age=604800, stale-while-revalidate=86400');
+  if (isGzip(buf)) response.header(HttpHeader.ContentEncoding, 'gzip');
+  return response;
+}
+
+/**
+ * Does a buffer look like a gzipped document instead of raw json
+ *
+ * Determined by checking the first two bytes are the gzip magic bytes `0x1f 0x8b`
+ *
+ * @see https://en.wikipedia.org/wiki/Gzip
+ *
+ */
+export function isGzip(b: Buffer): boolean {
+  return b[0] === 0x1f && b[1] === 0x8b;
+}

package/src/util/etag.ts

@@ -0,0 +1,20 @@
+import { sha256base58 } from '@basemaps/config';
+import { HttpHeader, LambdaHttpRequest } from '@linzjs/lambda';
+
+export const Etag = {
+  key(obj: unknown): string {
+    if (Buffer.isBuffer(obj) || typeof obj === 'string') return sha256base58(obj);
+    return sha256base58(JSON.stringify(obj));
+  },
+
+  isNotModified(req: LambdaHttpRequest, cacheKey: string): boolean {
+    // If the user has supplied a IfNoneMatch Header and it contains the full sha256 sum for our
+    // etag this tile has not been modified.
+    const ifNoneMatch = req.header(HttpHeader.IfNoneMatch);
+    if (ifNoneMatch != null && ifNoneMatch.indexOf(cacheKey) > -1) {
+      req.set('cache', { hit: true, match: ifNoneMatch });
+      return true;
+    }
+    return false;
+  },
+};

package/src/util/response.ts

@@ -0,0 +1,4 @@
+import { LambdaHttpResponse } from '@linzjs/lambda';
+
+export const NotFound = (): LambdaHttpResponse => new LambdaHttpResponse(404, 'Not Found');
+export const NotModified = (): LambdaHttpResponse => new LambdaHttpResponse(304, 'Not modified');

package/src/util/source.cache.ts

@@ -0,0 +1,71 @@
+import { fsa } from '@basemaps/shared';
+import { ChunkSourceBase } from '@chunkd/core';
+import { CogTiff } from '@cogeotiff/core';
+import { Cotar } from '@cotar/core';
+import { St } from './source.tracer.js';
+import { SwappingLru } from './swapping.lru.js';
+
+export type LruStrut = LruStrutCotar | LruStrutCog;
+
+export interface LruStrutCotar {
+  type: 'cotar';
+  value: Promise<Cotar>;
+  _value?: Cotar;
+}
+
+export interface LruStrutCog {
+  type: 'cog';
+  value: Promise<CogTiff>;
+  _value?: CogTiff;
+}
+
+class LruStrutObj<T extends LruStrut> {
+  ob: T;
+  constructor(ob: T) {
+    this.ob = ob;
+    if (this.ob._value == null) this.ob.value.then((c) => (this.ob._value = c));
+  }
+
+  get size(): number {
+    const val = this.ob._value;
+    if (val == null) return 0;
+    return val.source.chunkSize * (val.source as ChunkSourceBase).chunks.size;
+  }
+}
+
+export class SourceCache {
+  cache: SwappingLru<LruStrutObj<LruStrutCotar | LruStrutCog>>;
+  constructor(maxSize: number) {
+    this.cache = new SwappingLru<LruStrutObj<LruStrut>>(maxSize);
+  }
+
+  getCog(location: string): Promise<CogTiff> {
+    const existing = this.cache.get(location)?.ob;
+
+    if (existing != null) {
+      if (existing.type === 'cog') return existing.value;
+      throw new Error(`Existing object of type: ${existing.type} made for location: ${location}`);
+    }
+    const source = fsa.source(location);
+    St.trace(source);
+    const value = CogTiff.create(source);
+    this.cache.set(location, new LruStrutObj({ type: 'cog', value }));
+    return value;
+  }
+
+  getCotar(location: string): Promise<Cotar> {
+    const existing = this.cache.get(location)?.ob;
+
+    if (existing != null) {
+      if (existing.type === 'cotar') return existing.value as Promise<Cotar>;
+      throw new Error(`Existing object of type: ${existing.type} made for location: ${location}`);
+    }
+    const source = fsa.source(location);
+    St.trace(source);
+    const value = Cotar.fromTar(source);
+    this.cache.set(location, new LruStrutObj({ type: 'cotar', value }));
+    return value;
+  }
+}
+
+export const CoSources = new SourceCache(256 * 1024 * 1024);

package/src/{source.tracer.ts → util/source.tracer.ts}

@@ -1,6 +1,8 @@
+import { sha256base58 } from '@basemaps/config';
 import { ChunkSource } from '@chunkd/core';
 
 interface SourceRequest {
+  id?: string;
   offset: number;
   length?: number;
   source: string;
@@ -20,6 +22,8 @@ export class SourceTracer {
     const originFetch = source.fetchBytes;
     source.fetchBytes = async (offset: number, length?: number): Promise<ArrayBuffer> => {
       const request: SourceRequest = { source: source.uri, offset, length };
+      const traceId = sha256base58(`${request.source}:${request.offset}:${request.length}`);
+      request.id = traceId;
       this.requests.push(request);
       const startTime = Date.now();
 

package/src/util/swapping.lru.ts

@@ -0,0 +1,63 @@
+export class SwappingLru<T extends { size: number }> {
+  cacheA: Map<string, T> = new Map();
+  cacheB: Map<string, T> = new Map();
+  maxSize: number;
+
+  hits = 0;
+  misses = 0;
+  resets = 0;
+
+  _lastCheckedAt = -1;
+
+  constructor(maxSize: number) {
+    this.maxSize = maxSize;
+  }
+
+  get(id: string): T | null {
+    const cacheA = this.cacheA.get(id);
+    if (cacheA) {
+      this.hits++;
+      return cacheA;
+    }
+
+    const cacheB = this.cacheB.get(id);
+    if (cacheB == null) {
+      this.misses++;
+      return null;
+    }
+
+    this.hits++;
+    // If a object is still useful move it into the main cache
+    this.cacheA.set(id, cacheB);
+    this.cacheB.delete(id);
+    return cacheB;
+  }
+
+  /** Reset the cache */
+  clear(): void {
+    this.cacheA.clear();
+    this.cacheB.clear();
+  }
+
+  set(id: string, tiff: T): void {
+    this.cacheA.set(id, tiff);
+    this.check();
+  }
+
+  /** Validate the size of the cache has not exploded */
+  check(): void {
+    this._lastCheckedAt = Date.now();
+    if (this.maxSize <= 0) return;
+    if (this.currentSize <= this.maxSize) return;
+    this.resets++;
+    this.cacheB = this.cacheA;
+    this.cacheA = new Map();
+  }
+
+  /** Calculate the total number of bytes used by this cache */
+  get currentSize(): number {
+    let size = 0;
+    for (const value of this.cacheA.values()) size += value.size;
+    return size;
+  }
+}