@basemaps/lambda-tiler 6.29.0 → 6.32.0

package/src/routes/tile.etag.ts

@@ -1,36 +0,0 @@
-import { Composition } from '@basemaps/tiler';
-import { createHash } from 'crypto';
-import { TileDataXyz } from '@basemaps/shared';
-import { HttpHeader, LambdaHttpRequest } from '@linzjs/lambda';
-import { basename } from 'path';
-
-export const TileEtag = {
-  // To force a full cache invalidation change this number
-  RenderId: 1,
-
-  key(object: Record<string, unknown>): string {
-    return createHash('sha256').update(JSON.stringify(object)).digest('base64');
-  },
-
-  /** Generate a unique ETag for this tile */
-  generate(compositions: Composition[], xyzData: TileDataXyz): string {
-    // We cannot serialize the CogTiff inside of composition so replace it with the source name
-    const layers = compositions.map((c) => {
-      return { ...c, tiff: basename(c.tiff.source.uri) };
-    });
-
-    const xyz = { ...xyzData, tileMatrix: xyzData.tileMatrix.identifier };
-    return TileEtag.key({ xyz, layers, RenderId: TileEtag.RenderId });
-  },
-
-  isNotModified(req: LambdaHttpRequest, cacheKey: string): boolean {
-    // If the user has supplied a IfNoneMatch Header and it contains the full sha256 sum for our
-    // etag this tile has not been modified.
-    const ifNoneMatch = req.header(HttpHeader.IfNoneMatch);
-    if (ifNoneMatch != null && ifNoneMatch.indexOf(cacheKey) > -1) {
-      req.set('cache', { key: cacheKey, hit: true, match: ifNoneMatch });
-      return true;
-    }
-    return false;
-  },
-};

package/src/routes/tile.ts

@@ -1,23 +0,0 @@
-import { LambdaHttpRequest, LambdaHttpResponse } from '@linzjs/lambda';
-import { isValidApiKey } from '../api.key.js';
-import { Router } from '../router.js';
-import { attribution } from './attribution.js';
-import { NotFound } from './response.js';
-import { tileJson } from './tile.json.js';
-import { styleJson } from './tile.style.json.js';
-import { wmts } from './tile.wmts.js';
-import { tileXyz } from './tile.xyz.js';
-
-export async function Tiles(req: LambdaHttpRequest): Promise<LambdaHttpResponse> {
-  const { rest } = Router.action(req);
-  if (rest.length < 1) return NotFound;
-  const apiKey = Router.apiKey(req);
-  if (!isValidApiKey(apiKey)) return new LambdaHttpResponse(400, 'Invalid API Key');
-
-  const fileName = rest[rest.length - 1].toLowerCase();
-  if (fileName === 'attribution.json') return attribution(req);
-  if (fileName === 'wmtscapabilities.xml') return wmts(req);
-  if (fileName === 'tile.json') return tileJson(req);
-  if (fileName.endsWith('json') && rest[rest.length - 2] === 'style') return styleJson(req, fileName);
-  return tileXyz(req);
-}

package/src/tiff.cache.ts

@@ -1,51 +0,0 @@
-import { ChunkSourceBase } from '@chunkd/core';
-import { CogTiff } from '@cogeotiff/core';
-
-/** Limit the caching of tiffs */
-export class TiffCache {
-  static cacheA: Map<string, CogTiff> = new Map();
-  static cacheB: Map<string, CogTiff> = new Map();
-
-  /** 256 MB Cache */
-  static MaxCacheSizeBytes = 256 * 1024 * 1024;
-
-  static get(id: string): CogTiff | null {
-    const cacheA = TiffCache.cacheA.get(id);
-    if (cacheA) return cacheA;
-
-    const cacheB = TiffCache.cacheB.get(id);
-    if (cacheB == null) return null;
-    // If a object is still useful move it into the main cache
-    TiffCache.cacheA.set(id, cacheB);
-    TiffCache.cacheB.delete(id);
-    return cacheB;
-  }
-
-  /** Reset the cache */
-  static clear(): void {
-    this.cacheA.clear();
-    this.cacheB.clear();
-  }
-
-  static set(id: string, tiff: CogTiff): void {
-    TiffCache.cacheA.set(id, tiff);
-    TiffCache.check();
-  }
-
-  /** Validate the size of the cache has not exploded */
-  static check(): void {
-    if (TiffCache.MaxCacheSizeBytes <= 0) return;
-    if (TiffCache.currentSize <= TiffCache.MaxCacheSizeBytes) return;
-    TiffCache.cacheB = TiffCache.cacheA;
-    TiffCache.cacheA = new Map();
-  }
-
-  /** Calculate the total number of bytes used by this tiff cache */
-  static get currentSize(): number {
-    let size = 0;
-    for (const value of TiffCache.cacheA.values()) {
-      size += value.source.chunkSize * (value.source as ChunkSourceBase).chunks.size;
-    }
-    return size;
-  }
-}

package/src/tile.set.cache.ts

@@ -1,111 +0,0 @@
-import { TileSetNameParser, TileSetType } from '@basemaps/config';
-import { TileMatrixSet, TileMatrixSets } from '@basemaps/geo';
-import { Config } from '@basemaps/shared';
-import { TileSet } from './tile.set.js';
-import { TileSetRaster } from './tile.set.raster.js';
-import { TileSetVector } from './tile.set.vector.js';
-
-export class TileSetCache {
-  /** Duration to cache tile sets for */
-  CacheTime = 30_000;
-
-  /** Cache the fetch requests */
-  cache: Map<string, { time: number; value: Promise<TileSet | null> }> = new Map();
-  /** Cache initialized tile sets */
-  tileSets: Map<string, TileSet> = new Map();
-
-  id(tileSet: TileSet): string;
-  id(name: string, tileMatrix: TileMatrixSet): string;
-  id(name: string | TileSet, tileMatrix?: TileMatrixSet): string {
-    if (typeof name === 'string') {
-      const nameComp = TileSetNameParser.parse(name);
-      return `${TileSetNameParser.componentsToName(nameComp)}_${tileMatrix?.identifier}`;
-    }
-
-    return `${name.fullName}_${name.tileMatrix.identifier}`;
-  }
-
-  add(tileSet: TileSet, expiresAt = Date.now() + this.CacheTime): void {
-    const id = this.id(tileSet);
-    if (this.cache.has(id)) throw new Error('Trying to add duplicate tile set:' + id);
-    this.cache.set(id, { time: expiresAt, value: Promise.resolve(tileSet) });
-  }
-
-  get(name: string, tileMatrix: TileMatrixSet): Promise<TileSet | null> {
-    const tsId = this.id(name, tileMatrix);
-    let existing = this.cache.get(tsId);
-    // Validate the data is current every ~30 seconds
-    if (existing == null || Date.now() - existing.time > 0) {
-      const value = this.loadTileSet(name, tileMatrix);
-      existing = { time: Date.now(), value };
-      this.cache.set(tsId, existing);
-    }
-    return existing.value;
-  }
-
-  async loadTileSet(name: string, tileMatrix: TileMatrixSet): Promise<TileSet | null> {
-    const nameComp = TileSetNameParser.parse(name);
-    const tileSetId = this.id(name, tileMatrix);
-
-    if (nameComp.layer != null) {
-      const parentName = TileSetNameParser.componentsToName({ ...nameComp, layer: undefined });
-      const parent = await this.get(parentName, tileMatrix);
-      if (parent == null || parent.type === TileSetType.Vector) return null;
-      return parent.child(nameComp.layer);
-    }
-
-    const dbId = Config.TileSet.id(name);
-    const tileSet = await Config.TileSet.get(dbId);
-    if (tileSet == null) {
-      this.cache.delete(tileSetId);
-      return null;
-    }
-
-    // If we already have a copy and it hasn't been modified just return it
-    const existing = this.tileSets.get(tileSetId);
-    if (existing != null && existing?.tileSet.updatedAt === tileSet.updatedAt) {
-      return existing;
-    }
-
-    if (Config.isTileSetRaster(tileSet)) {
-      const ts = new TileSetRaster(name, tileMatrix);
-      await ts.init(tileSet);
-      this.tileSets.set(tileSetId, ts);
-      return ts;
-    }
-
-    const ts = new TileSetVector(name, tileMatrix);
-    await ts.init(tileSet);
-    this.tileSets.set(tileSetId, ts);
-    return ts;
-  }
-
-  async getAll(name: string, tileMatrix?: TileMatrixSet | null): Promise<TileSet[]> {
-    const nameComp = TileSetNameParser.parse(name);
-    const tileMatrices = tileMatrix == null ? Array.from(TileMatrixSets.Defaults.values()) : [tileMatrix];
-
-    const promises: Promise<TileSet | null>[] = [];
-    for (const tileMatrix of tileMatrices) promises.push(this.get(name, tileMatrix));
-    const tileMatrixSets = await Promise.all(promises);
-
-    const tileSets: TileSetRaster[] = [];
-    for (const parent of tileMatrixSets) {
-      if (parent == null) continue;
-      if (parent.type === TileSetType.Vector) continue;
-
-      tileSets.push(parent);
-      if (nameComp.layer != null) {
-        parent.components.name = nameComp.name;
-      } else if (parent.imagery != null && parent.imagery.size > 1) {
-        for (const imageId of parent.imagery.keys()) {
-          const childImg = parent.child(imageId);
-          if (childImg == null) continue;
-          tileSets.push(childImg);
-        }
-      }
-    }
-    return tileSets.sort((a, b) => a.title.localeCompare(b.title));
-  }
-}
-
-export const TileSets = new TileSetCache();

package/src/tile.set.raster.ts

@@ -1,228 +0,0 @@
-import {
-  ConfigImagery,
-  ConfigLayer,
-  ConfigTileSetRaster,
-  TileSetNameComponents,
-  TileSetNameParser,
-  TileSetType,
-} from '@basemaps/config';
-import { Bounds, Epsg, ImageFormat, Tile, TileMatrixSet, TileMatrixSets, VectorFormat } from '@basemaps/geo';
-import { Config, Env, fsa, LogType, TileDataXyz, titleizeImageryName } from '@basemaps/shared';
-import { Tiler } from '@basemaps/tiler';
-import { TileMakerSharp } from '@basemaps/tiler-sharp';
-import { CogTiff } from '@cogeotiff/core';
-import { HttpHeader, LambdaHttpRequest, LambdaHttpResponse } from '@linzjs/lambda';
-import { Metrics } from '@linzjs/metrics';
-import pLimit from 'p-limit';
-import { NotFound, NotModified } from './routes/response.js';
-import { TileEtag } from './routes/tile.etag.js';
-import { St } from './source.tracer.js';
-import { TiffCache } from './tiff.cache.js';
-import { TileSets } from './tile.set.cache.js';
-
-const LoadingQueue = pLimit(Env.getNumber(Env.TiffConcurrency, 25));
-
-export const TileComposer = new TileMakerSharp(256);
-
-export interface TileSetResponse {
-  buffer: Buffer;
-  metrics: Metrics;
-  layersUsed: number;
-  layersTotal: number;
-  contentType: string;
-}
-
-const DefaultResizeKernel = { in: 'lanczos3', out: 'lanczos3' } as const;
-const DefaultBackground = { r: 0, g: 0, b: 0, alpha: 0 };
-
-export function getTiffName(name: string): string {
-  if (name.endsWith('.tif') || name.endsWith('.tiff')) return name;
-  return `${name}.tiff`;
-}
-
-export class TileSetRaster {
-  type: TileSetType.Raster = TileSetType.Raster;
-
-  tileMatrix: TileMatrixSet;
-  tiler: Tiler;
-  imagery: Map<string, ConfigImagery>;
-  extentOverride: Bounds;
-
-  components: TileSetNameComponents;
-  tileSet: ConfigTileSetRaster;
-
-  constructor(name: string, tileMatrix: TileMatrixSet) {
-    this.components = TileSetNameParser.parse(name);
-    this.tileMatrix = tileMatrix;
-    this.tiler = new Tiler(this.tileMatrix);
-  }
-
-  get id(): string {
-    return TileSets.id(this.fullName, this.tileMatrix);
-  }
-
-  get fullName(): string {
-    return TileSetNameParser.componentsToName(this.components);
-  }
-
-  get title(): string {
-    return this.tileSet?.title ?? this.components.name;
-  }
-
-  get description(): string {
-    return this.tileSet?.description ?? '';
-  }
-
-  get extent(): Bounds {
-    return this.extentOverride ?? this.tileMatrix.extent;
-  }
-
-  /** Preferred default imagery format */
-  get format(): ImageFormat {
-    return this.tileSet.format ?? ImageFormat.Webp;
-  }
-
-  async init(record: ConfigTileSetRaster): Promise<void> {
-    this.tileSet = record;
-    this.imagery = await Config.getAllImagery(this.tileSet.layers, this.tileMatrix.projection);
-  }
-
-  async initTiffs(tile: Tile, log: LogType): Promise<CogTiff[]> {
-    const tiffs = this.getTiffsForTile(tile, log);
-    let failed = false;
-    // Remove any tiffs that failed to load
-    const promises = tiffs.map((c) => {
-      return LoadingQueue(async () => {
-        try {
-          await c.init();
-        } catch (error) {
-          log.warn({ error, tiff: c.source.uri }, 'TiffLoadFailed');
-          failed = true;
-        }
-      });
-    });
-    await Promise.all(promises);
-    if (failed) return tiffs.filter((f) => f.images.length > 0);
-    return tiffs;
-  }
-
-  public async tile(req: LambdaHttpRequest, xyz: TileDataXyz): Promise<LambdaHttpResponse> {
-    if (xyz.ext === VectorFormat.MapboxVectorTiles) return NotFound;
-    const tiffs = await this.initTiffs(xyz, req.log);
-    const layers = await this.tiler.tile(tiffs, xyz.x, xyz.y, xyz.z);
-
-    // Generate a unique hash given the full URI, the layers used and a renderId
-    const cacheKey = TileEtag.generate(layers, xyz);
-    req.set('layers', layers.length);
-    if (TileEtag.isNotModified(req, cacheKey)) return NotModified;
-
-    const res = await TileComposer.compose({
-      layers,
-      format: xyz.ext,
-      background: this.tileSet.background ?? DefaultBackground,
-      resizeKernel: this.tileSet.resizeKernel ?? DefaultResizeKernel,
-      metrics: req.timer,
-    });
-
-    req.set('layersUsed', res.layers);
-    req.set('bytes', res.buffer.byteLength);
-
-    const response = new LambdaHttpResponse(200, 'ok');
-    response.header(HttpHeader.ETag, cacheKey);
-    response.header(HttpHeader.CacheControl, 'public, max-age=604800');
-    response.buffer(res.buffer, 'image/' + xyz.ext);
-    return response;
-  }
-
-  /**
-   * Get a list of tiffs in the rendering order that is needed to render the tile
-   * @param tms tile matrix set to describe the tiling scheme
-   * @param tile tile to render
-   */
-  public getTiffsForTile(tile: Tile, log?: LogType): CogTiff[] {
-    const output: CogTiff[] = [];
-    const tileBounds = this.tileMatrix.tileToSourceBounds(tile);
-
-    // All zoom level config is stored as Google zoom levels
-    const filterZoom = TileMatrixSet.convertZoomLevel(tile.z, this.tileMatrix, TileMatrixSets.get(Epsg.Google));
-    for (const layer of this.tileSet.layers) {
-      if (layer.maxZoom != null && filterZoom > layer.maxZoom) continue;
-      if (layer.minZoom != null && filterZoom < layer.minZoom) continue;
-
-      const imgId = layer[this.tileMatrix.projection.code];
-      if (imgId == null) {
-        log?.warn({ layer: layer.name, projection: this.tileMatrix.projection.code }, 'Failed to lookup imagery');
-        continue;
-      }
-
-      const imagery = this.imagery.get(imgId);
-      if (imagery == null) {
-        log?.warn(
-          { layer: layer.name, projection: this.tileMatrix.projection.code, imgId },
-          'Failed to lookup imagery',
-        );
-        continue;
-      }
-      if (!tileBounds.intersects(Bounds.fromJson(imagery.bounds))) continue;
-
-      for (const tiff of this.getCogsForTile(imagery, tileBounds)) output.push(tiff);
-    }
-    return output;
-  }
-
-  private getCogsForTile(record: ConfigImagery, tileBounds: Bounds): CogTiff[] {
-    const output: CogTiff[] = [];
-    for (const c of record.files) {
-      if (!tileBounds.intersects(Bounds.fromJson(c))) continue;
-      const tiffPath = fsa.join(record.uri, getTiffName(c.name));
-
-      let existing = TiffCache.get(tiffPath);
-      if (existing == null) {
-        const source = fsa.source(tiffPath);
-        if (source == null) {
-          throw new Error(`Failed to create CogSource from  ${tiffPath}`);
-        }
-
-        St.trace(source);
-        existing = new CogTiff(source);
-        TiffCache.set(tiffPath, existing);
-      }
-
-      output.push(existing);
-    }
-
-    return output;
-  }
-
-  /** Look up imagery by imageryId or by image name */
-  findImagery(imgId: string): ConfigImagery | null {
-    const existing = this.imagery.get(imgId);
-    if (existing != null) return existing;
-    for (const img of this.imagery.values()) {
-      if (img.name === imgId) return img;
-    }
-    return null;
-  }
-
-  child(imgId: string): TileSetRaster | null {
-    const image = this.findImagery(imgId);
-    if (image == null) return null;
-    const childName = TileSetNameParser.componentsToName({ ...this.components, layer: image.name });
-    const child = new TileSetRaster(childName, this.tileMatrix);
-    // use parent data as prototype for child;
-    child.tileSet = { ...this.tileSet };
-    child.tileSet.background = undefined;
-    const title = this.tileSet?.title ?? this.tileSet?.name;
-    child.tileSet.title = `${title} ${titleizeImageryName(image.name)}`;
-    child.extentOverride = Bounds.fromJson(image.bounds);
-
-    const layer: ConfigLayer = { name: image.name, minZoom: 0, maxZoom: 100 };
-    layer[this.tileMatrix.projection.code] = image.id;
-
-    child.tileSet.layers = [layer];
-    child.imagery = new Map();
-    child.imagery.set(image.id, image);
-
-    return child;
-  }
-}

package/src/tile.set.ts

@@ -1,4 +0,0 @@
-import { TileSetRaster } from './tile.set.raster.js';
-import { TileSetVector } from './tile.set.vector.js';
-
-export type TileSet = TileSetVector | TileSetRaster;

package/src/tile.set.vector.ts

@@ -1,79 +0,0 @@
-import { ConfigTileSetVector, TileSetNameComponents, TileSetNameParser, TileSetType } from '@basemaps/config';
-import { GoogleTms, TileMatrixSet, VectorFormat } from '@basemaps/geo';
-import { fsa, TileDataXyz } from '@basemaps/shared';
-import { Cotar } from '@cotar/core';
-import { HttpHeader, LambdaHttpRequest, LambdaHttpResponse } from '@linzjs/lambda';
-import { NotFound } from './routes/response.js';
-import { TileSets } from './tile.set.cache.js';
-import { St } from './source.tracer.js';
-
-class CotarCache {
-  cache = new Map<string, Promise<Cotar | null>>();
-
-  get(uri: string): Promise<Cotar | null> {
-    let cotar = this.cache.get(uri);
-    if (cotar == null) {
-      const source = fsa.source(uri);
-      St.trace(source);
-      cotar = Cotar.fromTar(source);
-      this.cache.set(uri, cotar);
-    }
-    return cotar;
-  }
-}
-
-export const Layers = new CotarCache();
-
-export class TileSetVector {
-  type: TileSetType.Vector = TileSetType.Vector;
-  components: TileSetNameComponents;
-  tileMatrix: TileMatrixSet;
-  tileSet: ConfigTileSetVector;
-  constructor(name: string, tileMatrix: TileMatrixSet) {
-    this.components = TileSetNameParser.parse(name);
-    this.tileMatrix = tileMatrix;
-  }
-
-  async init(record: ConfigTileSetVector): Promise<void> {
-    this.tileSet = record;
-  }
-
-  /** What format does tile set use */
-  get format(): VectorFormat {
-    return VectorFormat.MapboxVectorTiles;
-  }
-
-  get id(): string {
-    return TileSets.id(this.fullName, this.tileMatrix);
-  }
-
-  get fullName(): string {
-    return TileSetNameParser.componentsToName(this.components);
-  }
-
-  async tile(req: LambdaHttpRequest, xyz: TileDataXyz): Promise<LambdaHttpResponse> {
-    if (xyz.ext !== VectorFormat.MapboxVectorTiles) return NotFound;
-    if (xyz.tileMatrix.identifier !== GoogleTms.identifier) return NotFound;
-    if (this.tileSet.layers.length > 1) return new LambdaHttpResponse(500, 'Too many layers in tileset');
-    const [layer] = this.tileSet.layers;
-    if (layer[3857] == null) return new LambdaHttpResponse(500, 'Layer url not found from tileset Config');
-
-    req.timer.start('cotar:load');
-    const cotar = await Layers.get(layer[3857]);
-    if (cotar == null) return new LambdaHttpResponse(500, 'Failed to load VectorTiles');
-    req.timer.end('cotar:load');
-
-    // Flip Y coordinate because MBTiles files are TMS.
-    const y = (1 << xyz.z) - 1 - xyz.y;
-
-    req.timer.start('cotar:tile');
-    const tile = await cotar.get(`tiles/${xyz.z}/${xyz.x}/${y}.pbf.gz`);
-    if (tile == null) return NotFound;
-    req.timer.end('cotar:tile');
-
-    const response = new LambdaHttpResponse(200, 'Ok');
-    response.buffer(Buffer.from(tile), 'application/x-protobuf');
-    response.header(HttpHeader.ContentEncoding, 'gzip');
-    return response;
-  }
-}

package/src/validate.ts

@@ -1,32 +0,0 @@
-import { Projection, TileDataXyz } from '@basemaps/shared';
-import { LambdaHttpRequest, LambdaHttpResponse } from '@linzjs/lambda';
-
-export const ValidateTilePath = {
-  /**
-   * Validate that the tile request is somewhat valid
-   * - Valid projection
-   * - Valid range
-   *
-   * @throws LambdaHttpResponse for tile requests that are not valid
-   *
-   * @param req request to validate
-   * @param xyzData
-   */
-  validate(req: LambdaHttpRequest, xyzData: TileDataXyz): void {
-    const { tileMatrix, x, y, z, ext } = xyzData;
-    req.set('xyz', { x, y, z });
-    req.set('projection', tileMatrix.projection.code);
-    req.set('tileMatrix', tileMatrix.identifier);
-    req.set('extension', ext);
-    req.set('tileSet', xyzData.name);
-
-    if (z > tileMatrix.maxZoom || z < 0) throw new LambdaHttpResponse(404, `Zoom not found: ${z}`);
-
-    const zoom = tileMatrix.zooms[z];
-    if (x < 0 || x > zoom.matrixWidth) throw new LambdaHttpResponse(404, `X not found: ${x}`);
-    if (y < 0 || y > zoom.matrixHeight) throw new LambdaHttpResponse(404, `Y not found: ${y}`);
-
-    const latLon = Projection.tileCenterToLatLon(tileMatrix, xyzData);
-    req.set('location', latLon);
-  },
-};