@basemaps/lambda-tiler 6.34.0 → 6.35.0

package/src/util/__test__/config.loader.test.ts

@@ -0,0 +1,116 @@
+import { base58, ConfigProviderMemory } from '@basemaps/config';
+import { fsa } from '@basemaps/shared';
+import { LambdaHttpResponse } from '@linzjs/lambda';
+import o from 'ospec';
+import { createSandbox } from 'sinon';
+import { FsMemory } from '../../routes/__tests__/memory.fs.js';
+import { FakeData } from '../../__tests__/config.data.js';
+import { Api, mockRequest, mockUrlRequest } from '../../__tests__/xyz.util.js';
+import { CachedConfig } from '../config.cache.js';
+import { ConfigLoader } from '../config.loader.js';
+
+o.spec('ConfigLoader', () => {
+  const memory = new FsMemory();
+  const config = new ConfigProviderMemory();
+  const sandbox = createSandbox();
+
+  o.before(() => {
+    fsa.register('memory://', memory);
+  });
+
+  o.beforeEach(() => {
+    sandbox.stub(ConfigLoader, 'getDefaultConfig').resolves(config);
+  });
+
+  o.afterEach(() => {
+    sandbox.restore();
+    config.objects.clear();
+    memory.files.clear();
+    CachedConfig.cache.clear();
+  });
+
+  o('should return default config', async () => {
+    const provider = await ConfigLoader.load(mockRequest('/v1/fonts.json'));
+    o(provider).deepEquals(config);
+  });
+
+  o('should Not working with wrong config url', async () => {
+    const error = await ConfigLoader.load(
+      mockUrlRequest('/v1/tiles/🦄 🌈/NZTM2000Quad/tile.json', `?config=notapath`, Api.header),
+    )
+      .then(() => null)
+      .catch((e) => e);
+
+    o(error instanceof LambdaHttpResponse).equals(true);
+    o((error as LambdaHttpResponse).status).equals(400);
+    o((error as LambdaHttpResponse).statusDescription).equals('Invalid config location');
+  });
+
+  o('should Not working with wrong protocol', async () => {
+    const error = await ConfigLoader.load(
+      mockUrlRequest('/v1/tiles/🦄 🌈/NZTM2000Quad/tile.json', `?config=memory1://linz-basemaps/config`, Api.header),
+    )
+      .then(() => null)
+      .catch((e) => e);
+
+    o(error instanceof LambdaHttpResponse).equals(true);
+    o((error as LambdaHttpResponse).status).equals(400);
+    o((error as LambdaHttpResponse).statusDescription).equals('Invalid configuration location protocol:memory1');
+  });
+
+  o('should Not working with wrong s3 bucket', async () => {
+    const error = await ConfigLoader.load(
+      mockUrlRequest('/v1/tiles/🦄 🌈/NZTM2000Quad/tile.json', `?config=s3://wrong-bucket/config`, Api.header),
+    )
+      .then(() => null)
+      .catch((e) => e);
+
+    o(error instanceof LambdaHttpResponse).equals(true);
+    o((error as LambdaHttpResponse).status).equals(400);
+    o((error as LambdaHttpResponse).statusDescription).equals(
+      'Bucket: "wrong-bucket" is not a allowed bucket location',
+    );
+  });
+
+  const location = 'memory://linz-basemaps/config.json';
+
+  o('should Not working with no file in the path', async () => {
+    const error = await ConfigLoader.load(
+      mockUrlRequest('/v1/tiles/🦄 🌈/NZTM2000Quad/tile.json', `?config=${location}`, Api.header),
+    )
+      .then(() => null)
+      .catch((e) => e);
+
+    o(error instanceof LambdaHttpResponse).equals(true);
+    o((error as LambdaHttpResponse).status).equals(404);
+    o((error as LambdaHttpResponse).statusDescription).equals(`Config not found at ${location}`);
+  });
+
+  o('should get expected config file', async () => {
+    const expectedConfig = new ConfigProviderMemory();
+    expectedConfig.put(FakeData.tileSetRaster('aerial'));
+    await fsa.write(location, Buffer.from(JSON.stringify(expectedConfig.toJson())));
+
+    const provider = await ConfigLoader.load(
+      mockUrlRequest('/v1/tiles/🦄 🌈/NZTM2000Quad/tile.json', `?config=${location}`, Api.header),
+    );
+
+    o(await provider.Imagery.get('aerial')).deepEquals(await expectedConfig.Imagery.get('aerial'));
+  });
+
+  o('should get expected config file with base58 location', async () => {
+    const expectedConfig = new ConfigProviderMemory();
+    expectedConfig.put(FakeData.tileSetVector('topographic'));
+    await fsa.write(location, Buffer.from(JSON.stringify(expectedConfig.toJson())));
+
+    const provider = await ConfigLoader.load(
+      mockUrlRequest(
+        '/v1/tiles/🦄 🌈/NZTM2000Quad/tile.json',
+        `?config=${base58.encode(Buffer.from(location))}`,
+        Api.header,
+      ),
+    );
+
+    o(await provider.Imagery.get('topographic')).deepEquals(await expectedConfig.Imagery.get('topographic'));
+  });
+});

package/src/util/assets.provider.ts

@@ -1,5 +1,7 @@
+import { Env } from '@basemaps/shared';
 import { fsa } from '@chunkd/fs';
 import { LambdaHttpResponse, LambdaHttpRequest, HttpHeader } from '@linzjs/lambda';
+import { ConfigLoader } from './config.loader.js';
 import { isGzip } from './cotar.serve.js';
 import { Etag } from './etag.js';
 import { NotFound, NotModified } from './response.js';
@@ -15,21 +17,13 @@ export class AssetProvider {
    * s3://linz-basemaps/assets/assets-b4ff211a.tar.co # Remote Cotar
    */
 
-  /** Path of the assets location */
-  path: string | undefined;
-
-  set(path?: string): void {
-    this.path = path;
-  }
-
-  async get(fileName: string): Promise<Buffer | null> {
-    if (this.path == null) return null;
+  async get(path: string, fileName: string): Promise<Buffer | null> {
     // get assets file from cotar
-    if (this.path.endsWith('.tar.co')) return await this.getFromCotar(this.path, fileName);
+    if (path.endsWith('.tar.co')) return await this.getFromCotar(path, fileName);
 
     // get assets file for directory
     try {
-      const filePath = fsa.join(this.path, fileName);
+      const filePath = fsa.join(path, fileName);
       return await fsa.read(filePath);
     } catch (e: any) {
       if (e.code === 404) return null;
@@ -51,7 +45,12 @@ export class AssetProvider {
    * - Content-Type from the parameter contentType
    */
   async serve(req: LambdaHttpRequest, file: string, contentType: string): Promise<LambdaHttpResponse> {
-    const buf = await assetProvider.get(file);
+    const config = await ConfigLoader.load(req);
+    let assetLocation = Env.get(Env.AssetLocation);
+    if (config == null) return NotFound();
+    if (config.assets != null) assetLocation = config.assets;
+    if (assetLocation == null) return NotFound();
+    const buf = await assetProvider.get(assetLocation, file);
     if (buf == null) return NotFound();
     const cacheKey = Etag.key(buf);
     if (Etag.isNotModified(req, cacheKey)) return NotModified();

package/src/util/config.cache.ts

@@ -1,16 +1,21 @@
 import { ConfigBundled, ConfigProviderMemory } from '@basemaps/config';
-import { fsa } from '@chunkd/fs';
+import { fsa } from '@basemaps/shared';
 import { SwappingLru } from './swapping.lru.js';
 
 class LruConfig {
-  configProvider: Promise<ConfigProviderMemory>;
+  configProvider: Promise<ConfigProviderMemory | null>;
 
   constructor(config: Promise<ConfigBundled>) {
-    this.configProvider = config.then((c) => {
-      const configProvider = ConfigProviderMemory.fromJson(c);
-      configProvider.createVirtualTileSets();
-      return configProvider;
-    });
+    this.configProvider = config
+      .then((c) => {
+        const configProvider = ConfigProviderMemory.fromJson(c);
+        configProvider.createVirtualTileSets();
+        return configProvider;
+      })
+      .catch((e) => {
+        if (e.code === 404) return null;
+        throw e;
+      });
   }
 
   get size(): number {
@@ -25,20 +30,15 @@ export class ConfigCache {
     this.cache = new SwappingLru<LruConfig>(maxSize);
   }
 
-  getConfig(location: string): Promise<ConfigProviderMemory | null> {
+  get(location: string): Promise<ConfigProviderMemory | null> {
     const existing = this.cache.get(location)?.configProvider;
     if (existing != null) return existing;
-    try {
-      const configJson = fsa.readJson<ConfigBundled>(location);
-      const config = new LruConfig(configJson);
-      this.cache.set(location, config);
-      return config.configProvider;
-    } catch (e: any) {
-      if (e.code === 404) return Promise.resolve(null);
-      throw e;
-    }
+    const configJson = fsa.readJson<ConfigBundled>(location);
+    const config = new LruConfig(configJson);
+    this.cache.set(location, config);
+    return config.configProvider;
   }
 }
 
-/** Cache 20 configs(Around 500KB each)*/
+/** Cache 20 configs (Around <30KB -> 5MB each)*/
 export const CachedConfig = new ConfigCache(20);

package/src/util/config.loader.ts

@@ -0,0 +1,50 @@
+import { base58, BasemapsConfigProvider, isBase58 } from '@basemaps/config';
+import { LambdaHttpResponse } from '@linzjs/lambda';
+import { parseUri } from '@chunkd/core';
+import { LambdaHttpRequest } from '@linzjs/lambda';
+import { CachedConfig } from './config.cache.js';
+import { getDefaultConfig } from '@basemaps/shared';
+
+// FIXME load this from process.env COG BUCKETS?
+const SafeBuckets = new Set(['linz-workflow-artifacts', 'linz-basemaps']);
+const SafeProtocols = new Set(['s3', 'memory']);
+
+export class ConfigLoader {
+  /** Exposed for testing */
+  static async getDefaultConfig(): Promise<BasemapsConfigProvider> {
+    return getDefaultConfig();
+  }
+
+  /** Lookup the config path from a request and return a standardized location */
+  static extract(req: LambdaHttpRequest): string | null {
+    const rawLocation = req.query.get('config');
+    if (rawLocation == null) return null;
+    if (rawLocation.includes('/')) return base58.encode(Buffer.from(rawLocation));
+    return rawLocation;
+  }
+
+  static async load(req: LambdaHttpRequest): Promise<BasemapsConfigProvider> {
+    const rawLocation = req.query.get('config');
+    if (rawLocation == null) return this.getDefaultConfig();
+
+    const configLocation = isBase58(rawLocation) ? Buffer.from(base58.decode(rawLocation)).toString() : rawLocation;
+
+    const r = parseUri(configLocation);
+
+    if (r == null) throw new LambdaHttpResponse(400, 'Invalid config location');
+    if (!SafeProtocols.has(r.protocol)) {
+      throw new LambdaHttpResponse(400, `Invalid configuration location protocol:${r.protocol}`);
+    }
+    if (!SafeBuckets.has(r.bucket)) {
+      throw new LambdaHttpResponse(400, `Bucket: "${r.bucket}" is not a allowed bucket location`);
+    }
+
+    req.set('config', configLocation);
+    req.timer.start('config:load');
+    return CachedConfig.get(configLocation).then((f) => {
+      req.timer.end('config:load');
+      if (f == null) throw new LambdaHttpResponse(404, `Config not found at ${configLocation}`);
+      return f;
+    });
+  }
+}

package/src/wmts.capability.ts

@@ -1,6 +1,6 @@
-import { Config, ConfigImagery, ConfigLayer, ConfigTileSet, standardizeLayerName } from '@basemaps/config';
+import { ConfigImagery, ConfigLayer, ConfigTileSet, standardizeLayerName } from '@basemaps/config';
 import { Bounds, GoogleTms, ImageFormat, TileMatrixSet, WmtsProvider } from '@basemaps/geo';
-import { Projection, V, VNodeElement } from '@basemaps/shared';
+import { Projection, toQueryString, V, VNodeElement } from '@basemaps/shared';
 import { ImageFormatOrder } from '@basemaps/tiler';
 import { BoundingBox } from '@cogeotiff/core';
 import { BBox } from '@linzjs/geojson';
@@ -36,6 +36,8 @@ export interface WmtsCapabilitiesParams {
   apiKey?: string;
   /** Limit the output to the following image formats other wise @see ImageFormatOrder */
   formats?: ImageFormat[] | null;
+  /** Config location */
+  config?: string | null;
 }
 
 /** Number of decimal places to use in lat lng */
@@ -56,6 +58,7 @@ export class WmtsCapabilities {
   provider?: WmtsProvider;
   tileSet: ConfigTileSet;
   apiKey?: string;
+  config?: string | null;
   tileMatrixSets = new Map<string, TileMatrixSet>();
   imagery: Map<string, ConfigImagery>;
   formats: ImageFormat[];
@@ -65,6 +68,7 @@ export class WmtsCapabilities {
     this.httpBase = params.httpBase;
     this.provider = params.provider;
     this.tileSet = params.tileSet;
+    this.config = params.config;
     this.isIndividualLayers = params.isIndividualLayers;
     for (const tms of params.tileMatrix) this.tileMatrixSets.set(tms.identifier, tms);
     this.apiKey = params.apiKey;
@@ -72,17 +76,6 @@ export class WmtsCapabilities {
     this.imagery = params.imagery;
   }
 
-  async loadImagery(): Promise<void> {
-    const ids = new Set<string>();
-    for (const tms of this.tileMatrixSets.values()) {
-      for (const layer of this.tileSet.layers) {
-        const layerId = layer[tms.projection.code];
-        if (layerId != null) ids.add(layerId);
-      }
-    }
-    this.imagery = await Config.Imagery.getAll(ids);
-  }
-
   buildWgs84BoundingBox(tms: TileMatrixSet, layers: Bounds[]): VNodeElement {
     let bbox: BBox;
     if (layers.length > 0) {
@@ -164,7 +157,8 @@ export class WmtsCapabilities {
   }
 
   buildTileUrl(tileSetId: string, suffix: string): string {
-    const apiSuffix = this.apiKey ? `?api=${this.apiKey}` : '';
+    const query = toQueryString({ api: this.apiKey, config: this.config });
+
     return [
       this.httpBase,
       'v1',
@@ -173,7 +167,7 @@ export class WmtsCapabilities {
       '{TileMatrixSet}',
       '{TileMatrix}',
       '{TileCol}',
-      `{TileRow}.${suffix}${apiSuffix}`,
+      `{TileRow}.${suffix}${query}`,
     ].join('/');
   }