@basemaps/lambda-tiler 6.16.1 → 6.20.0

package/src/__test__/xyz.test.ts

@@ -3,17 +3,21 @@ import { TileMatrixSets } from '@basemaps/geo';
 import { Config, Env, LogConfig, VNodeParser } from '@basemaps/shared';
 import { round } from '@basemaps/test/build/rounding.js';
 import o from 'ospec';
+import sinon from 'sinon';
 import { handleRequest } from '../index.js';
-import { TileComposer } from '../routes/tile.js';
 import { TileEtag } from '../routes/tile.etag.js';
 import { TileSets } from '../tile.set.cache.js';
+import { TileComposer } from '../tile.set.raster.js';
 import { FakeTileSet, mockRequest, Provider } from './xyz.util.js';
-import sinon from 'sinon';
+
 const sandbox = sinon.createSandbox();
 
 const TileSetNames = ['aerial', 'aerial@head', 'aerial@beta', '01E7PJFR9AMQFJ05X9G7FQ3XMW'];
 /* eslint-disable @typescript-eslint/explicit-function-return-type */
 o.spec('LambdaXyz', () => {
+  const host = 'https://tiles.test';
+  const origPublicUrlBase = process.env[Env.PublicUrlBase];
+
   /** Generate mock ALBEvent */
 
   let rasterMock = o.spy();
@@ -26,6 +30,8 @@ o.spec('LambdaXyz', () => {
   const apiKeyHeader = { 'x-linz-api-key': 'd01f7w7rnhdzg0p7fyrc9v9ard1' };
 
   o.beforeEach(() => {
+    process.env[Env.PublicUrlBase] = host;
+
     LogConfig.disable();
     // tileMock = o.spy(() => tileMockData) as any;
     rasterMock = o.spy(() => {
@@ -54,6 +60,7 @@ o.spec('LambdaXyz', () => {
     TileSets.cache.clear();
     TileComposer.compose = origCompose;
     TileEtag.generate = origTileEtag;
+    process.env[Env.PublicUrlBase] = origPublicUrlBase;
     sandbox.restore();
   });
 
@@ -134,12 +141,8 @@ o.spec('LambdaXyz', () => {
   });
 
   o.spec('WMTSCapabilities', () => {
-    const origPublicUrlBase = process.env[Env.PublicUrlBase];
-    o.after(() => {
-      process.env[Env.PublicUrlBase] = origPublicUrlBase;
-    });
-
     o('should 304 if a xml is not modified', async () => {
+      delete process.env[Env.PublicUrlBase];
       o.timeout(1000);
       const key = 'NuirTK8fozzCJV1iG1FznmdHhKvk6WaWuDhhEA1d40c=';
       const request = mockRequest('/v1/tiles/WMTSCapabilities.xml', 'get', {
@@ -148,6 +151,7 @@ o.spec('LambdaXyz', () => {
       });
 
       const res = await handleRequest(request);
+
       if (res.status === 200) {
         o(res.header('eTaG')).equals(key); // this line is useful for discovering the new etag
         return;
@@ -160,9 +164,6 @@ o.spec('LambdaXyz', () => {
     });
 
     o('should serve WMTSCapabilities for tile_set', async () => {
-      console.log('\n\nTestStart');
-      process.env[Env.PublicUrlBase] = 'https://tiles.test';
-
       const request = mockRequest('/v1/tiles/aerial@beta/WMTSCapabilities.xml', 'get', apiKeyHeader);
 
       const res = await handleRequest(request);
@@ -185,13 +186,10 @@ o.spec('LambdaXyz', () => {
   });
 
   o.spec('tileJson', () => {
-    const origPublicUrlBase = process.env[Env.PublicUrlBase];
-    o.after(() => {
-      process.env[Env.PublicUrlBase] = origPublicUrlBase;
-    });
-
     o('should 304 if a json is not modified', async () => {
-      const key = 'XecTdbcdjCyzB1MHOOQbrOkD2TTJ0ORh4JuXqhxXEE0=';
+      // delete process.env[Env.PublicUrlBase];
+
+      const key = 'BBfQpNXA3Q90jlFrLSOZhxbvfOh7OpN1OEE+BylpbHw=';
       const request = mockRequest('/v1/tiles/tile.json', 'get', { 'if-none-match': key, ...apiKeyHeader });
 
       const res = await handleRequest(request);
@@ -206,10 +204,20 @@ o.spec('LambdaXyz', () => {
       o(request.logContext['cache']).deepEquals({ key, match: key, hit: true });
     });
 
-    o('should serve tile json for tile_set', async () => {
-      process.env[Env.PublicUrlBase] = 'https://tiles.test';
+    o('should 200 if a invalid etag is given', async () => {
+      const key = 'ABCXecTdbcdjCyzB1MHOOQbrOkD2TTJ0ORh4JuXqhxXEE0=';
+      const request = mockRequest('/v1/tiles/tile.json', 'get', { 'if-none-match': key, ...apiKeyHeader });
 
-      const request = mockRequest('/v1/tiles/topolike/Google/tile.json', 'get', apiKeyHeader);
+      const res = await handleRequest(request);
+      o(res.status).equals(200);
+      o(res.header('etag')).equals('BBfQpNXA3Q90jlFrLSOZhxbvfOh7OpN1OEE+BylpbHw=');
+      const out = JSON.parse(Buffer.from(res.body ?? '', 'base64').toString());
+      o(out.tiles[0].startsWith('https://tiles.test/v1/tiles/tile.json/undefined/{z}/{x}/{y}.pbf?api=')).equals(true);
+      o(request.logContext['cache']).deepEquals(undefined);
+    });
+
+    o('should serve tile json for tile_set', async () => {
+      const request = mockRequest('/v1/tiles/topographic/Google/tile.json', 'get', apiKeyHeader);
 
       const res = await handleRequest(request);
       o(res.status).equals(200);
@@ -218,7 +226,7 @@ o.spec('LambdaXyz', () => {
 
       const body = Buffer.from(res.body ?? '', 'base64').toString();
       o(JSON.parse(body)).deepEquals({
-        tiles: [`https://tiles.test/v1/tiles/topolike/Google/{z}/{x}/{y}.pbf?api=${apiKey}`],
+        tiles: [`https://tiles.test/v1/tiles/topographic/Google/{z}/{x}/{y}.pbf?api=${apiKey}`],
         minzoom: 0,
         maxzoom: 15,
         format: 'pbf',
@@ -228,15 +236,8 @@ o.spec('LambdaXyz', () => {
   });
 
   o.spec('styleJson', () => {
-    const origPublicUrlBase = process.env[Env.PublicUrlBase];
-    o.after(() => {
-      process.env[Env.PublicUrlBase] = origPublicUrlBase;
-    });
-
     o('should not found style json', async () => {
-      process.env[Env.PublicUrlBase] = 'https://tiles.test';
-
-      const request = mockRequest('/v1/tiles/topolike/Google/style/topolike.json', 'get', apiKeyHeader);
+      const request = mockRequest('/v1/tiles/topographic/Google/style/topographic.json', 'get', apiKeyHeader);
 
       sandbox.stub(Config.Style, 'get').resolves(null);
 
@@ -245,27 +246,24 @@ o.spec('LambdaXyz', () => {
     });
 
     o('should serve style json', async () => {
-      const host = 'https://tiles.test';
-      process.env[Env.PublicUrlBase] = host;
-
-      const request = mockRequest('/v1/tiles/topolike/Google/style/topolike.json', 'get', apiKeyHeader);
+      const request = mockRequest('/v1/tiles/topographic/Google/style/topographic.json', 'get', apiKeyHeader);
 
       const fakeStyle: StyleJson = {
         version: 8,
         id: 'test',
-        name: 'topolike',
+        name: 'topographic',
         sources: {
           basemaps_vector: {
             type: 'vector',
-            url: `${host}/vector`,
+            url: `/vector`,
           },
           basemaps_raster: {
             type: 'raster',
-            tiles: [`${host}/raster`],
+            tiles: [`/raster`],
           },
           basemaps_raster_encode: {
             type: 'raster',
-            tiles: [`${host}/raster/{z}/{x}/{y}.webp`], // Shouldn't encode the {}
+            tiles: [`/raster/{z}/{x}/{y}.webp`], // Shouldn't encode the {}
           },
           test_vector: {
             type: 'vector',
@@ -289,14 +287,14 @@ o.spec('LambdaXyz', () => {
             minzoom: 0,
           },
         ],
-        glyphs: 'glyphs',
-        sprite: 'sprite',
+        glyphs: '/glyphs',
+        sprite: '/sprite',
         metadata: { id: 'test' },
       };
 
       const fakeRecord = {
-        id: 'st_topolike_production',
-        name: 'topolike',
+        id: 'st_topographic_production',
+        name: 'topographic',
         style: fakeStyle,
       };
 
@@ -305,7 +303,7 @@ o.spec('LambdaXyz', () => {
       const res = await handleRequest(request);
       o(res.status).equals(200);
       o(res.header('content-type')).equals('application/json');
-      o(res.header('cache-control')).equals('max-age=120');
+      o(res.header('cache-control')).equals('no-store');
 
       const body = Buffer.from(res.body ?? '', 'base64').toString();
       fakeStyle.sources.basemaps_vector = {
@@ -320,6 +318,10 @@ o.spec('LambdaXyz', () => {
         type: 'raster',
         tiles: [`${host}/raster/{z}/{x}/{y}.webp?api=${apiKey}`],
       };
+
+      fakeStyle.sprite = `${host}/sprite`;
+      fakeStyle.glyphs = `${host}/glyphs`;
+
       o(JSON.parse(body)).deepEquals(fakeStyle);
     });
   });

package/src/cli/dump.ts

@@ -4,7 +4,7 @@ import { ImageFormat } from '@basemaps/tiler';
 import { LambdaAlbRequest } from '@linzjs/lambda';
 import { Context } from 'aws-lambda';
 import { promises as fs } from 'fs';
-import { TileRoute } from '../routes/tile.js';
+import { tileXyz } from '../routes/tile.xyz.js';
 import { TileSets } from '../tile.set.cache.js';
 import { TileSet } from '../tile.set.js';
 import { TileSetLocal } from './tile.set.local.js';
@@ -48,7 +48,7 @@ async function main(): Promise<void> {
     logger,
   );
 
-  const tileData = await TileRoute.tile(ctx);
+  const tileData = await tileXyz(ctx);
 
   const headers: Record<string, any> = {};
   for (const [key, value] of tileData.headers) headers[key] = value;

package/src/index.ts

@@ -4,6 +4,9 @@ import { Ping, Version } from './routes/api.js';
 import { Health } from './routes/health.js';
 import { Tiles } from './routes/tile.js';
 import { Router } from './router.js';
+import { createHash } from 'crypto';
+import { Imagery } from './routes/imagery.js';
+import { Esri } from './routes/esri/rest.js';
 
 const app = new Router();
 
@@ -11,6 +14,8 @@ app.get('ping', Ping);
 app.get('health', Health);
 app.get('version', Version);
 app.get('tiles', Tiles);
+app.get('imagery', Imagery);
+app.get('esri', Esri);
 
 let slowTimer: NodeJS.Timer | null = null;
 export async function handleRequest(req: LambdaHttpRequest): Promise<LambdaHttpResponse> {
@@ -21,6 +26,11 @@ export async function handleRequest(req: LambdaHttpRequest): Promise<LambdaHttpR
 
   req.set('name', 'LambdaTiler');
   try {
+    const apiKey = Router.apiKey(req);
+    if (apiKey != null) {
+      const apiKeyHash = createHash('sha256').update(apiKey).digest('base64');
+      req.set('api', apiKeyHash);
+    }
     return await app.handle(req);
   } finally {
     if (slowTimer) clearTimeout(slowTimer);

package/src/routes/esri/rest.ts

@@ -0,0 +1,90 @@
+import { GoogleTms, TileMatrixSet } from '@basemaps/geo';
+import { tileXyzFromPath } from '@basemaps/shared';
+import { LambdaHttpRequest, LambdaHttpResponse } from '@linzjs/lambda';
+import { Router } from '../../router.js';
+import { TileSets } from '../../tile.set.cache.js';
+import { NotFound } from '../response.js';
+
+export async function vectorTileServer(
+  req: LambdaHttpRequest,
+  layerId: string,
+  tms: TileMatrixSet,
+): Promise<LambdaHttpResponse> {
+  if (tms.identifier !== GoogleTms.identifier) return NotFound;
+  const extent = {
+    xmin: tms.extent.x,
+    ymin: tms.extent.y,
+    xmax: tms.extent.right,
+    ymax: tms.extent.bottom,
+    // TODO where is wkid from
+    spatialReference: { wkid: 102100, latestWkid: tms.projection.code },
+  };
+
+  const res = new LambdaHttpResponse(200, 'ok');
+  res.json({
+    currentVersion: 10.4,
+    name: layerId,
+    capabilities: 'TilesOnly',
+    type: 'indexedVector',
+    tileMap: 'tilemap',
+    defaultStyles: 'resources/styles',
+    tiles: ['tiles/{z}/{x}/{y}.pbf'],
+    exportTilesAllowed: false,
+    maxExportTilesCount: 0,
+    initialExtent: extent,
+    fullExtent: extent,
+    minScale: tms.zooms[0].scaleDenominator,
+    maxScale: tms.zooms[tms.zooms.length - 1].scaleDenominator,
+    tileInfo: {
+      // TODO are all the pbf 256x256?
+      rows: 256,
+      cols: 256,
+      dpi: 96,
+      format: 'pbf',
+      origin: { x: tms.extent.x, y: tms.extent.bottom },
+      spatialReference: { wkid: 102100, latestWkid: tms.projection.code },
+      lods: tms.zooms.map((c, i) => {
+        return {
+          level: i,
+          scale: c.scaleDenominator,
+          resolution: c.scaleDenominator * 0.28e-3,
+        };
+      }),
+    },
+    resourceInfo: {
+      styleVersion: 8,
+      tileCompression: 'gzip',
+      cacheInfo: { storageInfo: { packetSize: 128, storageFormat: 'compactV2' } },
+    },
+  });
+  return res;
+}
+
+/**
+ * /v1/esri/services/:layerId/VectorTileServer
+ *
+ * @example http://localhost:5000/v1/esri/services/topographic/VectorTileServer
+ */
+export async function Esri(req: LambdaHttpRequest): Promise<LambdaHttpResponse> {
+  const { rest } = Router.action(req);
+  if (rest[0] !== 'services') return NotFound;
+  const layerId = rest[1];
+  if (layerId == null) return NotFound;
+
+  const serviceId = rest[2];
+  if (serviceId !== 'VectorTileServer') return NotFound;
+  if (rest.length === 3) return vectorTileServer(req, layerId, GoogleTms);
+
+  if (rest[rest.length - 1].endsWith('.pbf')) {
+    const generatedPath = [layerId, GoogleTms.identifier, ...rest.slice(rest.length - 3)];
+    const xyz = tileXyzFromPath(generatedPath);
+    if (xyz == null) return NotFound;
+    req.timer.start('tileset:load');
+    const tileSet = await TileSets.get(xyz.name, xyz.tileMatrix);
+    req.timer.end('tileset:load');
+    if (tileSet == null) return NotFound;
+    return await tileSet.tile(req, xyz);
+  }
+
+  return new LambdaHttpResponse(200, 'ok');
+}

package/src/routes/imagery.ts

@@ -0,0 +1,52 @@
+import { Config } from '@basemaps/config';
+import { fsa } from '@basemaps/shared';
+import { HttpHeader, LambdaHttpRequest, LambdaHttpResponse } from '@linzjs/lambda';
+import { createHash } from 'crypto';
+import { promisify } from 'util';
+import { gzip } from 'zlib';
+import { Router } from '../router.js';
+import { NotModified } from './response.js';
+import { TileEtag } from './tile.etag.js';
+
+const gzipP = promisify(gzip);
+
+function isAllowedFile(f: string): boolean {
+  if (f.endsWith('.geojson')) return true;
+  if (f.endsWith('.json')) return true;
+  return false;
+}
+
+/**
+ * Get metadata around the imagery such as the source bounding box or the bounding box of the COGS
+ *
+ * @example
+ * - /v1/imagery/:imageryId/source.geojson
+ * - /v1/imagery/:imageryId/covering.geojson
+ */
+export async function Imagery(req: LambdaHttpRequest): Promise<LambdaHttpResponse> {
+  const { rest } = Router.action(req);
+  const [imageryId, requestType] = rest;
+  if (!isAllowedFile(requestType)) return new LambdaHttpResponse(404, 'Not found');
+
+  const imagery = await Config.Imagery.get(imageryId);
+  if (imagery == null) return new LambdaHttpResponse(404, 'Not found');
+
+  const targetPath = fsa.join(imagery.uri, requestType);
+
+  try {
+    const buf = await fsa.read(targetPath);
+    const cacheKey = createHash('sha256').update(buf).digest('base64');
+
+    if (TileEtag.isNotModified(req, cacheKey)) return NotModified;
+
+    const response = new LambdaHttpResponse(200, 'ok');
+    response.header(HttpHeader.ETag, cacheKey);
+    response.header(HttpHeader.ContentEncoding, 'gzip');
+    response.buffer(await gzipP(buf), 'application/json');
+    req.set('bytes', buf.byteLength);
+    return response;
+  } catch (e) {
+    req.log.warn({ targetPath }, 'ImageryMetadata:Failed');
+    return new LambdaHttpResponse(404, 'Not found');
+  }
+}

package/src/routes/response.ts

@@ -0,0 +1,4 @@
+import { LambdaHttpResponse } from '@linzjs/lambda';
+
+export const NotFound = new LambdaHttpResponse(404, 'Not Found');
+export const NotModified = new LambdaHttpResponse(304, 'Not modified');

package/src/routes/tile.json.ts

@@ -0,0 +1,44 @@
+import { Env } from '@basemaps/shared';
+import { HttpHeader, LambdaHttpRequest, LambdaHttpResponse } from '@linzjs/lambda';
+import { createHash } from 'crypto';
+import { Router } from '../router.js';
+import { NotModified } from './response.js';
+import { TileEtag } from './tile.etag.js';
+
+export interface TileJson {
+  tiles: string[];
+  minzoom: number;
+  maxzoom: number;
+  format: string;
+  tilejson: string;
+}
+
+export async function tileJson(req: LambdaHttpRequest): Promise<LambdaHttpResponse> {
+  const { version, rest, name } = Router.action(req);
+  const apiKey = Router.apiKey(req);
+  const host = Env.get(Env.PublicUrlBase) ?? '';
+  const tileUrl = `${host}/${version}/${name}/${rest[0]}/${rest[1]}/{z}/{x}/{y}.pbf?api=${apiKey}`;
+
+  const tileJson: TileJson = {
+    tiles: [tileUrl],
+    minzoom: 0,
+    maxzoom: 15,
+    format: 'pbf',
+    tilejson: '2.0.0',
+  };
+
+  const json = JSON.stringify(tileJson);
+
+  const data = Buffer.from(json);
+
+  const cacheKey = createHash('sha256').update(data).digest('base64');
+
+  if (TileEtag.isNotModified(req, cacheKey)) return NotModified;
+
+  const response = new LambdaHttpResponse(200, 'ok');
+  response.header(HttpHeader.ETag, cacheKey);
+  response.header(HttpHeader.CacheControl, 'max-age=120');
+  response.buffer(data, 'application/json');
+  req.set('bytes', data.byteLength);
+  return response;
+}

package/src/routes/tile.style.json.ts

@@ -0,0 +1,77 @@
+import { Sources, StyleJson } from '@basemaps/config';
+import { Config, Env } from '@basemaps/shared';
+import { fsa } from '@chunkd/fs';
+import { HttpHeader, LambdaHttpRequest, LambdaHttpResponse } from '@linzjs/lambda';
+import { createHash } from 'crypto';
+import { URL } from 'url';
+import { Router } from '../router.js';
+import { NotFound, NotModified } from './response.js';
+import { TileEtag } from './tile.etag.js';
+
+/**
+ * Convert relative URLS into a full hostname url
+ * @param url possible url to update
+ * @param apiKey ApiKey to append with ?api= if required
+ * @returns Updated Url or empty string if url is empty
+ */
+export function convertRelativeUrl(url?: string, apiKey?: string): string {
+  if (url == null) return '';
+  const host = Env.get(Env.PublicUrlBase) ?? '';
+  if (!url.startsWith('/')) return url; // Not relative ignore
+  const fullUrl = new URL(fsa.join(host, url));
+  if (apiKey) fullUrl.searchParams.set('api', apiKey);
+  return fullUrl.toString().replace(/%7B/g, '{').replace(/%7D/g, '}');
+}
+
+export async function styleJson(req: LambdaHttpRequest, fileName: string): Promise<LambdaHttpResponse> {
+  const apiKey = Router.apiKey(req);
+  if (apiKey == null) return new LambdaHttpResponse(400, 'Invalid API Key.');
+  const styleName = fileName.split('.json')[0];
+
+  // Get style Config from db
+  const dbId = Config.Style.id(styleName);
+  const styleConfig = await Config.Style.get(dbId);
+  if (styleConfig == null) return NotFound;
+
+  // Prepare sources and add linz source
+  const style = styleConfig.style;
+  const sources: Sources = {};
+  for (const [key, value] of Object.entries(style.sources)) {
+    if (value.type === 'vector') {
+      value.url = convertRelativeUrl(value.url, apiKey);
+    } else if (value.type === 'raster' && Array.isArray(value.tiles)) {
+      for (let i = 0; i < value.tiles.length; i++) {
+        value.tiles[i] = convertRelativeUrl(value.tiles[i], apiKey);
+      }
+    }
+    sources[key] = value;
+  }
+
+  // prepare Style.json
+  const styleJson: StyleJson = {
+    /** Style.json version 8 */
+    version: 8,
+    id: style.id,
+    name: style.name,
+    sources,
+    layers: style.layers,
+    metadata: style.metadata || {},
+    glyphs: convertRelativeUrl(style.glyphs),
+    sprite: convertRelativeUrl(style.sprite),
+  };
+
+  const json = JSON.stringify(styleJson);
+
+  const data = Buffer.from(json);
+
+  const cacheKey = createHash('sha256').update(data).digest('base64');
+
+  if (TileEtag.isNotModified(req, cacheKey)) return NotModified;
+
+  const response = new LambdaHttpResponse(200, 'ok');
+  response.header(HttpHeader.ETag, cacheKey);
+  response.header(HttpHeader.CacheControl, 'no-store');
+  response.buffer(data, 'application/json');
+  req.set('bytes', data.byteLength);
+  return response;
+}