@basemaps/lambda-tiler 6.15.0

package/src/cli/validate.ts

@@ -0,0 +1,49 @@
+import { GoogleTms } from '@basemaps/geo';
+import { Env, fsa, LogConfig, TileSetName } from '@basemaps/shared';
+import { CogTiff } from '@cogeotiff/core';
+import pLimit from 'p-limit';
+import { TileSets } from '../tile.set.cache.js';
+import { TileSetRaster } from '../tile.set.raster.js';
+
+const Q = pLimit(Env.getNumber(Env.TiffConcurrency, 25));
+
+/**
+ * CLI to iterate over all imagery sets that have been defined and determine if all the COGS are present and optimized
+ */
+async function main(): Promise<void> {
+  const tileSet = await TileSets.get(TileSetName.aerial, GoogleTms);
+  if (tileSet == null) throw new Error('No tile set found');
+  if (tileSet.isVector()) throw new Error('Invalid tile set type');
+
+  let errorCount = 0;
+  const logger = LogConfig.get();
+  for (const imagery of tileSet.imagery.values()) {
+    const path = TileSetRaster.basePath(imagery);
+    logger.info({ path, cogs: imagery.files.length }, 'TestingMosaic');
+
+    const promises = imagery.files.map(({ name }) => {
+      return Q(async () => {
+        try {
+          const uri = fsa.source(TileSetRaster.basePath(imagery, name));
+          if (uri == null) throw new Error('Failed to load uri: ' + TileSetRaster.basePath(imagery, name));
+          const source = new CogTiff(uri);
+          await source.init();
+          if (!source.options.isCogOptimized) {
+            logger.error({ path, name }, 'NotOptimized');
+          }
+        } catch (e) {
+          logger.error({ err: e, path, name }, 'Failed');
+          errorCount++;
+          if (errorCount > 10) {
+            logger.fatal('Too many errors');
+            process.exit(1);
+          }
+        }
+      });
+    });
+
+    await Promise.all(promises);
+  }
+}
+
+main().catch((err: Error) => LogConfig.get().fatal({ err }, 'Failed'));

package/src/index.ts

@@ -0,0 +1,31 @@
+import { lf, LambdaHttpRequest, LambdaHttpResponse } from '@linzjs/lambda';
+import { LogConfig } from '@basemaps/shared';
+import { Ping, Version } from './routes/api.js';
+import { Health } from './routes/health.js';
+import { Tiles } from './routes/tile.js';
+import { Router } from './router.js';
+
+const app = new Router();
+
+app.get('ping', Ping);
+app.get('health', Health);
+app.get('version', Version);
+app.get('tiles', Tiles);
+
+let slowTimer: NodeJS.Timer | null = null;
+export async function handleRequest(req: LambdaHttpRequest): Promise<LambdaHttpResponse> {
+  // Warn if a request takes more than 10 seconds to process
+  if (slowTimer) clearTimeout(slowTimer);
+  slowTimer = setTimeout(() => req.log.warn(req.logContext, 'Lambda:Slow'), 10_000);
+  slowTimer.unref();
+
+  req.set('name', 'LambdaTiler');
+  try {
+    return await app.handle(req);
+  } finally {
+    if (slowTimer) clearTimeout(slowTimer);
+    slowTimer = null;
+  }
+}
+
+export const handler = lf.http(handleRequest, LogConfig.get());

package/src/router.ts

@@ -0,0 +1,56 @@
+import { Const } from '@basemaps/shared';
+import { HttpHeader, LambdaHttpRequest, LambdaHttpResponse } from '@linzjs/lambda';
+
+export type ReqCallback = (req: LambdaHttpRequest) => Promise<LambdaHttpResponse>;
+
+export interface ActionData {
+  version: string;
+  name: string;
+  rest: string[];
+}
+
+export class Router {
+  static action(req: LambdaHttpRequest): ActionData {
+    const path = req.path;
+    const [version, name, ...rest] = (path[0] === '/' ? path.slice(1) : path).split('/');
+    if (name == null) return { version: 'v1', name: version, rest: [] };
+    return { version, name, rest };
+  }
+
+  static apiKey(req: LambdaHttpRequest): string | undefined {
+    const apiKey = req.query.get(Const.ApiKey.QueryString) ?? req.header('X-LINZ-Api-Key');
+    if (apiKey != null && !Array.isArray(apiKey)) {
+      req.set(Const.ApiKey.QueryString, this.apiKey);
+      return apiKey;
+    }
+    return;
+  }
+
+  private handlers: Record<string, ReqCallback> = {};
+
+  async handle(req: LambdaHttpRequest): Promise<LambdaHttpResponse> {
+    // Allow cross origin requests
+    if (req.method === 'options') {
+      return new LambdaHttpResponse(200, 'Options', {
+        [HttpHeader.Cors]: '*',
+        'Access-Control-Allow-Credentials': 'false',
+        'Access-Control-Allow-Methods': 'OPTIONS,GET,PUT,POST,DELETE',
+      });
+    }
+
+    if (req.method !== 'GET') return new LambdaHttpResponse(405, 'Method not allowed');
+
+    const action = Router.action(req);
+    const handler = action.version === 'v1' ? this.handlers[action.name] : null;
+    if (handler == null) return new LambdaHttpResponse(404, 'Not Found');
+
+    const response = await handler(req);
+    response.header(HttpHeader.Cors, '*');
+    return response;
+  }
+
+  get(path: string, handler: ReqCallback): void {
+    if (this.handlers[path] != null) throw new Error(path + ' already registered');
+    this.handlers[path] = handler;
+  }
+}

package/src/routes/__test__/attribution.test.ts

@@ -0,0 +1,441 @@
+import { ConfigImagery, ConfigLayer, ConfigProvider } from '@basemaps/config';
+import { EpsgCode, GoogleTms, NamedBounds, Nztm2000QuadTms, Nztm2000Tms, Stac, TileMatrixSets } from '@basemaps/geo';
+import { HttpHeader } from '@linzjs/lambda';
+import { Config } from '@basemaps/shared';
+import { mockFileOperator } from '@basemaps/shared/build/file/__test__/file.operator.test.helper.js';
+import { round } from '@basemaps/test/build/rounding.js';
+import o from 'ospec';
+import sinon from 'sinon';
+const sandbox = sinon.createSandbox();
+import { TileSets } from '../../tile.set.cache.js';
+import { TileSetRaster } from '../../tile.set.raster.js';
+import { FakeTileSet, mockRequest, Provider } from '../../__test__/xyz.util.js';
+import { attribution, createAttributionCollection } from '../attribution.js';
+import { TileEtag } from '../tile.etag.js';
+import { Attribution } from '@basemaps/attribution';
+
+const ExpectedJson = {
+  id: 'aerial_WebMercatorQuad',
+  type: 'FeatureCollection',
+  stac_version: '1.0.0-beta.2',
+  stac_extensions: ['single-file-stac'],
+  title: 'aerial:title',
+  description: 'aerial:description',
+  features: [
+    {
+      type: 'Feature',
+      stac_version: '1.0.0-beta.2',
+      id: 'im_imageId1_item',
+      collection: 'im_imageId1',
+      assets: {},
+      links: [],
+      bbox: [-22.5, 48.9225, -11.25, 55.7766],
+      geometry: {
+        type: 'MultiPolygon',
+        coordinates: [
+          [
+            [
+              [-22.5, 48.9225],
+              [-19.6875, 48.9225],
+              [-19.6875, 49.838],
+              [-22.5, 49.838],
+              [-22.5, 48.9225],
+            ],
+          ],
+        ],
+      },
+      properties: {
+        datetime: null,
+        start_datetime: '2011-01-01T00:00:00Z',
+        end_datetime: '2014-01-01T00:00:00Z',
+      },
+    },
+    {
+      type: 'Feature',
+      stac_version: '1.0.0-beta.2',
+      id: 'im_imageId2_item',
+      collection: 'im_imageId2',
+      assets: {},
+      links: [],
+      bbox: [-11.25, 48.9225, 0, 55.7766],
+      geometry: {
+        type: 'MultiPolygon',
+        coordinates: [
+          [
+            [
+              [-11.25, 48.9225],
+              [-8.4375, 48.9225],
+              [-8.4375, 49.838],
+              [-11.25, 49.838],
+              [-11.25, 48.9225],
+            ],
+          ],
+        ],
+      },
+      properties: {
+        datetime: null,
+        start_datetime: '2013-01-01T00:00:00Z',
+        end_datetime: '2015-01-01T00:00:00Z',
+      },
+    },
+    {
+      type: 'Feature',
+      stac_version: '1.0.0-beta.2',
+      id: 'im_imageId3_item',
+      collection: 'im_imageId3',
+      assets: {},
+      links: [],
+      bbox: [0, 48.9225, 11.25, 55.7766],
+      geometry: {
+        type: 'MultiPolygon',
+        coordinates: [
+          [
+            [
+              [0, 48.9225],
+              [2.8125, 48.9225],
+              [2.8125, 49.838],
+              [0, 49.838],
+              [0, 48.9225],
+            ],
+          ],
+        ],
+      },
+      properties: {
+        datetime: null,
+        start_datetime: '2015-01-01T00:00:00Z',
+        end_datetime: '2018-01-01T00:00:00Z',
+      },
+    },
+    {
+      type: 'Feature',
+      stac_version: '1.0.0-beta.2',
+      id: 'im_imageId4_item',
+      collection: 'im_imageId4',
+      assets: {},
+      links: [],
+      bbox: [-22.5, 48.9225, -11.25, 55.7766],
+      geometry: {
+        type: 'MultiPolygon',
+        coordinates: [
+          [
+            [
+              [-22.5, 48.9225],
+              [-19.6875, 48.9225],
+              [-19.6875, 49.838],
+              [-22.5, 49.838],
+              [-22.5, 48.9225],
+            ],
+          ],
+        ],
+      },
+      properties: {
+        datetime: null,
+        start_datetime: '2017-01-01T00:00:00Z',
+        end_datetime: '2019-01-01T00:00:00Z',
+      },
+    },
+  ],
+  collections: [
+    {
+      stac_version: '1.0.0-beta.2',
+      license: 'CC BY 4.0',
+      id: 'im_imageId1',
+      providers: [
+        {
+          name: 'the name',
+          url: 'https://example.provider.com',
+          roles: ['host'],
+        },
+      ],
+      title: 'Hastings-district urban 2011-13 0.1m',
+      description: 'No description',
+      extent: {
+        spatial: {
+          bbox: [[-22.5, 48.9225, -11.25, 55.7766]],
+        },
+        temporal: {
+          interval: [['2011-01-01T00:00:00Z', '2014-01-01T00:00:00Z']],
+        },
+      },
+      links: [],
+      summaries: {
+        gsd: [0.75],
+        'linz:zoom': {
+          min: 14,
+          max: 16,
+        },
+        'linz:priority': [1000],
+      },
+    },
+    {
+      stac_version: '1.0.0-beta.2',
+      license: 'CC BY 4.0',
+      id: 'im_imageId2',
+      providers: [
+        {
+          name: 'the name',
+          url: 'https://example.provider.com',
+          roles: ['host'],
+        },
+      ],
+      title: 'Hastings-district urban 2013-14 0.1m',
+      description: 'No description',
+      extent: {
+        spatial: {
+          bbox: [[-11.25, 48.9225, 0, 55.7766]],
+        },
+        temporal: {
+          interval: [['2013-01-01T00:00:00Z', '2015-01-01T00:00:00Z']],
+        },
+      },
+      links: [],
+      summaries: {
+        gsd: [0.75],
+        'linz:zoom': {
+          min: 15,
+          max: 17,
+        },
+        'linz:priority': [1001],
+      },
+    },
+    {
+      stac_version: '1.0.0-beta.2',
+      license: 'CC BY 4.0',
+      id: 'im_imageId3',
+      providers: [
+        {
+          name: 'the name',
+          url: 'https://example.provider.com',
+          roles: ['host'],
+        },
+      ],
+      title: 'Hastings-district urban 2015-17 0.1m',
+      description: 'No description',
+      extent: {
+        spatial: {
+          bbox: [[0, 48.9225, 11.25, 55.7766]],
+        },
+        temporal: {
+          interval: [['2015-01-01T00:00:00Z', '2018-01-01T00:00:00Z']],
+        },
+      },
+      links: [],
+      summaries: {
+        gsd: [0.75],
+        'linz:zoom': {
+          min: 16,
+          max: 18,
+        },
+        'linz:priority': [1002],
+      },
+    },
+    {
+      stac_version: '1.0.0-beta.2',
+      license: 'CC BY 4.0',
+      id: 'im_imageId4',
+      providers: [
+        {
+          name: 'the name',
+          url: 'https://example.provider.com',
+          roles: ['host'],
+        },
+      ],
+      title: 'Hastings-district urban 2017-18 0.1m',
+      description: 'No description',
+      extent: {
+        spatial: {
+          bbox: [[-22.5, 48.9225, -11.25, 55.7766]],
+        },
+        temporal: {
+          interval: [['2017-01-01T00:00:00Z', '2019-01-01T00:00:00Z']],
+        },
+      },
+      links: [],
+      summaries: {
+        gsd: [0.75],
+        'linz:zoom': {
+          min: 14,
+          max: 16,
+        },
+        'linz:priority': [1003],
+      },
+    },
+  ],
+  links: [],
+};
+function makeImageRecord(id: string, name: string, x = 10, year = 2019): ConfigImagery {
+  return {
+    id,
+    name,
+    projection: EpsgCode.Google,
+    year,
+    uri: 's3://bucket/path/' + name,
+    bounds: GoogleTms.tileToSourceBounds({ x, y: 10, z: 5 }),
+    resolution: 750,
+    files: [0, 1].map((i) => {
+      const b = GoogleTms.tileToSourceBounds({ x, y: 10, z: 5 }).toJson() as NamedBounds;
+      b.name = name + i;
+      b.width /= 8;
+      b.height /= 8;
+      b.x += i * b.width;
+      return b;
+    }),
+    createdAt: Date.now(),
+    updatedAt: Date.now(),
+  };
+}
+o.spec('attribution', () => {
+  o.spec('fetch', () => {
+    const origTileEtag = TileEtag.generate;
+    const generateMock = o.spy(() => 'foo');
+    const TileSetNames = ['aerial', 'aerial@head', 'aerial@beta', '01E7PJFR9AMQFJ05X9G7FQ3XMW'];
+
+    const mockFs = mockFileOperator();
+
+    o.beforeEach(() => {
+      mockFs.setup();
+      TileEtag.generate = generateMock;
+      // Mock the tile generation
+      for (const tileSetName of TileSetNames) {
+        for (const tileMatrix of TileMatrixSets.Defaults.values()) {
+          const tileSet = new FakeTileSet(tileSetName, tileMatrix);
+          TileSets.add(tileSet);
+          const layers: ConfigLayer[] = [];
+          const imagery = new Map<string, ConfigImagery>();
+          const addRule = (id: string, name: string, minZoom = 10): void => {
+            const imgId = Config.Imagery.id(id);
+            imagery.set(imgId, makeImageRecord(imgId, name, minZoom));
+            layers.push({
+              [3857]: imgId,
+              name,
+              minZoom,
+              maxZoom: minZoom + 2,
+            });
+          };
+          addRule('imageId1', 'hastings-district_urban_2011-13_0.1m', 14);
+          addRule('imageId2', 'hastings-district_urban_2013-14_0.1m', 15);
+          addRule('imageId3', 'hastings-district_urban_2015-17_0.1m', 16);
+          addRule('imageId4', 'hastings-district_urban_2017-18_0.1m', 14);
+          tileSet.tileSet.layers = layers;
+          tileSet.imagery = imagery;
+        }
+      }
+      sandbox.stub(Config.Provider, 'get').callsFake(() => Promise.resolve(Provider));
+    });
+
+    o.afterEach(() => {
+      sandbox.restore();
+      mockFs.teardown();
+      TileSets.cache.clear();
+      TileEtag.generate = origTileEtag;
+    });
+
+    o('should notFound', async () => {
+      const request = mockRequest(`/v1/attribution/aerial/1234/summary.json`);
+      const res = await attribution(request);
+
+      o(res.status).equals(404);
+    });
+
+    o('should 200 with etag mismatch', async () => {
+      mockFs.jsStore['s3://bucket/path/image1/collection.json'] = {
+        extent: {
+          spatial: { bbox: [1, 2, 3, 4] },
+          temporal: { interval: [['2018-02-03T01:02:03Z', '2018-09-13T11:32:43Z']] },
+        },
+        title: 'image one',
+        description: 'image one description',
+        license: Stac.License,
+        providers: [
+          {
+            name: 'p1',
+          },
+        ],
+        summaries: {
+          gsd: [0.75],
+        },
+      };
+      mockFs.jsStore['s3://bucket/path/image2/collection.json'] = {
+        extent: {
+          spatial: { bbox: [5, 6, 7, 8] },
+          temporal: { interval: [['2016-02-03T01:02:03Z', '2018-09-13T11:32:43Z']] },
+        },
+        summaries: {
+          gsd: [0.5],
+        },
+      };
+
+      const request = mockRequest(`/v1/attribution/aerial/EPSG:3857/summary.json`);
+      const res = await attribution(request);
+
+      o(res.status).equals(200);
+      o(res.header(HttpHeader.ETag)).equals('3edkgmltK4/LUyTCTYU9MeiNSwlfUvJAx/qORSisUzM=');
+      o(res.header(HttpHeader.CacheControl)).equals('public, max-age=86400, stale-while-revalidate=604800');
+
+      const body = round(JSON.parse(res.body as string), 4);
+      o(body).deepEquals(ExpectedJson);
+    });
+
+    o('should 304 with etag match', async () => {
+      const request = mockRequest(`/v1/attribution/aerial/EPSG:3857/summary.json`, 'get', {
+        [HttpHeader.IfNoneMatch]: '3edkgmltK4/LUyTCTYU9MeiNSwlfUvJAx/qORSisUzM=',
+      });
+      const res = await attribution(request);
+
+      o(res.status).equals(304);
+    });
+
+    o('should parse attribution', async () => {
+      const request = mockRequest(`/v1/attribution/aerial/EPSG:3857/summary.json`);
+      const res = await attribution(request);
+
+      const json = JSON.parse(res.body as string);
+
+      const attr = Attribution.fromStac(json);
+      o(attr.attributions.length).equals(4);
+      o(attr.attributions[0].minZoom).equals(14);
+      o(attr.attributions[0].maxZoom).equals(16);
+    });
+  });
+
+  o.spec('ImageryRule', () => {
+    const fakeIm = { name: 'someName' } as ConfigImagery;
+    const fakeHost = { serviceProvider: {} } as ConfigProvider;
+    const fakeLayer = { [2193]: 'id', name: 'image', minZoom: 9, maxZoom: 16 };
+
+    o('should generate for NZTM', () => {
+      const ts = new TileSetRaster('Fake', Nztm2000Tms);
+      ts.tileSet = { ...ts.tileSet, layers: [fakeLayer] };
+
+      const output = createAttributionCollection(ts, null, fakeIm, fakeLayer, fakeHost, null as any);
+      o(output.title).equals('SomeName');
+      o(output.summaries['linz:zoom']).deepEquals({ min: 5, max: 11 });
+    });
+
+    o('should generate with correct zooms for NZTM2000Quad', () => {
+      const ts = new TileSetRaster('Fake', Nztm2000QuadTms);
+      ts.tileSet = { ...ts.tileSet, layers: [fakeLayer] };
+      const output = createAttributionCollection(ts, null, fakeIm, fakeLayer, fakeHost, null as any);
+      o(output.title).equals('SomeName');
+      o(output.summaries['linz:zoom']).deepEquals({ min: 7, max: 14 });
+    });
+
+    o('should generate with correct zooms for gebco NZTM2000Quad', () => {
+      const fakeGebco = { ...fakeLayer, minZoom: 0, maxZoom: 15 };
+      const ts = new TileSetRaster('Fake', Nztm2000QuadTms);
+      ts.tileSet = { ...ts.tileSet, layers: [fakeLayer] };
+      const output = createAttributionCollection(ts, null, fakeIm, fakeGebco, fakeHost, null as any);
+      o(output.title).equals('SomeName');
+      o(output.summaries['linz:zoom']).deepEquals({ min: 0, max: 13 });
+    });
+
+    o('should generate with correct zooms for nz sentinel NZTM2000Quad', () => {
+      const fakeGebco = { ...fakeLayer, minZoom: 0, maxZoom: 32 };
+      const ts = new TileSetRaster('Fake', Nztm2000QuadTms);
+      ts.tileSet = { ...ts.tileSet, layers: [fakeLayer] };
+      const output = createAttributionCollection(ts, null, fakeIm, fakeGebco, fakeHost, null as any);
+      o(output.title).equals('SomeName');
+      o(output.summaries['linz:zoom']).deepEquals({ min: 0, max: Nztm2000QuadTms.maxZoom });
+    });
+  });
+});

package/src/routes/__test__/health.test.ts

@@ -0,0 +1,89 @@
+import { GoogleTms } from '@basemaps/geo';
+import { LogConfig } from '@basemaps/shared';
+import { LambdaAlbRequest, LambdaHttpRequest, LambdaHttpResponse } from '@linzjs/lambda';
+import { Context } from 'aws-lambda';
+import o from 'ospec';
+import sinon from 'sinon';
+import { TileSets } from '../../tile.set.cache.js';
+import { TileSetRaster } from '../../tile.set.raster.js';
+import { getTestBuffer, Health, TestTiles } from '../health.js';
+
+const ctx: LambdaHttpRequest = new LambdaAlbRequest(
+  {
+    requestContext: null as any,
+    httpMethod: 'get',
+    path: '/v1/tiles/health',
+    body: null,
+    isBase64Encoded: false,
+  },
+  {} as Context,
+  LogConfig.get(),
+);
+
+o.spec('health', async () => {
+  const sandbox = sinon.createSandbox();
+
+  const tileSet = new TileSetRaster('health', GoogleTms);
+  o.beforeEach(() => {
+    sandbox.stub(TileSets, 'get').resolves(tileSet);
+  });
+
+  o.afterEach(() => {
+    sandbox.restore();
+  });
+
+  o('Should return bad response', async () => {
+    // Given ... a bad get tile response
+    const BadResponse = new LambdaHttpResponse(500, 'Can not get Tile Set.');
+    sandbox.stub(tileSet, 'tile').resolves(BadResponse);
+
+    // When ...
+    const res = await Health(ctx);
+
+    // Then ...
+    o(res.status).equals(500);
+    o(res.statusDescription).equals('Can not get Tile Set.');
+  });
+
+  const Response1 = new LambdaHttpResponse(200, 'ok');
+  const Response2 = new LambdaHttpResponse(200, 'ok');
+
+  o.before(async () => {
+    const testTileFile1 = await getTestBuffer(TestTiles[0]);
+    Response1.buffer(testTileFile1);
+    const testTileFile2 = await getTestBuffer(TestTiles[1]);
+    Response2.buffer(testTileFile2);
+  });
+  // Prepare mock test tile response based on the static test tiles
+
+  o('Should give a 200 response', async () => {
+    o.timeout(500);
+
+    // Given ... a series good get tile response
+    const callback = sandbox.stub(tileSet, 'tile');
+    callback.onCall(0).resolves(Response1);
+    callback.onCall(1).resolves(Response2);
+
+    // When ...
+    const res = await Health(ctx);
+
+    // Then ...
+    o(res.status).equals(200);
+    o(res.statusDescription).equals('ok');
+  });
+
+  o('Should return mis-match tile response', async () => {
+    o.timeout(500);
+    // Given ... a bad get tile response for second get tile
+    const callback = sandbox.stub(tileSet, 'tile');
+    callback.onCall(0).resolves(Response1);
+    callback.onCall(1).resolves(Response1);
+
+    // When ...
+    const res = await Health(ctx);
+
+    // Then ...
+    o(res.status).equals(500);
+    o(res.statusDescription).equals('TileSet does not match.');
+  });
+});

package/src/routes/api.ts

@@ -0,0 +1,19 @@
+import { LambdaHttpResponse, HttpHeader } from '@linzjs/lambda';
+
+const OkResponse = new LambdaHttpResponse(200, 'ok');
+OkResponse.header(HttpHeader.CacheControl, 'no-store');
+
+export async function Health(): Promise<LambdaHttpResponse> {
+  return OkResponse;
+}
+
+export async function Ping(): Promise<LambdaHttpResponse> {
+  return OkResponse;
+}
+
+export async function Version(): Promise<LambdaHttpResponse> {
+  const response = new LambdaHttpResponse(200, 'ok');
+  response.header(HttpHeader.CacheControl, 'no-store');
+  response.json({ version: process.env.GIT_VERSION ?? 'dev', hash: process.env.GIT_HASH });
+  return response;
+}