@basemaps/lambda-tiler 6.32.1 → 6.34.0

package/src/routes/__tests__/fonts.test.ts

@@ -2,8 +2,9 @@ import { Env } from '@basemaps/shared';
 import { fsa } from '@chunkd/fs';
 import o from 'ospec';
 import { handler } from '../../index.js';
+import { assetProvider } from '../../util/assets.provider.js';
 import { mockRequest } from '../../__tests__/xyz.util.js';
-import { fontList, getFonts } from '../fonts.js';
+import { fontList } from '../fonts.js';
 import { FsMemory } from './memory.fs.js';
 
 o.spec('/v1/fonts', () => {
@@ -11,52 +12,30 @@ o.spec('/v1/fonts', () => {
   o.before(() => {
     fsa.register('memory://', memory);
   });
+  const assetLocation = process.env[Env.AssetLocation];
 
   o.beforeEach(() => {
     process.env[Env.AssetLocation] = 'memory://';
+    assetProvider.set('memory://');
   });
 
   o.afterEach(() => {
-    delete process.env[Env.AssetLocation];
+    assetProvider.set(assetLocation);
     memory.files.clear();
   });
 
-  o('should list font types', async () => {
-    await Promise.all([
-      fsa.write('memory://fonts/Roboto Thin/0-255.pbf', Buffer.from('')),
-      fsa.write('memory://fonts/Roboto Thin/256-512.pbf', Buffer.from('')),
-      fsa.write('memory://fonts/Roboto Black/0-255.pbf', Buffer.from('')),
-    ]);
-
-    const fonts = await getFonts('memory://fonts/');
-    o(fonts).deepEquals(['Roboto Black', 'Roboto Thin']);
-  });
-
-  o('should return empty list if no fonts found', async () => {
-    const res = await fontList(mockRequest('/v1/fonts.json'));
-    o(res.status).equals(200);
-    o(res.body).equals('[]');
-    o(res.header('etag')).notEquals(undefined);
-    o(res.header('cache-control')).equals('public, max-age=604800, stale-while-revalidate=86400');
-  });
-
-  o('should return 404 if no assets defined', async () => {
-    delete process.env[Env.AssetLocation];
+  o('should return 404 if no font found', async () => {
     const res = await fontList(mockRequest('/v1/fonts.json'));
     o(res.status).equals(404);
   });
 
   o('should return a list of fonts found', async () => {
-    await Promise.all([
-      fsa.write('memory://fonts/Roboto Thin/0-255.pbf', Buffer.from('')),
-      fsa.write('memory://fonts/Roboto Thin/256-512.pbf', Buffer.from('')),
-      fsa.write('memory://fonts/Roboto Black/0-255.pbf', Buffer.from('')),
-    ]);
+    await fsa.write('memory://fonts.json', Buffer.from(JSON.stringify(['Roboto Black', 'Roboto Thin'])));
     const res = await fontList(mockRequest('/v1/fonts.json'));
     o(res.status).equals(200);
     o(res.header('content-type')).equals('application/json');
     o(res.header('content-encoding')).equals(undefined);
-    o(res.body).equals(JSON.stringify(['Roboto Black', 'Roboto Thin']));
+    o(res._body?.toString()).equals(JSON.stringify(['Roboto Black', 'Roboto Thin']));
   });
 
   o('should get the correct font', async () => {
@@ -83,4 +62,10 @@ o.spec('/v1/fonts', () => {
     o(res255.header('etag')).notEquals(undefined);
     o(res255.header('cache-control')).equals('public, max-age=604800, stale-while-revalidate=86400');
   });
+
+  o('should return 404 if no asset location set', async () => {
+    assetProvider.set(undefined);
+    const res = await fontList(mockRequest('/v1/fonts.json'));
+    o(res.status).equals(404);
+  });
 });

package/src/routes/__tests__/sprites.test.ts

@@ -1,26 +1,31 @@
 import { Env } from '@basemaps/shared';
 import { fsa } from '@chunkd/fs';
 import o from 'ospec';
+import { createSandbox } from 'sinon';
 import { gunzipSync, gzipSync } from 'zlib';
 import { handler } from '../../index.js';
+import { assetProvider } from '../../util/assets.provider.js';
 import { mockRequest } from '../../__tests__/xyz.util.js';
 import { FsMemory } from './memory.fs.js';
 
 o.spec('/v1/sprites', () => {
   const memory = new FsMemory();
+  const sandbox = createSandbox();
   o.before(() => {
     fsa.register('memory://', memory);
   });
+  const assetLocation = process.env[Env.AssetLocation];
 
   o.beforeEach(() => {
     process.env[Env.AssetLocation] = 'memory://';
+    assetProvider.set('memory://');
   });
 
   o.afterEach(() => {
-    delete process.env[Env.AssetLocation];
+    assetProvider.set(assetLocation);
     memory.files.clear();
+    sandbox.restore();
   });
-
   o('should return 404 if no assets defined', async () => {
     delete process.env[Env.AssetLocation];
     const res404 = await handler.router.handle(mockRequest('/v1/sprites/topographic.json'));

package/src/routes/__tests__/xyz.test.ts

@@ -63,7 +63,8 @@ o.spec('/v1/tiles', () => {
 
   ['png', 'webp', 'jpeg', 'avif'].forEach((fmt) => {
     o(`should 200 with empty ${fmt} if a tile is out of bounds`, async () => {
-      // tiler.tile = async () => [];
+      o.timeout(1_000);
+
       const res = await handler.router.handle(
         mockRequest(`/v1/tiles/aerial/global-mercator/0/0/0.${fmt}`, 'get', Api.header),
       );
@@ -71,7 +72,6 @@ o.spec('/v1/tiles', () => {
       o(res.header('content-type')).equals(`image/${fmt}`);
       o(res.header('etag')).notEquals(undefined);
       o(res.header('cache-control')).equals('public, max-age=604800, stale-while-revalidate=86400');
-      //   o(rasterMock.calls.length).equals(1);
     });
   });
 

package/src/routes/fonts.ts

@@ -1,76 +1,16 @@
-import { Env } from '@basemaps/shared';
-import { fsa } from '@chunkd/fs';
-import { HttpHeader, LambdaHttpRequest, LambdaHttpResponse } from '@linzjs/lambda';
+import { LambdaHttpRequest, LambdaHttpResponse } from '@linzjs/lambda';
 import path from 'path';
-import { isGzip, serveFromCotar } from '../util/cotar.serve.js';
-import { Etag } from '../util/etag.js';
-import { NotFound, NotModified } from '../util/response.js';
+import { assetProvider } from '../util/assets.provider.js';
 
 interface FontGet {
   Params: { fontStack: string; range: string };
 }
 
 export async function fontGet(req: LambdaHttpRequest<FontGet>): Promise<LambdaHttpResponse> {
-  const assetLocation = Env.get(Env.AssetLocation);
-  if (assetLocation == null) return NotFound();
-
   const targetFile = path.join('fonts', req.params.fontStack, req.params.range) + '.pbf';
-  if (assetLocation.endsWith('.tar.co')) {
-    return serveFromCotar(req, assetLocation, targetFile, 'application/x-protobuf');
-  }
-
-  try {
-    const filePath = fsa.join(assetLocation, targetFile);
-    const buf = await fsa.read(filePath);
-
-    const cacheKey = Etag.key(buf);
-    if (Etag.isNotModified(req, cacheKey)) return NotModified();
-
-    const response = LambdaHttpResponse.ok().buffer(buf, 'application/x-protobuf');
-    response.header(HttpHeader.ETag, cacheKey);
-    response.header(HttpHeader.CacheControl, 'public, max-age=604800, stale-while-revalidate=86400');
-    if (isGzip(buf)) response.header(HttpHeader.ContentEncoding, 'gzip');
-    return response;
-  } catch (e: any) {
-    if (e.code === 404) return NotFound();
-    throw e;
-  }
-}
-
-/** Get the unique name of folders is a path that contain .pbf files */
-export async function getFonts(fontPath: string): Promise<string[]> {
-  const fonts = new Set<string>();
-
-  // TODO use {recursive: false}
-  for await (const font of fsa.list(fontPath)) {
-    if (!font.endsWith('.pbf')) continue;
-    const dirName = path.basename(path.dirname(font)); // TODO this only works for /a/b.pbf and not /a/b/c.pbf
-    if (dirName.includes('/')) continue;
-    fonts.add(dirName);
-  }
-  // Ensure the fonts are alphabetical
-  return [...fonts].sort();
+  return assetProvider.serve(req, targetFile, 'application/x-protobuf');
 }
 
 export async function fontList(req: LambdaHttpRequest): Promise<LambdaHttpResponse> {
-  const assetLocation = Env.get(Env.AssetLocation);
-  if (assetLocation == null) return NotFound();
-
-  if (assetLocation.endsWith('.tar.co')) return serveFromCotar(req, assetLocation, 'fonts.json', 'application/json');
-
-  try {
-    const filePath = fsa.join(assetLocation, '/fonts');
-    const fonts = await getFonts(filePath);
-
-    const cacheKey = Etag.key(fonts);
-    if (Etag.isNotModified(req, cacheKey)) return NotModified();
-
-    const response = LambdaHttpResponse.ok().buffer(JSON.stringify(fonts), 'application/json');
-    response.header(HttpHeader.ETag, cacheKey);
-    response.header(HttpHeader.CacheControl, 'public, max-age=604800, stale-while-revalidate=86400');
-    return response;
-  } catch (e: any) {
-    if (e.code === 404) return NotFound();
-    throw e;
-  }
+  return assetProvider.serve(req, 'fonts.json', 'application/json');
 }

package/src/routes/sprites.ts

@@ -1,10 +1,8 @@
-import { Env } from '@basemaps/shared';
 import { fsa } from '@chunkd/fs';
 import path from 'path';
-import { HttpHeader, LambdaHttpRequest, LambdaHttpResponse } from '@linzjs/lambda';
-import { NotFound, NotModified } from '../util/response.js';
-import { isGzip, serveFromCotar } from '../util/cotar.serve.js';
-import { Etag } from '../util/etag.js';
+import { LambdaHttpRequest, LambdaHttpResponse } from '@linzjs/lambda';
+import { NotFound } from '../util/response.js';
+import { assetProvider } from '../util/assets.provider.js';
 
 interface SpriteGet {
   Params: {
@@ -17,31 +15,10 @@ Extensions.set('.png', 'image/png');
 Extensions.set('.json', 'application/json');
 
 export async function spriteGet(req: LambdaHttpRequest<SpriteGet>): Promise<LambdaHttpResponse> {
-  const assetLocation = Env.get(Env.AssetLocation);
-  if (assetLocation == null) return NotFound();
-
   const extension = path.extname(req.params.spriteName);
   const mimeType = Extensions.get(extension);
   if (mimeType == null) return NotFound();
 
   const targetFile = fsa.join('sprites', req.params.spriteName);
-  if (assetLocation.endsWith('.tar.co')) return serveFromCotar(req, assetLocation, targetFile, mimeType);
-
-  try {
-    const filePath = fsa.join(assetLocation, targetFile);
-    req.set('target', filePath);
-
-    const buf = await fsa.read(filePath);
-    const cacheKey = Etag.key(buf);
-    if (Etag.isNotModified(req, cacheKey)) return NotModified();
-
-    const response = LambdaHttpResponse.ok().buffer(buf, mimeType);
-    response.header(HttpHeader.ETag, cacheKey);
-    response.header(HttpHeader.CacheControl, 'public, max-age=604800, stale-while-revalidate=86400');
-    if (isGzip(buf)) response.header(HttpHeader.ContentEncoding, 'gzip');
-    return response;
-  } catch (e: any) {
-    if (e.code === 404) return NotFound();
-    throw e;
-  }
+  return assetProvider.serve(req, targetFile, mimeType);
 }

package/src/routes/tile.xyz.vector.ts

@@ -3,7 +3,7 @@ import { GoogleTms, VectorFormat } from '@basemaps/geo';
 import { HttpHeader, LambdaHttpRequest, LambdaHttpResponse } from '@linzjs/lambda';
 import { isGzip } from '../util/cotar.serve.js';
 import { Etag } from '../util/etag.js';
-import { NotFound, NotModified } from '../util/response.js';
+import { NoContent, NotFound, NotModified } from '../util/response.js';
 import { CoSources } from '../util/source.cache.js';
 import { TileXyz } from '../util/validate.js';
 
@@ -34,7 +34,7 @@ export const tileXyzVector = {
 
     req.timer.start('cotar:tile');
     const tile = await cotar.get(tilePath);
-    if (tile == null) return NotFound();
+    if (tile == null) return NoContent();
     req.timer.end('cotar:tile');
 
     const tileBuffer = Buffer.from(tile);

package/src/util/assets.provider.ts

@@ -0,0 +1,67 @@
+import { fsa } from '@chunkd/fs';
+import { LambdaHttpResponse, LambdaHttpRequest, HttpHeader } from '@linzjs/lambda';
+import { isGzip } from './cotar.serve.js';
+import { Etag } from './etag.js';
+import { NotFound, NotModified } from './response.js';
+import { CoSources } from './source.cache.js';
+
+export class AssetProvider {
+  /**
+   * Assets can be ready from the following locations.
+   *
+   * /home/blacha/config/build/assets # Local File
+   * /home/blacha/config/build/assets.tar.co # Local Cotar
+   * s3://linz-baesmaps/assets/ # Remote location
+   * s3://linz-basemaps/assets/assets-b4ff211a.tar.co # Remote Cotar
+   */
+
+  /** Path of the assets location */
+  path: string | undefined;
+
+  set(path?: string): void {
+    this.path = path;
+  }
+
+  async get(fileName: string): Promise<Buffer | null> {
+    if (this.path == null) return null;
+    // get assets file from cotar
+    if (this.path.endsWith('.tar.co')) return await this.getFromCotar(this.path, fileName);
+
+    // get assets file for directory
+    try {
+      const filePath = fsa.join(this.path, fileName);
+      return await fsa.read(filePath);
+    } catch (e: any) {
+      if (e.code === 404) return null;
+      throw e;
+    }
+  }
+
+  async getFromCotar(path: string, fileName: string): Promise<Buffer | null> {
+    const cotar = await CoSources.getCotar(path);
+    const data = await cotar.get(fileName);
+    return data ? Buffer.from(data) : data;
+  }
+
+  /**
+   *  Load a assets from local path or cotar returning the file back as a LambdaResponse
+   *
+   * This will also set two headers
+   * - Content-Encoding if the file starts with gzip magic
+   * - Content-Type from the parameter contentType
+   */
+  async serve(req: LambdaHttpRequest, file: string, contentType: string): Promise<LambdaHttpResponse> {
+    const buf = await assetProvider.get(file);
+    if (buf == null) return NotFound();
+    const cacheKey = Etag.key(buf);
+    if (Etag.isNotModified(req, cacheKey)) return NotModified();
+
+    const response = LambdaHttpResponse.ok().buffer(buf, contentType);
+    response.header(HttpHeader.ETag, cacheKey);
+    response.header(HttpHeader.CacheControl, 'public, max-age=604800, stale-while-revalidate=86400');
+    if (isGzip(buf)) response.header(HttpHeader.ContentEncoding, 'gzip');
+    return response;
+  }
+}
+
+export const assetProvider = new AssetProvider();

package/src/util/config.cache.ts

@@ -0,0 +1,44 @@
+import { ConfigBundled, ConfigProviderMemory } from '@basemaps/config';
+import { fsa } from '@chunkd/fs';
+import { SwappingLru } from './swapping.lru.js';
+
+class LruConfig {
+  configProvider: Promise<ConfigProviderMemory>;
+
+  constructor(config: Promise<ConfigBundled>) {
+    this.configProvider = config.then((c) => {
+      const configProvider = ConfigProviderMemory.fromJson(c);
+      configProvider.createVirtualTileSets();
+      return configProvider;
+    });
+  }
+
+  get size(): number {
+    // Return size 1 for the config and cache the number of configs based on size number.
+    return 1;
+  }
+}
+
+export class ConfigCache {
+  cache: SwappingLru<LruConfig>;
+  constructor(maxSize: number) {
+    this.cache = new SwappingLru<LruConfig>(maxSize);
+  }
+
+  getConfig(location: string): Promise<ConfigProviderMemory | null> {
+    const existing = this.cache.get(location)?.configProvider;
+    if (existing != null) return existing;
+    try {
+      const configJson = fsa.readJson<ConfigBundled>(location);
+      const config = new LruConfig(configJson);
+      this.cache.set(location, config);
+      return config.configProvider;
+    } catch (e: any) {
+      if (e.code === 404) return Promise.resolve(null);
+      throw e;
+    }
+  }
+}
+
+/** Cache 20 configs(Around 500KB each)*/
+export const CachedConfig = new ConfigCache(20);

package/src/util/response.ts

@@ -1,4 +1,7 @@
-import { LambdaHttpResponse } from '@linzjs/lambda';
+import { LambdaHttpRequest, LambdaHttpResponse } from '@linzjs/lambda';
 
 export const NotFound = (): LambdaHttpResponse => new LambdaHttpResponse(404, 'Not Found');
 export const NotModified = (): LambdaHttpResponse => new LambdaHttpResponse(304, 'Not modified');
+export const NoContent = (): LambdaHttpResponse => new LambdaHttpResponse(204, 'No Content');
+export const OkResponse = (req: LambdaHttpRequest): LambdaHttpResponse =>
+  new LambdaHttpResponse(200, 'ok').json({ id: req.id, correlationId: req.correlationId, message: 'ok' });

package/src/util/source.cache.ts

@@ -68,4 +68,5 @@ export class SourceCache {
   }
 }
 
-export const CoSources = new SourceCache(256 * 1024 * 1024);
+/** Approx ~1.4GB 2x~700MB caches */
+export const CoSources = new SourceCache(700 * 1000 * 1000);

package/src/util/swapping.lru.ts

@@ -6,6 +6,7 @@ export class SwappingLru<T extends { size: number }> {
   hits = 0;
   misses = 0;
   resets = 0;
+  clears = 0;
 
   _lastCheckedAt = -1;
 
@@ -37,6 +38,7 @@ export class SwappingLru<T extends { size: number }> {
   clear(): void {
     this.cacheA.clear();
     this.cacheB.clear();
+    this.clears++;
   }
 
   set(id: string, tiff: T): void {
@@ -50,6 +52,11 @@ export class SwappingLru<T extends { size: number }> {
     if (this.maxSize <= 0) return;
     if (this.currentSize <= this.maxSize) return;
     this.resets++;
+    // Paranoia if we are resetting too often something is wrong, reset the entire cache.
+    if (this.resets > 100) {
+      this.resets = 0;
+      return this.clear();
+    }
     this.cacheB = this.cacheA;
     this.cacheA = new Map();
   }

package/src/util/validate.ts

@@ -1,8 +1,7 @@
 import { ImageFormat, TileMatrixSet, TileMatrixSets, VectorFormat } from '@basemaps/geo';
-import { Const, Projection } from '@basemaps/shared';
+import { Const, isValidApiKey, Projection } from '@basemaps/shared';
 import { getImageFormat } from '@basemaps/tiler';
 import { LambdaHttpRequest, LambdaHttpResponse } from '@linzjs/lambda';
-import * as ulid from 'ulid';
 import { TileXyzGet } from '../routes/tile.xyz';
 
 export interface TileXyz {
@@ -16,31 +15,6 @@ export interface TileMatrixRequest {
   Params: { tileMatrix?: string };
 }
 
-const OneHourMs = 60 * 60 * 1000;
-const OneDayMs = 24 * OneHourMs;
-const MaxApiAgeMs = 91 * OneDayMs;
-
-export interface ApiKeyStatus {
-  valid: boolean;
-  message: 'ok' | 'malformed' | 'missing' | 'expired';
-}
-
-export function isValidApiKey(apiKey?: string | null): ApiKeyStatus {
-  if (apiKey == null) return { valid: false, message: 'missing' };
-  if (!apiKey.startsWith('c') && !apiKey.startsWith('d')) return { valid: false, message: 'malformed' };
-  const ulidId = apiKey.slice(1).toUpperCase();
-  try {
-    const ulidTime = ulid.decodeTime(ulidId);
-    if (apiKey.startsWith('d')) return { valid: true, message: 'ok' };
-
-    if (Date.now() - ulidTime > MaxApiAgeMs) return { valid: false, message: 'expired' };
-  } catch (e) {
-    return { valid: false, message: 'malformed' };
-  }
-
-  return { valid: true, message: 'ok' };
-}
-
 export const Validate = {
   /**
    * Validate that the api key exists and is valid

package/tsconfig.json

@@ -6,6 +6,7 @@
   },
   "include": ["src/**/*"],
   "references": [
+    { "path": "../config" },
     { "path": "../shared" },
     { "path": "../geo" },
     { "path": "../tiler" },