@basemaps/lambda-tiler 6.28.1 → 6.29.0

package/dist/package.json

@@ -0,0 +1,39 @@
+{
+  "name": "@basemaps/lambda-tiler",
+  "version": "6.29.0",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/linz/basemaps.git",
+    "directory": "packages/lambda-tiler"
+  },
+  "author": {
+    "name": "Land Information New Zealand",
+    "url": "https://linz.govt.nz",
+    "organization": true
+  },
+  "type": "commonjs",
+  "engines": {
+    "node": "^12.20.0 || ^14.13.1 || >=16.0.0"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "main": "index.js",
+  "license": "MIT",
+  "dependencies": {
+    "sharp": "0.30.7"
+  },
+  "bundle": {
+    "entry": "src/index.ts",
+    "outdir": "dist/",
+    "external": [
+      "aws-sdk",
+      "pino-pretty",
+      "sharp"
+    ]
+  },
+  "scripts": {
+    "test": "ospec --globs 'build/**/*.test.js'",
+    "bundle": "./bundle.sh"
+  }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@basemaps/lambda-tiler",
-  "version": "6.28.1",
+  "version": "6.29.0",
   "repository": {
     "type": "git",
     "url": "https://github.com/linz/basemaps.git",
@@ -22,13 +22,13 @@
   "types": "./build/index.d.ts",
   "license": "MIT",
   "dependencies": {
-    "@basemaps/config": "^6.28.1",
+    "@basemaps/config": "^6.29.0",
     "@basemaps/geo": "^6.28.1",
     "@basemaps/lambda": "^6.7.0",
-    "@basemaps/shared": "^6.28.1",
-    "@basemaps/tiler": "^6.28.1",
-    "@basemaps/tiler-sharp": "^6.28.1",
-    "@chunkd/fs": "^8.1.0",
+    "@basemaps/shared": "^6.29.0",
+    "@basemaps/tiler": "^6.29.0",
+    "@basemaps/tiler-sharp": "^6.29.0",
+    "@chunkd/fs": "^8.4.0",
     "@cogeotiff/core": "^7.0.0",
     "@cotar/core": "^5.3.0",
     "@linzjs/geojson": "^6.28.1",
@@ -36,7 +36,7 @@
     "p-limit": "^4.0.0",
     "path-to-regexp": "^6.1.0",
     "pixelmatch": "^5.1.0",
-    "sharp": "0.29.2"
+    "sharp": "^0.30.7"
   },
   "bundle": {
     "entry": "src/index.ts",
@@ -56,8 +56,8 @@
     "@types/aws-lambda": "^8.10.75",
     "@types/node": "^17.0.34",
     "@types/pixelmatch": "^5.0.0",
-    "@types/sharp": "0.29.2",
+    "@types/sharp": "^0.30.3",
     "pretty-json-log": "^1.0.0"
   },
-  "gitHead": "f387e0365bcfc7391fd8d0665195b246c93532cf"
+  "gitHead": "9097d2f045d734d0d5acd968f026fe29ab6afcda"
 }

package/scripts/create.deployment.package.mjs

@@ -24,7 +24,7 @@ export function getPackageVersion(packageName) {
 // the bundle is a commonjs module
 parentPackage.type = 'commonjs';
 parentPackage.main = 'index.js';
-parentPackage.dependencies = { sharp: '0.29.2' };
+parentPackage.dependencies = { sharp: getPackageVersion('sharp') };
 
 // Clean up
 delete parentPackage.types;

package/src/__tests__/xyz.util.ts

@@ -36,7 +36,6 @@ export class FakeTileSetVector extends TileSetVector {
 }
 
 export const Provider: ConfigProvider = {
-  createdAt: Date.now(),
   name: 'main',
   id: 'pv_main_production',
   updatedAt: Date.now(),

package/src/index.ts

@@ -8,6 +8,8 @@ import { createHash } from 'crypto';
 import { Imagery } from './routes/imagery.js';
 import { Esri } from './routes/esri/rest.js';
 import { St } from './source.tracer.js';
+import { spriteGet } from './routes/sprites.js';
+import { fontGet, fontList } from './routes/fonts.js';
 
 const app = new Router();
 
@@ -49,4 +51,9 @@ export async function handleRequest(req: LambdaHttpRequest): Promise<LambdaHttpR
 }
 
 export const handler = lf.http(LogConfig.get());
+
+handler.router.get('/v1/sprites/:spriteName', spriteGet);
+handler.router.get('/v1/fonts.json', fontList);
+handler.router.get('/v1/fonts/:fontStack/:range.pbf', fontGet);
+
 handler.router.get('*', handleRequest);

package/src/routes/__tests__/attribution.test.ts

@@ -279,7 +279,6 @@ function makeImageRecord(id: string, name: string, x = 10): ConfigImagery {
       b.x += i * b.width;
       return b;
     }),
-    createdAt: Date.now(),
     updatedAt: Date.now(),
   };
 }

package/src/routes/__tests__/fonts.test.ts

@@ -0,0 +1,71 @@
+import { Env } from '@basemaps/shared';
+import { fsa } from '@chunkd/fs';
+import o from 'ospec';
+import { handler } from '../../index.js';
+import { mockRequest } from '../../__tests__/xyz.util.js';
+import { fontList, getFonts } from '../fonts.js';
+import { FsMemory } from './memory.fs.js';
+
+o.spec('/v1/fonts', () => {
+  const memory = new FsMemory();
+  o.before(() => {
+    fsa.register('memory://', memory);
+  });
+
+  o.beforeEach(() => {
+    process.env[Env.AssetLocation] = 'memory://';
+  });
+
+  o.afterEach(() => {
+    delete process.env[Env.AssetLocation];
+    memory.files.clear();
+  });
+
+  o('should list font types', async () => {
+    await Promise.all([
+      fsa.write('memory://fonts/Roboto Thin/0-255.pbf', Buffer.from('')),
+      fsa.write('memory://fonts/Roboto Thin/256-512.pbf', Buffer.from('')),
+      fsa.write('memory://fonts/Roboto Black/0-255.pbf', Buffer.from('')),
+    ]);
+
+    const fonts = await getFonts('memory://fonts/');
+    o(fonts).deepEquals(['Roboto Black', 'Roboto Thin']);
+  });
+
+  o('should return empty list if no fonts found', async () => {
+    const res = await fontList();
+    o(res.status).equals(200);
+    o(res.body).equals('[]');
+  });
+
+  o('should return 404 if no assets defined', async () => {
+    delete process.env[Env.AssetLocation];
+    const res = await fontList();
+    o(res.status).equals(404);
+  });
+
+  o('should return a list of fonts found', async () => {
+    await Promise.all([
+      fsa.write('memory://fonts/Roboto Thin/0-255.pbf', Buffer.from('')),
+      fsa.write('memory://fonts/Roboto Thin/256-512.pbf', Buffer.from('')),
+      fsa.write('memory://fonts/Roboto Black/0-255.pbf', Buffer.from('')),
+    ]);
+    const res = await fontList();
+    o(res.status).equals(200);
+    o(res.header('content-type')).equals('application/json');
+    o(res.header('content-encoding')).equals(undefined);
+    o(res.body).equals(JSON.stringify(['Roboto Black', 'Roboto Thin']));
+  });
+
+  o('should get the correct font', async () => {
+    await fsa.write('memory://fonts/Roboto Thin/0-255.pbf', Buffer.from(''));
+
+    const res255 = await handler.router.handle(mockRequest('/v1/fonts/Roboto Thin/0-255.pbf'));
+    o(res255.status).equals(200);
+    o(res255.header('content-type')).equals('application/x-protobuf');
+    o(res255.header('content-encoding')).equals(undefined);
+
+    const res404 = await handler.router.handle(mockRequest('/v1/fonts/Roboto Thin/256-512.pbf'));
+    o(res404.status).equals(404);
+  });
+});

package/src/routes/__tests__/memory.fs.ts

@@ -0,0 +1,77 @@
+import { ChunkSource, CompositeError, FileInfo, FileSystem } from '@chunkd/core';
+import { Readable } from 'stream';
+
+export async function toBuffer(stream: Readable): Promise<Buffer> {
+  return new Promise<Buffer>((resolve, reject) => {
+    const buf: Buffer[] = [];
+
+    stream.on('data', (chunk) => buf.push(chunk));
+    stream.on('end', () => resolve(Buffer.concat(buf)));
+    stream.on('error', (err) => reject(`error converting stream - ${err}`));
+  });
+}
+export function toReadable(r: string | Buffer | Readable): Readable {
+  if (typeof r === 'string') r = Buffer.from(r);
+  return Readable.from(r);
+}
+
+// TODO this should be provided by @chunkd/fs as it already provides a memory source
+export class FsMemory implements FileSystem {
+  protocol = 'memory';
+
+  files: Map<string, Buffer> = new Map();
+
+  async read(filePath: string): Promise<Buffer> {
+    const data = this.files.get(filePath);
+    if (data == null) throw new CompositeError('Not found', 404, new Error());
+    return data;
+  }
+
+  stream(filePath: string): Readable {
+    const buf = this.files.get(filePath);
+    if (buf == null) throw new CompositeError('Not found', 404, new Error());
+    return toReadable(buf);
+  }
+
+  async write(filePath: string, buffer: string | Buffer | Readable): Promise<void> {
+    if (typeof buffer === 'string') {
+      this.files.set(filePath, Buffer.from(buffer));
+      return;
+    }
+    if (Buffer.isBuffer(buffer)) {
+      this.files.set(filePath, buffer);
+      return;
+    }
+    const buf = await toBuffer(buffer);
+    this.files.set(filePath, buf);
+  }
+
+  async *list(filePath: string): AsyncGenerator<string> {
+    for (const file of this.files.keys()) {
+      if (file.startsWith(filePath)) yield file;
+    }
+  }
+
+  async *details(filePath: string): AsyncGenerator<FileInfo> {
+    for await (const file of this.list(filePath)) {
+      const data = await this.head(file);
+      if (data == null) continue;
+      yield data;
+    }
+  }
+
+  async exists(filePath: string): Promise<boolean> {
+    const dat = await this.head(filePath);
+    return dat != null;
+  }
+
+  async head(filePath: string): Promise<FileInfo | null> {
+    const buf = this.files.get(filePath);
+    if (buf == null) return null;
+    return { path: filePath, size: buf.length };
+  }
+
+  source(): ChunkSource {
+    throw new Error('Method not implemented.');
+  }
+}

package/src/routes/__tests__/sprites.test.ts

@@ -0,0 +1,64 @@
+import { Env } from '@basemaps/shared';
+import { fsa } from '@chunkd/fs';
+import o from 'ospec';
+import { gunzipSync, gzipSync } from 'zlib';
+import { handler } from '../../index.js';
+import { mockRequest } from '../../__tests__/xyz.util.js';
+import { FsMemory } from './memory.fs.js';
+
+o.spec('/v1/sprites', () => {
+  const memory = new FsMemory();
+  o.before(() => {
+    fsa.register('memory://', memory);
+  });
+
+  o.beforeEach(() => {
+    process.env[Env.AssetLocation] = 'memory://';
+  });
+
+  o.afterEach(() => {
+    delete process.env[Env.AssetLocation];
+    memory.files.clear();
+  });
+
+  o('should return 404 if no assets defined', async () => {
+    delete process.env[Env.AssetLocation];
+    const res404 = await handler.router.handle(mockRequest('/v1/sprites/topographic.json'));
+    o(res404.status).equals(404);
+  });
+
+  o('should fetch a json document', async () => {
+    await Promise.all([
+      fsa.write('memory://sprites/topographic.json', Buffer.from(JSON.stringify({ test: true }))),
+      fsa.write('memory://sprites/topographic.png', Buffer.from('')),
+    ]);
+    const res = await handler.router.handle(mockRequest('/v1/sprites/topographic.json'));
+    o(res.status).equals(200);
+    o(res.header('content-type')).equals('application/json');
+    o(res.header('content-encoding')).equals(undefined);
+
+    o(JSON.parse(Buffer.from(res.body, 'base64').toString())).deepEquals({ test: true });
+  });
+
+  o('should fetch a png', async () => {
+    await Promise.all([
+      fsa.write('memory://sprites/topographic.json', Buffer.from(JSON.stringify({ test: true }))),
+      fsa.write('memory://sprites/topographic@2x.png', Buffer.from('')),
+    ]);
+    const res = await handler.router.handle(mockRequest('/v1/sprites/topographic@2x.png'));
+    o(res.status).equals(200);
+    o(res.header('content-type')).equals('image/png');
+  });
+
+  o('should detect gziped files and set content-encoding', async () => {
+    await Promise.all([
+      fsa.write('memory://sprites/topographic.json', gzipSync(Buffer.from(JSON.stringify({ test: true })))),
+    ]);
+    const res = await handler.router.handle(mockRequest('/v1/sprites/topographic.json'));
+    o(res.status).equals(200);
+    o(res.header('content-type')).equals('application/json');
+    o(res.header('content-encoding')).equals('gzip');
+
+    o(JSON.parse(gunzipSync(Buffer.from(res.body, 'base64')).toString())).deepEquals({ test: true });
+  });
+});

package/src/routes/fonts.ts

@@ -0,0 +1,54 @@
+import { Env } from '@basemaps/shared';
+import { fsa } from '@chunkd/fs';
+import { LambdaHttpRequest, LambdaHttpResponse } from '@linzjs/lambda';
+import path from 'path';
+import { NotFound } from './response.js';
+
+interface FontGet {
+  Params: { fontStack: string; range: string };
+}
+
+export async function fontGet(req: LambdaHttpRequest<FontGet>): Promise<LambdaHttpResponse> {
+  const assetLocation = Env.get(Env.AssetLocation);
+  if (assetLocation == null) return NotFound;
+
+  try {
+    const filePath = fsa.join(assetLocation, path.join('fonts', req.params.fontStack, req.params.range)) + '.pbf';
+    const buf = await fsa.read(filePath);
+
+    return LambdaHttpResponse.ok().buffer(buf, 'application/x-protobuf');
+  } catch (e: any) {
+    if (e.code === 404) return NotFound;
+    throw e;
+  }
+}
+
+/** Get the unique name of folders is a path that contain .pbf files */
+export async function getFonts(fontPath: string): Promise<string[]> {
+  const fonts = new Set<string>();
+
+  // TODO use {recursive: false}
+  for await (const font of fsa.list(fontPath)) {
+    if (!font.endsWith('.pbf')) continue;
+    const dirName = path.basename(path.dirname(font)); // TODO this only works for /a/b.pbf and not /a/b/c.pbf
+    if (dirName.includes('/')) continue;
+    fonts.add(dirName);
+  }
+  // Ensure the fonts are alphabetical
+  return [...fonts].sort();
+}
+
+export async function fontList(): Promise<LambdaHttpResponse> {
+  const assetLocation = Env.get(Env.AssetLocation);
+  if (assetLocation == null) return NotFound;
+
+  try {
+    const filePath = fsa.join(assetLocation, '/fonts');
+    const fonts = await getFonts(filePath);
+
+    return LambdaHttpResponse.ok().buffer(JSON.stringify(fonts), 'application/json');
+  } catch (e: any) {
+    if (e.code === 404) return NotFound;
+    throw e;
+  }
+}

package/src/routes/sprites.ts

@@ -0,0 +1,49 @@
+import { Env } from '@basemaps/shared';
+import { fsa } from '@chunkd/fs';
+import path from 'path';
+import { LambdaHttpRequest, LambdaHttpResponse } from '@linzjs/lambda';
+import { NotFound } from './response.js';
+
+interface SpriteGet {
+  Params: {
+    spriteName: string;
+  };
+}
+
+const Extensions = new Map();
+Extensions.set('.png', 'image/png');
+Extensions.set('.json', 'application/json');
+
+/**
+ * Does a buffer look like a gzipped document instead of raw json
+ *
+ * Determined by checking the first two bytes are the gzip magic bytes `0x1f 0x8b`
+ *
+ * @see https://en.wikipedia.org/wiki/Gzip
+ *
+ */
+function isGzip(b: Buffer): boolean {
+  return b[0] === 0x1f && b[1] === 0x8b;
+}
+
+export async function spriteGet(req: LambdaHttpRequest<SpriteGet>): Promise<LambdaHttpResponse> {
+  const spriteLocation = Env.get(Env.AssetLocation);
+  if (spriteLocation == null) return NotFound;
+
+  const extension = path.extname(req.params.spriteName);
+  const mimeType = Extensions.get(extension);
+  if (mimeType == null) return NotFound;
+
+  try {
+    const filePath = fsa.join(spriteLocation, fsa.join('/sprites', req.params.spriteName));
+    req.set('target', filePath);
+
+    const buf = await fsa.read(filePath);
+    const res = LambdaHttpResponse.ok().buffer(buf, mimeType);
+    if (isGzip(buf)) res.header('content-encoding', 'gzip');
+    return res;
+  } catch (e: any) {
+    if (e.code === 404) return NotFound;
+    throw e;
+  }
+}