@basemaps/lambda-tiler 6.36.0 → 6.37.0

package/dist/node_modules/.package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "@basemaps/lambda-tiler",
-  "version": "6.36.0",
+  "version": "6.37.0",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
@@ -229,9 +229,9 @@
       "integrity": "sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg=="
     },
     "node_modules/node-abi": {
-      "version": "3.26.0",
-      "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.26.0.tgz",
-      "integrity": "sha512-jRVtMFTChbi2i/jqo/i2iP9634KMe+7K1v35mIdj3Mn59i5q27ZYhn+sW6npISM/PQg7HrP2kwtRBMmh5Uvzdg==",
+      "version": "3.30.0",
+      "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.30.0.tgz",
+      "integrity": "sha512-qWO5l3SCqbwQavymOmtTVuCWZE23++S+rxyoHjXqUmPyzRcaoI4lA2gO55/drddGnedAyjA7sk76SfQ5lfUMnw==",
       "dependencies": {
         "semver": "^7.3.5"
       },

package/dist/node_modules/node-abi/abi_registry.json

@@ -67,6 +67,13 @@
     "future": false,
     "abi": "108"
   },
+  {
+    "runtime": "node",
+    "target": "19.0.0",
+    "lts": false,
+    "future": false,
+    "abi": "111"
+  },
   {
     "abi": "70",
     "future": false,
@@ -216,7 +223,7 @@
   },
   {
     "abi": "110",
-    "future": true,
+    "future": false,
     "lts": false,
     "runtime": "electron",
     "target": "22.0.0-alpha.1"

package/dist/node_modules/node-abi/package.json

@@ -1,6 +1,6 @@
 {
   "name": "node-abi",
-  "version": "3.26.0",
+  "version": "3.30.0",
   "description": "Get the Node ABI for a given target and runtime, and vice versa.",
   "main": "index.js",
   "scripts": {

package/dist/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@basemaps/lambda-tiler",
-  "version": "6.36.0",
+  "version": "6.37.0",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "@basemaps/lambda-tiler",
-      "version": "6.36.0",
+      "version": "6.37.0",
       "license": "MIT",
       "dependencies": {
         "sharp": "0.30.7"
@@ -240,9 +240,9 @@
       "integrity": "sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg=="
     },
     "node_modules/node-abi": {
-      "version": "3.26.0",
-      "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.26.0.tgz",
-      "integrity": "sha512-jRVtMFTChbi2i/jqo/i2iP9634KMe+7K1v35mIdj3Mn59i5q27ZYhn+sW6npISM/PQg7HrP2kwtRBMmh5Uvzdg==",
+      "version": "3.30.0",
+      "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.30.0.tgz",
+      "integrity": "sha512-qWO5l3SCqbwQavymOmtTVuCWZE23++S+rxyoHjXqUmPyzRcaoI4lA2gO55/drddGnedAyjA7sk76SfQ5lfUMnw==",
       "dependencies": {
         "semver": "^7.3.5"
       },
@@ -650,9 +650,9 @@
       "integrity": "sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg=="
     },
     "node-abi": {
-      "version": "3.26.0",
-      "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.26.0.tgz",
-      "integrity": "sha512-jRVtMFTChbi2i/jqo/i2iP9634KMe+7K1v35mIdj3Mn59i5q27ZYhn+sW6npISM/PQg7HrP2kwtRBMmh5Uvzdg==",
+      "version": "3.30.0",
+      "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.30.0.tgz",
+      "integrity": "sha512-qWO5l3SCqbwQavymOmtTVuCWZE23++S+rxyoHjXqUmPyzRcaoI4lA2gO55/drddGnedAyjA7sk76SfQ5lfUMnw==",
       "requires": {
         "semver": "^7.3.5"
       }

package/dist/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@basemaps/lambda-tiler",
-  "version": "6.36.0",
+  "version": "6.37.0",
   "repository": {
     "type": "git",
     "url": "https://github.com/linz/basemaps.git",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@basemaps/lambda-tiler",
-  "version": "6.36.0",
+  "version": "6.37.0",
   "repository": {
     "type": "git",
     "url": "https://github.com/linz/basemaps.git",
@@ -22,13 +22,13 @@
   "types": "./build/index.d.ts",
   "license": "MIT",
   "dependencies": {
-    "@basemaps/config": "^6.36.0",
-    "@basemaps/geo": "^6.32.1",
-    "@basemaps/shared": "^6.36.0",
-    "@basemaps/tiler": "^6.35.0",
-    "@basemaps/tiler-sharp": "^6.35.0",
-    "@chunkd/fs": "^8.5.1",
-    "@cogeotiff/core": "^7.0.0",
+    "@basemaps/config": "^6.37.0",
+    "@basemaps/geo": "^6.37.0",
+    "@basemaps/shared": "^6.37.0",
+    "@basemaps/tiler": "^6.37.0",
+    "@basemaps/tiler-sharp": "^6.37.0",
+    "@chunkd/fs": "^10.0.2",
+    "@cogeotiff/core": "^7.2.0",
     "@cotar/core": "^5.4.0",
     "@linzjs/geojson": "^6.32.1",
     "@linzjs/lambda": "^4.0.0",
@@ -51,12 +51,12 @@
     "bundle": "./bundle.sh"
   },
   "devDependencies": {
-    "@basemaps/attribution": "^6.32.1",
+    "@basemaps/attribution": "^6.37.0",
+    "@chunkd/source-memory": "^10.0.0",
     "@types/aws-lambda": "^8.10.75",
-    "@types/node": "^17.0.34",
     "@types/pixelmatch": "^5.0.0",
     "@types/sharp": "^0.30.3",
     "pretty-json-log": "^1.0.0"
   },
-  "gitHead": "c232694ad1f1186b7b56a8af1def56ef165df8fd"
+  "gitHead": "98c97100161b159272c1826c4e668b3e805ec214"
 }

package/src/__tests__/config.data.ts

@@ -10,7 +10,7 @@ import {
 } from '@basemaps/config';
 import { ImageFormat, VectorFormat } from '@basemaps/geo';
 import { fsa } from '@basemaps/shared';
-import { FsMemory } from '../routes/__tests__/memory.fs.js';
+import { FsMemory } from '@chunkd/source-memory';
 
 export const TileSetAerial: ConfigTileSetRaster = {
   id: 'ts_aerial',

package/src/routes/__tests__/fonts.test.ts

@@ -9,7 +9,7 @@ import { ConfigLoader } from '../../util/config.loader.js';
 import { CoSources } from '../../util/source.cache.js';
 import { Api, mockRequest, mockUrlRequest } from '../../__tests__/xyz.util.js';
 import { fontList } from '../fonts.js';
-import { FsMemory } from './memory.fs.js';
+import { FsMemory } from '@chunkd/source-memory';
 
 o.spec('/v1/fonts', () => {
   const memory = new FsMemory();

package/src/routes/__tests__/memory.fs.ts

@@ -1,4 +1,3 @@
-import { ChunkSource, CompositeError, FileInfo, FileSystem } from '@chunkd/core';
 import { Readable } from 'stream';
 
 export async function toBuffer(stream: Readable): Promise<Buffer> {
@@ -14,64 +13,3 @@ export function toReadable(r: string | Buffer | Readable): Readable {
   if (typeof r === 'string') r = Buffer.from(r);
   return Readable.from(r);
 }
-
-// TODO this should be provided by @chunkd/fs as it already provides a memory source
-export class FsMemory implements FileSystem {
-  protocol = 'memory';
-
-  files: Map<string, Buffer> = new Map();
-
-  async read(filePath: string): Promise<Buffer> {
-    const data = this.files.get(filePath);
-    if (data == null) throw new CompositeError('Not found', 404, new Error());
-    return data;
-  }
-
-  stream(filePath: string): Readable {
-    const buf = this.files.get(filePath);
-    if (buf == null) throw new CompositeError('Not found', 404, new Error());
-    return toReadable(buf);
-  }
-
-  async write(filePath: string, buffer: string | Buffer | Readable): Promise<void> {
-    if (typeof buffer === 'string') {
-      this.files.set(filePath, Buffer.from(buffer));
-      return;
-    }
-    if (Buffer.isBuffer(buffer)) {
-      this.files.set(filePath, buffer);
-      return;
-    }
-    const buf = await toBuffer(buffer);
-    this.files.set(filePath, buf);
-  }
-
-  async *list(filePath: string): AsyncGenerator<string> {
-    for (const file of this.files.keys()) {
-      if (file.startsWith(filePath)) yield file;
-    }
-  }
-
-  async *details(filePath: string): AsyncGenerator<FileInfo> {
-    for await (const file of this.list(filePath)) {
-      const data = await this.head(file);
-      if (data == null) continue;
-      yield data;
-    }
-  }
-
-  async exists(filePath: string): Promise<boolean> {
-    const dat = await this.head(filePath);
-    return dat != null;
-  }
-
-  async head(filePath: string): Promise<FileInfo | null> {
-    const buf = this.files.get(filePath);
-    if (buf == null) return null;
-    return { path: filePath, size: buf.length };
-  }
-
-  source(): ChunkSource {
-    throw new Error('Method not implemented.');
-  }
-}

package/src/routes/__tests__/sprites.test.ts

@@ -6,7 +6,7 @@ import { gunzipSync, gzipSync } from 'zlib';
 import { handler } from '../../index.js';
 import { ConfigLoader } from '../../util/config.loader.js';
 import { mockRequest } from '../../__tests__/xyz.util.js';
-import { FsMemory } from './memory.fs.js';
+import { FsMemory } from '@chunkd/source-memory';
 
 o.spec('/v1/sprites', () => {
   const memory = new FsMemory();

package/src/routes/__tests__/tile.json.test.ts

@@ -8,7 +8,7 @@ import { ConfigLoader } from '../../util/config.loader.js';
 import { CoSources } from '../../util/source.cache.js';
 import { FakeData } from '../../__tests__/config.data.js';
 import { Api, mockRequest, mockUrlRequest } from '../../__tests__/xyz.util.js';
-import { FsMemory } from './memory.fs.js';
+import { FsMemory } from '@chunkd/source-memory';
 
 o.spec('/v1/tiles/:tileSet/:tileMatrix/tile.json', () => {
   const config = new ConfigProviderMemory();

package/src/routes/tile.xyz.raster.ts

@@ -4,6 +4,7 @@ import { Env, fsa } from '@basemaps/shared';
 import { Tiler } from '@basemaps/tiler';
 import { TileMakerSharp } from '@basemaps/tiler-sharp';
 import { CogTiff } from '@cogeotiff/core';
+import { Cotar } from '@cotar/core';
 import { HttpHeader, LambdaHttpRequest, LambdaHttpResponse } from '@linzjs/lambda';
 import pLimit from 'p-limit';
 import { ConfigLoader } from '../util/config.loader.js';
@@ -20,13 +21,15 @@ export function getTiffName(name: string): string {
   return `${name}.tiff`;
 }
 
+export type CloudArchive = CogTiff | Cotar;
+
 export const TileComposer = new TileMakerSharp(256);
 
 const DefaultResizeKernel = { in: 'lanczos3', out: 'lanczos3' } as const;
 const DefaultBackground = { r: 0, g: 0, b: 0, alpha: 0 };
 
 export const TileXyzRaster = {
-  async getTiffsForTile(req: LambdaHttpRequest, tileSet: ConfigTileSetRaster, xyz: TileXyz): Promise<string[]> {
+  async getAssetsForTile(req: LambdaHttpRequest, tileSet: ConfigTileSetRaster, xyz: TileXyz): Promise<string[]> {
     const config = await ConfigLoader.load(req);
     const imagery = await getAllImagery(config, tileSet.layers, [xyz.tileMatrix.projection]);
 
@@ -55,6 +58,12 @@ export const TileXyzRaster = {
       }
       if (!tileBounds.intersects(Bounds.fromJson(img.bounds))) continue;
 
+      // FIXME is this meant to be >= <=
+      if (img.overviews && img.overviews.maxZoom >= filterZoom && img.overviews.minZoom <= filterZoom) {
+        output.push(fsa.join(img.uri, img.overviews.path));
+        continue;
+      }
+
       for (const c of img.files) {
         if (!tileBounds.intersects(Bounds.fromJson(c))) continue;
         const tiffPath = fsa.join(img.uri, getTiffName(c.name));
@@ -67,26 +76,32 @@ export const TileXyzRaster = {
   async tile(req: LambdaHttpRequest, tileSet: ConfigTileSetRaster, xyz: TileXyz): Promise<LambdaHttpResponse> {
     if (xyz.tileType === VectorFormat.MapboxVectorTiles) return NotFound();
 
-    const tiffPaths = await this.getTiffsForTile(req, tileSet, xyz);
-    const cacheKey = Etag.key(tiffPaths);
+    const assetPaths = await this.getAssetsForTile(req, tileSet, xyz);
+    const cacheKey = Etag.key(assetPaths);
     if (Etag.isNotModified(req, cacheKey)) return NotModified();
 
-    const toLoad: Promise<CogTiff | null>[] = [];
-    for (const tiffPath of tiffPaths) {
+    const toLoad: Promise<CloudArchive | null>[] = [];
+    for (const assetPath of assetPaths) {
       toLoad.push(
-        LoadingQueue(() => {
-          return CoSources.getCog(tiffPath).catch((error) => {
-            req.log.warn({ error, tiff: tiffPath }, 'TiffLoadFailed');
+        LoadingQueue((): Promise<CloudArchive | null> => {
+          if (assetPath.endsWith('.tar.co')) {
+            return CoSources.getCotar(assetPath).catch((error) => {
+              req.log.warn({ error, tiff: assetPath }, 'Load:Cotar:Failed');
+              return null;
+            });
+          }
+          return CoSources.getCog(assetPath).catch((error) => {
+            req.log.warn({ error, tiff: assetPath }, 'Load:Tiff:Failed');
             return null;
           });
         }),
       );
     }
 
-    const tiffs = (await Promise.all(toLoad)).filter((f) => f != null) as CogTiff[];
+    const assets = (await Promise.all(toLoad)).filter((f) => f != null) as CloudArchive[];
 
     const tiler = new Tiler(xyz.tileMatrix);
-    const layers = await tiler.tile(tiffs, xyz.tile.x, xyz.tile.y, xyz.tile.z);
+    const layers = await tiler.tile(assets, xyz.tile.x, xyz.tile.y, xyz.tile.z);
 
     const res = await TileComposer.compose({
       layers,

package/src/util/__test__/config.loader.test.ts

@@ -3,7 +3,7 @@ import { fsa } from '@basemaps/shared';
 import { LambdaHttpResponse } from '@linzjs/lambda';
 import o from 'ospec';
 import { createSandbox } from 'sinon';
-import { FsMemory } from '../../routes/__tests__/memory.fs.js';
+import { FsMemory } from '@chunkd/source-memory';
 import { FakeData } from '../../__tests__/config.data.js';
 import { Api, mockRequest, mockUrlRequest } from '../../__tests__/xyz.util.js';
 import { CachedConfig } from '../config.cache.js';

package/src/util/config.loader.ts

@@ -6,7 +6,7 @@ import { CachedConfig } from './config.cache.js';
 import { getDefaultConfig } from '@basemaps/shared';
 
 // FIXME load this from process.env COG BUCKETS?
-const SafeBuckets = new Set(['linz-workflow-artifacts', 'linz-basemaps']);
+const SafeBuckets = new Set(['linz-workflow-artifacts', 'linz-basemaps', 'linz-basemaps-staging']);
 const SafeProtocols = new Set(['s3', 'memory']);
 
 export class ConfigLoader {

package/src/wmts.capability.ts

@@ -64,6 +64,9 @@ export class WmtsCapabilities {
   formats: ImageFormat[];
   isIndividualLayers = false;
 
+  minZoom = 0;
+  maxZoom = 32;
+
   constructor(params: WmtsCapabilitiesParams) {
     this.httpBase = params.httpBase;
     this.provider = params.provider;
@@ -267,7 +270,7 @@ export class WmtsCapabilities {
       V('ows:Identifier', tms.identifier),
       V('ows:SupportedCRS', tms.projection.toUrn()),
       tms.def.wellKnownScaleSet ? V('WellKnownScaleSet', tms.def.wellKnownScaleSet) : null,
-      ...tms.def.tileMatrix.map((c) => {
+      ...tms.def.tileMatrix.slice(this.minZoom, this.maxZoom).map((c) => {
         return V('TileMatrix', [
           V('ows:Identifier', c.identifier),
           V('ScaleDenominator', c.scaleDenominator),