@basemaps/server 6.26.0 → 6.28.1

package/CHANGELOG.md

@@ -3,6 +3,41 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+## [6.28.1](https://github.com/linz/basemaps/compare/v6.28.0...v6.28.1) (2022-06-07)
+
+**Note:** Version bump only for package @basemaps/server
+
+
+
+
+
+# [6.28.0](https://github.com/linz/basemaps/compare/v6.27.0...v6.28.0) (2022-06-06)
+
+
+### Features
+
+* **docker:** add a container of basemaps-server and basemaps-landing ([#2225](https://github.com/linz/basemaps/issues/2225)) ([13f8144](https://github.com/linz/basemaps/commit/13f814446de0b2896e0ccca620d4b0a017380c14))
+* **server:** bundle basemaps-server cli so its easier to install ([#2218](https://github.com/linz/basemaps/issues/2218)) ([8457b66](https://github.com/linz/basemaps/commit/8457b66be6d0f54decf43b515bb78853cefbc8ed))
+* **server:** provide a better error when loading configuration bundles ([#2222](https://github.com/linz/basemaps/issues/2222)) ([8318192](https://github.com/linz/basemaps/commit/83181920c8a9e061babd38a8ffd0dec93830dced))
+* **sprites:** create sprites using sharp ([#2235](https://github.com/linz/basemaps/issues/2235)) ([e7b6a9e](https://github.com/linz/basemaps/commit/e7b6a9e9c95359dc866b40e7a6988837a71d9d96))
+
+
+
+
+
+# [6.27.0](https://github.com/linz/basemaps/compare/v6.26.0...v6.27.0) (2022-05-29)
+
+
+### Features
+
+* **config:** add configuration parser and bundler ([#2200](https://github.com/linz/basemaps/issues/2200)) ([795e3f2](https://github.com/linz/basemaps/commit/795e3f224ee0b4cd1e66a242d05a1fd5357cae3a))
+* **lambda-cog:** New lambda Cog for import api. ([#2207](https://github.com/linz/basemaps/issues/2207)) ([79f4ae7](https://github.com/linz/basemaps/commit/79f4ae70ea3fc16a37dd575b843a0b60a1365df4))
+* **server:** use a bundled `@basemaps/landing` to serve static assets ([#2202](https://github.com/linz/basemaps/issues/2202)) ([c60f518](https://github.com/linz/basemaps/commit/c60f518893fe037a03f8bfd489c84d8427481678))
+
+
+
+
+
 # [6.26.0](https://github.com/linz/basemaps/compare/v6.25.0...v6.26.0) (2022-05-12)
 
 **Note:** Version bump only for package @basemaps/server

package/README.md

@@ -2,14 +2,42 @@
 
 WMTS/XYZ Tile server command line interface.
 
-This wraps the @basemaps/lambda-tiler into a standalone express http server.
+This wraps the @basemaps/tiler into a standalone express http server.
 
 ## Usage
+Basemaps server expects a folder tree full of configuration, with multiple tilesets and styles configuration files.
 
+```bash
+basemaps-server --config path/to/config/
 ```
-basemaps-server --config path/to/config
+
+### Usage docker
+
+The server is also published as a docker container 
+
+```bash
+docker run -it \
+    --volume $PWD/config:/config \
+    --volume $PWD/tiffs:$PWD/tiffs \
+    -p 5000:5000 \
+     ghcr.io/linz/basemaps-server:v6.28.0 --config /config
+```
+
+Where `${PWD}/config` contains all the configuration and `${PWD}/tiffs` is all the relevant tiff files
+
+### Bundled configuration
+
+Basemaps server can also be configured with a single configuration bundle file, which can be created from a configuration folder
+
+By bundling the configuration the startup time is greatly reduced as each individual tiff file does not have to be scanned for dimensions, it does mean that new tiffs will not be picked up until a new bundle is created.
+
+```bash
+basemaps-server --config path/to/config/ --bundle config.bundle.json
+basemaps-server --config config.bundle.json
 ```
 
+
+### Usage with LINZ imagery
 Usage with basemaps config, you will need access to basemaps' imagery cache 
 
 please contact basemaps@linz.govt.nz if you need access.
@@ -20,6 +48,8 @@ basemaps-server --config basemaps-config/config
 ```
 
 
+
+
 ### Direct TIFF access
 
 If you have a folder of tiffs the basemaps-server can index the folder and create tiles from
@@ -32,4 +62,18 @@ For example given a structure where there are two folders full of tiffs
 /images/10_geographx_nz_texture_shade_2012_8-0m_01FHRPYJ5FV1XAARZAC4T4K6MC/*.tiff
 ```
 
-running `basemaps-server --config /images` will create two tile sets one for each folder `gebco_2021...` and `geographx_nz_t...` and then also create a combined layer in the order the tiffs are found.
+running `basemaps-server --config /images --no-config` will create two tile sets one for each folder `gebco_2021...` and `geographx_nz_t...` and then also create a combined layer in the order the tiffs are found.
+
+
+## Developing
+
+When running the `@basemaps/server` in development mode, ensure `@basemaps/landing` page has been bundled
+
+```bash
+yarn
+yarn build
+
+npx lerna run bundle --stream # Bundle everything
+```
+
+This will package all the static assets into `landing/dist` and will be served from `http://localhost:5000`

package/build/cli.d.ts

@@ -1,14 +1,13 @@
-import { Command, flags } from '@oclif/command';
-export declare class BasemapsServerCommand extends Command {
-    static description: string;
-    static flags: {
-        verbose: import("@oclif/parser/lib/flags").IBooleanFlag<boolean>;
-        port: import("@oclif/parser/lib/flags").IOptionFlag<number>;
-        dynamo: flags.IOptionFlag<string | undefined>;
-        config: flags.IOptionFlag<string | undefined>;
-    };
+import { LogType } from '@basemaps/shared';
+import { BaseCommandLine } from '@basemaps/shared/build/cli/base.js';
+export declare class BasemapsServerCommand extends BaseCommandLine {
+    config: import("@rushstack/ts-command-line").CommandLineStringParameter;
+    ignoreConfig: import("@rushstack/ts-command-line").CommandLineFlagParameter;
+    bundle: import("@rushstack/ts-command-line").CommandLineStringParameter;
+    port: import("@rushstack/ts-command-line").CommandLineIntegerParameter;
+    constructor();
     static args: never[];
-    loadFromPath(configPath: string, serverUrl: string): Promise<void>;
-    run(): Promise<void>;
+    loadTiffs(tiffPath: string, serverUrl: string, logger: LogType): Promise<void>;
+    onExecute(): Promise<void>;
 }
 //# sourceMappingURL=cli.d.ts.map

package/build/cli.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAkBhD,qBAAa,qBAAsB,SAAQ,OAAO;IAChD,MAAM,CAAC,WAAW,SAAuD;IACzE,MAAM,CAAC,KAAK;;;;;MAKV;IAEF,MAAM,CAAC,IAAI,UAAM;IAEX,YAAY,CAAC,UAAU,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IA+ClE,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;CAsB3B"}
+{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":"AAGA,OAAO,EAAsC,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAC/E,OAAO,EAAE,eAAe,EAAW,MAAM,oCAAoC,CAAC;AAoB9E,qBAAa,qBAAsB,SAAQ,eAAe;IACxD,MAAM,kEAIH;IACH,YAAY,gEAGT;IACH,MAAM,kEAIH;IACH,IAAI,mEAKD;;IASH,MAAM,CAAC,IAAI,UAAM;IAEX,SAAS,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IA0B9E,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;CAkDjC"}

package/build/cli.js

@@ -1,13 +1,11 @@
-// Configure the logging before importing everything
-import { ConfigPrefix, ConfigProviderDynamo, ConfigProviderMemory, parseRgba } from '@basemaps/config';
+import { ConfigJson, ConfigProviderDynamo, ConfigProviderMemory } from '@basemaps/config';
 import { TileSetLocal } from '@basemaps/lambda-tiler/build/cli/tile.set.local.js';
 import { TileSets } from '@basemaps/lambda-tiler/build/tile.set.cache.js';
-import { Config, Env, LogConfig } from '@basemaps/shared';
-import { fsa } from '@linzjs/s3fs';
-import { Command, flags } from '@oclif/command';
+import { Config, Const, Env, fsa, LogConfig } from '@basemaps/shared';
+import { BaseCommandLine, CliInfo } from '@basemaps/shared/build/cli/base.js';
 import { basename, dirname } from 'path';
-import { BasemapsServer } from './server.js';
-const logger = LogConfig.get();
+import { createServer } from './server.js';
+CliInfo.package = 'basemaps/server';
 const BaseProvider = {
     id: 'pv_linz',
     version: 1,
@@ -19,12 +17,39 @@ const BaseProvider = {
         },
     },
 };
-export class BasemapsServerCommand extends Command {
-    async loadFromPath(configPath, serverUrl) {
+const DefaultPort = 5000;
+export class BasemapsServerCommand extends BaseCommandLine {
+    constructor() {
+        super({
+            toolFilename: 'bms',
+            toolDescription: 'Create a WMTS/XYZ Tile server from basemaps config',
+        });
+        this.config = this.defineStringParameter({
+            argumentName: 'CONFIG',
+            parameterLongName: '--config',
+            description: 'Configuration source to use',
+        });
+        this.ignoreConfig = this.defineFlagParameter({
+            parameterLongName: '--no-config',
+            description: 'Assume no config and just load tiffs from the configuration path',
+        });
+        this.bundle = this.defineStringParameter({
+            argumentName: 'BUNDLE',
+            parameterLongName: '--bundle',
+            description: 'Compile the configuration into a bundle and output path',
+        });
+        this.port = this.defineIntegerParameter({
+            argumentName: 'PORT',
+            parameterLongName: '--port',
+            description: 'port to use',
+            defaultValue: DefaultPort,
+        });
+    }
+    async loadTiffs(tiffPath, serverUrl, logger) {
         const config = new ConfigProviderMemory();
         Config.setConfigProvider(config);
         const tifSets = new Map();
-        for await (const file of fsa.listDetails(configPath)) {
+        for await (const file of fsa.details(tiffPath)) {
             const lowerPath = file.path.toLowerCase();
             if (lowerPath.endsWith('.tiff') || lowerPath.endsWith('.tif')) {
                 const tiffPath = dirname(file.path);
@@ -40,58 +65,58 @@ export class BasemapsServerCommand extends Command {
                 if (!config.objects.has('pv_linz'))
                     config.put(BaseProvider);
             }
-            if (!lowerPath.endsWith('.json'))
-                continue;
-            logger.trace({ path: file.path, size: file.size }, 'Config:Load');
-            const jsonData = await fsa.read(file.path).then((c) => JSON.parse(c.toString()));
-            if (jsonData.id == null) {
-                logger.warn({ path: file.path }, 'Config:Invalid Missing "id"');
-                continue;
-            }
-            // Last modified now, make sure its considered fresh
-            jsonData.updatedAt = Date.now();
-            // TODO we should really use something like zod to validate this
-            config.put(jsonData);
-            if (!Config.TileSet.is(jsonData) || !Config.isTileSetRaster(jsonData))
-                continue;
-            if (typeof jsonData.background === 'string')
-                jsonData.background = parseRgba(jsonData.background);
-            const tileSet = Config.unprefix(ConfigPrefix.TileSet, jsonData.id);
-            if (jsonData.name == null)
-                jsonData.name = tileSet;
-            const wmtsUrl = `${serverUrl}/v1/tiles/${tileSet}/WMTSCapabilities.xml`;
-            logger.info({ tileSetId: jsonData.id, wmtsUrl }, 'TileSet:Loaded');
         }
     }
-    async run() {
-        var _a;
-        const { flags } = this.parse(BasemapsServerCommand);
-        if (flags.verbose)
-            logger.level = 'debug';
-        const ServerUrl = `http://localhost:${flags.port}`;
-        if (flags.config != null) {
-            logger.info({ path: flags.config }, 'Starting Server');
-            await this.loadFromPath(flags.config, ServerUrl);
+    async onExecute() {
+        var _a, _b;
+        await super.onExecute();
+        const logger = LogConfig.get();
+        logger.level = 'debug';
+        const config = (_a = this.config.value) !== null && _a !== void 0 ? _a : 'dynamodb://' + Const.TileMetadata.TableName;
+        const port = this.port.value;
+        const ServerUrl = (_b = Env.get(Env.PublicUrlBase)) !== null && _b !== void 0 ? _b : `http://localhost:${port}`;
+        // Force a default url base so WMTS requests know their relative url
+        process.env[Env.PublicUrlBase] = ServerUrl;
+        if (config.startsWith('dynamodb://')) {
+            // Load config from dynamodb table
+            const table = config.slice('dynamodb://'.length);
+            logger.info({ path: config, table, mode: 'dynamo' }, 'Starting Server');
+            Config.setConfigProvider(new ConfigProviderDynamo(table));
+        }
+        else if (config.endsWith('.json')) {
+            // Bundled config
+            logger.info({ path: config, mode: 'config' }, 'Starting Server');
+            const configJson = await fsa.read(config);
+            const mem = ConfigProviderMemory.fromJson(JSON.parse(configJson.toString()));
+            Config.setConfigProvider(mem);
         }
-        else if (flags.dynamo != null) {
-            logger.info({ dynamo: flags.dynamo }, 'Starting Server');
-            Config.setConfigProvider(new ConfigProviderDynamo(flags.dynamo));
+        else if (this.ignoreConfig.value) {
+            // Load config directly from tiff files
+            logger.info({ path: config, mode: 'tiffs' }, 'Starting Server');
+            await this.loadTiffs(config, ServerUrl, logger);
         }
         else {
-            throw new Error('Either a configuration path or dynamodb table name must be supplied');
+            const mem = await ConfigJson.fromPath(config, logger);
+            const bundlePath = this.bundle.value;
+            if (bundlePath) {
+                await fsa.writeJson(bundlePath, mem.toJson());
+                logger.info({ path: bundlePath }, 'ConfigBundled');
+                return;
+            }
+            // Assume the folder is a collection of config files
+            logger.info({ path: config, mode: 'config' }, 'Starting Server');
+            mem.createVirtualTileSets();
+            Config.setConfigProvider(mem);
         }
-        // Force a default url base so WMTS requests know their relative url
-        process.env[Env.PublicUrlBase] = (_a = process.env[Env.PublicUrlBase]) !== null && _a !== void 0 ? _a : `http://localhost:${flags.port}`;
-        BasemapsServer.listen(flags.port, () => {
+        if (this.bundle.value != null) {
+            throw new Error('--bundle path provided without providing a configuration path');
+        }
+        createServer(logger).listen(port !== null && port !== void 0 ? port : DefaultPort, '0.0.0.0', () => {
             logger.info({ url: ServerUrl }, 'ServerStarted');
         });
     }
 }
-BasemapsServerCommand.description = 'Create a WMTS/XYZ Tile server for basemaps config';
-BasemapsServerCommand.flags = {
-    verbose: flags.boolean(),
-    port: flags.integer({ default: 5000 }),
-    dynamo: flags.string({ description: 'Dynamodb table', required: false }),
-    config: flags.string({ description: 'Configuration path', required: false }),
-};
 BasemapsServerCommand.args = [];
+new BasemapsServerCommand().executeWithoutErrorHandling().catch((c) => {
+    console.error(c);
+});

package/build/index.d.ts

@@ -1,2 +1,2 @@
-export { BasemapsServer } from './server.js';
+export { createServer } from './server.js';
 //# sourceMappingURL=index.d.ts.map

package/build/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC"}

package/build/index.js

@@ -1 +1 @@
-export { BasemapsServer } from './server.js';
+export { createServer } from './server.js';

package/build/server.d.ts

@@ -1,3 +1,4 @@
-/// <reference types="node" />
-export declare const BasemapsServer: import("fastify").FastifyInstance<import("http").Server, import("http").IncomingMessage, import("http").ServerResponse, import("fastify").FastifyLoggerInstance> & PromiseLike<import("fastify").FastifyInstance<import("http").Server, import("http").IncomingMessage, import("http").ServerResponse, import("fastify").FastifyLoggerInstance>>;
+import { LogType } from '@basemaps/shared';
+import { FastifyInstance } from 'fastify';
+export declare function createServer(logger: LogType): FastifyInstance;
 //# sourceMappingURL=server.d.ts.map

package/build/server.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":";AAMA,eAAO,MAAM,cAAc,kVAAY,CAAC"}
+{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAI3C,OAAgB,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAuBnD,wBAAgB,YAAY,CAAC,MAAM,EAAE,OAAO,GAAG,eAAe,CAwC7D"}

package/build/server.js

@@ -1,8 +1,11 @@
 import { handler } from '@basemaps/lambda-tiler';
+import fastifyStatic from '@fastify/static';
 import { lf } from '@linzjs/lambda';
 import fastify from 'fastify';
+import { createRequire } from 'module';
+import path from 'path';
 import ulid from 'ulid';
-export const BasemapsServer = fastify();
+import { URL } from 'url';
 function isAlbResult(r) {
     if (typeof r !== 'object')
         return false;
@@ -11,31 +14,54 @@ function isAlbResult(r) {
     return 'statusCode' in r;
 }
 const instanceId = ulid.ulid();
-BasemapsServer.get('/v1/*', async (req, res) => {
-    const event = {
-        httpMethod: 'GET',
-        requestContext: { elb: { targetGroupArn: 'arn:fake' } },
-        path: req.url,
-        headers: req.headers,
-        queryStringParameters: req.query,
-        body: null,
-        isBase64Encoded: false,
-    };
-    if (req.query.api == null)
-        req.query.api = 'c' + instanceId;
-    handler(event, {}, (err, r) => {
-        var _a;
-        if (err || !isAlbResult(r)) {
-            lf.Logger.fatal({ err }, 'RequestFailed');
-            res.send(err);
-            return;
-        }
-        res.status(r.statusCode);
-        for (const [key, value] of Object.entries((_a = r.headers) !== null && _a !== void 0 ? _a : {}))
-            res.header(key, String(value));
-        if (r.body)
-            res.send(Buffer.from(r.body, r.isBase64Encoded ? 'base64' : 'utf8'));
-        else
-            res.send('Not found');
+function getLandingLocation() {
+    if (typeof require !== 'undefined' && typeof require.resolve === 'function') {
+        return require.resolve('@basemaps/landing/dist');
+    }
+    else {
+        const require = createRequire(import.meta.url);
+        return require.resolve('@basemaps/landing/dist');
+    }
+}
+export function createServer(logger) {
+    const BasemapsServer = fastify();
+    const landingLocation = getLandingLocation();
+    if (landingLocation == null) {
+        logger.warn('Server:Landing:Failed');
+    }
+    else {
+        const root = path.join(path.dirname(landingLocation), '..', 'dist');
+        logger.info({ path: root }, 'Server:Landing');
+        BasemapsServer.register(fastifyStatic, { root });
+    }
+    BasemapsServer.get('/v1/*', async (req, res) => {
+        const url = new URL(`${req.protocol}://${req.hostname}${req.url}`);
+        const event = {
+            httpMethod: 'GET',
+            requestContext: { elb: { targetGroupArn: 'arn:fake' } },
+            path: url.pathname,
+            headers: req.headers,
+            queryStringParameters: req.query,
+            body: null,
+            isBase64Encoded: false,
+        };
+        if (req.query.api == null)
+            req.query.api = 'c' + instanceId;
+        handler(event, {}, (err, r) => {
+            var _a;
+            if (err || !isAlbResult(r)) {
+                lf.Logger.fatal({ err }, 'RequestFailed');
+                res.send(err);
+                return;
+            }
+            res.status(r.statusCode);
+            for (const [key, value] of Object.entries((_a = r.headers) !== null && _a !== void 0 ? _a : {}))
+                res.header(key, String(value));
+            if (r.body)
+                res.send(Buffer.from(r.body, r.isBase64Encoded ? 'base64' : 'utf8'));
+            else
+                res.send('Not found');
+        });
     });
-});
+    return BasemapsServer;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@basemaps/server",
-  "version": "6.26.0",
+  "version": "6.28.1",
   "repository": {
     "type": "git",
     "url": "https://github.com/linz/basemaps.git",
@@ -19,27 +19,45 @@
   "main": "./build/index.js",
   "types": "./build/index.d.ts",
   "scripts": {
-    "test": "ospec --globs 'build/**/*.test.js'"
+    "test": "ospec --globs 'build/**/*.test.js'",
+    "bundle": "../../scripts/bundle.js package.json"
   },
   "publishConfig": {
     "access": "public"
   },
   "bin": {
-    "basemaps-server": "./bin/basemaps-server.js"
+    "basemaps-server": "./bin/basemaps-server.cjs"
   },
+  "bundle": [
+    {
+      "entry": "src/cli.ts",
+      "outfile": "bin/basemaps-server.cjs",
+      "executable": true,
+      "external": [
+        "pino-pretty",
+        "sharp"
+      ]
+    }
+  ],
   "files": [
     "build/",
     "bin/"
   ],
+  "optionalDependencies": {
+    "@basemaps/landing": "^6.28.1"
+  },
   "dependencies": {
-    "@basemaps/config": "^6.26.0",
-    "@basemaps/geo": "^6.26.0",
-    "@basemaps/lambda-tiler": "^6.26.0",
-    "@basemaps/shared": "^6.26.0",
-    "@linzjs/lambda": "^2.0.0",
-    "@oclif/command": "^1.8.0",
+    "sharp": "^0.30.2"
+  },
+  "devDependencies": {
+    "@basemaps/config": "^6.28.1",
+    "@basemaps/geo": "^6.28.1",
+    "@basemaps/lambda-tiler": "^6.28.1",
+    "@basemaps/landing": "^6.27.0",
+    "@basemaps/shared": "^6.28.1",
+    "@fastify/static": "^5.0.2",
     "fastify": "^3.27.4",
     "pretty-json-log": "^1.0.0"
   },
-  "gitHead": "1b3a7eb8aad1e0c9bb2ca64e0dcfe13b16e08a50"
+  "gitHead": "f387e0365bcfc7391fd8d0665195b246c93532cf"
 }

package/bin/basemaps-server.js

@@ -1,11 +0,0 @@
-#!/usr/bin/env node
-
-if (process.env.AWS_REGION == null) process.env.AWS_REGION = process.env.AWS_DEFAULT_REGION;
-
-import { BasemapsServerCommand } from '../build/cli.js';
-import Errors from '@oclif/errors/handle.js';
-
-BasemapsServerCommand.run(void 0, import.meta.url).catch((error) => {
-  if (error.oclif) return Errors(error);
-  console.log(error);
-});