@basemaps/cli 6.13.0 → 6.16.1

package/CHANGELOG.md

@@ -3,6 +3,52 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+## [6.16.1](https://github.com/linz/basemaps/compare/v6.16.0...v6.16.1) (2021-11-30)
+
+**Note:** Version bump only for package @basemaps/cli
+
+
+
+
+
+# [6.16.0](https://github.com/linz/basemaps/compare/v6.15.0...v6.16.0) (2021-11-29)
+
+
+### Bug Fixes
+
+* **lambda-tiler:** p-limit is a needed dependency ([#1998](https://github.com/linz/basemaps/issues/1998)) ([dfb1b25](https://github.com/linz/basemaps/commit/dfb1b2575e9b40b96ffa4bdcfa8f1496b18ae25e))
+
+
+
+
+
+# [6.15.0](https://github.com/linz/basemaps/compare/v6.14.2...v6.15.0) (2021-11-28)
+
+
+### Bug Fixes
+
+* **cli:** ensure all files from the job.json are created before logging JobComplete ([#1959](https://github.com/linz/basemaps/issues/1959)) ([78fd35a](https://github.com/linz/basemaps/commit/78fd35ad863c424069d79c8668e9e5b3195ccec9))
+
+
+
+
+
+# [6.14.0](https://github.com/linz/basemaps/compare/v6.13.0...v6.14.0) (2021-10-27)
+
+
+### Bug Fixes
+
+* **cli:** s3 keys should not start with a "/" ([#1935](https://github.com/linz/basemaps/issues/1935)) ([bc9cc22](https://github.com/linz/basemaps/commit/bc9cc224c0c4da6a85c193c8eb3524108d31c17d))
+
+
+### Features
+
+* **cli:** compress text documents with gzip before uploading ([#1936](https://github.com/linz/basemaps/issues/1936)) ([7a6e090](https://github.com/linz/basemaps/commit/7a6e090d0cc26372425b664d6788569e39fd5cca))
+
+
+
+
+
 # [6.13.0](https://github.com/linz/basemaps/compare/v6.12.2...v6.13.0) (2021-10-25)
 
 

package/build/cli/cogify/action.cog.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"action.cog.d.ts","sourceRoot":"","sources":["../../../src/cli/cogify/action.cog.ts"],"names":[],"mappings":"AAAA,OAAO,EAAyC,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAClF,OAAO,EACL,iBAAiB,EAIlB,MAAM,4BAA4B,CAAC;AAIpC,OAAO,EAAE,UAAU,EAAE,MAAM,2BAA2B,CAAC;AAGvD,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAM5C,qBAAa,eAAgB,SAAQ,iBAAiB;IACpD,OAAO,CAAC,GAAG,CAAC,CAA6B;IACzC,OAAO,CAAC,IAAI,CAAC,CAA6B;IAC1C,OAAO,CAAC,MAAM,CAAC,CAA2B;IAC1C,OAAO,CAAC,QAAQ,CAAC,CAA8B;;IAU/C,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI;IAgC7B,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;IAqEhC,2DAA2D;IACrD,cAAc,CAAC,GAAG,EAAE,UAAU,EAAE,MAAM,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAsBrE,SAAS,CAAC,kBAAkB,IAAI,IAAI;CA4BrC"}
+{"version":3,"file":"action.cog.d.ts","sourceRoot":"","sources":["../../../src/cli/cogify/action.cog.ts"],"names":[],"mappings":"AAAA,OAAO,EAAyC,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAClF,OAAO,EACL,iBAAiB,EAIlB,MAAM,4BAA4B,CAAC;AAIpC,OAAO,EAAE,UAAU,EAAE,MAAM,2BAA2B,CAAC;AAGvD,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAM5C,qBAAa,eAAgB,SAAQ,iBAAiB;IACpD,OAAO,CAAC,GAAG,CAAC,CAA6B;IACzC,OAAO,CAAC,IAAI,CAAC,CAA6B;IAC1C,OAAO,CAAC,MAAM,CAAC,CAA2B;IAC1C,OAAO,CAAC,QAAQ,CAAC,CAA8B;;IAU/C,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI;IAgC7B,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;IAqEhC,2DAA2D;IACrD,cAAc,CAAC,GAAG,EAAE,UAAU,EAAE,MAAM,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAoBrE,SAAS,CAAC,kBAAkB,IAAI,IAAI;CA4BrC"}

package/build/cli/cogify/action.cog.js

@@ -107,23 +107,21 @@ export class ActionCogCreate extends CommandLineAction {
     async checkJobStatus(job, logger) {
         const basePath = job.getJobPath();
         const expectedTiffs = new Set();
+        const jobSize = job.output.files.length;
+        for (const file of job.output.files)
+            expectedTiffs.add(`${file.name}.tiff`);
         for await (const fileName of fsa.list(basePath)) {
             const basename = path.basename(fileName);
             // Look for tile tiffs only
             if (!basename.includes('-') || !basename.endsWith('.tiff'))
                 continue;
-            expectedTiffs.add(basename);
-        }
-        const jobSize = job.output.files.length;
-        for (const file of job.output.files) {
-            if (expectedTiffs.has(`${file.name}.tiff`))
-                expectedTiffs.delete(`${file.name}.tiff`);
+            expectedTiffs.delete(basename);
         }
         if (expectedTiffs.size === 0) {
-            logger.info({ tiffCount: jobSize }, 'CogCreate:JobComplete');
+            logger.info({ tiffCount: jobSize, tiffTotal: jobSize }, 'CogCreate:JobComplete');
         }
         else {
-            logger.info({ tiffCount: jobSize, remainingTiff: expectedTiffs.size }, 'CogCreate:JobProgress');
+            logger.info({ tiffCount: jobSize, tiffRemaining: expectedTiffs.size }, 'CogCreate:JobProgress');
         }
     }
     onDefineParameters() {

package/build/cli/util.d.ts

@@ -13,5 +13,5 @@ export declare function getStaticBucket(): Promise<string | null>;
  * @param cacheControl cache control headers @example 'public, max-age=604800, immutable'
  * @returns whether the item was updated
  */
-export declare function uploadStaticFile(path: string, target: string, contentType: string, cacheControl: string): Promise<boolean>;
+export declare function uploadStaticFile(filePath: string, target: string, contentType: string, cacheControl: string): Promise<boolean>;
 //# sourceMappingURL=util.d.ts.map

package/build/cli/util.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"util.d.ts","sourceRoot":"","sources":["../../src/cli/util.ts"],"names":[],"mappings":"AAeA;;GAEG;AACH,wBAAsB,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,UAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CA8BjF;AAED,eAAO,MAAM,OAAO,cAAc,CAAC;AAEnC,wBAAsB,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAQjF;AAOD,wBAAsB,eAAe,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAQ9D;AAED;;;;;;;GAOG;AACH,wBAAsB,gBAAgB,CACpC,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,MAAM,EACd,WAAW,EAAE,MAAM,EACnB,YAAY,EAAE,MAAM,GACnB,OAAO,CAAC,OAAO,CAAC,CAqBlB"}
+{"version":3,"file":"util.d.ts","sourceRoot":"","sources":["../../src/cli/util.ts"],"names":[],"mappings":"AAkBA;;GAEG;AACH,wBAAsB,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,UAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CA8BjF;AAED,eAAO,MAAM,OAAO,cAAc,CAAC;AAEnC,wBAAsB,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAQjF;AAOD,wBAAsB,eAAe,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAQ9D;AAMD;;;;;;;GAOG;AACH,wBAAsB,gBAAgB,CACpC,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,MAAM,EACd,WAAW,EAAE,MAAM,EACnB,YAAY,EAAE,MAAM,GACnB,OAAO,CAAC,OAAO,CAAC,CA6BlB"}

package/build/cli/util.js

@@ -4,6 +4,9 @@ import CloudFront from 'aws-sdk/clients/cloudfront.js';
 import S3 from 'aws-sdk/clients/s3.js';
 import { CliId } from './base.cli.js';
 import crypto from 'crypto';
+import path from 'path';
+import { gzip } from 'zlib';
+import { promisify } from 'util';
 // Cloudfront has to be defined in us-east-1
 const cloudFormation = new CloudFormation({ region: 'us-east-1' });
 const cloudFront = new CloudFront({ region: 'us-east-1' });
@@ -70,6 +73,9 @@ export async function getStaticBucket() {
     staticBucket = (_d = bucket.OutputValue) !== null && _d !== void 0 ? _d : null;
     return staticBucket;
 }
+/** Extensions that should be gzipped before uploading */
+const CompressExt = new Set(['.js', '.html', '.css', '.map', '.json', '.svg']);
+const gzipPromise = promisify(gzip);
 /**
  * Upload a file to the static bucket
  * @param path source file
@@ -78,23 +84,30 @@ export async function getStaticBucket() {
  * @param cacheControl cache control headers @example 'public, max-age=604800, immutable'
  * @returns whether the item was updated
  */
-export async function uploadStaticFile(path, target, contentType, cacheControl) {
-    const fileData = await fsa.read(path);
+export async function uploadStaticFile(filePath, target, contentType, cacheControl) {
+    const fileData = await fsa.read(filePath);
     const hash = crypto.createHash('sha512').update(fileData).digest('base64');
+    // S3 keys should not start with a `/`
+    if (target.startsWith('/'))
+        target = target.slice(1);
     const bucket = await getStaticBucket();
     if (bucket == null)
         throw new Error('Unable to find static bucket');
     const existing = await getHash(bucket, target);
     if (hash === existing)
         return false;
+    // Should we compress the file before uploading
+    const ext = path.extname(target);
+    const contentEncoding = CompressExt.has(ext) ? 'gzip' : undefined;
     await s3
         .putObject({
         Bucket: bucket,
         Key: target,
-        Body: fileData,
+        Body: contentEncoding === 'gzip' ? await gzipPromise(fileData, { level: 9 }) : fileData,
         Metadata: { [HashKey]: hash },
         ContentType: contentType,
         CacheControl: cacheControl,
+        ContentEncoding: contentEncoding,
     })
         .promise();
     return true;

package/build/cog/builder.d.ts

@@ -2,7 +2,7 @@ import { Epsg, TileMatrixSet } from '@basemaps/geo';
 import { LogType } from '@basemaps/shared';
 import { ChunkSource } from '@chunkd/core';
 import { CogTiff } from '@cogeotiff/core';
-import pLimit from 'p-limit';
+import { LimitFunction } from 'p-limit';
 import { Cutline } from './cutline.js';
 import { CogBuilderMetadata, SourceMetadata } from './types.js';
 export declare const InvalidProjectionCode = 32767;
@@ -20,7 +20,7 @@ export declare const CacheFolder = "./.cache";
  */
 export declare function guessProjection(wkt: string | null): Epsg | null;
 export declare class CogBuilder {
-    q: pLimit.Limit;
+    q: LimitFunction;
     logger: LogType;
     targetTms: TileMatrixSet;
     srcProj?: Epsg;

package/build/cog/builder.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"builder.d.ts","sourceRoot":"","sources":["../../src/cog/builder.ts"],"names":[],"mappings":"AAAA,OAAO,EAAU,IAAI,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC5D,OAAO,EAAyC,OAAO,EAAc,MAAM,kBAAkB,CAAC;AAC9F,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,EAAE,OAAO,EAAuB,MAAM,iBAAiB,CAAC;AAG/D,OAAO,MAAM,MAAM,SAAS,CAAC;AAG7B,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EAAE,kBAAkB,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAEhE,eAAO,MAAM,qBAAqB,QAAQ,CAAC;AAC3C,eAAO,MAAM,YAAY,OAAO,CAAC;AACjC,eAAO,MAAM,WAAW,aAAa,CAAC;AAEtC;;;;;;;;;GASG;AACH,wBAAgB,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,CAO/D;AAED,qBAAa,UAAU;IACrB,CAAC,EAAE,MAAM,CAAC,KAAK,CAAC;IAChB,MAAM,EAAE,OAAO,CAAC;IAChB,SAAS,EAAE,aAAa,CAAC;IACzB,OAAO,CAAC,EAAE,IAAI,CAAC;IAGf,kBAAkB,cAAqB;IAEvC;;OAEG;gBACS,SAAS,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,IAAI;IAO1F;;;OAGG;IACG,MAAM,CAAC,OAAO,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC,cAAc,CAAC;IAoF7D,cAAc,CAAC,IAAI,EAAE,OAAO,GAAG,IAAI;IAuBnC;;;;OAIG;IACH,UAAU,CAAC,IAAI,EAAE,OAAO,GAAG,MAAM,GAAG,IAAI;IAaxC,wFAAwF;YAC1E,WAAW;IA2BzB;;;;OAIG;IACG,KAAK,CAAC,KAAK,EAAE,WAAW,EAAE,EAAE,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,kBAAkB,CAAC;CAWjF"}
+{"version":3,"file":"builder.d.ts","sourceRoot":"","sources":["../../src/cog/builder.ts"],"names":[],"mappings":"AAAA,OAAO,EAAU,IAAI,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC5D,OAAO,EAAyC,OAAO,EAAc,MAAM,kBAAkB,CAAC;AAC9F,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,EAAE,OAAO,EAAuB,MAAM,iBAAiB,CAAC;AAG/D,OAAe,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAGhD,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EAAE,kBAAkB,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAEhE,eAAO,MAAM,qBAAqB,QAAQ,CAAC;AAC3C,eAAO,MAAM,YAAY,OAAO,CAAC;AACjC,eAAO,MAAM,WAAW,aAAa,CAAC;AAEtC;;;;;;;;;GASG;AACH,wBAAgB,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,CAO/D;AAED,qBAAa,UAAU;IACrB,CAAC,EAAE,aAAa,CAAC;IACjB,MAAM,EAAE,OAAO,CAAC;IAChB,SAAS,EAAE,aAAa,CAAC;IACzB,OAAO,CAAC,EAAE,IAAI,CAAC;IAGf,kBAAkB,cAAqB;IAEvC;;OAEG;gBACS,SAAS,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,IAAI;IAO1F;;;OAGG;IACG,MAAM,CAAC,OAAO,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC,cAAc,CAAC;IAoF7D,cAAc,CAAC,IAAI,EAAE,OAAO,GAAG,IAAI;IAuBnC;;;;OAIG;IACH,UAAU,CAAC,IAAI,EAAE,OAAO,GAAG,MAAM,GAAG,IAAI;IAaxC,wFAAwF;YAC1E,WAAW;IA2BzB;;;;OAIG;IACG,KAAK,CAAC,KAAK,EAAE,WAAW,EAAE,EAAE,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,kBAAkB,CAAC;CAWjF"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@basemaps/cli",
-  "version": "6.13.0",
+  "version": "6.16.1",
   "private": false,
   "repository": {
     "type": "git",
@@ -40,13 +40,13 @@
   "dependencies": {
     "@basemaps/config": "^6.10.0",
     "@basemaps/geo": "^6.10.0",
-    "@basemaps/shared": "^6.12.0",
+    "@basemaps/shared": "^6.16.1",
     "@chunkd/fs": "^7.3.1",
     "@cogeotiff/core": "^6.0.2",
     "@linzjs/geojson": "^6.10.0",
     "@rushstack/ts-command-line": "^4.3.13",
     "ansi-colors": "^4.1.1",
-    "p-limit": "^3.0.1",
+    "p-limit": "^4.0.0",
     "pretty-json-log": "^1.0.0",
     "zod": "^3.2.0"
   },
@@ -60,5 +60,5 @@
   "files": [
     "build/"
   ],
-  "gitHead": "212aa596ad3f545020691b33b0afa0594d95e88f"
+  "gitHead": "5859ec22302eac1126b887d6399656272b3611cb"
 }