@basemaps/cli 6.28.1 → 6.29.0

package/CHANGELOG.md

@@ -3,6 +3,27 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+# [6.29.0](https://github.com/linz/basemaps/compare/v6.28.1...v6.29.0) (2022-06-27)
+
+
+### Bug Fixes
+
+* **cli:** ensure cli can run inside of docker ([#2273](https://github.com/linz/basemaps/issues/2273)) ([8184167](https://github.com/linz/basemaps/commit/81841674efba2f86d9a39d01af62fccb1fe6f70f))
+* **cli:** ensure errors are thrown ([#2248](https://github.com/linz/basemaps/issues/2248)) ([c0923fe](https://github.com/linz/basemaps/commit/c0923fe137ce36c610c6e13332292d5c7f573c16))
+* **cli:** give each internal job a fresh copy of all data ([#2250](https://github.com/linz/basemaps/issues/2250)) ([52d59de](https://github.com/linz/basemaps/commit/52d59def25358e5a0f1fae5506edee6d8b17dbad))
+* **cli:** look into batch to get exact list of tiffs being processed ([#2249](https://github.com/linz/basemaps/issues/2249)) ([69b722e](https://github.com/linz/basemaps/commit/69b722ea3190488231baf3b7023ce83e60e432c1))
+* **config:** include missing zod dependency ([#2245](https://github.com/linz/basemaps/issues/2245)) ([94914dc](https://github.com/linz/basemaps/commit/94914dc728b60c51d6382c4460a1bd4e233f00c5))
+
+
+### Features
+
+* **cli:** merge all cli from config-cli package and server package into cli package. ([#2260](https://github.com/linz/basemaps/issues/2260)) ([73afd61](https://github.com/linz/basemaps/commit/73afd610740f69e67504a0610c94a407a6ba4091))
+* **config:** create a hash of config bundles and use bundle created timestamp for records ([#2274](https://github.com/linz/basemaps/issues/2274)) ([bd9c7bb](https://github.com/linz/basemaps/commit/bd9c7bbf3f651417b60ba6ad2ca655f89f1f5cd9))
+
+
+
+
+
 ## [6.28.1](https://github.com/linz/basemaps/compare/v6.28.0...v6.28.1) (2022-06-07)
 
 **Note:** Version bump only for package @basemaps/cli

package/README.md

@@ -1,35 +1,69 @@
-# @basemaps/cog
+# @basemaps/cli
 
-Create a collection of cloud optimized geotiff's from a collection of geotiff, that is optimized to be used in `@basemaps/tiler`
+CreateThis package is to control the configuration in the LINZ basemaps product.
 
 ## Install
 
 This script requires docker to be installed
 
-To install cogify
+To install
 
 ```bash
-npm i @basemaps/cog
+npm i @basemaps/cli
 ```
 
-## Usage
+## Usage -- Bundle
 
-Create a list of COG's to create
+Bundle config files into config bundle json from a given config path.
 
 ```bash
-cogify -V job --source ./source_folder/ --output ./source_folder/cogify/
+./bin/bmc.js bundle --config config/ --output config.json
 ```
 
-Build a specific COG
+## Usage -- Import
+
+Import all configs from a bundled config.json into dynamo db from a given config path
 
 ```bash
-cogify -V cog --job ./cogs/01DYREBEEFFXEPBAYBED2TMAFJ/job.json --name 1-2-3 --commit
+./bin/bmc.js import --config config.json --commit
 ```
 
-Build all the COGs using aws BATCH
+## Usage -- Screenshots
+
+Dump the screenshots from basemaps production
 
 ```bash
-cogify -V batch --job ./cogs/01DYREBEEFFXEPBAYBED2TMAFJ/job.json --commit
+./bin/bmc.js screenshot
+```
+
+Dump the screenshots from different host
+
+```bash
+./bin/bmc.js screenshot --host HOST
+
+```
+
+Dump the screenshots with config file
+
+```bash
+./bin/bmc.js screenshot --config s3://..../config.json.gz
+
+```
+
+## Usage -- cogify
+
+Create a collection of cloud optimized geotiff's from a collection of geotiff, that is optimized to be used in `@basemaps/tiler`
+
+Create a list of COG's to create
+
+```bash
+./bin/bmc.js -V job --source ./source_folder/ --output ./source_folder/cogify/
+```
+
+Build a specific COG
+
+```bash
+./bin/bmc.js -V cog --job ./cogs/01DYREBEEFFXEPBAYBED2TMAFJ/job.json --name 1-2-3 --commit
 ```
 
 ## Advanced Job creation
@@ -39,7 +73,7 @@ Useful configuration options for `cogify job`
 ### Min Tile zoom `--min-zoom :number`
 
 using the argument `--min-zoom` will configure the highest COG tile that can be created
-for example, `--min-zoom 10` means tiles of zoom `0 - 10`  can be created but `11` cannot.
+for example, `--min-zoom 10` means tiles of zoom `0 - 10` can be created but `11` cannot.
 
 This is useful to control the size of COGS across varying imagery sets World level z0-5 vs region z12-15 vs specific area 20+
 
@@ -59,7 +93,6 @@ Outputs two GeoJSON files to provide a representation of the source and target i
 
 The best way to build a cog collection is to create multiple JOBS with `--geojson` enabled while adjusting `--max-cogs` and `--min-zoom` when a good covering is found the job can then be submitted to AWS for processing.
 
-
 Using a large area of interest as shown in the imagery set below and is showing three different sets of parameters, The shaded green area is the original source imagery
 
 ![Example Covering](./static/example-covering.png)

package/build/cli/bin.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=bin.d.ts.map

package/build/cli/bin.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"bin.d.ts","sourceRoot":"","sources":["../../src/cli/bin.ts"],"names":[],"mappings":""}

package/build/cli/bin.js

@@ -0,0 +1,3 @@
+Error.stackTraceLimit = 100;
+import { BasemapsConfigCommandLine } from './index.js';
+new BasemapsConfigCommandLine().run();

package/build/cli/cogify/__tests__/batch.job.test.js

@@ -1,3 +1,4 @@
+import { LogConfig } from '@basemaps/shared';
 import o from 'ospec';
 import { BatchJob, extractResolutionFromName } from '../batch.job.js';
 o.spec('action.batch', () => {
@@ -12,9 +13,10 @@ o.spec('action.batch', () => {
     });
     o('should create valid jobNames', () => {
         const fakeJob = { id: '01FHRPYJ5FV1XAARZAC4T4K6MC', name: 'geographx_nz_texture_shade_2012_8-0m' };
-        o(BatchJob.id(fakeJob, ['0'])).equals('01FHRPYJ5FV1XAARZAC4T4K6MC-9af5e139bbb3e502-0');
+        o(BatchJob.id(fakeJob, ['0'])).equals('01FHRPYJ5FV1XAARZAC4T4K6MC-9af5e139bbb3e502-1x-0');
         fakeJob.name = 'ōtorohanga_urban_2021_0.1m_RGB';
-        o(BatchJob.id(fakeJob, ['0'])).equals('01FHRPYJ5FV1XAARZAC4T4K6MC-5294acface81c107-0');
+        o(BatchJob.id(fakeJob, ['0'])).equals('01FHRPYJ5FV1XAARZAC4T4K6MC-5294acface81c107-1x-0');
+        o(BatchJob.id(fakeJob)).equals('01FHRPYJ5FV1XAARZAC4T4K6MC-5294acface81c107-');
     });
     o('should truncate job names to 128 characters', () => {
         const fakeJob = { id: '01FHRPYJ5FV1XAARZAC4T4K6MC', name: 'geographx_nz_texture_shade_2012_8-0m' };
@@ -22,9 +24,9 @@ o.spec('action.batch', () => {
             'this is a really long file name',
             'it should over flow 128 characters',
             'so it should be truncated at some point.tiff',
-        ])).equals('01FHRPYJ5FV1XAARZAC4T4K6MC-9af5e139bbb3e502-it should over flow 128 characters_so it should be truncated at some point.tiff_this');
+        ])).equals('01FHRPYJ5FV1XAARZAC4T4K6MC-9af5e139bbb3e502-3x-it should over flow 128 characters_so it should be truncated at some point.tiff_t');
     });
-    o('should prepare valid chunk jobs', async () => {
+    o.spec('ChunkJobs', () => {
         const fakeGsd = 0.9;
         const ChunkJobSmall = 4097;
         const ChunkJobMiddle = 8193;
@@ -47,23 +49,47 @@ o.spec('action.batch', () => {
             { name: '1-2-14', width: ChunkJobMiddle * fakeGsd - 1 }, // Middle Job 70
         ];
         const fakeJob = { id: '01FHRPYJ5FV1XAARZAC4T4K6MC', output: { files: fakeFiles, gsd: fakeGsd } };
-        o(await BatchJob.getJobs(fakeJob)).deepEquals([
-            [fakeFiles[2].name],
-            [fakeFiles[5].name],
-            [
-                fakeFiles[0].name,
-                fakeFiles[1].name,
-                fakeFiles[3].name,
-                fakeFiles[4].name,
-                fakeFiles[6].name,
-                fakeFiles[7].name,
-                fakeFiles[8].name,
-                fakeFiles[9].name,
-                fakeFiles[10].name,
-                fakeFiles[11].name,
-            ],
-            [fakeFiles[13].name],
-            [fakeFiles[12].name, fakeFiles[14].name], // Second Chunk
-        ]);
+        o('should prepare valid chunk jobs', async () => {
+            o(BatchJob.getJobs(fakeJob, new Set(), LogConfig.get())).deepEquals([
+                [fakeFiles[2].name],
+                [fakeFiles[5].name],
+                [
+                    fakeFiles[0].name,
+                    fakeFiles[1].name,
+                    fakeFiles[3].name,
+                    fakeFiles[4].name,
+                    fakeFiles[6].name,
+                    fakeFiles[7].name,
+                    fakeFiles[8].name,
+                    fakeFiles[9].name,
+                    fakeFiles[10].name,
+                    fakeFiles[11].name,
+                ],
+                [fakeFiles[13].name],
+                [fakeFiles[12].name, fakeFiles[14].name], // Second Chunk
+            ]);
+        });
+        o('should skip tiffs that exist', () => {
+            const existing = new Set([
+                `${fakeFiles[0].name}.tiff`,
+                `${fakeFiles[1].name}.tiff`,
+                `${fakeFiles[13].name}.tiff`,
+            ]);
+            o(BatchJob.getJobs(fakeJob, existing, LogConfig.get())).deepEquals([
+                [fakeFiles[2].name],
+                [fakeFiles[5].name],
+                [
+                    fakeFiles[3].name,
+                    fakeFiles[4].name,
+                    fakeFiles[6].name,
+                    fakeFiles[7].name,
+                    fakeFiles[8].name,
+                    fakeFiles[9].name,
+                    fakeFiles[10].name,
+                    fakeFiles[11].name,
+                ],
+                [fakeFiles[12].name, fakeFiles[14].name], // Second Chunk
+            ]);
+        });
     });
 });

package/build/cli/cogify/action.cog.d.ts

@@ -2,7 +2,7 @@ import { LogType } from '@basemaps/shared';
 import { CommandLineAction } from '@rushstack/ts-command-line';
 import { CogStacJob } from '../../cog/cog.stac.job.js';
 import { CogJob } from '../../cog/types.js';
-export declare class ActionCogCreate extends CommandLineAction {
+export declare class CommandCogCreate extends CommandLineAction {
     private job?;
     private name?;
     private commit?;
@@ -10,6 +10,7 @@ export declare class ActionCogCreate extends CommandLineAction {
     constructor();
     getNames(job: CogJob): Set<string> | null;
     onExecute(): Promise<void>;
+    processTiff(job: CogStacJob, tiffName: string, tmpFolder: string, isCommit: boolean, logger: LogType): Promise<void>;
     /** Check to see how many tiffs are remaining in the job */
     checkJobStatus(job: CogStacJob, logger: LogType): Promise<void>;
     protected onDefineParameters(): void;

package/build/cli/cogify/action.cog.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"action.cog.d.ts","sourceRoot":"","sources":["../../../src/cli/cogify/action.cog.ts"],"names":[],"mappings":"AAAA,OAAO,EAAiD,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAE1F,OAAO,EACL,iBAAiB,EAKlB,MAAM,4BAA4B,CAAC;AAIpC,OAAO,EAAE,UAAU,EAAE,MAAM,2BAA2B,CAAC;AAGvD,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAQ5C,qBAAa,eAAgB,SAAQ,iBAAiB;IACpD,OAAO,CAAC,GAAG,CAAC,CAA6B;IACzC,OAAO,CAAC,IAAI,CAAC,CAAiC;IAC9C,OAAO,CAAC,MAAM,CAAC,CAA2B;IAC1C,OAAO,CAAC,QAAQ,CAAC,CAAkC;;IAUnD,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI;IA0CnC,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;IAqFhC,2DAA2D;IACrD,cAAc,CAAC,GAAG,EAAE,UAAU,EAAE,MAAM,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAqCrE,SAAS,CAAC,kBAAkB,IAAI,IAAI;CA4BrC"}
+{"version":3,"file":"action.cog.d.ts","sourceRoot":"","sources":["../../../src/cli/cogify/action.cog.ts"],"names":[],"mappings":"AAAA,OAAO,EAAiD,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAE1F,OAAO,EACL,iBAAiB,EAKlB,MAAM,4BAA4B,CAAC;AAIpC,OAAO,EAAE,UAAU,EAAE,MAAM,2BAA2B,CAAC;AAGvD,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAQ5C,qBAAa,gBAAiB,SAAQ,iBAAiB;IACrD,OAAO,CAAC,GAAG,CAAC,CAA6B;IACzC,OAAO,CAAC,IAAI,CAAC,CAAiC;IAC9C,OAAO,CAAC,MAAM,CAAC,CAA2B;IAC1C,OAAO,CAAC,QAAQ,CAAC,CAAkC;;IAUnD,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI;IA0CnC,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;IAkD1B,WAAW,CACf,GAAG,EAAE,UAAU,EACf,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,OAAO,EACjB,MAAM,EAAE,OAAO,GACd,OAAO,CAAC,IAAI,CAAC;IA2ChB,2DAA2D;IACrD,cAAc,CAAC,GAAG,EAAE,UAAU,EAAE,MAAM,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAqCrE,SAAS,CAAC,kBAAkB,IAAI,IAAI;CA4BrC"}

package/build/cli/cogify/action.cog.js

@@ -12,7 +12,7 @@ import path from 'path';
 import { insertConfigImagery, insertConfigTileSet } from './imagery.config.js';
 import { JobStatus } from '@basemaps/config';
 import { prepareUrl } from '../util.js';
-export class ActionCogCreate extends CommandLineAction {
+export class CommandCogCreate extends CommandLineAction {
     constructor() {
         super({
             actionName: 'cog',
@@ -55,12 +55,12 @@ export class ActionCogCreate extends CommandLineAction {
         return output;
     }
     async onExecute() {
-        var _a, _b, _c, _d;
-        const jobFn = (_a = this.job) === null || _a === void 0 ? void 0 : _a.value;
-        if (jobFn == null)
+        var _a, _b, _c;
+        const jobLocation = (_a = this.job) === null || _a === void 0 ? void 0 : _a.value;
+        if (jobLocation == null)
             throw new Error('Missing job name');
-        const job = await CogStacJob.load(jobFn);
         const isCommit = (_c = (_b = this.commit) === null || _b === void 0 ? void 0 : _b.value) !== null && _c !== void 0 ? _c : false;
+        const job = await CogStacJob.load(jobLocation);
         const logger = LogConfig.get().child({
             correlationId: job.id,
             imageryName: job.name,
@@ -69,49 +69,15 @@ export class ActionCogCreate extends CommandLineAction {
         LogConfig.set(logger);
         logger.info('CogCreate:Start');
         const gdalVersion = await Gdal.version(logger);
-        logger.info({ version: gdalVersion }, 'CogCreate:GdalVersion');
+        logger.info({ gdalVersion }, 'CogCreate:GdalVersion');
         const names = this.getNames(job);
         if (names == null || names.size === 0)
             return;
         const tmpFolder = await makeTempFolder(`basemaps-${job.id}-${CliId}`);
         try {
             for (const name of names) {
-                const tiffFolder = await makeTiffFolder(tmpFolder, name);
-                const targetPath = job.getJobPath(`${name}.tiff`);
-                fsa.configure(job.output.location);
-                const outputExists = await fsa.exists(targetPath);
-                logger.info({ targetPath, outputExists }, 'CogCreate:CheckExists');
-                // Output file exists don't try and overwrite it
-                if (outputExists) {
-                    logger.warn({ targetPath }, 'CogCreate:OutputExists');
-                    await this.checkJobStatus(job, logger);
-                    continue;
-                }
-                let cutlineJson;
-                if (job.output.cutline != null) {
-                    const cutlinePath = job.getJobPath('cutline.geojson.gz');
-                    logger.info({ path: cutlinePath }, 'CogCreate:UsingCutLine');
-                    cutlineJson = await Cutline.loadCutline(cutlinePath);
-                }
-                else {
-                    logger.warn('NoCutLine');
-                }
-                const cutline = new Cutline(job.tileMatrix, cutlineJson, (_d = job.output.cutline) === null || _d === void 0 ? void 0 : _d.blend, job.output.oneCogCovering);
-                const tmpVrtPath = await CogVrt.buildVrt(tiffFolder, job, cutline, name, logger);
-                if (tmpVrtPath == null) {
-                    logger.warn({ name }, 'CogCreate:NoMatchingSourceImagery');
-                    return;
-                }
-                const tmpTiff = fsa.join(tiffFolder, `${name}.tiff`);
-                await buildCogForName(job, name, tmpVrtPath, tmpTiff, logger, isCommit);
-                logger.info({ target: targetPath }, 'CogCreate:StoreTiff');
-                if (isCommit) {
-                    await fsa.write(targetPath, createReadStream(tmpTiff));
-                    await this.checkJobStatus(job, logger);
-                }
-                else {
-                    logger.warn({ name }, 'DryRun:Done');
-                }
+                const tiffJob = await CogStacJob.load(jobLocation);
+                await this.processTiff(tiffJob, name, tmpFolder, isCommit, logger.child({ tiffName: name }));
             }
         }
         catch (e) {
@@ -129,12 +95,53 @@ export class ActionCogCreate extends CommandLineAction {
                 else
                     throw new Error('Unable update the Processing Job status:' + jobFailed.id);
             }
+            // Ensure the error is thrown
+            throw e;
         }
         finally {
             // Cleanup!
             await fs.rm(tmpFolder, { recursive: true });
         }
     }
+    async processTiff(job, tiffName, tmpFolder, isCommit, logger) {
+        var _a;
+        const tiffFolder = await makeTiffFolder(tmpFolder, tiffName);
+        const targetPath = job.getJobPath(`${tiffName}.tiff`);
+        fsa.configure(job.output.location);
+        const outputExists = await fsa.exists(targetPath);
+        logger.info({ targetPath, outputExists }, 'CogCreate:CheckExists');
+        // Output file exists don't try and overwrite it
+        if (outputExists) {
+            logger.warn({ targetPath }, 'CogCreate:OutputExists');
+            await this.checkJobStatus(job, logger);
+            return;
+        }
+        let cutlineJson;
+        if (job.output.cutline != null) {
+            const cutlinePath = job.getJobPath('cutline.geojson.gz');
+            logger.info({ path: cutlinePath }, 'CogCreate:UsingCutLine');
+            cutlineJson = await Cutline.loadCutline(cutlinePath);
+        }
+        else {
+            logger.warn('CutLine:Skip');
+        }
+        const cutline = new Cutline(job.tileMatrix, cutlineJson, (_a = job.output.cutline) === null || _a === void 0 ? void 0 : _a.blend, job.output.oneCogCovering);
+        const tmpVrtPath = await CogVrt.buildVrt(tiffFolder, job, cutline, tiffName, logger);
+        if (tmpVrtPath == null) {
+            logger.warn('CogCreate:NoMatchingSourceImagery');
+            return;
+        }
+        const tmpTiff = fsa.join(tiffFolder, `${tiffName}.tiff`);
+        await buildCogForName(job, tiffName, tmpVrtPath, tmpTiff, logger, isCommit);
+        logger.info({ target: targetPath }, 'CogCreate:StoreTiff');
+        if (isCommit) {
+            await fsa.write(targetPath, createReadStream(tmpTiff));
+            await this.checkJobStatus(job, logger);
+        }
+        else {
+            logger.warn('DryRun:Done');
+        }
+    }
     /** Check to see how many tiffs are remaining in the job */
     async checkJobStatus(job, logger) {
         const basePath = job.getJobPath();

package/build/cli/cogify/action.job.d.ts

@@ -6,7 +6,7 @@ export declare class CLiInputData {
     externalId: CommandLineStringParameter;
     constructor(parent: CommandLineAction, prefix: string);
 }
-export declare class ActionJobCreate extends CommandLineAction {
+export declare class CommandJobCreate extends CommandLineAction {
     private source;
     private output;
     private maxConcurrency;

package/build/cli/cogify/action.job.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"action.job.d.ts","sourceRoot":"","sources":["../../../src/cli/cogify/action.job.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAuB,MAAM,kBAAkB,CAAC;AACnE,OAAO,EACL,iBAAiB,EAGjB,0BAA0B,EAC3B,MAAM,4BAA4B,CAAC;AAMpC,qBAAa,YAAY;IACvB,IAAI,EAAE,0BAA0B,CAAC;IACjC,OAAO,EAAE,0BAA0B,CAAC;IACpC,UAAU,EAAE,0BAA0B,CAAC;gBAE3B,MAAM,EAAE,iBAAiB,EAAE,MAAM,EAAE,MAAM;CAsBtD;AAED,qBAAa,eAAgB,SAAQ,iBAAiB;IACpD,OAAO,CAAC,MAAM,CAAe;IAC7B,OAAO,CAAC,MAAM,CAAe;IAC7B,OAAO,CAAC,cAAc,CAA8B;IACpD,OAAO,CAAC,OAAO,CAA6B;IAC5C,OAAO,CAAC,YAAY,CAA8B;IAClD,OAAO,CAAC,UAAU,CAA6B;IAC/C,OAAO,CAAC,mBAAmB,CAA6B;IACxD,OAAO,CAAC,WAAW,CAA2B;IAC9C,OAAO,CAAC,OAAO,CAA8B;IAC7C,OAAO,CAAC,gBAAgB,CAA8B;IACtD,OAAO,CAAC,UAAU,CAA6B;IAC/C,OAAO,CAAC,MAAM,CAA2B;IACzC,OAAO,CAAC,QAAQ,CAA6B;IAC7C,OAAO,CAAC,YAAY,CAA6B;;IAUjD,QAAQ,CAAC,MAAM,EAAE,YAAY,GAAG,UAAU;IAkBpC,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;IA0DhC,SAAS,CAAC,kBAAkB,IAAI,IAAI;CAyFrC"}
+{"version":3,"file":"action.job.d.ts","sourceRoot":"","sources":["../../../src/cli/cogify/action.job.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAuB,MAAM,kBAAkB,CAAC;AACnE,OAAO,EACL,iBAAiB,EAGjB,0BAA0B,EAC3B,MAAM,4BAA4B,CAAC;AAMpC,qBAAa,YAAY;IACvB,IAAI,EAAE,0BAA0B,CAAC;IACjC,OAAO,EAAE,0BAA0B,CAAC;IACpC,UAAU,EAAE,0BAA0B,CAAC;gBAE3B,MAAM,EAAE,iBAAiB,EAAE,MAAM,EAAE,MAAM;CAsBtD;AAED,qBAAa,gBAAiB,SAAQ,iBAAiB;IACrD,OAAO,CAAC,MAAM,CAAe;IAC7B,OAAO,CAAC,MAAM,CAAe;IAC7B,OAAO,CAAC,cAAc,CAA8B;IACpD,OAAO,CAAC,OAAO,CAA6B;IAC5C,OAAO,CAAC,YAAY,CAA8B;IAClD,OAAO,CAAC,UAAU,CAA6B;IAC/C,OAAO,CAAC,mBAAmB,CAA6B;IACxD,OAAO,CAAC,WAAW,CAA2B;IAC9C,OAAO,CAAC,OAAO,CAA8B;IAC7C,OAAO,CAAC,gBAAgB,CAA8B;IACtD,OAAO,CAAC,UAAU,CAA6B;IAC/C,OAAO,CAAC,MAAM,CAA2B;IACzC,OAAO,CAAC,QAAQ,CAA6B;IAC7C,OAAO,CAAC,YAAY,CAA6B;;IAUjD,QAAQ,CAAC,MAAM,EAAE,YAAY,GAAG,UAAU;IAkBpC,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;IA0DhC,SAAS,CAAC,kBAAkB,IAAI,IAAI;CAyFrC"}

package/build/cli/cogify/action.job.js

@@ -26,7 +26,7 @@ export class CLiInputData {
         });
     }
 }
-export class ActionJobCreate extends CommandLineAction {
+export class CommandJobCreate extends CommandLineAction {
     constructor() {
         super({
             actionName: 'job',

package/build/cli/cogify/batch.job.d.ts

@@ -9,6 +9,8 @@ import { CogJob } from '../../cog/types.js';
  */
 export declare function extractResolutionFromName(name: string): number;
 export declare class BatchJob {
+    static _batch: Batch;
+    static get batch(): Batch;
     /**
      * Create a id for a job
      *
@@ -17,8 +19,8 @@ export declare class BatchJob {
      * @param fileNames output filename
      * @returns job id
      */
-    static id(job: CogJob, fileNames: string[]): string;
-    static batchOne(jobPath: string, job: CogJob, batch: Batch, names: string[], isCommit: boolean): Promise<{
+    static id(job: CogJob, fileNames?: string[]): string;
+    static batchOne(jobPath: string, job: CogJob, names: string[], isCommit: boolean): Promise<{
         jobName: string;
         jobId: string;
         memory: number;
@@ -27,12 +29,12 @@ export declare class BatchJob {
      * List all the current jobs in batch and their statuses
      * @returns a map of JobName to if their status is "ok" (not failed)
      */
-    static getCurrentJobList(batch: Batch): Promise<Map<string, boolean>>;
+    static getCurrentJobList(job: CogJob, logger: LogType): Promise<Set<string>>;
     static batchJob(job: CogJob, commit: boolean | undefined, logger: LogType): Promise<void>;
     /**
      * Prepare the jobs from job files, and chunk the small images into single
      * @returns List of jobs including single job and chunk jobs.
      */
-    static getJobs(job: CogJob): Promise<string[][]>;
+    static getJobs(job: CogJob, existing: Set<string>, log: LogType): string[][];
 }
 //# sourceMappingURL=batch.job.d.ts.map

package/build/cli/cogify/batch.job.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"batch.job.d.ts","sourceRoot":"","sources":["../../../src/cli/cogify/batch.job.ts"],"names":[],"mappings":"AACA,OAAO,EAAuB,OAAO,EAAc,MAAM,kBAAkB,CAAC;AAC5E,OAAO,KAAK,MAAM,0BAA0B,CAAC;AAE7C,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAa5C;;;;;GAKG;AACH,wBAAgB,yBAAyB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAI9D;AAED,qBAAa,QAAQ;IACnB;;;;;;;OAOG;IACH,MAAM,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,GAAG,MAAM;WAOtC,QAAQ,CACnB,OAAO,EAAE,MAAM,EACf,GAAG,EAAE,MAAM,EACX,KAAK,EAAE,KAAK,EACZ,KAAK,EAAE,MAAM,EAAE,EACf,QAAQ,EAAE,OAAO,GAChB,OAAO,CAAC;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAmC9D;;;OAGG;WACU,iBAAiB,CAAC,KAAK,EAAE,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;WAqB9D,QAAQ,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,qBAAQ,EAAE,MAAM,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAyElF;;;OAGG;WACU,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;CA4BvD"}
+{"version":3,"file":"batch.job.d.ts","sourceRoot":"","sources":["../../../src/cli/cogify/batch.job.ts"],"names":[],"mappings":"AACA,OAAO,EAAuB,OAAO,EAAc,MAAM,kBAAkB,CAAC;AAC5E,OAAO,KAAK,MAAM,0BAA0B,CAAC;AAG7C,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAa5C;;;;;GAKG;AACH,wBAAgB,yBAAyB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAI9D;AAED,qBAAa,QAAQ;IACnB,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC;IACrB,MAAM,KAAK,KAAK,IAAI,KAAK,CAKxB;IACD;;;;;;;OAOG;IACH,MAAM,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM;WAQvC,QAAQ,CACnB,OAAO,EAAE,MAAM,EACf,GAAG,EAAE,MAAM,EACX,KAAK,EAAE,MAAM,EAAE,EACf,QAAQ,EAAE,OAAO,GAChB,OAAO,CAAC;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAmC9D;;;OAGG;WACU,iBAAiB,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;WA0CrE,QAAQ,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,qBAAQ,EAAE,MAAM,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IA+ClF;;;OAGG;IACH,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,OAAO,GAAG,MAAM,EAAE,EAAE;CAkC7E"}

package/build/cli/cogify/batch.job.js

@@ -2,6 +2,7 @@ import { TileMatrixSet } from '@basemaps/geo';
 import { Env, fsa, LogConfig, Projection } from '@basemaps/shared';
 import Batch from 'aws-sdk/clients/batch.js';
 import { createHash } from 'crypto';
+import { basename } from 'path';
 const JobQueue = 'CogBatchJobQueue';
 const JobDefinition = 'CogBatchJob';
 const ChunkJobMax = 1000;
@@ -24,6 +25,14 @@ export function extractResolutionFromName(name) {
     return parseFloat(matches[1].replace('-', '.')) * 1000;
 }
 export class BatchJob {
+    static get batch() {
+        var _a, _b;
+        if (this._batch)
+            return this._batch;
+        const region = (_b = (_a = Env.get('AWS_REGION')) !== null && _a !== void 0 ? _a : Env.get('AWS_DEFAULT_REGION')) !== null && _b !== void 0 ? _b : 'ap-southeast-2';
+        this._batch = new Batch({ region });
+        return this._batch;
+    }
     /**
      * Create a id for a job
      *
@@ -35,10 +44,12 @@ export class BatchJob {
     static id(job, fileNames) {
         // Job names are uncontrolled so hash the name and grab a small slice to use as a identifier
         const jobName = createHash('sha256').update(job.name).digest('hex').slice(0, 16);
+        if (fileNames == null)
+            return `${job.id}-${jobName}-`;
         fileNames.sort((a, b) => a.localeCompare(b));
-        return `${job.id}-${jobName}-${fileNames.join('_')}`.slice(0, 128);
+        return `${job.id}-${jobName}-${fileNames.length}x-${fileNames.join('_')}`.slice(0, 128);
     }
-    static async batchOne(jobPath, job, batch, names, isCommit) {
+    static async batchOne(jobPath, job, names, isCommit) {
         const jobName = BatchJob.id(job, names);
         let memory = 3900;
         if (names.length === 1) {
@@ -55,7 +66,7 @@ export class BatchJob {
         let commandStr = ['-V', 'cog', '--job', jobPath, '--commit'];
         for (const name of names)
             commandStr = commandStr.concat(['--name', name]);
-        const batchJob = await batch
+        const batchJob = await this.batch
             .submitJob({
             jobName,
             jobQueue: JobQueue,
@@ -73,77 +84,75 @@ export class BatchJob {
      * List all the current jobs in batch and their statuses
      * @returns a map of JobName to if their status is "ok" (not failed)
      */
-    static async getCurrentJobList(batch) {
+    static async getCurrentJobList(job, logger) {
+        var _a;
+        const jobPrefix = BatchJob.id(job);
         // For some reason AWS only lets us query one status at a time.
-        const allStatuses = ['SUBMITTED', 'PENDING', 'RUNNABLE', 'STARTING', 'RUNNING', 'SUCCEEDED', 'FAILED'];
-        const allJobs = await Promise.all(allStatuses.map((jobStatus) => batch.listJobs({ jobQueue: JobQueue, jobStatus }).promise()));
-        const okMap = new Map();
+        const allStatuses = ['SUBMITTED', 'PENDING', 'RUNNABLE', 'STARTING', 'RUNNING' /* 'SUCCEEDED' */];
+        // Succeeded is not needed as we check to see if the output file exists, if it succeeds and the output file doesn't exist then something has gone wrong
+        const allJobs = await Promise.all(allStatuses.map((jobStatus) => this.batch.listJobs({ jobQueue: JobQueue, jobStatus }).promise()));
+        const jobIds = new Set();
+        // Find all the relevant jobs that start with our job prefix
         for (const status of allJobs) {
             for (const job of status.jobSummaryList) {
-                if (job.status === 'FAILED' && okMap.get(job.jobName) !== true) {
-                    okMap.set(job.jobName, false);
-                }
-                else {
-                    okMap.set(job.jobName, true);
+                if (!job.jobName.startsWith(jobPrefix))
+                    continue;
+                jobIds.add(job.jobId);
+            }
+        }
+        // Inspect all the jobs for what files are being "processed"
+        const tiffs = new Set();
+        let allJobIds = [...jobIds];
+        while (allJobIds.length > 0) {
+            logger.info({ jobCount: allJobIds.length }, 'JobFetch');
+            const jobList = allJobIds.slice(0, 100);
+            allJobIds = allJobIds.slice(100);
+            const describedJobs = await this.batch.describeJobs({ jobs: jobList }).promise();
+            if (describedJobs.jobs == null)
+                continue;
+            for (const job of describedJobs.jobs) {
+                const jobCommand = (_a = job.container) === null || _a === void 0 ? void 0 : _a.command;
+                if (jobCommand == null)
+                    continue;
+                // Extract the tiff names from the job command
+                for (let i = 0; i < jobCommand.length; i++) {
+                    if (jobCommand[i] === '--name')
+                        tiffs.add(jobCommand[i + 1]);
                 }
             }
         }
-        return okMap;
+        return tiffs;
     }
     static async batchJob(job, commit = false, logger) {
-        var _a;
         const jobPath = job.getJobPath('job.json');
         if (!jobPath.startsWith('s3://')) {
             throw new Error(`AWS Batch collection.json have to be in S3, jobPath:${jobPath}`);
         }
         LogConfig.set(logger.child({ correlationId: job.id, imageryName: job.name }));
-        const region = (_a = Env.get('AWS_DEFAULT_REGION')) !== null && _a !== void 0 ? _a : 'ap-southeast-2';
-        const batch = new Batch({ region });
         fsa.configure(job.output.location);
-        const runningJobs = await BatchJob.getCurrentJobList(batch);
-        // Prepare chunk job and individual jobs based on imagery size.
-        const jobs = await this.getJobs(job);
         // Get all the existing output tiffs
-        const targetPath = job.getJobPath();
-        const existTiffs = [];
+        const existTiffs = new Set();
         for await (const fileName of fsa.list(job.getJobPath())) {
             if (fileName.endsWith('.tiff'))
-                existTiffs.push(fileName);
+                existTiffs.add(basename(fileName));
         }
-        const toSubmit = [];
-        for (const names of jobs) {
-            // Check existence of batch job running
-            const jobName = BatchJob.id(job, names);
-            const isRunning = runningJobs.get(jobName);
-            if (isRunning) {
-                logger.info({ jobName }, 'JobRunning');
-                continue;
-            }
-            // Check existence of all the output tiffs.
-            let allExists = true;
-            for (const name of names) {
-                if (!existTiffs.includes(job.getJobPath(`${name}.tiff`)))
-                    allExists = false;
-            }
-            if (allExists) {
-                logger.info({ targetPath, names }, 'FileExists');
-                continue;
-            }
-            // Ready to submit
-            toSubmit.push(names);
-        }
-        if (toSubmit.length === 0) {
+        const runningJobs = await this.getCurrentJobList(job, logger);
+        for (const tiffName of runningJobs)
+            existTiffs.add(`${tiffName}.tiff`);
+        // Prepare chunk job and individual jobs based on imagery size.
+        const jobs = await this.getJobs(job, existTiffs, logger);
+        if (jobs.length === 0) {
             logger.info('NoJobs');
             return;
         }
         logger.info({
             jobTotal: job.output.files.length,
-            jobLeft: toSubmit.length,
+            jobLeft: jobs.length,
             jobQueue: JobQueue,
             jobDefinition: JobDefinition,
         }, 'JobSubmit');
-        for (const names of toSubmit) {
-            const jobStatus = await BatchJob.batchOne(jobPath, job, batch, names, commit);
+        for (const names of jobs) {
+            const jobStatus = await BatchJob.batchOne(jobPath, job, names, commit);
             logger.info(jobStatus, 'JobSubmitted');
         }
         if (!commit) {
@@ -155,11 +164,16 @@ export class BatchJob {
      * Prepare the jobs from job files, and chunk the small images into single
      * @returns List of jobs including single job and chunk jobs.
      */
-    static async getJobs(job) {
+    static getJobs(job, existing, log) {
         const jobs = [];
         let chunkJob = [];
         let chunkUnit = 0; // Calculate the chunkUnit based on the size
         for (const file of job.output.files) {
+            const outputFile = `${file.name}.tiff`;
+            if (existing.has(outputFile)) {
+                log.debug({ fileName: outputFile }, 'Skip:Exists');
+                continue;
+            }
             const imageSize = file.width / job.output.gsd;
             if (imageSize > 16385) {
                 jobs.push([file.name]);