@basemaps/cli 6.25.0 → 6.28.0

package/CHANGELOG.md

@@ -3,6 +3,51 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+# [6.28.0](https://github.com/linz/basemaps/compare/v6.27.0...v6.28.0) (2022-06-06)
+
+
+### Features
+
+* **server:** bundle basemaps-server cli so its easier to install ([#2218](https://github.com/linz/basemaps/issues/2218)) ([8457b66](https://github.com/linz/basemaps/commit/8457b66be6d0f54decf43b515bb78853cefbc8ed))
+
+
+
+
+
+# [6.27.0](https://github.com/linz/basemaps/compare/v6.26.0...v6.27.0) (2022-05-29)
+
+
+### Bug Fixes
+
+* **cli:** Action Cog get processing Id after read job json. ([#2204](https://github.com/linz/basemaps/issues/2204)) ([e459d2a](https://github.com/linz/basemaps/commit/e459d2a12363ded2a825c4266c79357324d17b97))
+
+
+### Features
+
+* **cli:** allow overriding imagery names ([#2169](https://github.com/linz/basemaps/issues/2169)) ([5c3bdd8](https://github.com/linz/basemaps/commit/5c3bdd89b664dd85df2b48d709653b71bdc348f7))
+* **cli:** Chunk batch job based on the size. ([#2217](https://github.com/linz/basemaps/issues/2217)) ([770430c](https://github.com/linz/basemaps/commit/770430ce388f13f2690fa680e113c4e830df312a))
+* **cli:** Insert imagery and tileset config after cog creation complete ([#2191](https://github.com/linz/basemaps/issues/2191)) ([3ea5efd](https://github.com/linz/basemaps/commit/3ea5efd049b956f882a05c90471d764efb5d39fd))
+* **cli:** Make a sseperate folder for each tiff for group cogs batch job. ([#2213](https://github.com/linz/basemaps/issues/2213)) ([e2aef5e](https://github.com/linz/basemaps/commit/e2aef5e460e059a7f8f59cc143a606255d7d77e0))
+* **cli:** make cogs will update the process job status if exists. ([#2180](https://github.com/linz/basemaps/issues/2180)) ([855ce1c](https://github.com/linz/basemaps/commit/855ce1cb1f7b8bff575be342184e5ac387684f09))
+* **cli:** Pass processingId to action job cli for the batch job. ([#2199](https://github.com/linz/basemaps/issues/2199)) ([f962fe2](https://github.com/linz/basemaps/commit/f962fe21859d44fe6ace79aed4611f3343a0d1bd))
+* **cli:** Upgrade action cog to be able to group small jobs into a single batch job. ([#2210](https://github.com/linz/basemaps/issues/2210)) ([8a7de91](https://github.com/linz/basemaps/commit/8a7de912a3b633bfac631e61b0480f2839ceb2cc))
+* **lambda-cog:** New lambda Cog for import api. ([#2207](https://github.com/linz/basemaps/issues/2207)) ([79f4ae7](https://github.com/linz/basemaps/commit/79f4ae70ea3fc16a37dd575b843a0b60a1365df4))
+* **lambda-tiler:** Add file number limitation to import api. ([#2203](https://github.com/linz/basemaps/issues/2203)) ([4694e29](https://github.com/linz/basemaps/commit/4694e29a4444810391e72b290ec64f8b8541c369))
+* **lambda-tiler:** Increase limit of total file size. ([#2205](https://github.com/linz/basemaps/issues/2205)) ([5246ea0](https://github.com/linz/basemaps/commit/5246ea0879a4bf6b20770fb633d63afac778d54d))
+* **lambda-tiler:** Load MaxImagePixelSize from import api. ([#2206](https://github.com/linz/basemaps/issues/2206)) ([b851934](https://github.com/linz/basemaps/commit/b85193401bc5df2ecb689c3801f62af71696cdf5))
+
+
+
+
+
+# [6.26.0](https://github.com/linz/basemaps/compare/v6.25.0...v6.26.0) (2022-05-12)
+
+**Note:** Version bump only for package @basemaps/cli
+
+
+
+
+
 # [6.25.0](https://github.com/linz/basemaps/compare/v6.24.2...v6.25.0) (2022-05-11)
 
 **Note:** Version bump only for package @basemaps/cli

package/build/cli/base.cli.d.ts

@@ -1,19 +1 @@
-#!/usr/bin/env node
-import { CommandLineParser } from '@rushstack/ts-command-line';
-import 'source-map-support/register.js';
-/** Useful traceability information  */
-export declare const CliInfo: {
-    package: string;
-    version: string;
-    hash: string;
-};
-/** Unique Id for this instance of the cli being run */
-export declare const CliId: string;
-export declare abstract class BaseCommandLine extends CommandLineParser {
-    verbose: import("@rushstack/ts-command-line").CommandLineFlagParameter;
-    extraVerbose: import("@rushstack/ts-command-line").CommandLineFlagParameter;
-    protected onExecute(): Promise<void>;
-    protected onDefineParameters(): void;
-    run(): void;
-}
 //# sourceMappingURL=base.cli.d.ts.map

package/build/cli/base.cli.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"base.cli.d.ts","sourceRoot":"","sources":["../../src/cli/base.cli.ts"],"names":[],"mappings":";AAGA,OAAO,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAC;AAC/D,OAAO,gCAAgC,CAAC;AAGxC,uCAAuC;AACvC,eAAO,MAAM,OAAO,EAAE;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAKrE,CAAC;AAEF,uDAAuD;AACvD,eAAO,MAAM,KAAK,QAAc,CAAC;AAEjC,8BAAsB,eAAgB,SAAQ,iBAAiB;IAC7D,OAAO,gEAIJ;IACH,YAAY,gEAIT;IAEH,SAAS,CAAC,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;IAepC,SAAS,CAAC,kBAAkB,IAAI,IAAI;IAI7B,GAAG,IAAI,IAAI;CAUnB"}
+{"version":3,"file":"base.cli.d.ts","sourceRoot":"","sources":["../../src/cli/base.cli.ts"],"names":[],"mappings":""}

package/build/cli/base.cli.js

@@ -1,60 +1 @@
-#!/usr/bin/env node
-var _a, _b;
-import { LogConfig, LoggerFatalError } from '@basemaps/shared';
-import { GitTag } from '@basemaps/shared/build/cli/git.tag.js';
-import { CommandLineParser } from '@rushstack/ts-command-line';
-import 'source-map-support/register.js';
-import * as ulid from 'ulid';
-/** Useful traceability information  */
-export const CliInfo = {
-    // Detect unlinked packages looks for this string since its a package name, slightly work around it
-    package: '@' + 'basemaps/cli',
-    version: (_a = process.env.GIT_VERSION) !== null && _a !== void 0 ? _a : GitTag().version,
-    hash: (_b = process.env.GIT_HASH) !== null && _b !== void 0 ? _b : GitTag().hash,
-};
-/** Unique Id for this instance of the cli being run */
-export const CliId = ulid.ulid();
-export class BaseCommandLine extends CommandLineParser {
-    constructor() {
-        super(...arguments);
-        this.verbose = this.defineFlagParameter({
-            parameterLongName: '--verbose',
-            parameterShortName: '-v',
-            description: 'Show extra logging detail',
-        });
-        this.extraVerbose = this.defineFlagParameter({
-            parameterLongName: '--vv',
-            parameterShortName: '-V',
-            description: 'Show extra extra logging detail',
-        });
-    }
-    onExecute() {
-        if (this.verbose.value) {
-            LogConfig.get().level = 'debug';
-        }
-        else if (this.extraVerbose.value) {
-            LogConfig.get().level = 'trace';
-        }
-        else {
-            LogConfig.get().level = 'info';
-        }
-        const logger = LogConfig.get().child({ id: CliId });
-        logger.info(CliInfo, 'CliStart');
-        LogConfig.set(logger);
-        return super.onExecute();
-    }
-    onDefineParameters() {
-        // Nothing
-    }
-    run() {
-        this.executeWithoutErrorHandling().catch((err) => {
-            if (err instanceof LoggerFatalError) {
-                LogConfig.get().fatal(err.obj, err.message);
-            }
-            else {
-                LogConfig.get().fatal({ err }, 'Failed to run command');
-            }
-            process.exit(1);
-        });
-    }
-}
+"use strict";

package/build/cli/cogify/__tests__/batch.job.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=batch.job.test.d.ts.map

package/build/cli/cogify/__tests__/batch.job.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"batch.job.test.d.ts","sourceRoot":"","sources":["../../../../src/cli/cogify/__tests__/batch.job.test.ts"],"names":[],"mappings":""}

package/build/cli/cogify/__tests__/batch.job.test.js

@@ -0,0 +1,69 @@
+import o from 'ospec';
+import { BatchJob, extractResolutionFromName } from '../batch.job.js';
+o.spec('action.batch', () => {
+    o('extractResolutionFromName', () => {
+        o(extractResolutionFromName('2013')).equals(-1);
+        o(extractResolutionFromName('new_zealand_sentinel_2018-19_10m')).equals(10000);
+        o(extractResolutionFromName('abc2017def_1.00m')).equals(1000);
+        o(extractResolutionFromName('wellington_urban_2017_0.10m')).equals(100);
+        o(extractResolutionFromName('wellington_urban_2017_0-10m')).equals(100);
+        o(extractResolutionFromName('wellington_urban_2017_1.00m')).equals(1000);
+        o(extractResolutionFromName('wellington_urban_2017_0.025m')).equals(25);
+    });
+    o('should create valid jobNames', () => {
+        const fakeJob = { id: '01FHRPYJ5FV1XAARZAC4T4K6MC', name: 'geographx_nz_texture_shade_2012_8-0m' };
+        o(BatchJob.id(fakeJob, ['0'])).equals('01FHRPYJ5FV1XAARZAC4T4K6MC-9af5e139bbb3e502-0');
+        fakeJob.name = 'ōtorohanga_urban_2021_0.1m_RGB';
+        o(BatchJob.id(fakeJob, ['0'])).equals('01FHRPYJ5FV1XAARZAC4T4K6MC-5294acface81c107-0');
+    });
+    o('should truncate job names to 128 characters', () => {
+        const fakeJob = { id: '01FHRPYJ5FV1XAARZAC4T4K6MC', name: 'geographx_nz_texture_shade_2012_8-0m' };
+        o(BatchJob.id(fakeJob, [
+            'this is a really long file name',
+            'it should over flow 128 characters',
+            'so it should be truncated at some point.tiff',
+        ])).equals('01FHRPYJ5FV1XAARZAC4T4K6MC-9af5e139bbb3e502-it should over flow 128 characters_so it should be truncated at some point.tiff_this');
+    });
+    o('should prepare valid chunk jobs', async () => {
+        const fakeGsd = 0.9;
+        const ChunkJobSmall = 4097;
+        const ChunkJobMiddle = 8193;
+        const ChunkJobLarge = 16385;
+        const fakeFiles = [
+            { name: '1-2-0', width: ChunkJobSmall * fakeGsd - 1 },
+            { name: '1-2-1', width: ChunkJobSmall * fakeGsd - 1 },
+            { name: '1-2-2', width: ChunkJobLarge * fakeGsd + 1 },
+            { name: '1-2-3', width: ChunkJobMiddle * fakeGsd - 1 },
+            { name: '1-2-4', width: ChunkJobMiddle * fakeGsd - 1 },
+            { name: '1-2-5', width: ChunkJobLarge * fakeGsd + 0.1 },
+            { name: '1-2-6', width: ChunkJobMiddle * fakeGsd - 1 },
+            { name: '1-2-7', width: ChunkJobLarge * fakeGsd - 1 },
+            { name: '1-2-8', width: ChunkJobLarge * fakeGsd - 1 },
+            { name: '1-2-9', width: ChunkJobLarge * fakeGsd - 1 },
+            { name: '1-2-10', width: ChunkJobLarge * fakeGsd - 1 },
+            { name: '1-2-11', width: ChunkJobLarge * fakeGsd - 1 },
+            { name: '1-2-12', width: ChunkJobSmall * fakeGsd - 1 },
+            { name: '1-2-13', width: ChunkJobLarge * fakeGsd + 1 },
+            { name: '1-2-14', width: ChunkJobMiddle * fakeGsd - 1 }, // Middle Job 70
+        ];
+        const fakeJob = { id: '01FHRPYJ5FV1XAARZAC4T4K6MC', output: { files: fakeFiles, gsd: fakeGsd } };
+        o(await BatchJob.getJobs(fakeJob)).deepEquals([
+            [fakeFiles[2].name],
+            [fakeFiles[5].name],
+            [
+                fakeFiles[0].name,
+                fakeFiles[1].name,
+                fakeFiles[3].name,
+                fakeFiles[4].name,
+                fakeFiles[6].name,
+                fakeFiles[7].name,
+                fakeFiles[8].name,
+                fakeFiles[9].name,
+                fakeFiles[10].name,
+                fakeFiles[11].name,
+            ],
+            [fakeFiles[13].name],
+            [fakeFiles[12].name, fakeFiles[14].name], // Second Chunk
+        ]);
+    });
+});

package/build/cli/cogify/__tests__/semver.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"semver.test.d.ts","sourceRoot":"","sources":["../../../../src/cli/cogify/__tests__/semver.test.ts"],"names":[],"mappings":""}

package/build/cli/cogify/action.cog.d.ts

@@ -8,7 +8,7 @@ export declare class ActionCogCreate extends CommandLineAction {
     private commit?;
     private cogIndex?;
     constructor();
-    getName(job: CogJob): string | null;
+    getNames(job: CogJob): Set<string> | null;
     onExecute(): Promise<void>;
     /** Check to see how many tiffs are remaining in the job */
     checkJobStatus(job: CogStacJob, logger: LogType): Promise<void>;

package/build/cli/cogify/action.cog.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"action.cog.d.ts","sourceRoot":"","sources":["../../../src/cli/cogify/action.cog.ts"],"names":[],"mappings":"AAAA,OAAO,EAAyC,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAClF,OAAO,EACL,iBAAiB,EAIlB,MAAM,4BAA4B,CAAC;AAIpC,OAAO,EAAE,UAAU,EAAE,MAAM,2BAA2B,CAAC;AAGvD,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAM5C,qBAAa,eAAgB,SAAQ,iBAAiB;IACpD,OAAO,CAAC,GAAG,CAAC,CAA6B;IACzC,OAAO,CAAC,IAAI,CAAC,CAA6B;IAC1C,OAAO,CAAC,MAAM,CAAC,CAA2B;IAC1C,OAAO,CAAC,QAAQ,CAAC,CAA8B;;IAU/C,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI;IAgC7B,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;IAqEhC,2DAA2D;IACrD,cAAc,CAAC,GAAG,EAAE,UAAU,EAAE,MAAM,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAoBrE,SAAS,CAAC,kBAAkB,IAAI,IAAI;CA4BrC"}
+{"version":3,"file":"action.cog.d.ts","sourceRoot":"","sources":["../../../src/cli/cogify/action.cog.ts"],"names":[],"mappings":"AAAA,OAAO,EAAiD,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAE1F,OAAO,EACL,iBAAiB,EAKlB,MAAM,4BAA4B,CAAC;AAIpC,OAAO,EAAE,UAAU,EAAE,MAAM,2BAA2B,CAAC;AAGvD,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAQ5C,qBAAa,eAAgB,SAAQ,iBAAiB;IACpD,OAAO,CAAC,GAAG,CAAC,CAA6B;IACzC,OAAO,CAAC,IAAI,CAAC,CAAiC;IAC9C,OAAO,CAAC,MAAM,CAAC,CAA2B;IAC1C,OAAO,CAAC,QAAQ,CAAC,CAAkC;;IAUnD,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI;IA0CnC,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;IAqFhC,2DAA2D;IACrD,cAAc,CAAC,GAAG,EAAE,UAAU,EAAE,MAAM,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAqCrE,SAAS,CAAC,kBAAkB,IAAI,IAAI;CA4BrC"}

package/build/cli/cogify/action.cog.js

@@ -1,4 +1,5 @@
-import { Env, fsa, LogConfig, LoggerFatalError } from '@basemaps/shared';
+import { Config, Env, fsa, LogConfig, LoggerFatalError } from '@basemaps/shared';
+import { CliId } from '@basemaps/shared/build/cli/base.js';
 import { CommandLineAction, } from '@rushstack/ts-command-line';
 import { createReadStream, promises as fs } from 'fs';
 import { buildCogForName } from '../../cog/cog.js';
@@ -6,9 +7,11 @@ import { CogStacJob } from '../../cog/cog.stac.job.js';
 import { CogVrt } from '../../cog/cog.vrt.js';
 import { Cutline } from '../../cog/cutline.js';
 import { Gdal } from '../../gdal/gdal.js';
-import { CliId } from '../base.cli.js';
-import { makeTempFolder } from '../folder.js';
+import { makeTempFolder, makeTiffFolder } from '../folder.js';
 import path from 'path';
+import { insertConfigImagery, insertConfigTileSet } from './imagery.config.js';
+import { JobStatus } from '@basemaps/config';
+import { prepareUrl } from '../util.js';
 export class ActionCogCreate extends CommandLineAction {
     constructor() {
         super({
@@ -17,8 +20,9 @@ export class ActionCogCreate extends CommandLineAction {
             documentation: 'Create a COG for the specified cog name',
         });
     }
-    getName(job) {
+    getNames(job) {
         var _a, _b;
+        const output = new Set();
         const { files } = job.output;
         const batchIndex = Env.getNumber(Env.BatchIndex, -1);
         if (batchIndex > -1) {
@@ -26,21 +30,29 @@ export class ActionCogCreate extends CommandLineAction {
             if (name == null) {
                 throw new LoggerFatalError({ cogIndex: batchIndex, tileMax: files.length - 1 }, 'Failed to find cog name from batch index');
             }
-            return name;
+            output.add(name);
+            return output;
         }
-        const cogIndex = (_a = this.cogIndex) === null || _a === void 0 ? void 0 : _a.value;
-        if (cogIndex != null) {
-            const { name } = files[cogIndex];
-            if (name == null) {
-                throw new LoggerFatalError({ cogIndex, tileMax: files.length - 1 }, 'Failed to find cog name from index');
+        const cogIndex = (_a = this.cogIndex) === null || _a === void 0 ? void 0 : _a.values;
+        if (cogIndex != null && cogIndex.length > 0) {
+            for (const i of cogIndex) {
+                if (i < 0 || i >= files.length) {
+                    throw new LoggerFatalError({ cogIndex: i, tileMax: files.length - 1 }, 'Failed to find cog name from index');
+                }
+                const { name } = files[i];
+                output.add(name);
             }
-            return name;
+            return output;
         }
-        const name = (_b = this.name) === null || _b === void 0 ? void 0 : _b.value;
-        if (name == null || !files.find((r) => r.name === name)) {
-            throw new LoggerFatalError({ name, names: files.map((r) => r.name).join(', ') }, 'Name does not exist inside job');
+        const names = (_b = this.name) === null || _b === void 0 ? void 0 : _b.values;
+        if (names == null)
+            throw new LoggerFatalError({ names }, 'Names cannot be null');
+        for (const name of names) {
+            if (!files.find((r) => r.name === name))
+                throw new LoggerFatalError({ name, names: files.map((r) => r.name).join(', ') }, 'Name does not exist inside job');
+            output.add(name);
         }
-        return name;
+        return output;
     }
     async onExecute() {
         var _a, _b, _c, _d;
@@ -58,44 +70,64 @@ export class ActionCogCreate extends CommandLineAction {
         logger.info('CogCreate:Start');
         const gdalVersion = await Gdal.version(logger);
         logger.info({ version: gdalVersion }, 'CogCreate:GdalVersion');
-        const name = this.getName(job);
-        if (name == null)
-            return;
-        const targetPath = job.getJobPath(`${name}.tiff`);
-        fsa.configure(job.output.location);
-        const outputExists = await fsa.exists(targetPath);
-        logger.info({ targetPath, outputExists }, 'CogCreate:CheckExists');
-        // Output file exists don't try and overwrite it
-        if (outputExists) {
-            logger.warn({ targetPath }, 'CogCreate:OutputExists');
+        const names = this.getNames(job);
+        if (names == null || names.size === 0)
             return;
-        }
         const tmpFolder = await makeTempFolder(`basemaps-${job.id}-${CliId}`);
         try {
-            let cutlineJson;
-            if (job.output.cutline != null) {
-                const cutlinePath = job.getJobPath('cutline.geojson.gz');
-                logger.info({ path: cutlinePath }, 'CogCreate:UsingCutLine');
-                cutlineJson = await Cutline.loadCutline(cutlinePath);
-            }
-            else {
-                logger.warn('NoCutLine');
+            for (const name of names) {
+                const tiffFolder = await makeTiffFolder(tmpFolder, name);
+                const targetPath = job.getJobPath(`${name}.tiff`);
+                fsa.configure(job.output.location);
+                const outputExists = await fsa.exists(targetPath);
+                logger.info({ targetPath, outputExists }, 'CogCreate:CheckExists');
+                // Output file exists don't try and overwrite it
+                if (outputExists) {
+                    logger.warn({ targetPath }, 'CogCreate:OutputExists');
+                    await this.checkJobStatus(job, logger);
+                    continue;
+                }
+                let cutlineJson;
+                if (job.output.cutline != null) {
+                    const cutlinePath = job.getJobPath('cutline.geojson.gz');
+                    logger.info({ path: cutlinePath }, 'CogCreate:UsingCutLine');
+                    cutlineJson = await Cutline.loadCutline(cutlinePath);
+                }
+                else {
+                    logger.warn('NoCutLine');
+                }
+                const cutline = new Cutline(job.tileMatrix, cutlineJson, (_d = job.output.cutline) === null || _d === void 0 ? void 0 : _d.blend, job.output.oneCogCovering);
+                const tmpVrtPath = await CogVrt.buildVrt(tiffFolder, job, cutline, name, logger);
+                if (tmpVrtPath == null) {
+                    logger.warn({ name }, 'CogCreate:NoMatchingSourceImagery');
+                    return;
+                }
+                const tmpTiff = fsa.join(tiffFolder, `${name}.tiff`);
+                await buildCogForName(job, name, tmpVrtPath, tmpTiff, logger, isCommit);
+                logger.info({ target: targetPath }, 'CogCreate:StoreTiff');
+                if (isCommit) {
+                    await fsa.write(targetPath, createReadStream(tmpTiff));
+                    await this.checkJobStatus(job, logger);
+                }
+                else {
+                    logger.warn({ name }, 'DryRun:Done');
+                }
             }
-            const cutline = new Cutline(job.tileMatrix, cutlineJson, (_d = job.output.cutline) === null || _d === void 0 ? void 0 : _d.blend, job.output.oneCogCovering);
-            const tmpVrtPath = await CogVrt.buildVrt(tmpFolder, job, cutline, name, logger);
-            if (tmpVrtPath == null) {
-                logger.warn({ name }, 'CogCreate:NoMatchingSourceImagery');
-                return;
-            }
-            const tmpTiff = fsa.join(tmpFolder, `${name}.tiff`);
-            await buildCogForName(job, name, tmpVrtPath, tmpTiff, logger, isCommit);
-            logger.info({ target: targetPath }, 'CogCreate:StoreTiff');
-            if (isCommit) {
-                await fsa.write(targetPath, createReadStream(tmpTiff));
-                await this.checkJobStatus(job, logger);
-            }
-            else {
-                logger.warn('DryRun:Done');
+        }
+        catch (e) {
+            const processingId = job.json.processingId;
+            if (processingId != null) {
+                // Update job status if this is the processing job.
+                const jobConfig = await Config.ProcessingJob.get(processingId);
+                if (jobConfig == null)
+                    throw new Error('Unable to find Job Processing Config:' + processingId);
+                const jobFailed = jobConfig;
+                jobFailed.status = JobStatus.Fail;
+                jobFailed.error = String(e);
+                if (Config.ProcessingJob.isWriteable())
+                    await Config.ProcessingJob.put(jobFailed);
+                else
+                    throw new Error('Unable update the Processing Job status:' + jobFailed.id);
             }
         }
         finally {
@@ -118,7 +150,25 @@ export class ActionCogCreate extends CommandLineAction {
             expectedTiffs.delete(basename);
         }
         if (expectedTiffs.size === 0) {
-            logger.info({ tiffCount: jobSize, tiffTotal: jobSize }, 'CogCreate:JobComplete');
+            // Insert Imagery and TileSet Config
+            await insertConfigImagery(job, logger);
+            await insertConfigTileSet(job, logger);
+            // Update job status if this is the processing job.
+            const url = await prepareUrl(job);
+            const processingId = job.json.processingId;
+            if (processingId != null) {
+                const jobConfig = await Config.ProcessingJob.get(processingId);
+                if (jobConfig == null)
+                    throw new Error('Unable to find Job Processing Config:' + processingId);
+                const jobComplete = jobConfig;
+                jobComplete.status = JobStatus.Complete;
+                jobComplete.url = url;
+                if (Config.ProcessingJob.isWriteable())
+                    await Config.ProcessingJob.put(jobConfig);
+                else
+                    throw new Error('Unable update the Processing Job status:' + jobConfig.id);
+            }
+            logger.info({ tiffCount: jobSize, tiffTotal: jobSize, url }, 'CogCreate:JobComplete');
         }
         else {
             logger.info({ tiffCount: jobSize, tiffRemaining: expectedTiffs.size }, 'CogCreate:JobProgress');
@@ -131,16 +181,16 @@ export class ActionCogCreate extends CommandLineAction {
             description: 'Job config source to access',
             required: true,
         });
-        this.name = this.defineStringParameter({
+        this.name = this.defineStringListParameter({
             argumentName: 'NAME',
             parameterLongName: '--name',
-            description: 'cog name to process',
+            description: 'list of cog names to process',
             required: false,
         });
-        this.cogIndex = this.defineIntegerParameter({
+        this.cogIndex = this.defineIntegerListParameter({
             argumentName: 'COG_INDEX',
             parameterLongName: '--cog-index',
-            description: 'index of cog to process',
+            description: 'list of cog indexes to process',
             required: false,
         });
         this.commit = this.defineFlagParameter({

package/build/cli/cogify/action.job.d.ts

@@ -20,6 +20,7 @@ export declare class ActionJobCreate extends CommandLineAction {
     private tileMatrix;
     private oneCog;
     private fileList;
+    private processingId;
     constructor();
     fsConfig(source: CLiInputData): FileConfig;
     onExecute(): Promise<void>;

package/build/cli/cogify/action.job.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"action.job.d.ts","sourceRoot":"","sources":["../../../src/cli/cogify/action.job.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAuB,MAAM,kBAAkB,CAAC;AACnE,OAAO,EACL,iBAAiB,EAGjB,0BAA0B,EAC3B,MAAM,4BAA4B,CAAC;AAMpC,qBAAa,YAAY;IACvB,IAAI,EAAE,0BAA0B,CAAC;IACjC,OAAO,EAAE,0BAA0B,CAAC;IACpC,UAAU,EAAE,0BAA0B,CAAC;gBAE3B,MAAM,EAAE,iBAAiB,EAAE,MAAM,EAAE,MAAM;CAsBtD;AAED,qBAAa,eAAgB,SAAQ,iBAAiB;IACpD,OAAO,CAAC,MAAM,CAAe;IAC7B,OAAO,CAAC,MAAM,CAAe;IAC7B,OAAO,CAAC,cAAc,CAA8B;IACpD,OAAO,CAAC,OAAO,CAA6B;IAC5C,OAAO,CAAC,YAAY,CAA8B;IAClD,OAAO,CAAC,UAAU,CAA6B;IAC/C,OAAO,CAAC,mBAAmB,CAA6B;IACxD,OAAO,CAAC,WAAW,CAA2B;IAC9C,OAAO,CAAC,OAAO,CAA8B;IAC7C,OAAO,CAAC,gBAAgB,CAA8B;IACtD,OAAO,CAAC,UAAU,CAA6B;IAC/C,OAAO,CAAC,MAAM,CAA2B;IACzC,OAAO,CAAC,QAAQ,CAA6B;;IAU7C,QAAQ,CAAC,MAAM,EAAE,YAAY,GAAG,UAAU;IAkBpC,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;IAyDhC,SAAS,CAAC,kBAAkB,IAAI,IAAI;CAkFrC"}
+{"version":3,"file":"action.job.d.ts","sourceRoot":"","sources":["../../../src/cli/cogify/action.job.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAuB,MAAM,kBAAkB,CAAC;AACnE,OAAO,EACL,iBAAiB,EAGjB,0BAA0B,EAC3B,MAAM,4BAA4B,CAAC;AAMpC,qBAAa,YAAY;IACvB,IAAI,EAAE,0BAA0B,CAAC;IACjC,OAAO,EAAE,0BAA0B,CAAC;IACpC,UAAU,EAAE,0BAA0B,CAAC;gBAE3B,MAAM,EAAE,iBAAiB,EAAE,MAAM,EAAE,MAAM;CAsBtD;AAED,qBAAa,eAAgB,SAAQ,iBAAiB;IACpD,OAAO,CAAC,MAAM,CAAe;IAC7B,OAAO,CAAC,MAAM,CAAe;IAC7B,OAAO,CAAC,cAAc,CAA8B;IACpD,OAAO,CAAC,OAAO,CAA6B;IAC5C,OAAO,CAAC,YAAY,CAA8B;IAClD,OAAO,CAAC,UAAU,CAA6B;IAC/C,OAAO,CAAC,mBAAmB,CAA6B;IACxD,OAAO,CAAC,WAAW,CAA2B;IAC9C,OAAO,CAAC,OAAO,CAA8B;IAC7C,OAAO,CAAC,gBAAgB,CAA8B;IACtD,OAAO,CAAC,UAAU,CAA6B;IAC/C,OAAO,CAAC,MAAM,CAA2B;IACzC,OAAO,CAAC,QAAQ,CAA6B;IAC7C,OAAO,CAAC,YAAY,CAA6B;;IAUjD,QAAQ,CAAC,MAAM,EAAE,YAAY,GAAG,UAAU;IAkBpC,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;IA0DhC,SAAS,CAAC,kBAAkB,IAAI,IAAI;CAyFrC"}

package/build/cli/cogify/action.job.js

@@ -3,7 +3,7 @@ import { fsa } from '@basemaps/shared';
 import { CommandLineAction, } from '@rushstack/ts-command-line';
 import { CogJobFactory, MaxConcurrencyDefault } from '../../cog/job.factory.js';
 import { GdalCogBuilderDefaults, GdalResamplingOptions } from '../../gdal/gdal.config.js';
-import { CliId } from '../base.cli.js';
+import { CliId } from '@basemaps/shared/build/cli/base.js';
 export class CLiInputData {
     constructor(parent, prefix) {
         this.path = parent.defineStringParameter({
@@ -52,7 +52,7 @@ export class ActionJobCreate extends CommandLineAction {
         };
     }
     async onExecute() {
-        var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p, _q, _r, _s;
+        var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p, _q, _r, _s, _t;
         const sourceLocation = this.fsConfig(this.source);
         const outputLocation = this.fsConfig(this.output);
         fsa.configure(this.fsConfig(this.source));
@@ -95,9 +95,10 @@ export class ActionJobCreate extends CommandLineAction {
                 id: (_m = (_l = this.overrideId) === null || _l === void 0 ? void 0 : _l.value) !== null && _m !== void 0 ? _m : CliId,
                 projection: Epsg.tryGet((_o = this.sourceProjection) === null || _o === void 0 ? void 0 : _o.value),
                 resampling,
+                processingId: (_p = this.processingId) === null || _p === void 0 ? void 0 : _p.value,
             },
-            batch: (_q = (_p = this.submitBatch) === null || _p === void 0 ? void 0 : _p.value) !== null && _q !== void 0 ? _q : false,
-            oneCogCovering: (_s = (_r = this.oneCog) === null || _r === void 0 ? void 0 : _r.value) !== null && _s !== void 0 ? _s : false,
+            batch: (_r = (_q = this.submitBatch) === null || _q === void 0 ? void 0 : _q.value) !== null && _r !== void 0 ? _r : false,
+            oneCogCovering: (_t = (_s = this.oneCog) === null || _s === void 0 ? void 0 : _s.value) !== null && _t !== void 0 ? _t : false,
         };
         await CogJobFactory.create(ctx);
     }
@@ -171,5 +172,11 @@ export class ActionJobCreate extends CommandLineAction {
             description: 'supply a list of files to use as source imagery',
             required: false,
         });
+        this.processingId = this.defineStringParameter({
+            argumentName: 'PROCESSING_ID',
+            parameterLongName: '--processing-id',
+            description: 'pass the job processing id',
+            required: false,
+        });
     }
 }

package/build/cli/cogify/{action.batch.d.ts → batch.job.d.ts}

@@ -1,5 +1,4 @@
 import { LogType } from '@basemaps/shared';
-import { CommandLineAction } from '@rushstack/ts-command-line';
 import Batch from 'aws-sdk/clients/batch.js';
 import { CogJob } from '../../cog/types.js';
 /**
@@ -9,21 +8,17 @@ import { CogJob } from '../../cog/types.js';
  * @returns resolution (millimeters), -1 for failure to parse
  */
 export declare function extractResolutionFromName(name: string): number;
-export declare class ActionBatchJob extends CommandLineAction {
-    private job?;
-    private commit?;
-    private oneCog?;
-    constructor();
+export declare class BatchJob {
     /**
      * Create a id for a job
      *
      * This needs to be within `[a-Z_-]` upto 128 characters log
      * @param job job to process
-     * @param fileName output filename
+     * @param fileNames output filename
      * @returns job id
      */
-    static id(job: CogJob, fileName: string): string;
-    static batchOne(jobPath: string, job: CogJob, batch: Batch, name: string, isCommit: boolean): Promise<{
+    static id(job: CogJob, fileNames: string[]): string;
+    static batchOne(jobPath: string, job: CogJob, batch: Batch, names: string[], isCommit: boolean): Promise<{
         jobName: string;
         jobId: string;
         memory: number;
@@ -33,8 +28,11 @@ export declare class ActionBatchJob extends CommandLineAction {
      * @returns a map of JobName to if their status is "ok" (not failed)
      */
     static getCurrentJobList(batch: Batch): Promise<Map<string, boolean>>;
-    onExecute(): Promise<void>;
-    static batchJob(job: CogJob, commit: boolean | undefined, oneCog: string | undefined, logger: LogType): Promise<void>;
-    protected onDefineParameters(): void;
+    static batchJob(job: CogJob, commit: boolean | undefined, logger: LogType): Promise<void>;
+    /**
+     * Prepare the jobs from job files, and chunk the small images into single
+     * @returns List of jobs including single job and chunk jobs.
+     */
+    static getJobs(job: CogJob): Promise<string[][]>;
 }
-//# sourceMappingURL=action.batch.d.ts.map
+//# sourceMappingURL=batch.job.d.ts.map

package/build/cli/cogify/batch.job.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"batch.job.d.ts","sourceRoot":"","sources":["../../../src/cli/cogify/batch.job.ts"],"names":[],"mappings":"AACA,OAAO,EAAuB,OAAO,EAAc,MAAM,kBAAkB,CAAC;AAC5E,OAAO,KAAK,MAAM,0BAA0B,CAAC;AAE7C,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAa5C;;;;;GAKG;AACH,wBAAgB,yBAAyB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAI9D;AAED,qBAAa,QAAQ;IACnB;;;;;;;OAOG;IACH,MAAM,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,GAAG,MAAM;WAOtC,QAAQ,CACnB,OAAO,EAAE,MAAM,EACf,GAAG,EAAE,MAAM,EACX,KAAK,EAAE,KAAK,EACZ,KAAK,EAAE,MAAM,EAAE,EACf,QAAQ,EAAE,OAAO,GAChB,OAAO,CAAC;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAmC9D;;;OAGG;WACU,iBAAiB,CAAC,KAAK,EAAE,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;WAqB9D,QAAQ,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,qBAAQ,EAAE,MAAM,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAyElF;;;OAGG;WACU,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;CA4BvD"}