@basemaps/cli 6.8.0 → 6.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +58 -0
- package/basemaps.js +2 -0
- package/build/cli/base.cli.d.ts +1 -1
- package/build/cli/base.cli.d.ts.map +1 -1
- package/build/cli/base.cli.js +22 -33
- package/build/cli/basemaps/action.invalidate.d.ts +1 -1
- package/build/cli/basemaps/action.invalidate.d.ts.map +1 -1
- package/build/cli/basemaps/action.invalidate.js +14 -18
- package/build/cli/basemaps/action.tileset.info.d.ts +1 -1
- package/build/cli/basemaps/action.tileset.info.d.ts.map +1 -1
- package/build/cli/basemaps/action.tileset.info.js +18 -23
- package/build/cli/basemaps/index.d.ts +2 -2
- package/build/cli/basemaps/index.d.ts.map +1 -1
- package/build/cli/basemaps/index.js +7 -16
- package/build/cli/basemaps/tileset.action.d.ts.map +1 -1
- package/build/cli/basemaps/tileset.action.js +4 -8
- package/build/cli/basemaps/tileset.util.d.ts +1 -1
- package/build/cli/basemaps/tileset.util.d.ts.map +1 -1
- package/build/cli/basemaps/tileset.util.js +20 -28
- package/build/cli/cli.table.d.ts.map +1 -1
- package/build/cli/cli.table.js +2 -7
- package/build/cli/cogify/__test__/action.batch.test.js +11 -14
- package/build/cli/cogify/__test__/semver.test.js +50 -53
- package/build/cli/cogify/action.batch.d.ts +2 -2
- package/build/cli/cogify/action.batch.d.ts.map +1 -1
- package/build/cli/cogify/action.batch.js +16 -22
- package/build/cli/cogify/action.cog.d.ts +1 -1
- package/build/cli/cogify/action.cog.d.ts.map +1 -1
- package/build/cli/cogify/action.cog.js +29 -33
- package/build/cli/cogify/action.job.d.ts.map +1 -1
- package/build/cli/cogify/action.job.js +21 -26
- package/build/cli/cogify/index.d.ts +2 -2
- package/build/cli/cogify/index.d.ts.map +1 -1
- package/build/cli/cogify/index.js +9 -13
- package/build/cli/cogify/semver.util.d.ts.map +1 -1
- package/build/cli/cogify/semver.util.js +2 -5
- package/build/cli/folder.js +6 -11
- package/build/cli/tag.action.js +1 -4
- package/build/cli/util.js +11 -16
- package/build/cog/__test__/builder.test.js +31 -35
- package/build/cog/__test__/cog.stac.job.test.js +59 -62
- package/build/cog/__test__/cog.test.js +26 -29
- package/build/cog/__test__/cog.vrt.test.js +90 -90
- package/build/cog/__test__/cutline.test.js +71 -71
- package/build/cog/__test__/source.tiff.testhelper.d.ts +1 -1
- package/build/cog/__test__/source.tiff.testhelper.d.ts.map +1 -1
- package/build/cog/__test__/source.tiff.testhelper.js +6 -9
- package/build/cog/builder.d.ts +3 -3
- package/build/cog/builder.d.ts.map +1 -1
- package/build/cog/builder.js +43 -48
- package/build/cog/cog.d.ts +2 -2
- package/build/cog/cog.d.ts.map +1 -1
- package/build/cog/cog.js +15 -19
- package/build/cog/cog.stac.job.d.ts +10 -10
- package/build/cog/cog.stac.job.d.ts.map +1 -1
- package/build/cog/cog.stac.job.js +48 -52
- package/build/cog/cog.vrt.d.ts +2 -2
- package/build/cog/cog.vrt.d.ts.map +1 -1
- package/build/cog/cog.vrt.js +16 -18
- package/build/cog/constants.js +2 -5
- package/build/cog/cutline.d.ts +21 -21
- package/build/cog/cutline.d.ts.map +1 -1
- package/build/cog/cutline.js +53 -58
- package/build/cog/job.factory.d.ts +2 -2
- package/build/cog/job.factory.d.ts.map +1 -1
- package/build/cog/job.factory.js +25 -41
- package/build/cog/stac.d.ts +1 -1
- package/build/cog/stac.d.ts.map +1 -1
- package/build/cog/stac.js +2 -5
- package/build/cog/types.d.ts +1 -1
- package/build/cog/types.d.ts.map +1 -1
- package/build/cog/types.js +1 -2
- package/build/gdal/__test__/gdal.progress.test.js +11 -14
- package/build/gdal/__test__/gdal.test.js +35 -38
- package/build/gdal/gdal.cog.d.ts +2 -2
- package/build/gdal/gdal.cog.d.ts.map +1 -1
- package/build/gdal/gdal.cog.js +14 -18
- package/build/gdal/gdal.command.d.ts +1 -1
- package/build/gdal/gdal.command.d.ts.map +1 -1
- package/build/gdal/gdal.command.js +4 -9
- package/build/gdal/gdal.config.d.ts.map +1 -1
- package/build/gdal/gdal.config.js +4 -7
- package/build/gdal/gdal.d.ts +1 -1
- package/build/gdal/gdal.d.ts.map +1 -1
- package/build/gdal/gdal.docker.d.ts +1 -1
- package/build/gdal/gdal.docker.d.ts.map +1 -1
- package/build/gdal/gdal.docker.js +8 -13
- package/build/gdal/gdal.js +7 -11
- package/build/gdal/gdal.local.d.ts +1 -1
- package/build/gdal/gdal.local.d.ts.map +1 -1
- package/build/gdal/gdal.local.js +2 -6
- package/build/gdal/gdal.progress.d.ts.map +1 -1
- package/build/gdal/gdal.progress.js +2 -6
- package/build/index.d.ts +5 -5
- package/build/index.d.ts.map +1 -1
- package/build/index.js +4 -11
- package/{cogify → cogify.js} +1 -1
- package/package.json +15 -13
- package/basemaps +0 -2
|
@@ -1,64 +1,61 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
});
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
ospec_1.default('should support release candidates', () => {
|
|
33
|
-
ospec_1.default(semver_util_1.SemVer.toNumber('0.0.1-rc1')).equals(1);
|
|
34
|
-
ospec_1.default(semver_util_1.SemVer.toNumber('1.0.0-rc2')).equals(semver_util_1.SemVer.toNumber('1.0.0'));
|
|
1
|
+
import o from 'ospec';
|
|
2
|
+
import { SemVer } from '../semver.util.js';
|
|
3
|
+
o.spec('VersionCompare', () => {
|
|
4
|
+
o.spec('toNumber', () => {
|
|
5
|
+
o('should make correct numbers', () => {
|
|
6
|
+
o(SemVer.toNumber('0.0.1')).equals(1);
|
|
7
|
+
o(SemVer.toNumber('0.1.0')).equals(1024);
|
|
8
|
+
o(SemVer.toNumber('0.1.1')).equals(1025);
|
|
9
|
+
o(SemVer.toNumber('0.1.1023')).equals(2047);
|
|
10
|
+
o(SemVer.toNumber('1.0.0')).equals(1048576);
|
|
11
|
+
});
|
|
12
|
+
o('should not overflow', () => {
|
|
13
|
+
o(SemVer.toNumber('1023.1023.1023') > 0).equals(true);
|
|
14
|
+
});
|
|
15
|
+
o('should support up to three dots', () => {
|
|
16
|
+
o(SemVer.toNumber('')).equals(0);
|
|
17
|
+
o(SemVer.toNumber('1')).equals(1048576);
|
|
18
|
+
o(SemVer.toNumber('1.0')).equals(1048576);
|
|
19
|
+
o(SemVer.toNumber('1.0.0')).equals(1048576);
|
|
20
|
+
o(SemVer.toNumber('2')).equals(2097152);
|
|
21
|
+
o(SemVer.toNumber('')).equals(0);
|
|
22
|
+
});
|
|
23
|
+
o('should be getting bigger', () => {
|
|
24
|
+
o(SemVer.toNumber('1.0.0') > SemVer.toNumber('0.0.0')).equals(true);
|
|
25
|
+
o(SemVer.toNumber('1.0.1') > SemVer.toNumber('1.0.0')).equals(true);
|
|
26
|
+
o(SemVer.toNumber('1.1.1') > SemVer.toNumber('1.1.0')).equals(true);
|
|
27
|
+
o(SemVer.toNumber('2.1.1') > SemVer.toNumber('1.1.0')).equals(true);
|
|
28
|
+
});
|
|
29
|
+
o('should support release candidates', () => {
|
|
30
|
+
o(SemVer.toNumber('0.0.1-rc1')).equals(1);
|
|
31
|
+
o(SemVer.toNumber('1.0.0-rc2')).equals(SemVer.toNumber('1.0.0'));
|
|
35
32
|
});
|
|
36
33
|
});
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
});
|
|
46
|
-
|
|
34
|
+
o.spec('fromNumber', () => {
|
|
35
|
+
o('should create versions', () => {
|
|
36
|
+
o(SemVer.fromNumber(1)).equals('0.0.1');
|
|
37
|
+
o(SemVer.fromNumber(1023)).equals('0.0.1023');
|
|
38
|
+
o(SemVer.fromNumber(1024)).equals('0.1.0');
|
|
39
|
+
o(SemVer.fromNumber(1025)).equals('0.1.1');
|
|
40
|
+
o(SemVer.fromNumber(1048576)).equals('1.0.0');
|
|
41
|
+
o(SemVer.fromNumber(2097152)).equals('2.0.0');
|
|
42
|
+
});
|
|
43
|
+
o('should round trip', () => {
|
|
47
44
|
for (const num of ['0.1.0', '1.2.3', '3.2.1']) {
|
|
48
|
-
|
|
45
|
+
o(SemVer.fromNumber(SemVer.toNumber(num))).equals(num)(`Should roundtrip ${num}`);
|
|
49
46
|
}
|
|
50
47
|
});
|
|
51
48
|
});
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
49
|
+
o.spec('compare', () => {
|
|
50
|
+
o('should compare versions', () => {
|
|
51
|
+
o(SemVer.compare('0.0.0', '0.0.1')).equals(-1);
|
|
52
|
+
o(SemVer.compare('0.0.1', '0.0.0')).equals(1);
|
|
53
|
+
o(SemVer.compare('0.0.0', '0.0.0')).equals(0);
|
|
57
54
|
});
|
|
58
|
-
|
|
55
|
+
o('should sort versions', () => {
|
|
59
56
|
const versions = ['1.0.0', '2.0.0', '0.0.1', '1.2.3'];
|
|
60
|
-
versions.sort(
|
|
61
|
-
|
|
57
|
+
versions.sort(SemVer.compare);
|
|
58
|
+
o(versions).deepEquals(['0.0.1', '1.0.0', '1.2.3', '2.0.0']);
|
|
62
59
|
});
|
|
63
60
|
});
|
|
64
61
|
});
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { LogType } from '@basemaps/shared';
|
|
2
2
|
import { CommandLineAction } from '@rushstack/ts-command-line';
|
|
3
|
-
import Batch from 'aws-sdk/clients/batch';
|
|
4
|
-
import { CogJob } from '../../cog/types';
|
|
3
|
+
import Batch from 'aws-sdk/clients/batch.js';
|
|
4
|
+
import { CogJob } from '../../cog/types.js';
|
|
5
5
|
/**
|
|
6
6
|
* Attempt to parse a resolution from a imagery name
|
|
7
7
|
* @example `wellington_urban_2017_0.10m` -> 100
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"action.batch.d.ts","sourceRoot":"","sources":["../../../src/cli/cogify/action.batch.ts"],"names":[],"mappings":"AACA,OAAO,EAAuB,OAAO,EAAc,MAAM,kBAAkB,CAAC;AAC5E,OAAO,EAAE,iBAAiB,EAAwD,MAAM,4BAA4B,CAAC;AACrH,OAAO,KAAK,MAAM,
|
|
1
|
+
{"version":3,"file":"action.batch.d.ts","sourceRoot":"","sources":["../../../src/cli/cogify/action.batch.ts"],"names":[],"mappings":"AACA,OAAO,EAAuB,OAAO,EAAc,MAAM,kBAAkB,CAAC;AAC5E,OAAO,EAAE,iBAAiB,EAAwD,MAAM,4BAA4B,CAAC;AACrH,OAAO,KAAK,MAAM,0BAA0B,CAAC;AAE7C,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAS5C;;;;;GAKG;AACH,wBAAgB,yBAAyB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAI9D;AAED,qBAAa,cAAe,SAAQ,iBAAiB;IACnD,OAAO,CAAC,GAAG,CAAC,CAA6B;IACzC,OAAO,CAAC,MAAM,CAAC,CAA2B;IAC1C,OAAO,CAAC,MAAM,CAAC,CAA6B;;IAU5C,MAAM,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM;WAI/B,QAAQ,CACnB,OAAO,EAAE,MAAM,EACf,GAAG,EAAE,MAAM,EACX,KAAK,EAAE,KAAK,EACZ,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,OAAO,GAChB,OAAO,CAAC;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IA6B9D;;;OAGG;WACU,iBAAiB,CAAC,KAAK,EAAE,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAqBrE,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;WAanB,QAAQ,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,qBAAQ,EAAE,MAAM,EAAE,MAAM,GAAG,SAAS,EAAE,MAAM,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAyD9G,SAAS,CAAC,kBAAkB,IAAI,IAAI;CAqBrC"}
|
|
@@ -1,12 +1,8 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
const shared_1 = require("@basemaps/shared");
|
|
7
|
-
const ts_command_line_1 = require("@rushstack/ts-command-line");
|
|
8
|
-
const batch_1 = tslib_1.__importDefault(require("aws-sdk/clients/batch"));
|
|
9
|
-
const cog_stac_job_1 = require("../../cog/cog.stac.job");
|
|
1
|
+
import { TileMatrixSet } from '@basemaps/geo';
|
|
2
|
+
import { Env, fsa, LogConfig, Projection } from '@basemaps/shared';
|
|
3
|
+
import { CommandLineAction } from '@rushstack/ts-command-line';
|
|
4
|
+
import Batch from 'aws-sdk/clients/batch.js';
|
|
5
|
+
import { CogStacJob } from '../../cog/cog.stac.job.js';
|
|
10
6
|
const JobQueue = 'CogBatchJobQueue';
|
|
11
7
|
const JobDefinition = 'CogBatchJob';
|
|
12
8
|
/** The base alignment level used by GDAL, Tiffs that are bigger or smaller than this should scale the compute resources */
|
|
@@ -18,14 +14,13 @@ const ResolutionRegex = /((?:\d[\.\-])?\d+)m/;
|
|
|
18
14
|
* @param name Imagery name to parse
|
|
19
15
|
* @returns resolution (millimeters), -1 for failure to parse
|
|
20
16
|
*/
|
|
21
|
-
function extractResolutionFromName(name) {
|
|
17
|
+
export function extractResolutionFromName(name) {
|
|
22
18
|
const matches = name.match(ResolutionRegex);
|
|
23
19
|
if (matches == null)
|
|
24
20
|
return -1;
|
|
25
21
|
return parseFloat(matches[1].replace('-', '.')) * 1000;
|
|
26
22
|
}
|
|
27
|
-
|
|
28
|
-
class ActionBatchJob extends ts_command_line_1.CommandLineAction {
|
|
23
|
+
export class ActionBatchJob extends CommandLineAction {
|
|
29
24
|
constructor() {
|
|
30
25
|
super({
|
|
31
26
|
actionName: 'batch',
|
|
@@ -38,8 +33,8 @@ class ActionBatchJob extends ts_command_line_1.CommandLineAction {
|
|
|
38
33
|
}
|
|
39
34
|
static async batchOne(jobPath, job, batch, name, isCommit) {
|
|
40
35
|
const jobName = ActionBatchJob.id(job, name);
|
|
41
|
-
const tile =
|
|
42
|
-
const alignmentLevels =
|
|
36
|
+
const tile = TileMatrixSet.nameToTile(name);
|
|
37
|
+
const alignmentLevels = Projection.findAlignmentLevels(job.tileMatrix, tile, job.source.gsd);
|
|
43
38
|
// Give 25% more memory to larger jobs
|
|
44
39
|
const resDiff = 1 + Math.max(alignmentLevels - MagicAlignmentLevel, 0) * 0.25;
|
|
45
40
|
const memory = 3900 * resDiff;
|
|
@@ -87,18 +82,18 @@ class ActionBatchJob extends ts_command_line_1.CommandLineAction {
|
|
|
87
82
|
if (((_a = this.job) === null || _a === void 0 ? void 0 : _a.value) == null) {
|
|
88
83
|
throw new Error('Failed to read parameters');
|
|
89
84
|
}
|
|
90
|
-
await ActionBatchJob.batchJob(await
|
|
85
|
+
await ActionBatchJob.batchJob(await CogStacJob.load(this.job.value), (_b = this.commit) === null || _b === void 0 ? void 0 : _b.value, (_c = this.oneCog) === null || _c === void 0 ? void 0 : _c.value, LogConfig.get());
|
|
91
86
|
}
|
|
92
87
|
static async batchJob(job, commit = false, oneCog, logger) {
|
|
93
88
|
var _a;
|
|
94
89
|
const jobPath = job.getJobPath('job.json');
|
|
95
|
-
if (!
|
|
90
|
+
if (!jobPath.startsWith('s3://')) {
|
|
96
91
|
throw new Error(`AWS Batch collection.json have to be in S3, jobPath:${jobPath}`);
|
|
97
92
|
}
|
|
98
|
-
|
|
99
|
-
const region = (_a =
|
|
100
|
-
const batch = new
|
|
101
|
-
|
|
93
|
+
LogConfig.set(logger.child({ correlationId: job.id, imageryName: job.name }));
|
|
94
|
+
const region = (_a = Env.get('AWS_DEFAULT_REGION')) !== null && _a !== void 0 ? _a : 'ap-southeast-2';
|
|
95
|
+
const batch = new Batch({ region });
|
|
96
|
+
fsa.configure(job.output.location);
|
|
102
97
|
const runningJobs = await ActionBatchJob.getCurrentJobList(batch);
|
|
103
98
|
const stats = await Promise.all(job.output.files.map(async ({ name }) => {
|
|
104
99
|
if (oneCog != null && oneCog !== name)
|
|
@@ -110,7 +105,7 @@ class ActionBatchJob extends ts_command_line_1.CommandLineAction {
|
|
|
110
105
|
return { name, ok: true };
|
|
111
106
|
}
|
|
112
107
|
const targetPath = job.getJobPath(`${name}.tiff`);
|
|
113
|
-
const exists = await
|
|
108
|
+
const exists = await fsa.exists(targetPath);
|
|
114
109
|
if (exists)
|
|
115
110
|
logger.info({ targetPath }, 'FileExists');
|
|
116
111
|
return { name, ok: exists };
|
|
@@ -155,4 +150,3 @@ class ActionBatchJob extends ts_command_line_1.CommandLineAction {
|
|
|
155
150
|
});
|
|
156
151
|
}
|
|
157
152
|
}
|
|
158
|
-
exports.ActionBatchJob = ActionBatchJob;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"action.cog.d.ts","sourceRoot":"","sources":["../../../src/cli/cogify/action.cog.ts"],"names":[],"mappings":"AACA,OAAO,
|
|
1
|
+
{"version":3,"file":"action.cog.d.ts","sourceRoot":"","sources":["../../../src/cli/cogify/action.cog.ts"],"names":[],"mappings":"AACA,OAAO,EACL,iBAAiB,EAIlB,MAAM,4BAA4B,CAAC;AAOpC,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAK5C,qBAAa,eAAgB,SAAQ,iBAAiB;IACpD,OAAO,CAAC,GAAG,CAAC,CAA6B;IACzC,OAAO,CAAC,IAAI,CAAC,CAA6B;IAC1C,OAAO,CAAC,MAAM,CAAC,CAA2B;IAC1C,OAAO,CAAC,QAAQ,CAAC,CAA8B;;IAU/C,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI;IAgC7B,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;IA8DhC,SAAS,CAAC,kBAAkB,IAAI,IAAI;CA4BrC"}
|
|
@@ -1,17 +1,14 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
const base_cli_1 = require("../base.cli");
|
|
13
|
-
const folder_1 = require("../folder");
|
|
14
|
-
class ActionCogCreate extends ts_command_line_1.CommandLineAction {
|
|
1
|
+
import { Env, fsa, LogConfig, LoggerFatalError } from '@basemaps/shared';
|
|
2
|
+
import { CommandLineAction, } from '@rushstack/ts-command-line';
|
|
3
|
+
import { createReadStream, promises as fs } from 'fs';
|
|
4
|
+
import { buildCogForName } from '../../cog/cog.js';
|
|
5
|
+
import { CogStacJob } from '../../cog/cog.stac.job.js';
|
|
6
|
+
import { CogVrt } from '../../cog/cog.vrt.js';
|
|
7
|
+
import { Cutline } from '../../cog/cutline.js';
|
|
8
|
+
import { Gdal } from '../../gdal/gdal.js';
|
|
9
|
+
import { CliId } from '../base.cli.js';
|
|
10
|
+
import { makeTempFolder } from '../folder.js';
|
|
11
|
+
export class ActionCogCreate extends CommandLineAction {
|
|
15
12
|
constructor() {
|
|
16
13
|
super({
|
|
17
14
|
actionName: 'cog',
|
|
@@ -22,11 +19,11 @@ class ActionCogCreate extends ts_command_line_1.CommandLineAction {
|
|
|
22
19
|
getName(job) {
|
|
23
20
|
var _a, _b;
|
|
24
21
|
const { files } = job.output;
|
|
25
|
-
const batchIndex =
|
|
22
|
+
const batchIndex = Env.getNumber(Env.BatchIndex, -1);
|
|
26
23
|
if (batchIndex > -1) {
|
|
27
24
|
const { name } = files[batchIndex];
|
|
28
25
|
if (name == null) {
|
|
29
|
-
throw new
|
|
26
|
+
throw new LoggerFatalError({ cogIndex: batchIndex, tileMax: files.length - 1 }, 'Failed to find cog name from batch index');
|
|
30
27
|
}
|
|
31
28
|
return name;
|
|
32
29
|
}
|
|
@@ -34,13 +31,13 @@ class ActionCogCreate extends ts_command_line_1.CommandLineAction {
|
|
|
34
31
|
if (cogIndex != null) {
|
|
35
32
|
const { name } = files[cogIndex];
|
|
36
33
|
if (name == null) {
|
|
37
|
-
throw new
|
|
34
|
+
throw new LoggerFatalError({ cogIndex, tileMax: files.length - 1 }, 'Failed to find cog name from index');
|
|
38
35
|
}
|
|
39
36
|
return name;
|
|
40
37
|
}
|
|
41
38
|
const name = (_b = this.name) === null || _b === void 0 ? void 0 : _b.value;
|
|
42
39
|
if (name == null || !files.find((r) => r.name === name)) {
|
|
43
|
-
throw new
|
|
40
|
+
throw new LoggerFatalError({ name, names: files.map((r) => r.name).join(', ') }, 'Name does not exist inside job');
|
|
44
41
|
}
|
|
45
42
|
return name;
|
|
46
43
|
}
|
|
@@ -49,46 +46,46 @@ class ActionCogCreate extends ts_command_line_1.CommandLineAction {
|
|
|
49
46
|
const jobFn = (_a = this.job) === null || _a === void 0 ? void 0 : _a.value;
|
|
50
47
|
if (jobFn == null)
|
|
51
48
|
throw new Error('Missing job name');
|
|
52
|
-
const job = await
|
|
49
|
+
const job = await CogStacJob.load(jobFn);
|
|
53
50
|
const isCommit = (_c = (_b = this.commit) === null || _b === void 0 ? void 0 : _b.value) !== null && _c !== void 0 ? _c : false;
|
|
54
|
-
const logger =
|
|
55
|
-
|
|
56
|
-
const gdalVersion = await
|
|
51
|
+
const logger = LogConfig.get().child({ correlationId: job.id, imageryName: job.name });
|
|
52
|
+
LogConfig.set(logger);
|
|
53
|
+
const gdalVersion = await Gdal.version(logger);
|
|
57
54
|
logger.info({ version: gdalVersion }, 'GdalVersion');
|
|
58
55
|
const name = this.getName(job);
|
|
59
56
|
if (name == null)
|
|
60
57
|
return;
|
|
61
58
|
const targetPath = job.getJobPath(`${name}.tiff`);
|
|
62
|
-
|
|
63
|
-
const outputExists = await
|
|
59
|
+
fsa.configure(job.output.location);
|
|
60
|
+
const outputExists = await fsa.exists(targetPath);
|
|
64
61
|
logger.info({ targetPath, outputExists }, 'CheckExists');
|
|
65
62
|
// Output file exists don't try and overwrite it
|
|
66
63
|
if (outputExists) {
|
|
67
64
|
logger.warn({ targetPath }, 'OutputExists');
|
|
68
65
|
return;
|
|
69
66
|
}
|
|
70
|
-
const tmpFolder = await
|
|
67
|
+
const tmpFolder = await makeTempFolder(`basemaps-${job.id}-${CliId}`);
|
|
71
68
|
try {
|
|
72
69
|
let cutlineJson;
|
|
73
70
|
if (job.output.cutline != null) {
|
|
74
71
|
const cutlinePath = job.getJobPath('cutline.geojson.gz');
|
|
75
72
|
logger.info({ path: cutlinePath }, 'UsingCutLine');
|
|
76
|
-
cutlineJson = await
|
|
73
|
+
cutlineJson = await Cutline.loadCutline(cutlinePath);
|
|
77
74
|
}
|
|
78
75
|
else {
|
|
79
76
|
logger.warn('NoCutLine');
|
|
80
77
|
}
|
|
81
|
-
const cutline = new
|
|
82
|
-
const tmpVrtPath = await
|
|
78
|
+
const cutline = new Cutline(job.tileMatrix, cutlineJson, (_d = job.output.cutline) === null || _d === void 0 ? void 0 : _d.blend, job.output.oneCogCovering);
|
|
79
|
+
const tmpVrtPath = await CogVrt.buildVrt(tmpFolder, job, cutline, name, logger);
|
|
83
80
|
if (tmpVrtPath == null) {
|
|
84
81
|
logger.warn({ name, tileMatrix: job.tileMatrix.identifier }, 'NoMatchingSourceImagery');
|
|
85
82
|
return;
|
|
86
83
|
}
|
|
87
|
-
const tmpTiff =
|
|
88
|
-
await
|
|
84
|
+
const tmpTiff = fsa.join(tmpFolder, `${name}.tiff`);
|
|
85
|
+
await buildCogForName(job, name, tmpVrtPath, tmpTiff, logger, isCommit);
|
|
89
86
|
logger.info({ target: targetPath }, 'StoreTiff');
|
|
90
87
|
if (isCommit) {
|
|
91
|
-
await
|
|
88
|
+
await fsa.write(targetPath, createReadStream(tmpTiff));
|
|
92
89
|
}
|
|
93
90
|
else {
|
|
94
91
|
logger.warn('DryRun:Done');
|
|
@@ -96,7 +93,7 @@ class ActionCogCreate extends ts_command_line_1.CommandLineAction {
|
|
|
96
93
|
}
|
|
97
94
|
finally {
|
|
98
95
|
// Cleanup!
|
|
99
|
-
await
|
|
96
|
+
await fs.rmdir(tmpFolder, { recursive: true });
|
|
100
97
|
}
|
|
101
98
|
}
|
|
102
99
|
onDefineParameters() {
|
|
@@ -125,4 +122,3 @@ class ActionCogCreate extends ts_command_line_1.CommandLineAction {
|
|
|
125
122
|
});
|
|
126
123
|
}
|
|
127
124
|
}
|
|
128
|
-
exports.ActionCogCreate = ActionCogCreate;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"action.job.d.ts","sourceRoot":"","sources":["../../../src/cli/cogify/action.job.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAuB,MAAM,kBAAkB,CAAC;AACnE,OAAO,
|
|
1
|
+
{"version":3,"file":"action.job.d.ts","sourceRoot":"","sources":["../../../src/cli/cogify/action.job.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAuB,MAAM,kBAAkB,CAAC;AACnE,OAAO,EACL,iBAAiB,EAGjB,0BAA0B,EAC3B,MAAM,4BAA4B,CAAC;AAMpC,qBAAa,YAAY;IACvB,IAAI,EAAE,0BAA0B,CAAC;IACjC,OAAO,EAAE,0BAA0B,CAAC;IACpC,UAAU,EAAE,0BAA0B,CAAC;gBAE3B,MAAM,EAAE,iBAAiB,EAAE,MAAM,EAAE,MAAM;CAsBtD;AAED,qBAAa,eAAgB,SAAQ,iBAAiB;IACpD,OAAO,CAAC,MAAM,CAAe;IAC7B,OAAO,CAAC,MAAM,CAAe;IAC7B,OAAO,CAAC,cAAc,CAA8B;IACpD,OAAO,CAAC,OAAO,CAA6B;IAC5C,OAAO,CAAC,YAAY,CAA8B;IAClD,OAAO,CAAC,UAAU,CAA6B;IAC/C,OAAO,CAAC,mBAAmB,CAA6B;IACxD,OAAO,CAAC,WAAW,CAA2B;IAC9C,OAAO,CAAC,OAAO,CAA8B;IAC7C,OAAO,CAAC,gBAAgB,CAA8B;IACtD,OAAO,CAAC,UAAU,CAA6B;IAC/C,OAAO,CAAC,MAAM,CAA2B;IACzC,OAAO,CAAC,QAAQ,CAA6B;;IAU7C,QAAQ,CAAC,MAAM,EAAE,YAAY,GAAG,UAAU;IAkBpC,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;IAyDhC,SAAS,CAAC,kBAAkB,IAAI,IAAI;CAkFrC"}
|
|
@@ -1,13 +1,10 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
const gdal_config_1 = require("../../gdal/gdal.config");
|
|
9
|
-
const base_cli_1 = require("../base.cli");
|
|
10
|
-
class CLiInputData {
|
|
1
|
+
import { Epsg, GoogleTms, TileMatrixSets } from '@basemaps/geo';
|
|
2
|
+
import { fsa } from '@basemaps/shared';
|
|
3
|
+
import { CommandLineAction, } from '@rushstack/ts-command-line';
|
|
4
|
+
import { CogJobFactory, MaxConcurrencyDefault } from '../../cog/job.factory.js';
|
|
5
|
+
import { GdalCogBuilderDefaults, GdalResamplingOptions } from '../../gdal/gdal.config.js';
|
|
6
|
+
import { CliId } from '../base.cli.js';
|
|
7
|
+
export class CLiInputData {
|
|
11
8
|
constructor(parent, prefix) {
|
|
12
9
|
this.path = parent.defineStringParameter({
|
|
13
10
|
argumentName: prefix.toUpperCase(),
|
|
@@ -29,8 +26,7 @@ class CLiInputData {
|
|
|
29
26
|
});
|
|
30
27
|
}
|
|
31
28
|
}
|
|
32
|
-
|
|
33
|
-
class ActionJobCreate extends ts_command_line_1.CommandLineAction {
|
|
29
|
+
export class ActionJobCreate extends CommandLineAction {
|
|
34
30
|
constructor() {
|
|
35
31
|
super({
|
|
36
32
|
actionName: 'job',
|
|
@@ -42,7 +38,7 @@ class ActionJobCreate extends ts_command_line_1.CommandLineAction {
|
|
|
42
38
|
if (source.path.value == null) {
|
|
43
39
|
throw new Error('Invalid path');
|
|
44
40
|
}
|
|
45
|
-
if (!
|
|
41
|
+
if (!source.path.value.startsWith('s3://')) {
|
|
46
42
|
return { type: 'local', path: source.path.value };
|
|
47
43
|
}
|
|
48
44
|
if (source.roleArn.value == null) {
|
|
@@ -59,18 +55,18 @@ class ActionJobCreate extends ts_command_line_1.CommandLineAction {
|
|
|
59
55
|
var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p, _q, _r, _s;
|
|
60
56
|
const sourceLocation = this.fsConfig(this.source);
|
|
61
57
|
const outputLocation = this.fsConfig(this.output);
|
|
62
|
-
|
|
63
|
-
|
|
58
|
+
fsa.configure(this.fsConfig(this.source));
|
|
59
|
+
fsa.configure(this.fsConfig(this.output));
|
|
64
60
|
let cutline = undefined;
|
|
65
61
|
if ((_a = this.cutline) === null || _a === void 0 ? void 0 : _a.value) {
|
|
66
62
|
cutline = { href: this.cutline.value, blend: (_b = this.cutlineBlend.value) !== null && _b !== void 0 ? _b : 20 };
|
|
67
63
|
}
|
|
68
|
-
const tileMatrix =
|
|
64
|
+
const tileMatrix = TileMatrixSets.find((_c = this.tileMatrix.value) !== null && _c !== void 0 ? _c : GoogleTms.identifier);
|
|
69
65
|
if (tileMatrix == null)
|
|
70
66
|
throw new Error('Invalid target-projection');
|
|
71
67
|
let resampling;
|
|
72
68
|
if (((_d = this.overrideWarpReample) === null || _d === void 0 ? void 0 : _d.value) != null) {
|
|
73
|
-
const warp =
|
|
69
|
+
const warp = GdalResamplingOptions[(_e = this.overrideWarpReample) === null || _e === void 0 ? void 0 : _e.value];
|
|
74
70
|
if (warp == null) {
|
|
75
71
|
throw new Error('Invalid override-warp-resampling');
|
|
76
72
|
}
|
|
@@ -81,7 +77,7 @@ class ActionJobCreate extends ts_command_line_1.CommandLineAction {
|
|
|
81
77
|
}
|
|
82
78
|
const fileListPath = (_f = this.fileList) === null || _f === void 0 ? void 0 : _f.value;
|
|
83
79
|
if (fileListPath != null) {
|
|
84
|
-
const fileData = await
|
|
80
|
+
const fileData = await fsa.read(fileListPath);
|
|
85
81
|
sourceLocation.files = fileData
|
|
86
82
|
.toString()
|
|
87
83
|
.trim()
|
|
@@ -94,16 +90,16 @@ class ActionJobCreate extends ts_command_line_1.CommandLineAction {
|
|
|
94
90
|
cutline,
|
|
95
91
|
tileMatrix,
|
|
96
92
|
override: {
|
|
97
|
-
concurrency: (_h = (_g = this.maxConcurrency) === null || _g === void 0 ? void 0 : _g.value) !== null && _h !== void 0 ? _h :
|
|
98
|
-
quality: (_k = (_j = this.quality) === null || _j === void 0 ? void 0 : _j.value) !== null && _k !== void 0 ? _k :
|
|
99
|
-
id: (_m = (_l = this.overrideId) === null || _l === void 0 ? void 0 : _l.value) !== null && _m !== void 0 ? _m :
|
|
100
|
-
projection:
|
|
93
|
+
concurrency: (_h = (_g = this.maxConcurrency) === null || _g === void 0 ? void 0 : _g.value) !== null && _h !== void 0 ? _h : MaxConcurrencyDefault,
|
|
94
|
+
quality: (_k = (_j = this.quality) === null || _j === void 0 ? void 0 : _j.value) !== null && _k !== void 0 ? _k : GdalCogBuilderDefaults.quality,
|
|
95
|
+
id: (_m = (_l = this.overrideId) === null || _l === void 0 ? void 0 : _l.value) !== null && _m !== void 0 ? _m : CliId,
|
|
96
|
+
projection: Epsg.tryGet((_o = this.sourceProjection) === null || _o === void 0 ? void 0 : _o.value),
|
|
101
97
|
resampling,
|
|
102
98
|
},
|
|
103
99
|
batch: (_q = (_p = this.submitBatch) === null || _p === void 0 ? void 0 : _p.value) !== null && _q !== void 0 ? _q : false,
|
|
104
100
|
oneCogCovering: (_s = (_r = this.oneCog) === null || _r === void 0 ? void 0 : _r.value) !== null && _s !== void 0 ? _s : false,
|
|
105
101
|
};
|
|
106
|
-
await
|
|
102
|
+
await CogJobFactory.create(ctx);
|
|
107
103
|
}
|
|
108
104
|
onDefineParameters() {
|
|
109
105
|
this.source = new CLiInputData(this, 'source');
|
|
@@ -113,7 +109,7 @@ class ActionJobCreate extends ts_command_line_1.CommandLineAction {
|
|
|
113
109
|
parameterLongName: '--concurrency',
|
|
114
110
|
parameterShortName: '-c',
|
|
115
111
|
description: 'Maximum number of requests to use at one time',
|
|
116
|
-
defaultValue:
|
|
112
|
+
defaultValue: MaxConcurrencyDefault,
|
|
117
113
|
required: false,
|
|
118
114
|
});
|
|
119
115
|
this.cutline = this.defineStringParameter({
|
|
@@ -161,7 +157,7 @@ class ActionJobCreate extends ts_command_line_1.CommandLineAction {
|
|
|
161
157
|
argumentName: 'TILE_MATRIX',
|
|
162
158
|
parameterLongName: '--target-tile-matrix-set',
|
|
163
159
|
description: 'The TileMatrixSet to generate the COGS for',
|
|
164
|
-
defaultValue:
|
|
160
|
+
defaultValue: GoogleTms.identifier,
|
|
165
161
|
required: false,
|
|
166
162
|
});
|
|
167
163
|
this.oneCog = this.defineFlagParameter({
|
|
@@ -177,4 +173,3 @@ class ActionJobCreate extends ts_command_line_1.CommandLineAction {
|
|
|
177
173
|
});
|
|
178
174
|
}
|
|
179
175
|
}
|
|
180
|
-
exports.ActionJobCreate = ActionJobCreate;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import 'source-map-support/register';
|
|
3
|
-
import { BaseCommandLine } from '../base.cli';
|
|
2
|
+
import 'source-map-support/register.js';
|
|
3
|
+
import { BaseCommandLine } from '../base.cli.js';
|
|
4
4
|
export declare class CogifyCommandLine extends BaseCommandLine {
|
|
5
5
|
constructor();
|
|
6
6
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/cli/cogify/index.ts"],"names":[],"mappings":";AACA,OAAO,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/cli/cogify/index.ts"],"names":[],"mappings":";AACA,OAAO,gCAAgC,CAAC;AACxC,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAKjD,qBAAa,iBAAkB,SAAQ,eAAe;;CAUrD"}
|
|
@@ -1,22 +1,18 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
const action_cog_1 = require("./action.cog");
|
|
9
|
-
const action_job_1 = require("./action.job");
|
|
10
|
-
class CogifyCommandLine extends base_cli_1.BaseCommandLine {
|
|
2
|
+
import 'source-map-support/register.js';
|
|
3
|
+
import { BaseCommandLine } from '../base.cli.js';
|
|
4
|
+
import { ActionBatchJob } from './action.batch.js';
|
|
5
|
+
import { ActionCogCreate } from './action.cog.js';
|
|
6
|
+
import { ActionJobCreate } from './action.job.js';
|
|
7
|
+
export class CogifyCommandLine extends BaseCommandLine {
|
|
11
8
|
constructor() {
|
|
12
9
|
super({
|
|
13
10
|
toolFilename: 'cogify',
|
|
14
11
|
toolDescription: 'Cloud optimized geotiff utilities',
|
|
15
12
|
});
|
|
16
|
-
this.addAction(new
|
|
17
|
-
this.addAction(new
|
|
18
|
-
this.addAction(new
|
|
13
|
+
this.addAction(new ActionCogCreate());
|
|
14
|
+
this.addAction(new ActionJobCreate());
|
|
15
|
+
this.addAction(new ActionBatchJob());
|
|
19
16
|
}
|
|
20
17
|
}
|
|
21
|
-
exports.CogifyCommandLine = CogifyCommandLine;
|
|
22
18
|
new CogifyCommandLine().run();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"semver.util.d.ts","sourceRoot":"","sources":["../../../src/cli/cogify/semver.util.ts"],"names":[],"mappings":"AAOA,eAAO,MAAM,MAAM;
|
|
1
|
+
{"version":3,"file":"semver.util.d.ts","sourceRoot":"","sources":["../../../src/cli/cogify/semver.util.ts"],"names":[],"mappings":"AAOA,eAAO,MAAM,MAAM;IACjB;;;;;;OAMG;gBACS,MAAM,GAAG,MAAM;IAmB3B;;;;OAIG;kBACW,MAAM,GAAG,MAAM;IAO7B;;;;;OAKG;gBACS,MAAM,MAAM,MAAM,GAAG,MAAM;CAGxC,CAAC"}
|
|
@@ -1,13 +1,10 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.SemVer = void 0;
|
|
4
1
|
/** major.minor.patch */
|
|
5
2
|
const MaxLength = 3;
|
|
6
3
|
/** fits into a int32 with out forcing a uint conversion */
|
|
7
4
|
const EncodeBits = 10;
|
|
8
5
|
/** Max size for a single semver number (1024) */
|
|
9
6
|
const MaxEncode = 2 ** EncodeBits - 1;
|
|
10
|
-
|
|
7
|
+
export const SemVer = {
|
|
11
8
|
/**
|
|
12
9
|
* Encode a semver string into a number
|
|
13
10
|
* @example
|
|
@@ -53,6 +50,6 @@ exports.SemVer = {
|
|
|
53
50
|
* @returns 0 if equal, >0 if vA > vB, <0 otherwise
|
|
54
51
|
*/
|
|
55
52
|
compare(vA, vB) {
|
|
56
|
-
return
|
|
53
|
+
return SemVer.toNumber(vA) - SemVer.toNumber(vB);
|
|
57
54
|
},
|
|
58
55
|
};
|
package/build/cli/folder.js
CHANGED
|
@@ -1,16 +1,11 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
const tslib_1 = require("tslib");
|
|
5
|
-
const shared_1 = require("@basemaps/shared");
|
|
6
|
-
const fs_1 = require("fs");
|
|
7
|
-
const path = tslib_1.__importStar(require("path"));
|
|
1
|
+
import { Env } from '@basemaps/shared';
|
|
2
|
+
import { promises as fs } from 'fs';
|
|
3
|
+
import * as path from 'path';
|
|
8
4
|
/** Make a temp folder inside TEMP_FOLDER's path */
|
|
9
|
-
async function makeTempFolder(folder) {
|
|
5
|
+
export async function makeTempFolder(folder) {
|
|
10
6
|
var _a;
|
|
11
|
-
const tempPath = (_a =
|
|
7
|
+
const tempPath = (_a = Env.get(Env.TempFolder)) !== null && _a !== void 0 ? _a : '/tmp';
|
|
12
8
|
const folderPath = path.join(tempPath, folder);
|
|
13
|
-
await
|
|
9
|
+
await fs.mkdir(folderPath, { recursive: true });
|
|
14
10
|
return folderPath;
|
|
15
11
|
}
|
|
16
|
-
exports.makeTempFolder = makeTempFolder;
|
package/build/cli/tag.action.js
CHANGED
|
@@ -1,8 +1,5 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.TagActions = void 0;
|
|
4
1
|
/** Commonly used definitions for cli args */
|
|
5
|
-
|
|
2
|
+
export const TagActions = {
|
|
6
3
|
Version: {
|
|
7
4
|
argumentName: 'VERSION',
|
|
8
5
|
parameterLongName: '--version',
|