@cogeotiff/cli 7.2.1 → 8.0.1

package/src/commands/info.ts

@@ -0,0 +1,199 @@
+import { fsa } from '@chunkd/fs';
+import { CogTiff, TiffTag, TiffTagGeo, TiffTagId, TiffTagValueType, TiffVersion, toHex } from '@cogeotiff/core';
+import { CogTiffImage } from '@cogeotiff/core/src/cog.tiff.image.js';
+import c from 'ansi-colors';
+import { command, flag, option, optional, restPositionals } from 'cmd-ts';
+import { ActionUtil, CliResultMap } from '../action.util.js';
+import { CliTable } from '../cli.table.js';
+import { DefaultArgs, Url } from '../common.js';
+import { FetchLog } from '../fs.js';
+import { ensureS3fs, setupLogger } from '../log.js';
+import { toByteSizeString } from '../util.bytes.js';
+
+function round(num: number): number {
+  const opt = 10 ** 4;
+  return Math.floor(num * opt) / opt;
+}
+
+export const commandInfo = command({
+  name: 'info',
+  args: {
+    ...DefaultArgs,
+    path: option({ short: 'f', long: 'file', type: optional(Url) }),
+    tags: flag({ short: 't', long: 'tags', description: 'Dump tiff Tags' }),
+    offsets: flag({ short: 'o', long: 'offsets', description: 'Load byte offsets too' }),
+    paths: restPositionals({ type: Url, description: 'Files to process' }),
+  },
+  async handler(args) {
+    const logger = setupLogger(args);
+    const paths = [...args.paths, args.path].filter((f) => f != null) as URL[];
+
+    for (const path of paths) {
+      if (path.protocol === 's3:') await ensureS3fs();
+      logger.debug('Tiff:load', { path: path?.href });
+      FetchLog.reset();
+
+      const source = fsa.source(path);
+      const tiff = await new CogTiff(source).init();
+
+      const header = [
+        { key: 'Tiff type', value: `${TiffVersion[tiff.version]} (v${String(tiff.version)})` },
+        {
+          key: 'Bytes read',
+          value: `${toByteSizeString(FetchLog.bytesRead)} (${FetchLog.fetches.length} Chunk${
+            FetchLog.fetches.length === 1 ? '' : 's'
+          })`,
+        },
+      ];
+
+      const firstImage = tiff.images[0];
+      const isGeoLocated = firstImage.isGeoLocated;
+      const images = [
+        { key: 'Compression', value: firstImage.compression },
+        isGeoLocated ? { key: 'Origin', value: firstImage.origin.map(round).join(', ') } : null,
+        isGeoLocated ? { key: 'Resolution', value: firstImage.resolution.map(round).join(', ') } : null,
+        isGeoLocated ? { key: 'BoundingBox', value: firstImage.bbox.map(round).join(', ') } : null,
+        firstImage.epsg ? { key: 'EPSG', value: `EPSG:${firstImage.epsg} (https://epsg.io/${firstImage.epsg})` } : null,
+        { key: 'Images', value: '\n' + TiffImageInfoTable.print(tiff.images, '\t').join('\n') },
+      ];
+
+      const ghostOptions = [...(tiff.options?.options.entries() ?? [])];
+      const gdalMetadata = parseGdalMetadata(firstImage);
+      const gdal = [
+        {
+          key: 'COG optimized',
+          value: String(tiff.options?.isCogOptimized),
+          enabled: tiff.options?.isCogOptimized === true,
+        },
+        { key: 'COG broken', value: String(tiff.options?.isBroken), enabled: tiff.options?.isBroken === true },
+        {
+          key: 'Ghost Options',
+          value: '\n' + ghostOptions.map((c) => `\t\t${c[0]} = ${c[1]}`).join('\n'),
+          enabled: ghostOptions.length > 0,
+        },
+        {
+          key: 'Metadata',
+          value: '\n' + gdalMetadata?.map((c) => `\t\t${c}`).join('\n'),
+          enabled: gdalMetadata != null,
+        },
+      ];
+
+      const result: CliResultMap[] = [
+        { keys: header },
+        { title: 'Images', keys: images },
+        { title: 'GDAL', keys: gdal, enabled: gdal.filter((g) => g.enabled == null || g.enabled).length > 0 },
+      ];
+      if (args.tags) {
+        for (const img of tiff.images) {
+          const tiffTags = [...img.tags.values()];
+          // Load the first 25 values of offset arrays
+          if (args.offsets) {
+            for (const tag of tiffTags) {
+              if (tag.type !== 'offset') continue;
+              // for (let i = 0; i < Math.min(tag.count, 100); i++) tag.getValueAt(i);
+            }
+          }
+
+          result.push({
+            title: `Image: ${img.id} - Tiff tags`,
+            keys: tiffTags.map(formatTag),
+          });
+          await img.loadGeoTiffTags();
+          if (img.tagsGeo.size > 0) {
+            const tiffTagsGeo = [...img.tagsGeo.entries()];
+            const keys = tiffTagsGeo.map(([key, value]) => formatGeoTag(key, value));
+            if (keys.length > 0) {
+              result.push({ title: `Image: ${img.id} - Geo Tiff tags`, keys });
+            }
+          }
+        }
+      }
+
+      const msg = ActionUtil.formatResult(`\n${c.bold('COG File Info')} - ${c.bold(path.href)}`, result);
+      console.log(msg.join('\n'));
+
+      await source.close?.();
+    }
+  },
+});
+
+const TiffImageInfoTable = new CliTable<CogTiffImage>();
+TiffImageInfoTable.add({ name: 'Id', width: 4, get: (_i, index) => String(index) });
+TiffImageInfoTable.add({ name: 'Size', width: 20, get: (i) => `${i.size.width}x${i.size.height}` });
+TiffImageInfoTable.add({
+  name: 'Tile Size',
+  width: 20,
+  get: (i) => `${i.tileSize.width}x${i.tileSize.height}`,
+  enabled: (i) => i.isTiled(),
+});
+TiffImageInfoTable.add({
+  name: 'Tile Count',
+  width: 20,
+  get: (i) => `${i.tileCount.x}x${i.tileCount.y} (${i.tileCount.x * i.tileCount.y})`,
+  enabled: (i) => i.isTiled(),
+});
+TiffImageInfoTable.add({
+  name: 'Strip Count',
+  width: 20,
+  get: (i) => `${i.tags.get(TiffTagId.StripOffsets)?.count}`,
+  enabled: (i) => !i.isTiled(),
+});
+TiffImageInfoTable.add({
+  name: 'Resolution',
+  width: 20,
+  get: (i) => `${round(i.resolution[0])}`,
+  enabled: (i) => i.isGeoLocated,
+});
+
+// Show compression only if it varies between images
+TiffImageInfoTable.add({
+  name: 'Compression',
+  width: 20,
+  get: (i) => i.compression,
+  enabled: (i) => {
+    const formats = new Set();
+    i.tiff.images.forEach((f) => formats.add(f.compression));
+    return formats.size > 1;
+  },
+});
+
+/**
+ * Parse out the GDAL Metadata to be more friendly to read
+ *
+ * TODO using a XML Parser will make this even better
+ * @param img
+ */
+function parseGdalMetadata(img: CogTiffImage): string[] | null {
+  const metadata = img.value(TiffTagId.GdalMetadata);
+  if (typeof metadata !== 'string') return null;
+  if (!metadata.startsWith('<GDALMetadata>')) return null;
+  return metadata
+    .replace('<GDALMetadata>\n', '')
+    .replace('</GDALMetadata>\n', '')
+    .replace('\n\x00', '')
+    .split('\n')
+    .map((c) => c.trim());
+}
+
+function formatTag(tag: TiffTag): { key: string; value: string } {
+  const tagName = TiffTagId[tag.id];
+  const tagDebug = `(${TiffTagValueType[tag.dataType]}${tag.count > 1 ? ' x' + tag.count : ''}`;
+  const key = `${c.dim(toHex(tag.id)).padEnd(7, ' ')} ${String(tagName)} ${c.dim(tagDebug)})`.padEnd(50, ' ');
+
+  if (Array.isArray(tag.value)) return { key, value: JSON.stringify(tag.value.slice(0, 250)) };
+
+  let tagString = JSON.stringify(tag.value) ?? c.dim('null');
+  if (tagString.length > 256) tagString = tagString.slice(0, 250) + '...';
+  return { key, value: tagString };
+}
+
+function formatGeoTag(tagId: TiffTagGeo, value: string | number): { key: string; value: string } {
+  const tagName = TiffTagGeo[tagId];
+  const key = `${c.dim(toHex(tagId)).padEnd(7, ' ')} ${String(tagName).padEnd(30)}`;
+
+  if (Array.isArray(value)) return { key, value: JSON.stringify(value.slice(0, 250)) };
+
+  let tagString = JSON.stringify(value) ?? c.dim('null');
+  if (tagString.length > 256) tagString = tagString.slice(0, 250) + '...';
+  return { key, value: tagString };
+}

package/src/common.ts

@@ -0,0 +1,20 @@
+import { Type, flag } from 'cmd-ts';
+import { pathToFileURL } from 'url';
+
+export const verbose = flag({ long: 'verbose', description: 'Verbose logging', short: 'v' });
+export const extraVerbose = flag({ long: 'extra-verbose', description: 'Extra verbose logging', short: 'V' });
+
+export const DefaultArgs = {
+  verbose,
+  extraVerbose,
+};
+
+export const Url: Type<string, URL> = {
+  async from(s: string): Promise<URL> {
+    try {
+      return new URL(s);
+    } catch (e) {
+      return pathToFileURL(s);
+    }
+  },
+};

package/src/fs.ts

@@ -0,0 +1,18 @@
+import { SourceCallback, SourceMiddleware, SourceRequest } from '@chunkd/source';
+import { logger } from './log.js';
+
+export const FetchLog: SourceMiddleware & { reset(): void; fetches: SourceRequest[]; bytesRead: number } = {
+  name: 'source:log',
+  fetch(req: SourceRequest, next: SourceCallback) {
+    this.fetches.push(req);
+    this.bytesRead += req.length ?? 0;
+    logger.info('Tiff:fetch', { href: req.source.url.href, offset: req.offset, length: req.length });
+    return next(req);
+  },
+  reset() {
+    this.fetches = [];
+    this.bytesRead = 0;
+  },
+  fetches: [],
+  bytesRead: 0,
+};

package/src/index.ts

@@ -1,9 +1,12 @@
-import 'source-map-support/register.js';
-import { CogInfoCommandLine } from './cli.cog.info.js';
-import { logger } from './cli.log.js';
+import { subcommands } from 'cmd-ts';
+import { commandInfo } from './commands/info.js';
+import { commandDump } from './commands/dump.js';
 
-const cogInfo: CogInfoCommandLine = new CogInfoCommandLine();
-cogInfo.executeWithoutErrorHandling().catch((error) => {
-    logger.fatal({ error }, 'Failed to run');
-    process.exit(1);
+export const cmd = subcommands({
+  name: 'cogeotiff',
+  description: 'COG utilities',
+  cmds: {
+    info: commandInfo,
+    dump: commandDump,
+  },
 });

package/src/log.ts

@@ -0,0 +1,43 @@
+import { FsHttp, fsa } from '@chunkd/fs';
+import { SourceCache, SourceChunk } from '@chunkd/middleware';
+import { log } from '@linzjs/tracing';
+import { FetchLog } from './fs.js';
+
+// Cache the last 10MB of chunks for reuse
+export const sourceCache = new SourceCache({ size: 10 * 1024 * 1024 });
+export const sourceChunk = new SourceChunk({ size: 32 * 1024 });
+
+export function setupLogger(cfg: { verbose?: boolean; extraVerbose?: boolean }): typeof log {
+  if (cfg.verbose) {
+    log.level = 'debug';
+  } else if (cfg.extraVerbose) {
+    log.level = 'trace';
+  } else {
+    log.level = 'warn';
+  }
+
+  fsa.register('http://', new FsHttp());
+  fsa.register('https://', new FsHttp());
+
+  // Order of these are really important
+  // Chunk all requests into 32KB chunks
+  fsa.middleware.push(sourceChunk);
+  // Cache the last 10MB of chunks for reuse
+  fsa.middleware.push(sourceCache);
+
+  fsa.middleware.push(FetchLog);
+
+  return log;
+}
+
+export const logger = log;
+
+/** S3 client adds approx 300ms to the cli startup time, so only register it if needed */
+export async function ensureS3fs(): Promise<void> {
+  if (fsa.systems.find((f) => f.prefix.startsWith('s3'))) return;
+
+  const S3Client = await import('@aws-sdk/client-s3');
+  const FsAwsS3 = await import('@chunkd/fs-aws');
+
+  fsa.register('s3://', new FsAwsS3.FsAwsS3(new S3Client.S3Client({})));
+}

package/src/util.bytes.ts

@@ -8,11 +8,11 @@ const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB'];
  * @param bytes  byte count to convert
  */
 export function toByteSizeString(bytes: number): string {
-    if (bytes === 1) return '1 Byte';
-    const i = Math.floor(Math.log(bytes) / Math.log(1024));
-    const output = bytes / Math.pow(1024, i);
-    const byteSize = sizes[i];
+  if (bytes === 1) return '1 Byte';
+  const i = Math.floor(Math.log(bytes) / Math.log(1024));
+  const output = bytes / Math.pow(1024, i);
+  const byteSize = sizes[i];
 
-    if (i === 1) return `${Math.round(output)} ${byteSize}`;
-    return `${Math.floor(output * 100) / 100} ${byteSize}`;
+  if (i === 1) return `${Math.round(output)} ${byteSize}`;
+  return `${Math.floor(output * 100) / 100} ${byteSize}`;
 }

package/src/util.tile.ts

@@ -1,14 +1,18 @@
 import { CogTiff, TiffMimeType } from '@cogeotiff/core';
-import { promises as fs } from 'fs';
-import * as path from 'path';
-import type pino from 'pino';
+import { log } from '@linzjs/tracing';
+import { promises as fs } from 'node:fs';
 
-const FileExtension: { [key: string]: string } = {
-    [TiffMimeType.JPEG]: 'jpeg',
-    [TiffMimeType.JP2]: 'jp2',
-    [TiffMimeType.WEBP]: 'webp',
-    [TiffMimeType.LZW]: 'lzw',
-    [TiffMimeType.DEFLATE]: 'deflate',
+const FileExtension: Record<string, string> = {
+  [TiffMimeType.Jpeg]: 'jpeg',
+  [TiffMimeType.Jp2]: 'jp2',
+  [TiffMimeType.Webp]: 'webp',
+  [TiffMimeType.Lzw]: 'lzw',
+  [TiffMimeType.Deflate]: 'deflate',
+  [TiffMimeType.None]: 'bin',
+  [TiffMimeType.JpegXl]: 'jpeg',
+  [TiffMimeType.Zstd]: 'zstd',
+  [TiffMimeType.Lerc]: 'lerc',
+  [TiffMimeType.Lzma]: 'lzma',
 };
 
 /**
@@ -22,31 +26,27 @@ const FileExtension: { [key: string]: string } = {
  * @returns tile name eg `001_002_12.png`
  */
 export function getTileName(mimeType: string, index: number, x: number, y: number): string {
-    const xS = `${x}`.padStart(3, '0');
-    const yS = `${y}`.padStart(3, '0');
-
-    const fileExt: string = FileExtension[mimeType];
-    if (fileExt == null) {
-        throw new Error(`Unable to process tile type:${mimeType}`);
-    }
-
-    return `${xS}_${yS}_${index}.${fileExt}`;
+  const xS = `${x}`.padStart(3, '0');
+  const yS = `${y}`.padStart(3, '0');
+  const fileExt = FileExtension[mimeType] ?? 'unknown';
+  return `${xS}_${yS}_${index}.${fileExt}`;
 }
 
 export async function writeTile(
-    tif: CogTiff,
-    x: number,
-    y: number,
-    index: number,
-    outputPath: string,
-    logger: pino.Logger,
-): Promise<void> {
-    const tile = await tif.getTile(x, y, index);
-    if (tile == null) {
-        logger.debug({ index, x, y }, 'TileEmpty');
-        return;
-    }
-    const fileName = getTileName(tile.mimeType, index, x, y);
-    await fs.writeFile(path.join(outputPath, fileName), tile.bytes);
-    logger.debug({ index, x, y, fileName, bytes: tile.bytes.length }, 'TileWrite');
+  tiff: CogTiff,
+  x: number,
+  y: number,
+  index: number,
+  outputPath: URL,
+  logger: typeof log,
+): Promise<string | null> {
+  const tile = await tiff.images[index].getTile(x, y);
+  if (tile == null) {
+    logger.debug('Tile:Empty', { source: tiff.source.url.href, index, x, y });
+    return null;
+  }
+  const fileName = getTileName(tile.mimeType, index, x, y);
+  await fs.writeFile(new URL(fileName, outputPath), Buffer.from(tile.bytes));
+  logger.debug('Tile:Write', { source: tiff.source.url.href, index, x, y, fileName, bytes: tile.bytes.byteLength });
+  return fileName;
 }

package/tsconfig.json

@@ -1,10 +1,10 @@
 {
-    "extends": "../../tsconfig.base.json",
-    "compilerOptions": {
-        "rootDir": "./src",
-        "outDir": "./build",
-        "lib": ["es2018", "dom"]
-    },
-    "include": ["src/**/*"],
-    "references": [{ "path": "../core" }]
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "rootDir": "./src",
+    "outDir": "./build",
+    "lib": ["es2018", "dom"]
+  },
+  "include": ["src"],
+  "references": [{ "path": "../core" }]
 }

package/build/action.dump.tile.d.ts

@@ -1,24 +0,0 @@
-import { CogTiff } from '@cogeotiff/core';
-import { CommandLineAction } from '@rushstack/ts-command-line';
-export interface GeoJsonPolygon {
-    type: 'Feature';
-    geometry: {
-        type: 'Polygon';
-        coordinates: [[[number, number], [number, number], [number, number], [number, number], [number, number]]];
-    };
-    properties: Record<string, unknown>;
-}
-export declare class ActionDumpTile extends CommandLineAction {
-    private file;
-    private imageIndex;
-    private output;
-    private outputCount;
-    private logger;
-    constructor();
-    dumpBounds(tif: CogTiff, output: string, index: number): Promise<void>;
-    dumpIndex(tif: CogTiff, output: string, index: number): Promise<void>;
-    dumpTiles(tif: CogTiff, output: string, index: number): Promise<void>;
-    onExecute(): Promise<void>;
-    protected onDefineParameters(): void;
-}
-//# sourceMappingURL=action.dump.tile.d.ts.map

package/build/action.dump.tile.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"action.dump.tile.d.ts","sourceRoot":"","sources":["../src/action.dump.tile.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAe,MAAM,iBAAiB,CAAC;AACvD,OAAO,EAAE,iBAAiB,EAA2D,MAAM,4BAA4B,CAAC;AAoBxH,MAAM,WAAW,cAAc;IAC3B,IAAI,EAAE,SAAS,CAAC;IAChB,QAAQ,EAAE;QACN,IAAI,EAAE,SAAS,CAAC;QAChB,WAAW,EAAE,CAAC,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC;KAC7G,CAAC;IACF,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACvC;AAwBD,qBAAa,cAAe,SAAQ,iBAAiB;IACjD,OAAO,CAAC,IAAI,CAA2C;IACvD,OAAO,CAAC,UAAU,CAA4C;IAC9D,OAAO,CAAC,MAAM,CAA2C;IACzD,OAAO,CAAC,WAAW,CAAK;IACxB,OAAO,CAAC,MAAM,CAAc;;IAYtB,UAAU,CAAC,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAoCtE,SAAS,CAAC,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IA8BrE,SAAS,CAAC,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAwBrE,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;IAuDhC,SAAS,CAAC,kBAAkB,IAAI,IAAI;CAyBvC"}