@basemaps/cli-raster 8.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +70 -0
- package/build/bin.d.ts +1 -0
- package/build/bin.js +16 -0
- package/build/bin.js.map +1 -0
- package/build/cogify/__test__/covering.test.d.ts +1 -0
- package/build/cogify/__test__/covering.test.js +64 -0
- package/build/cogify/__test__/covering.test.js.map +1 -0
- package/build/cogify/__test__/extract.test.d.ts +1 -0
- package/build/cogify/__test__/extract.test.js +34 -0
- package/build/cogify/__test__/extract.test.js.map +1 -0
- package/build/cogify/cli/__test__/cli.cover.test.d.ts +1 -0
- package/build/cogify/cli/__test__/cli.cover.test.js +48 -0
- package/build/cogify/cli/__test__/cli.cover.test.js.map +1 -0
- package/build/cogify/cli/__test__/cli.topo.test.d.ts +1 -0
- package/build/cogify/cli/__test__/cli.topo.test.js +53 -0
- package/build/cogify/cli/__test__/cli.topo.test.js.map +1 -0
- package/build/cogify/cli/cli.cog.d.ts +55 -0
- package/build/cogify/cli/cli.cog.js +353 -0
- package/build/cogify/cli/cli.cog.js.map +1 -0
- package/build/cogify/cli/cli.cover.d.ts +30 -0
- package/build/cogify/cli/cli.cover.js +157 -0
- package/build/cogify/cli/cli.cover.js.map +1 -0
- package/build/cogify/cli/cli.topo.d.ts +50 -0
- package/build/cogify/cli/cli.topo.js +169 -0
- package/build/cogify/cli/cli.topo.js.map +1 -0
- package/build/cogify/cli/cli.validate.d.ts +15 -0
- package/build/cogify/cli/cli.validate.js +30 -0
- package/build/cogify/cli/cli.validate.js.map +1 -0
- package/build/cogify/cli.d.ts +121 -0
- package/build/cogify/cli.js +15 -0
- package/build/cogify/cli.js.map +1 -0
- package/build/cogify/covering/covering.d.ts +28 -0
- package/build/cogify/covering/covering.js +126 -0
- package/build/cogify/covering/covering.js.map +1 -0
- package/build/cogify/covering/cutline.d.ts +29 -0
- package/build/cogify/covering/cutline.js +109 -0
- package/build/cogify/covering/cutline.js.map +1 -0
- package/build/cogify/covering/tile.cover.d.ts +38 -0
- package/build/cogify/covering/tile.cover.js +198 -0
- package/build/cogify/covering/tile.cover.js.map +1 -0
- package/build/cogify/gdal/gdal.command.d.ts +27 -0
- package/build/cogify/gdal/gdal.command.js +175 -0
- package/build/cogify/gdal/gdal.command.js.map +1 -0
- package/build/cogify/gdal/gdal.runner.d.ts +42 -0
- package/build/cogify/gdal/gdal.runner.js +194 -0
- package/build/cogify/gdal/gdal.runner.js.map +1 -0
- package/build/cogify/stac.d.ts +138 -0
- package/build/cogify/stac.js +25 -0
- package/build/cogify/stac.js.map +1 -0
- package/build/cogify/topo/extract.d.ts +82 -0
- package/build/cogify/topo/extract.js +195 -0
- package/build/cogify/topo/extract.js.map +1 -0
- package/build/cogify/topo/slug.d.ts +10 -0
- package/build/cogify/topo/slug.js +18 -0
- package/build/cogify/topo/slug.js.map +1 -0
- package/build/cogify/topo/stac.creation.d.ts +47 -0
- package/build/cogify/topo/stac.creation.js +171 -0
- package/build/cogify/topo/stac.creation.js.map +1 -0
- package/build/download.d.ts +57 -0
- package/build/download.js +144 -0
- package/build/download.js.map +1 -0
- package/build/hash.stream.d.ts +20 -0
- package/build/hash.stream.js +62 -0
- package/build/hash.stream.js.map +1 -0
- package/build/index.d.ts +1 -0
- package/build/index.js +2 -0
- package/build/index.js.map +1 -0
- package/build/preset.d.ts +15 -0
- package/build/preset.js +66 -0
- package/build/preset.js.map +1 -0
- package/dist/index.cjs +85880 -0
- package/package.json +61 -0
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import { EpsgCode } from '@basemaps/geo';
|
|
2
|
+
const Slugs = {
|
|
3
|
+
[EpsgCode.Nztm2000]: 'new-zealand-mainland',
|
|
4
|
+
[EpsgCode.Citm2000]: 'chatham-islands',
|
|
5
|
+
};
|
|
6
|
+
/**
|
|
7
|
+
* Attempts to map the given EpsgCode enum to a slug.
|
|
8
|
+
*
|
|
9
|
+
* @param epsg: The EpsgCode enum to map to a slug
|
|
10
|
+
*
|
|
11
|
+
* @returns if succeeded, a slug string. Otherwise, null.
|
|
12
|
+
*/
|
|
13
|
+
export function mapEpsgToSlug(epsg, logger) {
|
|
14
|
+
const slug = Slugs[epsg];
|
|
15
|
+
logger?.info({ found: slug != null }, 'mapEpsgToSlug()');
|
|
16
|
+
return slug ?? null;
|
|
17
|
+
}
|
|
18
|
+
//# sourceMappingURL=slug.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"slug.js","sourceRoot":"","sources":["../../../src/cogify/topo/slug.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAGzC,MAAM,KAAK,GAAsC;IAC/C,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,sBAAsB;IAC3C,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,iBAAiB;CACvC,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,UAAU,aAAa,CAAC,IAAc,EAAE,MAAgB;IAC5D,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC;IAEzB,MAAM,EAAE,IAAI,CAAC,EAAE,KAAK,EAAE,IAAI,IAAI,IAAI,EAAE,EAAE,iBAAiB,CAAC,CAAC;IACzD,OAAO,IAAI,IAAI,IAAI,CAAC;AACtB,CAAC"}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import { Bounds, Epsg } from '@basemaps/geo';
|
|
2
|
+
import { LogType } from '@basemaps/shared';
|
|
3
|
+
import { CogifyStacCollection, TopoStacItem } from '../stac.js';
|
|
4
|
+
import { TiffItem } from './extract.js';
|
|
5
|
+
/**
|
|
6
|
+
* Creates a StacItem with additional properties specific to topo-raster maps.
|
|
7
|
+
*
|
|
8
|
+
* @param fileName: The map sheet's filename.
|
|
9
|
+
* @example "CJ10" or "CJ10_v1-00"
|
|
10
|
+
*
|
|
11
|
+
* @param tiffItem The TiffItem from which to create a TopoStacItem.
|
|
12
|
+
*
|
|
13
|
+
* @returns a TopoStacItem object.
|
|
14
|
+
*/
|
|
15
|
+
export declare function createBaseStacItem(fileName: string, tiffItem: TiffItem, logger?: LogType): TopoStacItem;
|
|
16
|
+
/**
|
|
17
|
+
* Creates StacItems from a list of TiffItems and a sublist of those identified as the latest versions by map code.
|
|
18
|
+
*
|
|
19
|
+
* @param scale: The scale of the imagery described by the TiffItems.
|
|
20
|
+
* @example topo250
|
|
21
|
+
*
|
|
22
|
+
* @param resolution: The resolution of the imagery described by the TiffItems.
|
|
23
|
+
* @example gridless_600dpi
|
|
24
|
+
*
|
|
25
|
+
* @param all: The list of TiffItems from which to create StacItems.
|
|
26
|
+
*
|
|
27
|
+
* @param latest: The sublist of TiffItems identifying the latest versions by map code, from which to also create StacItems.
|
|
28
|
+
*
|
|
29
|
+
* This function creates two directories:
|
|
30
|
+
* - StacItems that will live in the "topo[50|250]" directory
|
|
31
|
+
* - StacItems that will live in the "topo[50|250]_latest" directory
|
|
32
|
+
*
|
|
33
|
+
* All versions need a StacItem that will live in the topo[50/250] directory
|
|
34
|
+
* The latest versions needs a second StacItem that will live in the topo[50|250]_latest directory
|
|
35
|
+
*
|
|
36
|
+
* @returns An object containing the StacItems that will live in the topo[50/250] directory (all), and those that will live in
|
|
37
|
+
* the topo[50|250]_latest directory (latest).
|
|
38
|
+
*/
|
|
39
|
+
export declare function createStacItems(scale: string, resolution: string, all: TiffItem[], latest: Map<string, TiffItem>, logger?: LogType): {
|
|
40
|
+
all: TopoStacItem[];
|
|
41
|
+
latest: TopoStacItem[];
|
|
42
|
+
};
|
|
43
|
+
export declare function createStacCollection(title: string, linzSlug: string, epsgCode: Epsg, imageryBounds: Bounds, items: TopoStacItem[], logger?: LogType): CogifyStacCollection;
|
|
44
|
+
export declare function writeStacFiles(target: URL, items: TopoStacItem[], collection: CogifyStacCollection, logger: LogType): Promise<{
|
|
45
|
+
items: URL[];
|
|
46
|
+
collection: URL;
|
|
47
|
+
}>;
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
import { Projection } from '@basemaps/geo';
|
|
2
|
+
import { fsa } from '@basemaps/shared';
|
|
3
|
+
import { CliDate, CliId, CliInfo } from '@basemaps/shared/build/cli/info.js';
|
|
4
|
+
/**
|
|
5
|
+
* Creates a StacItem with additional properties specific to topo-raster maps.
|
|
6
|
+
*
|
|
7
|
+
* @param fileName: The map sheet's filename.
|
|
8
|
+
* @example "CJ10" or "CJ10_v1-00"
|
|
9
|
+
*
|
|
10
|
+
* @param tiffItem The TiffItem from which to create a TopoStacItem.
|
|
11
|
+
*
|
|
12
|
+
* @returns a TopoStacItem object.
|
|
13
|
+
*/
|
|
14
|
+
export function createBaseStacItem(fileName, tiffItem, logger) {
|
|
15
|
+
logger?.info({ fileName }, 'createBaseStacItem()');
|
|
16
|
+
const proj = Projection.get(tiffItem.epsg.code);
|
|
17
|
+
const feature = proj.boundsToGeoJsonFeature(tiffItem.bounds);
|
|
18
|
+
const bbox = proj.boundsToWgs84BoundingBox(tiffItem.bounds);
|
|
19
|
+
const sourceLink = {
|
|
20
|
+
rel: 'linz_basemaps:source',
|
|
21
|
+
href: tiffItem.source.href,
|
|
22
|
+
type: 'image/tiff; application=geotiff',
|
|
23
|
+
'linz_basemaps:source_width': tiffItem.size.width,
|
|
24
|
+
'linz_basemaps:source_height': tiffItem.size.height,
|
|
25
|
+
};
|
|
26
|
+
const item = {
|
|
27
|
+
type: 'Feature',
|
|
28
|
+
stac_version: '1.0.0',
|
|
29
|
+
id: fileName,
|
|
30
|
+
links: [
|
|
31
|
+
{ rel: 'self', href: `./${fileName}.json`, type: 'application/json' },
|
|
32
|
+
{ rel: 'collection', href: './collection.json', type: 'application/json' },
|
|
33
|
+
{ rel: 'parent', href: './collection.json', type: 'application/json' },
|
|
34
|
+
sourceLink,
|
|
35
|
+
],
|
|
36
|
+
assets: {},
|
|
37
|
+
stac_extensions: ['https://stac-extensions.github.io/file/v2.0.0/schema.json'],
|
|
38
|
+
properties: {
|
|
39
|
+
datetime: CliDate,
|
|
40
|
+
'linz:map_sheet': tiffItem.mapCode,
|
|
41
|
+
version: tiffItem.version.replace('-', '.'), // e.g. "v1-00" to "v1.00"
|
|
42
|
+
'proj:epsg': tiffItem.epsg.code,
|
|
43
|
+
'linz_basemaps:options': {
|
|
44
|
+
tileMatrix: tiffItem.tileMatrix.identifier,
|
|
45
|
+
sourceEpsg: tiffItem.epsg.code,
|
|
46
|
+
},
|
|
47
|
+
'linz_basemaps:generated': {
|
|
48
|
+
package: CliInfo.package,
|
|
49
|
+
hash: CliInfo.hash,
|
|
50
|
+
version: CliInfo.version,
|
|
51
|
+
datetime: CliDate,
|
|
52
|
+
},
|
|
53
|
+
},
|
|
54
|
+
geometry: feature.geometry,
|
|
55
|
+
bbox,
|
|
56
|
+
collection: CliId,
|
|
57
|
+
};
|
|
58
|
+
return item;
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Creates StacItems from a list of TiffItems and a sublist of those identified as the latest versions by map code.
|
|
62
|
+
*
|
|
63
|
+
* @param scale: The scale of the imagery described by the TiffItems.
|
|
64
|
+
* @example topo250
|
|
65
|
+
*
|
|
66
|
+
* @param resolution: The resolution of the imagery described by the TiffItems.
|
|
67
|
+
* @example gridless_600dpi
|
|
68
|
+
*
|
|
69
|
+
* @param all: The list of TiffItems from which to create StacItems.
|
|
70
|
+
*
|
|
71
|
+
* @param latest: The sublist of TiffItems identifying the latest versions by map code, from which to also create StacItems.
|
|
72
|
+
*
|
|
73
|
+
* This function creates two directories:
|
|
74
|
+
* - StacItems that will live in the "topo[50|250]" directory
|
|
75
|
+
* - StacItems that will live in the "topo[50|250]_latest" directory
|
|
76
|
+
*
|
|
77
|
+
* All versions need a StacItem that will live in the topo[50/250] directory
|
|
78
|
+
* The latest versions needs a second StacItem that will live in the topo[50|250]_latest directory
|
|
79
|
+
*
|
|
80
|
+
* @returns An object containing the StacItems that will live in the topo[50/250] directory (all), and those that will live in
|
|
81
|
+
* the topo[50|250]_latest directory (latest).
|
|
82
|
+
*/
|
|
83
|
+
export function createStacItems(scale, resolution, all, latest, logger) {
|
|
84
|
+
// create origin StacItem files
|
|
85
|
+
const allStacItems = all.map((item) => {
|
|
86
|
+
const latestTiff = latest.get(item.mapCode);
|
|
87
|
+
if (latestTiff == null)
|
|
88
|
+
throw new Error(`Failed to find latest item for map code '${item.mapCode}'`);
|
|
89
|
+
const originStacItem = createBaseStacItem(`${item.mapCode}_${item.version}`, item, logger);
|
|
90
|
+
// add link referencing the 'latest version' origin StacItem file that will live in the same directory
|
|
91
|
+
originStacItem.links.push({
|
|
92
|
+
href: `./${latestTiff.mapCode}_${latestTiff.version}.json`,
|
|
93
|
+
rel: 'latest-version',
|
|
94
|
+
type: 'application/json',
|
|
95
|
+
});
|
|
96
|
+
return originStacItem;
|
|
97
|
+
});
|
|
98
|
+
// create latest StacItem files
|
|
99
|
+
const latestStacItems = Array.from(latest.values()).map((item) => {
|
|
100
|
+
const latestStacItem = createBaseStacItem(item.mapCode, item, logger);
|
|
101
|
+
// add link referencing this StacItem's origin file that will live in the topo[50/250] directory
|
|
102
|
+
latestStacItem.links.push({
|
|
103
|
+
// directory into which we save this StacItem file: <target>/<scale>_latest/<resolution>/<espg>/[latest_stac_item]
|
|
104
|
+
// directory inside which we save this StacItem's origin file: <target>/<scale>/<resolution>/<espg>/[origin_stac_item]
|
|
105
|
+
//
|
|
106
|
+
// `../../../` takes us up to the <target> directory
|
|
107
|
+
href: `../../../${scale}/${resolution}/${item.epsg.code}/${item.mapCode}_${item.version}.json`,
|
|
108
|
+
rel: 'derived_from',
|
|
109
|
+
type: 'application/json',
|
|
110
|
+
});
|
|
111
|
+
return latestStacItem;
|
|
112
|
+
});
|
|
113
|
+
return { latest: latestStacItems, all: allStacItems };
|
|
114
|
+
}
|
|
115
|
+
export function createStacCollection(title, linzSlug, epsgCode, imageryBounds, items, logger) {
|
|
116
|
+
logger?.info({ items: items.length }, 'CreateStacCollection()');
|
|
117
|
+
const proj = Projection.get(epsgCode);
|
|
118
|
+
const bbox = proj.boundsToWgs84BoundingBox(imageryBounds);
|
|
119
|
+
const collection = {
|
|
120
|
+
type: 'Collection',
|
|
121
|
+
stac_version: '1.0.0',
|
|
122
|
+
id: CliId,
|
|
123
|
+
title,
|
|
124
|
+
description: 'Topographic maps of New Zealand',
|
|
125
|
+
license: 'CC-BY-4.0',
|
|
126
|
+
links: [
|
|
127
|
+
// TODO: We not have an ODR bucket for the linz-topographic yet.
|
|
128
|
+
// {
|
|
129
|
+
// rel: 'root',
|
|
130
|
+
// href: 'https://nz-imagery.s3.ap-southeast-2.amazonaws.com/catalog.json',
|
|
131
|
+
// type: 'application/json',
|
|
132
|
+
// },
|
|
133
|
+
{ rel: 'self', href: './collection.json', type: 'application/json' },
|
|
134
|
+
...items.map((item) => {
|
|
135
|
+
return {
|
|
136
|
+
href: `./${item.id}.json`,
|
|
137
|
+
rel: 'item',
|
|
138
|
+
type: 'application/json',
|
|
139
|
+
};
|
|
140
|
+
}),
|
|
141
|
+
],
|
|
142
|
+
providers: [{ name: 'Land Information New Zealand', roles: ['host', 'licensor', 'processor', 'producer'] }],
|
|
143
|
+
'linz:lifecycle': 'ongoing',
|
|
144
|
+
'linz:geospatial_category': 'topographic-maps',
|
|
145
|
+
'linz:region': 'new-zealand',
|
|
146
|
+
'linz:security_classification': 'unclassified',
|
|
147
|
+
'linz:slug': linzSlug,
|
|
148
|
+
extent: {
|
|
149
|
+
spatial: { bbox: [bbox] },
|
|
150
|
+
// Default the temporal time today if no times were found as it is required for STAC
|
|
151
|
+
temporal: { interval: [[CliDate, null]] },
|
|
152
|
+
},
|
|
153
|
+
stac_extensions: ['https://stac-extensions.github.io/file/v2.0.0/schema.json'],
|
|
154
|
+
};
|
|
155
|
+
return collection;
|
|
156
|
+
}
|
|
157
|
+
export async function writeStacFiles(target, items, collection, logger) {
|
|
158
|
+
// Create collection json for all topo50-latest items.
|
|
159
|
+
logger.info({ itemCount: items.length, collectionId: collection.id, target: target.href }, 'Stac:Output');
|
|
160
|
+
const itemPaths = [];
|
|
161
|
+
for (const item of items) {
|
|
162
|
+
const itemPath = new URL(`${item.id}.json`, target);
|
|
163
|
+
itemPaths.push(itemPath);
|
|
164
|
+
await fsa.write(itemPath, JSON.stringify(item, null, 2));
|
|
165
|
+
}
|
|
166
|
+
const collectionPath = new URL('collection.json', target);
|
|
167
|
+
logger.info({ collectionId: collection.id, target: collectionPath.href }, 'Stac:Output:Collection');
|
|
168
|
+
await fsa.write(collectionPath, JSON.stringify(collection, null, 2));
|
|
169
|
+
return { items: itemPaths, collection: collectionPath };
|
|
170
|
+
}
|
|
171
|
+
//# sourceMappingURL=stac.creation.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"stac.creation.js","sourceRoot":"","sources":["../../../src/cogify/topo/stac.creation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAgB,UAAU,EAAE,MAAM,eAAe,CAAC;AACzD,OAAO,EAAE,GAAG,EAAW,MAAM,kBAAkB,CAAC;AAChD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,oCAAoC,CAAC;AAM7E;;;;;;;;;GASG;AACH,MAAM,UAAU,kBAAkB,CAAC,QAAgB,EAAE,QAAkB,EAAE,MAAgB;IACvF,MAAM,EAAE,IAAI,CAAC,EAAE,QAAQ,EAAE,EAAE,sBAAsB,CAAC,CAAC;IAEnD,MAAM,IAAI,GAAG,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAChD,MAAM,OAAO,GAAG,IAAI,CAAC,sBAAsB,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC7D,MAAM,IAAI,GAAG,IAAI,CAAC,wBAAwB,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IAE5D,MAAM,UAAU,GAAqB;QACnC,GAAG,EAAE,sBAAsB;QAC3B,IAAI,EAAE,QAAQ,CAAC,MAAM,CAAC,IAAI;QAC1B,IAAI,EAAE,iCAAiC;QACvC,4BAA4B,EAAE,QAAQ,CAAC,IAAI,CAAC,KAAK;QACjD,6BAA6B,EAAE,QAAQ,CAAC,IAAI,CAAC,MAAM;KACpD,CAAC;IAEF,MAAM,IAAI,GAAiB;QACzB,IAAI,EAAE,SAAS;QACf,YAAY,EAAE,OAAO;QACrB,EAAE,EAAE,QAAQ;QACZ,KAAK,EAAE;YACL,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,QAAQ,OAAO,EAAE,IAAI,EAAE,kBAAkB,EAAE;YACrE,EAAE,GAAG,EAAE,YAAY,EAAE,IAAI,EAAE,mBAAmB,EAAE,IAAI,EAAE,kBAAkB,EAAE;YAC1E,EAAE,GAAG,EAAE,QAAQ,EAAE,IAAI,EAAE,mBAAmB,EAAE,IAAI,EAAE,kBAAkB,EAAE;YACtE,UAAU;SACX;QACD,MAAM,EAAE,EAAE;QACV,eAAe,EAAE,CAAC,2DAA2D,CAAC;QAC9E,UAAU,EAAE;YACV,QAAQ,EAAE,OAAO;YACjB,gBAAgB,EAAE,QAAQ,CAAC,OAAO;YAClC,OAAO,EAAE,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,0BAA0B;YACvE,WAAW,EAAE,QAAQ,CAAC,IAAI,CAAC,IAAI;YAC/B,uBAAuB,EAAE;gBACvB,UAAU,EAAE,QAAQ,CAAC,UAAU,CAAC,UAAU;gBAC1C,UAAU,EAAE,QAAQ,CAAC,IAAI,CAAC,IAAI;aAC/B;YACD,yBAAyB,EAAE;gBACzB,OAAO,EAAE,OAAO,CAAC,OAAO;gBACxB,IAAI,EAAE,OAAO,CAAC,IAAI;gBAClB,OAAO,EAAE,OAAO,CAAC,OAAO;gBACxB,QAAQ,EAAE,OAAO;aAClB;SACF;QACD,QAAQ,EAAE,OAAO,CAAC,QAA0B;QAC5C,IAAI;QACJ,UAAU,EAAE,KAAK;KAClB,CAAC;IAEF,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,MAAM,UAAU,eAAe,CAC7B,KAAa,EACb,UAAkB,EAClB,GAAe,EACf,MAA6B,EAC7B,MAAgB;IAEhB,+BAA+B;IAC/B,MAAM,YAAY,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACpC,MAAM,UAAU,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC5C,IAAI,UAAU,IAAI,IAAI;YAAE,MAAM,IAAI,KAAK,CAAC,4CAA4C,IAAI,CAAC,OAAO,GAAG,CAAC,CAAC;QAErG,MAAM,cAAc,GAAG,kBAAkB,CAAC,GAAG,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;QAE3F,sGAAsG;QACtG,cAAc,CAAC,KAAK,CAAC,IAAI,CAAC;YACxB,IAAI,EAAE,KAAK,UAAU,CAAC,OAAO,IAAI,UAAU,CAAC,OAAO,OAAO;YAC1D,GAAG,EAAE,gBAAgB;YACrB,IAAI,EAAE,kBAAkB;SACzB,CAAC,CAAC;QAEH,OAAO,cAAc,CAAC;IACxB,CAAC,CAAC,CAAC;IAEH,+BAA+B;IAC/B,MAAM,eAAe,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QAC/D,MAAM,cAAc,GAAG,kBAAkB,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;QAEtE,gGAAgG;QAChG,cAAc,CAAC,KAAK,CAAC,IAAI,CAAC;YACxB,kHAAkH;YAClH,sHAAsH;YACtH,EAAE;YACF,oDAAoD;YACpD,IAAI,EAAE,YAAY,KAAK,IAAI,UAAU,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,OAAO;YAC9F,GAAG,EAAE,cAAc;YACnB,IAAI,EAAE,kBAAkB;SACzB,CAAC,CAAC;QAEH,OAAO,cAAc,CAAC;IACxB,CAAC,CAAC,CAAC;IAEH,OAAO,EAAE,MAAM,EAAE,eAAe,EAAE,GAAG,EAAE,YAAY,EAAE,CAAC;AACxD,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,KAAa,EACb,QAAgB,EAChB,QAAc,EACd,aAAqB,EACrB,KAAqB,EACrB,MAAgB;IAEhB,MAAM,EAAE,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,EAAE,wBAAwB,CAAC,CAAC;IAEhE,MAAM,IAAI,GAAG,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IACtC,MAAM,IAAI,GAAG,IAAI,CAAC,wBAAwB,CAAC,aAAa,CAAC,CAAC;IAE1D,MAAM,UAAU,GAAyB;QACvC,IAAI,EAAE,YAAY;QAClB,YAAY,EAAE,OAAO;QACrB,EAAE,EAAE,KAAK;QACT,KAAK;QACL,WAAW,EAAE,iCAAiC;QAC9C,OAAO,EAAE,WAAW;QACpB,KAAK,EAAE;YACL,gEAAgE;YAChE,IAAI;YACJ,iBAAiB;YACjB,6EAA6E;YAC7E,8BAA8B;YAC9B,KAAK;YACL,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,mBAAmB,EAAE,IAAI,EAAE,kBAAkB,EAAE;YACpE,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;gBACpB,OAAO;oBACL,IAAI,EAAE,KAAK,IAAI,CAAC,EAAE,OAAO;oBACzB,GAAG,EAAE,MAAM;oBACX,IAAI,EAAE,kBAAkB;iBACzB,CAAC;YACJ,CAAC,CAAC;SACH;QACD,SAAS,EAAE,CAAC,EAAE,IAAI,EAAE,8BAA8B,EAAE,KAAK,EAAE,CAAC,MAAM,EAAE,UAAU,EAAE,WAAW,EAAE,UAAU,CAAC,EAAE,CAAC;QAC3G,gBAAgB,EAAE,SAAS;QAC3B,0BAA0B,EAAE,kBAAkB;QAC9C,aAAa,EAAE,aAAa;QAC5B,8BAA8B,EAAE,cAAc;QAC9C,WAAW,EAAE,QAAQ;QACrB,MAAM,EAAE;YACN,OAAO,EAAE,EAAE,IAAI,EAAE,CAAC,IAAI,CAAC,EAAE;YACzB,oFAAoF;YACpF,QAAQ,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,EAAE;SAC1C;QACD,eAAe,EAAE,CAAC,2DAA2D,CAAC;KAC/E,CAAC;IAEF,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,MAAW,EACX,KAAqB,EACrB,UAAgC,EAChC,MAAe;IAEf,sDAAsD;IACtD,MAAM,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,KAAK,CAAC,MAAM,EAAE,YAAY,EAAE,UAAU,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,CAAC,IAAI,EAAE,EAAE,aAAa,CAAC,CAAC;IAE1G,MAAM,SAAS,GAAU,EAAE,CAAC;IAE5B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,GAAG,IAAI,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;QACpD,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAEzB,MAAM,GAAG,CAAC,KAAK,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAC3D,CAAC;IAED,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,iBAAiB,EAAE,MAAM,CAAC,CAAC;IAC1D,MAAM,CAAC,IAAI,CAAC,EAAE,YAAY,EAAE,UAAU,CAAC,EAAE,EAAE,MAAM,EAAE,cAAc,CAAC,IAAI,EAAE,EAAE,wBAAwB,CAAC,CAAC;IAEpG,MAAM,GAAG,CAAC,KAAK,CAAC,cAAc,EAAE,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAErE,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,cAAc,EAAE,CAAC;AAC1D,CAAC"}
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import { LogType } from '@basemaps/shared';
|
|
2
|
+
import { LimitFunction } from 'p-limit';
|
|
3
|
+
export interface SourceFile {
|
|
4
|
+
/** Source location */
|
|
5
|
+
url: URL;
|
|
6
|
+
/** List of items that need the source */
|
|
7
|
+
items: string[];
|
|
8
|
+
/** Location to the local file */
|
|
9
|
+
asset?: Promise<URL>;
|
|
10
|
+
/** Number of bytes in the file */
|
|
11
|
+
size?: number;
|
|
12
|
+
/** multihash of the file if it exists */
|
|
13
|
+
hash?: string;
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Store assets used for creating COGs, (Tiffs, Cutlines) etc in a temporary location
|
|
17
|
+
*
|
|
18
|
+
* While also sharing them between multiple COG creation processes saving redownloading the files when they are needed
|
|
19
|
+
* for multiple output COGS
|
|
20
|
+
*/
|
|
21
|
+
export declare class SourceDownloader {
|
|
22
|
+
/** Limit the concurrency of the downloads @default 10 */
|
|
23
|
+
Q: LimitFunction;
|
|
24
|
+
/** Mapping of source location to target location */
|
|
25
|
+
items: Map<string, SourceFile>;
|
|
26
|
+
/** Local cache location */
|
|
27
|
+
cachePath: URL;
|
|
28
|
+
/**
|
|
29
|
+
* Unique set of hosts eg "s3://linz-basemaps",
|
|
30
|
+
* This has to be strings as each instance of a URL is unique.
|
|
31
|
+
*/
|
|
32
|
+
hosts: Map<string, URL>;
|
|
33
|
+
constructor(cachePath: URL);
|
|
34
|
+
/** Register that a item needs a specific source file for processing */
|
|
35
|
+
register(url: URL, itemId: string): void;
|
|
36
|
+
/** Once a item is done with a asset clean it up if no other items need it */
|
|
37
|
+
done(url: URL, itemId: string, logger: LogType): Promise<boolean>;
|
|
38
|
+
/**
|
|
39
|
+
* Get the local location of a remote asset
|
|
40
|
+
*
|
|
41
|
+
* This will start downloading the file into the temporary location
|
|
42
|
+
*/
|
|
43
|
+
get(url: URL, logger: LogType): Promise<URL>;
|
|
44
|
+
_checked: Map<string, Promise<unknown>>;
|
|
45
|
+
/** Validate that the host can be read from before attempting to stream files */
|
|
46
|
+
_checkHost(url: URL): Promise<unknown>;
|
|
47
|
+
/**
|
|
48
|
+
* Download a file into a hashed target location, using @see {this.Q} to manage concurrency of downloads
|
|
49
|
+
*
|
|
50
|
+
* Pattern `/${this.cachePath}/source/${sha256base58(href)}`
|
|
51
|
+
* @example
|
|
52
|
+
* ```typescript
|
|
53
|
+
* "/tmp/01H01JMN98AC36VVPNCYGQ8D5X/source/GnBQfpFe8QBTzJgHP9fXABBZR9xVEmPo87Zcec9n177S.tiff"
|
|
54
|
+
* ```
|
|
55
|
+
*/
|
|
56
|
+
_downloadFile(asset: SourceFile, logger: LogType): Promise<URL>;
|
|
57
|
+
}
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
import { extname } from 'node:path';
|
|
2
|
+
import { sha256base58 } from '@basemaps/config';
|
|
3
|
+
import { fsa } from '@basemaps/shared';
|
|
4
|
+
import pLimit from 'p-limit';
|
|
5
|
+
import { HashTransform } from './hash.stream.js';
|
|
6
|
+
/**
|
|
7
|
+
* Store assets used for creating COGs, (Tiffs, Cutlines) etc in a temporary location
|
|
8
|
+
*
|
|
9
|
+
* While also sharing them between multiple COG creation processes saving redownloading the files when they are needed
|
|
10
|
+
* for multiple output COGS
|
|
11
|
+
*/
|
|
12
|
+
export class SourceDownloader {
|
|
13
|
+
constructor(cachePath) {
|
|
14
|
+
/** Limit the concurrency of the downloads @default 10 */
|
|
15
|
+
Object.defineProperty(this, "Q", {
|
|
16
|
+
enumerable: true,
|
|
17
|
+
configurable: true,
|
|
18
|
+
writable: true,
|
|
19
|
+
value: void 0
|
|
20
|
+
});
|
|
21
|
+
/** Mapping of source location to target location */
|
|
22
|
+
Object.defineProperty(this, "items", {
|
|
23
|
+
enumerable: true,
|
|
24
|
+
configurable: true,
|
|
25
|
+
writable: true,
|
|
26
|
+
value: void 0
|
|
27
|
+
});
|
|
28
|
+
/** Local cache location */
|
|
29
|
+
Object.defineProperty(this, "cachePath", {
|
|
30
|
+
enumerable: true,
|
|
31
|
+
configurable: true,
|
|
32
|
+
writable: true,
|
|
33
|
+
value: void 0
|
|
34
|
+
});
|
|
35
|
+
/**
|
|
36
|
+
* Unique set of hosts eg "s3://linz-basemaps",
|
|
37
|
+
* This has to be strings as each instance of a URL is unique.
|
|
38
|
+
*/
|
|
39
|
+
Object.defineProperty(this, "hosts", {
|
|
40
|
+
enumerable: true,
|
|
41
|
+
configurable: true,
|
|
42
|
+
writable: true,
|
|
43
|
+
value: new Map()
|
|
44
|
+
});
|
|
45
|
+
Object.defineProperty(this, "_checked", {
|
|
46
|
+
enumerable: true,
|
|
47
|
+
configurable: true,
|
|
48
|
+
writable: true,
|
|
49
|
+
value: new Map()
|
|
50
|
+
});
|
|
51
|
+
this.cachePath = cachePath;
|
|
52
|
+
this.Q = pLimit(10);
|
|
53
|
+
this.items = new Map();
|
|
54
|
+
}
|
|
55
|
+
/** Register that a item needs a specific source file for processing */
|
|
56
|
+
register(url, itemId) {
|
|
57
|
+
const assets = this.items.get(url.href) ?? { items: [], url };
|
|
58
|
+
assets.items.push(itemId);
|
|
59
|
+
this.items.set(url.href, assets);
|
|
60
|
+
if (url.protocol !== 'file:') {
|
|
61
|
+
const host = new URL(url.href);
|
|
62
|
+
host.pathname = '/'; // remove the file path;
|
|
63
|
+
this.hosts.set(host.href, url);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
/** Once a item is done with a asset clean it up if no other items need it */
|
|
67
|
+
async done(url, itemId, logger) {
|
|
68
|
+
const asset = this.items.get(url.href);
|
|
69
|
+
if (asset == null)
|
|
70
|
+
throw new Error('Asset was not registered to be downloaded: ' + url.href);
|
|
71
|
+
// Remove the itemId
|
|
72
|
+
asset.items = asset.items.filter((f) => f !== itemId);
|
|
73
|
+
if (asset.items.length > 0)
|
|
74
|
+
return false;
|
|
75
|
+
if (asset.asset == null)
|
|
76
|
+
return false;
|
|
77
|
+
// No more items need this asset, clean it up
|
|
78
|
+
const targetFile = await asset.asset;
|
|
79
|
+
logger.debug({ source: asset.url, target: targetFile }, 'Cog:Source:Cleanup');
|
|
80
|
+
await fsa.delete(targetFile);
|
|
81
|
+
return true;
|
|
82
|
+
}
|
|
83
|
+
/**
|
|
84
|
+
* Get the local location of a remote asset
|
|
85
|
+
*
|
|
86
|
+
* This will start downloading the file into the temporary location
|
|
87
|
+
*/
|
|
88
|
+
get(url, logger) {
|
|
89
|
+
const asset = this.items.get(url.href);
|
|
90
|
+
if (asset == null)
|
|
91
|
+
throw new Error('Asset was not registered to be downloaded: ' + url.href);
|
|
92
|
+
if (asset.asset)
|
|
93
|
+
return asset.asset;
|
|
94
|
+
asset.asset = this._downloadFile(asset, logger);
|
|
95
|
+
return asset.asset;
|
|
96
|
+
}
|
|
97
|
+
/** Validate that the host can be read from before attempting to stream files */
|
|
98
|
+
_checkHost(url) {
|
|
99
|
+
if (url.protocol === 'file:')
|
|
100
|
+
return Promise.resolve();
|
|
101
|
+
const host = url.hostname;
|
|
102
|
+
let ret = this._checked.get(host);
|
|
103
|
+
if (ret)
|
|
104
|
+
return ret;
|
|
105
|
+
ret = fsa.head(url);
|
|
106
|
+
this._checked.set(host, ret);
|
|
107
|
+
return ret;
|
|
108
|
+
}
|
|
109
|
+
/**
|
|
110
|
+
* Download a file into a hashed target location, using @see {this.Q} to manage concurrency of downloads
|
|
111
|
+
*
|
|
112
|
+
* Pattern `/${this.cachePath}/source/${sha256base58(href)}`
|
|
113
|
+
* @example
|
|
114
|
+
* ```typescript
|
|
115
|
+
* "/tmp/01H01JMN98AC36VVPNCYGQ8D5X/source/GnBQfpFe8QBTzJgHP9fXABBZR9xVEmPo87Zcec9n177S.tiff"
|
|
116
|
+
* ```
|
|
117
|
+
*/
|
|
118
|
+
_downloadFile(asset, logger) {
|
|
119
|
+
return this.Q(async () => {
|
|
120
|
+
const newFileName = sha256base58(Buffer.from(asset.url.href)) + extname(asset.url.href);
|
|
121
|
+
const targetFile = new URL(`source/${newFileName}`, this.cachePath);
|
|
122
|
+
await this._checkHost(asset.url);
|
|
123
|
+
logger.trace({ source: asset.url, target: targetFile }, 'Cog:Source:Download');
|
|
124
|
+
const hashStream = fsa.readStream(asset.url).pipe(new HashTransform('sha256'));
|
|
125
|
+
const startTime = performance.now();
|
|
126
|
+
await fsa.write(targetFile, hashStream);
|
|
127
|
+
const duration = performance.now() - startTime;
|
|
128
|
+
asset.size = hashStream.size;
|
|
129
|
+
asset.hash = hashStream.multihash;
|
|
130
|
+
const stat = await fsa.head(targetFile);
|
|
131
|
+
logger.info({
|
|
132
|
+
source: asset.url,
|
|
133
|
+
target: targetFile,
|
|
134
|
+
items: asset.items,
|
|
135
|
+
size: stat?.size,
|
|
136
|
+
hash: asset.hash,
|
|
137
|
+
assetSize: asset.size,
|
|
138
|
+
duration,
|
|
139
|
+
}, 'Cog:Source:Download:Done');
|
|
140
|
+
return targetFile;
|
|
141
|
+
});
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
//# sourceMappingURL=download.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"download.js","sourceRoot":"","sources":["../src/download.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAChD,OAAO,EAAE,GAAG,EAAW,MAAM,kBAAkB,CAAC;AAChD,OAAO,MAAyB,MAAM,SAAS,CAAC;AAEhD,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAcjD;;;;;GAKG;AACH,MAAM,OAAO,gBAAgB;IAa3B,YAAY,SAAc;QAZ1B,yDAAyD;QACzD;;;;;WAAiB;QACjB,oDAAoD;QACpD;;;;;WAA+B;QAC/B,2BAA2B;QAC3B;;;;;WAAe;QAEf;;;WAGG;QACH;;;;mBAA0B,IAAI,GAAG,EAAE;WAAC;QAiDpC;;;;mBAAW,IAAI,GAAG,EAA4B;WAAC;QA/C7C,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,CAAC,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC;QACpB,IAAI,CAAC,KAAK,GAAG,IAAI,GAAG,EAAsB,CAAC;IAC7C,CAAC;IAED,uEAAuE;IACvE,QAAQ,CAAC,GAAQ,EAAE,MAAc;QAC/B,MAAM,MAAM,GAAe,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC;QAC1E,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC1B,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACjC,IAAI,GAAG,CAAC,QAAQ,KAAK,OAAO,EAAE,CAAC;YAC7B,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAC/B,IAAI,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,wBAAwB;YAC7C,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QACjC,CAAC;IACH,CAAC;IAED,6EAA6E;IAC7E,KAAK,CAAC,IAAI,CAAC,GAAQ,EAAE,MAAc,EAAE,MAAe;QAClD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACvC,IAAI,KAAK,IAAI,IAAI;YAAE,MAAM,IAAI,KAAK,CAAC,6CAA6C,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC;QAE7F,oBAAoB;QACpB,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC;QACtD,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC;YAAE,OAAO,KAAK,CAAC;QACzC,IAAI,KAAK,CAAC,KAAK,IAAI,IAAI;YAAE,OAAO,KAAK,CAAC;QACtC,6CAA6C;QAC7C,MAAM,UAAU,GAAG,MAAM,KAAK,CAAC,KAAK,CAAC;QACrC,MAAM,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,EAAE,oBAAoB,CAAC,CAAC;QAC9E,MAAM,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QAC7B,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;;OAIG;IACH,GAAG,CAAC,GAAQ,EAAE,MAAe;QAC3B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACvC,IAAI,KAAK,IAAI,IAAI;YAAE,MAAM,IAAI,KAAK,CAAC,6CAA6C,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC;QAC7F,IAAI,KAAK,CAAC,KAAK;YAAE,OAAO,KAAK,CAAC,KAAK,CAAC;QAEpC,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QAChD,OAAO,KAAK,CAAC,KAAK,CAAC;IACrB,CAAC;IAGD,gFAAgF;IAChF,UAAU,CAAC,GAAQ;QACjB,IAAI,GAAG,CAAC,QAAQ,KAAK,OAAO;YAAE,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;QACvD,MAAM,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAC;QAC1B,IAAI,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAClC,IAAI,GAAG;YAAE,OAAO,GAAG,CAAC;QACpB,GAAG,GAAG,GAAG,CAAC,IAAI,CAAC,GAAG,CAAqB,CAAC;QACxC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAC7B,OAAO,GAAG,CAAC;IACb,CAAC;IAED;;;;;;;;OAQG;IACH,aAAa,CAAC,KAAiB,EAAE,MAAe;QAC9C,OAAO,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE;YACvB,MAAM,WAAW,GAAG,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACxF,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,UAAU,WAAW,EAAE,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;YAEpE,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACjC,MAAM,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,EAAE,qBAAqB,CAAC,CAAC;YAC/E,MAAM,UAAU,GAAG,GAAG,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,aAAa,CAAC,QAAQ,CAAC,CAAC,CAAC;YAC/E,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;YACpC,MAAM,GAAG,CAAC,KAAK,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;YACxC,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;YAE/C,KAAK,CAAC,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC;YAC7B,KAAK,CAAC,IAAI,GAAG,UAAU,CAAC,SAAS,CAAC;YAClC,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YACxC,MAAM,CAAC,IAAI,CACT;gBACE,MAAM,EAAE,KAAK,CAAC,GAAG;gBACjB,MAAM,EAAE,UAAU;gBAClB,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,IAAI,EAAE,IAAI,EAAE,IAAI;gBAChB,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,SAAS,EAAE,KAAK,CAAC,IAAI;gBACrB,QAAQ;aACT,EACD,0BAA0B,CAC3B,CAAC;YACF,OAAO,UAAU,CAAC;QACpB,CAAC,CAAC,CAAC;IACL,CAAC;CACF"}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { Transform, TransformCallback } from 'stream';
|
|
2
|
+
export declare class HashTransform extends Transform {
|
|
3
|
+
/** hash function in use */
|
|
4
|
+
private hash;
|
|
5
|
+
/** type of hash used @example "sha256" */
|
|
6
|
+
hashType: string;
|
|
7
|
+
/** Number of bytes processed */
|
|
8
|
+
size: number;
|
|
9
|
+
/** digest of hash */
|
|
10
|
+
private _digestHex?;
|
|
11
|
+
constructor(hashType: string);
|
|
12
|
+
_transform(chunk: Buffer, _encoding: BufferEncoding, callback: TransformCallback): void;
|
|
13
|
+
/**
|
|
14
|
+
* Digest the hash into a multihash
|
|
15
|
+
*
|
|
16
|
+
* once digested the hash cannot be updated.
|
|
17
|
+
*/
|
|
18
|
+
get multihash(): string;
|
|
19
|
+
get digestHex(): string;
|
|
20
|
+
}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import { createHash } from 'crypto';
|
|
2
|
+
import { Transform } from 'stream';
|
|
3
|
+
export class HashTransform extends Transform {
|
|
4
|
+
constructor(hashType) {
|
|
5
|
+
super();
|
|
6
|
+
/** hash function in use */
|
|
7
|
+
Object.defineProperty(this, "hash", {
|
|
8
|
+
enumerable: true,
|
|
9
|
+
configurable: true,
|
|
10
|
+
writable: true,
|
|
11
|
+
value: void 0
|
|
12
|
+
});
|
|
13
|
+
/** type of hash used @example "sha256" */
|
|
14
|
+
Object.defineProperty(this, "hashType", {
|
|
15
|
+
enumerable: true,
|
|
16
|
+
configurable: true,
|
|
17
|
+
writable: true,
|
|
18
|
+
value: void 0
|
|
19
|
+
});
|
|
20
|
+
/** Number of bytes processed */
|
|
21
|
+
Object.defineProperty(this, "size", {
|
|
22
|
+
enumerable: true,
|
|
23
|
+
configurable: true,
|
|
24
|
+
writable: true,
|
|
25
|
+
value: 0
|
|
26
|
+
});
|
|
27
|
+
/** digest of hash */
|
|
28
|
+
Object.defineProperty(this, "_digestHex", {
|
|
29
|
+
enumerable: true,
|
|
30
|
+
configurable: true,
|
|
31
|
+
writable: true,
|
|
32
|
+
value: void 0
|
|
33
|
+
});
|
|
34
|
+
this.hashType = hashType;
|
|
35
|
+
this.hash = createHash(hashType);
|
|
36
|
+
}
|
|
37
|
+
_transform(chunk, _encoding, callback) {
|
|
38
|
+
// Only update the hash if it has not been digested
|
|
39
|
+
if (this._digestHex)
|
|
40
|
+
return callback(new Error(`Conflict: Hash has already been digested`));
|
|
41
|
+
this.hash.update(chunk);
|
|
42
|
+
this.size += chunk.byteLength ?? chunk.length;
|
|
43
|
+
callback(null, chunk);
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Digest the hash into a multihash
|
|
47
|
+
*
|
|
48
|
+
* once digested the hash cannot be updated.
|
|
49
|
+
*/
|
|
50
|
+
get multihash() {
|
|
51
|
+
if (this.hashType !== 'sha256')
|
|
52
|
+
throw new Error('Invalid hashType requires "sha256" ' + this.hashType);
|
|
53
|
+
return `1220` + this.digestHex;
|
|
54
|
+
}
|
|
55
|
+
get digestHex() {
|
|
56
|
+
if (this._digestHex)
|
|
57
|
+
return this._digestHex;
|
|
58
|
+
this._digestHex = this.hash.digest('hex');
|
|
59
|
+
return this._digestHex;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
//# sourceMappingURL=hash.stream.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"hash.stream.js","sourceRoot":"","sources":["../src/hash.stream.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAQ,MAAM,QAAQ,CAAC;AAC1C,OAAO,EAAE,SAAS,EAAqB,MAAM,QAAQ,CAAC;AAEtD,MAAM,OAAO,aAAc,SAAQ,SAAS;IAU1C,YAAY,QAAgB;QAC1B,KAAK,EAAE,CAAC;QAVV,2BAA2B;QACnB;;;;;WAAW;QACnB,0CAA0C;QAC1C;;;;;WAAiB;QACjB,gCAAgC;QAChC;;;;mBAAO,CAAC;WAAC;QACT,qBAAqB;QACb;;;;;WAAoB;QAI1B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;IACnC,CAAC;IAEQ,UAAU,CAAC,KAAa,EAAE,SAAyB,EAAE,QAA2B;QACvF,mDAAmD;QACnD,IAAI,IAAI,CAAC,UAAU;YAAE,OAAO,QAAQ,CAAC,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC,CAAC;QAE5F,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACxB,IAAI,CAAC,IAAI,IAAI,KAAK,CAAC,UAAU,IAAI,KAAK,CAAC,MAAM,CAAC;QAC9C,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IACxB,CAAC;IAED;;;;OAIG;IACH,IAAI,SAAS;QACX,IAAI,IAAI,CAAC,QAAQ,KAAK,QAAQ;YAAE,MAAM,IAAI,KAAK,CAAC,qCAAqC,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC;QACvG,OAAO,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC;IACjC,CAAC;IAED,IAAI,SAAS;QACX,IAAI,IAAI,CAAC,UAAU;YAAE,OAAO,IAAI,CAAC,UAAU,CAAC;QAC5C,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC1C,OAAO,IAAI,CAAC,UAAU,CAAC;IACzB,CAAC;CACF"}
|
package/build/index.d.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export { CogifyCli } from './cogify/cli.js';
|
package/build/index.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC"}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { CogifyCreationOptions } from './cogify/stac.js';
|
|
2
|
+
export declare const CogifyDefaults: {
|
|
3
|
+
readonly compression: "webp";
|
|
4
|
+
readonly blockSize: 512;
|
|
5
|
+
readonly quality: 90;
|
|
6
|
+
readonly warpResampling: "bilinear";
|
|
7
|
+
readonly overviewResampling: "lanczos";
|
|
8
|
+
};
|
|
9
|
+
export interface Preset {
|
|
10
|
+
name: string;
|
|
11
|
+
options: Partial<CogifyCreationOptions>;
|
|
12
|
+
}
|
|
13
|
+
export declare const Presets: {
|
|
14
|
+
[x: string]: Preset;
|
|
15
|
+
};
|
package/build/preset.js
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
export const CogifyDefaults = {
|
|
2
|
+
compression: 'webp',
|
|
3
|
+
blockSize: 512,
|
|
4
|
+
quality: 90,
|
|
5
|
+
warpResampling: 'bilinear',
|
|
6
|
+
overviewResampling: 'lanczos',
|
|
7
|
+
};
|
|
8
|
+
const webP = {
|
|
9
|
+
name: 'webp',
|
|
10
|
+
options: {
|
|
11
|
+
blockSize: CogifyDefaults.blockSize,
|
|
12
|
+
compression: CogifyDefaults.compression,
|
|
13
|
+
quality: CogifyDefaults.quality,
|
|
14
|
+
warpResampling: CogifyDefaults.warpResampling,
|
|
15
|
+
overviewResampling: CogifyDefaults.overviewResampling,
|
|
16
|
+
},
|
|
17
|
+
};
|
|
18
|
+
const webP80 = {
|
|
19
|
+
name: 'webp_80',
|
|
20
|
+
options: {
|
|
21
|
+
blockSize: CogifyDefaults.blockSize,
|
|
22
|
+
compression: CogifyDefaults.compression,
|
|
23
|
+
quality: 80,
|
|
24
|
+
warpResampling: CogifyDefaults.warpResampling,
|
|
25
|
+
overviewResampling: CogifyDefaults.overviewResampling,
|
|
26
|
+
},
|
|
27
|
+
};
|
|
28
|
+
const lerc1mm = {
|
|
29
|
+
name: 'lerc_1mm',
|
|
30
|
+
options: {
|
|
31
|
+
blockSize: 512,
|
|
32
|
+
compression: 'lerc',
|
|
33
|
+
maxZError: 0.001,
|
|
34
|
+
maxZErrorOverview: 0.01,
|
|
35
|
+
warpResampling: 'bilinear',
|
|
36
|
+
overviewResampling: 'bilinear',
|
|
37
|
+
},
|
|
38
|
+
};
|
|
39
|
+
const lerc10mm = {
|
|
40
|
+
name: 'lerc_10mm',
|
|
41
|
+
options: {
|
|
42
|
+
blockSize: 512,
|
|
43
|
+
compression: 'lerc',
|
|
44
|
+
maxZError: 0.01,
|
|
45
|
+
maxZErrorOverview: 0.02,
|
|
46
|
+
warpResampling: 'bilinear',
|
|
47
|
+
overviewResampling: 'bilinear',
|
|
48
|
+
},
|
|
49
|
+
};
|
|
50
|
+
const lzw = {
|
|
51
|
+
name: 'lzw',
|
|
52
|
+
options: {
|
|
53
|
+
blockSize: 512,
|
|
54
|
+
compression: 'lzw',
|
|
55
|
+
warpResampling: 'bilinear',
|
|
56
|
+
overviewResampling: 'bilinear',
|
|
57
|
+
},
|
|
58
|
+
};
|
|
59
|
+
export const Presets = {
|
|
60
|
+
[webP.name]: webP,
|
|
61
|
+
[webP80.name]: webP80,
|
|
62
|
+
[lerc1mm.name]: lerc1mm,
|
|
63
|
+
[lerc10mm.name]: lerc10mm,
|
|
64
|
+
[lzw.name]: lzw,
|
|
65
|
+
};
|
|
66
|
+
//# sourceMappingURL=preset.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"preset.js","sourceRoot":"","sources":["../src/preset.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,MAAM,cAAc,GAAG;IAC5B,WAAW,EAAE,MAAM;IACnB,SAAS,EAAE,GAAG;IACd,OAAO,EAAE,EAAE;IACX,cAAc,EAAE,UAAU;IAC1B,kBAAkB,EAAE,SAAS;CACrB,CAAC;AAOX,MAAM,IAAI,GAAW;IACnB,IAAI,EAAE,MAAM;IACZ,OAAO,EAAE;QACP,SAAS,EAAE,cAAc,CAAC,SAAS;QACnC,WAAW,EAAE,cAAc,CAAC,WAAW;QACvC,OAAO,EAAE,cAAc,CAAC,OAAO;QAC/B,cAAc,EAAE,cAAc,CAAC,cAAc;QAC7C,kBAAkB,EAAE,cAAc,CAAC,kBAAkB;KACtD;CACF,CAAC;AAEF,MAAM,MAAM,GAAW;IACrB,IAAI,EAAE,SAAS;IACf,OAAO,EAAE;QACP,SAAS,EAAE,cAAc,CAAC,SAAS;QACnC,WAAW,EAAE,cAAc,CAAC,WAAW;QACvC,OAAO,EAAE,EAAE;QACX,cAAc,EAAE,cAAc,CAAC,cAAc;QAC7C,kBAAkB,EAAE,cAAc,CAAC,kBAAkB;KACtD;CACF,CAAC;AAEF,MAAM,OAAO,GAAW;IACtB,IAAI,EAAE,UAAU;IAChB,OAAO,EAAE;QACP,SAAS,EAAE,GAAG;QACd,WAAW,EAAE,MAAM;QACnB,SAAS,EAAE,KAAK;QAChB,iBAAiB,EAAE,IAAI;QACvB,cAAc,EAAE,UAAU;QAC1B,kBAAkB,EAAE,UAAU;KAC/B;CACF,CAAC;AAEF,MAAM,QAAQ,GAAW;IACvB,IAAI,EAAE,WAAW;IACjB,OAAO,EAAE;QACP,SAAS,EAAE,GAAG;QACd,WAAW,EAAE,MAAM;QACnB,SAAS,EAAE,IAAI;QACf,iBAAiB,EAAE,IAAI;QACvB,cAAc,EAAE,UAAU;QAC1B,kBAAkB,EAAE,UAAU;KAC/B;CACF,CAAC;AAEF,MAAM,GAAG,GAAW;IAClB,IAAI,EAAE,KAAK;IACX,OAAO,EAAE;QACP,SAAS,EAAE,GAAG;QACd,WAAW,EAAE,KAAK;QAClB,cAAc,EAAE,UAAU;QAC1B,kBAAkB,EAAE,UAAU;KAC/B;CACF,CAAC;AAEF,MAAM,CAAC,MAAM,OAAO,GAAG;IACrB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI;IACjB,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM;IACrB,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,OAAO;IACvB,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,QAAQ;IACzB,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,GAAG;CAChB,CAAC"}
|