@basemaps/lambda-tiler 6.25.0 → 6.28.0

package/build/import/make.cog.d.ts

@@ -1,5 +0,0 @@
-import { JobCreationContext } from '@basemaps/cli/build/cog/cog.stac.job';
-import { TileMatrixSet } from '@basemaps/geo';
-import { RoleConfig } from './imagery.find.js';
-export declare function getJobCreationContext(path: string, tileMatrix: TileMatrixSet, role: RoleConfig, files: string[]): Promise<JobCreationContext>;
-//# sourceMappingURL=make.cog.d.ts.map

package/build/import/make.cog.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"make.cog.d.ts","sourceRoot":"","sources":["../../src/import/make.cog.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,sCAAsC,CAAC;AAC1E,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAE9C,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAE/C,wBAAsB,qBAAqB,CACzC,IAAI,EAAE,MAAM,EACZ,UAAU,EAAE,aAAa,EACzB,IAAI,EAAE,UAAU,EAChB,KAAK,EAAE,MAAM,EAAE,GACd,OAAO,CAAC,kBAAkB,CAAC,CAkB7B"}

package/build/import/make.cog.js

@@ -1,21 +0,0 @@
-import { Env } from '@basemaps/shared';
-export async function getJobCreationContext(path, tileMatrix, role, files) {
-    const bucket = Env.get(Env.ImportImageryBucket);
-    if (bucket == null)
-        throw new Error('Output AWS s3 bucket Not Found.');
-    const ctx = {
-        override: {
-            projection: tileMatrix.projection,
-            resampling: {
-                warp: 'bilinear',
-                overview: 'lanczos',
-            },
-        },
-        outputLocation: { type: 's3', path: `s3://${bucket}` },
-        sourceLocation: { type: 's3', path, ...role, files: files },
-        batch: true,
-        tileMatrix,
-        oneCogCovering: false,
-    };
-    return ctx;
-}

package/build/routes/__test__/attribution.test.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"attribution.test.d.ts","sourceRoot":"","sources":["../../../src/routes/__test__/attribution.test.ts"],"names":[],"mappings":""}

package/build/routes/__test__/health.test.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"health.test.d.ts","sourceRoot":"","sources":["../../../src/routes/__test__/health.test.ts"],"names":[],"mappings":""}

package/build/routes/__test__/imagery.test.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"imagery.test.d.ts","sourceRoot":"","sources":["../../../src/routes/__test__/imagery.test.ts"],"names":[],"mappings":""}

package/build/routes/import.d.ts

@@ -1,9 +0,0 @@
-import { LambdaHttpRequest, LambdaHttpResponse } from '@linzjs/lambda';
-/**
- * Trigger import imagery job by this endpoint
- *
- * @example
- * - /v1/import?path=s3://linz-imagery-staging/2022-03/wellington_rural_2022_delivery_1
- */
-export declare function Import(req: LambdaHttpRequest): Promise<LambdaHttpResponse>;
-//# sourceMappingURL=import.d.ts.map

package/build/routes/import.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"import.d.ts","sourceRoot":"","sources":["../../src/routes/import.ts"],"names":[],"mappings":"AAAA,OAAO,EAAc,iBAAiB,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;AASnF;;;;;GAKG;AACH,wBAAsB,MAAM,CAAC,GAAG,EAAE,iBAAiB,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAkDhF"}

package/build/routes/import.js

@@ -1,61 +0,0 @@
-import { HttpHeader, LambdaHttpResponse } from '@linzjs/lambda';
-import { Config, Const, fsa } from '@basemaps/shared';
-import { createHash } from 'crypto';
-import { findImagery, RoleRegister } from '../import/imagery.find.js';
-import { Nztm2000Tms, TileMatrixSets } from '@basemaps/geo';
-import { getJobCreationContext } from '../import/make.cog.js';
-import { ConfigProviderDynamo } from '@basemaps/config';
-import { CogJobFactory } from '@basemaps/cli';
-/**
- * Trigger import imagery job by this endpoint
- *
- * @example
- * - /v1/import?path=s3://linz-imagery-staging/2022-03/wellington_rural_2022_delivery_1
- */
-export async function Import(req) {
-    const path = req.query.get('path');
-    const projection = req.query.get('p');
-    // Parse projection as target, default to process both NZTM2000Quad
-    let targetTms = Nztm2000Tms;
-    if (projection != null) {
-        const tileMatrix = TileMatrixSets.find(projection);
-        if (tileMatrix == null)
-            return new LambdaHttpResponse(404, 'Target projection Not found');
-        targetTms = tileMatrix;
-    }
-    // Find the imagery from s3
-    if (path == null || !path.startsWith('s3://'))
-        return new LambdaHttpResponse(500, 'Invalid s3 path');
-    const role = await RoleRegister.findRole(path);
-    if (role == null)
-        return new LambdaHttpResponse(500, 'Unable to Access the bucket');
-    const files = await findImagery(path);
-    if (files.length === 0)
-        return new LambdaHttpResponse(404, 'Imagery Not Found');
-    // Prepare Cog jobs
-    const ctx = await getJobCreationContext(path, targetTms, role, files);
-    const id = createHash('sha256').update(JSON.stringify(ctx)).digest('base64');
-    const jobId = Config.ProcessingJob.id(id);
-    let jobConfig = await Config.ProcessingJob.get(jobId);
-    if (jobConfig == null) {
-        // Add id back to JobCreationContext
-        ctx.override.id = id;
-        ctx.outputLocation.path = fsa.join(ctx.outputLocation.path, id);
-        // Start processing job
-        await CogJobFactory.create(ctx);
-        jobConfig = {
-            id: jobId,
-            name: path,
-            status: 'processing',
-        };
-        const config = new ConfigProviderDynamo(Const.TileMetadata.TableName);
-        await config.ProcessingJob.put(jobConfig);
-    }
-    const json = JSON.stringify(jobConfig);
-    const data = Buffer.from(json);
-    const response = new LambdaHttpResponse(200, 'ok');
-    response.header(HttpHeader.CacheControl, 'no-store');
-    response.buffer(data, 'application/json');
-    req.set('bytes', data.byteLength);
-    return response;
-}

package/src/__test__/tile.import.test.ts

@@ -1,140 +0,0 @@
-import { JobCreationContext } from '@basemaps/cli/build/cog/cog.stac.job';
-import { Nztm2000Tms } from '@basemaps/geo';
-import { Config, Env, fsa, LogConfig } from '@basemaps/shared';
-import o from 'ospec';
-import { createHash } from 'crypto';
-import sinon from 'sinon';
-import { LambdaAlbRequest, LambdaHttpRequest } from '@linzjs/lambda';
-import { Context } from 'aws-lambda';
-import { Import } from '../routes/import.js';
-import { RoleConfig } from '../import/imagery.find.js';
-import { CogJobFactory } from '@basemaps/cli';
-import { ConfigProcessingJob } from '@basemaps/config';
-
-o.spec('Import', () => {
-  const sandbox = sinon.createSandbox();
-  const outputBucket = 'testOutputBucket';
-  const configBucket = 'testConfigBucket';
-  const origConfigBucket = process.env[Env.AwsRoleConfigBucket];
-  const origOutputBucket = process.env[Env.ImportImageryBucket];
-  o.beforeEach(() => {
-    process.env[Env.AwsRoleConfigBucket] = configBucket;
-    process.env[Env.ImportImageryBucket] = outputBucket;
-  });
-
-  o.afterEach(() => {
-    process.env[Env.AwsRoleConfigBucket] = origConfigBucket;
-    process.env[Env.ImportImageryBucket] = origOutputBucket;
-    sandbox.restore();
-  });
-
-  const tileMatrix = Nztm2000Tms;
-  const bucket = 'testSourceBucket';
-  const path = `s3://${bucket}/imagery/`;
-  const role: RoleConfig = {
-    bucket,
-    accountId: '123456',
-    roleArn: 'arn:aws:iam::123456:role/read-role',
-  };
-
-  const files = [`${path}/1.tiff`, `${path}/2.tiff`];
-  async function* listFiles(): AsyncGenerator<string, any, unknown> {
-    for (const key in files) yield files[key];
-  }
-
-  const ctx: JobCreationContext = {
-    override: {
-      projection: tileMatrix.projection,
-      resampling: {
-        warp: 'bilinear',
-        overview: 'lanczos',
-      },
-    },
-    outputLocation: { type: 's3' as const, path: `s3://${outputBucket}` },
-    sourceLocation: { type: 's3', path: path, ...role, files: files },
-    batch: true,
-    tileMatrix,
-    oneCogCovering: false,
-  };
-
-  const id = createHash('sha256').update(JSON.stringify(ctx)).digest('base64');
-  const jobId = Config.ProcessingJob.id(id);
-
-  function getRequest(path: string, projection: string): LambdaHttpRequest {
-    return new LambdaAlbRequest(
-      {
-        requestContext: null as any,
-        httpMethod: 'get',
-        path: '/v1/tiles/import',
-        body: null,
-        isBase64Encoded: false,
-        queryStringParameters: { path: path, p: projection },
-      },
-      {} as Context,
-      LogConfig.get(),
-    );
-  }
-
-  o('should return projection not found', async () => {
-    // Given ... wrong projection
-    const req = getRequest(path, '0000');
-
-    // When ... Then ...
-    const res = await Import(req);
-    o(res.body).equals('{"status":404,"message":"Target projection Not found"}');
-  });
-
-  o('should return Invalid s3 location', async () => {
-    // Given... wrong s3 path
-    const req = getRequest('s3::testbucket/', '2193');
-
-    // When ...Then ...
-    const res = await Import(req);
-    o(res.body).equals('{"status":500,"message":"Invalid s3 path"}');
-  });
-
-  o('should return Unable to access bucket', async () => {
-    // Given... different bucket have no access role
-    sandbox.stub(fsa, 'readJson').resolves({ buckets: [role] });
-    const req = getRequest(`s3://wrong-bucket/imagery/`, '2193');
-
-    // When ...Then ...
-    const res = await Import(req);
-    o(res.body).equals('{"status":500,"message":"Unable to Access the bucket"}');
-  });
-
-  o('should return Imagery not found', async () => {
-    // Given... none imagery find from bucket
-    sandbox.stub(fsa, 'readJson').resolves({ buckets: [role] });
-    sandbox.stub(fsa, 'list').callsFake(async function* () {
-      yield `${path}1.json`;
-    });
-
-    const req = getRequest(path, '2193');
-
-    // When ...Then ...
-    const res = await Import(req);
-    o(res.body).equals('{"status":404,"message":"Imagery Not Found"}');
-  });
-
-  o('should return 200 with existing import', async () => {
-    // Given... different bucket have no access role
-    sandbox.stub(fsa, 'readJson').resolves({ buckets: [role] });
-    sandbox.stub(fsa, 'list').callsFake(listFiles);
-    sandbox.stub(CogJobFactory, 'create').resolves(undefined);
-
-    const jobConfig = {
-      id: jobId,
-      name: path,
-      status: 'complete',
-    } as ConfigProcessingJob;
-    sandbox.stub(Config.ProcessingJob, 'get').resolves(jobConfig);
-    const req = getRequest(path, '2193');
-
-    // When ...Then ...
-    const res = await Import(req);
-    o(res.status).equals(200);
-    const body = Buffer.from(res.body ?? '', 'base64').toString();
-    o(JSON.parse(body)).deepEquals(jobConfig);
-  });
-});

package/src/import/imagery.find.ts

@@ -1,60 +0,0 @@
-import { AwsCredentials } from '@chunkd/source-aws-v2';
-import { fsa } from '@chunkd/fs';
-import { Env } from '@basemaps/shared';
-
-export interface RoleConfig {
-  bucket: string;
-  accountId: string;
-  roleArn: string;
-  externalId?: string;
-  roleSessionDuration?: number;
-}
-
-interface BucketConfig {
-  v: number;
-  buckets: RoleConfig[];
-  version: string;
-  package: string;
-  hash: string;
-  updatedAt: string;
-}
-
-export class RoleRegister {
-  /** Get all imagery source aws roles */
-  static async _loadRoles(): Promise<RoleConfig[]> {
-    const configBucket = Env.get(Env.AwsRoleConfigBucket);
-    if (configBucket == null) return [];
-    const configPath = `s3://${configBucket}/config.json`;
-    const config: BucketConfig = await fsa.readJson(configPath);
-    const roles = [];
-    for (const role of config.buckets) {
-      fsa.register(
-        's3://' + role.bucket,
-        AwsCredentials.fsFromRole(role.roleArn, role.externalId, role.roleSessionDuration),
-      );
-      roles.push(role);
-    }
-    return roles;
-  }
-
-  static _loadRolesPromise: Promise<RoleConfig[]> | undefined;
-  static loadRoles(): Promise<RoleConfig[]> {
-    if (RoleRegister._loadRolesPromise == null) RoleRegister._loadRolesPromise = this._loadRoles();
-    return RoleRegister._loadRolesPromise;
-  }
-
-  static async findRole(path: string): Promise<RoleConfig | undefined> {
-    const roles = await this.loadRoles();
-    return roles.find((f) => path.startsWith(`s3://${f.bucket}`));
-  }
-}
-
-/** Search for the imagery across all of our buckets */
-export async function findImagery(path: string): Promise<string[]> {
-  const files: string[] = [];
-  for await (const key of fsa.list(path)) {
-    const searchKey = key.toLowerCase();
-    if (searchKey.endsWith('.tif') || searchKey.endsWith('.tiff')) files.push(key);
-  }
-  return files;
-}

package/src/import/make.cog.ts

@@ -1,29 +0,0 @@
-import { JobCreationContext } from '@basemaps/cli/build/cog/cog.stac.job';
-import { TileMatrixSet } from '@basemaps/geo';
-import { Env } from '@basemaps/shared';
-import { RoleConfig } from './imagery.find.js';
-
-export async function getJobCreationContext(
-  path: string,
-  tileMatrix: TileMatrixSet,
-  role: RoleConfig,
-  files: string[],
-): Promise<JobCreationContext> {
-  const bucket = Env.get(Env.ImportImageryBucket);
-  if (bucket == null) throw new Error('Output AWS s3 bucket Not Found.');
-  const ctx: JobCreationContext = {
-    override: {
-      projection: tileMatrix.projection,
-      resampling: {
-        warp: 'bilinear',
-        overview: 'lanczos',
-      },
-    },
-    outputLocation: { type: 's3' as const, path: `s3://${bucket}` },
-    sourceLocation: { type: 's3', path, ...role, files: files },
-    batch: true,
-    tileMatrix,
-    oneCogCovering: false,
-  };
-  return ctx;
-}

package/src/routes/import.ts

@@ -1,66 +0,0 @@
-import { HttpHeader, LambdaHttpRequest, LambdaHttpResponse } from '@linzjs/lambda';
-import { Config, Const, fsa } from '@basemaps/shared';
-import { createHash } from 'crypto';
-import { findImagery, RoleRegister } from '../import/imagery.find.js';
-import { Nztm2000Tms, TileMatrixSets } from '@basemaps/geo';
-import { getJobCreationContext } from '../import/make.cog.js';
-import { ConfigProcessingJob, ConfigProviderDynamo } from '@basemaps/config';
-import { CogJobFactory } from '@basemaps/cli';
-
-/**
- * Trigger import imagery job by this endpoint
- *
- * @example
- * - /v1/import?path=s3://linz-imagery-staging/2022-03/wellington_rural_2022_delivery_1
- */
-export async function Import(req: LambdaHttpRequest): Promise<LambdaHttpResponse> {
-  const path = req.query.get('path');
-  const projection = req.query.get('p');
-
-  // Parse projection as target, default to process both NZTM2000Quad
-  let targetTms = Nztm2000Tms;
-  if (projection != null) {
-    const tileMatrix = TileMatrixSets.find(projection);
-    if (tileMatrix == null) return new LambdaHttpResponse(404, 'Target projection Not found');
-    targetTms = tileMatrix;
-  }
-
-  // Find the imagery from s3
-  if (path == null || !path.startsWith('s3://')) return new LambdaHttpResponse(500, 'Invalid s3 path');
-  const role = await RoleRegister.findRole(path);
-  if (role == null) return new LambdaHttpResponse(500, 'Unable to Access the bucket');
-  const files = await findImagery(path);
-  if (files.length === 0) return new LambdaHttpResponse(404, 'Imagery Not Found');
-
-  // Prepare Cog jobs
-  const ctx = await getJobCreationContext(path, targetTms, role, files);
-
-  const id = createHash('sha256').update(JSON.stringify(ctx)).digest('base64');
-  const jobId = Config.ProcessingJob.id(id);
-  let jobConfig = await Config.ProcessingJob.get(jobId);
-  if (jobConfig == null) {
-    // Add id back to JobCreationContext
-    ctx.override!.id = id;
-    ctx.outputLocation.path = fsa.join(ctx.outputLocation.path, id);
-
-    // Start processing job
-    await CogJobFactory.create(ctx);
-    jobConfig = {
-      id: jobId,
-      name: path,
-      status: 'processing',
-    } as ConfigProcessingJob;
-
-    const config = new ConfigProviderDynamo(Const.TileMetadata.TableName);
-    await config.ProcessingJob.put(jobConfig);
-  }
-
-  const json = JSON.stringify(jobConfig);
-  const data = Buffer.from(json);
-
-  const response = new LambdaHttpResponse(200, 'ok');
-  response.header(HttpHeader.CacheControl, 'no-store');
-  response.buffer(data, 'application/json');
-  req.set('bytes', data.byteLength);
-  return response;
-}