chiitiler 1.14.2 → 1.15.0

package/README.md

@@ -99,7 +99,7 @@ you can pass server options via environment variables
 | CHIITILER_PROCESSES                | 1        | num of chiitiler processes. 0 means all-CPUs   |
 | CHIITILER_DEBUG                    | false    | debug mode                                     |
 | CHIITILER_STREAM_MODE              | false    | stream mode                                    |
-| CHIITILER_CACHE_METHOD             | none     | cache method, `none`, `memory`, `file` or `s3` |
+| CHIITILER_CACHE_METHOD             | none     | cache method, `none`, `memory`, `file`, `s3` or `gcs` |
 | CHIITILER_CACHE_TTL_SEC            | 3600     | cache ttl, effect to `memory` and `file`       |
 | CHIITILER_MEMORYCACHE_MAXITEMCOUNT | 1000     | max items for memorycache                      |
 | CHIITILER_FILECACHE_DIR            | .cache   | filecache directory                            |
@@ -107,6 +107,11 @@ you can pass server options via environment variables
 | CHIITILER_S3_REGION                | us-east1 | s3 bucket region for caching/fetching          |
 | CHIITILER_S3_ENDPOINT              |          | s3 endpoint for caching/fetching               |
 | CHIITILER_S3_FORCE_PATH_STYLE      | false    | force path style for s3, needed for minio      |
+| CHIITILER_GCS_CACHE_BUCKE          |          | gcs cache bucket name                          |
+| CHIITILER_GCS_CACHE_PREFIX         |          | gcs cache prefix                               |
+| CHIITILER_GCS_PROJECT_ID           |          | gcs project id                                 |
+| CHIITILER_GCS_KEY_FILENAME         |          | gcs key filename                               |
+| CHIITILER_GCS_API_ENDPOINT         |          | gcs api endpoint                               |
 
 ### debug page
 
@@ -115,11 +120,18 @@ you can pass server options via environment variables
     - You can pass style.json url: <http://localhost:3000/debug?url=https://tile.openstreetmap.jp/styles/osm-bright/style.json>
   - editor page: <http://localhost:3000/editor>
 
+## deployment
+
+### AWS CDK
+
+- you can deploy chiitiler with AWS CDK, check [cdk](./cdk)
+
 ## supported protocols in style.json
 
 - `http://` or `https://` protocol are used in Style Specification
 - In addition, chiitiler supports following protocols:
   - `s3://` for S3 bucket
+  - `gs://` for Google Cloud Storage bucket
   - `file://` for file system
   - `mbtiles://` for MBTIles files
   - `pmtiles://` for PMTiles, remote or local or s3
@@ -162,6 +174,13 @@ you can pass server options via environment variables
       ],
       "maxzoom": 6
     },
+    "gcs": {
+      "type": "vector",
+      "tiles": [
+        "gs://tiles/{z}/{x}/{y}.pbf"
+      ],
+      "maxzoom": 6
+    },
     "cog": {
       "type": "raster",
       "tiles": [
@@ -211,6 +230,26 @@ you can pass server options via environment variables
         "circle-color": "green"
       }
     },
+    {
+      "id": "pmtiles-s3",
+      "source": "pmtiles-s3",
+      "source-layer": "P2921",
+      "type": "circle",
+      "paint": {
+        "circle-radius": 3,
+        "circle-color": "purple"
+      }
+    },
+    {
+      "id": "gcs",
+      "source": "gcs",
+      "source-layer": "P2921",
+      "type": "circle",
+      "paint": {
+        "circle-radius": 3,
+        "circle-color": "purple"
+      }
+    },
     {
       "id": "cog",
       "source": "cog",
@@ -252,6 +291,11 @@ const s3Cache = ChiitilerCache.s3Cache({
 });
 // credentials are loaded from environment variables: AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY
 
+const gcsCache = ChiitilerCache.gcsCache({
+    bucket: 'chiitiler',
+});
+// credentials are loaded from environment variables: GOOGLE_APPLICATION_CREDENTIALS
+
 const tileBuf = await getRenderedTileBuffer({
     stylejson: 'https://example.com/style.json', // or StyleSpecification object
     z: 0,
@@ -278,4 +322,5 @@ const bboxBuf = await getRenderedBboxBuffer({
 
 ## development
 
-- run `docker compose up`
+- To debug S3, run `docker compose up`
+- To debug GCS, change `CHIITILER_CACHE_METHOD` to `gcs` in `docker-compose.yml`, and then run `docker compose up`

package/dist/cache/gcs.d.ts

@@ -0,0 +1,10 @@
+import { type Cache } from './index.js';
+type GCSCacheOptions = {
+    bucket: string;
+    prefix?: string;
+    projectId?: string;
+    keyFilename?: string;
+    apiEndpoint?: string;
+};
+declare function gcsCache(options: GCSCacheOptions): Cache;
+export { gcsCache };

package/dist/cache/gcs.js

@@ -0,0 +1,66 @@
+import { getStorageClient } from '../gcs.js';
+function gcsCache(options) {
+    if (options.prefix?.endsWith('/')) {
+        options.prefix = options.prefix.slice(0, -1);
+    }
+    const storageClient = getStorageClient({
+        projectId: options.projectId,
+        keyFilename: options.keyFilename,
+        apiEndpoint: options.apiEndpoint,
+    });
+    const bucket = storageClient.bucket(options.bucket);
+    function bucketFile(key) {
+        const name = escapeFileName(key);
+        const nameWithPrefix = options.prefix ? `${options.prefix}/${name}` : name;
+        return bucket.file(nameWithPrefix);
+    }
+    return {
+        name: 'gcs',
+        set: async function (key, value) {
+            const file = bucketFile(key);
+            try {
+                await file.save(value, {
+                    resumable: false,
+                });
+            }
+            catch (e) {
+                console.log(`[error]: ${e}`);
+            }
+        },
+        get: async function (key) {
+            const file = bucketFile(key);
+            try {
+                const [content] = await file.download();
+                return Buffer.from(content);
+            }
+            catch {
+                // miss or any error
+                return undefined;
+            }
+        },
+    };
+}
+function escapeFileName(url) {
+    return url
+        .replace(/\//g, '_') // replace slashes with underscores
+        .replace(/\?/g, '-') // replace question marks with dashes
+        .replace(/&/g, '-') // replace ampersands with dashes
+        .replace(/=/g, '-') // replace equals signs with dashes
+        .replace(/%/g, '-') // replace percent signs with dashes
+        .replace(/#/g, '-') // replace hash signs with dashes
+        .replace(/:/g, '-') // replace colons with dashes
+        .replace(/\+/g, '-') // replace plus signs with dashes
+        .replace(/ /g, '-') // replace spaces with dashes
+        .replace(/</g, '-') // replace less than signs with dashes
+        .replace(/>/g, '-') // replace greater than signs with dashes
+        .replace(/\*/g, '-') // replace asterisks with dashes
+        .replace(/\|/g, '-') // replace vertical bars with dashes
+        .replace(/"/g, '-') // replace double quotes with dashes
+        .replace(/'/g, '-') // replace single quotes with dashes
+        .replace(/\?/g, '-') // replace question marks with dashes
+        .replace(/\./g, '-') // replace dots with dashes
+        .replace(/,/g, '-') // replace commas with dashes
+        .replace(/;/g, '-') // replace semicolons with dashes
+        .replace(/\\/g, '-'); // replace backslashes with dashes
+}
+export { gcsCache };

package/dist/cache/index.d.ts

@@ -1,5 +1,6 @@
 import { memoryCache } from './memory.js';
 import { s3Cache } from './s3.js';
+import { gcsCache } from './gcs.js';
 import { fileCache } from './file.js';
 type Value = Buffer;
 type Cache = {
@@ -8,4 +9,4 @@ type Cache = {
     set: (key: string, value: Value) => Promise<void>;
 };
 declare const noneCache: () => Cache;
-export { noneCache, memoryCache, s3Cache, fileCache, type Value, type Cache };
+export { noneCache, memoryCache, s3Cache, gcsCache, fileCache, type Value, type Cache };

package/dist/cache/index.js

@@ -1,9 +1,10 @@
 import { memoryCache } from './memory.js';
 import { s3Cache } from './s3.js';
+import { gcsCache } from './gcs.js';
 import { fileCache } from './file.js';
 const noneCache = () => ({
     name: 'none',
     get: async () => undefined,
     set: async () => undefined,
 });
-export { noneCache, memoryCache, s3Cache, fileCache };
+export { noneCache, memoryCache, s3Cache, gcsCache, fileCache };

package/dist/cli.js

@@ -21,6 +21,14 @@ function parseCacheStrategy(method, options) {
             endpoint: options.s3Endpoint,
             forcePathStyle: options.s3ForcePathStyle,
         });
+    if (method === 'gcs')
+        return caches.gcsCache({
+            bucket: options.gcsCacheBucket,
+            projectId: options.gcsProjectId,
+            keyFilename: options.gcsKeyFilename,
+            prefix: options.gcsCachePrefix,
+            apiEndpoint: options.gcsApiEndpoint,
+        });
     // command-line is not specified -> try to read from env
     const cacheEnv = process.env.CHIITILER_CACHE_METHOD;
     if (cacheEnv === 'memory')
@@ -40,6 +48,14 @@ function parseCacheStrategy(method, options) {
             endpoint: process.env.CHIITILER_S3_ENDPOINT,
             forcePathStyle: process.env.CHIITILER_S3_FORCE_PATH_STYLE === 'true',
         });
+    if (cacheEnv === 'gcs')
+        return caches.gcsCache({
+            bucket: process.env.CHIITILER_GCS_CACHE_BUCKET ?? '',
+            prefix: process.env.CHIITILER_GCS_CACHE_PREFIX,
+            projectId: process.env.CHIITILER_GCS_PROJECT_ID,
+            keyFilename: process.env.CHIITILER_GCS_KEY_FILENAME,
+            apiEndpoint: process.env.CHIITILER_GCS_API_ENDPOINT,
+        });
     // undefined or invalid
     return caches.noneCache();
 }
@@ -88,6 +104,11 @@ export function createProgram() {
         .option('-s3b --s3-cache-bucket <bucket-name>', 's3 cache bucket name', '')
         .option('-s3e --s3-endpoint <url>', 's3 endpoint url', '')
         .option('-3p --s3-force-path-style', 's3 force path style', '')
+        .option('-gcsb --gcs-cache-bucket <bucket-name>', 'gcs cache bucket name', '')
+        .option('-gcsp --gcs-project-id <project-id>', 'gcs project id', '')
+        .option('-gcsk --gcs-key-filename <key-filename>', 'gcs key filename', '')
+        .option('-gcsp --gcs-cache-prefix <prefix>', 'gcs cache prefix', '')
+        .option('-gcse --gcs-api-endpoint <api-endpoint>', 'gcs api endpoint', '')
         .option('-p --port <port>', 'port number')
         .option('-r --stream', 'stream mode')
         .option('-D --debug', 'debug mode')
@@ -101,6 +122,11 @@ export function createProgram() {
                 s3Region: options.s3Region,
                 s3Endpoint: options.s3Endpoint,
                 s3ForcePathStyle: options.s3ForcePathStyle === 'true',
+                gcsCacheBucket: options.gcsCacheBucket,
+                gcsCachePrefix: options.gcsCachePrefix,
+                gcsProjectId: options.gcsProjectId,
+                gcsKeyFilename: options.gcsKeyFilename,
+                gcsApiEndpoint: options.gcsApiEndpoint,
             }),
             port: parsePort(options.port),
             debug: parseDebug(options.debug),

package/dist/gcs.d.ts

@@ -0,0 +1,7 @@
+import { Storage } from '@google-cloud/storage';
+declare const getStorageClient: ({ projectId, keyFilename, apiEndpoint }: {
+    projectId?: string;
+    keyFilename?: string;
+    apiEndpoint?: string;
+}) => Storage;
+export { getStorageClient };

package/dist/gcs.js

@@ -0,0 +1,14 @@
+import { Storage } from '@google-cloud/storage';
+let storageClient; // singleton
+const getStorageClient = function ({ projectId, keyFilename, apiEndpoint }) {
+    if (storageClient !== undefined)
+        return storageClient;
+    const storageOptions = {
+        projectId,
+        keyFilename,
+        apiEndpoint
+    };
+    storageClient = new Storage(storageOptions);
+    return storageClient;
+};
+export { getStorageClient };

package/dist/source/gcs.d.ts

@@ -0,0 +1,2 @@
+declare function getGCSSource(uri: string): Promise<Buffer<ArrayBufferLike> | null>;
+export { getGCSSource };

package/dist/source/gcs.js

@@ -0,0 +1,21 @@
+import { getStorageClient } from '../gcs.js';
+async function getGCSSource(uri) {
+    const storageClient = getStorageClient({
+        projectId: process.env.CHIITILER_GCS_PROJECT_ID,
+        keyFilename: process.env.CHIITILER_GCS_KEY_FILENAME,
+        apiEndpoint: process.env.CHIITILER_GCS_API_ENDPOINT,
+    });
+    const bucket = uri.replace('gs://', '').split('/')[0];
+    const path = uri.replace(`gs://${bucket}/`, '');
+    try {
+        const file = storageClient.bucket(bucket).file(path);
+        const [buffer] = await file.download();
+        return buffer;
+    }
+    catch (e) {
+        if (e.code !== 404)
+            console.log(e);
+        return null;
+    }
+}
+export { getGCSSource };

package/dist/source/index.js

@@ -3,6 +3,7 @@ import { getHttpSource } from './http.js';
 import { getPmtilesSource } from './pmtiles.js';
 import { getMbtilesSource } from './mbtiles.js';
 import { getS3Source } from './s3.js';
+import { getGCSSource } from './gcs.js';
 import { getCogSource } from './cog.js';
 import { noneCache } from '../cache/index.js';
 /**
@@ -19,6 +20,8 @@ async function getSource(uri, cache = noneCache()) {
         data = await getFilesystemSource(uri);
     else if (uri.startsWith('s3://'))
         data = await getS3Source(uri);
+    else if (uri.startsWith('gs://'))
+        data = await getGCSSource(uri);
     else if (uri.startsWith('mbtiles://'))
         data = await getMbtilesSource(uri);
     else if (uri.startsWith('pmtiles://'))

package/package.json

@@ -1,7 +1,7 @@
 {
 	"type": "module",
 	"name": "chiitiler",
-	"version": "1.14.2",
+	"version": "1.15.0",
 	"description": "Tiny map rendering server for MapLibre Style Spec",
 	"main": "./dist/index.js",
 	"types": "./dist/index.d.ts",
@@ -33,6 +33,7 @@
 	},
 	"dependencies": {
 		"@aws-sdk/client-s3": "^3.418.0",
+		"@google-cloud/storage": "^7.15.2",
 		"@hono/node-server": "^1.12.0",
 		"@mapbox/sphericalmercator": "^1.2.0",
 		"@mapbox/tilebelt": "^1.0.2",