@tmlmobilidade/databases 20260511.1606.49 → 20260513.1439.37

package/dist/interfaces/api-cache/hub-serverdb-keys.d.ts

@@ -0,0 +1,44 @@
+export declare const SERVERDB_KEYS: Readonly<{
+    FACILITIES: {
+        BOAT_STATIONS: string;
+        HELPDESKS: string;
+        LIGHT_RAIL_STATIONS: string;
+        PIPS: string;
+        SCHOOLS: string;
+        STORES: string;
+        SUBWAY_STATIONS: string;
+        TRAIN_STATIONS: string;
+    };
+    LOCATIONS: {
+        DISTRICTS: string;
+        LOCALITIES: string;
+        MUNICIPALITIES: string;
+        PARISHES: string;
+        REGIONS: string;
+    };
+    NETWORK: {
+        ALERTS: {
+            ALL: string;
+            PROTOBUF: string;
+            SENT_NOTIFICATIONS: string;
+        };
+        DATES: string;
+        LINES: string;
+        PATTERNS: {
+            BASE: string;
+            ID: (id: string) => string;
+        };
+        PERIODS: string;
+        PLANS: string;
+        ROUTES: string;
+        SHAPES: {
+            BASE: string;
+            ID: (id: string) => string;
+        };
+        STOPS: string;
+        VEHICLES: {
+            ALL: string;
+            PROTOBUF: string;
+        };
+    };
+}>;

package/dist/interfaces/api-cache/hub-serverdb-keys.js

@@ -0,0 +1,45 @@
+/* * */
+export const SERVERDB_KEYS = Object.freeze({
+    FACILITIES: {
+        BOAT_STATIONS: 'facilities:boat_stations',
+        HELPDESKS: 'facilities:helpdesks',
+        LIGHT_RAIL_STATIONS: 'facilities:light_rail_stations',
+        PIPS: 'facilities:pips',
+        SCHOOLS: 'facilities:schools',
+        STORES: 'facilities:stores',
+        SUBWAY_STATIONS: 'facilities:subway_stations',
+        TRAIN_STATIONS: 'facilities:train_stations',
+    },
+    LOCATIONS: {
+        DISTRICTS: 'locations:districts',
+        LOCALITIES: 'locations:localities',
+        MUNICIPALITIES: 'locations:municipalities',
+        PARISHES: 'locations:parishes',
+        REGIONS: 'locations:regions',
+    },
+    NETWORK: {
+        ALERTS: {
+            ALL: 'network:alerts:all',
+            PROTOBUF: 'network:alerts:protobuf',
+            SENT_NOTIFICATIONS: 'network:alerts:sent_notifications',
+        },
+        DATES: 'network:dates',
+        LINES: 'network:lines',
+        PATTERNS: {
+            BASE: 'network:patterns',
+            ID: (id) => `network:patterns:${id}`,
+        },
+        PERIODS: 'network:periods',
+        PLANS: 'network:plans',
+        ROUTES: 'network:routes',
+        SHAPES: {
+            BASE: 'network:shapes',
+            ID: (id) => `network:shapes:${id}`,
+        },
+        STOPS: 'network:stops',
+        VEHICLES: {
+            ALL: 'network:vehicles:all',
+            PROTOBUF: 'network:vehicles:protobuf',
+        },
+    },
+});

package/dist/interfaces/api-cache/index.d.ts

@@ -1,2 +1,3 @@
+export * from './hub-serverdb-keys.js';
 export * from './interface.js';
 export * from './keys.js';

package/dist/interfaces/api-cache/index.js

@@ -1,2 +1,3 @@
+export * from './hub-serverdb-keys.js';
 export * from './interface.js';
 export * from './keys.js';

package/dist/interfaces/api-cache/interface.d.ts

@@ -1,4 +1,4 @@
-import { type ApiCacheKey } from './keys.js';
+import { type ApiCacheKey, type ApiCacheKeyParams } from './keys.js';
 import { type RedisClientType } from 'redis';
 declare class ApiCacheClass {
     private static _instance;
@@ -8,7 +8,7 @@ declare class ApiCacheClass {
      */
     static getInstance(): Promise<ApiCacheClass>;
     /**
-     * Deletes all keys from the cache that are not defined in `ApiCacheKeyValues`.
+     * Deletes all keys from the cache that are not allowed by {@link isAllowedHubApiCacheKey}.
      * This method is useful for maintaining a clean state free of stale
      * or irrelevant cache entries that consume storage and memory resources.
      * @returns A promise that resolves when the cleaning process is complete.
@@ -22,22 +22,38 @@ declare class ApiCacheClass {
      * @throws Will throw an error if the deletion process fails.
      */
     delete(key: ApiCacheKey): Promise<void>;
+    /**
+     * Deletes multiple cache entries.
+     * @param keys The list of keys to delete.
+     */
+    deleteMany(keys: string[]): Promise<void>;
     /**
      * Retrieves a cache entry by its key.
      * @param key The key of the cache entry to retrieve.
+     * @param params Optional params to replace `{named}` tokens in the key.
      * @returns A promise that resolves with the cache entry value,
      * or `null` if not found.
      * @throws Will throw an error if the retrieval process fails.
      */
-    get(key: ApiCacheKey): Promise<null | string>;
+    get(key: ApiCacheKey, params?: ApiCacheKeyParams): Promise<null | string>;
+    /**
+     * Scans cache keys by pattern.
+     * @param pattern The redis pattern to match.
+     * @returns A promise resolving with all matching keys.
+     */
+    scan(pattern: string): Promise<string[]>;
     /**
      * Saves a cache entry with an optional time-to-live (TTL).
      * @param key The key of the cache entry to save.
      * @param value The value of the cache entry to save. Must be a string.
      * @param ttl Optional time-to-live for the cache entry in seconds.
      * If not provided, the entry will persist indefinitely.
+     * @param params Optional params to replace `{named}` tokens in the key.
      */
-    set(key: ApiCacheKey, value: string, ttl?: number): Promise<void>;
+    set(key: ApiCacheKey, value: string, options: {
+        params?: ApiCacheKeyParams;
+        ttl?: number;
+    }): Promise<void>;
     protected connectToClient(): Promise<RedisClientType>;
     /**
      * Initializes the Redis client.

package/dist/interfaces/api-cache/interface.js

@@ -1,6 +1,6 @@
 /* * */
 import { GORedisClient } from '../../clients/go-redis.js';
-import { ApiCacheKeyValues } from './keys.js';
+import { resolveApiCacheKey } from './keys.js';
 import { asyncSingletonProxy } from '@tmlmobilidade/utils';
 /* * */
 class ApiCacheClass {
@@ -28,7 +28,7 @@ class ApiCacheClass {
         return await this._instance;
     }
     /**
-     * Deletes all keys from the cache that are not defined in `ApiCacheKeyValues`.
+     * Deletes all keys from the cache that are not allowed by {@link isAllowedHubApiCacheKey}.
      * This method is useful for maintaining a clean state free of stale
      * or irrelevant cache entries that consume storage and memory resources.
      * @returns A promise that resolves when the cleaning process is complete.
@@ -36,7 +36,7 @@ class ApiCacheClass {
      */
     async clean() {
         const allKeys = await this.client.keys('*');
-        const keysToDelete = allKeys.filter(key => !ApiCacheKeyValues.includes(key));
+        const keysToDelete = allKeys.filter(key => key);
         if (keysToDelete.length)
             await this.client.del(keysToDelete);
     }
@@ -49,38 +49,60 @@ class ApiCacheClass {
     async delete(key) {
         await this.client.del(key);
     }
+    /**
+     * Deletes multiple cache entries.
+     * @param keys The list of keys to delete.
+     */
+    async deleteMany(keys) {
+        if (!keys.length)
+            return;
+        await this.client.del(keys);
+    }
     /**
      * Retrieves a cache entry by its key.
      * @param key The key of the cache entry to retrieve.
+     * @param params Optional params to replace `{named}` tokens in the key.
      * @returns A promise that resolves with the cache entry value,
      * or `null` if not found.
      * @throws Will throw an error if the retrieval process fails.
      */
-    async get(key) {
-        const result = await this.client.get(key);
+    async get(key, params) {
+        const parsedKey = resolveApiCacheKey(key, params);
+        const result = await this.client.get(parsedKey);
         if (typeof result !== 'string')
             return null;
         return result;
     }
+    /**
+     * Scans cache keys by pattern.
+     * @param pattern The redis pattern to match.
+     * @returns A promise resolving with all matching keys.
+     */
+    async scan(pattern) {
+        const keys = [];
+        for await (const key of this.client.scanIterator({ MATCH: pattern, TYPE: 'string' })) {
+            keys.push(String(key));
+        }
+        return keys;
+    }
     /**
      * Saves a cache entry with an optional time-to-live (TTL).
      * @param key The key of the cache entry to save.
      * @param value The value of the cache entry to save. Must be a string.
      * @param ttl Optional time-to-live for the cache entry in seconds.
      * If not provided, the entry will persist indefinitely.
+     * @param params Optional params to replace `{named}` tokens in the key.
      */
-    async set(key, value, ttl) {
+    async set(key, value, options) {
+        const parsedKey = resolveApiCacheKey(key, options?.params);
         // Validate value type before setting cache
         if (typeof value !== 'string')
-            throw new Error(`[ApiCache] Value must be a string. Got "${typeof value}" for key "${key}".`);
-        // Check if key is valid before setting cache
-        if (!ApiCacheKeyValues.includes(key))
-            throw new Error(`[ApiCache] Invalid cache key "${key}". Allowed keys are: ${ApiCacheKeyValues.join(', ')}.`);
+            throw new Error(`[ApiCache] Value must be a string. Got "${typeof value}" for key "${parsedKey}".`);
         // Set cache with optional TTL
-        if (ttl)
-            await this.client.set(key, value, { expiration: { type: 'EX', value: ttl } });
+        if (options?.ttl)
+            await this.client.set(parsedKey, value, { expiration: { type: 'EX', value: options.ttl } });
         else
-            await this.client.set(key, value);
+            await this.client.set(parsedKey, value);
     }
     connectToClient() {
         return GORedisClient.getClient();

package/dist/interfaces/api-cache/keys.d.ts

@@ -1,2 +1,4 @@
-export declare const ApiCacheKeyValues: readonly ["hub:alerts:published:json", "hub:alerts:published:gtfs", "hub:alerts:published:rss", "hub:plans:approved:json", "hub:plans:gtfs", "hub:plans:gtfs:cm"];
+export type ApiCacheKeyParams = Record<string, boolean | number | string>;
+export declare const ApiCacheKeyValues: readonly ["hub:alerts:published:json", "hub:alerts:published:gtfs", "hub:alerts:published:rss", "hub:plans:approved:json", "hub:plans:gtfs", "hub:plans:gtfs:cm", "hub:facilities:json", "hub:facilities:helpdesks:json", "hub:facilities:boat_stations:json", "hub:facilities:light_rail_stations:json", "hub:facilities:subway_stations:json", "hub:facilities:train_stations:json", "hub:facilities:pips:json", "hub:facilities:schools:json", "hub:facilities:stores:json", "hub:locations:municipalities:json", "hub:locations:districts:json", "hub:locations:localities:json", "hub:locations:parishes:json", "hub:network:dates:json", "hub:network:periods:json", "hub:network:stops:json", "hub:network:lines:json", "hub:network:routes:json", "hub:network:plans:json", "hub:network:vehicles:json", "hub:network:vehicles:protobuf:json", "hub:network:patterns:{patternId}:json", "hub:network:shapes:{shapeId}:json"];
 export type ApiCacheKey = typeof ApiCacheKeyValues[number];
+export declare function resolveApiCacheKey(key: ApiCacheKey, params?: ApiCacheKeyParams): string;

package/dist/interfaces/api-cache/keys.js

@@ -6,4 +6,41 @@ export const ApiCacheKeyValues = [
     'hub:plans:approved:json',
     'hub:plans:gtfs',
     'hub:plans:gtfs:cm',
+    'hub:facilities:json',
+    'hub:facilities:helpdesks:json',
+    'hub:facilities:boat_stations:json',
+    'hub:facilities:light_rail_stations:json',
+    'hub:facilities:subway_stations:json',
+    'hub:facilities:train_stations:json',
+    'hub:facilities:pips:json',
+    'hub:facilities:schools:json',
+    'hub:facilities:stores:json',
+    'hub:locations:municipalities:json',
+    'hub:locations:districts:json',
+    'hub:locations:localities:json',
+    'hub:locations:parishes:json',
+    'hub:network:dates:json',
+    'hub:network:periods:json',
+    'hub:network:stops:json',
+    'hub:network:lines:json',
+    'hub:network:routes:json',
+    'hub:network:plans:json',
+    'hub:network:vehicles:json',
+    'hub:network:vehicles:protobuf:json',
+    'hub:network:patterns:{patternId}:json',
+    'hub:network:shapes:{shapeId}:json',
 ];
+export function resolveApiCacheKey(key, params) {
+    const resolvedKey = key.replace(/\{([^{}]+)\}/g, (_match, paramName) => {
+        const value = params?.[paramName];
+        if (typeof value === 'undefined' || value === null) {
+            throw new Error(`[ApiCache] Missing key param "${paramName}" for key "${key}".`);
+        }
+        return String(value);
+    });
+    const unresolvedParam = resolvedKey.match(/\{([^{}]+)\}/);
+    if (unresolvedParam) {
+        throw new Error(`[ApiCache] Missing key param "${unresolvedParam[1]}" for key "${key}".`);
+    }
+    return resolvedKey;
+}

package/dist/interfaces/vehicle-events/raw-vehicle-events.d.ts

@@ -1,3 +1,8 @@
+/**
+ * Raw Vehicle Events represent the raw vehicle events as they are received from the data sources.
+ * These are stored in MongoDB for persistence and later processing.
+ * These documents are also different depending on the data source and version of the vehicle event.
+**/
 import { MongoInterfaceTemplate } from '../../templates/mongodb.js';
 import { type SimplifiedMongoIndex } from '../../types/mongo/index-description.js';
 import { type RawVehicleEvent } from '@tmlmobilidade/types';

package/dist/interfaces/vehicle-events/raw-vehicle-events.js

@@ -1,4 +1,8 @@
-/* * */
+/**
+ * Raw Vehicle Events represent the raw vehicle events as they are received from the data sources.
+ * These are stored in MongoDB for persistence and later processing.
+ * These documents are also different depending on the data source and version of the vehicle event.
+**/
 import { PCGIRawClient } from '../../clients/pcgi-raw.js';
 import { MongoInterfaceTemplate } from '../../templates/mongodb.js';
 import { RawVehicleEventSchema } from '@tmlmobilidade/types';

package/dist/interfaces/vehicle-events/simplified-vehicle-events.d.ts

@@ -1,11 +1,17 @@
+/**
+ * Simplified Vehicle Events represent a simplified version of the raw vehicle events.
+ * These are stored in ClickHouse for performance and scalability reasons.
+**/
 import { ClickHouseInterfaceTemplate } from '../../templates/clickhouse.js';
-import { type ClickHouseSchema } from '../../types/index.js';
+import { type ClickHouseSchema, ClickHouseTableEngine } from '../../types/index.js';
 import { type SimplifiedVehicleEvent } from '@tmlmobilidade/types';
 declare class SimplifiedVehicleEventsNewClass extends ClickHouseInterfaceTemplate<SimplifiedVehicleEvent> {
     private static _instance;
     readonly databaseName = "operation";
-    readonly orderBy = "(created_at, trip_id)";
-    readonly partitionBy = "toYYYYMMDD(fromUnixTimestamp64Milli(created_at))";
+    readonly engine: ClickHouseTableEngine;
+    readonly orderBy = "(trip_id, vehicle_id, agency_id, created_at)";
+    readonly partitionBy = "toYYYYMM(fromUnixTimestamp64Milli(created_at))";
+    readonly primaryKey = "(trip_id, vehicle_id)";
     readonly schema: ClickHouseSchema<{
         _id: string;
         created_at: number & {

package/dist/interfaces/vehicle-events/simplified-vehicle-events.js

@@ -1,4 +1,7 @@
-/* * */
+/**
+ * Simplified Vehicle Events represent a simplified version of the raw vehicle events.
+ * These are stored in ClickHouse for performance and scalability reasons.
+**/
 import { GOClickHouseClient } from '../../clients/go-clickhouse.js';
 import { ClickHouseInterfaceTemplate } from '../../templates/clickhouse.js';
 import { asyncSingletonProxy } from '@tmlmobilidade/utils';
@@ -28,8 +31,10 @@ class SimplifiedVehicleEventsNewClass extends ClickHouseInterfaceTemplate {
     //
     static _instance = null;
     databaseName = 'operation';
-    orderBy = '(created_at, trip_id)';
-    partitionBy = 'toYYYYMMDD(fromUnixTimestamp64Milli(created_at))';
+    engine = 'ReplacingMergeTree';
+    orderBy = '(trip_id, vehicle_id, agency_id, created_at)';
+    partitionBy = 'toYYYYMM(fromUnixTimestamp64Milli(created_at))';
+    primaryKey = '(trip_id, vehicle_id)';
     schema = tableSchema;
     tableName = 'simplified_vehicle_events';
     /**

package/dist/templates/clickhouse.d.ts

@@ -16,6 +16,7 @@ export declare abstract class ClickHouseInterfaceTemplate<T extends object> {
     protected readonly manageSchema: boolean;
     protected readonly orderBy: string;
     protected readonly partitionBy: null | string;
+    protected readonly primaryKey: null | string;
     private client;
     /**
      * Disallow direct instantiation of the service.

package/dist/templates/clickhouse.js

@@ -17,6 +17,7 @@ export class ClickHouseInterfaceTemplate {
     manageSchema = true;
     orderBy = '_id';
     partitionBy = null;
+    primaryKey = null;
     client;
     /**
      * Disallow direct instantiation of the service.
@@ -209,6 +210,7 @@ export class ClickHouseInterfaceTemplate {
 			CREATE TABLE IF NOT EXISTS "${this.databaseName}"."${this.tableName}" (
 				${Object.entries(this.schema).map(([key, column]) => `${key} ${column.type}`).join(', ')}
 			) ENGINE = ${this.getEngineString()}
+			${this.primaryKey ? `PRIMARY KEY (${this.primaryKey})` : ''}
 			${this.orderBy ? `ORDER BY (${this.orderBy})` : ''}
 			${this.partitionBy ? `PARTITION BY (${this.partitionBy})` : ''}
 		`;

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@tmlmobilidade/databases",
-	"version": "20260511.1606.49",
+	"version": "20260513.1439.37",
 	"author": {
 		"email": "iso@tmlmobilidade.pt",
 		"name": "TML-ISO"