@atmosx/event-product-parser 2.0.1

package/src/@submodules/utils.ts

@@ -0,0 +1,222 @@
+/*
+                                            _               _     __   __
+         /\  | |                           | |             (_)    \ \ / /
+        /  \ | |_ _ __ ___   ___  ___ _ __ | |__   ___ _ __ _  ___ \ V / 
+       / /\ \| __| "_ ` _ \ / _ \/ __| "_ \| "_ \ / _ \ "__| |/ __| > <  
+      / ____ \ |_| | | | | | (_) \__ \ |_) | | | |  __/ |  | | (__ / . \ 
+     /_/    \_\__|_| |_| |_|\___/|___/ .__/|_| |_|\___|_|  |_|\___/_/ \_\
+                                     | |                                 
+                                     |_|                                                                                                                
+    
+    Written by: KiyoWx (k3yomi)                
+*/
+
+import * as loader from '../bootstrap';
+import * as types from '../types';
+import StanzaParser from '../@parsers/stanza';
+import EventParser from '../@parsers/events';
+import Xmpp from './xmpp';
+
+export class Utils { 
+    
+    /**
+     * @function sleep
+     * @description
+     *     Pauses execution for a specified number of milliseconds.
+     *
+     * @static
+     * @async
+     * @param {number} ms
+     * @returns {Promise<void>}
+     */
+    public static async sleep(ms: number): Promise<void> {
+        return new Promise(resolve => setTimeout(resolve, ms));
+    }
+
+    /**
+     * @function warn
+     * @description
+     *     Emits a log event and prints a warning to the console. Throttles repeated
+     *     warnings within a short interval unless `force` is `true`.
+     *
+     * @static
+     * @param {string} message
+     * @param {boolean} [force=false]
+     * @returns {void}
+     */
+    public static warn(message: string, force: boolean = false): void {
+        loader.cache.events.emit('log', message)
+        if (!loader.settings.journal) return;
+        if (loader.cache.lastWarn != null && (Date.now() - loader.cache.lastWarn < 500) && !force) return;
+        loader.cache.lastWarn = Date.now();
+        console.warn(`\x1b[33m[ATMOSX-PARSER]\x1b[0m [${new Date().toLocaleString()}] ${message}`);
+    }
+
+    /**
+     * @function loadCollectionCache
+     * @description
+     *      Loads cached stanzas from the database, validates them, and processes them through the event parser. 
+     *      Only processes stanzas that are not marked to be ignored and match the CAP preferences.
+     *     
+     * @static
+     * @async
+     * @returns {Promise<void>}
+     */
+    public static async loadCollectionCache(): Promise<void> {
+        try {
+            const settings = loader.settings as types.ClientSettingsTypes;
+            if (settings.noaa_weather_wire_service_settings.cache.enabled) {
+                const maxRows = settings.noaa_weather_wire_service_settings.cache.max_db_cache_size ?? 5000;
+                const rows = await loader.cache.db.prepare(`SELECT * FROM stanzas ORDER BY rowid DESC LIMIT ?`)
+                    .all(maxRows) as { rowid: number; stanza: string }[];
+                this.warn(loader.definitions.messages.dump_cache.replace(`{count}`, rows.length.toString()), true);
+                const eventsToProcess = rows
+                    .map(row => {return JSON.parse(row.stanza)})
+                    .filter(validate => {
+                        if (!validate) return false;
+                        const skip = validate.ignore ||
+                            (validate.isCap && !settings.noaa_weather_wire_service_settings.preferences.cap_only) ||
+                            (!validate.isCap && settings.noaa_weather_wire_service_settings.preferences.cap_only) ||
+                            (validate.isCap && !validate.isCapDescription);
+                        return !skip;
+                    });
+                await Promise.all(eventsToProcess.map(validate => EventParser.eventHandler(validate)));
+                this.warn(loader.definitions.messages.dump_cache_complete, true);
+                return;
+            }
+        } catch (error: any) {
+            Utils.warn(`Failed to load cache: ${error.stack}`);
+        }
+    }
+
+    /**
+     * @function loadGeoJsonData
+     * @description
+     *     Fetches GeoJSON data from the National Weather Service endpoint and
+     *     passes it to the event parser for processing.
+     *
+     * @static
+     * @async
+     * @returns {Promise<void>}
+     */
+    public static async loadGeoJsonData(): Promise<void> {
+        try {
+            const settings = loader.settings as types.ClientSettingsTypes;
+            const response = await this.createHttpRequest<types.GenericHTTPResponse >(
+                settings.national_weather_service_settings.endpoint
+            );
+            if (response.error) return;
+            EventParser.eventHandler({
+                message: JSON.stringify(response.message),
+                attributes: {},
+                isCap: true,
+                isApi: true,
+                isPVtec: false,
+                isUGC: false,
+                isCapDescription: false,
+                awipsType: { type: 'api', prefix: 'AP' },
+                ignore: false
+            });
+        } catch (error: unknown) {
+            const msg = error instanceof Error ? error.message : String(error);
+            Utils.warn(`Failed to load National Weather Service GeoJSON Data: ${msg}`);
+        }
+    }
+
+    /**
+     * @function createHttpRequest
+     * @description
+     *     Performs an HTTP GET request with default headers and timeout, returning
+     *     either the response data or an error message.
+     *
+     * @static
+     * @template T
+     * @param {string} url
+     * @param {types.HTTPSettings} [options]
+     * @returns {Promise<{ error: boolean; message: T | string }>}
+     */
+    public static async createHttpRequest<T = unknown>(url: string, options?: types.HTTPSettings): Promise<{ error: boolean; message: T | string }> {   
+        const defaultOptions = { 
+            timeout: 10000,
+            headers: { 
+                "User-Agent": "AtmosphericX",
+                "Accept": "application/geo+json, text/plain, */*; q=0.9",
+                "Accept-Language": "en-US,en;q=0.9"
+            }
+        };
+        const requestOptions = {
+            ...defaultOptions,
+            ...options,
+            headers: { ...defaultOptions.headers, ...(options?.headers ?? {}) }
+        };
+        try {
+            const resp = await loader.packages.axios.get<T>(url, {
+                headers: requestOptions.headers,
+                timeout: requestOptions.timeout,
+                maxRedirects: 0,
+                validateStatus: (status) => status === 200 || status === 500
+            });
+            return { error: false, message: resp.data };
+        } catch (err: unknown) {
+            const msg = err instanceof Error ? err.message : String(err);
+            return { error: true, message: msg };
+        }
+    }
+
+    /**
+     * @function handleCronJob
+     * @description
+     *     Performs scheduled tasks for NWWS XMPP session maintenance or GeoJSON data
+     *     updates depending on the job type.
+     *
+     * @static
+     * @param {boolean} isWire
+     * @returns {void}
+     */
+    public static handleCronJob(isWire: boolean): void {
+        try {
+            const settings = loader.settings as types.ClientSettingsTypes;
+            const cache = settings.noaa_weather_wire_service_settings.cache;
+            const reconnections = settings.noaa_weather_wire_service_settings.reconnection_settings;
+            if (isWire) {
+                if (reconnections.enabled) {
+                    void Xmpp.isSessionReconnectionEligible(reconnections.interval);
+                }
+            } else {
+                void this.loadGeoJsonData();
+            }
+        } catch (error: unknown) {
+            const msg = error instanceof Error ? error.message : String(error);
+            Utils.warn(`Failed to perform scheduled tasks (${isWire ? 'NWWS' : 'GeoJSON'}): ${msg}`);
+        }
+    }
+
+    /**
+     * @function mergeClientSettings
+     * @description
+     *     Recursively merges a ClientSettings object into a target object,
+     *     preserving nested structures and overriding existing values.
+     *
+     * @static
+     * @param {Record<string, unknown>} target
+     * @param {types.ClientSettingsTypes} settings
+     * @returns {Record<string, unknown>}
+     */
+    public static mergeClientSettings(target: Record<string, unknown>, settings: types.ClientSettingsTypes): Record<string, unknown> {
+        for (const key in settings) {
+            if (!Object.prototype.hasOwnProperty.call(settings, key)) continue;
+            const value = settings[key];
+            if (value && typeof value === 'object' && !Array.isArray(value)) {
+                if (!target[key] || typeof target[key] !== 'object' || Array.isArray(target[key])) {
+                    target[key] = {};
+                }
+                this.mergeClientSettings(target[key] as Record<string, unknown>, value as types.ClientSettingsTypes);
+            } else {
+                target[key] = value;
+            }
+        }
+        return target;
+    }
+}
+
+export default Utils;

package/src/@submodules/xmpp.ts

@@ -0,0 +1,142 @@
+/*
+                                            _               _     __   __
+         /\  | |                           | |             (_)    \ \ / /
+        /  \ | |_ _ __ ___   ___  ___ _ __ | |__   ___ _ __ _  ___ \ V / 
+       / /\ \| __| "_ ` _ \ / _ \/ __| "_ \| "_ \ / _ \ "__| |/ __| > <  
+      / ____ \ |_| | | | | | (_) \__ \ |_) | | | |  __/ |  | | (__ / . \ 
+     /_/    \_\__|_| |_| |_|\___/|___/ .__/|_| |_|\___|_|  |_|\___/_/ \_\
+                                     | |                                 
+                                     |_|                                                                                                                
+    
+    Written by: KiyoWx (k3yomi)                
+*/
+
+import * as loader from '../bootstrap';
+import * as types from '../types';
+import Utils from './utils';
+import StanzaParser from '../@parsers/stanza';
+import Database from '../@submodules/database';
+import EventParser from '../@parsers/events';
+
+export class Xmpp { 
+
+    /** 
+     * @function isSessionReconnectionEligible
+     * @description
+     *     Checks if the XMPP session has been inactive longer than the given interval
+     *     and, if so, attempts a controlled reconnection.
+     *
+     * @async
+     * @static
+     * @param {number} currentInterval
+     * @returns {Promise<void>}
+     */
+    public static async isSessionReconnectionEligible(currentInterval: number): Promise<void> {
+        const settings = loader.settings as types.ClientSettingsTypes;
+        const lastStanzaElapsed = Date.now() - loader.cache.lastStanza;
+        const threshold = currentInterval * 1000;
+        if ((!loader.cache.isConnected && !loader.cache.sigHalt) || !loader.cache.session) { return; }
+        if (lastStanzaElapsed < threshold) { return; }
+        if (loader.cache.attemptingReconnect) { return;  }
+        loader.cache.attemptingReconnect = true;
+        loader.cache.isConnected = false;
+        loader.cache.totalReconnects += 1;
+        try {
+            loader.cache.events.emit("onReconnection", {
+                reconnects: loader.cache.totalReconnects,
+                lastStanza: lastStanzaElapsed,
+                lastName: settings.noaa_weather_wire_service_settings.credentials.nickname,
+            });
+            await loader.cache.session.stop().catch(() => {});
+            await loader.cache.session.start().catch(() => {});
+        } catch (err) {
+            Utils.warn(`XMPP reconnection failed: ${(err as Error).message}`);
+        } finally {
+            loader.cache.attemptingReconnect = false;
+        }
+    } 
+
+    /**
+     * @function deploySession
+     * @description
+     *     Initializes the NOAA Weather Wire Service (NWWS-OI) XMPP client session and
+     *     manages its lifecycle events including connection, disconnection, errors,
+     *     and message handling.
+     *
+     * @async
+     * @static
+     * @returns {Promise<void>}
+     */
+    public static async deploySession(): Promise<void> {
+        const settings = loader.settings as types.ClientSettingsTypes;
+        settings.noaa_weather_wire_service_settings.credentials.nickname ??= settings.noaa_weather_wire_service_settings.credentials.username;
+        loader.cache.session = loader.packages.xmpp.client({
+            service: 'xmpp://nwws-oi.weather.gov',
+            domain: 'nwws-oi.weather.gov',
+            username: settings.noaa_weather_wire_service_settings.credentials.username,
+            password: settings.noaa_weather_wire_service_settings.credentials.password,
+        });
+        loader.cache.session.on('online', async (address: string) => {
+            const now = Date.now();
+            if (loader.cache.lastConnect && now - loader.cache.lastConnect < 10_000) {
+                loader.cache.sigHalt = true;
+                Utils.warn(loader.definitions.messages.reconnect_too_fast);
+                await Utils.sleep(2_000);
+                await loader.cache.session.stop().catch(() => {});
+                return;
+            }
+            loader.cache.isConnected = true;
+            loader.cache.sigHalt = false;
+            loader.cache.lastConnect = now;
+            loader.cache.session.send(loader.packages.xmpp.xml('presence', {
+                to: `nwws@conference.nwws-oi.weather.gov/${settings.noaa_weather_wire_service_settings.credentials.nickname}`,
+                xmlns: 'http://jabber.org/protocol/muc',
+            }));
+            loader.cache.events.emit('onConnection', settings.noaa_weather_wire_service_settings.credentials.nickname);
+            if (loader.cache.attemptingReconnect) return;
+            loader.cache.attemptingReconnect = true;
+            await Utils.sleep(15_000);
+            loader.cache.attemptingReconnect = false;
+        });
+        loader.cache.session.on('offline', () => {
+            loader.cache.isConnected = false;
+            loader.cache.sigHalt = true;
+            Utils.warn('XMPP connection went offline');
+        });
+        loader.cache.session.on('error', (error: Error) => {
+            loader.cache.isConnected = false;
+            loader.cache.sigHalt = true;
+            Utils.warn(`XMPP connection error: ${error.message}`);
+        });
+        loader.cache.session.on('stanza', async (stanza: any) => {
+            try {
+                loader.cache.lastStanza = Date.now();
+                if (stanza.is('message')) {
+                    const validate = StanzaParser.validate(stanza);
+                    const skipMessage = validate.ignore ||
+                            (validate.isCap && !settings.noaa_weather_wire_service_settings.preferences.cap_only) ||
+                            (!validate.isCap && settings.noaa_weather_wire_service_settings.preferences.cap_only) ||
+                            (validate.isCap && !validate.isCapDescription);
+                    if (skipMessage) return;
+                    await EventParser.eventHandler(validate);
+                    await Database.stanzaCacheImport(validate);
+                    loader.cache.events.emit('onMessage', validate);
+                }
+                if (stanza.is('presence') && stanza.attrs.from?.startsWith('nwws@conference.nwws-oi.weather.gov/')) {
+                    const occupant = stanza.attrs.from.split('/').slice(1).join('/');
+                    loader.cache.events.emit('onOccupant', {
+                        occupant,
+                        type: stanza.attrs.type === 'unavailable' ? 'unavailable' : 'available',
+                    });
+                }
+            } catch (err: unknown) {
+                Utils.warn(`Error processing stanza: ${(err as Error).message}`);
+            }
+        });
+        try { await loader.cache.session.start(); } catch (err: unknown) {
+            Utils.warn(`Failed to start XMPP session: ${(err as Error).message}`);
+        }
+    }
+}
+
+export default Xmpp;

package/src/bootstrap.ts

@@ -0,0 +1,190 @@
+/*
+                                            _               _     __   __
+         /\  | |                           | |             (_)    \ \ / /
+        /  \ | |_ _ __ ___   ___  ___ _ __ | |__   ___ _ __ _  ___ \ V / 
+       / /\ \| __| '_ ` _ \ / _ \/ __| '_ \| '_ \ / _ \ '__| |/ __| > <  
+      / ____ \ |_| | | | | | (_) \__ \ |_) | | | |  __/ |  | | (__ / . \ 
+     /_/    \_\__|_| |_| |_|\___/|___/ .__/|_| |_|\___|_|  |_|\___/_/ \_\
+                                     | |                                 
+                                     |_|                                                                                                                
+    
+    Written by: k3yomi@GitHub                        
+*/
+
+import * as fs from 'fs';
+import * as path from 'path';
+import * as events from 'events';
+import * as xmpp from '@xmpp/client';
+import * as shapefile from 'shapefile';
+import * as xml2js from 'xml2js';
+import * as jobs from 'croner';
+import * as polygonClipping from 'polygon-clipping';
+import sqlite3 from 'better-sqlite3';
+import axios from 'axios';
+import crypto from 'crypto';
+import os from 'os';
+import say from 'say';
+import child from 'child_process';
+import jszip from 'jszip';
+
+
+import * as dictEvents from './@dictionaries/events';
+import * as dictAwips from './@dictionaries/awips';
+import * as dictSignatures from './@dictionaries/signatures';
+import * as dictICAOs from './@dictionaries/icao';
+
+
+
+export const packages = {
+    fs, path, events, xmpp, 
+    shapefile, xml2js, sqlite3, jobs, axios, 
+    crypto, os, say, child, polygonClipping, jszip
+};
+
+export const cache = {
+    isReady: true,
+    sigHalt: false,
+    isConnected: false,
+    attemptingReconnect: false,
+    totalReconnects: 0,
+    lastStanza: null,
+    session: null,
+    lastConnect: null,
+    db: null,
+    lastWarn: null,
+    totalLocationWarns: 0,
+    events: new events.EventEmitter(),
+    isProcessingAudioQueue: false,
+    audioQueue: [],
+};
+
+export const settings = { 
+    database: path.join(process.cwd(), 'shapefiles.db'),
+    is_wire: true,
+    journal: true,
+    noaa_weather_wire_service_settings: {
+        reconnection_settings: {
+            enabled: true,
+            interval: 60,
+        },
+        credentials: {
+            username: null,
+            password: null,
+            nickname: "AtmosphericX Standalone Parser",
+        },   
+        cache: {
+            enabled: true,
+            max_db_history: 5000,
+            max_db_cache_size: 1000,
+        },
+        preferences: {
+            disable_ugc: false,
+            disable_vtec: false,
+            disable_text: false,
+            cap_only: false,
+        }
+    },
+    national_weather_service_settings: {
+        interval: 15,
+        endpoint: `https://api.weather.gov/alerts/active`,
+    },
+    global_settings: {
+        parent_events_only: true,
+        better_event_parsing: true,
+        ignore_geometry_parsing: false,
+        shapefile_coordinates: false,
+        shapefile_skip: 15,
+        filtering: {
+            events: [],
+            filtered_icao: [],
+            ignored_icao: [],
+            ignored_events: [`Xx`, `Test Message`],
+            ugc_filter: [],
+            state_filter: [],
+            check_expired: true,
+            ignore_test_products: true,
+        },
+        eas_settings: {
+            directory: null,
+            intro_wav: null,
+        }
+    }
+};
+
+
+export const definitions = {
+    events: dictEvents.events,
+    actions: dictEvents.actions,
+    status: dictEvents.status,
+    productTypes: dictEvents.types,
+    correlations: dictEvents.status_correlations,
+    offshore: dictEvents.offshore,
+    awips: dictAwips.awips,
+    causes: dictEvents.causes,
+    records: dictEvents.records,
+    severity: dictEvents.severity,
+    cancelSignatures: dictSignatures.cancel_signatures,
+    messageSignatures: dictSignatures.message_signatures,
+    tags: dictSignatures.tags,
+    ICAO: dictICAOs.icaos,
+    enhancedEvents: [
+        {"Tornado Warning": {
+            "Tornado Emergency": { description: "tornado emergency", condition: (tornadoThreatTag: string) => tornadoThreatTag === 'OBSERVED'},
+            "PDS Tornado Warning": { description: "particularly dangerous situation", condition: (damageThreatTag: string) => damageThreatTag === 'CONSIDERABLE'},
+            "Confirmed Tornado Warning": { condition: (tornadoThreatTag: string) => tornadoThreatTag === 'OBSERVED'},
+            "Radar Indicated Tornado Warning": {condition: (tornadoThreatTag: string) => tornadoThreatTag !== 'OBSERVED'},
+        }},
+        {"Special Marine Warning": {
+            "Tornadic Special Marine Warning": {condition: (tornadoThreatTag: string) => tornadoThreatTag !== 'POSSIBLE'},
+        }},
+        {"Tornado Watch": {
+            "PDS Tornado Watch": { description: "particularly dangerous situation"}
+        }},
+        {"Flash Flood Warning": {
+            "Flash Flood Emergency": { description: "flash flood emergency", },
+            "Considerable Flash Flood Warning": { condition: (damageThreatTag: string) => damageThreatTag === 'CONSIDERABLE' },
+        }},
+        {"Severe Thunderstorm Warning": {
+            "EDS Severe Thunderstorm Warning": {description: "extremely dangerous situation"},
+            "Destructive Severe Thunderstorm Warning": {condition: (damageThreatTag: string) => damageThreatTag === 'DESTRUCTIVE'},
+            "Considerable Severe Thunderstorm Warning": {condition: (damageThreatTag: string) => damageThreatTag === 'CONSIDERABLE'},
+        }},
+    ],
+    shapefiles_directory: [
+        {name: "us_counties", id: "C", link: "https://www.weather.gov/source/gis/Shapefiles/County/c_18mr25.zip"},
+        {name: "us_states_territories", id: "Z", link: "https://www.weather.gov/source/gis/Shapefiles/County/s_18mr25.zip"},
+        {name: "fire_weather_zones", id: "Z", link: "https://www.weather.gov/source/gis/Shapefiles/WSOM/fz18mr25.zip"},
+        {name: "costal_marine_zones", id: "Z", link: "https://www.weather.gov/source/gis/Shapefiles/WSOM/mz18mr25.zip"},
+        {name: "offshore_marine_zones", id: "Z", link: "https://www.weather.gov/source/gis/Shapefiles/WSOM/oz18mr25.zip"},
+        {name: "public_forecast_zones", id: "Z", link: "https://www.weather.gov/source/gis/Shapefiles/WSOM/z_18mr25.zip"},
+        {name: "county_warning_areas", id: "Z", link: "https://www.weather.gov/source/gis/Shapefiles/WSOM/w_18mr25.zip"},
+        {name: "river_forecast_boundaries", id: "Z", link: "https://www.weather.gov/source/gis/Shapefiles/Misc/rf05mr24.zip"},
+    ],
+    regular_expressions: {
+        pvtec: new RegExp(`[OTEX].(NEW|CON|EXT|EXA|EXB|UPG|CAN|EXP|COR|ROU).[A-Z]{4}.[A-Z]{2}.[WAYSFON].[0-9]{4}.[0-9]{6}T[0-9]{4}Z-[0-9]{6}T[0-9]{4}Z`, "g"),
+        hvtec: new RegExp(`[a-zA-Z0-9]{4}.[A-Z0-9].[A-Z]{2}.[0-9]{6}T[0-9]{4}Z.[0-9]{6}T[0-9]{4}Z.[0-9]{6}T[0-9]{4}Z.[A-Z]{2}`, "imu"),
+        wmo: new RegExp(`[A-Z0-9]{6}\\s[A-Z]{4}\\s\\d{6}`, "imu"),
+        ugc1: new RegExp(`(\\w{2}[CZ](\\d{3}((-|>)\\s?(\\n\\n)?))+)`, "imu"),
+        ugc2: new RegExp(`(\\d{6}(-|>)\\s?(\\n\\n)?)`, "imu"),
+        ugc3: new RegExp(`(\\d{6})(?=-|$)`, "imu"),
+        dateline: new RegExp(`\\d{3,4}\\s*(AM|PM)?\\s*[A-Z]{2,4}\\s+[A-Z]{3,}\\s+[A-Z]{3,}\\s+\\d{1,2}\\s+\\d{4}`, "gim"),
+    },
+    messages: {
+        shapefile_creation: `DO NOT EXIT UNTIL THE SHAPEFILES ARE DONE COMPLETING! IF YOU CLOSE YOUR PROJECT, THE SHAPEFILES WILL NOT BE CREATED AND YOU WILL NEED TO DELETE ${settings.database} AND RESTART TO CREATE THEM AGAIN!`,
+        shapefile_creation_finished: `Shapefiles have finished completing and you can now use the parser.`,
+        not_ready: `You can not create another instance without shutting down the current one first, please make sure to call the stop() method first!`,
+        invalid_nickname: `The nickname you provided is invalid, please provide a valid nickname to continue.`,
+        eas_no_directory: `You have not set a directory for EAS audio files to be saved to, please set the 'directory' setting in the global settings to enable EAS audio generation.`,
+        reconnect_too_fast: `The client is attempting to reconnect too fast. This may be due to network instability. Reconnection attempt has been halted for safety.`,
+        dump_cache: `Found {count} cached events and will begin dumping them shortly. This may take a while depending on the number of cached events.`,
+        dump_cache_complete: `Completed dumping all cached alert files.`,
+    }
+};
+
+
+process.on('uncaughtException', (err: any) => {
+    if (err?.code === 'ETIMEDOUT') { return; }
+    if (err?.code === 'ECONNRESET') { return; }
+    if (err?.code === 'EHOSTUNREACH') { return; }
+    throw err;
+})