@atmosx/event-product-parser 3.0.63 → 3.0.75

package/src/components/database/ImportBoundaries.ts

@@ -0,0 +1,168 @@
+/*
+              _                             _               _     __   __
+         /\  | |                           | |             (_)    \ \ / /
+        /  \ | |_ _ __ ___   ___  ___ _ __ | |__   ___ _ __ _  ___ \ V / 
+       / /\ \| __| '_ ` _ \ / _ \/ __| '_ \| '_ \ / _ \ '__| |/ __| > <  
+      / ____ \ |_| | | | | | (_) \__ \ |_) | | | |  __/ |  | | (__ / . \ 
+     /_/    \_\__|_| |_| |_|\___/|___/ .__/|_| |_|\___|_|  |_|\___/_/ \_\
+                                     | |                            
+                                     |_|                                                                                                                
+
+    Created with ♥ by the AtmosphericX Team (KiyoWx, StarflightWx, & CJ Ziegler)
+    Discord: https://atmosphericx-discord.scriptkitty.cafe
+    Ko-Fi: https://ko-fi.com/k3yomi
+    Documentation: https://atmosphericx.scriptkitty.cafe/documentation
+
+    Independent Package: @atmosx/event-product-parser
+
+*/
+
+import { EnumStateFIPS } from "@Enums/States"
+import { Bootstrap } from "@Bootstrap"
+import { CreateHttp } from "@Utilities/CreateHttp"
+import { CreateQuery } from "@Database/CreateQuery"
+import { SetWarning } from "@Utilities/SetWarning"
+import { feature } from "topojson-client"
+
+type BoundaryRow = {
+    id: string
+    type: `state` | `county`
+    state: string
+    name: string
+    geometry: string
+}
+
+export const ImportBoundaries = async (): Promise<void> => {
+    try {
+        const existing = CreateQuery({ Query: `SELECT type, COUNT(*) AS count FROM boundaries GROUP BY type`}) as { type: string, count: number }[]
+        const existingStates = existing.find(row => row.type === `state`)?.count ?? 0
+        const existingCounties = existing.find(row => row.type === `county`)?.count ?? 0
+
+        if (existingStates > 0 || existingCounties > 0) {
+            SetWarning({ Message: `Boundary database already contains ${existingStates} states and ${existingCounties} counties, skipping import` })
+            return
+        }
+
+        const boundaries = await CreateHttp({
+            URL: Bootstrap.Settings.BoundarySettings.BoundaryDatabase,
+            Timeout: 5000
+        })
+
+        if (boundaries.error || !boundaries.message) {
+            SetWarning({ Message: `Failed to download boundary database` })
+            return
+        }
+
+        let database: any
+
+        try {
+            database = JSON.parse(boundaries.message)
+        } catch {
+            SetWarning({ Message: `Failed to parse boundary database` })
+            return
+        }
+
+        if (database?.type !== `Topology` || !database?.objects?.states || !database?.objects?.counties) {
+            SetWarning({ Message: `Boundary database is missing states or counties` })
+            return
+        }
+
+        let states: any
+        let counties: any
+
+        try {
+            states = feature(database, database.objects.states)
+            counties = feature(database, database.objects.counties)
+        } catch (error: unknown) {
+            const message = error instanceof Error ? error.message : String(error)
+            SetWarning({ Message: `Failed to convert boundary database: ${message}` })
+            return
+        }
+
+        if (!Array.isArray(states?.features) || !Array.isArray(counties?.features)) {
+            SetWarning({ Message: `Boundary database contains invalid state or county data` })
+            return
+        }
+
+        const insert = `INSERT OR REPLACE INTO boundaries (id, type, state, name, geometry) VALUES (?, ?, ?, ?, ?)`
+        const stateRows: BoundaryRow[] = []
+        const countyRows: BoundaryRow[] = []
+        let skippedStates = 0
+        let skippedCounties = 0
+
+        for (const state of states.features) {
+            if (!state?.geometry || state.id === undefined || state.id === null) {
+                skippedStates++
+                continue
+            }
+
+            const fips = String(state.id).padStart(2, `0`)
+            const abbreviation = EnumStateFIPS[fips]
+
+            if (!abbreviation) {
+                skippedStates++
+                continue
+            }
+
+            stateRows.push({
+                id: fips,
+                type: `state`,
+                state: abbreviation,
+                name: state.properties?.name ?? abbreviation,
+                geometry: JSON.stringify(state.geometry)
+            })
+        }
+
+        for (const county of counties.features) {
+            if (!county?.geometry || county.id === undefined || county.id === null) {
+                skippedCounties++
+                continue
+            }
+
+            const fips = String(county.id).padStart(5, `0`)
+            const stateFIPS = fips.slice(0, 2)
+            const abbreviation = EnumStateFIPS[stateFIPS]
+
+            if (!abbreviation) {
+                skippedCounties++
+                continue
+            }
+
+            countyRows.push({
+                id: fips,
+                type: `county`,
+                state: abbreviation,
+                name: county.properties?.name ?? fips,
+                geometry: JSON.stringify(county.geometry)
+            })
+        }
+
+        SetWarning({ Message: `Importing ${stateRows.length} states and ${countyRows.length} counties` })
+
+        const transaction = Bootstrap.Database.transaction((rows: BoundaryRow[]) => {
+            for (const row of rows) {
+                CreateQuery({
+                    Query: insert,
+                    Parameters: [row.id, row.type, row.state, row.name, row.geometry]
+                })
+            }
+        })
+
+        transaction(stateRows)
+        transaction(countyRows)
+
+        const imported = CreateQuery({ Query: `SELECT type, COUNT(*) AS count FROM boundaries GROUP BY type`}) as { type: string, count: number }[]
+        const importedStates = imported.find(row => row.type === `state`)?.count ?? 0
+        const importedCounties = imported.find(row => row.type === `county`)?.count ?? 0
+
+        if (importedStates !== stateRows.length || importedCounties !== countyRows.length) {
+            SetWarning({ Message: `Boundary import verification failed: ${importedStates} states and ${importedCounties} counties` })
+            return
+        }
+
+        SetWarning({ Message: `Imported ${importedStates} states and ${importedCounties} counties` })
+    } catch (error: unknown) {
+        const message = error instanceof Error ? error.message : String(error)
+        SetWarning({ Message: `An error occurred while importing boundaries: ${message}` })
+    }
+}

package/src/{@modules/@database → components/database}/ImportBroadcastify.ts

@@ -11,38 +11,29 @@
     Created with ♥ by the AtmosphericX Team (KiyoWx, StarflightWx, & CJ Ziegler)
     Discord: https://atmosphericx-discord.scriptkitty.cafe
     Ko-Fi: https://ko-fi.com/k3yomi
-    Documentation: http://localhost/documentation | https://atmosphericx.scriptkitty.cafe/documentation
+    Documentation: https://atmosphericx.scriptkitty.cafe/documentation
 
-    Internal Package: @atmosx/event-product-parser
+    Independent Package: @atmosx/event-product-parser
 
 */
 
-import { Bootstrap } from "@bootstrap"
-import { CreateHttp } from "@utilities/CreateHttp"
-import { SetWarning } from "@utilities/SetWarning"
+import { Bootstrap } from "@Bootstrap"
+import { CreateHttp } from "@Utilities/CreateHttp"
+import { SetWarning } from "@Utilities/SetWarning"
+import { CreateQuery } from "@Database/CreateQuery";
 
 export const ImportBroadcastify = async (): Promise<void> => {
     const settings = Bootstrap.Settings;
-    
-    const isBroadcastifyImported = Bootstrap.Database
-        .prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name='broadcastify';`)
-        .get();
-
-    if (!settings.BroadcastifySettings.BroadcastifyAttachments) { return }
-    if (isBroadcastifyImported) { return } 
-
     const broadcastify = await CreateHttp({
         URL: settings.BroadcastifySettings.BroadcastifyDatabase,
         Timeout: 5000,
     });
     if (!broadcastify.error) {  
-        Bootstrap.Database.prepare(`CREATE TABLE broadcastify ( state TEXT, county TEXT, feed TEXT, type TEXT, link TEXT);`)
-            .run();
         const states = JSON.parse(broadcastify.message);
-        const insert = Bootstrap.Database.prepare(`INSERT INTO broadcastify (state, county, feed, type, link) VALUES (?, ?, ?, ?, ?)`);
+        const insert = `INSERT INTO broadcastify (state, county, feed, type, link) VALUES (?, ?, ?, ?, ?)`;
         const transaction = Bootstrap.Database.transaction((rows: any[]) => {
             for (const row of rows) {
-                insert.run(row);
+                CreateQuery({ Query: insert, Parameters: row });
             }
         });
         const batch: any[] = [];

package/src/components/database/ImportCities.ts

@@ -0,0 +1,87 @@
+/*
+              _                             _               _     __   __
+         /\  | |                           | |             (_)    \ \ / /
+        /  \ | |_ _ __ ___   ___  ___ _ __ | |__   ___ _ __ _  ___ \ V / 
+       / /\ \| __| '_ ` _ \ / _ \/ __| '_ \| '_ \ / _ \ '__| |/ __| > <  
+      / ____ \ |_| | | | | | (_) \__ \ |_) | | | |  __/ |  | | (__ / . \ 
+     /_/    \_\__|_| |_| |_|\___/|___/ .__/|_| |_|\___|_|  |_|\___/_/ \_\
+                                     | |                            
+                                     |_|                                                                                                                
+
+    Created with ♥ by the AtmosphericX Team (KiyoWx, StarflightWx, & CJ Ziegler)
+    Discord: https://atmosphericx-discord.scriptkitty.cafe
+    Ko-Fi: https://ko-fi.com/k3yomi
+    Documentation: https://atmosphericx.scriptkitty.cafe/documentation
+
+    Independent Package: @atmosx/event-product-parser
+
+*/
+import { EnumStates } from "@Enums/States"
+import { Bootstrap } from "@Bootstrap"
+import { CreateHttp } from "@Utilities/CreateHttp"
+import { CreateQuery } from "@Database/CreateQuery"
+import { SetWarning } from "@Utilities/SetWarning"
+
+interface CityRow {
+    id: string
+    name: string
+    state: string
+    county: string
+    population: number | string
+    lat: number | string
+    lon: number | string
+}
+
+export const ImportCities = async (): Promise<void> => {
+    const cities = await CreateHttp({
+        URL: Bootstrap.Settings.BoundarySettings.CityDatabase,
+        Timeout: 5000
+    })
+
+    if (cities.error || !cities.message) {
+        SetWarning({ Message: `Failed to download city database` })
+        return
+    }
+
+    let database: unknown
+
+    try {
+        database = JSON.parse(cities.message)
+    } catch {
+        SetWarning({ Message: `Failed to parse city database` })
+        return
+    }
+
+    if (!Array.isArray(database)) {
+        SetWarning({ Message: `Failed to parse city database` })
+        return
+    }
+
+    const StateCodes: Record<string, string> = Object.fromEntries(
+        Object.entries(EnumStates).map(([code, name]) => [name, code])
+    );
+    const insert = `INSERT INTO cities (id, name, state, county, population, lat, lon) VALUES (?, ?, ?, ?, ?, ?, ?)`
+    const cityRows: CityRow[] = []
+    for (const city of database) {
+        if (city.country != `US`) continue
+        cityRows.push({
+            id: city.id,
+            name: city.name,
+            state: StateCodes[city.admin1] ?? null,
+            county: city.admin2,
+            population: city.pop ?? null,
+            lat: city.lat,
+            lon: city.lon
+        })
+    }
+    SetWarning({ Message: `Importing ${cityRows.length} cities` })
+    const transaction = Bootstrap.Database.transaction((rows: CityRow[]) => {
+        for (const row of rows) {
+            CreateQuery({
+                Query: insert,
+                Parameters: [row.id, row.name, row.state, row.county, row.population, row.lat, row.lon]
+            })
+        }
+    })
+    transaction(cityRows)
+}

package/src/{@modules/@database → components/database}/ImportShapefiles.ts

@@ -11,17 +11,17 @@
     Created with ♥ by the AtmosphericX Team (KiyoWx, StarflightWx, & CJ Ziegler)
     Discord: https://atmosphericx-discord.scriptkitty.cafe
     Ko-Fi: https://ko-fi.com/k3yomi
-    Documentation: http://localhost/documentation | https://atmosphericx.scriptkitty.cafe/documentation
+    Documentation: https://atmosphericx.scriptkitty.cafe/documentation
 
-    Internal Package: @atmosx/event-product-parser
+    Independent Package: @atmosx/event-product-parser
 
 */
 
-import { TypeSettings } from "types/Settings"
-import { EnumShapefiles } from "@enums/Shapefiles"
-import { Bootstrap } from "@bootstrap"
-import { SetSleep } from "@utilities/SetSleep"
-import { SetWarning } from "@utilities/SetWarning"
+import { EnumShapefiles } from "@Enums/Shapefiles"
+import { Bootstrap } from "@Bootstrap"
+import { SetSleep } from "@Utilities/SetSleep"
+import { SetWarning } from "@Utilities/SetWarning"
+import { CreateQuery } from "@Database/CreateQuery"
 import { existsSync, mkdirSync, writeFileSync, unlinkSync, rm } from "fs"
 import { resolve, extname} from "path"
 import { loadAsync } from "jszip"
@@ -29,10 +29,8 @@ import { read } from "shapefile"
 
 export const ImportShapefiles = async (): Promise<void> => {
     try { 
-        const tShapefiles = Bootstrap.Database
-            .prepare(`SELECT COUNT(*) AS count FROM shapefiles`)
-            .get().count;
-        if (tShapefiles === 0) {
+        const tShapefiles = CreateQuery({ Query: `SELECT COUNT(*) AS count FROM shapefiles` })[0] as { count: number };
+        if (tShapefiles.count === 0) {
             await SetSleep({Timeout: 1e3});
             for (const shapefile of EnumShapefiles) {
                 const response = await fetch(shapefile.link);
@@ -55,8 +53,7 @@ export const ImportShapefiles = async (): Promise<void> => {
                     filepath,
                 );
                 SetWarning({ Message: `Importing ${features.length} features from ${shapefile.name}_${shapefile.id} for Shapefiles` })
-                const insert = Bootstrap.Database
-                    .prepare(`INSERT OR REPLACE INTO shapefiles (id, location, geometry) VALUES (?, ?, ?)`)
+                const insert = `INSERT INTO shapefiles (id, location, source, geometry) VALUES (?, ?, ?, ?)`;
                 const transaction = Bootstrap.Database.transaction((entries: any[]) => {
                     for (const entry of entries) {
                         const { properties, geometry } = entry;
@@ -77,7 +74,7 @@ export const ImportShapefiles = async (): Promise<void> => {
                             final = properties.ID ?? properties.WFO
                             location = properties.NAME;
                         }
-                        insert.run(final, location, JSON.stringify(geometry));
+                        CreateQuery({ Query: insert, Parameters: [final, location, `${shapefile.name}_${shapefile.id}`, JSON.stringify(geometry)] })
                     }
                 })
                 unlinkSync(`${filepath}.shp`)
@@ -86,7 +83,8 @@ export const ImportShapefiles = async (): Promise<void> => {
             }
             rm(resolve(__dirname, 'shapefiles'), { recursive: true, force: true }, () => {});
         }
-    } catch (error) {
-        SetWarning( {Message: `An error occurred while compiling shapefiles: ${error.message}` }) 
+    } catch (error: unknown) {
+        const message = error instanceof Error ? error.message : String(error);
+        SetWarning( {Message: `An error occurred while compiling shapefiles: ${message}` }) 
     }
 }

package/src/{@modules/@database → components/database}/ImportStanza.ts

@@ -11,37 +11,33 @@
     Created with ♥ by the AtmosphericX Team (KiyoWx, StarflightWx, & CJ Ziegler)
     Discord: https://atmosphericx-discord.scriptkitty.cafe
     Ko-Fi: https://ko-fi.com/k3yomi
-    Documentation: http://localhost/documentation | https://atmosphericx.scriptkitty.cafe/documentation
+    Documentation: https://atmosphericx.scriptkitty.cafe/documentation
 
-    Internal Package: @atmosx/event-product-parser
+    Independent Package: @atmosx/event-product-parser
 
 */
 
-import { TypeSettings } from "types/Settings"
-import { TypeStanzaCompiled } from "types/StanzaCompiled"
-import { Bootstrap } from "@bootstrap"
-import { SetWarning } from "@utilities/SetWarning"
+import { TypeSettings } from "Types/Settings"
+import { TypeStanzaCompiled } from "Types/StanzaCompiled"
+import { Bootstrap } from "@Bootstrap"
+import { SetWarning } from "@Utilities/SetWarning"
+import { CreateQuery } from "@Database/CreateQuery"
 
 export const ImportStanza = async (Stanza: TypeStanzaCompiled): Promise<void> => {
     const settings = Bootstrap.Settings as TypeSettings;
     try { 
         if (!settings.NOAAWeatherWireServiceSettings.CacheSettings.Enabled) { return }
-        Bootstrap.Database
-            .prepare(`INSERT OR IGNORE INTO stanzas (type, stanza, issued) VALUES (?, ?, ?)`)
-            .run(Stanza.Type.Type, JSON.stringify(Stanza), Stanza.Attributes.issue)
-        const count = Bootstrap.Database
-            .prepare(`SELECT COUNT(*) as total FROM stanzas`)
-            .get() as { total: number }
+        CreateQuery({ Query: `INSERT OR IGNORE INTO stanzas (type, stanza, issued) VALUES (?, ?, ?)`, Parameters: [Stanza.Type.Type, JSON.stringify(Stanza), Stanza.Attributes.issue] })
+        const count = CreateQuery({ Query: `SELECT COUNT(*) as total FROM stanzas` }) as { total: number };
         const max = settings.NOAAWeatherWireServiceSettings.CacheSettings.MaxDatabaseHistory;
         if (count.total > max) { 
             const toDelete = count.total - max;
             if (toDelete > 0) {
-                Bootstrap.Database
-                    .prepare(`DELETE FROM stanzas WHERE id IN (SELECT id FROM stanzas ORDER BY issued ASC LIMIT ?)`)
-                    .run(toDelete);
+                CreateQuery({ Query: `DELETE FROM stanzas WHERE id IN (SELECT id FROM stanzas ORDER BY issued ASC LIMIT ?)`, Parameters: [toDelete] })
             }
         }
-    } catch (error) {
-        SetWarning({ Message: `An error occurred while importing stanza: ${error.message}` })
+    } catch (error: unknown) {
+        const message = error instanceof Error ? error.message : String(error);
+        SetWarning({ Message: `An error occurred while importing stanza: ${message}` })
     }
 }

package/src/components/database/InitializeDatabase.ts

@@ -0,0 +1,71 @@
+/*
+              _                             _               _     __   __
+         /\  | |                           | |             (_)    \ \ / /
+        /  \ | |_ _ __ ___   ___  ___ _ __ | |__   ___ _ __ _  ___ \ V / 
+       / /\ \| __| '_ ` _ \ / _ \/ __| '_ \| '_ \ / _ \ '__| |/ __| > <  
+      / ____ \ |_| | | | | | (_) \__ \ |_) | | | |  __/ |  | | (__ / . \ 
+     /_/    \_\__|_| |_| |_|\___/|___/ .__/|_| |_|\___|_|  |_|\___/_/ \_\
+                                     | |                            
+                                     |_|                                                                                                                
+
+    Created with ♥ by the AtmosphericX Team (KiyoWx, StarflightWx, & CJ Ziegler)
+    Discord: https://atmosphericx-discord.scriptkitty.cafe
+    Ko-Fi: https://ko-fi.com/k3yomi
+    Documentation: https://atmosphericx.scriptkitty.cafe/documentation
+
+    Independent Package: @atmosx/event-product-parser
+
+*/
+
+import { TypeSettings } from "Types/Settings"
+import { Bootstrap } from "@Bootstrap"
+import { SetWarning } from "@Utilities/SetWarning"
+import { ImportShapefiles } from "@Database/ImportShapefiles"
+import { ImportBroadcastify } from "@Database/ImportBroadcastify"
+import { ImportBoundaries } from "@Database/ImportBoundaries"
+import { ImportCities } from "@Database/ImportCities"
+import { CreateQuery } from "@Database/CreateQuery"
+import { existsSync, writeFileSync } from "fs"
+import sqlite3 from "better-sqlite3"
+
+
+export const InitializeDatabase = async (): Promise<void> => {
+    const settings = Bootstrap.Settings as TypeSettings;
+    try { 
+        if (!existsSync(settings.Database)) { 
+            writeFileSync(settings.Database, '')
+            SetWarning({ Message: `Creating new database at ${settings.Database}` })
+        }
+        Bootstrap.Database = new sqlite3(settings.Database);
+        CreateQuery({ Query: `CREATE TABLE IF NOT EXISTS stanzas ( id INTEGER PRIMARY KEY AUTOINCREMENT, type TEXT, issued TEXT, stanza TEXT )` })
+        const isNeedingShapefiles = CreateQuery({ Query: `SELECT name FROM sqlite_master WHERE type='table' AND name='shapefiles';` }) as { name: string }[];
+        const isNeedingBroadcastify = CreateQuery({ Query: `SELECT name FROM sqlite_master WHERE type='table' AND name='broadcastify';` }) as { name: string }[];
+        const isNeedingBoundaries = CreateQuery({ Query: `SELECT name FROM sqlite_master WHERE type='table' AND name='boundaries';` }) as { name: string }[];
+        const isNeedingCities = CreateQuery({ Query: `SELECT name FROM sqlite_master WHERE type='table' AND name='cities';` }) as { name: string }[];
+
+        if (isNeedingShapefiles.length === 0 || isNeedingBroadcastify.length === 0 || isNeedingBoundaries.length === 0 || isNeedingCities.length === 0) {
+            CreateQuery({ Query: `CREATE TABLE IF NOT EXISTS broadcastify ( state TEXT, county TEXT, feed TEXT, type TEXT, link TEXT)` })
+            CreateQuery({ Query: `CREATE TABLE IF NOT EXISTS shapefiles ( id, location TEXT, source TEXT, geometry TEXT )` })
+            CreateQuery({ Query: `CREATE INDEX IF NOT EXISTS idx_shapefiles_id ON shapefiles (id)` })
+            CreateQuery({ Query: `CREATE TABLE IF NOT EXISTS boundaries (id TEXT PRIMARY KEY, type TEXT NOT NULL, state TEXT, name TEXT, geometry TEXT NOT NULL );` })
+            CreateQuery({ Query: `CREATE TABLE IF NOT EXISTS cities ( id TEXT PRIMARY KEY, name TEXT, state TEXT, county TEXT, population TEXT, lat REAL NOT NULL, lon REAL NOT NULL);`});
+            SetWarning({Message: `Required database tables are currently building, please ${Bootstrap.Colors.Red}DO NOT${Bootstrap.Colors.Reset} close your terminal. The building will not finish and will remain incomplete. If you do mess up, you will need to delete ${settings.Database} and restart the application.` })
+            if (isNeedingCities.length === 0) {
+                await ImportCities();
+            }
+            if (isNeedingBroadcastify.length === 0) {
+                await ImportBroadcastify();
+            }
+            if (isNeedingBoundaries.length === 0) {
+                await ImportBoundaries();
+            }
+            if (isNeedingShapefiles.length === 0) {
+                await ImportShapefiles();
+            }
+            SetWarning({ Message: `Database initialization complete. You may now close your terminal or continue using the application.` })
+        }
+    } catch (error: unknown) {
+        const message = error instanceof Error ? error.message : String(error);
+        SetWarning({ Message: `An error occurred while initializing the database: ${message}` })
+    }
+}