hfbcast 1.0.0

package/sqlite/build.ts

@@ -0,0 +1,127 @@
+import * as fs from 'fs';
+import * as url from 'url';
+import { AppDataSource } from "./data-source";
+import {Site} from "./entity/Site";
+import {Broadcast} from "./entity/Broadcast";
+import {JSDOM} from "jsdom";
+const jquery = require("jquery");
+
+function parseBroadcasts(sourceName) {
+
+    return new Promise(resolve => {
+
+        const broadcasts = [];
+        const sites = [];
+
+        fs.readFile(`../sources/shortwave_${sourceName}.html`, 'utf8', async function(err, testHTML) {
+
+            global.document = new JSDOM( testHTML );
+            const $ = jquery(global.document.window);
+            const rows = $('table.results tr');
+
+            rows.each(async (i, row) => {
+
+                // if (i===0 || i>100) return false;
+                if (i===0) return false;
+                if (i%1000 === 0) console.log(sourceName, i);
+
+                const cells = $(row).find('td');
+
+                const fq = Number.parseFloat($(cells[0]).html());
+                const startStopTime = $(cells[1]).html().split('-');
+                const station = $(cells[2]).html();
+                const country = $(cells[3]).html();
+                const language = $(cells[4]).html();
+                const days = $(cells[5]).html();
+                const link = $(cells[6]).html();
+                let lon=0, lat=0, siteName='unknown';
+                try {
+                    const a = $(link);
+                    // console.log(a.length);
+                    siteName = a.length === 1 ? a.html() : link;
+                    const coordString = (new url.URL(a.attr('href'))).searchParams.get('center').split(',');
+                    lat = Number.parseFloat(coordString[0]);
+                    lon = Number.parseFloat(coordString[1]);
+                }
+                catch (e) {
+                }
+
+                const startHour = Number.parseInt(startStopTime[0].substring(0,2));
+                const startMin = Number.parseInt(startStopTime[0].substring(2,4));
+                const stopHour = Number.parseInt(startStopTime[1].substring(0,2));
+                const stopMin = Number.parseInt(startStopTime[1].substring(2,4));
+
+                let daysInteger = 0;
+                for (let i = 0; i < days.length; i++) {
+                    const dayInt = parseInt(days.charAt(i));
+                    if (!isNaN(dayInt)) {
+                        daysInteger += Math.pow(2, dayInt-1);
+                    }
+                }
+
+                // console.log(fq, lon, lat, startStopTime, startHour, stopHour, days);
+
+                const site = new Site();
+                site.name = siteName;
+                site.power = 0;
+                site.lon = lon;
+                site.lat = lat;
+
+                const broadcast = new Broadcast();
+                broadcast.frequency = fq;
+                broadcast.site = site;
+                broadcast.days = daysInteger;
+                broadcast.startTime = startMin + 60*startHour;
+                broadcast.endTime = stopMin + 60*stopHour;
+                broadcast.station = station;
+                broadcast.country = country;
+                broadcast.language = language;
+                broadcast.source = sourceName;
+
+                sites.push(site);
+                broadcasts.push(broadcast);
+            })
+
+            resolve(broadcasts);
+        });
+    })
+}
+
+AppDataSource.initialize().then(async () => {
+
+    console.log('PARSING HTML ...');
+
+    const broadcasts1: any = await parseBroadcasts('AOKI');
+    console.log('...', broadcasts1.length, 'BROADCASTS FOUND IN AOKI');
+    const broadcasts2: any = await parseBroadcasts('EIBI');
+    console.log('...', broadcasts2.length, 'BROADCASTS FOUND IN EIBI');
+    const broadcasts3: any = await parseBroadcasts('HFCC');
+    console.log('...', broadcasts3.length, 'BROADCASTS FOUND IN HFCC');
+
+    const broadcasts = [... broadcasts1, ... broadcasts2, ... broadcasts3]
+
+    console.log(broadcasts.length, 'BROADCASTS FOUND IN TOTAL');
+
+    console.log('POPULATING DATABASE ...');
+
+    for (let i=0; i<broadcasts.length; i++) {
+        try {
+            if (i%1000 === 0) console.log(i);
+            // await siteRepository.save(broadcasts[i].site);
+
+            const siteName = broadcasts[i].site.name;
+            const site = await AppDataSource.manager.findOne(Site, {where: {name: siteName}});
+            if (site) {
+                broadcasts[i].site = site;
+            }
+            else {
+               await AppDataSource.manager.save(broadcasts[i].site);
+            }
+            await AppDataSource.manager.save(broadcasts[i]);
+        }
+        catch (e) {
+            console.log(i, 'NOT UNIQUE', broadcasts[i]);
+        }
+    }
+
+}).catch(error => console.log(error))

package/sqlite/data-source.ts

@@ -0,0 +1,17 @@
+import "reflect-metadata"
+import { DataSource } from "typeorm"
+import { Site } from "./entity/Site"
+import { Broadcast } from "./entity/Broadcast"
+
+const currentTime = new Date();
+
+export const AppDataSource = new DataSource({
+    type: "sqlite",
+    // database: `database_${currentTime.getFullYear()}-${currentTime.getMonth()}-${currentTime.getDate()}_${currentTime.getHours()}${currentTime.getMinutes()}${currentTime.getSeconds()}.sqlite`,
+    database: `database.sqlite`,
+    synchronize: true,
+    logging: false,
+    entities: [Site, Broadcast],
+    migrations: [],
+    subscribers: [],
+})

package/sqlite/entity/Broadcast.ts

@@ -0,0 +1,38 @@
+import {Entity, PrimaryGeneratedColumn, Column, ManyToOne, JoinColumn, Unique} from "typeorm";
+import {Site} from "./Site";
+
+@Entity()
+@Unique(['frequency', 'days', 'startTime', 'endTime', 'station', 'country', 'language', 'site'])
+export class Broadcast {
+
+    @PrimaryGeneratedColumn()
+    id: number;
+
+    @Column({nullable: true})
+    frequency: number;
+
+    @Column({nullable: true})
+    days: number;
+
+    @Column({nullable: true})
+    startTime: number;
+
+    @Column({nullable: true})
+    endTime: number;
+
+    @Column({nullable: true})
+    station: string;
+
+    @Column({nullable: true})
+    country: string;
+
+    @Column({nullable: true})
+    language: string;
+
+    @Column({nullable: true})
+    source: string;
+
+    @ManyToOne(type => Site)
+    @JoinColumn()
+    site: Site;
+}

package/sqlite/entity/Site.ts

@@ -0,0 +1,26 @@
+import {Entity, PrimaryGeneratedColumn, Column, Unique, OneToMany} from "typeorm";
+import {Broadcast} from "./Broadcast";
+
+@Entity()
+@Unique(['name'])
+export class Site {
+
+    @PrimaryGeneratedColumn()
+    id: number;
+
+    @Column()
+    name: string;
+
+    @Column()
+    power: number;
+
+    @Column()
+    lon: number;
+
+    @Column()
+    lat: number;
+
+    @OneToMany(type => Broadcast, broadcasts => broadcasts.site)
+    broadcasts: Broadcast[];
+
+}

package/sqlite/package.json

@@ -0,0 +1,17 @@
+{
+   "name": "sqlite",
+   "version": "1.0.1",
+   "description": "",
+   "dependencies": {
+      "@types/node": "^18.13.0",
+      "jsdom": "^21.1.0",
+      "reflect-metadata": "^0.1.13",
+      "sqlite3": "^5.1.4",
+      "ts-node": "^10.9.1",
+      "typeorm": "^0.3.12",
+      "typescript": "^4.9.5"
+   },
+   "scripts": {
+      "build": "ts-node build.ts"
+   }
+}

package/src/demo.html

@@ -0,0 +1,13 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <title>Title</title>
+</head>
+<body style="font-family: monospace">
+database url ......... <input value="http://localhost:3333/database.sqlite.gz" style="width:300px"><br>
+min frequency (kHz) .. <input value="0"><br>
+max frequency (kHz) .. <input value="30000"><br><br>
+<button>get broadcasts</button><br>
+</body>
+</html>

package/src/demo.js

@@ -0,0 +1,6 @@
+import {BroadcastDb} from "hfbcast"
+
+(new BroadcastDb('database.sqlite.gz')).then(db => {
+    const result = db.getBroadcastsBetween(0,30000);
+    console.log(result);
+})

package/src/index.d.ts

@@ -0,0 +1,33 @@
+/**
+ * A broadcast database entry
+ */
+declare type broadcastEntry = {
+    values: any[],
+    columns: any[]
+}
+
+/**
+ * A database class for HF broadcasts
+ */
+export declare class BroadcastDb {
+
+    /**
+     * Constructor
+     * @param {string} gzUrl URL of a gzipped sqlite3 database file
+     */
+    constructor();
+
+    /**
+     * Get broadcasts between two frequencies given in KHz
+     * @param {number} minFreqkHz minimum frequency
+     * @param {number} maxFreqkHz maximum frequency
+     */
+    getBroadcastsBetween(minFreqkHz: number, maxFreqkHz: number): broadcastEntry[];
+
+    /**
+     * Get broadcasts in an interval around at a center frequency in KHz
+     * @param {number} frequencykHz center frequency
+     * @param {number} intervalkHz frequency interval
+     */
+    getBroadcastsAt(frequencyKHz: number, intervalKHz: number): broadcastEntry[];
+}

package/src/index.js

@@ -0,0 +1,83 @@
+import dumm from "sql.js";
+import pako from "pako";
+import sqlwasm from 'url-loader!sql.js/dist/sql-wasm.wasm'
+import gzUrl from "file-loader?name=database.sqlite.gz!../assets/database.sqlite.gz";
+
+async function openDb(url) {
+    return new Promise(resolve => {
+        fetch(sqlwasm).then(res => res.blob()).then(async blob => {
+            const sqlPromise = dumm({
+                locateFile: () => URL.createObjectURL(blob)
+            });
+            const dataPromise = fetch(url).then(res => res.arrayBuffer());
+            const [SQL, buf] = await Promise.all([sqlPromise, dataPromise])
+            const db = new SQL.Database(new Uint8Array(pako.ungzip(buf)));
+            resolve(db);
+        })
+    })
+}
+
+export class BroadcastDb {
+
+    constructor() {
+        return new Promise(async resolve => {
+            this.db = await openDb(gzUrl);
+            resolve(this)
+        })
+    }
+
+    getBroadcastsBetween(minFreqkHz, maxFreqkHz) {
+
+        const now = new Date();
+        const currentDay = now.getUTCDay();
+        const currentHour = now.getUTCHours();
+        const currentMinute = now.getUTCMinutes();
+        const currentTime = currentMinute + 60*currentHour;
+
+        const query = `SELECT * FROM broadcast 
+         JOIN site on broadcast.siteId = site.id 
+         WHERE broadcast.startTime <= ${currentTime} AND 
+           broadcast.endTime >= ${currentTime} AND 
+           NOT (site.lat = 0 AND site.lon = 0) AND
+         broadcast.days & (1 << ${currentDay}) != 0 AND
+         broadcast.frequency <= ${maxFreqkHz} AND
+         broadcast.frequency >= ${minFreqkHz}
+    `
+        const res = this.db.exec(query);
+
+        return res[0];
+    }
+
+    getBroadcastsAt(frequencyKHz, intervalKHz) {
+
+        const now = new Date();
+        const currentDay = now.getUTCDay();
+        const currentHour = now.getUTCHours();
+        const currentMinute = now.getUTCMinutes();
+        const currentTime = currentMinute + 60*currentHour;
+
+        const query = `SELECT * FROM broadcast 
+         JOIN site on broadcast.siteId = site.id 
+         WHERE broadcast.startTime <= ${currentTime} AND 
+               broadcast.endTime >= ${currentTime} AND 
+               NOT (site.lat = 0 AND site.lon = 0) AND
+             broadcast.days & (1 << ${currentDay}) != 0 AND
+             broadcast.frequency <= ${intervalKHz + frequencyKHz/1000} AND
+             broadcast.frequency >= ${-intervalKHz + frequencyKHz/1000}
+        `
+
+        const broadcasts = [];
+        const res = this.db.exec(query);
+        if (res.length > 0) {
+            res[0].values.forEach(c => {
+                const broadcast = {};
+                c.forEach((colvalue, i) => {
+                    broadcast[res[0].columns[i]] = colvalue;
+                })
+                broadcasts.push(broadcast);
+            })
+        }
+
+        return broadcasts;
+    }
+}

package/tsconfig.json

@@ -0,0 +1,15 @@
+{
+   "compilerOptions": {
+      "lib": [
+         "es5",
+         "es6"
+      ],
+      "target": "es5",
+      "module": "commonjs",
+      "moduleResolution": "node",
+      "outDir": "./build",
+      "emitDecoratorMetadata": true,
+      "experimentalDecorators": true,
+      "sourceMap": true
+   }
+}

package/webpack.config.js

@@ -0,0 +1,28 @@
+const HtmlWebpackPlugin = require('html-webpack-plugin');
+
+module.exports = {
+    mode: 'development',
+    entry: {
+        rfdfhfbroadcast: './src/index.js',
+        demo: './src/demo.js'
+    },
+    output: {
+        filename: '[name].js',
+        library: 'rfdfhfbroadcast',
+        libraryTarget: 'umd'
+    },
+    resolve: {
+        extensions: ['.tsx', '.ts', '.js'],
+        fallback: { crypto: false, path: false, fs: false, util: false }
+    },
+    devServer: {
+        port: 4444,
+        static: ['assets']
+    },
+    plugins: [
+        new HtmlWebpackPlugin({
+            chunks: ['demo'],
+            template: "./src/demo.html"
+        })
+    ],
+}