@powercalc/power-router 1.0.63 → 1.0.65

package/src/router/services/Eurostat.ts

@@ -1,83 +1,126 @@
-import { readFileSync } from "fs";
-import { join } from "path";
-import { Country } from "./Loader";
-
-process.env["NODE_TLS_REJECT_UNAUTHORIZED"] = '0';
+import { readFileSync } from 'fs'
+import { join } from 'path'
+import { Country } from './Loader'
+import { Format, QueryOptions } from '../interfaces/queryoptions'
 
+process.env['NODE_TLS_REJECT_UNAUTHORIZED'] = '0'
 
 interface Fuel {
-    chartType: string;
-    siec: string;
-    nrg_bal: string;
-    unit: string;
-    freq: string;
+  chartType: string
+  siec: string
+  nrg_bal: string
+  unit: string
+  freq: string
 }
 
 interface Fuels {
-    [key: string]: Fuel;
+  [key: string]: Fuel
 }
 
-
 export class EurostatAPI {
-    countries: Country[] = [];
-    constructor() {
-        const countryPath = join(__dirname, '..', 'config/countries.json')
-        const countriesString = readFileSync(countryPath, 'utf-8')
-        this.countries = JSON.parse(countriesString);
-    }
-    private fuels: Fuels = {
-        gas: { chartType: 'NRG_CB_GASM', siec: 'G3000', nrg_bal: 'IC_OBS', unit: 'TJ_GCV', freq: 'M' },
-        gas4electricity: { chartType: 'NRG_CB_GASM', siec: 'G3000', nrg_bal: 'TI_EHG_MAP', unit: 'TJ_GCV', freq: 'M' },
-        gasoline: { chartType: 'NRG_CB_OILM', siec: 'O4652', nrg_bal: 'GID_OBS', unit: 'THS_T', freq: 'M' },
-        diesel: { chartType: 'NRG_CB_OILM', siec: 'O46711', nrg_bal: 'GID_OBS', unit: 'THS_T', freq: 'M' },
-        kerosin: { chartType: 'NRG_CB_OILM', siec: 'O4661', nrg_bal: 'GID_OBS', unit: 'THS_T', freq: 'M' }
-    };
-    private energyConversionFactors = new Map([
-        ['KG_EQ', 12988.48], // kg of oil equivalent to J
-        ['TJ_GCV', 10 ** 12], // joules per terajoule of gross calorific value
-        ['THS_T', 41868000000000], // joules per thousand metric tons of oil equivalent
-    ]);
+  countries: Country[] = []
+  constructor() {
+    const countryPath = join(__dirname, '..', 'config/countries.json')
+    const countriesString = readFileSync(countryPath, 'utf-8')
+    this.countries = JSON.parse(countriesString)
+  }
+  private fuels: Fuels = {
+    gas: {
+      chartType: 'NRG_CB_GASM',
+      siec: 'G3000',
+      nrg_bal: 'IC_OBS',
+      unit: 'TJ_GCV',
+      freq: 'M',
+    },
+    gas4electricity: {
+      chartType: 'NRG_CB_GASM',
+      siec: 'G3000',
+      nrg_bal: 'TI_EHG_MAP',
+      unit: 'TJ_GCV',
+      freq: 'M',
+    },
+    gasoline: {
+      chartType: 'NRG_CB_OILM',
+      siec: 'O4652',
+      nrg_bal: 'GID_OBS',
+      unit: 'THS_T',
+      freq: 'M',
+    },
+    diesel: {
+      chartType: 'NRG_CB_OILM',
+      siec: 'O46711',
+      nrg_bal: 'GID_OBS',
+      unit: 'THS_T',
+      freq: 'M',
+    },
+    kerosin: {
+      chartType: 'NRG_CB_OILM',
+      siec: 'O4661',
+      nrg_bal: 'GID_OBS',
+      unit: 'THS_T',
+      freq: 'M',
+    },
+  }
+  private energyConversionFactors = new Map([
+    ['KG_EQ', 12988.48], // kg of oil equivalent to J
+    ['TJ_GCV', 10 ** 12], // joules per terajoule of gross calorific value
+    ['THS_T', 41868000000000], // joules per thousand metric tons of oil equivalent
+  ])
 
-    convertToMWh(value: number, unit: string) {
-        const MWh = value * (this.energyConversionFactors.get(unit) || 0) / 3600 / 10 ** 6;
-        return Math.round(MWh);
-    }
+  convertToMWh(value: number, unit: string) {
+    const MWh =
+      (value * (this.energyConversionFactors.get(unit) || 0)) / 3600 / 10 ** 6
+    return Math.round(MWh)
+  }
 
-    private async fetchData(url: string, type: string, unit: string): Promise<any> {
-        const response = await fetch(url);
-        const data: any = await response.json();
-        try {
-            const time = Object.keys(data.dimension.time.category.index);
-            return {
-                time: time.map(item => new Date(item).getTime()),
-                [type]: Object.values(data.value).map((item: any) => this.convertToMWh(item, unit))
-            }
-        } catch (e) {
-            return { 
-                [type]:[],
-                time: [] 
-            }
-        }
+  private async fetchData(
+    url: string,
+    type: string,
+    unit: string
+  ): Promise<any> {
+    const response = await fetch(url)
+    const data: any = await response.json()
+    try {
+      const time = Object.keys(data.dimension.time.category.index)
+      return {
+        time: time.map((item) => new Date(item).getTime()),
+        [type]: Object.values(data.value).map((item: any) =>
+          this.convertToMWh(item, unit)
+        ),
+      }
+    } catch (e) {
+      return {
+        [type]: [],
+        time: [],
+      }
     }
+  }
 
-    async load(countryCode: string): Promise<any> {
-        const countryShort = this.countries.find(item => item.code === countryCode)?.short;
-        const baseUrl = 'https://ec.europa.eu/eurostat/api/dissemination/sdmx/2.1/data/';
-        const geo = `.${countryShort}`;
-
-        const promises = []
-        for (const [type, fuel] of Object.entries(this.fuels)) {
-            const { chartType, siec, nrg_bal, unit, freq } = fuel;
-            const select = `${chartType}/${freq}.${nrg_bal}.${siec}.${unit}`;
-            const rest = `?format=JSON&lang=en&startPeriod=2015-01&endPeriod=2050-12`;
-            const url = `${baseUrl}${select}${geo}${rest}`;
-            promises.push(this.fetchData(url, type, unit))
-        }
-        const res = await Promise.all(promises)
-        let merged: any = {};
-        res.forEach(item => {
-            merged = { ...merged, ...item };
-        })
-        return merged; // this.removeElGas(merged);
+  async load(countryCode: string, queryOptions: QueryOptions): Promise<any> {
+    const countryShort = this.countries.find(
+      (item) => item.code === countryCode
+    )?.short
+    const baseUrl =
+      'https://ec.europa.eu/eurostat/api/dissemination/sdmx/2.1/data/'
+    const geo = `.${countryShort}`
+    const urls = [];
+    const promises = []
+    for (const [type, fuel] of Object.entries(this.fuels)) {
+      const { chartType, siec, nrg_bal, unit, freq } = fuel
+      const select = `${chartType}/${freq}.${nrg_bal}.${siec}.${unit}`
+      const rest = `?format=JSON&lang=en&startPeriod=2015-01&endPeriod=2050-12`
+      const url = `${baseUrl}${select}${geo}${rest}`
+      urls.push(url)
+      promises.push(this.fetchData(url, type, unit))
+    }
+    if(queryOptions.format === Format.SOURCES){
+        return urls;
     }
+    const res = await Promise.all(promises)
+    let merged: any = {}
+    res.forEach((item) => {
+      merged = { ...merged, ...item }
+    })
+    return merged // this.removeElGas(merged);
+  }
 }

package/src/router/services/LoadService.ts

@@ -1,96 +1,145 @@
-import { readFileSync } from "fs";
-import { join } from "path";
-import { EntsoeLoader, LoaderConfig } from "./Loader";
-import { DataPoint, EdifactParser } from "./ParseEdifact";
-import { ParseInstalled } from "./ParseInstalled";
-import { CommonTimestamps } from "./CommonTimestamps";
-import { EurostatAPI } from "./Eurostat";
-import { Types } from "../interfaces/types";
-import { Format, QueryOptions } from "../interfaces/queryoptions";
-import { NaturalGasPrices } from "./NaturalGasPrice";
+import { readFileSync } from 'fs'
+import { join } from 'path'
+import { EntsoeLoader, LoaderConfig } from './Loader'
+import { DataPoint, EdifactParser } from './ParseEdifact'
+import { ParseInstalled } from './ParseInstalled'
+import { CommonTimestamps } from './CommonTimestamps'
+import { EurostatAPI } from './Eurostat'
+import { Types } from '../interfaces/types'
+import { Format, QueryOptions } from '../interfaces/queryoptions'
+import { NaturalGasPrices } from './NaturalGasPrice'
+import { Entsoe } from '../interfaces/entsoe'
+import { EnergyCharts } from './EnergyCharts'
 
 export class LoadService {
-    country = '';
-    loader: EntsoeLoader;
-    parser: EdifactParser;
-    parseInstalled: ParseInstalled;
+  country = ''
+  loader: EntsoeLoader
+  parser: EdifactParser
+  parseInstalled: ParseInstalled
 
-    constructor(private loaderConfig: LoaderConfig) {
-        const typesPath = join(__dirname, '../config/types.json'); // Set the path to your config file
-        const types = JSON.parse(readFileSync(typesPath, 'utf8')) as Types; // Read the config file from disk
+  constructor(private loaderConfig: LoaderConfig) {
+    const typesPath = join(__dirname, '../config/types.json') // Set the path to your config file
+    const types = JSON.parse(readFileSync(typesPath, 'utf8')) as Types // Read the config file from disk
 
-        this.loader = new EntsoeLoader(loaderConfig);
-        this.parser = new EdifactParser(types);
-        this.parseInstalled = new ParseInstalled(types);
+    this.loader = new EntsoeLoader(loaderConfig)
+    this.parser = new EdifactParser(types)
+    this.parseInstalled = new ParseInstalled(types)
+  }
+  async getPowerEnergyCharts(
+    countryCode: string,
+    year: number,
+    month: number,
+  ): Promise<any> {
+    const startMonth = new Date(Date.UTC(year, month - 1)).toISOString()
+    const endMonth = new Date(Date.UTC(year, month)).toISOString()
+    const energyCharts = new EnergyCharts()
+    const power = await energyCharts.load(countryCode, startMonth, endMonth)
+    return power
+  }
+  async getPower(
+    countryCode: string,
+    year: number,
+    month: number,
+    queryOptions?: QueryOptions
+  ): Promise<any> {
+    const startMonth = new Date(Date.UTC(year, month - 1)).toISOString()
+    const endMonth = new Date(Date.UTC(year, month)).toISOString()
+    const wanted = ['generation', 'price', 'consumption']
+    if (queryOptions?.format === Format.SOURCES) {
+      return this.loader.makeUrls(wanted, countryCode, year, month)
     }
-
-    async getPower(countryCode: string, year: number, month: number, queryOptions?: QueryOptions):Promise<any> {
-        const wanted = ['generation', 'price', 'consumption'];
-        if (queryOptions?.format === Format.SOURCES) {
-            return this.loader.makeUrls(wanted, countryCode, year, month);
-        }
-        const edifactAsJson = await this.loader.load(wanted, countryCode, year, month, queryOptions);
-        if (queryOptions?.format === Format.EDIJSON) {
-            return edifactAsJson;
-        }
-        const parsed = await this.parser.parseAll(edifactAsJson);
-        if (queryOptions?.format === Format.KEYVALUE) {
-            return parsed;
-        }
-        const startMonth = (new Date(Date.UTC(year, month - 1))).toISOString()
-        const endMonth = (new Date(Date.UTC(year, month))).toISOString()
-        const ready = CommonTimestamps.transform(parsed, startMonth, endMonth);
-        return ready;
+    const edifactAsJson = await this.loader.load(
+      wanted,
+      countryCode,
+      year,
+      month,
+      queryOptions
+    )
+    if (queryOptions?.format === Format.EDIJSON) {
+      return edifactAsJson
     }
-
-    async getInstalled(countryCode: string, reload: boolean) {
-        const edifactAsJson = await this.loader.loadInstalled(countryCode, reload)
-        const ready = await this.parseInstalled.parse(edifactAsJson);
-        return ready;
+    const parsed = await this.parser.parseAll(edifactAsJson)
+    if (queryOptions?.format === Format.KEYVALUE) {
+      return parsed
+    }
+    const ready: any = CommonTimestamps.transform(parsed, startMonth, endMonth)
+    //console.log('e.price',e.price)
+    /*
+    if (!e.price) {
+      e.price = ready.price
     }
+    */
+    return ready
+  }
 
-    async getHydrofill(countryCode: string, reload: boolean) {
-        const wanted = ['hydroFill'];
-        const endYear = new Date().getFullYear() + 1;
-        let all: DataPoint[] = []
-        for (let year = 2015; year < endYear; year++) {
-            const month = 1;
-            const edifactAsJson = await this.loader.load(wanted, countryCode, year, month, { reload })
-            const parsed = await this.parser.parseAll(edifactAsJson);
-            if (parsed?.hydroFill) {
-                const yearArr = Object.values(parsed?.hydroFill)?.[0];
-                if (yearArr) {
-                    all = all.concat(yearArr);
-                }
-            }
-        }
-        const keyValue = all.filter((obj, index, self) => {
-            return index === self.findIndex((o) => o.time === obj.time);
-        });
-        return {
-            time: keyValue.map(item => item.time),
-            hydroFill: keyValue.map(item => item.value)
+  async getInstalled(countryCode: string, queryOptions: QueryOptions) {
+    /*
+        if(queryOptions.format === Format.SOURCES){
+            return this.loader.makeUrls(['installed'], countryCode,1,1)
         }
+        */
+    const edifactAsJson = await this.loader.loadInstalled(
+      countryCode,
+      queryOptions
+    )
+    if (queryOptions.format === Format.SOURCES) {
+      return edifactAsJson
     }
+    const ready = await this.parseInstalled.parse(edifactAsJson as Entsoe[])
+    return ready
+  }
 
-    async getFossil(country: string) {
-        const eurostat = new EurostatAPI();
-        return await eurostat.load(country)
+  async getHydrofill(countryCode: string, queryOptions: QueryOptions) {
+    const wanted = ['hydroFill']
+    const endYear = new Date().getFullYear() + 1
+    let all: DataPoint[] = []
+    for (let year = 2015; year < endYear; year++) {
+      const month = 1
+      const edifactAsJson = await this.loader.load(
+        wanted,
+        countryCode,
+        year,
+        month,
+        queryOptions
+      )
+      console.log(edifactAsJson)
+      if (queryOptions.format === Format.SOURCES) {
+        return [edifactAsJson.hydroFill]
+      }
+      const parsed = await this.parser.parseAll(edifactAsJson)
+      if (parsed?.hydroFill) {
+        const yearArr = Object.values(parsed?.hydroFill)?.[0]
+        if (yearArr) {
+          all = all.concat(yearArr)
+        }
+      }
     }
-
-    async getNaturalGasPrices(country: string) {
-        const eurostat = new NaturalGasPrices();
-        return await eurostat.load(country)
+    const keyValue = all.filter((obj, index, self) => {
+      return index === self.findIndex((o) => o.time === obj.time)
+    })
+    return {
+      time: keyValue.map((item) => item.time),
+      hydroFill: keyValue.map((item) => item.value),
     }
+  }
 
+  async getFossil(country: string, queryOptions: QueryOptions) {
+    const eurostat = new EurostatAPI()
+    return await eurostat.load(country, queryOptions)
+  }
 
-    async getCountrydata(country: string, reload: boolean) {
-        console.log('reload', country, reload)
-        return {
-            fossil: await this.getFossil(country),
-            price: await this.getNaturalGasPrices(country),
-            installed: await this.getInstalled(country, reload),
-            hydrofill: await this.getHydrofill(country, reload)
-        }
+  async getNaturalGasPrices(country: string, queryOptions: QueryOptions) {
+    const eurostat = new NaturalGasPrices()
+    return await eurostat.load(country, queryOptions)
+  }
+
+  async getCountrydata(country: string, queryOptions: QueryOptions) {
+    console.log('reload', country, queryOptions.reload)
+    return {
+      fossil: await this.getFossil(country, queryOptions),
+      price: await this.getNaturalGasPrices(country, queryOptions),
+      installed: await this.getInstalled(country, queryOptions),
+      hydrofill: await this.getHydrofill(country, queryOptions),
     }
-}
+  }
+}

package/src/router/services/Loader.ts

@@ -4,7 +4,7 @@ import { readFile, writeFile } from 'node:fs/promises';
 import { join } from 'node:path';
 import { add, format, sub } from 'date-fns'
 import { Entsoe } from '../interfaces/entsoe';
-import { QueryOptions } from '../interfaces/queryoptions';
+import { Format, QueryOptions } from '../interfaces/queryoptions';
 
 export interface AllEntsoe {
   [key: string]: Entsoe;
@@ -41,8 +41,11 @@ export class EntsoeLoader {
     this.config = loaderConfig;
   }
 
-  async load(wanted: string[], countryCode: string, year: number, month: number, queryOptions?: QueryOptions): Promise<AllEntsoe> {
+  async load(wanted: string[], countryCode: string, year: number, month: number, queryOptions?: QueryOptions): Promise<any> {
     const urls = this.makeUrls(wanted, countryCode, year, month);
+    if(queryOptions?.format === Format.SOURCES){
+      return urls;
+    }
     const allJsons = await this.fetchAndMakeJson(Object.values(urls), queryOptions?.reload);
     const result: AllEntsoe = {}
     Object.keys(urls).forEach((key, i) => {
@@ -89,7 +92,8 @@ export class EntsoeLoader {
     return filteredUrls;
   }
 
-  async loadInstalled(countryCode: string, reload?: boolean) {
+  async loadInstalled(countryCode: string, queryOptions?: QueryOptions) {
+    console.log('8858', queryOptions)
     const currentYear = (new Date()).getFullYear() ;
     const urls = []
     for (let year = 2015; year <= currentYear; year++) {
@@ -98,7 +102,10 @@ export class EntsoeLoader {
       const installed = `${this.config.entsoeDomain}/api?documentType=A68&processType=A33&in_Domain=${countryCode}&periodStart=${periodStart}&periodEnd=${periodEnd}`;
       urls.push(installed)
     }
-    const allJsons = await this.fetchAndMakeJson(Object.values(urls), reload);
+    if(queryOptions?.format===Format.SOURCES){
+      return urls;
+    }
+    const allJsons = await this.fetchAndMakeJson(Object.values(urls), queryOptions?.reload);
     return allJsons;
   }
   async fetchAndMakeJson(urls: string[], reload?: boolean): Promise<Entsoe[]> {

package/src/router/services/NaturalGasPrice.ts

@@ -1,6 +1,7 @@
 import { readFileSync } from "fs"
 import { join } from "path"
 import { Country } from "./Loader"
+import { Format, QueryOptions } from "../interfaces/queryoptions"
 
 export class NaturalGasPrices {
   countries: Country[] = []
@@ -9,12 +10,15 @@ export class NaturalGasPrices {
     const countriesString = readFileSync(countryPath, 'utf-8')
     this.countries = JSON.parse(countriesString)
   }
-  async load(countryCode: string) {
+  async load(countryCode: string, queryOptions:QueryOptions) {
     const countryShort = this.countries.find(
       (item) => item.code === countryCode
     )?.short
 
     const url = `https://ec.europa.eu/eurostat/api/dissemination/sdmx/2.1/data/nrg_pc_203/S.4100.GJ10000-99999.KWH.X_TAX.EUR.${countryShort}/?format=JSON&lang=en&startPeriod=2014&endPeriod=2023`
+    if(queryOptions.format === Format.SOURCES){
+        return [url];
+    }
     const response = await fetch(url)
     const json = await response.json()
     const r = this.convert(json)

package/src/router/services/ParseEdifact.ts

@@ -106,12 +106,6 @@ export class EdifactParser {
 
     // Calculate time distance in milliseconds
     const timeDistance = utcEndTime.toMillis() - utcStartTime.toMillis();
-    if (timeDistance < duration && timeSerie.Period.length !== 1) {
-      console.log('timeDistance', timeDistance, duration, startIsoString, endIsoString)
-    }
-    if (timeDistance > duration) {
-      //console.log('timeDistance2', timeDistance, duration, startIsoString, endIsoString)
-    }
     // Create an array with time distance duration in UTC time using Luxon
     const timeSteps: number[] = [];
     for (let i = 0; i <= timeDistance; i += duration) {

package/src/router/services/batch/maxHydrofill.ts

@@ -14,7 +14,6 @@ export class Hydrofill {
       const h = await this.getMaxHydrofill(country.code)
       if (h.max) {
         const GW = Math.round(h.max / 1000) + ' GW'
-        console.log(country.name, GW)
         country.hydrofill = {
           min: h.min,
           max: h.max
@@ -22,7 +21,6 @@ export class Hydrofill {
       }
     }
     const countryFile = join(__dirname, '../../config', 'countrydata.json')
-    console.log(countryFile)
     const countryDataFile = join(cwd(), 'data/countrydata.json')
     writeFileSync(countryFile, JSON.stringify(countries, null, 2))
   }