@golemio/energetics 2.0.0 → 2.1.0-rc.2970808972

package/db/migrations/postgresql/sqls/20261019090000-enapo-consumption-history-invoice-zero-down.sql

@@ -0,0 +1,492 @@
+DROP MATERIALIZED VIEW IF EXISTS mv_enapo_om_consumption;
+
+CREATE MATERIALIZED VIEW mv_enapo_om_consumption AS
+WITH bounds AS (
+	SELECT
+		((date_trunc('month', now() AT TIME ZONE 'Europe/Prague') - interval '48 months')
+			AT TIME ZONE 'Europe/Prague') AS month_from_tstz,
+		(date_trunc('month', now() AT TIME ZONE 'Europe/Prague') - interval '48 months') AS month_from,
+		(date_trunc('month', now() AT TIME ZONE 'Europe/Prague')
+			AT TIME ZONE 'Europe/Prague') AS month_to_tstz,
+		date_trunc('month', now() AT TIME ZONE 'Europe/Prague') AS month_to
+),
+
+point AS (
+	SELECT DISTINCT gid, place_id, id_type, commodity
+	FROM v_enapo_building_consumption_points
+),
+
+electricity_metered AS (
+	SELECT
+		em.id AS place_id,
+		date_trunc('month', em."timestamp" AT TIME ZONE 'Europe/Prague')::date AS period_start,
+		sum(em.value) FILTER (WHERE em.var = 'EActi_VT_NT_DaySum') AS total_kwh,
+		sum(em.value) FILTER (WHERE em.var = 'EActi_VT_DaySum')    AS vt_kwh,
+		sum(em.value) FILTER (WHERE em.var = 'EActi_NT_DaySum')    AS nt_kwh,
+		-- Coverage is the max day-count across the three vars, not just the total's.
+		GREATEST(
+			count(*) FILTER (WHERE em.var = 'EActi_VT_NT_DaySum'),
+			count(*) FILTER (WHERE em.var = 'EActi_VT_DaySum'),
+			count(*) FILTER (WHERE em.var = 'EActi_NT_DaySum')
+		) AS days_covered
+	FROM enapo_measurements em
+	CROSS JOIN bounds b
+	WHERE em.source = 'predi_input'
+		AND em.var IN ('EActi_VT_NT_DaySum', 'EActi_VT_DaySum', 'EActi_NT_DaySum')
+		AND em."timestamp" >= b.month_from_tstz
+		AND em."timestamp" < b.month_to_tstz
+	GROUP BY em.id, date_trunc('month', em."timestamp" AT TIME ZONE 'Europe/Prague')::date
+),
+
+gas_core_month AS (
+	-- Cumulative 'core' reading (m3) at the first measured hour of each first-of-month day, kept per
+	-- (id, place_id): an id can carry several meters, one per PPAS place id, whose diffs are summed.
+	-- Not per device_serial_number - the integration stamps every refetched hour with the serial AVE
+	-- lists at fetch time, so the serial is unreliable for history and would split one meter in two.
+	SELECT DISTINCT ON (em.id, em.place_id, date_trunc('month', em."timestamp" AT TIME ZONE 'Europe/Prague'))
+		em.id,
+		em.place_id,
+		date_trunc('month', em."timestamp" AT TIME ZONE 'Europe/Prague')::date AS month_start,
+		em.value
+	FROM enapo_measurements em
+	CROSS JOIN bounds b
+	WHERE em.source = 'ppas_ave_api'
+		AND em.var = 'core'
+		AND (em."timestamp" AT TIME ZONE 'Europe/Prague')::date
+			= date_trunc('month', em."timestamp" AT TIME ZONE 'Europe/Prague')::date
+		AND em."timestamp" >= b.month_from_tstz
+		AND em."timestamp" < (b.month_to_tstz + interval '1 month')
+	ORDER BY em.id, em.place_id, date_trunc('month', em."timestamp" AT TIME ZONE 'Europe/Prague'), em."timestamp"
+),
+gas_metered AS (
+	SELECT
+		cur.id AS place_id,
+		cur.month_start AS period_start,
+		-- A meter reading only grows, so a negative month total means a meter was exchanged or reset:
+		-- the month stays empty. Checked on the total, so a tiny drop on a small meter does not empty it.
+		(CASE WHEN sum(nxt.value - cur.value) < 0 THEN NULL ELSE sum(nxt.value - cur.value) END) AS m3_sum,
+		((cur.month_start + interval '1 month')::date - cur.month_start) AS days_covered
+	FROM gas_core_month cur
+	INNER JOIN gas_core_month nxt
+		ON nxt.id = cur.id
+		AND nxt.place_id = cur.place_id
+		AND nxt.month_start = (cur.month_start + interval '1 month')::date
+	GROUP BY cur.id, cur.month_start
+),
+
+distribution_device AS (
+	SELECT DISTINCT ON (ins.eic, d.place_id, d.device_serial_number, d.date_from, d.date_to, d.kind, d.unit)
+		ins.eic,
+		d.place_id,
+		d.invoice_id,
+		d.date_from,
+		d.date_to,
+		d.kind,
+		d.unit,
+		d.consumption,
+		d.normometers_nm3,
+		d.gas_consumption_kwh,
+		d.combustion_heat
+	FROM enapo_ppas_distribution_invoice_device d
+	INNER JOIN enapo_ppas_distribution_invoice i
+		ON i.id = d.invoice_id
+	INNER JOIN enapo_ppas_distribution_installation ins
+		ON ins.invoice_id = d.invoice_id
+		AND ins.customer_id = i.customer_id
+		AND ins.place_id = d.place_id
+	WHERE COALESCE(i.is_canceled, false) = false
+		AND d.date_to >= d.date_from
+		AND d.consumption IS NOT NULL
+		AND d.kind IN ('ZIZWC', 'ZIABN3')
+		AND d.reading_type = 'STANDARD'
+		AND NOT EXISTS (
+			SELECT 1 FROM enapo_ppas_distribution_invoice s
+			WHERE s.preceding_id = i.id AND COALESCE(s.is_canceled, false) = false
+		)
+	ORDER BY ins.eic, d.place_id, d.device_serial_number, d.date_from, d.date_to, d.kind, d.unit,
+		i.facts_doc_date DESC, i.id DESC
+),
+commercial_device AS (
+	SELECT DISTINCT ON (ins.eic, d.place_id, d.device_serial_number, d.date_from, d.date_to, d.kind, d.unit)
+		ins.eic,
+		d.place_id,
+		d.invoice_id,
+		d.date_from,
+		d.date_to,
+		d.kind,
+		d.unit,
+		d.consumption,
+		d.normometers_nm3,
+		d.gas_consumption_kwh,
+		d.combustion_heat
+	FROM enapo_ppas_commercial_invoice_device d
+	INNER JOIN enapo_ppas_commercial_invoice i
+		ON i.id = d.invoice_id
+	INNER JOIN enapo_ppas_commercial_invoice_installations ins
+		ON ins.invoice_id = d.invoice_id
+		AND ins.customer_id = i.customer_id
+		AND ins.place_id = d.place_id
+	WHERE COALESCE(i.is_canceled, false) = false
+		AND d.date_to >= d.date_from
+		AND d.consumption IS NOT NULL
+		AND d.reading_type = 'STANDARD'
+		AND NOT EXISTS (
+			SELECT 1 FROM enapo_ppas_commercial_invoice s
+			WHERE s.preceding_id = i.id AND COALESCE(s.is_canceled, false) = false
+		)
+	ORDER BY ins.eic, d.place_id, d.device_serial_number, d.date_from, d.date_to, d.kind, d.unit,
+		i.facts_doc_date DESC, i.id DESC
+),
+
+invoice_device_month AS (
+	SELECT
+		'ppas_distribution_invoice' AS source,
+		d.eic AS place_id,
+		d.invoice_id,
+		gs.month::date AS period_start,
+		CASE WHEN d.unit = 'KWH' THEN NULL ELSE d.consumption END * o.share AS volume,
+		CASE WHEN d.unit = 'NM3' THEN 'Nm3' ELSE 'm3' END AS volume_unit,
+		d.normometers_nm3 * o.share AS nm3,
+		NULLIF(d.gas_consumption_kwh, 0) * o.share AS kwh,
+		(o.share < 1) AS is_estimated,
+		o.overlap_days AS days_covered
+	FROM distribution_device d
+	CROSS JOIN bounds b
+	CROSS JOIN LATERAL generate_series(
+		date_trunc('month', d.date_from::timestamp),
+		date_trunc('month', d.date_to::timestamp),
+		interval '1 month'
+	) gs(month)
+	CROSS JOIN LATERAL (
+		SELECT
+			ov.overlap_days,
+			ov.overlap_days::numeric / NULLIF(d.date_to - d.date_from + 1, 0) AS share
+		FROM (
+			SELECT LEAST(d.date_to, (gs.month + interval '1 month - 1 day')::date)
+				- GREATEST(d.date_from, gs.month::date) + 1 AS overlap_days
+		) ov
+	) o
+	WHERE gs.month >= b.month_from AND gs.month < b.month_to
+
+	UNION ALL
+
+	SELECT
+		'ppas_commercial_invoice',
+		d.eic,
+		d.invoice_id,
+		gs.month::date,
+		CASE WHEN d.unit = 'KWH' THEN NULL ELSE d.consumption END * o.share,
+		CASE WHEN d.unit = 'NM3' THEN 'Nm3' ELSE 'm3' END,
+		d.normometers_nm3 * o.share,
+		NULLIF(d.gas_consumption_kwh, 0) * o.share,
+		(o.share < 1),
+		o.overlap_days
+	FROM commercial_device d
+	CROSS JOIN bounds b
+	CROSS JOIN LATERAL generate_series(
+		date_trunc('month', d.date_from::timestamp),
+		date_trunc('month', d.date_to::timestamp),
+		interval '1 month'
+	) gs(month)
+	CROSS JOIN LATERAL (
+		SELECT
+			ov.overlap_days,
+			ov.overlap_days::numeric / NULLIF(d.date_to - d.date_from + 1, 0) AS share
+		FROM (
+			SELECT LEAST(d.date_to, (gs.month + interval '1 month - 1 day')::date)
+				- GREATEST(d.date_from, gs.month::date) + 1 AS overlap_days
+		) ov
+	) o
+	WHERE gs.month >= b.month_from AND gs.month < b.month_to
+),
+invoice_month AS (
+	SELECT
+		source,
+		place_id,
+		period_start,
+		sum(volume)           AS volume,
+		max(volume_unit)      AS volume_unit,
+		sum(nm3)              AS nm3,
+		sum(kwh)              AS kwh,
+		bool_or(is_estimated) AS is_estimated,
+		-- Clamp to the month's length: sum() would exceed it when meters are billed in parallel.
+		LEAST(
+			sum(days_covered),
+			(period_start + interval '1 month')::date - period_start
+		) AS days_covered,
+		array_agg(DISTINCT invoice_id::text ORDER BY invoice_id::text) AS invoice_ids
+	FROM invoice_device_month
+	GROUP BY source, place_id, period_start
+),
+invoice_year AS (
+	SELECT
+		source,
+		place_id,
+		date_trunc('year', period_start)::date AS period_start,
+		array_agg(DISTINCT invoice_id::text ORDER BY invoice_id::text) AS invoice_ids
+	FROM invoice_device_month
+	GROUP BY source, place_id, date_trunc('year', period_start)::date
+),
+
+porsenna_period AS (
+	SELECT *
+	FROM (
+		SELECT
+			d.identifier AS place_id,
+			c.commodity,
+			c."var" AS period_type,
+			f_enapo_safe_period(c."var", c."period") AS period_start,
+			c.value,
+			c.unit,
+			c."count" AS days_covered
+		FROM enapo_porsenna_consumption c
+		INNER JOIN enapo_porsenna_devices d
+			ON d.source = c.source AND d.id = c.device_id
+		WHERE d.identifier IS NOT NULL
+			AND (d."level" IS NULL OR d."level" = 'main')
+			AND c.commodity IS NOT NULL
+			AND c."var" IN ('yearly', 'monthly')
+	) parsed
+	WHERE parsed.period_start IS NOT NULL
+),
+
+eb_month AS (
+	-- Energy Broker history, full calendar months only: a period that is longer or does not start on
+	-- the 1st needs the pro-rata methodology, which is a separate step. One EAN/EIC can appear under
+	-- two idodbmistoom with identical values (a re-registered OM), so one row per month is kept instead
+	-- of summing them into a double. The highest idodbmisto_rspotrebyfa stands in for the newest row: the
+	-- one-off import has no timestamp, and in production the duplicates carry identical values.
+	SELECT DISTINCT ON (eb.ean_eic, eb.komodita, eb.datum_od)
+		eb.ean_eic AS place_id,
+		CASE eb.komodita WHEN 1 THEN 'electricity' WHEN 2 THEN 'gas' END AS commodity,
+		eb.datum_od AS period_start,
+		eb.spotreba_mwh * 1000 AS value_kwh,
+		eb.datum_do - eb.datum_od + 1 AS days_covered
+	FROM enapo_eb_consumption eb
+	CROSS JOIN bounds b
+	WHERE eb.spotreba_mwh IS NOT NULL
+		AND eb.datum_od = date_trunc('month', eb.datum_od)::date
+		AND eb.datum_do = (date_trunc('month', eb.datum_od) + interval '1 month - 1 day')::date
+		AND eb.datum_od >= b.month_from
+		AND eb.datum_od < b.month_to
+	ORDER BY eb.ean_eic, eb.komodita, eb.datum_od, eb.idodbmisto_rspotrebyfa DESC
+),
+
+live_monthly AS (
+	SELECT
+		p.gid,
+		p.place_id,
+		p.id_type,
+		p.commodity,
+		'predi_input'::varchar(50) AS source,
+		'monthly'::varchar(10) AS period_type,
+		e.period_start,
+		COALESCE(e.total_kwh, e.vt_kwh + e.nt_kwh, e.vt_kwh, e.nt_kwh) AS value,
+		'kWh'::varchar(10) AS unit,
+		COALESCE(e.total_kwh, e.vt_kwh + e.nt_kwh, e.vt_kwh, e.nt_kwh) AS value_kwh,
+		'EActi_VT_NT_DaySum'::varchar(50) AS internal_source_var,
+		e.days_covered,
+		false AS is_estimated,
+		'{}'::text[] AS invoice_ids
+	FROM point p
+	INNER JOIN electricity_metered e ON e.place_id = p.place_id
+	WHERE p.commodity = 'electricity'
+
+	UNION ALL
+
+	SELECT
+		p.gid,
+		p.place_id,
+		p.id_type,
+		p.commodity,
+		'ppas_ave_api',
+		'monthly',
+		g.period_start,
+		g.m3_sum,
+		'm3',
+		-- No conversion inputs for metered gas, but no consumption is 0 kWh whatever the conversion.
+		CASE WHEN g.m3_sum = 0 THEN 0 END,
+		'core',
+		g.days_covered,
+		false,
+		'{}'::text[]
+	FROM point p
+	INNER JOIN gas_metered g ON g.place_id = p.place_id
+	WHERE p.commodity = 'gas'
+
+	UNION ALL
+
+	SELECT
+		p.gid,
+		p.place_id,
+		p.id_type,
+		p.commodity,
+		i.source,
+		'monthly',
+		i.period_start,
+		COALESCE(i.volume, i.nm3::numeric),
+		CASE WHEN i.volume IS NOT NULL THEN i.volume_unit ELSE 'Nm3' END,
+		i.kwh,
+		NULL,
+		i.days_covered,
+		i.is_estimated,
+		i.invoice_ids
+	FROM point p
+	INNER JOIN invoice_month i ON i.place_id = p.place_id
+	WHERE p.commodity = 'gas'
+),
+
+monthly AS (
+	SELECT * FROM live_monthly
+
+	UNION ALL
+
+	SELECT
+		p.gid,
+		p.place_id,
+		p.id_type,
+		p.commodity,
+		'energy_broker',
+		'monthly',
+		e.period_start,
+		e.value_kwh,
+		'kWh',
+		e.value_kwh,
+		NULL,
+		e.days_covered,
+		false,
+		'{}'::text[]
+	FROM point p
+	INNER JOIN eb_month e ON e.place_id = p.place_id AND e.commodity = p.commodity
+	-- A live source that read the whole month in kWh wins, so the overlap is not charted twice. A pro-rata
+	-- invoice or a partially covered month does not win: the EB figure is billed for the whole month. Nor
+	-- does a volume-only row (the PPAS gas meter): EB is the only kWh for that month, which the combined
+	-- kWh series needs - unless the meter read no consumption, which is 0 kWh and wins.
+	WHERE NOT EXISTS (
+		SELECT 1
+		FROM live_monthly l
+		WHERE l.place_id = e.place_id
+			AND l.commodity = e.commodity
+			AND l.period_start = e.period_start
+			AND l.value_kwh IS NOT NULL
+			AND NOT l.is_estimated
+			AND l.days_covered >= (e.period_start + interval '1 month')::date - e.period_start
+	)
+),
+
+yearly_rollup AS (
+	SELECT
+		gid,
+		place_id,
+		id_type,
+		commodity,
+		source,
+		'yearly'::varchar(10) AS period_type,
+		date_trunc('year', period_start)::date AS period_start,
+		sum(value) AS value,
+		unit,
+		-- count(*) = count(value_kwh) is an all-non-null check: only sum value_kwh when every month has one.
+		CASE WHEN count(*) = count(value_kwh) THEN sum(value_kwh) END AS value_kwh,
+		max(internal_source_var) AS internal_source_var,
+		sum(days_covered) AS days_covered,
+		bool_or(is_estimated) AS is_estimated
+	FROM monthly m
+	GROUP BY gid, place_id, id_type, commodity, source, date_trunc('year', period_start)::date, unit
+),
+yearly AS (
+	SELECT
+		r.gid,
+		r.place_id,
+		r.id_type,
+		r.commodity,
+		r.source,
+		r.period_type,
+		r.period_start,
+		r.value,
+		r.unit,
+		r.value_kwh,
+		r.internal_source_var,
+		r.days_covered,
+		r.is_estimated,
+		COALESCE(iy.invoice_ids, '{}'::text[]) AS invoice_ids
+	FROM yearly_rollup r
+	LEFT JOIN invoice_year iy
+		ON iy.source = r.source AND iy.place_id = r.place_id AND iy.period_start = r.period_start
+
+	UNION ALL
+
+	SELECT
+		p.gid,
+		p.place_id,
+		p.id_type,
+		p.commodity,
+		'porsenna',
+		'yearly',
+		pp.period_start,
+		pp.value,
+		pp.unit,
+		CASE WHEN lower(pp.unit) = 'kwh' THEN pp.value ELSE NULL END,
+		NULL,
+		pp.days_covered,
+		false,
+		'{}'::text[]
+	FROM point p
+	INNER JOIN porsenna_period pp
+		ON pp.place_id = p.place_id AND pp.commodity = p.commodity
+	WHERE pp.period_type = 'yearly'
+)
+
+SELECT
+	gid::varchar AS gid,
+	place_id::varchar(50) AS place_id,
+	max(id_type)::varchar(10) AS id_type,
+	commodity::varchar(20) AS commodity,
+	source::varchar(50) AS source,
+	period_type::varchar(10) AS period_type,
+	period_start,
+	sum(value)::numeric(30, 6) AS value,
+	unit::varchar(10) AS unit,
+	(CASE WHEN count(*) = count(value_kwh) THEN sum(value_kwh) END)::numeric(30, 6) AS value_kwh,
+	max(internal_source_var)::varchar(50) AS internal_source_var,
+	COALESCE(max(invoice_ids), '{}'::text[]) AS invoice_ids,
+	max(days_covered)::int AS coverage_count,
+	(
+		CASE period_type
+			WHEN 'yearly' THEN (make_date(extract(year FROM period_start)::int + 1, 1, 1) - period_start)
+			ELSE (period_start + interval '1 month')::date - period_start
+		END
+	)::int AS expected_count,
+	bool_or(is_estimated) AS is_estimated
+FROM (
+	SELECT * FROM monthly
+	UNION ALL
+	SELECT * FROM yearly
+) all_periods
+WHERE value IS NOT NULL
+GROUP BY gid, place_id, commodity, source, unit, period_type, period_start;
+
+
+COMMENT ON MATERIALIZED VIEW mv_enapo_om_consumption IS
+	'Monthly and yearly consumption per consumption point per source, for the last 48 months, behind the DeD building detail endpoint. One row per (gid, place_id, commodity, source, unit, period_type, period_start). Refreshed by EnapoConsumptionHistoryRefreshTask (integration engine).';
+
+COMMENT ON COLUMN mv_enapo_om_consumption.place_id IS
+	'EAN / EIC / OM code, as in enapo_gid_mapping.place_id - NOT enapo_measurements.place_id, which holds a PPAS-internal place id. Measurements are joined through enapo_measurements.id.';
+COMMENT ON COLUMN mv_enapo_om_consumption.value IS
+	'Consumption for the period in `unit`. Electricity is kWh (the upstream T1_Wh_total naming is wrong); metered gas is m3, the difference between the cumulative `core` reading on the first day of the next month and the first day of the current month; invoiced gas follows the invoice unit; Porsenna carries its own unit; Energy Broker (enapo_eb_consumption, full calendar months only, suppressed where a live source read the whole month) is kWh converted from MWh for both electricity and gas.';
+COMMENT ON COLUMN mv_enapo_om_consumption.value_kwh IS
+	'Derived comparable energy: for gas, normometers_nm3 * combustion_heat (verified exact). NULL when the conversion inputs are missing - `value` + `unit` stay authoritative. NULL for metered gas (no normometers_nm3 input), except 0 when no gas was consumed.';
+COMMENT ON COLUMN mv_enapo_om_consumption.internal_source_var IS
+	'Which enapo_measurements.var produced the row; NULL for invoice and Porsenna rows. Provenance for debugging, not part of the public response.';
+COMMENT ON COLUMN mv_enapo_om_consumption.invoice_ids IS
+	'Billing documents the row was allocated from, sorted and deduplicated; empty for metered and Porsenna rows. A period can be allocated from several invoices (a yearly roll-up usually is), which is why it is an array.';
+COMMENT ON COLUMN mv_enapo_om_consumption.coverage_count IS
+	'Days of the period that carry data. Days for every commodity and both period types, so it is comparable with expected_count.';
+COMMENT ON COLUMN mv_enapo_om_consumption.expected_count IS 'Days in the period (28-31, or 365/366).';
+COMMENT ON COLUMN mv_enapo_om_consumption.is_estimated IS
+	'True when the value was allocated pro rata from a billing period spanning more than one month, rather than read for the period itself.';
+
+CREATE UNIQUE INDEX mv_enapo_om_consumption_uidx
+	ON mv_enapo_om_consumption (gid, place_id, commodity, source, unit, period_type, period_start);
+
+CREATE INDEX mv_enapo_om_consumption_gid_period_idx
+	ON mv_enapo_om_consumption (gid, period_type, period_start);