@aiaiai-pt/frankctl 0.5.1 → 0.7.0

package/dist/template-packs/cira/01_organization.yaml

@@ -0,0 +1,68 @@
+# Stage 01: 11 transport municipalities plus the Busway operator.
+# Expected complete cardinality from the live source profile: 12.
+
+apiVersion: frank.platform/v1
+kind: Pipeline
+metadata:
+  name: cira_transport_organization
+spec:
+  name: cira_transport_organization
+  description: Canonical CIRA transport municipalities and operator organizations.
+  source_ids:
+    - cira-bdtransportes-rede
+  schedule_config:
+    type: manual
+  steps:
+    - name: organization_rows
+      kind: custom_sql
+      params:
+        sql: |-
+          SELECT concat('municipality:',cast(cast(fid AS bigint) AS varchar)) code,
+                 municipio name,
+                 CAST(NULL AS varchar) email,
+                 CASE WHEN geometry_wkt IS NOT NULL THEN
+                   concat(
+                     '{"type":"MultiPolygon","coordinates":[',
+                     json_format(json_extract(json_parse(to_geojson_geometry(ST_GeometryFromText(geometry_wkt))), '$.coordinates')),
+                     ']}'
+                   )
+                 END boundary_geojson
+          FROM tenant_00000000_cira_bdtransportes_rede.municipios
+          UNION ALL
+          SELECT DISTINCT concat('operator:',cast(cast(id_operador_estrutura AS bigint) AS varchar)) code,
+                          nome name,
+                          CAST(NULL AS varchar) email,
+                          CAST(NULL AS varchar) boundary_geojson
+          FROM tenant_00000000_cira_bdtransportes_rede.carreirastotal
+      depends_on: []
+      emits_to: backing_dataset
+      config:
+        output_layer: silver
+---
+apiVersion: frank.platform/v1
+kind: BackingDataset
+metadata:
+  name: cira_transport_organization
+spec:
+  entity_type_id: organization
+  entity_type_name: Organization
+  ontology_tenant_id: cira
+  pipeline: cira_transport_organization
+  sync_mode: manual
+  primary_key_column: code
+  title_key_column: name
+  property_mappings:
+    - column: code
+      property: code
+      type: varchar
+      is_primary_key: true
+    - column: name
+      property: name
+      type: varchar
+    - column: email
+      property: email
+      type: varchar
+    - column: boundary_geojson
+      property: boundary
+      type: varchar
+      sql_transform_id: json_parse

package/dist/template-packs/cira/02_freguesia.yaml

@@ -0,0 +1,73 @@
+# Stage 02: CIRA transport freguesias. Run after stage 01 so every
+# organization relationship resolves through organization.code.
+# Expected complete cardinality from the live source profile: 74.
+
+apiVersion: frank.platform/v1
+kind: Pipeline
+metadata:
+  name: cira_transport_freguesia
+spec:
+  name: cira_transport_freguesia
+  description: Canonical CIRA transport freguesias with municipality ownership.
+  source_ids:
+    - cira-bdtransportes-rede
+  schedule_config:
+    type: manual
+  steps:
+    - name: freguesia_rows
+      kind: custom_sql
+      params:
+        sql: |-
+          SELECT dicofre code,
+                 freguesia name,
+                 concat('municipality:',cast(cast(idmunicipio AS bigint) AS varchar)) municipality_code,
+                 concat('municipality:',cast(cast(idmunicipio AS bigint) AS varchar)) organization_code,
+                 CASE WHEN geometry_wkt IS NOT NULL THEN
+                   concat(
+                     '{"type":"MultiPolygon","coordinates":[',
+                     json_format(json_extract(json_parse(to_geojson_geometry(ST_GeometryFromText(geometry_wkt))), '$.coordinates')),
+                     ']}'
+                   )
+                 END geometry_geojson
+          FROM tenant_00000000_cira_bdtransportes_rede.freguesias
+          WHERE dicofre IS NOT NULL
+      depends_on: []
+      emits_to: backing_dataset
+      config:
+        output_layer: silver
+---
+apiVersion: frank.platform/v1
+kind: BackingDataset
+metadata:
+  name: cira_transport_freguesia
+  dependsOn:
+    - entityTypeId: organization
+      ontologyTenantId: cira
+spec:
+  entity_type_id: freguesia
+  entity_type_name: Freguesia
+  ontology_tenant_id: cira
+  pipeline: cira_transport_freguesia
+  sync_mode: manual
+  primary_key_column: code
+  title_key_column: name
+  property_mappings:
+    - column: code
+      property: code
+      type: varchar
+      is_primary_key: true
+    - column: name
+      property: name
+      type: varchar
+    - column: municipality_code
+      property: municipality_code
+      type: varchar
+    - column: geometry_geojson
+      property: geometry
+      type: varchar
+      sql_transform_id: json_parse
+    - column: organization_code
+      property: organization
+      is_relationship: true
+      target_type: organization
+      target_key: code

package/dist/template-packs/cira/03_transit_route.yaml

@@ -0,0 +1,70 @@
+# Stage 03: complete CIRA route projection. Run after stage 01 so every
+# operator relationship resolves through organization.code.
+# Expected complete cardinality from the live source profile: 8,964.
+
+apiVersion: frank.platform/v1
+kind: Pipeline
+metadata:
+  name: cira_transport_routes
+spec:
+  name: cira_transport_routes
+  description: Canonical CIRA public-transport routes and their operator.
+  source_ids:
+    - cira-bdtransportes-rede
+  schedule_config:
+    type: manual
+  steps:
+    - name: transit_route_rows
+      kind: custom_sql
+      params:
+        sql: |-
+          SELECT coda route_key,
+                 designacao long_name,
+                 'bus' route_type,
+                 nullif(regexp_replace(color_hex,'^#',''),'') color,
+                 concat_ws(' · ',nullif(cod_car_operador,''),nullif(origem,''),nullif(destino,''),nullif(sentido,''),nullif(tipo,''),nullif(classificacao,'')) description,
+                 concat('operator:',cast(cast(id_operador_estrutura AS bigint) AS varchar)) operator_code
+          FROM tenant_00000000_cira_bdtransportes_rede.carreirastotal
+          WHERE coda IS NOT NULL
+      depends_on: []
+      emits_to: backing_dataset
+      config:
+        output_layer: silver
+---
+apiVersion: frank.platform/v1
+kind: BackingDataset
+metadata:
+  name: cira_transport_routes
+  dependsOn:
+    - entityTypeId: organization
+      ontologyTenantId: cira
+spec:
+  entity_type_id: transit_route
+  entity_type_name: Transit Route
+  ontology_tenant_id: cira
+  pipeline: cira_transport_routes
+  sync_mode: manual
+  primary_key_column: route_key
+  title_key_column: long_name
+  property_mappings:
+    - column: route_key
+      property: short_name
+      type: varchar
+      is_primary_key: true
+    - column: long_name
+      property: long_name
+      type: varchar
+    - column: route_type
+      property: route_type
+      type: varchar
+    - column: color
+      property: color
+      type: varchar
+    - column: description
+      property: description
+      type: varchar
+    - column: operator_code
+      property: operator
+      is_relationship: true
+      target_type: organization
+      target_key: code

package/dist/template-packs/cira/04_transit_stop.yaml

@@ -0,0 +1,57 @@
+# Stage 04: complete CIRA public-stop projection.
+# Expected complete cardinality from the live source profile: 2,411.
+
+apiVersion: frank.platform/v1
+kind: Pipeline
+metadata:
+  name: cira_transport_stops
+spec:
+  name: cira_transport_stops
+  description: Canonical CIRA public-transport stops with GeoJSON locations.
+  source_ids:
+    - cira-bdtransportes-rede
+  schedule_config:
+    type: manual
+  steps:
+    - name: transit_stop_rows
+      kind: custom_sql
+      params:
+        sql: |-
+          SELECT concat('stop:',cast(cast(id_paragem AS bigint) AS varchar)) stop_key,
+                 designacao_paragem name,
+                 nullif(trim(municipio),'') zone,
+                 CASE WHEN geometry_wkt IS NOT NULL THEN to_geojson_geometry(ST_GeometryFromText(geometry_wkt)) END location_geojson
+          FROM tenant_00000000_cira_bdtransportes_rede.paragenspublicogeral
+          WHERE id_paragem IS NOT NULL
+      depends_on: []
+      emits_to: backing_dataset
+      config:
+        output_layer: silver
+---
+apiVersion: frank.platform/v1
+kind: BackingDataset
+metadata:
+  name: cira_transport_stops
+spec:
+  entity_type_id: transit_stop
+  entity_type_name: Transit Stop
+  ontology_tenant_id: cira
+  pipeline: cira_transport_stops
+  sync_mode: manual
+  primary_key_column: stop_key
+  title_key_column: name
+  property_mappings:
+    - column: stop_key
+      property: code
+      type: varchar
+      is_primary_key: true
+    - column: name
+      property: name
+      type: varchar
+    - column: zone
+      property: zone
+      type: varchar
+    - column: location_geojson
+      property: location
+      type: varchar
+      sql_transform_id: json_parse

package/dist/template-packs/cira/05_air_quality_station.yaml

@@ -0,0 +1,83 @@
+# Stage 05: station natural keys from the raw Kunak and VDL tables.
+# This deliberately avoids the existing duplicate-name Silver projection.
+# Expected complete cardinality from the live source profile: 13 unique names.
+
+apiVersion: frank.platform/v1
+kind: Pipeline
+metadata:
+  name: cira_air_quality_stations
+spec:
+  name: cira_air_quality_stations
+  description: Canonical unique CIRA air-quality stations from Kunak and VDL.
+  source_ids:
+    - Kunak Cloud - Ubiwhere
+    - VDL - AirQuality - Monitar
+  schedule_config:
+    type: manual
+  steps:
+    - name: air_quality_station_rows
+      kind: custom_sql
+      params:
+        sql: |-
+          WITH kunak AS (
+            SELECT serial_number,
+                   max(tag) label,
+                   max(try_cast(coalesce(nullif(trim(location__latitude),''),try(json_extract_scalar(location,'$.latitude'))) AS double)) latitude,
+                   max(try_cast(coalesce(nullif(trim(location__longitude),''),try(json_extract_scalar(location,'$.longitude'))) AS double)) longitude
+            FROM tenant_00000000_kunak_cloud_ubiwhere.kunak_stations
+            WHERE serial_number IS NOT NULL
+            GROUP BY serial_number
+          ), vdl AS (
+            SELECT system
+            FROM tenant_00000000_vdl_airquality_monitar.sensors
+            WHERE system IS NOT NULL
+            GROUP BY system
+          )
+          SELECT concat('kunak:',serial_number) station_key,
+                 label alternate_name,
+                 'Kunak' data_provider,
+                 'kunak_cloud_ubiwhere' source,
+                 CASE WHEN latitude IS NOT NULL AND longitude IS NOT NULL THEN to_geojson_geometry(ST_GeometryFromText(format('POINT (%s %s)',longitude,latitude))) END location_geojson
+          FROM kunak
+          UNION ALL
+          SELECT concat('vdl:',system),
+                 system,
+                 'Monitar',
+                 'vdl_airquality_monitar',
+                 CAST(NULL AS varchar)
+          FROM vdl
+      depends_on: []
+      emits_to: backing_dataset
+      config:
+        output_layer: silver
+---
+apiVersion: frank.platform/v1
+kind: BackingDataset
+metadata:
+  name: cira_air_quality_stations
+spec:
+  entity_type_id: air_quality_station
+  entity_type_name: Air Quality Station
+  ontology_tenant_id: cira
+  pipeline: cira_air_quality_stations
+  sync_mode: manual
+  primary_key_column: station_key
+  title_key_column: station_key
+  property_mappings:
+    - column: station_key
+      property: name
+      type: varchar
+      is_primary_key: true
+    - column: alternate_name
+      property: alternate_name
+      type: varchar
+    - column: data_provider
+      property: data_provider
+      type: varchar
+    - column: source
+      property: source
+      type: varchar
+    - column: location_geojson
+      property: location
+      type: varchar
+      sql_transform_id: json_parse

package/dist/template-packs/cira/06_air_quality_observed.yaml

@@ -0,0 +1,99 @@
+# Stage 06: the complete publishable Kunak observation subset for the five
+# source pollutant tags supported by the target schema. Run after stage 05 so
+# ref_station resolves.
+#
+# Live raw profile (2026-03-23 through 2026-06-22): 3,169,139 raw measurement rows
+# across 115,665 raw timestamps. The five-tag predicate retains 542,406
+# pollutant-tag rows and 26,216 canonical observation timestamps. Keeping this
+# predicate prevents publishing 89,449 timestamp entities whose every mapped
+# pollutant would be null. Pressure, temperature, humidity, and battery are not
+# target fields; other target pollutant fields are absent from this Kunak tag set.
+
+apiVersion: frank.platform/v1
+kind: Pipeline
+metadata:
+  name: cira_air_quality_observed
+spec:
+  name: cira_air_quality_observed
+  description: Complete Kunak air-quality observations linked to canonical stations.
+  source_ids:
+    - Kunak Cloud - Ubiwhere
+  schedule_config:
+    type: manual
+  steps:
+    - name: air_quality_observed_rows
+      kind: custom_sql
+      params:
+        sql: |-
+          SELECT concat('kunak:0425100035:',cast(ts AS varchar)) observation_key,
+                 from_unixtime(cast(ts AS double)/1000.0) date_observed,
+                 'Kunak' data_provider,
+                 'kunak_cloud_ubiwhere' source,
+                 'kunak:0425100035' station_name,
+                 'kunak:0425100035' ref_station_key,
+                 max(CASE WHEN sensor_tag='PM1' THEN try_cast(value AS double) END) pm1,
+                 max(CASE WHEN sensor_tag='PM2.5' THEN try_cast(value AS double) END) pm25,
+                 max(CASE WHEN sensor_tag='PM10' THEN try_cast(value AS double) END) pm10,
+                 max(CASE WHEN sensor_tag='NO2 GCc' THEN try_cast(value AS double) END) no2,
+                 max(CASE WHEN sensor_tag='CO GCc' THEN try_cast(value AS double) END) co
+          FROM tenant_00000000_kunak_cloud_ubiwhere.device_0425100035
+          WHERE ts IS NOT NULL
+            AND sensor_tag IN ('PM1','PM2.5','PM10','NO2 GCc','CO GCc')
+          GROUP BY ts
+      depends_on: []
+      emits_to: backing_dataset
+      config:
+        output_layer: silver
+---
+apiVersion: frank.platform/v1
+kind: BackingDataset
+metadata:
+  name: cira_air_quality_observed
+  dependsOn:
+    - entityTypeId: air_quality_station
+      ontologyTenantId: cira
+spec:
+  entity_type_id: air_quality_observed
+  entity_type_name: Air Quality Observed
+  ontology_tenant_id: cira
+  pipeline: cira_air_quality_observed
+  sync_mode: manual
+  primary_key_column: observation_key
+  title_key_column: observation_key
+  property_mappings:
+    - column: observation_key
+      property: name
+      type: varchar
+      is_primary_key: true
+    - column: date_observed
+      property: date_observed
+      type: timestamp with time zone
+    - column: data_provider
+      property: data_provider
+      type: varchar
+    - column: source
+      property: source
+      type: varchar
+    - column: station_name
+      property: station_name
+      type: varchar
+    - column: ref_station_key
+      property: ref_station
+      is_relationship: true
+      target_type: air_quality_station
+      target_key: name
+    - column: pm1
+      property: pm1
+      type: double
+    - column: pm25
+      property: pm25
+      type: double
+    - column: pm10
+      property: pm10
+      type: double
+    - column: no2
+      property: no2
+      type: double
+    - column: co
+      property: co
+      type: double

package/dist/template-packs/cira/07_transit_trip.yaml

@@ -0,0 +1,106 @@
+# Stage 07: complete CIRA trips after the repaired WFS iterator publishes the
+# authoritative 237,808-row schedule snapshot. The point-in-time source profile
+# has 8,964 unique, non-null trip IDs; the count is evidence, not a row limit.
+
+apiVersion: frank.platform/v1
+kind: Source
+metadata:
+  name: cira-bdtransportes-horarios
+spec:
+  name: cira-bdtransportes-horarios
+  pattern_id: geoserver_wfs
+  source_config:
+    base_url: https://transportes.regiaodeaveiro.pt/geoserver
+    workspace: cimo
+    resources:
+      - ParagensTotalSchedule
+    wfs_version: 2.0.0
+    srs_name: EPSG:4326
+    page_size: 10000
+  # Match one iterator activity to one WFS page. The 5k cluster default
+  # re-fetches every 10k page for its second half, which is correct but adds a
+  # slow duplicate sorted-offset request on this 237,808-row source.
+  max_items_per_chunk: 10000
+  schedule_type: manual
+  streams:
+    - name: ParagensTotalSchedule
+      sync_mode: full_refresh
+      write_disposition: replace
+      primary_key_path:
+        - trip_id
+        - Ordem
+        - IdParagem
+      is_enabled: true
+---
+apiVersion: frank.platform/v1
+kind: Pipeline
+metadata:
+  name: cira_transport_trips
+spec:
+  name: cira_transport_trips
+  description: Complete CIRA transit trips with deterministic final-stop headsigns.
+  source_ids:
+    - cira-bdtransportes-horarios
+  schedule_config:
+    type: manual
+  steps:
+    - name: transit_trip_rows
+      kind: custom_sql
+      params:
+        sql: |-
+          WITH final_stop AS (
+            SELECT trip_id service_id,
+                   try_cast(sentidoparagem AS integer) direction,
+                   nullif(trim(paragem),'') headsign,
+                   coda route_key,
+                   row_number() OVER (
+                     PARTITION BY trip_id
+                     ORDER BY try_cast(ordem AS bigint) DESC NULLS LAST,
+                              try_cast(id_paragem AS bigint) DESC NULLS LAST,
+                              _feature_id DESC NULLS LAST
+                   ) stop_rank
+            FROM tenant_00000000_cira_bdtransportes_horarios.paragenstotalschedule
+            WHERE trip_id IS NOT NULL
+          )
+          SELECT service_id,
+                 direction,
+                 headsign,
+                 route_key
+          FROM final_stop
+          WHERE stop_rank=1
+      depends_on: []
+      emits_to: backing_dataset
+      config:
+        output_layer: silver
+---
+apiVersion: frank.platform/v1
+kind: BackingDataset
+metadata:
+  name: cira_transport_trips
+  dependsOn:
+    - entityTypeId: transit_route
+      ontologyTenantId: cira
+spec:
+  entity_type_id: transit_trip
+  entity_type_name: Transit Trip
+  ontology_tenant_id: cira
+  pipeline: cira_transport_trips
+  sync_mode: manual
+  primary_key_column: service_id
+  title_key_column: service_id
+  property_mappings:
+    - column: service_id
+      property: service_id
+      type: varchar
+      is_primary_key: true
+    - column: direction
+      property: direction
+      type: integer
+    - column: headsign
+      property: headsign
+      type: varchar
+    - column: route_key
+      property: route
+      is_relationship: true
+      target_type: transit_route
+      target_key: short_name

package/dist/template-packs/cira/08_vehicle_position.yaml

@@ -0,0 +1,81 @@
+# Stage 08: the current CIRA AVL snapshot. The upstream point-in-time profile
+# at 2026-07-21 20:11 UTC contained 405 rows with 405 unique, non-null partition
+# IDs (up from 403 minutes earlier). Every full refresh replaces that evidence
+# point with the then-current set.
+
+apiVersion: frank.platform/v1
+kind: Source
+metadata:
+  name: cira-bdtransportes-avl
+spec:
+  name: cira-bdtransportes-avl
+  pattern_id: geoserver_wfs
+  source_config:
+    base_url: https://transportes.regiaodeaveiro.pt/geoserver
+    workspace: cimo
+    resources:
+      - carreirasae_ultimaposicao
+    wfs_version: 2.0.0
+    srs_name: EPSG:4326
+    page_size: 5000
+    primary_keys:
+      - partition_id
+  schedule_type: manual
+  streams:
+    - name: carreirasae_ultimaposicao
+      sync_mode: full_refresh
+      write_disposition: replace
+      primary_key_path:
+        - partition_id
+      is_enabled: true
+---
+apiVersion: frank.platform/v1
+kind: Pipeline
+metadata:
+  name: cira_transport_vehicle_positions
+spec:
+  name: cira_transport_vehicle_positions
+  description: Current CIRA vehicle positions without an unproven trip relationship.
+  source_ids:
+    - cira-bdtransportes-avl
+  schedule_config:
+    type: manual
+  steps:
+    - name: vehicle_position_rows
+      kind: custom_sql
+      params:
+        sql: |-
+          SELECT partition_id label,
+                 CASE WHEN geometry_wkt IS NOT NULL THEN to_geojson_geometry(ST_GeometryFromText(geometry_wkt)) END location_geojson,
+                 try_cast(data AS timestamp) recorded_at
+          FROM tenant_00000000_cira_bdtransportes_avl.carreirasae_ultimaposicao
+          WHERE partition_id IS NOT NULL
+      depends_on: []
+      emits_to: backing_dataset
+      config:
+        output_layer: silver
+---
+apiVersion: frank.platform/v1
+kind: BackingDataset
+metadata:
+  name: cira_transport_vehicle_positions
+spec:
+  entity_type_id: vehicle_position
+  entity_type_name: Vehicle Position
+  ontology_tenant_id: cira
+  pipeline: cira_transport_vehicle_positions
+  sync_mode: manual
+  primary_key_column: label
+  title_key_column: label
+  property_mappings:
+    - column: label
+      property: label
+      type: varchar
+      is_primary_key: true
+    - column: location_geojson
+      property: location
+      type: varchar
+      sql_transform_id: json_parse
+    - column: recorded_at
+      property: recorded_at
+      type: timestamp