@aiaiai-pt/frankctl 0.5.1 → 0.7.1

package/dist/template-packs/cira/09_transdev_aveirobus_gtfs.yaml

@@ -0,0 +1,709 @@
+# Git-owned Transdev/AveiroBus Pipeline + BackingDataset bundle (issue #600).
+#
+# Prerequisite: apply the stable Source `cira-transdev-aveirobus-gtfs` from an
+# ignored chmod-0600 manifest containing literal S3 credentials. The non-apply
+# Source shape is documented under dev_docs/examples/sources/. This tracked
+# bundle contains no Source document and is safe to apply only after that Source
+# and its eleven Bronze streams exist.
+
+apiVersion: frank.platform/v1
+kind: Pipeline
+metadata:
+  name: cira_transdev_gtfs_agency
+spec:
+  name: cira_transdev_gtfs_agency
+  description: Historical Transdev GTFS agency projection; expected rows 1.
+  source_ids:
+    - cira-transdev-aveirobus-gtfs
+  schedule_config:
+    type: manual
+  steps:
+    - name: gtfs_agency_rows
+      kind: custom_sql
+      params:
+        sql: |-
+          SELECT trim(cast(agency_id AS varchar)) external_id,
+                 nullif(trim(cast(agency_name AS varchar)),'') name,
+                 nullif(trim(cast(agency_url AS varchar)),'') url,
+                 nullif(trim(cast(agency_timezone AS varchar)),'') timezone,
+                 nullif(trim(cast(agency_phone AS varchar)),'') phone,
+                 nullif(trim(cast(agency_lang AS varchar)),'') language
+          FROM tenant_00000000_cira_transdev_aveirobus_gtfs.agency
+          WHERE nullif(trim(cast(agency_id AS varchar)),'') IS NOT NULL
+      depends_on: []
+      emits_to: backing_dataset
+      config:
+        output_layer: silver
+---
+apiVersion: frank.platform/v1
+kind: BackingDataset
+metadata:
+  name: cira_transdev_gtfs_agency
+spec:
+  entity_type_id: gtfs_agency
+  entity_type_name: GTFS Agency
+  ontology_tenant_id: cira
+  pipeline: cira_transdev_gtfs_agency
+  sync_mode: manual
+  primary_key_column: external_id
+  title_key_column: name
+  property_mappings:
+    - column: external_id
+      property: external_id
+      type: varchar
+      is_primary_key: true
+    - column: name
+      property: name
+      type: varchar
+    - column: url
+      property: url
+      type: varchar
+    - column: timezone
+      property: timezone
+      type: varchar
+    - column: phone
+      property: phone
+      type: varchar
+    - column: language
+      property: language
+      type: varchar
+---
+apiVersion: frank.platform/v1
+kind: Pipeline
+metadata:
+  name: cira_transdev_gtfs_service
+spec:
+  name: cira_transdev_gtfs_service
+  description: Historical Transdev GTFS service identities, including exception-only services; expected rows 11.
+  source_ids:
+    - cira-transdev-aveirobus-gtfs
+  schedule_config:
+    type: manual
+  steps:
+    - name: gtfs_service_rows
+      kind: custom_sql
+      params:
+        sql: |-
+          WITH service_ids AS (
+            SELECT service_id
+            FROM tenant_00000000_cira_transdev_aveirobus_gtfs.calendar
+            UNION
+            SELECT service_id
+            FROM tenant_00000000_cira_transdev_aveirobus_gtfs.calendar_dates
+          )
+          SELECT trim(cast(service_id AS varchar)) external_id,
+                 trim(cast(service_id AS varchar)) name,
+                 CAST(NULL AS varchar) description
+          FROM service_ids
+          WHERE nullif(trim(cast(service_id AS varchar)),'') IS NOT NULL
+      depends_on: []
+      emits_to: backing_dataset
+      config:
+        output_layer: silver
+---
+apiVersion: frank.platform/v1
+kind: BackingDataset
+metadata:
+  name: cira_transdev_gtfs_service
+spec:
+  entity_type_id: gtfs_service
+  entity_type_name: GTFS Service
+  ontology_tenant_id: cira
+  pipeline: cira_transdev_gtfs_service
+  sync_mode: manual
+  primary_key_column: external_id
+  title_key_column: name
+  property_mappings:
+    - column: external_id
+      property: external_id
+      type: varchar
+      is_primary_key: true
+    - column: name
+      property: name
+      type: varchar
+    - column: description
+      property: description
+      type: varchar
+---
+apiVersion: frank.platform/v1
+kind: Pipeline
+metadata:
+  name: cira_transdev_gtfs_calendar_rule
+spec:
+  name: cira_transdev_gtfs_calendar_rule
+  description: Historical Transdev weekly service rules; expected rows 10.
+  source_ids:
+    - cira-transdev-aveirobus-gtfs
+  schedule_config:
+    type: manual
+  steps:
+    - name: gtfs_calendar_rule_rows
+      kind: custom_sql
+      params:
+        sql: |-
+          SELECT trim(cast(service_id AS varchar)) external_id,
+                 try_cast(monday AS integer)=1 monday,
+                 try_cast(tuesday AS integer)=1 tuesday,
+                 try_cast(wednesday AS integer)=1 wednesday,
+                 try_cast(thursday AS integer)=1 thursday,
+                 try_cast(friday AS integer)=1 friday,
+                 try_cast(saturday AS integer)=1 saturday,
+                 try_cast(sunday AS integer)=1 sunday,
+                 CAST(date_parse(cast(cast(start_date AS bigint) AS varchar),'%Y%m%d') AS date) start_date,
+                 CAST(date_parse(cast(cast(end_date AS bigint) AS varchar),'%Y%m%d') AS date) end_date,
+                 trim(cast(service_id AS varchar)) service_external_id
+          FROM tenant_00000000_cira_transdev_aveirobus_gtfs.calendar
+          WHERE nullif(trim(cast(service_id AS varchar)),'') IS NOT NULL
+      depends_on: []
+      emits_to: backing_dataset
+      config:
+        output_layer: silver
+---
+apiVersion: frank.platform/v1
+kind: BackingDataset
+metadata:
+  name: cira_transdev_gtfs_calendar_rule
+  dependsOn:
+    - entityTypeId: gtfs_service
+      ontologyTenantId: cira
+spec:
+  entity_type_id: gtfs_calendar_rule
+  entity_type_name: GTFS Calendar Rule
+  ontology_tenant_id: cira
+  pipeline: cira_transdev_gtfs_calendar_rule
+  sync_mode: manual
+  primary_key_column: external_id
+  title_key_column: external_id
+  property_mappings:
+    - column: external_id
+      property: external_id
+      type: varchar
+      is_primary_key: true
+    - column: monday
+      property: monday
+      type: boolean
+    - column: tuesday
+      property: tuesday
+      type: boolean
+    - column: wednesday
+      property: wednesday
+      type: boolean
+    - column: thursday
+      property: thursday
+      type: boolean
+    - column: friday
+      property: friday
+      type: boolean
+    - column: saturday
+      property: saturday
+      type: boolean
+    - column: sunday
+      property: sunday
+      type: boolean
+    - column: start_date
+      property: start_date
+      type: date
+    - column: end_date
+      property: end_date
+      type: date
+    - column: service_external_id
+      property: service
+      is_relationship: true
+      target_type: gtfs_service
+      target_key: external_id
+---
+apiVersion: frank.platform/v1
+kind: Pipeline
+metadata:
+  name: cira_transdev_gtfs_calendar_date_rule
+spec:
+  name: cira_transdev_gtfs_calendar_date_rule
+  description: Historical Transdev service-date exceptions; expected rows 1624.
+  source_ids:
+    - cira-transdev-aveirobus-gtfs
+  schedule_config:
+    type: manual
+  steps:
+    - name: gtfs_calendar_date_rule_rows
+      kind: custom_sql
+      params:
+        sql: |-
+          SELECT concat(
+                   trim(cast(service_id AS varchar)),
+                   ':',
+                   trim(cast(date AS varchar))
+                 ) external_id,
+                 CAST(date_parse(cast(cast(date AS bigint) AS varchar),'%Y%m%d') AS date) applies_on,
+                 try_cast(exception_type AS integer) exception_type,
+                 trim(cast(service_id AS varchar)) service_external_id
+          FROM tenant_00000000_cira_transdev_aveirobus_gtfs.calendar_dates
+          WHERE nullif(trim(cast(service_id AS varchar)),'') IS NOT NULL
+            AND nullif(trim(cast(date AS varchar)),'') IS NOT NULL
+      depends_on: []
+      emits_to: backing_dataset
+      config:
+        output_layer: silver
+---
+apiVersion: frank.platform/v1
+kind: BackingDataset
+metadata:
+  name: cira_transdev_gtfs_calendar_date_rule
+  dependsOn:
+    - entityTypeId: gtfs_service
+      ontologyTenantId: cira
+spec:
+  entity_type_id: gtfs_calendar_date_rule
+  entity_type_name: GTFS Calendar Date Rule
+  ontology_tenant_id: cira
+  pipeline: cira_transdev_gtfs_calendar_date_rule
+  sync_mode: manual
+  primary_key_column: external_id
+  title_key_column: external_id
+  property_mappings:
+    - column: external_id
+      property: external_id
+      type: varchar
+      is_primary_key: true
+    - column: applies_on
+      property: applies_on
+      type: date
+    - column: exception_type
+      property: exception_type
+      type: integer
+    - column: service_external_id
+      property: service
+      is_relationship: true
+      target_type: gtfs_service
+      target_key: external_id
+---
+apiVersion: frank.platform/v1
+kind: Pipeline
+metadata:
+  name: cira_transdev_gtfs_shape
+spec:
+  name: cira_transdev_gtfs_shape
+  description: Historical Transdev ordered GTFS shapes; expected rows 56.
+  source_ids:
+    - cira-transdev-aveirobus-gtfs
+  schedule_config:
+    type: manual
+  steps:
+    - name: gtfs_shape_rows
+      kind: custom_sql
+      params:
+        sql: |-
+          WITH shape_points AS (
+            SELECT trim(cast(shape_id AS varchar)) external_id,
+                   try_cast(shape_pt_sequence AS bigint) point_sequence,
+                   try_cast(shape_pt_lon AS double) longitude,
+                   try_cast(shape_pt_lat AS double) latitude
+            FROM tenant_00000000_cira_transdev_aveirobus_gtfs.shapes
+            WHERE nullif(trim(cast(shape_id AS varchar)),'') IS NOT NULL
+          )
+          SELECT external_id,
+                 to_geojson_geometry(
+                   ST_LineString(
+                     array_agg(ST_Point(longitude,latitude) ORDER BY point_sequence)
+                   )
+                 ) geometry_geojson
+          FROM shape_points
+          WHERE point_sequence IS NOT NULL
+            AND longitude BETWEEN -180.0 AND 180.0
+            AND latitude BETWEEN -90.0 AND 90.0
+          GROUP BY external_id
+          HAVING count(*)>=2
+      depends_on: []
+      emits_to: backing_dataset
+      config:
+        output_layer: silver
+---
+apiVersion: frank.platform/v1
+kind: BackingDataset
+metadata:
+  name: cira_transdev_gtfs_shape
+spec:
+  entity_type_id: gtfs_shape
+  entity_type_name: GTFS Shape
+  ontology_tenant_id: cira
+  pipeline: cira_transdev_gtfs_shape
+  sync_mode: manual
+  primary_key_column: external_id
+  title_key_column: external_id
+  property_mappings:
+    - column: external_id
+      property: external_id
+      type: varchar
+      is_primary_key: true
+    - column: geometry_geojson
+      property: geometry
+      type: varchar
+      sql_transform_id: json_parse
+---
+apiVersion: frank.platform/v1
+kind: Pipeline
+metadata:
+  name: cira_transdev_gtfs_route
+spec:
+  name: cira_transdev_gtfs_route
+  description: Historical Transdev GTFS routes linked to the single feed agency; expected rows 13.
+  source_ids:
+    - cira-transdev-aveirobus-gtfs
+  schedule_config:
+    type: manual
+  steps:
+    - name: gtfs_route_rows
+      kind: custom_sql
+      params:
+        sql: |-
+          WITH only_agency AS (
+            SELECT min(trim(cast(agency_id AS varchar))) agency_external_id
+            FROM tenant_00000000_cira_transdev_aveirobus_gtfs.agency
+            WHERE nullif(trim(cast(agency_id AS varchar)),'') IS NOT NULL
+          )
+          SELECT trim(cast(r.route_id AS varchar)) external_id,
+                 nullif(trim(cast(r.route_short_name AS varchar)),'') short_name,
+                 nullif(trim(cast(r.route_long_name AS varchar)),'') long_name,
+                 nullif(trim(cast(r.route_desc AS varchar)),'') description,
+                 try_cast(r.route_type AS integer) route_type,
+                 nullif(trim(cast(r.route_url AS varchar)),'') url,
+                 try_cast(r.route_sort_order AS integer) sort_order,
+                 nullif(trim(cast(r.route_color AS varchar)),'') color,
+                 nullif(trim(cast(r.route_text_color AS varchar)),'') text_color,
+                 coalesce(
+                   nullif(trim(cast(r.agency_id AS varchar)),''),
+                   a.agency_external_id
+                 ) agency_external_id
+          FROM tenant_00000000_cira_transdev_aveirobus_gtfs.routes r
+          CROSS JOIN only_agency a
+          WHERE nullif(trim(cast(r.route_id AS varchar)),'') IS NOT NULL
+      depends_on: []
+      emits_to: backing_dataset
+      config:
+        output_layer: silver
+---
+apiVersion: frank.platform/v1
+kind: BackingDataset
+metadata:
+  name: cira_transdev_gtfs_route
+  dependsOn:
+    - entityTypeId: gtfs_agency
+      ontologyTenantId: cira
+spec:
+  entity_type_id: gtfs_route
+  entity_type_name: GTFS Route
+  ontology_tenant_id: cira
+  pipeline: cira_transdev_gtfs_route
+  sync_mode: manual
+  primary_key_column: external_id
+  title_key_column: external_id
+  property_mappings:
+    - column: external_id
+      property: external_id
+      type: varchar
+      is_primary_key: true
+    - column: short_name
+      property: short_name
+      type: varchar
+    - column: long_name
+      property: long_name
+      type: varchar
+    - column: description
+      property: description
+      type: varchar
+    - column: route_type
+      property: route_type
+      type: integer
+    - column: url
+      property: url
+      type: varchar
+    - column: sort_order
+      property: sort_order
+      type: integer
+    - column: color
+      property: color
+      type: varchar
+    - column: text_color
+      property: text_color
+      type: varchar
+    - column: agency_external_id
+      property: agency
+      is_relationship: true
+      target_type: gtfs_agency
+      target_key: external_id
+---
+apiVersion: frank.platform/v1
+kind: Pipeline
+metadata:
+  name: cira_transdev_gtfs_stop
+spec:
+  name: cira_transdev_gtfs_stop
+  description: Historical Transdev ordinary GTFS stops; all 546 source rows have location_type=0.
+  source_ids:
+    - cira-transdev-aveirobus-gtfs
+  schedule_config:
+    type: manual
+  steps:
+    - name: gtfs_stop_rows
+      kind: custom_sql
+      params:
+        sql: |-
+          WITH stop_rows AS (
+            SELECT trim(cast(stop_id AS varchar)) external_id,
+                   nullif(trim(cast(stop_code AS varchar)),'') code,
+                   nullif(trim(cast(stop_name AS varchar)),'') name,
+                   nullif(trim(cast(stop_desc AS varchar)),'') description,
+                   nullif(trim(cast(stop_url AS varchar)),'') url,
+                   try_cast(wheelchair_boarding AS integer) wheelchair_boarding,
+                   nullif(trim(cast(zone_id AS varchar)),'') fare_zone,
+                   try_cast(stop_lon AS double) longitude,
+                   try_cast(stop_lat AS double) latitude,
+                   coalesce(try_cast(location_type AS integer),0) location_type
+            FROM tenant_00000000_cira_transdev_aveirobus_gtfs.stops
+          )
+          SELECT external_id,
+                 code,
+                 name,
+                 description,
+                 url,
+                 wheelchair_boarding,
+                 fare_zone,
+                 CASE
+                   WHEN longitude BETWEEN -180.0 AND 180.0
+                    AND latitude BETWEEN -90.0 AND 90.0
+                   THEN to_geojson_geometry(ST_Point(longitude,latitude))
+                 END geometry_geojson
+          FROM stop_rows
+          WHERE external_id IS NOT NULL
+            AND location_type=0
+      depends_on: []
+      emits_to: backing_dataset
+      config:
+        output_layer: silver
+---
+apiVersion: frank.platform/v1
+kind: BackingDataset
+metadata:
+  name: cira_transdev_gtfs_stop
+spec:
+  entity_type_id: gtfs_stop
+  entity_type_name: GTFS Stop
+  ontology_tenant_id: cira
+  pipeline: cira_transdev_gtfs_stop
+  sync_mode: manual
+  primary_key_column: external_id
+  title_key_column: name
+  property_mappings:
+    - column: external_id
+      property: external_id
+      type: varchar
+      is_primary_key: true
+    - column: code
+      property: code
+      type: varchar
+    - column: name
+      property: name
+      type: varchar
+    - column: description
+      property: description
+      type: varchar
+    - column: url
+      property: url
+      type: varchar
+    - column: wheelchair_boarding
+      property: wheelchair_boarding
+      type: integer
+    - column: fare_zone
+      property: fare_zone
+      type: varchar
+    - column: geometry_geojson
+      property: geometry
+      type: varchar
+      sql_transform_id: json_parse
+---
+apiVersion: frank.platform/v1
+kind: Pipeline
+metadata:
+  name: cira_transdev_gtfs_trip
+spec:
+  name: cira_transdev_gtfs_trip
+  description: Historical Transdev GTFS trips linked to route, service, and optional shape; expected rows 441.
+  source_ids:
+    - cira-transdev-aveirobus-gtfs
+  schedule_config:
+    type: manual
+  steps:
+    - name: gtfs_trip_rows
+      kind: custom_sql
+      params:
+        sql: |-
+          SELECT trim(cast(trip_id AS varchar)) external_id,
+                 nullif(trim(cast(trip_headsign AS varchar)),'') headsign,
+                 nullif(trim(cast(trip_short_name AS varchar)),'') short_name,
+                 try_cast(direction_id AS integer) direction,
+                 nullif(trim(cast(block_id AS varchar)),'') block,
+                 try_cast(wheelchair_accessible AS integer) wheelchair_accessible,
+                 try_cast(bikes_allowed AS integer) bikes_allowed,
+                 trim(cast(route_id AS varchar)) route_external_id,
+                 trim(cast(service_id AS varchar)) service_external_id,
+                 nullif(trim(cast(shape_id AS varchar)),'') shape_external_id
+          FROM tenant_00000000_cira_transdev_aveirobus_gtfs.trips
+          WHERE nullif(trim(cast(trip_id AS varchar)),'') IS NOT NULL
+      depends_on: []
+      emits_to: backing_dataset
+      config:
+        output_layer: silver
+---
+apiVersion: frank.platform/v1
+kind: BackingDataset
+metadata:
+  name: cira_transdev_gtfs_trip
+  dependsOn:
+    - entityTypeId: gtfs_route
+      ontologyTenantId: cira
+    - entityTypeId: gtfs_service
+      ontologyTenantId: cira
+    - entityTypeId: gtfs_shape
+      ontologyTenantId: cira
+spec:
+  entity_type_id: gtfs_trip
+  entity_type_name: GTFS Trip
+  ontology_tenant_id: cira
+  pipeline: cira_transdev_gtfs_trip
+  sync_mode: manual
+  primary_key_column: external_id
+  title_key_column: headsign
+  property_mappings:
+    - column: external_id
+      property: external_id
+      type: varchar
+      is_primary_key: true
+    - column: headsign
+      property: headsign
+      type: varchar
+    - column: short_name
+      property: short_name
+      type: varchar
+    - column: direction
+      property: direction
+      type: integer
+    - column: block
+      property: block
+      type: varchar
+    - column: wheelchair_accessible
+      property: wheelchair_accessible
+      type: integer
+    - column: bikes_allowed
+      property: bikes_allowed
+      type: integer
+    - column: route_external_id
+      property: route
+      is_relationship: true
+      target_type: gtfs_route
+      target_key: external_id
+    - column: service_external_id
+      property: service
+      is_relationship: true
+      target_type: gtfs_service
+      target_key: external_id
+    - column: shape_external_id
+      property: shape
+      is_relationship: true
+      target_type: gtfs_shape
+      target_key: external_id
+---
+apiVersion: frank.platform/v1
+kind: Pipeline
+metadata:
+  name: cira_transdev_gtfs_stop_time
+spec:
+  name: cira_transdev_gtfs_stop_time
+  description: Historical Transdev GTFS stop times; service-day times can exceed 24:00:00 and remain strings; expected rows 16511.
+  source_ids:
+    - cira-transdev-aveirobus-gtfs
+  schedule_config:
+    type: manual
+  steps:
+    - name: gtfs_stop_time_rows
+      kind: custom_sql
+      params:
+        sql: |-
+          SELECT concat(
+                   trim(cast(trip_id AS varchar)),
+                   ':',
+                   cast(try_cast(stop_sequence AS bigint) AS varchar)
+                 ) external_id,
+                 nullif(trim(cast(arrival_time AS varchar)),'') arrival_time,
+                 nullif(trim(cast(departure_time AS varchar)),'') departure_time,
+                 try_cast(stop_sequence AS integer) stop_sequence,
+                 nullif(trim(cast(stop_headsign AS varchar)),'') stop_headsign,
+                 try_cast(pickup_type AS integer) pickup_type,
+                 try_cast(drop_off_type AS integer) drop_off_type,
+                 try_cast(shape_dist_traveled AS double) shape_dist_traveled,
+                 try_cast(timepoint AS integer) timepoint,
+                 trim(cast(trip_id AS varchar)) trip_external_id,
+                 trim(cast(stop_id AS varchar)) stop_external_id
+          FROM tenant_00000000_cira_transdev_aveirobus_gtfs.stop_times
+          WHERE nullif(trim(cast(trip_id AS varchar)),'') IS NOT NULL
+            AND try_cast(stop_sequence AS bigint) IS NOT NULL
+            AND nullif(trim(cast(stop_id AS varchar)),'') IS NOT NULL
+      depends_on: []
+      emits_to: backing_dataset
+      config:
+        output_layer: silver
+---
+apiVersion: frank.platform/v1
+kind: BackingDataset
+metadata:
+  name: cira_transdev_gtfs_stop_time
+  dependsOn:
+    - entityTypeId: gtfs_trip
+      ontologyTenantId: cira
+    - entityTypeId: gtfs_stop
+      ontologyTenantId: cira
+spec:
+  entity_type_id: gtfs_stop_time
+  entity_type_name: GTFS Stop Time
+  ontology_tenant_id: cira
+  pipeline: cira_transdev_gtfs_stop_time
+  sync_mode: manual
+  primary_key_column: external_id
+  title_key_column: external_id
+  property_mappings:
+    - column: external_id
+      property: external_id
+      type: varchar
+      is_primary_key: true
+    - column: arrival_time
+      property: arrival_time
+      type: varchar
+    - column: departure_time
+      property: departure_time
+      type: varchar
+    - column: stop_sequence
+      property: stop_sequence
+      type: integer
+    - column: stop_headsign
+      property: stop_headsign
+      type: varchar
+    - column: pickup_type
+      property: pickup_type
+      type: integer
+    - column: drop_off_type
+      property: drop_off_type
+      type: integer
+    - column: shape_dist_traveled
+      property: shape_dist_traveled
+      type: double
+    - column: timepoint
+      property: timepoint
+      type: integer
+    - column: trip_external_id
+      property: trip
+      is_relationship: true
+      target_type: gtfs_trip
+      target_key: external_id
+    - column: stop_external_id
+      property: stop
+      is_relationship: true
+      target_type: gtfs_stop
+      target_key: external_id