@aiaiai-pt/frankctl 0.5.1 → 0.7.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,526 @@
1
+ # Complete Git-owned CIRA Ciclope Pipeline + BackingDataset bundle (#601).
2
+ #
3
+ # Apply the ignored chmod-0600 Source manifest first, sync all nine Bronze
4
+ # streams, then apply this Source-free bundle. It projects only into existing
5
+ # CIRA E2 Civil Protection and G1 IoT entity types; it never owns schema.
6
+
7
+ apiVersion: frank.platform/v1
8
+ kind: Pipeline
9
+ metadata:
10
+ name: cira_ciclope_emergency_resources
11
+ spec:
12
+ name: cira_ciclope_emergency_resources
13
+ description: Installed CICLOPE fire-detection towers as E2 emergency resources.
14
+ source_ids:
15
+ - cira-ciclope
16
+ schedule_config:
17
+ type: manual
18
+ steps:
19
+ - name: emergency_resource_rows
20
+ kind: custom_sql
21
+ params:
22
+ sql: |-
23
+ WITH camera_rollup AS (
24
+ SELECT c.tower_id,
25
+ count(*) camera_count,
26
+ CASE
27
+ WHEN count_if(lower(coalesce(s.operational_status,''))='operational')=count(*)
28
+ THEN 'operational'
29
+ WHEN count_if(lower(coalesce(s.operational_status,''))='operational')=0
30
+ THEN 'unavailable'
31
+ ELSE 'degraded'
32
+ END resource_status
33
+ FROM tenant_00000000_cira_ciclope.cameras c
34
+ LEFT JOIN tenant_00000000_cira_ciclope.camera_status s
35
+ ON s.tower_id=c.tower_id AND s.camera_id=c.camera_id
36
+ GROUP BY c.tower_id
37
+ )
38
+ SELECT concat('ciclope:tower:',cast(cast(t.tower_id AS bigint) AS varchar)) code,
39
+ 'fire_detection_tower' resource_type,
40
+ coalesce(r.resource_status,'unknown') status,
41
+ cast(coalesce(r.camera_count,0) AS integer) capacity,
42
+ CASE
43
+ WHEN t.longitude BETWEEN -180.0 AND 180.0
44
+ AND t.latitude BETWEEN -90.0 AND 90.0
45
+ THEN to_geojson_geometry(ST_Point(t.longitude,t.latitude))
46
+ END location_geojson
47
+ FROM tenant_00000000_cira_ciclope.towers t
48
+ LEFT JOIN camera_rollup r ON r.tower_id=t.tower_id
49
+ WHERE t.tower_id IS NOT NULL
50
+ depends_on: []
51
+ emits_to: backing_dataset
52
+ config:
53
+ output_layer: silver
54
+ ---
55
+ apiVersion: frank.platform/v1
56
+ kind: BackingDataset
57
+ metadata:
58
+ name: cira_ciclope_emergency_resources
59
+ spec:
60
+ entity_type_id: emergency_resource
61
+ entity_type_name: Emergency Resource
62
+ ontology_tenant_id: cira
63
+ pipeline: cira_ciclope_emergency_resources
64
+ sync_mode: manual
65
+ primary_key_column: code
66
+ title_key_column: code
67
+ property_mappings:
68
+ - column: code
69
+ property: code
70
+ type: varchar
71
+ is_primary_key: true
72
+ - column: resource_type
73
+ property: resource_type
74
+ type: varchar
75
+ - column: status
76
+ property: status
77
+ type: varchar
78
+ - column: capacity
79
+ property: capacity
80
+ type: integer
81
+ - column: location_geojson
82
+ property: location
83
+ type: varchar
84
+ sql_transform_id: json_parse
85
+ ---
86
+ apiVersion: frank.platform/v1
87
+ kind: Pipeline
88
+ metadata:
89
+ name: cira_ciclope_iot_devices
90
+ spec:
91
+ name: cira_ciclope_iot_devices
92
+ description: CICLOPE cameras and installed meteo parameters as G1 IoT devices.
93
+ source_ids:
94
+ - cira-ciclope
95
+ schedule_config:
96
+ type: manual
97
+ steps:
98
+ - name: iot_device_rows
99
+ kind: custom_sql
100
+ params:
101
+ sql: |-
102
+ WITH tower_inventory AS (
103
+ SELECT tower_id,tower_name,latitude,longitude
104
+ FROM tenant_00000000_cira_ciclope.towers
105
+ ), latest_meteo AS (
106
+ SELECT tower_id,parameter_type,
107
+ max(coalesce(
108
+ try(from_iso8601_timestamp(cast(observed_at AS varchar))),
109
+ try_cast(observed_at AS timestamp(6) with time zone)
110
+ )) last_seen_at
111
+ FROM tenant_00000000_cira_ciclope.realtime_meteo
112
+ GROUP BY tower_id,parameter_type
113
+ ), camera_devices AS (
114
+ SELECT concat('ciclope:camera:',cast(cast(c.tower_id AS bigint) AS varchar),':',cast(cast(c.camera_id AS bigint) AS varchar)) code,
115
+ concat(coalesce(t.tower_name,concat('CICLOPE tower ',cast(cast(c.tower_id AS bigint) AS varchar))),
116
+ ' / ',coalesce(c.camera_name,concat('camera ',cast(cast(c.camera_id AS bigint) AS varchar)))) name,
117
+ concat('camera:',lower(coalesce(c.camera_type,'unknown'))) device_type,
118
+ coalesce(s.operational_status,'unknown') status,
119
+ CASE
120
+ WHEN t.longitude BETWEEN -180.0 AND 180.0
121
+ AND t.latitude BETWEEN -90.0 AND 90.0
122
+ THEN to_geojson_geometry(ST_Point(t.longitude,t.latitude))
123
+ END location_geojson,
124
+ from_iso8601_timestamp(s._extracted_at) last_seen_at
125
+ FROM tenant_00000000_cira_ciclope.cameras c
126
+ LEFT JOIN tenant_00000000_cira_ciclope.camera_status s
127
+ ON s.tower_id=c.tower_id AND s.camera_id=c.camera_id
128
+ LEFT JOIN tower_inventory t
129
+ ON t.tower_id=c.tower_id
130
+ WHERE c.tower_id IS NOT NULL AND c.camera_id IS NOT NULL
131
+ ), meteo_devices AS (
132
+ SELECT concat('ciclope:meteo-sensor:',cast(cast(m.tower_id AS bigint) AS varchar),':',lower(m.parameter_type)) code,
133
+ concat(coalesce(t.tower_name,concat('CICLOPE tower ',cast(cast(m.tower_id AS bigint) AS varchar))),
134
+ ' / ',m.parameter_type) name,
135
+ concat('weather_sensor:',lower(m.parameter_type)) device_type,
136
+ CASE WHEN lm.last_seen_at IS NULL THEN 'installed' ELSE 'operational' END status,
137
+ CASE
138
+ WHEN t.longitude BETWEEN -180.0 AND 180.0
139
+ AND t.latitude BETWEEN -90.0 AND 90.0
140
+ THEN to_geojson_geometry(ST_Point(t.longitude,t.latitude))
141
+ END location_geojson,
142
+ lm.last_seen_at
143
+ FROM tenant_00000000_cira_ciclope.meteo_sensors m
144
+ LEFT JOIN latest_meteo lm
145
+ ON lm.tower_id=m.tower_id AND lm.parameter_type=m.parameter_type
146
+ LEFT JOIN tower_inventory t
147
+ ON t.tower_id=m.tower_id
148
+ WHERE m.tower_id IS NOT NULL AND m.parameter_type IS NOT NULL
149
+ )
150
+ SELECT code,name,device_type,status,location_geojson,last_seen_at
151
+ FROM camera_devices
152
+ UNION ALL
153
+ SELECT code,name,device_type,status,location_geojson,last_seen_at
154
+ FROM meteo_devices
155
+ depends_on: []
156
+ emits_to: backing_dataset
157
+ config:
158
+ output_layer: silver
159
+ ---
160
+ apiVersion: frank.platform/v1
161
+ kind: BackingDataset
162
+ metadata:
163
+ name: cira_ciclope_iot_devices
164
+ spec:
165
+ entity_type_id: iot_device
166
+ entity_type_name: IoT Device
167
+ ontology_tenant_id: cira
168
+ pipeline: cira_ciclope_iot_devices
169
+ sync_mode: manual
170
+ primary_key_column: code
171
+ title_key_column: code
172
+ property_mappings:
173
+ - column: code
174
+ property: code
175
+ type: varchar
176
+ is_primary_key: true
177
+ - column: name
178
+ property: name
179
+ type: varchar
180
+ - column: device_type
181
+ property: device_type
182
+ type: varchar
183
+ - column: status
184
+ property: status
185
+ type: varchar
186
+ - column: location_geojson
187
+ property: location
188
+ type: varchar
189
+ sql_transform_id: json_parse
190
+ - column: last_seen_at
191
+ property: last_seen_at
192
+ type: timestamp with time zone
193
+ ---
194
+ apiVersion: frank.platform/v1
195
+ kind: Pipeline
196
+ metadata:
197
+ name: cira_ciclope_device_readings
198
+ spec:
199
+ name: cira_ciclope_device_readings
200
+ description: CICLOPE meteo observations and numeric camera state as G1 device readings.
201
+ source_ids:
202
+ - cira-ciclope
203
+ schedule_config:
204
+ type: manual
205
+ steps:
206
+ - name: device_reading_rows
207
+ kind: custom_sql
208
+ params:
209
+ sql: |-
210
+ WITH meteo_values AS (
211
+ SELECT record_key,tower_id,parameter_type,
212
+ coalesce(
213
+ try(from_iso8601_timestamp(cast(observed_at AS varchar))),
214
+ try_cast(observed_at AS timestamp(6) with time zone)
215
+ ) observed_at,
216
+ unit,value,0 source_priority
217
+ FROM tenant_00000000_cira_ciclope.realtime_meteo
218
+ UNION ALL
219
+ SELECT record_key,tower_id,parameter_type,
220
+ coalesce(
221
+ try(from_iso8601_timestamp(cast(observed_at AS varchar))),
222
+ try_cast(observed_at AS timestamp(6) with time zone)
223
+ ) observed_at,
224
+ unit,value,1 source_priority
225
+ FROM tenant_00000000_cira_ciclope.meteo_history
226
+ ), meteo_candidates AS (
227
+ SELECT concat('ciclope:meteo-reading:',cast(cast(tower_id AS bigint) AS varchar),':',
228
+ lower(parameter_type),':',lower(coalesce(unit,'unknown')),':',
229
+ to_iso8601(observed_at)) metric,
230
+ concat('ciclope:meteo-sensor:',cast(cast(tower_id AS bigint) AS varchar),':',lower(parameter_type)) device_code,
231
+ cast(value AS double) value,
232
+ observed_at recorded_at,
233
+ source_priority,
234
+ record_key
235
+ FROM meteo_values
236
+ WHERE tower_id IS NOT NULL
237
+ AND parameter_type IS NOT NULL
238
+ AND observed_at IS NOT NULL
239
+ AND value IS NOT NULL
240
+ ), meteo_distinct AS (
241
+ SELECT metric,device_code,value,recorded_at,
242
+ min(source_priority) source_priority,
243
+ min(record_key) record_key
244
+ FROM meteo_candidates
245
+ GROUP BY metric,device_code,value,recorded_at
246
+ ), meteo_rows AS (
247
+ SELECT CASE
248
+ WHEN variant_rank=1 THEN metric
249
+ ELSE concat(metric,':variant:',
250
+ substr(lower(to_hex(sha256(cast(record_key AS varbinary)))),1,16))
251
+ END metric,
252
+ device_code,value,recorded_at
253
+ FROM (
254
+ SELECT metric,device_code,value,recorded_at,record_key,
255
+ row_number() OVER (PARTITION BY metric
256
+ ORDER BY source_priority ASC,value DESC,record_key
257
+ ) variant_rank
258
+ FROM meteo_distinct
259
+ )
260
+ ), camera_rows AS (
261
+ SELECT concat('ciclope:camera-reading:',cast(cast(s.tower_id AS bigint) AS varchar),':',
262
+ cast(cast(s.camera_id AS bigint) AS varchar),':',u.metric_name,':',
263
+ to_iso8601(from_iso8601_timestamp(s._extracted_at))) metric,
264
+ concat('ciclope:camera:',cast(cast(s.tower_id AS bigint) AS varchar),':',cast(cast(s.camera_id AS bigint) AS varchar)) device_code,
265
+ u.reading_value value,
266
+ from_iso8601_timestamp(s._extracted_at) recorded_at
267
+ FROM tenant_00000000_cira_ciclope.camera_status s
268
+ CROSS JOIN UNNEST(
269
+ ARRAY['horizontal_fov_degrees','pan_angle_degrees','tilt_angle_degrees','vertical_fov_degrees','zoom_times'],
270
+ ARRAY[
271
+ cast(s.horizontal_fov_degrees AS double),
272
+ cast(s.pan_angle_degrees AS double),
273
+ cast(s.tilt_angle_degrees AS double),
274
+ cast(s.vertical_fov_degrees AS double),
275
+ cast(s.zoom_times AS double)
276
+ ]
277
+ ) AS u(metric_name,reading_value)
278
+ WHERE s.tower_id IS NOT NULL
279
+ AND s.camera_id IS NOT NULL
280
+ AND s._extracted_at IS NOT NULL
281
+ AND u.reading_value IS NOT NULL
282
+ )
283
+ SELECT metric,device_code,value,recorded_at FROM meteo_rows
284
+ UNION ALL
285
+ SELECT metric,device_code,value,recorded_at FROM camera_rows
286
+ depends_on: []
287
+ emits_to: backing_dataset
288
+ config:
289
+ output_layer: silver
290
+ ---
291
+ apiVersion: frank.platform/v1
292
+ kind: BackingDataset
293
+ metadata:
294
+ name: cira_ciclope_device_readings
295
+ dependsOn:
296
+ - entityTypeId: iot_device
297
+ ontologyTenantId: cira
298
+ spec:
299
+ entity_type_id: device_reading
300
+ entity_type_name: Device Reading
301
+ ontology_tenant_id: cira
302
+ pipeline: cira_ciclope_device_readings
303
+ sync_mode: manual
304
+ primary_key_column: metric
305
+ title_key_column: metric
306
+ property_mappings:
307
+ - column: metric
308
+ property: metric
309
+ type: varchar
310
+ is_primary_key: true
311
+ - column: value
312
+ property: value
313
+ type: double
314
+ - column: recorded_at
315
+ property: recorded_at
316
+ type: timestamp with time zone
317
+ - column: device_code
318
+ property: device
319
+ is_relationship: true
320
+ target_type: iot_device
321
+ target_key: code
322
+ ---
323
+ apiVersion: frank.platform/v1
324
+ kind: Pipeline
325
+ metadata:
326
+ name: cira_ciclope_emergency_incidents
327
+ spec:
328
+ name: cira_ciclope_emergency_incidents
329
+ description: Confirmed CICLOPE fire detections and occurrence entries as E2 incidents.
330
+ source_ids:
331
+ - cira-ciclope
332
+ schedule_config:
333
+ type: manual
334
+ steps:
335
+ - name: emergency_incident_rows
336
+ kind: custom_sql
337
+ params:
338
+ sql: |-
339
+ WITH alarm_candidates AS (
340
+ SELECT concat('ciclope:alarm:',cast(cast(a.alarm_id AS bigint) AS varchar)) code,
341
+ 'wildfire' incident_type,
342
+ 'confirmed' status,
343
+ coalesce(
344
+ try(from_iso8601_timestamp(cast(a.observed_at AS varchar))),
345
+ try_cast(a.observed_at AS timestamp(6) with time zone)
346
+ ) reported_at,
347
+ concat('CICLOPE confirmed automatic detection at tower ',cast(cast(a.tower_id AS bigint) AS varchar),
348
+ ', camera ',cast(cast(a.camera_id AS bigint) AS varchar)) description,
349
+ cast(NULL AS varchar) location_geojson,
350
+ 1 source_priority
351
+ FROM tenant_00000000_cira_ciclope.current_afd_alarms a
352
+ WHERE a.alarm_id IS NOT NULL AND lower(coalesce(a.classification,'')) = 'truealarm'
353
+ UNION ALL
354
+ SELECT concat('ciclope:alarm:',cast(cast(a.alarm_id AS bigint) AS varchar)) code,
355
+ 'wildfire' incident_type,
356
+ 'confirmed' status,
357
+ coalesce(
358
+ try(from_iso8601_timestamp(cast(a.observed_at AS varchar))),
359
+ try_cast(a.observed_at AS timestamp(6) with time zone)
360
+ ) reported_at,
361
+ concat('CICLOPE legacy provider-confirmed automatic detection at tower ',cast(cast(a.tower_id AS bigint) AS varchar),
362
+ ', camera ',cast(cast(a.camera_id AS bigint) AS varchar)) description,
363
+ cast(NULL AS varchar) location_geojson,
364
+ 2 source_priority
365
+ FROM tenant_00000000_cira_ciclope.true_afd_alarms a
366
+ WHERE a.alarm_id IS NOT NULL
367
+ ), confirmed_alarm_rows AS (
368
+ SELECT code,incident_type,status,reported_at,description,location_geojson
369
+ FROM (
370
+ SELECT alarm_candidates.*,
371
+ row_number() OVER (PARTITION BY code ORDER BY source_priority) source_rank
372
+ FROM alarm_candidates
373
+ ) ranked
374
+ WHERE source_rank=1
375
+ ), occurrence_rows AS (
376
+ SELECT concat('ciclope:occurrence-entry:',cast(cast(o.occurrence_entry_id AS bigint) AS varchar)) code,
377
+ coalesce(nullif(o.occurrence_type,''),'emergency') incident_type,
378
+ 'reported' status,
379
+ coalesce(
380
+ coalesce(
381
+ try(from_iso8601_timestamp(cast(o.occurrence_started_at AS varchar))),
382
+ try_cast(o.occurrence_started_at AS timestamp(6) with time zone)
383
+ ),
384
+ coalesce(
385
+ try(from_iso8601_timestamp(cast(o.entry_registered_at AS varchar))),
386
+ try_cast(o.entry_registered_at AS timestamp(6) with time zone)
387
+ )
388
+ ) reported_at,
389
+ o.remarks description,
390
+ CASE
391
+ WHEN o.longitude BETWEEN -180.0 AND 180.0
392
+ AND o.latitude BETWEEN -90.0 AND 90.0
393
+ THEN to_geojson_geometry(ST_Point(o.longitude,o.latitude))
394
+ END location_geojson
395
+ FROM tenant_00000000_cira_ciclope.occurrence_entries o
396
+ WHERE o.occurrence_entry_id IS NOT NULL
397
+ )
398
+ SELECT code,incident_type,status,reported_at,description,location_geojson
399
+ FROM confirmed_alarm_rows
400
+ UNION ALL
401
+ SELECT code,incident_type,status,reported_at,description,location_geojson
402
+ FROM occurrence_rows
403
+ depends_on: []
404
+ emits_to: backing_dataset
405
+ config:
406
+ output_layer: silver
407
+ ---
408
+ apiVersion: frank.platform/v1
409
+ kind: BackingDataset
410
+ metadata:
411
+ name: cira_ciclope_emergency_incidents
412
+ spec:
413
+ entity_type_id: emergency_incident
414
+ entity_type_name: Emergency Incident
415
+ ontology_tenant_id: cira
416
+ pipeline: cira_ciclope_emergency_incidents
417
+ sync_mode: manual
418
+ primary_key_column: code
419
+ title_key_column: code
420
+ property_mappings:
421
+ - column: code
422
+ property: code
423
+ type: varchar
424
+ is_primary_key: true
425
+ - column: incident_type
426
+ property: incident_type
427
+ type: varchar
428
+ - column: status
429
+ property: status
430
+ type: varchar
431
+ - column: reported_at
432
+ property: reported_at
433
+ type: timestamp with time zone
434
+ - column: description
435
+ property: description
436
+ type: varchar
437
+ - column: location_geojson
438
+ property: location
439
+ type: varchar
440
+ sql_transform_id: json_parse
441
+ ---
442
+ apiVersion: frank.platform/v1
443
+ kind: Pipeline
444
+ metadata:
445
+ name: cira_ciclope_emergency_alerts
446
+ spec:
447
+ name: cira_ciclope_emergency_alerts
448
+ description: Every current CICLOPE automatic fire detection as an E2 alert.
449
+ source_ids:
450
+ - cira-ciclope
451
+ schedule_config:
452
+ type: manual
453
+ steps:
454
+ - name: emergency_alert_rows
455
+ kind: custom_sql
456
+ params:
457
+ sql: |-
458
+ SELECT concat('ciclope:alarm:',cast(cast(alarm_id AS bigint) AS varchar)) code,
459
+ 'wildfire_detection' alert_type,
460
+ CASE
461
+ WHEN lower(coalesce(classification,''))='truealarm' THEN 'high'
462
+ WHEN lower(coalesce(classification,''))='falsealarm' THEN 'low'
463
+ ELSE 'unclassified'
464
+ END severity,
465
+ CASE
466
+ WHEN lower(coalesce(classification,''))='truealarm' THEN 'confirmed'
467
+ WHEN lower(coalesce(classification,''))='falsealarm' THEN 'dismissed'
468
+ ELSE 'active'
469
+ END status,
470
+ coalesce(
471
+ try(from_iso8601_timestamp(cast(observed_at AS varchar))),
472
+ try_cast(observed_at AS timestamp(6) with time zone)
473
+ ) issued_at,
474
+ concat('CICLOPE automatic fire detection at tower ',cast(cast(tower_id AS bigint) AS varchar),
475
+ ', camera ',cast(cast(camera_id AS bigint) AS varchar),
476
+ '; classification=',coalesce(classification,'None')) message,
477
+ CASE WHEN lower(coalesce(classification,'')) = 'truealarm'
478
+ THEN concat('ciclope:alarm:',cast(cast(alarm_id AS bigint) AS varchar))
479
+ END incident_code
480
+ FROM tenant_00000000_cira_ciclope.current_afd_alarms
481
+ WHERE alarm_id IS NOT NULL
482
+ depends_on: []
483
+ emits_to: backing_dataset
484
+ config:
485
+ output_layer: silver
486
+ ---
487
+ apiVersion: frank.platform/v1
488
+ kind: BackingDataset
489
+ metadata:
490
+ name: cira_ciclope_emergency_alerts
491
+ dependsOn:
492
+ - entityTypeId: emergency_incident
493
+ ontologyTenantId: cira
494
+ spec:
495
+ entity_type_id: emergency_alert
496
+ entity_type_name: Emergency Alert
497
+ ontology_tenant_id: cira
498
+ pipeline: cira_ciclope_emergency_alerts
499
+ sync_mode: manual
500
+ primary_key_column: code
501
+ title_key_column: code
502
+ property_mappings:
503
+ - column: code
504
+ property: code
505
+ type: varchar
506
+ is_primary_key: true
507
+ - column: alert_type
508
+ property: alert_type
509
+ type: varchar
510
+ - column: severity
511
+ property: severity
512
+ type: varchar
513
+ - column: status
514
+ property: status
515
+ type: varchar
516
+ - column: issued_at
517
+ property: issued_at
518
+ type: timestamp with time zone
519
+ - column: message
520
+ property: message
521
+ type: varchar
522
+ - column: incident_code
523
+ property: incident
524
+ is_relationship: true
525
+ target_type: emergency_incident
526
+ target_key: code
@@ -0,0 +1,156 @@
1
+ # Complete CIRA Cityfy agenda dataset (issue #589).
2
+ #
3
+ # One REST resource owns pagination and lands one canonical Bronze table. The
4
+ # connector advances ?page=1,2,... until the selected `data` array is empty;
5
+ # there is deliberately no maximum page. Reapply adopts the existing Source
6
+ # and converges the declared `events` stream in place. The five live legacy
7
+ # page streams are explicit disabled migration entries: their runtime state is
8
+ # preserved, but scheduled/direct Source runs cannot execute them.
9
+ #
10
+ # Live verification on 2026-07-22 returned 50/50/50/6 records and then an empty
11
+ # page from the `regiaodeaveiro-eventos` hostname below over valid TLS. The
12
+ # similarly named `regiaodeaveiro.hq1.cityfy.pt` endpoint returns HTTP 404;
13
+ # do not substitute it and do not add a certificate-verification bypass.
14
+
15
+ apiVersion: frank.platform/v1
16
+ kind: Source
17
+ metadata:
18
+ name: cira-cityfy-agenda
19
+ spec:
20
+ name: cira-cityfy-agenda
21
+ description: Complete Cityfy agenda as one connector-paginated events dataset.
22
+ pattern_id: rest_api
23
+ source_config:
24
+ base_url: https://regiaodeaveiro-eventos.hq1.cityfy.pt/api/v1
25
+ resources:
26
+ - /events
27
+ data_selector: data
28
+ paginator: page_number
29
+ page_param: page
30
+ base_page: 1
31
+ pagination_total_path: null
32
+ stop_after_empty_page: true
33
+ primary_keys:
34
+ - id
35
+ schedule_type: manual
36
+ streams:
37
+ - name: events
38
+ sync_mode: full_refresh
39
+ write_disposition: replace
40
+ primary_key_path:
41
+ - id
42
+ is_enabled: true
43
+ dest_table_name: events
44
+ # Migration tombstones for the page-per-stream configuration currently on
45
+ # the adopted live Source. They are not connector resources or Bronze
46
+ # inputs; declaring them disabled makes every Source execution path safe.
47
+ - name: events_page_1
48
+ is_enabled: false
49
+ - name: events_page_2
50
+ is_enabled: false
51
+ - name: events_page_3
52
+ is_enabled: false
53
+ - name: events_page_4
54
+ is_enabled: false
55
+ - name: events_page_5
56
+ is_enabled: false
57
+ ---
58
+ apiVersion: frank.platform/v1
59
+ kind: Pipeline
60
+ metadata:
61
+ name: cira_cityfy_events
62
+ spec:
63
+ name: cira_cityfy_events
64
+ description: Complete CIRA Cityfy agenda projection into canonical events.
65
+ source_ids:
66
+ - cira-cityfy-agenda
67
+ schedule_config:
68
+ type: manual
69
+ steps:
70
+ - name: event_rows
71
+ kind: custom_sql
72
+ params:
73
+ sql: |-
74
+ SELECT
75
+ concat('cityfy:', trim(cast(o."id" AS varchar))) AS source_key,
76
+ concat('cityfy:', trim(cast(o."id" AS varchar))) AS external_id,
77
+ nullif(trim(cast(o."title"."value" AS varchar)), '') AS name,
78
+ nullif(trim(cast(o."description"."value" AS varchar)), '') AS description,
79
+ 'Cityfy' AS data_provider,
80
+ CASE
81
+ WHEN cardinality(o."location"."value"."coordinates") = 2
82
+ AND element_at(o."location"."value"."coordinates", 1) BETWEEN -180.0 AND 180.0
83
+ AND element_at(o."location"."value"."coordinates", 2) BETWEEN -90.0 AND 90.0
84
+ THEN to_geojson_geometry(ST_Point(
85
+ element_at(o."location"."value"."coordinates", 1),
86
+ element_at(o."location"."value"."coordinates", 2)
87
+ ))
88
+ END AS location_geojson,
89
+ json_format(CAST(o."see_also"."value" AS JSON)) AS see_also_json,
90
+ from_iso8601_timestamp(o."date_created"."value") AS date_created,
91
+ from_iso8601_timestamp(o."date_modified"."value") AS date_modified,
92
+ from_iso8601_timestamp(o."date_started"."value") AS start_date,
93
+ from_iso8601_timestamp(o."date_finished"."value") AS end_date,
94
+ nullif(trim(cast(o."image"."value" AS varchar)), '') AS content_url
95
+ FROM tenant_00000000_cira_cityfy_agenda.events o
96
+ WHERE nullif(trim(cast(o."id" AS varchar)), '') IS NOT NULL
97
+ depends_on: []
98
+ emits_to: backing_dataset
99
+ config:
100
+ output_layer: silver
101
+ ---
102
+ apiVersion: frank.platform/v1
103
+ kind: BackingDataset
104
+ metadata:
105
+ name: cira_cityfy_events
106
+ spec:
107
+ entity_type_id: event
108
+ entity_type_name: Event
109
+ ontology_tenant_id: cira
110
+ pipeline: cira_cityfy_events
111
+ # Adopt the bounded-canary BackingDataset in place. This field is immutable
112
+ # after creation; preserving on_materialization avoids destructive
113
+ # recreation and is safe because Cityfy has no ontology dependencies.
114
+ sync_mode: on_materialization
115
+ primary_key_column: source_key
116
+ title_key_column: name
117
+ property_mappings:
118
+ - column: source_key
119
+ property: source
120
+ type: string
121
+ is_primary_key: true
122
+ - column: external_id
123
+ property: external_id
124
+ type: string
125
+ - column: name
126
+ property: name
127
+ type: string
128
+ - column: description
129
+ property: description
130
+ type: string
131
+ - column: data_provider
132
+ property: data_provider
133
+ type: string
134
+ - column: location_geojson
135
+ property: location
136
+ type: varchar
137
+ sql_transform_id: json_parse
138
+ - column: see_also_json
139
+ property: see_also
140
+ type: varchar
141
+ sql_transform_id: json_parse
142
+ - column: date_created
143
+ property: date_created
144
+ type: timestamp with time zone
145
+ - column: date_modified
146
+ property: date_modified
147
+ type: timestamp with time zone
148
+ - column: start_date
149
+ property: start_date
150
+ type: timestamp with time zone
151
+ - column: end_date
152
+ property: end_date
153
+ type: timestamp with time zone
154
+ - column: content_url
155
+ property: content_url
156
+ type: string