@aiaiai-pt/frankctl 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +23 -0
- package/README.md +11 -6
- package/dist/index.js +208 -197
- package/dist/template-packs/cira/01_organization.yaml +68 -0
- package/dist/template-packs/cira/02_freguesia.yaml +73 -0
- package/dist/template-packs/cira/03_transit_route.yaml +70 -0
- package/dist/template-packs/cira/04_transit_stop.yaml +57 -0
- package/dist/template-packs/cira/05_air_quality_station.yaml +83 -0
- package/dist/template-packs/cira/06_air_quality_observed.yaml +99 -0
- package/dist/template-packs/cira/07_transit_trip.yaml +106 -0
- package/dist/template-packs/cira/08_vehicle_position.yaml +81 -0
- package/dist/template-packs/cira/09_transdev_aveirobus_gtfs.yaml +709 -0
- package/dist/template-packs/cira/10_ciclope_emergency.yaml +526 -0
- package/dist/template-packs/cira/cityfy_agenda_events.yaml +156 -0
- package/dist/template-packs/cira/template-pack.yaml +109 -0
- package/package.json +2 -1
- package/skills/frank-cli/SKILL.md +36 -14
|
@@ -0,0 +1,526 @@
|
|
|
1
|
+
# Complete Git-owned CIRA Ciclope Pipeline + BackingDataset bundle (#601).
|
|
2
|
+
#
|
|
3
|
+
# Apply the ignored chmod-0600 Source manifest first, sync all nine Bronze
|
|
4
|
+
# streams, then apply this Source-free bundle. It projects only into existing
|
|
5
|
+
# CIRA E2 Civil Protection and G1 IoT entity types; it never owns schema.
|
|
6
|
+
|
|
7
|
+
apiVersion: frank.platform/v1
|
|
8
|
+
kind: Pipeline
|
|
9
|
+
metadata:
|
|
10
|
+
name: cira_ciclope_emergency_resources
|
|
11
|
+
spec:
|
|
12
|
+
name: cira_ciclope_emergency_resources
|
|
13
|
+
description: Installed CICLOPE fire-detection towers as E2 emergency resources.
|
|
14
|
+
source_ids:
|
|
15
|
+
- cira-ciclope
|
|
16
|
+
schedule_config:
|
|
17
|
+
type: manual
|
|
18
|
+
steps:
|
|
19
|
+
- name: emergency_resource_rows
|
|
20
|
+
kind: custom_sql
|
|
21
|
+
params:
|
|
22
|
+
sql: |-
|
|
23
|
+
WITH camera_rollup AS (
|
|
24
|
+
SELECT c.tower_id,
|
|
25
|
+
count(*) camera_count,
|
|
26
|
+
CASE
|
|
27
|
+
WHEN count_if(lower(coalesce(s.operational_status,''))='operational')=count(*)
|
|
28
|
+
THEN 'operational'
|
|
29
|
+
WHEN count_if(lower(coalesce(s.operational_status,''))='operational')=0
|
|
30
|
+
THEN 'unavailable'
|
|
31
|
+
ELSE 'degraded'
|
|
32
|
+
END resource_status
|
|
33
|
+
FROM tenant_00000000_cira_ciclope.cameras c
|
|
34
|
+
LEFT JOIN tenant_00000000_cira_ciclope.camera_status s
|
|
35
|
+
ON s.tower_id=c.tower_id AND s.camera_id=c.camera_id
|
|
36
|
+
GROUP BY c.tower_id
|
|
37
|
+
)
|
|
38
|
+
SELECT concat('ciclope:tower:',cast(cast(t.tower_id AS bigint) AS varchar)) code,
|
|
39
|
+
'fire_detection_tower' resource_type,
|
|
40
|
+
coalesce(r.resource_status,'unknown') status,
|
|
41
|
+
cast(coalesce(r.camera_count,0) AS integer) capacity,
|
|
42
|
+
CASE
|
|
43
|
+
WHEN t.longitude BETWEEN -180.0 AND 180.0
|
|
44
|
+
AND t.latitude BETWEEN -90.0 AND 90.0
|
|
45
|
+
THEN to_geojson_geometry(ST_Point(t.longitude,t.latitude))
|
|
46
|
+
END location_geojson
|
|
47
|
+
FROM tenant_00000000_cira_ciclope.towers t
|
|
48
|
+
LEFT JOIN camera_rollup r ON r.tower_id=t.tower_id
|
|
49
|
+
WHERE t.tower_id IS NOT NULL
|
|
50
|
+
depends_on: []
|
|
51
|
+
emits_to: backing_dataset
|
|
52
|
+
config:
|
|
53
|
+
output_layer: silver
|
|
54
|
+
---
|
|
55
|
+
apiVersion: frank.platform/v1
|
|
56
|
+
kind: BackingDataset
|
|
57
|
+
metadata:
|
|
58
|
+
name: cira_ciclope_emergency_resources
|
|
59
|
+
spec:
|
|
60
|
+
entity_type_id: emergency_resource
|
|
61
|
+
entity_type_name: Emergency Resource
|
|
62
|
+
ontology_tenant_id: cira
|
|
63
|
+
pipeline: cira_ciclope_emergency_resources
|
|
64
|
+
sync_mode: manual
|
|
65
|
+
primary_key_column: code
|
|
66
|
+
title_key_column: code
|
|
67
|
+
property_mappings:
|
|
68
|
+
- column: code
|
|
69
|
+
property: code
|
|
70
|
+
type: varchar
|
|
71
|
+
is_primary_key: true
|
|
72
|
+
- column: resource_type
|
|
73
|
+
property: resource_type
|
|
74
|
+
type: varchar
|
|
75
|
+
- column: status
|
|
76
|
+
property: status
|
|
77
|
+
type: varchar
|
|
78
|
+
- column: capacity
|
|
79
|
+
property: capacity
|
|
80
|
+
type: integer
|
|
81
|
+
- column: location_geojson
|
|
82
|
+
property: location
|
|
83
|
+
type: varchar
|
|
84
|
+
sql_transform_id: json_parse
|
|
85
|
+
---
|
|
86
|
+
apiVersion: frank.platform/v1
|
|
87
|
+
kind: Pipeline
|
|
88
|
+
metadata:
|
|
89
|
+
name: cira_ciclope_iot_devices
|
|
90
|
+
spec:
|
|
91
|
+
name: cira_ciclope_iot_devices
|
|
92
|
+
description: CICLOPE cameras and installed meteo parameters as G1 IoT devices.
|
|
93
|
+
source_ids:
|
|
94
|
+
- cira-ciclope
|
|
95
|
+
schedule_config:
|
|
96
|
+
type: manual
|
|
97
|
+
steps:
|
|
98
|
+
- name: iot_device_rows
|
|
99
|
+
kind: custom_sql
|
|
100
|
+
params:
|
|
101
|
+
sql: |-
|
|
102
|
+
WITH tower_inventory AS (
|
|
103
|
+
SELECT tower_id,tower_name,latitude,longitude
|
|
104
|
+
FROM tenant_00000000_cira_ciclope.towers
|
|
105
|
+
), latest_meteo AS (
|
|
106
|
+
SELECT tower_id,parameter_type,
|
|
107
|
+
max(coalesce(
|
|
108
|
+
try(from_iso8601_timestamp(cast(observed_at AS varchar))),
|
|
109
|
+
try_cast(observed_at AS timestamp(6) with time zone)
|
|
110
|
+
)) last_seen_at
|
|
111
|
+
FROM tenant_00000000_cira_ciclope.realtime_meteo
|
|
112
|
+
GROUP BY tower_id,parameter_type
|
|
113
|
+
), camera_devices AS (
|
|
114
|
+
SELECT concat('ciclope:camera:',cast(cast(c.tower_id AS bigint) AS varchar),':',cast(cast(c.camera_id AS bigint) AS varchar)) code,
|
|
115
|
+
concat(coalesce(t.tower_name,concat('CICLOPE tower ',cast(cast(c.tower_id AS bigint) AS varchar))),
|
|
116
|
+
' / ',coalesce(c.camera_name,concat('camera ',cast(cast(c.camera_id AS bigint) AS varchar)))) name,
|
|
117
|
+
concat('camera:',lower(coalesce(c.camera_type,'unknown'))) device_type,
|
|
118
|
+
coalesce(s.operational_status,'unknown') status,
|
|
119
|
+
CASE
|
|
120
|
+
WHEN t.longitude BETWEEN -180.0 AND 180.0
|
|
121
|
+
AND t.latitude BETWEEN -90.0 AND 90.0
|
|
122
|
+
THEN to_geojson_geometry(ST_Point(t.longitude,t.latitude))
|
|
123
|
+
END location_geojson,
|
|
124
|
+
from_iso8601_timestamp(s._extracted_at) last_seen_at
|
|
125
|
+
FROM tenant_00000000_cira_ciclope.cameras c
|
|
126
|
+
LEFT JOIN tenant_00000000_cira_ciclope.camera_status s
|
|
127
|
+
ON s.tower_id=c.tower_id AND s.camera_id=c.camera_id
|
|
128
|
+
LEFT JOIN tower_inventory t
|
|
129
|
+
ON t.tower_id=c.tower_id
|
|
130
|
+
WHERE c.tower_id IS NOT NULL AND c.camera_id IS NOT NULL
|
|
131
|
+
), meteo_devices AS (
|
|
132
|
+
SELECT concat('ciclope:meteo-sensor:',cast(cast(m.tower_id AS bigint) AS varchar),':',lower(m.parameter_type)) code,
|
|
133
|
+
concat(coalesce(t.tower_name,concat('CICLOPE tower ',cast(cast(m.tower_id AS bigint) AS varchar))),
|
|
134
|
+
' / ',m.parameter_type) name,
|
|
135
|
+
concat('weather_sensor:',lower(m.parameter_type)) device_type,
|
|
136
|
+
CASE WHEN lm.last_seen_at IS NULL THEN 'installed' ELSE 'operational' END status,
|
|
137
|
+
CASE
|
|
138
|
+
WHEN t.longitude BETWEEN -180.0 AND 180.0
|
|
139
|
+
AND t.latitude BETWEEN -90.0 AND 90.0
|
|
140
|
+
THEN to_geojson_geometry(ST_Point(t.longitude,t.latitude))
|
|
141
|
+
END location_geojson,
|
|
142
|
+
lm.last_seen_at
|
|
143
|
+
FROM tenant_00000000_cira_ciclope.meteo_sensors m
|
|
144
|
+
LEFT JOIN latest_meteo lm
|
|
145
|
+
ON lm.tower_id=m.tower_id AND lm.parameter_type=m.parameter_type
|
|
146
|
+
LEFT JOIN tower_inventory t
|
|
147
|
+
ON t.tower_id=m.tower_id
|
|
148
|
+
WHERE m.tower_id IS NOT NULL AND m.parameter_type IS NOT NULL
|
|
149
|
+
)
|
|
150
|
+
SELECT code,name,device_type,status,location_geojson,last_seen_at
|
|
151
|
+
FROM camera_devices
|
|
152
|
+
UNION ALL
|
|
153
|
+
SELECT code,name,device_type,status,location_geojson,last_seen_at
|
|
154
|
+
FROM meteo_devices
|
|
155
|
+
depends_on: []
|
|
156
|
+
emits_to: backing_dataset
|
|
157
|
+
config:
|
|
158
|
+
output_layer: silver
|
|
159
|
+
---
|
|
160
|
+
apiVersion: frank.platform/v1
|
|
161
|
+
kind: BackingDataset
|
|
162
|
+
metadata:
|
|
163
|
+
name: cira_ciclope_iot_devices
|
|
164
|
+
spec:
|
|
165
|
+
entity_type_id: iot_device
|
|
166
|
+
entity_type_name: IoT Device
|
|
167
|
+
ontology_tenant_id: cira
|
|
168
|
+
pipeline: cira_ciclope_iot_devices
|
|
169
|
+
sync_mode: manual
|
|
170
|
+
primary_key_column: code
|
|
171
|
+
title_key_column: code
|
|
172
|
+
property_mappings:
|
|
173
|
+
- column: code
|
|
174
|
+
property: code
|
|
175
|
+
type: varchar
|
|
176
|
+
is_primary_key: true
|
|
177
|
+
- column: name
|
|
178
|
+
property: name
|
|
179
|
+
type: varchar
|
|
180
|
+
- column: device_type
|
|
181
|
+
property: device_type
|
|
182
|
+
type: varchar
|
|
183
|
+
- column: status
|
|
184
|
+
property: status
|
|
185
|
+
type: varchar
|
|
186
|
+
- column: location_geojson
|
|
187
|
+
property: location
|
|
188
|
+
type: varchar
|
|
189
|
+
sql_transform_id: json_parse
|
|
190
|
+
- column: last_seen_at
|
|
191
|
+
property: last_seen_at
|
|
192
|
+
type: timestamp with time zone
|
|
193
|
+
---
|
|
194
|
+
apiVersion: frank.platform/v1
|
|
195
|
+
kind: Pipeline
|
|
196
|
+
metadata:
|
|
197
|
+
name: cira_ciclope_device_readings
|
|
198
|
+
spec:
|
|
199
|
+
name: cira_ciclope_device_readings
|
|
200
|
+
description: CICLOPE meteo observations and numeric camera state as G1 device readings.
|
|
201
|
+
source_ids:
|
|
202
|
+
- cira-ciclope
|
|
203
|
+
schedule_config:
|
|
204
|
+
type: manual
|
|
205
|
+
steps:
|
|
206
|
+
- name: device_reading_rows
|
|
207
|
+
kind: custom_sql
|
|
208
|
+
params:
|
|
209
|
+
sql: |-
|
|
210
|
+
WITH meteo_values AS (
|
|
211
|
+
SELECT record_key,tower_id,parameter_type,
|
|
212
|
+
coalesce(
|
|
213
|
+
try(from_iso8601_timestamp(cast(observed_at AS varchar))),
|
|
214
|
+
try_cast(observed_at AS timestamp(6) with time zone)
|
|
215
|
+
) observed_at,
|
|
216
|
+
unit,value,0 source_priority
|
|
217
|
+
FROM tenant_00000000_cira_ciclope.realtime_meteo
|
|
218
|
+
UNION ALL
|
|
219
|
+
SELECT record_key,tower_id,parameter_type,
|
|
220
|
+
coalesce(
|
|
221
|
+
try(from_iso8601_timestamp(cast(observed_at AS varchar))),
|
|
222
|
+
try_cast(observed_at AS timestamp(6) with time zone)
|
|
223
|
+
) observed_at,
|
|
224
|
+
unit,value,1 source_priority
|
|
225
|
+
FROM tenant_00000000_cira_ciclope.meteo_history
|
|
226
|
+
), meteo_candidates AS (
|
|
227
|
+
SELECT concat('ciclope:meteo-reading:',cast(cast(tower_id AS bigint) AS varchar),':',
|
|
228
|
+
lower(parameter_type),':',lower(coalesce(unit,'unknown')),':',
|
|
229
|
+
to_iso8601(observed_at)) metric,
|
|
230
|
+
concat('ciclope:meteo-sensor:',cast(cast(tower_id AS bigint) AS varchar),':',lower(parameter_type)) device_code,
|
|
231
|
+
cast(value AS double) value,
|
|
232
|
+
observed_at recorded_at,
|
|
233
|
+
source_priority,
|
|
234
|
+
record_key
|
|
235
|
+
FROM meteo_values
|
|
236
|
+
WHERE tower_id IS NOT NULL
|
|
237
|
+
AND parameter_type IS NOT NULL
|
|
238
|
+
AND observed_at IS NOT NULL
|
|
239
|
+
AND value IS NOT NULL
|
|
240
|
+
), meteo_distinct AS (
|
|
241
|
+
SELECT metric,device_code,value,recorded_at,
|
|
242
|
+
min(source_priority) source_priority,
|
|
243
|
+
min(record_key) record_key
|
|
244
|
+
FROM meteo_candidates
|
|
245
|
+
GROUP BY metric,device_code,value,recorded_at
|
|
246
|
+
), meteo_rows AS (
|
|
247
|
+
SELECT CASE
|
|
248
|
+
WHEN variant_rank=1 THEN metric
|
|
249
|
+
ELSE concat(metric,':variant:',
|
|
250
|
+
substr(lower(to_hex(sha256(cast(record_key AS varbinary)))),1,16))
|
|
251
|
+
END metric,
|
|
252
|
+
device_code,value,recorded_at
|
|
253
|
+
FROM (
|
|
254
|
+
SELECT metric,device_code,value,recorded_at,record_key,
|
|
255
|
+
row_number() OVER (PARTITION BY metric
|
|
256
|
+
ORDER BY source_priority ASC,value DESC,record_key
|
|
257
|
+
) variant_rank
|
|
258
|
+
FROM meteo_distinct
|
|
259
|
+
)
|
|
260
|
+
), camera_rows AS (
|
|
261
|
+
SELECT concat('ciclope:camera-reading:',cast(cast(s.tower_id AS bigint) AS varchar),':',
|
|
262
|
+
cast(cast(s.camera_id AS bigint) AS varchar),':',u.metric_name,':',
|
|
263
|
+
to_iso8601(from_iso8601_timestamp(s._extracted_at))) metric,
|
|
264
|
+
concat('ciclope:camera:',cast(cast(s.tower_id AS bigint) AS varchar),':',cast(cast(s.camera_id AS bigint) AS varchar)) device_code,
|
|
265
|
+
u.reading_value value,
|
|
266
|
+
from_iso8601_timestamp(s._extracted_at) recorded_at
|
|
267
|
+
FROM tenant_00000000_cira_ciclope.camera_status s
|
|
268
|
+
CROSS JOIN UNNEST(
|
|
269
|
+
ARRAY['horizontal_fov_degrees','pan_angle_degrees','tilt_angle_degrees','vertical_fov_degrees','zoom_times'],
|
|
270
|
+
ARRAY[
|
|
271
|
+
cast(s.horizontal_fov_degrees AS double),
|
|
272
|
+
cast(s.pan_angle_degrees AS double),
|
|
273
|
+
cast(s.tilt_angle_degrees AS double),
|
|
274
|
+
cast(s.vertical_fov_degrees AS double),
|
|
275
|
+
cast(s.zoom_times AS double)
|
|
276
|
+
]
|
|
277
|
+
) AS u(metric_name,reading_value)
|
|
278
|
+
WHERE s.tower_id IS NOT NULL
|
|
279
|
+
AND s.camera_id IS NOT NULL
|
|
280
|
+
AND s._extracted_at IS NOT NULL
|
|
281
|
+
AND u.reading_value IS NOT NULL
|
|
282
|
+
)
|
|
283
|
+
SELECT metric,device_code,value,recorded_at FROM meteo_rows
|
|
284
|
+
UNION ALL
|
|
285
|
+
SELECT metric,device_code,value,recorded_at FROM camera_rows
|
|
286
|
+
depends_on: []
|
|
287
|
+
emits_to: backing_dataset
|
|
288
|
+
config:
|
|
289
|
+
output_layer: silver
|
|
290
|
+
---
|
|
291
|
+
apiVersion: frank.platform/v1
|
|
292
|
+
kind: BackingDataset
|
|
293
|
+
metadata:
|
|
294
|
+
name: cira_ciclope_device_readings
|
|
295
|
+
dependsOn:
|
|
296
|
+
- entityTypeId: iot_device
|
|
297
|
+
ontologyTenantId: cira
|
|
298
|
+
spec:
|
|
299
|
+
entity_type_id: device_reading
|
|
300
|
+
entity_type_name: Device Reading
|
|
301
|
+
ontology_tenant_id: cira
|
|
302
|
+
pipeline: cira_ciclope_device_readings
|
|
303
|
+
sync_mode: manual
|
|
304
|
+
primary_key_column: metric
|
|
305
|
+
title_key_column: metric
|
|
306
|
+
property_mappings:
|
|
307
|
+
- column: metric
|
|
308
|
+
property: metric
|
|
309
|
+
type: varchar
|
|
310
|
+
is_primary_key: true
|
|
311
|
+
- column: value
|
|
312
|
+
property: value
|
|
313
|
+
type: double
|
|
314
|
+
- column: recorded_at
|
|
315
|
+
property: recorded_at
|
|
316
|
+
type: timestamp with time zone
|
|
317
|
+
- column: device_code
|
|
318
|
+
property: device
|
|
319
|
+
is_relationship: true
|
|
320
|
+
target_type: iot_device
|
|
321
|
+
target_key: code
|
|
322
|
+
---
|
|
323
|
+
apiVersion: frank.platform/v1
|
|
324
|
+
kind: Pipeline
|
|
325
|
+
metadata:
|
|
326
|
+
name: cira_ciclope_emergency_incidents
|
|
327
|
+
spec:
|
|
328
|
+
name: cira_ciclope_emergency_incidents
|
|
329
|
+
description: Confirmed CICLOPE fire detections and occurrence entries as E2 incidents.
|
|
330
|
+
source_ids:
|
|
331
|
+
- cira-ciclope
|
|
332
|
+
schedule_config:
|
|
333
|
+
type: manual
|
|
334
|
+
steps:
|
|
335
|
+
- name: emergency_incident_rows
|
|
336
|
+
kind: custom_sql
|
|
337
|
+
params:
|
|
338
|
+
sql: |-
|
|
339
|
+
WITH alarm_candidates AS (
|
|
340
|
+
SELECT concat('ciclope:alarm:',cast(cast(a.alarm_id AS bigint) AS varchar)) code,
|
|
341
|
+
'wildfire' incident_type,
|
|
342
|
+
'confirmed' status,
|
|
343
|
+
coalesce(
|
|
344
|
+
try(from_iso8601_timestamp(cast(a.observed_at AS varchar))),
|
|
345
|
+
try_cast(a.observed_at AS timestamp(6) with time zone)
|
|
346
|
+
) reported_at,
|
|
347
|
+
concat('CICLOPE confirmed automatic detection at tower ',cast(cast(a.tower_id AS bigint) AS varchar),
|
|
348
|
+
', camera ',cast(cast(a.camera_id AS bigint) AS varchar)) description,
|
|
349
|
+
cast(NULL AS varchar) location_geojson,
|
|
350
|
+
1 source_priority
|
|
351
|
+
FROM tenant_00000000_cira_ciclope.current_afd_alarms a
|
|
352
|
+
WHERE a.alarm_id IS NOT NULL AND lower(coalesce(a.classification,'')) = 'truealarm'
|
|
353
|
+
UNION ALL
|
|
354
|
+
SELECT concat('ciclope:alarm:',cast(cast(a.alarm_id AS bigint) AS varchar)) code,
|
|
355
|
+
'wildfire' incident_type,
|
|
356
|
+
'confirmed' status,
|
|
357
|
+
coalesce(
|
|
358
|
+
try(from_iso8601_timestamp(cast(a.observed_at AS varchar))),
|
|
359
|
+
try_cast(a.observed_at AS timestamp(6) with time zone)
|
|
360
|
+
) reported_at,
|
|
361
|
+
concat('CICLOPE legacy provider-confirmed automatic detection at tower ',cast(cast(a.tower_id AS bigint) AS varchar),
|
|
362
|
+
', camera ',cast(cast(a.camera_id AS bigint) AS varchar)) description,
|
|
363
|
+
cast(NULL AS varchar) location_geojson,
|
|
364
|
+
2 source_priority
|
|
365
|
+
FROM tenant_00000000_cira_ciclope.true_afd_alarms a
|
|
366
|
+
WHERE a.alarm_id IS NOT NULL
|
|
367
|
+
), confirmed_alarm_rows AS (
|
|
368
|
+
SELECT code,incident_type,status,reported_at,description,location_geojson
|
|
369
|
+
FROM (
|
|
370
|
+
SELECT alarm_candidates.*,
|
|
371
|
+
row_number() OVER (PARTITION BY code ORDER BY source_priority) source_rank
|
|
372
|
+
FROM alarm_candidates
|
|
373
|
+
) ranked
|
|
374
|
+
WHERE source_rank=1
|
|
375
|
+
), occurrence_rows AS (
|
|
376
|
+
SELECT concat('ciclope:occurrence-entry:',cast(cast(o.occurrence_entry_id AS bigint) AS varchar)) code,
|
|
377
|
+
coalesce(nullif(o.occurrence_type,''),'emergency') incident_type,
|
|
378
|
+
'reported' status,
|
|
379
|
+
coalesce(
|
|
380
|
+
coalesce(
|
|
381
|
+
try(from_iso8601_timestamp(cast(o.occurrence_started_at AS varchar))),
|
|
382
|
+
try_cast(o.occurrence_started_at AS timestamp(6) with time zone)
|
|
383
|
+
),
|
|
384
|
+
coalesce(
|
|
385
|
+
try(from_iso8601_timestamp(cast(o.entry_registered_at AS varchar))),
|
|
386
|
+
try_cast(o.entry_registered_at AS timestamp(6) with time zone)
|
|
387
|
+
)
|
|
388
|
+
) reported_at,
|
|
389
|
+
o.remarks description,
|
|
390
|
+
CASE
|
|
391
|
+
WHEN o.longitude BETWEEN -180.0 AND 180.0
|
|
392
|
+
AND o.latitude BETWEEN -90.0 AND 90.0
|
|
393
|
+
THEN to_geojson_geometry(ST_Point(o.longitude,o.latitude))
|
|
394
|
+
END location_geojson
|
|
395
|
+
FROM tenant_00000000_cira_ciclope.occurrence_entries o
|
|
396
|
+
WHERE o.occurrence_entry_id IS NOT NULL
|
|
397
|
+
)
|
|
398
|
+
SELECT code,incident_type,status,reported_at,description,location_geojson
|
|
399
|
+
FROM confirmed_alarm_rows
|
|
400
|
+
UNION ALL
|
|
401
|
+
SELECT code,incident_type,status,reported_at,description,location_geojson
|
|
402
|
+
FROM occurrence_rows
|
|
403
|
+
depends_on: []
|
|
404
|
+
emits_to: backing_dataset
|
|
405
|
+
config:
|
|
406
|
+
output_layer: silver
|
|
407
|
+
---
|
|
408
|
+
apiVersion: frank.platform/v1
|
|
409
|
+
kind: BackingDataset
|
|
410
|
+
metadata:
|
|
411
|
+
name: cira_ciclope_emergency_incidents
|
|
412
|
+
spec:
|
|
413
|
+
entity_type_id: emergency_incident
|
|
414
|
+
entity_type_name: Emergency Incident
|
|
415
|
+
ontology_tenant_id: cira
|
|
416
|
+
pipeline: cira_ciclope_emergency_incidents
|
|
417
|
+
sync_mode: manual
|
|
418
|
+
primary_key_column: code
|
|
419
|
+
title_key_column: code
|
|
420
|
+
property_mappings:
|
|
421
|
+
- column: code
|
|
422
|
+
property: code
|
|
423
|
+
type: varchar
|
|
424
|
+
is_primary_key: true
|
|
425
|
+
- column: incident_type
|
|
426
|
+
property: incident_type
|
|
427
|
+
type: varchar
|
|
428
|
+
- column: status
|
|
429
|
+
property: status
|
|
430
|
+
type: varchar
|
|
431
|
+
- column: reported_at
|
|
432
|
+
property: reported_at
|
|
433
|
+
type: timestamp with time zone
|
|
434
|
+
- column: description
|
|
435
|
+
property: description
|
|
436
|
+
type: varchar
|
|
437
|
+
- column: location_geojson
|
|
438
|
+
property: location
|
|
439
|
+
type: varchar
|
|
440
|
+
sql_transform_id: json_parse
|
|
441
|
+
---
|
|
442
|
+
apiVersion: frank.platform/v1
|
|
443
|
+
kind: Pipeline
|
|
444
|
+
metadata:
|
|
445
|
+
name: cira_ciclope_emergency_alerts
|
|
446
|
+
spec:
|
|
447
|
+
name: cira_ciclope_emergency_alerts
|
|
448
|
+
description: Every current CICLOPE automatic fire detection as an E2 alert.
|
|
449
|
+
source_ids:
|
|
450
|
+
- cira-ciclope
|
|
451
|
+
schedule_config:
|
|
452
|
+
type: manual
|
|
453
|
+
steps:
|
|
454
|
+
- name: emergency_alert_rows
|
|
455
|
+
kind: custom_sql
|
|
456
|
+
params:
|
|
457
|
+
sql: |-
|
|
458
|
+
SELECT concat('ciclope:alarm:',cast(cast(alarm_id AS bigint) AS varchar)) code,
|
|
459
|
+
'wildfire_detection' alert_type,
|
|
460
|
+
CASE
|
|
461
|
+
WHEN lower(coalesce(classification,''))='truealarm' THEN 'high'
|
|
462
|
+
WHEN lower(coalesce(classification,''))='falsealarm' THEN 'low'
|
|
463
|
+
ELSE 'unclassified'
|
|
464
|
+
END severity,
|
|
465
|
+
CASE
|
|
466
|
+
WHEN lower(coalesce(classification,''))='truealarm' THEN 'confirmed'
|
|
467
|
+
WHEN lower(coalesce(classification,''))='falsealarm' THEN 'dismissed'
|
|
468
|
+
ELSE 'active'
|
|
469
|
+
END status,
|
|
470
|
+
coalesce(
|
|
471
|
+
try(from_iso8601_timestamp(cast(observed_at AS varchar))),
|
|
472
|
+
try_cast(observed_at AS timestamp(6) with time zone)
|
|
473
|
+
) issued_at,
|
|
474
|
+
concat('CICLOPE automatic fire detection at tower ',cast(cast(tower_id AS bigint) AS varchar),
|
|
475
|
+
', camera ',cast(cast(camera_id AS bigint) AS varchar),
|
|
476
|
+
'; classification=',coalesce(classification,'None')) message,
|
|
477
|
+
CASE WHEN lower(coalesce(classification,'')) = 'truealarm'
|
|
478
|
+
THEN concat('ciclope:alarm:',cast(cast(alarm_id AS bigint) AS varchar))
|
|
479
|
+
END incident_code
|
|
480
|
+
FROM tenant_00000000_cira_ciclope.current_afd_alarms
|
|
481
|
+
WHERE alarm_id IS NOT NULL
|
|
482
|
+
depends_on: []
|
|
483
|
+
emits_to: backing_dataset
|
|
484
|
+
config:
|
|
485
|
+
output_layer: silver
|
|
486
|
+
---
|
|
487
|
+
apiVersion: frank.platform/v1
|
|
488
|
+
kind: BackingDataset
|
|
489
|
+
metadata:
|
|
490
|
+
name: cira_ciclope_emergency_alerts
|
|
491
|
+
dependsOn:
|
|
492
|
+
- entityTypeId: emergency_incident
|
|
493
|
+
ontologyTenantId: cira
|
|
494
|
+
spec:
|
|
495
|
+
entity_type_id: emergency_alert
|
|
496
|
+
entity_type_name: Emergency Alert
|
|
497
|
+
ontology_tenant_id: cira
|
|
498
|
+
pipeline: cira_ciclope_emergency_alerts
|
|
499
|
+
sync_mode: manual
|
|
500
|
+
primary_key_column: code
|
|
501
|
+
title_key_column: code
|
|
502
|
+
property_mappings:
|
|
503
|
+
- column: code
|
|
504
|
+
property: code
|
|
505
|
+
type: varchar
|
|
506
|
+
is_primary_key: true
|
|
507
|
+
- column: alert_type
|
|
508
|
+
property: alert_type
|
|
509
|
+
type: varchar
|
|
510
|
+
- column: severity
|
|
511
|
+
property: severity
|
|
512
|
+
type: varchar
|
|
513
|
+
- column: status
|
|
514
|
+
property: status
|
|
515
|
+
type: varchar
|
|
516
|
+
- column: issued_at
|
|
517
|
+
property: issued_at
|
|
518
|
+
type: timestamp with time zone
|
|
519
|
+
- column: message
|
|
520
|
+
property: message
|
|
521
|
+
type: varchar
|
|
522
|
+
- column: incident_code
|
|
523
|
+
property: incident
|
|
524
|
+
is_relationship: true
|
|
525
|
+
target_type: emergency_incident
|
|
526
|
+
target_key: code
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
# Complete CIRA Cityfy agenda dataset (issue #589).
|
|
2
|
+
#
|
|
3
|
+
# One REST resource owns pagination and lands one canonical Bronze table. The
|
|
4
|
+
# connector advances ?page=1,2,... until the selected `data` array is empty;
|
|
5
|
+
# there is deliberately no maximum page. Reapply adopts the existing Source
|
|
6
|
+
# and converges the declared `events` stream in place. The five live legacy
|
|
7
|
+
# page streams are explicit disabled migration entries: their runtime state is
|
|
8
|
+
# preserved, but scheduled/direct Source runs cannot execute them.
|
|
9
|
+
#
|
|
10
|
+
# Live verification on 2026-07-22 returned 50/50/50/6 records and then an empty
|
|
11
|
+
# page from the `regiaodeaveiro-eventos` hostname below over valid TLS. The
|
|
12
|
+
# similarly named `regiaodeaveiro.hq1.cityfy.pt` endpoint returns HTTP 404;
|
|
13
|
+
# do not substitute it and do not add a certificate-verification bypass.
|
|
14
|
+
|
|
15
|
+
apiVersion: frank.platform/v1
|
|
16
|
+
kind: Source
|
|
17
|
+
metadata:
|
|
18
|
+
name: cira-cityfy-agenda
|
|
19
|
+
spec:
|
|
20
|
+
name: cira-cityfy-agenda
|
|
21
|
+
description: Complete Cityfy agenda as one connector-paginated events dataset.
|
|
22
|
+
pattern_id: rest_api
|
|
23
|
+
source_config:
|
|
24
|
+
base_url: https://regiaodeaveiro-eventos.hq1.cityfy.pt/api/v1
|
|
25
|
+
resources:
|
|
26
|
+
- /events
|
|
27
|
+
data_selector: data
|
|
28
|
+
paginator: page_number
|
|
29
|
+
page_param: page
|
|
30
|
+
base_page: 1
|
|
31
|
+
pagination_total_path: null
|
|
32
|
+
stop_after_empty_page: true
|
|
33
|
+
primary_keys:
|
|
34
|
+
- id
|
|
35
|
+
schedule_type: manual
|
|
36
|
+
streams:
|
|
37
|
+
- name: events
|
|
38
|
+
sync_mode: full_refresh
|
|
39
|
+
write_disposition: replace
|
|
40
|
+
primary_key_path:
|
|
41
|
+
- id
|
|
42
|
+
is_enabled: true
|
|
43
|
+
dest_table_name: events
|
|
44
|
+
# Migration tombstones for the page-per-stream configuration currently on
|
|
45
|
+
# the adopted live Source. They are not connector resources or Bronze
|
|
46
|
+
# inputs; declaring them disabled makes every Source execution path safe.
|
|
47
|
+
- name: events_page_1
|
|
48
|
+
is_enabled: false
|
|
49
|
+
- name: events_page_2
|
|
50
|
+
is_enabled: false
|
|
51
|
+
- name: events_page_3
|
|
52
|
+
is_enabled: false
|
|
53
|
+
- name: events_page_4
|
|
54
|
+
is_enabled: false
|
|
55
|
+
- name: events_page_5
|
|
56
|
+
is_enabled: false
|
|
57
|
+
---
|
|
58
|
+
apiVersion: frank.platform/v1
|
|
59
|
+
kind: Pipeline
|
|
60
|
+
metadata:
|
|
61
|
+
name: cira_cityfy_events
|
|
62
|
+
spec:
|
|
63
|
+
name: cira_cityfy_events
|
|
64
|
+
description: Complete CIRA Cityfy agenda projection into canonical events.
|
|
65
|
+
source_ids:
|
|
66
|
+
- cira-cityfy-agenda
|
|
67
|
+
schedule_config:
|
|
68
|
+
type: manual
|
|
69
|
+
steps:
|
|
70
|
+
- name: event_rows
|
|
71
|
+
kind: custom_sql
|
|
72
|
+
params:
|
|
73
|
+
sql: |-
|
|
74
|
+
SELECT
|
|
75
|
+
concat('cityfy:', trim(cast(o."id" AS varchar))) AS source_key,
|
|
76
|
+
concat('cityfy:', trim(cast(o."id" AS varchar))) AS external_id,
|
|
77
|
+
nullif(trim(cast(o."title"."value" AS varchar)), '') AS name,
|
|
78
|
+
nullif(trim(cast(o."description"."value" AS varchar)), '') AS description,
|
|
79
|
+
'Cityfy' AS data_provider,
|
|
80
|
+
CASE
|
|
81
|
+
WHEN cardinality(o."location"."value"."coordinates") = 2
|
|
82
|
+
AND element_at(o."location"."value"."coordinates", 1) BETWEEN -180.0 AND 180.0
|
|
83
|
+
AND element_at(o."location"."value"."coordinates", 2) BETWEEN -90.0 AND 90.0
|
|
84
|
+
THEN to_geojson_geometry(ST_Point(
|
|
85
|
+
element_at(o."location"."value"."coordinates", 1),
|
|
86
|
+
element_at(o."location"."value"."coordinates", 2)
|
|
87
|
+
))
|
|
88
|
+
END AS location_geojson,
|
|
89
|
+
json_format(CAST(o."see_also"."value" AS JSON)) AS see_also_json,
|
|
90
|
+
from_iso8601_timestamp(o."date_created"."value") AS date_created,
|
|
91
|
+
from_iso8601_timestamp(o."date_modified"."value") AS date_modified,
|
|
92
|
+
from_iso8601_timestamp(o."date_started"."value") AS start_date,
|
|
93
|
+
from_iso8601_timestamp(o."date_finished"."value") AS end_date,
|
|
94
|
+
nullif(trim(cast(o."image"."value" AS varchar)), '') AS content_url
|
|
95
|
+
FROM tenant_00000000_cira_cityfy_agenda.events o
|
|
96
|
+
WHERE nullif(trim(cast(o."id" AS varchar)), '') IS NOT NULL
|
|
97
|
+
depends_on: []
|
|
98
|
+
emits_to: backing_dataset
|
|
99
|
+
config:
|
|
100
|
+
output_layer: silver
|
|
101
|
+
---
|
|
102
|
+
apiVersion: frank.platform/v1
|
|
103
|
+
kind: BackingDataset
|
|
104
|
+
metadata:
|
|
105
|
+
name: cira_cityfy_events
|
|
106
|
+
spec:
|
|
107
|
+
entity_type_id: event
|
|
108
|
+
entity_type_name: Event
|
|
109
|
+
ontology_tenant_id: cira
|
|
110
|
+
pipeline: cira_cityfy_events
|
|
111
|
+
# Adopt the bounded-canary BackingDataset in place. This field is immutable
|
|
112
|
+
# after creation; preserving on_materialization avoids destructive
|
|
113
|
+
# recreation and is safe because Cityfy has no ontology dependencies.
|
|
114
|
+
sync_mode: on_materialization
|
|
115
|
+
primary_key_column: source_key
|
|
116
|
+
title_key_column: name
|
|
117
|
+
property_mappings:
|
|
118
|
+
- column: source_key
|
|
119
|
+
property: source
|
|
120
|
+
type: string
|
|
121
|
+
is_primary_key: true
|
|
122
|
+
- column: external_id
|
|
123
|
+
property: external_id
|
|
124
|
+
type: string
|
|
125
|
+
- column: name
|
|
126
|
+
property: name
|
|
127
|
+
type: string
|
|
128
|
+
- column: description
|
|
129
|
+
property: description
|
|
130
|
+
type: string
|
|
131
|
+
- column: data_provider
|
|
132
|
+
property: data_provider
|
|
133
|
+
type: string
|
|
134
|
+
- column: location_geojson
|
|
135
|
+
property: location
|
|
136
|
+
type: varchar
|
|
137
|
+
sql_transform_id: json_parse
|
|
138
|
+
- column: see_also_json
|
|
139
|
+
property: see_also
|
|
140
|
+
type: varchar
|
|
141
|
+
sql_transform_id: json_parse
|
|
142
|
+
- column: date_created
|
|
143
|
+
property: date_created
|
|
144
|
+
type: timestamp with time zone
|
|
145
|
+
- column: date_modified
|
|
146
|
+
property: date_modified
|
|
147
|
+
type: timestamp with time zone
|
|
148
|
+
- column: start_date
|
|
149
|
+
property: start_date
|
|
150
|
+
type: timestamp with time zone
|
|
151
|
+
- column: end_date
|
|
152
|
+
property: end_date
|
|
153
|
+
type: timestamp with time zone
|
|
154
|
+
- column: content_url
|
|
155
|
+
property: content_url
|
|
156
|
+
type: string
|