openc3-cosmos-constellation 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/Rakefile +24 -0
- data/microservices/BRIDGE/bridge.py +546 -0
- data/microservices/BRIDGE/constellation_space/__init__.py +0 -0
- data/microservices/BRIDGE/constellation_space/results.py +512 -0
- data/plugin.txt +25 -0
- data/targets/CONSTELLATION/cmd_tlm/tlm.txt +78 -0
- data/targets/CONSTELLATION/target.txt +1 -0
- data/tools/constellationforecast/forecast-view.js +673 -0
- data/tools/constellationforecast/main.js +102 -0
- metadata +57 -0
|
@@ -0,0 +1,512 @@
|
|
|
1
|
+
"""Workflow results, as every connector here reads them: pulling a connection's runs, joining
|
|
2
|
+
a paged run whole, reading its items, and reporting what was applied.
|
|
3
|
+
|
|
4
|
+
`call(method, path, body=None)` is the bridge's own Constellation API call (its base URL, its key,
|
|
5
|
+
its timeouts); `log(message)` its own log line. The COSMOS plugin's gem carries a copy of this
|
|
6
|
+
module (as constellation_space/results.py), which its Rakefile makes at build time. Standard library only,
|
|
7
|
+
and it imports nothing else from this package, so that copy stands alone.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import ipaddress
|
|
11
|
+
import json
|
|
12
|
+
import os
|
|
13
|
+
import urllib.parse
|
|
14
|
+
import urllib.request
|
|
15
|
+
from collections.abc import Callable
|
|
16
|
+
from datetime import datetime, timedelta, timezone
|
|
17
|
+
from typing import Any
|
|
18
|
+
|
|
19
|
+
# The item shape these bridges read; results in another are skipped and logged, never guessed at.
|
|
20
|
+
SCHEMA = "constellation.item/v2"
|
|
21
|
+
# The most records the API sends in one page of a run; it pages a run with more (a few MB each).
|
|
22
|
+
PAGE_RECORDS = 400
|
|
23
|
+
PLATFORM = "https://platform.constellation.space"
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def site(api_url: str | None = None) -> str:
|
|
27
|
+
"""The Constellation site that serves `api_url` (default CONSTELLATION_API_URL), where its
|
|
28
|
+
keys and billing pages are: api.<domain> is platform.<domain>, <env>.api.<domain> is
|
|
29
|
+
<env>-platform.<domain>, and a personal sandbox's s-<id>.api.<domain> is
|
|
30
|
+
platform.s-<id>.<domain>. Any other API is told the public site."""
|
|
31
|
+
host = urllib.parse.urlparse(api_url or os.environ.get("CONSTELLATION_API_URL", "")).hostname
|
|
32
|
+
if host and host.startswith("api."):
|
|
33
|
+
return f"https://platform.{host[4:]}"
|
|
34
|
+
if host and ".api." in host:
|
|
35
|
+
env, domain = host.split(".api.", 1)
|
|
36
|
+
if env.startswith("s-"):
|
|
37
|
+
return f"https://platform.{env}.{domain}"
|
|
38
|
+
return f"https://{env}-platform.{domain}"
|
|
39
|
+
return PLATFORM
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def loopback(url: str) -> bool:
|
|
43
|
+
"""Whether `url` names this machine: localhost or a loopback address."""
|
|
44
|
+
host = urllib.parse.urlparse(url).hostname or ""
|
|
45
|
+
try:
|
|
46
|
+
return host == "localhost" or ipaddress.ip_address(host).is_loopback
|
|
47
|
+
except ValueError:
|
|
48
|
+
return False
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def origin(url: str) -> str:
|
|
52
|
+
"""A URL's scheme and host, lowercase, its port only when it is not the scheme's default, and
|
|
53
|
+
any name for this machine (localhost, 127.0.0.1, ::1) as localhost: two URLs with one origin
|
|
54
|
+
are the same API."""
|
|
55
|
+
u = urllib.parse.urlparse(url.strip())
|
|
56
|
+
scheme, host = u.scheme.lower(), "localhost" if loopback(url.strip()) else u.hostname or ""
|
|
57
|
+
host = f"[{host}]" if ":" in host else host
|
|
58
|
+
port = u.port if u.port not in (None, {"https": 443, "http": 80}.get(scheme)) else None
|
|
59
|
+
return f"{scheme}://{host}" + (f":{port}" if port else "")
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def key_safe(url: str) -> bool:
|
|
63
|
+
"""Whether an API key may go to `url`: over https, or plain http to this machine."""
|
|
64
|
+
scheme = urllib.parse.urlparse(url).scheme.lower()
|
|
65
|
+
return scheme == "https" or (scheme == "http" and loopback(url))
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def key_refusal(url: str) -> str:
|
|
69
|
+
"""Why an API key may not go to `url`, or "" when it may (key_safe)."""
|
|
70
|
+
if key_safe(url):
|
|
71
|
+
return ""
|
|
72
|
+
return f"not sending the API key to {url}: it is not https (plain http only to localhost)"
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def http(
|
|
76
|
+
method: str,
|
|
77
|
+
url: str,
|
|
78
|
+
body: Any = None,
|
|
79
|
+
headers: dict[str, str] | None = None,
|
|
80
|
+
raw: bytes | None = None,
|
|
81
|
+
timeout: float = 60,
|
|
82
|
+
) -> Any:
|
|
83
|
+
"""One HTTP call: `body` sent as JSON (or `raw` as is), the answer's JSON back, or None for an
|
|
84
|
+
answer that is not JSON. An HTTP error is raised as urllib's HTTPError."""
|
|
85
|
+
data = raw if raw is not None else None if body is None else json.dumps(body).encode()
|
|
86
|
+
req = urllib.request.Request(url, data=data, method=method, headers=headers or {})
|
|
87
|
+
if body is not None and raw is None:
|
|
88
|
+
req.add_header("Content-Type", "application/json")
|
|
89
|
+
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
|
90
|
+
payload = resp.read()
|
|
91
|
+
return json.loads(payload) if payload and payload[:1] in b"{[" else None
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def action_state(item: dict[str, Any]) -> str | None:
|
|
95
|
+
"""What an item's action is to a bridge. "advice": suggested, never an instruction.
|
|
96
|
+
"approved": a person approved it in Constellation, an instruction to carry out. "pending":
|
|
97
|
+
held for approval; the action itself is not sent, only action_status. "rejected": decided
|
|
98
|
+
against; the original stands. None: the item has no action. Any status other than approved
|
|
99
|
+
or rejected counts as held, so an item a bridge cannot read is never taken for a plain
|
|
100
|
+
forecast."""
|
|
101
|
+
status = item.get("action_status")
|
|
102
|
+
if status is not None and status != "approved":
|
|
103
|
+
return "rejected" if status == "rejected" else "pending"
|
|
104
|
+
if not item.get("action"):
|
|
105
|
+
return None if status is None else "pending"
|
|
106
|
+
return "approved" if status == "approved" else "advice"
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def action_key(item: dict[str, Any]) -> str:
|
|
110
|
+
"""Names the decision to carry out an action: its decision_id, which a standing approval keeps
|
|
111
|
+
in every run that carries it and a new approval of the same action changes. A result without
|
|
112
|
+
one is named by its content, as Constellation keys an action: the same verb on the same entity
|
|
113
|
+
to the same target."""
|
|
114
|
+
if decision := item.get("decision_id"):
|
|
115
|
+
return str(decision)
|
|
116
|
+
action = item.get("action") or {}
|
|
117
|
+
return f"{item['entity']['id']}|{action.get('verb') or ''}|{action.get('to') or ''}"
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
# How often a bridge checks in again: the platform reads a connection that stopped checking in
|
|
121
|
+
# as stopped. A check-in that failed is tried again sooner.
|
|
122
|
+
CHECKIN_S = 300
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
# How long an acted key is kept once its action is over: past it, no run sends the action again.
|
|
126
|
+
KEEP_ACTED = timedelta(days=7)
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def now() -> datetime:
|
|
130
|
+
return datetime.now(timezone.utc)
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def parse_time(value: Any) -> datetime | None:
|
|
134
|
+
"""An ISO 8601 time, UTC when it names no zone; None when it is not one."""
|
|
135
|
+
try:
|
|
136
|
+
at = datetime.fromisoformat(str(value).replace("Z", "+00:00"))
|
|
137
|
+
except ValueError:
|
|
138
|
+
return None
|
|
139
|
+
return at if at.tzinfo else at.replace(tzinfo=timezone.utc)
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def acted_until(item: dict[str, Any]) -> str:
|
|
143
|
+
"""When an acted key may be forgotten: the latest of the item's time, its end (a pass's end),
|
|
144
|
+
its action's cancel deadline and now, plus KEEP_ACTED, so a key is never dropped as soon as it
|
|
145
|
+
is written. "" (kept for good) when none of the item's is a time."""
|
|
146
|
+
action = item.get("action") or {}
|
|
147
|
+
times = [
|
|
148
|
+
parse_time(v) for v in (item.get("target_at"), item.get("until"), action.get("deadline"))
|
|
149
|
+
]
|
|
150
|
+
known = [t for t in times if t]
|
|
151
|
+
if not known:
|
|
152
|
+
return ""
|
|
153
|
+
return (max(*known, now()) + KEEP_ACTED).astimezone(timezone.utc).isoformat(timespec="seconds")
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def first_acted(items: list[dict[str, Any]], acted: dict[str, str]) -> int:
|
|
157
|
+
"""How many of these items are approved actions still open (before their cancel deadline)
|
|
158
|
+
carried out for the first time. A bridge that enforces or shows approvals on every run, rather
|
|
159
|
+
than acting once, counts each decision (action_key) once: `acted` maps each counted to when
|
|
160
|
+
it may be forgotten (acted_until), kept in the bridge's state, and loses keys long over."""
|
|
161
|
+
_drop_over(acted)
|
|
162
|
+
first = 0
|
|
163
|
+
for item in items:
|
|
164
|
+
key = action_key(item)
|
|
165
|
+
if action_state(item) == "approved" and not past_deadline(item) and key not in acted:
|
|
166
|
+
acted[key] = acted_until(item)
|
|
167
|
+
first += 1
|
|
168
|
+
return first
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def _drop_over(keys: dict[str, str]) -> bool:
|
|
172
|
+
"""Drops the keys whose time to be forgotten has passed; whether any was."""
|
|
173
|
+
cutoff = now()
|
|
174
|
+
over = [k for k, until in keys.items() if (t := parse_time(until)) and t < cutoff]
|
|
175
|
+
for key in over:
|
|
176
|
+
del keys[key]
|
|
177
|
+
return bool(over)
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def past_deadline(item: dict[str, Any]) -> bool:
|
|
181
|
+
"""Whether the item's action is past its cancel deadline: closed, never to be carried out."""
|
|
182
|
+
deadline = parse_time((item.get("action") or {}).get("deadline"))
|
|
183
|
+
return deadline is not None and deadline <= now()
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
class Acted:
|
|
187
|
+
"""The approved actions already carried out, in the file `path`: one line per action_key, a
|
|
188
|
+
tab, and when it may be forgotten (acted_until). Constellation sends an approved action again
|
|
189
|
+
(the run is republished when it is approved, and later runs carry it too), so a bridge checks
|
|
190
|
+
here before acting, and neither a later run nor a restart acts twice. Keys whose action is long
|
|
191
|
+
over are dropped whenever the file is written."""
|
|
192
|
+
|
|
193
|
+
def __init__(self, path: str) -> None:
|
|
194
|
+
self.path = path
|
|
195
|
+
self.keys: dict[str, str] = {}
|
|
196
|
+
try:
|
|
197
|
+
with open(path) as f:
|
|
198
|
+
for line in f.read().splitlines():
|
|
199
|
+
key, _, until = line.partition("\t")
|
|
200
|
+
if key:
|
|
201
|
+
self.keys[key] = until
|
|
202
|
+
except FileNotFoundError:
|
|
203
|
+
pass
|
|
204
|
+
if _drop_over(self.keys):
|
|
205
|
+
self._save()
|
|
206
|
+
|
|
207
|
+
def __contains__(self, item: dict[str, Any]) -> bool:
|
|
208
|
+
return action_key(item) in self.keys
|
|
209
|
+
|
|
210
|
+
def add(self, item: dict[str, Any]) -> None:
|
|
211
|
+
self.keys[action_key(item)] = acted_until(item)
|
|
212
|
+
_drop_over(self.keys)
|
|
213
|
+
self._save()
|
|
214
|
+
|
|
215
|
+
def _save(self) -> None:
|
|
216
|
+
with open(f"{self.path}.tmp", "w") as f:
|
|
217
|
+
f.writelines(f"{key}\t{until}\n" for key, until in self.keys.items())
|
|
218
|
+
os.replace(f"{self.path}.tmp", self.path)
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
# What reading a malformed item raises: a bridge skips, logs and counts the item, never retries it.
|
|
222
|
+
MALFORMED = (KeyError, TypeError, ValueError, AttributeError, IndexError)
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def check(item: Any) -> None:
|
|
226
|
+
"""Raises ValueError unless the item has what every bridge reads: an entity with a string id
|
|
227
|
+
and type, and times (target_at, until when it has one, and its action's deadline)."""
|
|
228
|
+
entity = item.get("entity") if isinstance(item, dict) else None
|
|
229
|
+
if not isinstance(entity, dict) or not all(
|
|
230
|
+
isinstance(entity.get(k), str) for k in ("id", "type")
|
|
231
|
+
):
|
|
232
|
+
raise ValueError("no entity with an id and a type")
|
|
233
|
+
datetime.fromisoformat(item["target_at"])
|
|
234
|
+
if item.get("until") is not None:
|
|
235
|
+
datetime.fromisoformat(item["until"])
|
|
236
|
+
action = item.get("action")
|
|
237
|
+
if action is not None and not isinstance(action, dict):
|
|
238
|
+
raise ValueError("its action is not an object")
|
|
239
|
+
if (action or {}).get("deadline") is not None:
|
|
240
|
+
datetime.fromisoformat(action["deadline"])
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def _detail(body: bytes | str) -> dict[str, Any]:
|
|
244
|
+
"""An API error's detail: FastAPI's {"detail": {...}} or a bare {"error": ...}."""
|
|
245
|
+
try:
|
|
246
|
+
answer = json.loads(body)
|
|
247
|
+
except ValueError:
|
|
248
|
+
answer = None
|
|
249
|
+
detail = answer.get("detail", answer) if isinstance(answer, dict) else None
|
|
250
|
+
return detail if isinstance(detail, dict) else {}
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def well_formed(
|
|
254
|
+
items: list[dict[str, Any]], run_id: Any, log: Callable[[str], None] = print
|
|
255
|
+
) -> list[dict[str, Any]]:
|
|
256
|
+
"""The items a bridge can read (check()); each other one is logged and skipped, so one
|
|
257
|
+
malformed item never holds up its run and the runs after it."""
|
|
258
|
+
good = []
|
|
259
|
+
for n, item in enumerate(items):
|
|
260
|
+
try:
|
|
261
|
+
check(item)
|
|
262
|
+
good.append(item)
|
|
263
|
+
except MALFORMED as exc:
|
|
264
|
+
log(f"skipped item {n} of run {run_id}: it is malformed ({exc!r})")
|
|
265
|
+
return good
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def refusal(
|
|
269
|
+
status: int, body: bytes | str, needs: str = "predictions:read", api_url: str | None = None
|
|
270
|
+
) -> str | None:
|
|
271
|
+
"""Why Constellation refused this bridge, and the fix, for a 401 or 403; None for any other
|
|
272
|
+
status. Neither clears by asking again, so a bridge stops polling on one. `needs` is the
|
|
273
|
+
scopes the bridge's key needs; the fix links the site of `api_url` (site())."""
|
|
274
|
+
if status not in (401, 403):
|
|
275
|
+
return None
|
|
276
|
+
detail = _detail(body)
|
|
277
|
+
fix = f"make a key with {needs} on {site(api_url)}/keys and set it as CONSTELLATION_API_KEY"
|
|
278
|
+
error = detail.get("error")
|
|
279
|
+
if status == 401:
|
|
280
|
+
return (
|
|
281
|
+
"Constellation does not accept CONSTELLATION_API_KEY (HTTP 401; wrong, revoked or"
|
|
282
|
+
f" expired): {fix}"
|
|
283
|
+
)
|
|
284
|
+
if error == "plan_required":
|
|
285
|
+
return (
|
|
286
|
+
f"this account's plan does not include {detail.get('feature')} (HTTP 403"
|
|
287
|
+
f" plan_required); the {detail.get('plan')} plan does. Upgrade on {site(api_url)}/billing"
|
|
288
|
+
)
|
|
289
|
+
if error == "insufficient_scope":
|
|
290
|
+
return f"CONSTELLATION_API_KEY lacks {detail.get('required')} (HTTP 403): {fix}"
|
|
291
|
+
if error == "live_key_required":
|
|
292
|
+
return f"CONSTELLATION_API_KEY is a test key; results need a live one (HTTP 403): {fix}"
|
|
293
|
+
return f"Constellation refused CONSTELLATION_API_KEY (HTTP 403 {error or 'forbidden'}): {fix}"
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
def held_off(
|
|
297
|
+
status: int, body: bytes | str, api_url: str | None = None
|
|
298
|
+
) -> tuple[str, float] | None:
|
|
299
|
+
"""Why Constellation asks this bridge to wait, and how many seconds, for a 402 or 429; None
|
|
300
|
+
for any other status. Asking sooner never helps: a lockout also blocks a corrected key."""
|
|
301
|
+
if status not in (402, 429):
|
|
302
|
+
return None
|
|
303
|
+
detail = _detail(body)
|
|
304
|
+
error = detail.get("error") or ("payment_required" if status == 402 else "too_many_requests")
|
|
305
|
+
retry = detail.get("retry_after_seconds")
|
|
306
|
+
wait = float(retry) if isinstance(retry, (int, float)) and retry > 0 else 0.0
|
|
307
|
+
if status == 402:
|
|
308
|
+
return (
|
|
309
|
+
f"this account's plan does not cover it (HTTP 402 {error}). Upgrade on"
|
|
310
|
+
f" {site(api_url)}/billing",
|
|
311
|
+
max(wait, 3600),
|
|
312
|
+
)
|
|
313
|
+
if error == "auth_lockout":
|
|
314
|
+
return (
|
|
315
|
+
"Constellation locked out this address after repeated refused keys (HTTP 429"
|
|
316
|
+
" auth_lockout); until it lifts it refuses even a corrected key",
|
|
317
|
+
max(wait, 60),
|
|
318
|
+
)
|
|
319
|
+
return f"Constellation asks this bridge to slow down (HTTP 429 {error})", max(wait, 60)
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
def plain(value: dict[str, Any]) -> Any:
|
|
323
|
+
"""An OTLP AnyValue back to plain JSON."""
|
|
324
|
+
if "kvlistValue" in value:
|
|
325
|
+
return {kv["key"]: plain(kv["value"]) for kv in value["kvlistValue"].get("values", [])}
|
|
326
|
+
if "arrayValue" in value:
|
|
327
|
+
return [plain(v) for v in value["arrayValue"].get("values", [])]
|
|
328
|
+
if "intValue" in value:
|
|
329
|
+
return int(value["intValue"])
|
|
330
|
+
for key in ("doubleValue", "boolValue", "stringValue"):
|
|
331
|
+
if key in value:
|
|
332
|
+
return value[key]
|
|
333
|
+
return None
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
def newer_shapes(records: list[dict[str, Any]]) -> set[Any]:
|
|
337
|
+
"""The item shapes among these records that this bridge does not read."""
|
|
338
|
+
return {
|
|
339
|
+
next(
|
|
340
|
+
(
|
|
341
|
+
plain(kv["value"])
|
|
342
|
+
for kv in r.get("attributes", [])
|
|
343
|
+
if kv["key"] == "constellation.schema"
|
|
344
|
+
),
|
|
345
|
+
None,
|
|
346
|
+
)
|
|
347
|
+
for r in records
|
|
348
|
+
} - {SCHEMA}
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
def unpack(
|
|
352
|
+
otlp: dict[str, Any], consumer: str, log: Callable[[str], None] = print
|
|
353
|
+
) -> list[tuple[dict[str, Any], list[dict[str, Any]]]]:
|
|
354
|
+
"""A results entry's parts for this connection: each resourceLogs (one per feed output of the
|
|
355
|
+
run) whose constellation.connection is `consumer`, as its resource attributes and its items.
|
|
356
|
+
Parts for other connections are left alone; a part in an item shape this bridge does not read
|
|
357
|
+
is skipped and logged."""
|
|
358
|
+
parts = []
|
|
359
|
+
for logs in otlp.get("resourceLogs", []):
|
|
360
|
+
resource = {
|
|
361
|
+
kv["key"]: plain(kv["value"]) for kv in logs.get("resource", {}).get("attributes", [])
|
|
362
|
+
}
|
|
363
|
+
if resource.get("constellation.connection") != consumer:
|
|
364
|
+
continue
|
|
365
|
+
records = [r for scope in logs.get("scopeLogs", []) for r in scope.get("logRecords", [])]
|
|
366
|
+
if newer := newer_shapes(records):
|
|
367
|
+
log(
|
|
368
|
+
f"skipped run {resource.get('constellation.run_id')}: its items are "
|
|
369
|
+
f"{', '.join(map(str, newer))} and this bridge reads {SCHEMA}"
|
|
370
|
+
)
|
|
371
|
+
continue
|
|
372
|
+
parts.append((resource, [plain(r["body"]) for r in records]))
|
|
373
|
+
return parts
|
|
374
|
+
|
|
375
|
+
|
|
376
|
+
def expired(page: dict[str, Any], cursor: str) -> str | None:
|
|
377
|
+
"""Why runs were lost when the feed no longer holds `cursor` (it sorts before the page's
|
|
378
|
+
oldest entry: runs after it expired unread); None when it still does."""
|
|
379
|
+
oldest = page.get("oldest")
|
|
380
|
+
if not cursor or not oldest or cursor >= oldest:
|
|
381
|
+
return None
|
|
382
|
+
return (
|
|
383
|
+
f"the feed no longer holds cursor {cursor}: any runs after it and before the oldest it"
|
|
384
|
+
f" holds, {oldest}, expired unread; reading on from there"
|
|
385
|
+
)
|
|
386
|
+
|
|
387
|
+
|
|
388
|
+
def results_page(call: Callable[..., Any], consumer: str, after: str) -> dict[str, Any]:
|
|
389
|
+
"""The next run after cursor `after`, its first page only (whole() joins the rest): one run
|
|
390
|
+
per page, as a run's results with their ledgers can be several MB."""
|
|
391
|
+
query = {"after": after, "limit": 1, "consumer": consumer, "page_records": PAGE_RECORDS}
|
|
392
|
+
return call("GET", f"/predictions/feed?{urllib.parse.urlencode(query)}")
|
|
393
|
+
|
|
394
|
+
|
|
395
|
+
def rest_of_run(call: Callable[..., Any], consumer: str, cursor: str, page: int) -> dict[str, Any]:
|
|
396
|
+
"""Page `page` of the run at `cursor`."""
|
|
397
|
+
query = {"cursor": cursor, "page": page, "page_records": PAGE_RECORDS, "consumer": consumer}
|
|
398
|
+
return call("GET", f"/predictions/feed?{urllib.parse.urlencode(query)}")
|
|
399
|
+
|
|
400
|
+
|
|
401
|
+
def whole(entry: dict[str, Any], fetch: Callable[[str, int], dict[str, Any]]) -> dict[str, Any]:
|
|
402
|
+
"""`entry` with every page of its run joined in: each page's records added to their own
|
|
403
|
+
resource, in order. `fetch(cursor, page)` reads one page. A run is applied whole or not at
|
|
404
|
+
all, never a page of it."""
|
|
405
|
+
entry = {
|
|
406
|
+
**entry,
|
|
407
|
+
"otlp": {**entry["otlp"], "resourceLogs": list(entry["otlp"]["resourceLogs"])},
|
|
408
|
+
}
|
|
409
|
+
while entry.get("next_page"):
|
|
410
|
+
more = fetch(entry["cursor"], entry["next_page"])
|
|
411
|
+
by_resource = {
|
|
412
|
+
json.dumps(r["resource"], sort_keys=True): r for r in entry["otlp"]["resourceLogs"]
|
|
413
|
+
}
|
|
414
|
+
for logs in more["otlp"]["resourceLogs"]:
|
|
415
|
+
key = json.dumps(logs["resource"], sort_keys=True)
|
|
416
|
+
if key in by_resource:
|
|
417
|
+
scope = by_resource[key]["scopeLogs"][0]
|
|
418
|
+
scope["logRecords"] = scope["logRecords"] + logs["scopeLogs"][0]["logRecords"]
|
|
419
|
+
else:
|
|
420
|
+
entry["otlp"]["resourceLogs"].append(logs)
|
|
421
|
+
by_resource[key] = logs
|
|
422
|
+
entry["next_page"] = more.get("next_page")
|
|
423
|
+
entry.pop("next_page", None)
|
|
424
|
+
return entry
|
|
425
|
+
|
|
426
|
+
|
|
427
|
+
def start_cursor(call: Callable[..., Any], consumer: str) -> str:
|
|
428
|
+
"""Where a first start reads from: just before the newest run with something for this
|
|
429
|
+
connection, so it applies that run and none of the older ones. The API answers it without
|
|
430
|
+
reading a run (from=latest, no entries)."""
|
|
431
|
+
query = {"from": "latest", "after": "", "limit": 20, "consumer": consumer, "page_records": 1}
|
|
432
|
+
page = call("GET", f"/predictions/feed?{urllib.parse.urlencode(query)}")
|
|
433
|
+
if page["entries"]:
|
|
434
|
+
raise RuntimeError(
|
|
435
|
+
"the API ignored GET /predictions/feed?from=latest, so a first start cannot find the"
|
|
436
|
+
" newest run: this bridge needs an API serving contract 1.5.0 or later"
|
|
437
|
+
)
|
|
438
|
+
return page["next"]
|
|
439
|
+
|
|
440
|
+
|
|
441
|
+
def report(
|
|
442
|
+
call: Callable[..., Any], consumer: str, cursor: str, applied: dict[str, int], target: str
|
|
443
|
+
) -> None:
|
|
444
|
+
"""Tells Constellation what the run at `cursor` became in the system, shown in Constellation;
|
|
445
|
+
raises when Constellation does not take it."""
|
|
446
|
+
call(
|
|
447
|
+
"POST",
|
|
448
|
+
"/predictions/feed/deliveries",
|
|
449
|
+
{
|
|
450
|
+
"consumer": consumer,
|
|
451
|
+
"run_id": cursor.split("#", 1)[1],
|
|
452
|
+
"cursor": cursor,
|
|
453
|
+
"applied": applied,
|
|
454
|
+
"target": target,
|
|
455
|
+
},
|
|
456
|
+
)
|
|
457
|
+
|
|
458
|
+
|
|
459
|
+
def kind(items: list[dict[str, Any]]) -> str:
|
|
460
|
+
"""What a run's items are, by what they carry, named as the template that makes them: the
|
|
461
|
+
Forecast page picks its view by it. Read only when the run does not name its template (the
|
|
462
|
+
contract does not require it)."""
|
|
463
|
+
explained = {k for i in items for k in i.get("explain") or {}}
|
|
464
|
+
if any(i["entity"]["type"] == "antenna" for i in items):
|
|
465
|
+
return "antenna-capacity"
|
|
466
|
+
if any((i.get("action") or {}).get("verb") == "swap" for i in items):
|
|
467
|
+
return "rain-fade-swap"
|
|
468
|
+
if any(i.get("series") for i in items):
|
|
469
|
+
return "modcod-plan"
|
|
470
|
+
for column, name in (("ledger", "audit-ledger"), ("rfi_db", "rfi-watch")):
|
|
471
|
+
if column in explained:
|
|
472
|
+
return name
|
|
473
|
+
return "pass-yield"
|
|
474
|
+
|
|
475
|
+
|
|
476
|
+
def made_at(otlp: dict[str, Any]) -> str | None:
|
|
477
|
+
"""When a run was made: the API stamps each of its records with the time it produced them."""
|
|
478
|
+
times = [
|
|
479
|
+
int(r["timeUnixNano"])
|
|
480
|
+
for logs in otlp.get("resourceLogs", [])
|
|
481
|
+
for scope in logs.get("scopeLogs", [])
|
|
482
|
+
for r in scope.get("logRecords", [])
|
|
483
|
+
if r.get("timeUnixNano")
|
|
484
|
+
]
|
|
485
|
+
if not times:
|
|
486
|
+
return None
|
|
487
|
+
return datetime.fromtimestamp(min(times) / 1e9, timezone.utc).isoformat().replace("+00:00", "Z")
|
|
488
|
+
|
|
489
|
+
|
|
490
|
+
def max_elevation(item: dict[str, Any]) -> float | None:
|
|
491
|
+
"""A pass's max elevation: the ledger entry the orbit wrote, under whichever node wrote it."""
|
|
492
|
+
for entry in (item.get("explain") or {}).values():
|
|
493
|
+
if isinstance(entry, dict) and entry.get("source") == "orbit" and entry.get("unit") == "deg":
|
|
494
|
+
return entry.get("value")
|
|
495
|
+
return None
|
|
496
|
+
|
|
497
|
+
|
|
498
|
+
QUANTILES = ("p10", "p50", "p90")
|
|
499
|
+
|
|
500
|
+
|
|
501
|
+
def expected_gb(item: dict[str, Any]) -> dict[str, Any] | None:
|
|
502
|
+
"""A pass's expected GB, {p10, p50, p90}, every one a number: the ledger's expected_gb, else
|
|
503
|
+
the item's value (the output column), as a copied workflow may have renamed the ledger node.
|
|
504
|
+
None when neither has all three."""
|
|
505
|
+
ledger = ((item.get("explain") or {}).get("expected_gb") or {}).get("value")
|
|
506
|
+
for gb in (ledger, item.get("value")):
|
|
507
|
+
if isinstance(gb, dict) and all(
|
|
508
|
+
isinstance(gb.get(q), int | float) and not isinstance(gb.get(q), bool)
|
|
509
|
+
for q in QUANTILES
|
|
510
|
+
):
|
|
511
|
+
return gb
|
|
512
|
+
return None
|
data/plugin.txt
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# LANGUAGE python
|
|
2
|
+
# Constellation workflow forecasts in COSMOS: a CONSTELLATION target whose packets the bridge
|
|
3
|
+
# microservice fills from the Constellation feed. Everything here is a forecast, not telemetry
|
|
4
|
+
# measured from a spacecraft or a ground station.
|
|
5
|
+
VARIABLE constellation_api_url https://api.constellation.space
|
|
6
|
+
VARIABLE_DESCRIPTION "Constellation API base URL"
|
|
7
|
+
VARIABLE consumer cosmos-bridge
|
|
8
|
+
VARIABLE_DESCRIPTION "This connection's name: workflows pick it as an output"
|
|
9
|
+
|
|
10
|
+
TARGET CONSTELLATION CONSTELLATION
|
|
11
|
+
|
|
12
|
+
# The Constellation Forecast view, reading what the bridge writes to COSMOS's config bucket.
|
|
13
|
+
TOOL constellationforecast "Constellation Forecast"
|
|
14
|
+
INLINE_URL main.js
|
|
15
|
+
ICON mdi-chart-timeline-variant
|
|
16
|
+
DISABLE_ERB
|
|
17
|
+
|
|
18
|
+
MICROSERVICE BRIDGE constellation-bridge
|
|
19
|
+
CMD python bridge.py
|
|
20
|
+
TARGET_NAME CONSTELLATION
|
|
21
|
+
ENV CONSTELLATION_API_URL <%= constellation_api_url %>
|
|
22
|
+
ENV CONSUMER <%= consumer %>
|
|
23
|
+
# Admin > Secrets: CONSTELLATION_API_KEY (a key with predictions:read). The bridge reads it there
|
|
24
|
+
# at each poll, so it can be added or changed after installing.
|
|
25
|
+
SECRET ENV CONSTELLATION_API_KEY CONSTELLATION_API_KEY
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
TELEMETRY CONSTELLATION NEXT_PASS BIG_ENDIAN "Forecast: the next predicted pass, not a scheduled contact"
|
|
2
|
+
ALLOW_SHORT
|
|
3
|
+
APPEND_ITEM SATELLITE 512 STRING "Satellite id"
|
|
4
|
+
APPEND_ITEM ANTENNA 512 STRING "Antenna id"
|
|
5
|
+
APPEND_ITEM AOS 256 STRING "Predicted acquisition of signal, UTC"
|
|
6
|
+
APPEND_ITEM LOS 256 STRING "Predicted loss of signal, UTC"
|
|
7
|
+
APPEND_ITEM EXPECTED_GB_P10 32 FLOAT "Expected GB, wet tail (p10)"
|
|
8
|
+
UNITS Gigabytes GB
|
|
9
|
+
APPEND_ITEM EXPECTED_GB_P50 32 FLOAT "Expected GB, median (p50)"
|
|
10
|
+
UNITS Gigabytes GB
|
|
11
|
+
APPEND_ITEM EXPECTED_GB_P90 32 FLOAT "Expected GB, best case (p90)"
|
|
12
|
+
UNITS Gigabytes GB
|
|
13
|
+
APPEND_ITEM MAX_ELEVATION 32 FLOAT "Peak elevation"
|
|
14
|
+
UNITS Degrees deg
|
|
15
|
+
|
|
16
|
+
TELEMETRY CONSTELLATION NEXT_24H BIG_ENDIAN "Forecast: every predicted pass in the next 24 hours, summed"
|
|
17
|
+
ALLOW_SHORT
|
|
18
|
+
APPEND_ITEM PASSES 32 UINT "Predicted passes"
|
|
19
|
+
APPEND_ITEM EXPECTED_GB_P10 32 FLOAT "Expected GB, wet tail (p10)"
|
|
20
|
+
UNITS Gigabytes GB
|
|
21
|
+
APPEND_ITEM EXPECTED_GB_P50 32 FLOAT "Expected GB, median (p50)"
|
|
22
|
+
UNITS Gigabytes GB
|
|
23
|
+
|
|
24
|
+
TELEMETRY CONSTELLATION SWAP BIG_ENDIAN "Forecast: the soonest open rain-fade swap, advised or approved"
|
|
25
|
+
ALLOW_SHORT
|
|
26
|
+
APPEND_ITEM OPEN 32 UINT "Rain-fade swaps open: advised or approved, deadline ahead"
|
|
27
|
+
LIMITS DEFAULT 1 ENABLED -1 -1 0.5 100000
|
|
28
|
+
APPEND_ITEM SATELLITE 512 STRING "Satellite id"
|
|
29
|
+
APPEND_ITEM ANTENNA 512 STRING "Antenna at risk"
|
|
30
|
+
APPEND_ITEM AOS 256 STRING "Predicted AOS of the pass at risk, UTC"
|
|
31
|
+
APPEND_ITEM KEEPS_GB_P10 32 FLOAT "GB the pass keeps if it rains (p10)"
|
|
32
|
+
UNITS Gigabytes GB
|
|
33
|
+
APPEND_ITEM EXPECTED_GB_P50 32 FLOAT "GB the pass delivers dry (p50)"
|
|
34
|
+
UNITS Gigabytes GB
|
|
35
|
+
APPEND_ITEM BACKUP_ANTENNA 512 STRING "Antenna to book instead, or NONE"
|
|
36
|
+
APPEND_ITEM BACKUP_AOS 256 STRING "Backup pass AOS, UTC"
|
|
37
|
+
APPEND_ITEM DEADLINE 256 STRING "Cancel the original by, UTC"
|
|
38
|
+
APPEND_ITEM MINUTES_TO_DEADLINE 32 FLOAT "Minutes left to decide (99999 when no swap is open)"
|
|
39
|
+
UNITS Minutes min
|
|
40
|
+
LIMITS DEFAULT 1 ENABLED 15 30 100000 100000
|
|
41
|
+
APPEND_ITEM STATUS 128 STRING "The swap shown: ADVISED, APPROVED (approved in Constellation: carry it out) or NONE"
|
|
42
|
+
APPEND_ITEM APPROVED 32 UINT "Open swaps approved in Constellation"
|
|
43
|
+
APPEND_ITEM PENDING 32 UINT "Swaps held for approval in Constellation"
|
|
44
|
+
|
|
45
|
+
TELEMETRY CONSTELLATION CAPACITY BIG_ENDIAN "Forecast: one day's capacity across the antennas, and the tightest"
|
|
46
|
+
ALLOW_SHORT
|
|
47
|
+
APPEND_ITEM DAY 128 STRING "Day, UTC"
|
|
48
|
+
APPEND_ITEM ANTENNAS 32 UINT "Antennas forecast"
|
|
49
|
+
APPEND_ITEM PASSES 32 UINT "Predicted passes"
|
|
50
|
+
APPEND_ITEM SELLABLE_MIN 32 FLOAT "Sellable contact minutes"
|
|
51
|
+
UNITS Minutes min
|
|
52
|
+
APPEND_ITEM GB_P10 32 FLOAT "Expected GB, wet tail (p10)"
|
|
53
|
+
UNITS Gigabytes GB
|
|
54
|
+
APPEND_ITEM GB_P50 32 FLOAT "Expected GB, median (p50)"
|
|
55
|
+
UNITS Gigabytes GB
|
|
56
|
+
APPEND_ITEM TIGHTEST_ANTENNA 512 STRING "Antenna with the least expected GB that day"
|
|
57
|
+
APPEND_ITEM TIGHTEST_GB_P50 32 FLOAT "Its expected GB (p50)"
|
|
58
|
+
UNITS Gigabytes GB
|
|
59
|
+
|
|
60
|
+
TELEMETRY CONSTELLATION MODCOD BIG_ENDIAN "Forecast: the next pass's MODCOD plan"
|
|
61
|
+
ALLOW_SHORT
|
|
62
|
+
APPEND_ITEM SATELLITE 512 STRING "Satellite id"
|
|
63
|
+
APPEND_ITEM ANTENNA 512 STRING "Antenna id"
|
|
64
|
+
APPEND_ITEM AOS 256 STRING "Predicted AOS, UTC"
|
|
65
|
+
APPEND_ITEM CHANGES 32 UINT "MODCOD changes over the pass"
|
|
66
|
+
APPEND_ITEM FIRST 256 STRING "MODCOD at AOS"
|
|
67
|
+
APPEND_ITEM LAST 256 STRING "MODCOD at LOS"
|
|
68
|
+
APPEND_ITEM MIN_SINR_DB 32 FLOAT "Lowest planned SINR over the pass"
|
|
69
|
+
UNITS Decibels dB
|
|
70
|
+
APPEND_ITEM PLANNED_PASSES 32 UINT "Passes with a plan still to fly"
|
|
71
|
+
|
|
72
|
+
TELEMETRY CONSTELLATION RUN BIG_ENDIAN "The workflow run these forecasts came from"
|
|
73
|
+
ALLOW_SHORT
|
|
74
|
+
APPEND_ITEM WORKFLOW 512 STRING "Workflow"
|
|
75
|
+
APPEND_ITEM TEMPLATE 256 STRING "Template"
|
|
76
|
+
APPEND_ITEM WORKFLOW_VERSION 32 UINT "Workflow version"
|
|
77
|
+
APPEND_ITEM RUN_ID 512 STRING "Run id"
|
|
78
|
+
APPEND_ITEM RESULTS 32 UINT "Results in the run"
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
LANGUAGE python
|