@withgateway/sdk 0.3.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -6
- package/dist/cli.js +4372 -2265
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +4362 -2256
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.d.mts +1 -1
- package/dist/files/index.d.ts +1 -1
- package/dist/files/index.js +259 -58
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +249 -50
- package/dist/files/index.mjs.map +1 -1
- package/dist/hooks/index.d.mts +1 -1
- package/dist/hooks/index.d.ts +1 -1
- package/dist/hooks/index.js +124 -11
- package/dist/hooks/index.js.map +1 -1
- package/dist/hooks/index.mjs +124 -11
- package/dist/hooks/index.mjs.map +1 -1
- package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
- package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
- package/dist/{index-CBqbB_0J.d.ts → index-DUHmS74p.d.ts} +1 -1
- package/dist/{index-D-AMnAMa.d.mts → index-mJhG3vZU.d.mts} +1 -1
- package/dist/index.d.mts +5 -5
- package/dist/index.d.ts +5 -5
- package/dist/index.js +2240 -789
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +2150 -739
- package/dist/index.mjs.map +1 -1
- package/dist/{media-CbcAfQvd.d.mts → media-BIApiPzS.d.mts} +21 -2
- package/dist/{media-CbcAfQvd.d.ts → media-BIApiPzS.d.ts} +21 -2
- package/dist/traces/index.d.mts +1 -1
- package/dist/traces/index.d.ts +1 -1
- package/dist/traces/index.js +253 -52
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +243 -44
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.d.mts +2 -2
- package/dist/tracing/index.d.ts +2 -2
- package/dist/tracing/index.js +75 -13
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +75 -13
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +893 -482
- package/dist/worlds/index.d.ts +893 -482
- package/dist/worlds/index.js +1968 -669
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +1960 -662
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
- package/runtime/gatewaysdk/environment/_bundle.py +2 -27
- package/runtime/gatewaysdk/environment/_materialize.py +48 -88
- package/runtime/gatewaysdk/environment/_world.py +155 -237
- package/runtime/gatewaysdk/environment/backing.py +133 -0
- package/runtime/gatewaysdk/environment/core.py +2 -17
- package/runtime/gatewaysdk/environment/legacy_sqlite.py +288 -0
- package/runtime/gatewaysdk/environment/reward_keys.py +25 -0
- package/runtime/gatewaysdk/environment/runtime.py +8 -13
- package/runtime/gatewaysdk/environment/schema/__init__.py +4 -4
- package/runtime/gatewaysdk/environment/schema/admin_routes.py +86 -0
- package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
- package/runtime/gatewaysdk/environment/schema/api.py +347 -892
- package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
- package/runtime/gatewaysdk/environment/schema/batch.py +44 -39
- package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +17 -5
- package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
- package/runtime/gatewaysdk/environment/schema/call_capture.py +309 -0
- package/runtime/gatewaysdk/environment/schema/call_log.py +503 -0
- package/runtime/gatewaysdk/environment/schema/call_redaction.py +526 -0
- package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
- package/runtime/gatewaysdk/environment/schema/compiler.py +334 -158
- package/runtime/gatewaysdk/environment/schema/conform.py +14 -39
- package/runtime/gatewaysdk/environment/schema/connector.py +113 -211
- package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
- package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +1 -0
- package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
- package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
- package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
- package/runtime/gatewaysdk/environment/schema/data.py +60 -216
- package/runtime/gatewaysdk/environment/schema/database/__init__.py +24 -0
- package/runtime/gatewaysdk/environment/schema/database/audit.py +121 -0
- package/runtime/gatewaysdk/environment/schema/database/base.py +936 -0
- package/runtime/gatewaysdk/environment/schema/database/budget.py +33 -0
- package/runtime/gatewaysdk/environment/schema/database/connections.py +101 -0
- package/runtime/gatewaysdk/environment/schema/database/engines.py +8 -0
- package/runtime/gatewaysdk/environment/schema/database/guard.py +215 -0
- package/runtime/gatewaysdk/environment/schema/database/ingest.py +154 -0
- package/runtime/gatewaysdk/environment/schema/database/matching.py +58 -0
- package/runtime/gatewaysdk/environment/schema/database/provenance.py +30 -0
- package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
- package/runtime/gatewaysdk/environment/schema/database/rows.py +53 -0
- package/runtime/gatewaysdk/environment/schema/database/runtime_rows.py +39 -0
- package/runtime/gatewaysdk/environment/schema/database/sqlite.py +393 -0
- package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
- package/runtime/gatewaysdk/environment/schema/domain_migration.py +236 -0
- package/runtime/gatewaysdk/environment/schema/door.py +899 -0
- package/runtime/gatewaysdk/environment/schema/envelope.py +227 -0
- package/runtime/gatewaysdk/environment/schema/extract.py +6 -13
- package/runtime/gatewaysdk/environment/schema/grading.py +90 -103
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +163 -12
- package/runtime/gatewaysdk/environment/schema/host.py +237 -1795
- package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
- package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
- package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
- package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
- package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
- package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
- package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
- package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
- package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
- package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
- package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
- package/runtime/gatewaysdk/environment/schema/journal.py +437 -0
- package/runtime/gatewaysdk/environment/schema/lanes.py +53 -0
- package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
- package/runtime/gatewaysdk/environment/schema/local.py +58 -170
- package/runtime/gatewaysdk/environment/schema/mapping.py +5 -25
- package/runtime/gatewaysdk/environment/schema/openapi_import.py +112 -73
- package/runtime/gatewaysdk/environment/schema/openapi_paging.py +147 -0
- package/runtime/gatewaysdk/environment/schema/openapi_payloads.py +288 -0
- package/runtime/gatewaysdk/environment/schema/openapi_routes.py +97 -102
- package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +113 -30
- package/runtime/gatewaysdk/environment/schema/platform.py +137 -187
- package/runtime/gatewaysdk/environment/schema/platform_import.py +2 -3
- package/runtime/gatewaysdk/environment/schema/platform_query.py +61 -5
- package/runtime/gatewaysdk/environment/schema/primary_key.py +61 -0
- package/runtime/gatewaysdk/environment/schema/proof.py +19 -12
- package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
- package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
- package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
- package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +178 -308
- package/runtime/gatewaysdk/environment/schema/served_names.py +58 -0
- package/runtime/gatewaysdk/environment/schema/session_files.py +275 -0
- package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
- package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +10 -11
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +49 -16
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +39 -9
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +27 -27
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +33 -33
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +13 -7
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +49 -1
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/openapi-import.md +50 -21
- package/runtime/gatewaysdk/environment/schema/slack.py +4 -8
- package/runtime/gatewaysdk/environment/schema/snapshot.py +516 -140
- package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
- package/runtime/gatewaysdk/environment/schema/store_contract.py +8 -14
- package/runtime/gatewaysdk/environment/schema/tasks.py +267 -276
- package/runtime/gatewaysdk/environment/schema/tools_world.py +146 -242
- package/runtime/gatewaysdk/environment/schema/ui.py +24 -5
- package/runtime/gatewaysdk/environment/schema/ui_static/app.js +100 -130
- package/runtime/gatewaysdk/environment/schema/ui_static/index.html +1 -8
- package/runtime/gatewaysdk/environment/schema/validation.py +235 -41
- package/runtime/gatewaysdk/environment/schema/verbs.py +32 -36
- package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
- package/runtime/gatewaysdk/environment/schema/workers.py +345 -96
- package/runtime/gatewaysdk/environment/schema/world_host.py +953 -0
- package/runtime/gatewaysdk/environment/schema/world_sql.py +81 -0
- package/runtime/gatewaysdk/environment/schema/world_tests.py +80 -50
- package/runtime/gatewaysdk/environment/score.py +57 -0
- package/runtime/gatewaysdk/environment/secret_values.py +36 -0
- package/runtime/gatewaysdk/environment/world.py +43 -4
- package/runtime/gatewaysdk/environment/world_session_contract.py +46 -5
- package/skills/agents/connector-template-author.md +5 -1
- package/skills/benchmarks-getting-started/SKILL.md +10 -11
- package/skills/connector-schema-authoring/SKILL.md +49 -16
- package/skills/connector-schema-authoring/connector-worlds.md +142 -11
- package/skills/connector-schema-authoring/world-schema.md +103 -51
- package/skills/world-data-ingestion/ROWS.md +39 -9
- package/skills/world-data-ingestion/SKILL.md +27 -27
- package/skills/worlds-getting-started/SKILL.md +33 -33
- package/skills/worlds-getting-started/references/cli-mcp-map.md +13 -7
- package/skills/worlds-getting-started/references/gotchas.md +49 -1
- package/skills/worlds-getting-started/references/journal-checks.md +182 -0
- package/skills/worlds-getting-started/references/openapi-import.md +50 -21
- package/runtime/gatewaysdk/environment/schema/store.py +0 -1013
|
@@ -0,0 +1,577 @@
|
|
|
1
|
+
"""The supervisor's half of its version workers (the worker's half is :mod:`worker`).
|
|
2
|
+
|
|
3
|
+
A session of a world that runs its own code lives in its version's worker; the
|
|
4
|
+
supervisor keeps the session's record and forwards its requests there. What a
|
|
5
|
+
forwarded request answers when the worker does not answer it:
|
|
6
|
+
|
|
7
|
+
* past its time, while the worker still answers its health check: that request's
|
|
8
|
+
``call_timeout`` (504); the worker and every session stay, and the session's next
|
|
9
|
+
requests answer ``call_running`` (503) while that call still holds its turn;
|
|
10
|
+
* dropped, while the worker still answers: ``busy`` (503, retry);
|
|
11
|
+
* the worker died or stopped answering its health check: ``worker_exited`` (502), at
|
|
12
|
+
once. The worker is replaced behind that answer and each of its sessions reopens on
|
|
13
|
+
the new one from what it had committed: the session's directory (its store, its
|
|
14
|
+
dependency worlds' and its call journal) is set aside before the new worker starts,
|
|
15
|
+
and the new worker continues the session from it
|
|
16
|
+
(:class:`session_files.RecoveredSession`); the session's next requests wait for
|
|
17
|
+
that. A worker that died idle is replaced the same way by the next request or the
|
|
18
|
+
lease sweeper, which also recycles any worker holding a turn past ``STUCK_TURN_S``,
|
|
19
|
+
its session still open here or not. What a worker answers about its turns is read
|
|
20
|
+
as untrusted: it runs tenant code.
|
|
21
|
+
|
|
22
|
+
Recovery never raises: a session that cannot be reopened leaves the registry (the
|
|
23
|
+
platform reads it as lost and reopens it from its last export), never left on a dead
|
|
24
|
+
worker; one the new worker may still have opened is deleted there before its id opens again.
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
from __future__ import annotations
|
|
28
|
+
|
|
29
|
+
import json
|
|
30
|
+
import math
|
|
31
|
+
import os
|
|
32
|
+
import shutil
|
|
33
|
+
import threading
|
|
34
|
+
import time
|
|
35
|
+
import urllib.parse
|
|
36
|
+
from collections.abc import Callable
|
|
37
|
+
from pathlib import Path
|
|
38
|
+
from typing import TYPE_CHECKING, Any, Protocol
|
|
39
|
+
|
|
40
|
+
from .admission import BUSY, RETRY_AFTER_S
|
|
41
|
+
from .call_log import CallJournal
|
|
42
|
+
from .handler_errors import log_line
|
|
43
|
+
from .lanes import Lanes
|
|
44
|
+
from .request_ledger import DRAIN_ANSWER_BYTES, LEFT_JOURNAL_BYTES
|
|
45
|
+
from .session_files import LEDGER_JOURNAL, RECOVERED_DIR, ExportFile, clear, real_dir
|
|
46
|
+
from .validation import ContractError, fail
|
|
47
|
+
from .workers import HEALTH_TIMEOUT_S, Worker, WorkerCapacity, WorkerGone, WorkerPool, WorkerTimeout, WorkerUnreached
|
|
48
|
+
|
|
49
|
+
if TYPE_CHECKING:
|
|
50
|
+
from .host_session import Session
|
|
51
|
+
|
|
52
|
+
#: The platform treats exactly this code (with a 404) as "the host lost the session".
|
|
53
|
+
UNKNOWN_SESSION = "unknown_session"
|
|
54
|
+
CALL_TIMEOUT = "call_timeout"
|
|
55
|
+
CALL_RUNNING = "call_running"
|
|
56
|
+
WORKER_EXITED = "worker_exited"
|
|
57
|
+
MOVING = "moving"
|
|
58
|
+
LOST = "lost"
|
|
59
|
+
#: A handoff export of a session this host is not moving: only its holder hands a session off.
|
|
60
|
+
NOT_MOVING = "not_moving"
|
|
61
|
+
#: A forwarded call's time on the worker: with a queue wait of up to ``admission.QUEUE_WAIT_S`` the host
|
|
62
|
+
#: answers before the platform's 60 s control timeout, so a slow handler is named, not read as a dead host.
|
|
63
|
+
CALL_TIMEOUT_S = 45.0
|
|
64
|
+
OPEN_TIMEOUT_S = 60.0
|
|
65
|
+
#: Under the platform's 330 s transfer timeout: an export copies a whole store.
|
|
66
|
+
EXPORT_TIMEOUT_S = 300.0
|
|
67
|
+
#: A turn held this long serves no request the supervisor still waits on: its worker is recycled.
|
|
68
|
+
STUCK_TURN_S = 3 * CALL_TIMEOUT_S
|
|
69
|
+
RUNNING_RETRY_AFTER_S = 5
|
|
70
|
+
JSON = {"content-type": "application/json"}
|
|
71
|
+
#: An open refused ``draining`` or ``at_capacity`` (503) goes to another replica; any other 503, ``busy``
|
|
72
|
+
#: for an id still closing included, sends the platform back later; 404 is the one answer it reads as a loss.
|
|
73
|
+
REFUSALS = {
|
|
74
|
+
"draining": 503,
|
|
75
|
+
"at_capacity": 503,
|
|
76
|
+
BUSY: 503,
|
|
77
|
+
CALL_RUNNING: 503,
|
|
78
|
+
MOVING: 503,
|
|
79
|
+
NOT_MOVING: 409,
|
|
80
|
+
UNKNOWN_SESSION: 404,
|
|
81
|
+
CALL_TIMEOUT: 504,
|
|
82
|
+
WORKER_EXITED: 502,
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def refusal(error: ContractError) -> tuple[int, dict, dict[str, str]]:
|
|
87
|
+
"""A refused session request as the host answers it: status, body and headers."""
|
|
88
|
+
detail = error.detail
|
|
89
|
+
code = str(detail.get("code", "host"))
|
|
90
|
+
retry = detail.get("retryAfter")
|
|
91
|
+
body = {"error": code, "detail": str(error), "message": str(detail.get("message", ""))}
|
|
92
|
+
if retry is None:
|
|
93
|
+
return REFUSALS.get(code, 400), body, {}
|
|
94
|
+
return REFUSALS.get(code, 400), {**body, "retryAfter": retry}, {"Retry-After": str(retry)}
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _age(value: object) -> float | None:
|
|
98
|
+
"""A turn's age as a worker reported it, or None unless it is a finite, non-negative number."""
|
|
99
|
+
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
|
100
|
+
return None
|
|
101
|
+
try:
|
|
102
|
+
age = float(value)
|
|
103
|
+
except OverflowError:
|
|
104
|
+
return None
|
|
105
|
+
return age if math.isfinite(age) and age >= 0 else None
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
class SessionRegistry(Protocol):
|
|
109
|
+
"""What the supervisor's registry does for :class:`VersionWorkers`."""
|
|
110
|
+
|
|
111
|
+
def recovering(self, worker: Worker) -> tuple[list[Session], list[Session]]: ...
|
|
112
|
+
|
|
113
|
+
def given_back(self, session: Session) -> None: ...
|
|
114
|
+
|
|
115
|
+
def worker_died(self, session: Session, entries: list[dict]) -> None: ...
|
|
116
|
+
|
|
117
|
+
def forget_lost(self, session: Session) -> None: ...
|
|
118
|
+
|
|
119
|
+
def close(self, session_id: str, reason: str = "closed") -> dict: ...
|
|
120
|
+
|
|
121
|
+
def tally(self, key: str) -> None: ...
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
class VersionWorkers:
|
|
125
|
+
"""Placement on, requests to and recovery of this supervisor's version workers."""
|
|
126
|
+
|
|
127
|
+
def __init__(self, pool: WorkerPool, registry: SessionRegistry) -> None:
|
|
128
|
+
self.pool = pool
|
|
129
|
+
self.registry = registry
|
|
130
|
+
self._claims = threading.Lock()
|
|
131
|
+
self._recoveries = Lanes()
|
|
132
|
+
self._overdue: dict[tuple[int, str], Worker] = {}
|
|
133
|
+
|
|
134
|
+
def home(self, version_id: str) -> Path:
|
|
135
|
+
"""Where a version's worker runs: its own ``WorldHost`` cache root (:mod:`worker`)."""
|
|
136
|
+
return self.pool.workers_root / version_id
|
|
137
|
+
|
|
138
|
+
def session_dir(self, version_id: str, session_id: str) -> Path:
|
|
139
|
+
return self.home(version_id) / "sessions" / session_id
|
|
140
|
+
|
|
141
|
+
def place(
|
|
142
|
+
self, version_id: str, session_id: str, body: dict, fetch: Callable[[Path], ExportFile | None], opens: Lanes
|
|
143
|
+
) -> Worker:
|
|
144
|
+
"""Open ``session_id`` on its version's worker with ``body``, claimed before placement so
|
|
145
|
+
another open cannot stop the worker as session-less meanwhile. ``fetch`` downloads an
|
|
146
|
+
export to restore into the session's directory once the worker runs (a worker clears
|
|
147
|
+
that directory when it starts); the worker reads it by path. One open per version at a
|
|
148
|
+
time (``opens``), so the worker's open timeout counts one open, never a queue."""
|
|
149
|
+
worker = self._worker_for(version_id)
|
|
150
|
+
self._claim(worker, session_id)
|
|
151
|
+
source = None
|
|
152
|
+
try:
|
|
153
|
+
source = fetch(self.session_dir(version_id, session_id))
|
|
154
|
+
placement = {**body, "restore": source.for_worker() if source is not None else None}
|
|
155
|
+
self._open_on(worker, session_id, placement, opens)
|
|
156
|
+
except BaseException:
|
|
157
|
+
self._release(worker, session_id)
|
|
158
|
+
raise
|
|
159
|
+
finally:
|
|
160
|
+
if source is not None:
|
|
161
|
+
source.discard()
|
|
162
|
+
return worker
|
|
163
|
+
|
|
164
|
+
def _open_on(self, worker: Worker, session_id: str, placement: dict, opens: Lanes) -> None:
|
|
165
|
+
"""The open on ``worker``, in its version's turn. One that raised after reaching the worker may still
|
|
166
|
+
complete there (an answer past the cut-off) or have left a copy, which no open of the id may find
|
|
167
|
+
later, so it is deleted there, an open still running abandoned, before the caller lets the claim go.
|
|
168
|
+
One that never reached it (the refusal's cause a :class:`WorkerUnreached`) left nothing to delete."""
|
|
169
|
+
try:
|
|
170
|
+
with opens.turn(worker.version_id):
|
|
171
|
+
self._control(worker, "POST", "/__host/sessions", placement, OPEN_TIMEOUT_S, "the open")
|
|
172
|
+
except Exception as error:
|
|
173
|
+
if not isinstance(error.__cause__, WorkerUnreached):
|
|
174
|
+
self._delete(worker, session_id)
|
|
175
|
+
raise
|
|
176
|
+
|
|
177
|
+
def control(
|
|
178
|
+
self, session: Session, method: str, verb: str, body: dict | None, *, timeout: float = CALL_TIMEOUT_S
|
|
179
|
+
) -> dict:
|
|
180
|
+
"""One session verb on the session's worker, answered as this host would answer it."""
|
|
181
|
+
what = f"tool '{body.get('tool')}'" if verb == "call" and body else verb
|
|
182
|
+
worker = self._serving(session)
|
|
183
|
+
path = f"/__host/sessions/{session.session_id}/{verb}"
|
|
184
|
+
return self._control(worker, method, path, body, timeout, what, session.session_id)
|
|
185
|
+
|
|
186
|
+
def outbox(self, session: Session, page: dict, timeout: float = CALL_TIMEOUT_S) -> dict:
|
|
187
|
+
"""A page of the session's request ledger outbox on its worker, asked even while an
|
|
188
|
+
overdue call holds the session's turn there (the outbox takes no turn), and read whole
|
|
189
|
+
within ``timeout``. A dead worker is refused, not recovered here: the drain keeps the
|
|
190
|
+
time it was given, and the reaper's recovery hands the session's journal over."""
|
|
191
|
+
path = f"/__host/sessions/{session.session_id}/requests"
|
|
192
|
+
worker = session.worker
|
|
193
|
+
assert worker is not None
|
|
194
|
+
if not worker.alive():
|
|
195
|
+
fail(WORKER_EXITED, f"the worker of session {session.session_id} exited; recovery hands its entries on")
|
|
196
|
+
return self._control(worker, "POST", path, page, timeout, "requests", session.session_id, DRAIN_ANSWER_BYTES)
|
|
197
|
+
|
|
198
|
+
def relay(self, session: Session, method: str, path: str, headers: dict[str, str], raw: bytes):
|
|
199
|
+
"""A data-plane request relayed as it came; the worker's own answer, whatever its status."""
|
|
200
|
+
what = f"{method} {urllib.parse.urlsplit(path).path}"
|
|
201
|
+
worker = self._serving(session)
|
|
202
|
+
return self._forward(worker, method, path, headers, raw, False, CALL_TIMEOUT_S, what, session.session_id)
|
|
203
|
+
|
|
204
|
+
def holds_turn(self, session: Session) -> bool:
|
|
205
|
+
"""Whether a call of the session past its time may still hold the session's turn on its worker:
|
|
206
|
+
one marked overdue that the worker does not say has ended. A worker that does not say holds it."""
|
|
207
|
+
worker = session.worker
|
|
208
|
+
if worker is None or not self._overdue_on(worker, session.session_id):
|
|
209
|
+
return False
|
|
210
|
+
turns = self._probe_turns(worker)
|
|
211
|
+
return turns is None or session.session_id in turns
|
|
212
|
+
|
|
213
|
+
def _overdue_on(self, worker: Worker, session_id: str) -> bool:
|
|
214
|
+
with self._claims:
|
|
215
|
+
return (id(worker), session_id) in self._overdue
|
|
216
|
+
|
|
217
|
+
def live(self, session: Session) -> Worker:
|
|
218
|
+
"""The session's worker, replaced first (and the session reopened on the new one) if it
|
|
219
|
+
died; a session no live worker holds afterwards is forgotten and answers ``unknown_session``.
|
|
220
|
+
A session that began closing meanwhile is served like any other: whoever then deletes it
|
|
221
|
+
on the new worker has drained the entries it resumed first."""
|
|
222
|
+
worker = session.worker
|
|
223
|
+
assert worker is not None
|
|
224
|
+
if worker.alive():
|
|
225
|
+
return worker
|
|
226
|
+
self.recover(worker)
|
|
227
|
+
current = session.worker
|
|
228
|
+
if current is None or not current.alive():
|
|
229
|
+
self.registry.forget_lost(session)
|
|
230
|
+
fail(UNKNOWN_SESSION, f"no session {session.session_id} on this host")
|
|
231
|
+
return current
|
|
232
|
+
|
|
233
|
+
def finish(self, session: Session) -> None:
|
|
234
|
+
"""Close a session that left the registry on its worker; a worker that is gone has closed it,
|
|
235
|
+
and what its journal kept of the session is handed over to the ledger."""
|
|
236
|
+
worker = session.worker
|
|
237
|
+
assert worker is not None
|
|
238
|
+
self._release(worker, session.session_id)
|
|
239
|
+
if not worker.alive():
|
|
240
|
+
return self._hand_over(session, self.session_dir(session.version_id, session.session_id))
|
|
241
|
+
self._delete(worker, session.session_id)
|
|
242
|
+
|
|
243
|
+
def _delete(self, worker: Worker, session_id: str) -> None:
|
|
244
|
+
"""``session_id`` deleted on ``worker`` while it runs; an open of it still running there is abandoned.
|
|
245
|
+
One the worker answers late still lands. One it never received would leave its copy holding an id this
|
|
246
|
+
host frees, so it is sent once more if the worker answers its health check, and the worker is recycled
|
|
247
|
+
when that one does not reach it either."""
|
|
248
|
+
if self._delivered(worker, session_id):
|
|
249
|
+
return
|
|
250
|
+
if worker.answers_health() and self._delivered(worker, session_id):
|
|
251
|
+
return
|
|
252
|
+
self._recycle(worker)
|
|
253
|
+
|
|
254
|
+
@staticmethod
|
|
255
|
+
def _delivered(worker: Worker, session_id: str) -> bool:
|
|
256
|
+
"""Whether a ``DELETE`` of ``session_id`` reached ``worker``, or there is no live worker to reach."""
|
|
257
|
+
if not worker.alive():
|
|
258
|
+
return True
|
|
259
|
+
try:
|
|
260
|
+
worker.request("DELETE", f"/__host/sessions/{session_id}", JSON, control=True, timeout=CALL_TIMEOUT_S)
|
|
261
|
+
except WorkerUnreached:
|
|
262
|
+
return False
|
|
263
|
+
except WorkerGone:
|
|
264
|
+
return True
|
|
265
|
+
return True
|
|
266
|
+
|
|
267
|
+
def export_path(self, session: Session, answer: dict) -> Path:
|
|
268
|
+
"""The file a worker's export landed in, admitted only from inside that worker's
|
|
269
|
+
directory for this session: the worker runs tenant code, so what it names is
|
|
270
|
+
checked, and the bytes are hashed by the supervisor, not trusted."""
|
|
271
|
+
exports = self.session_dir(session.version_id, session.session_id) / "exports"
|
|
272
|
+
path = Path(str(answer.get("path") or ""))
|
|
273
|
+
try:
|
|
274
|
+
resolved = path.resolve(strict=True)
|
|
275
|
+
resolved.relative_to(exports.resolve())
|
|
276
|
+
except (OSError, ValueError):
|
|
277
|
+
fail("host", f"the worker's export is not where a worker's exports go: {path}")
|
|
278
|
+
return resolved
|
|
279
|
+
|
|
280
|
+
def recover(self, dead: Worker, reason: str = "workers_replaced") -> None:
|
|
281
|
+
"""Replace ``dead`` (stopped first if it still runs) and reopen every session it held on
|
|
282
|
+
the new worker from what each committed; a session that cannot be reopened leaves the
|
|
283
|
+
registry (:meth:`_reopen_failed`). One recovery per version at a time; a later caller finds
|
|
284
|
+
it done. Never raises. Every session it takes is its own until it gives the session back
|
|
285
|
+
(:meth:`_give_back`)."""
|
|
286
|
+
held: list[Session] = []
|
|
287
|
+
try:
|
|
288
|
+
with self._recoveries.turn(dead.version_id):
|
|
289
|
+
if self.pool.peek(dead.version_id) is not dead:
|
|
290
|
+
return
|
|
291
|
+
self._stop(dead)
|
|
292
|
+
sessions, closing = self.registry.recovering(dead)
|
|
293
|
+
held = [*sessions, *closing]
|
|
294
|
+
aside = self._set_aside(dead.version_id, held)
|
|
295
|
+
fresh = self._replacement(dead)
|
|
296
|
+
for session in closing:
|
|
297
|
+
self._hand_over(session, aside.get(session.session_id))
|
|
298
|
+
for session in sessions:
|
|
299
|
+
self._reopen(fresh, session, aside.get(session.session_id))
|
|
300
|
+
self._forget_overdue(lambda worker: worker is dead)
|
|
301
|
+
self.registry.tally(reason)
|
|
302
|
+
finally:
|
|
303
|
+
self._give_back(held)
|
|
304
|
+
|
|
305
|
+
def _give_back(self, sessions: list[Session]) -> None:
|
|
306
|
+
"""Each session a recovery took, given back once the version's turn is; one that closed meanwhile
|
|
307
|
+
is drained and deleted on its new worker behind :class:`Finishing`, at tenant code's pace, holding
|
|
308
|
+
neither the version's next recovery nor a sweep. Each fails alone."""
|
|
309
|
+
for session in sessions:
|
|
310
|
+
try:
|
|
311
|
+
self.registry.given_back(session)
|
|
312
|
+
except Exception as error: # noqa: BLE001 - recovery never raises
|
|
313
|
+
log_line(f"session {session.session_id} did not leave its new worker: {type(error).__name__}")
|
|
314
|
+
|
|
315
|
+
def recover_dead(self) -> list[str]:
|
|
316
|
+
"""Recover every worker that died holding sessions; returns their versions."""
|
|
317
|
+
dead = self.pool.dead()
|
|
318
|
+
for worker in dead:
|
|
319
|
+
self.recover(worker)
|
|
320
|
+
return [worker.version_id for worker in dead]
|
|
321
|
+
|
|
322
|
+
def recover_stuck(self) -> list[str]:
|
|
323
|
+
"""Recycle every live worker holding a turn past ``STUCK_TURN_S`` (a handler that never
|
|
324
|
+
returned), whether or not that turn's session is still open here: its sessions reopen
|
|
325
|
+
from what they committed, and an id it still held opens again. Returns their versions."""
|
|
326
|
+
self._forget_overdue(lambda worker: not worker.alive())
|
|
327
|
+
stuck = [worker for worker in self.pool.live() if max(self._turns(worker).values(), default=0) > STUCK_TURN_S]
|
|
328
|
+
for worker in stuck:
|
|
329
|
+
self.recover(worker, "workers_recycled")
|
|
330
|
+
return [worker.version_id for worker in stuck]
|
|
331
|
+
|
|
332
|
+
def _serving(self, session: Session) -> Worker:
|
|
333
|
+
"""The session's live worker, refused ``unknown_session`` once its recovery could not reopen it (it
|
|
334
|
+
stays on its new worker only to be deleted there), and ``call_running`` while a call of the session
|
|
335
|
+
past its time still holds the session's turn there."""
|
|
336
|
+
worker = self.live(session)
|
|
337
|
+
if session.unrecovered:
|
|
338
|
+
fail(UNKNOWN_SESSION, f"no session {session.session_id} on this host")
|
|
339
|
+
key = (id(worker), session.session_id)
|
|
340
|
+
with self._claims:
|
|
341
|
+
overdue = key in self._overdue
|
|
342
|
+
held = self._turns(worker).get(session.session_id) if overdue else None
|
|
343
|
+
if held is None:
|
|
344
|
+
self._unmark_overdue(key)
|
|
345
|
+
return worker
|
|
346
|
+
fail(
|
|
347
|
+
CALL_RUNNING,
|
|
348
|
+
f"an earlier call on this session has run {held:.0f} s, past its time; "
|
|
349
|
+
f"retry after {RUNNING_RETRY_AFTER_S} s",
|
|
350
|
+
retryAfter=RUNNING_RETRY_AFTER_S,
|
|
351
|
+
)
|
|
352
|
+
|
|
353
|
+
@classmethod
|
|
354
|
+
def _turns(cls, worker: Worker) -> dict[str, float]:
|
|
355
|
+
return cls._probe_turns(worker) or {}
|
|
356
|
+
|
|
357
|
+
@staticmethod
|
|
358
|
+
def _probe_turns(worker: Worker) -> dict[str, float] | None:
|
|
359
|
+
"""Each session holding a turn on ``worker`` and how long it has held it, as the worker
|
|
360
|
+
says within ``HEALTH_TIMEOUT_S``; None when it cannot say. Never raises: the worker runs
|
|
361
|
+
tenant code, so its answer is cut off at the probe's deadline and size, and only a
|
|
362
|
+
finite, non-negative age of a session is kept."""
|
|
363
|
+
try:
|
|
364
|
+
status, payload = worker.probe("/__host/turns", deadline_s=HEALTH_TIMEOUT_S)
|
|
365
|
+
answer = json.loads(payload) if status == 200 else None
|
|
366
|
+
except (WorkerGone, ValueError, RecursionError):
|
|
367
|
+
return None
|
|
368
|
+
turns = answer.get("turns") if isinstance(answer, dict) else None
|
|
369
|
+
if not isinstance(turns, dict):
|
|
370
|
+
return None
|
|
371
|
+
ages = {str(sid): _age(age) for sid, age in turns.items()}
|
|
372
|
+
return {sid: age for sid, age in ages.items() if age is not None}
|
|
373
|
+
|
|
374
|
+
@staticmethod
|
|
375
|
+
def _stop(dead: Worker) -> None:
|
|
376
|
+
try:
|
|
377
|
+
dead.stop()
|
|
378
|
+
except Exception as error: # noqa: BLE001 - a worker that will not stop is replaced all the same
|
|
379
|
+
log_line(f"worker for version {dead.version_id} did not stop: {type(error).__name__}")
|
|
380
|
+
|
|
381
|
+
def _replacement(self, dead: Worker) -> Worker | None:
|
|
382
|
+
try:
|
|
383
|
+
return self.pool.replace(dead)
|
|
384
|
+
except Exception as error: # noqa: BLE001 - its sessions are lost either way; the platform reopens them
|
|
385
|
+
log_line(f"worker for version {dead.version_id} could not be replaced: {type(error).__name__}")
|
|
386
|
+
return None
|
|
387
|
+
|
|
388
|
+
def _set_aside(self, version_id: str, sessions: list[Session]) -> dict[str, Path]:
|
|
389
|
+
"""Each session's directory moved under the worker's ``recovered/``, before a new worker
|
|
390
|
+
clears its sessions directory; a session whose directory cannot be moved is left out."""
|
|
391
|
+
root = self.home(version_id) / RECOVERED_DIR
|
|
392
|
+
try:
|
|
393
|
+
if not real_dir(root):
|
|
394
|
+
clear(root)
|
|
395
|
+
root.mkdir(mode=0o700, exist_ok=True)
|
|
396
|
+
except OSError as error:
|
|
397
|
+
log_line(f"no directory to set version {version_id}'s sessions aside in: {type(error).__name__}")
|
|
398
|
+
return {}
|
|
399
|
+
moved = {session.session_id: self._move_aside(root, version_id, session.session_id) for session in sessions}
|
|
400
|
+
return {session_id: directory for session_id, directory in moved.items() if directory is not None}
|
|
401
|
+
|
|
402
|
+
def _move_aside(self, root: Path, version_id: str, session_id: str) -> Path | None:
|
|
403
|
+
source = self.session_dir(version_id, session_id)
|
|
404
|
+
target = root / session_id
|
|
405
|
+
try:
|
|
406
|
+
if not (real_dir(root) and real_dir(source.parent) and real_dir(source)):
|
|
407
|
+
return None
|
|
408
|
+
clear(target)
|
|
409
|
+
os.replace(source, target)
|
|
410
|
+
except OSError as error:
|
|
411
|
+
log_line(f"session {session_id} could not be set aside: {type(error).__name__}")
|
|
412
|
+
return None
|
|
413
|
+
return target
|
|
414
|
+
|
|
415
|
+
def _reopen(self, fresh: Worker | None, session: Session, directory: Path | None) -> None:
|
|
416
|
+
if fresh is None or directory is None or session.placement is None:
|
|
417
|
+
return self._lose(session, directory)
|
|
418
|
+
body = {**session.placement, "restore": {"recovered": str(directory)}}
|
|
419
|
+
self._claim(fresh, session.session_id)
|
|
420
|
+
try:
|
|
421
|
+
status, _, payload = fresh.request(
|
|
422
|
+
"POST", "/__host/sessions", JSON, json.dumps(body).encode("utf-8"), control=True, timeout=OPEN_TIMEOUT_S
|
|
423
|
+
)
|
|
424
|
+
self._answer(status, payload)
|
|
425
|
+
except Exception as error: # noqa: BLE001 - one session that cannot be reopened never strands the rest
|
|
426
|
+
return self._reopen_failed(fresh, session, directory, error)
|
|
427
|
+
session.worker = fresh
|
|
428
|
+
session.export = None
|
|
429
|
+
self.registry.worker_died(session, [])
|
|
430
|
+
self.registry.tally("sessions_recovered")
|
|
431
|
+
|
|
432
|
+
def _reopen_failed(self, fresh: Worker, session: Session, directory: Path, error: Exception) -> None:
|
|
433
|
+
"""A reopen that failed while ``fresh`` runs may still open there (an answer past the cut-off, or one
|
|
434
|
+
not read). The session is marked unrecovered and closed first, whatever closed it already, so no request
|
|
435
|
+
is relayed to ``fresh`` before both, then kept there, claimed: the recovery's give-back has it drained
|
|
436
|
+
and deleted there before its id takes a new open. With ``fresh`` gone too, it is forgotten."""
|
|
437
|
+
if not fresh.alive():
|
|
438
|
+
self._release(fresh, session.session_id)
|
|
439
|
+
return self._lose(session, directory, error)
|
|
440
|
+
self._unrecovered(session, error)
|
|
441
|
+
self.registry.close(session.session_id, reason=LOST)
|
|
442
|
+
session.worker = fresh
|
|
443
|
+
self._hand_over(session, directory)
|
|
444
|
+
|
|
445
|
+
def _lose(self, session: Session, directory: Path | None, error: BaseException | None = None) -> None:
|
|
446
|
+
self._hand_over(session, directory)
|
|
447
|
+
self.registry.forget_lost(session)
|
|
448
|
+
self._unrecovered(session, error)
|
|
449
|
+
|
|
450
|
+
def _unrecovered(self, session: Session, error: BaseException | None) -> None:
|
|
451
|
+
session.unrecovered = True
|
|
452
|
+
self.registry.tally("sessions_unrecovered")
|
|
453
|
+
why = f": {type(error).__name__}" if error is not None else ""
|
|
454
|
+
log_line(f"session {session.session_id} was not reopened on a new worker{why}; the platform reopens it")
|
|
455
|
+
|
|
456
|
+
def _hand_over(self, session: Session, directory: Path | None) -> None:
|
|
457
|
+
"""The request ledger entries a session's dead worker had not sent, given to the host's
|
|
458
|
+
ledger before the session goes (none when its directory was not set aside); a journal
|
|
459
|
+
that cannot be handed over costs that session its entries and no other session anything."""
|
|
460
|
+
try:
|
|
461
|
+
self.registry.worker_died(session, self._left(directory))
|
|
462
|
+
except Exception as error: # noqa: BLE001 - one session's journal never strands the rest
|
|
463
|
+
log_line(f"session {session.session_id} journal was not handed over: {type(error).__name__}")
|
|
464
|
+
|
|
465
|
+
@staticmethod
|
|
466
|
+
def _left(directory: Path | None) -> list[dict]:
|
|
467
|
+
if directory is None or not (real_dir(directory.parent) and real_dir(directory)):
|
|
468
|
+
return []
|
|
469
|
+
try:
|
|
470
|
+
return CallJournal.read_left(directory / LEDGER_JOURNAL, LEFT_JOURNAL_BYTES)
|
|
471
|
+
finally:
|
|
472
|
+
shutil.rmtree(directory, ignore_errors=True)
|
|
473
|
+
|
|
474
|
+
def _worker_for(self, version_id: str) -> Worker:
|
|
475
|
+
current = self.pool.peek(version_id)
|
|
476
|
+
if current is not None and not current.alive():
|
|
477
|
+
self.recover(current)
|
|
478
|
+
try:
|
|
479
|
+
return self.pool.get(version_id)
|
|
480
|
+
except WorkerCapacity as error:
|
|
481
|
+
fail("at_capacity", str(error))
|
|
482
|
+
|
|
483
|
+
def _claim(self, worker: Worker, session_id: str) -> None:
|
|
484
|
+
with self._claims:
|
|
485
|
+
worker.sessions.add(session_id)
|
|
486
|
+
|
|
487
|
+
def _release(self, worker: Worker, session_id: str) -> None:
|
|
488
|
+
with self._claims:
|
|
489
|
+
worker.sessions.discard(session_id)
|
|
490
|
+
if not worker.sessions:
|
|
491
|
+
worker.idle_since = time.time()
|
|
492
|
+
|
|
493
|
+
def _control(
|
|
494
|
+
self,
|
|
495
|
+
worker: Worker,
|
|
496
|
+
method: str,
|
|
497
|
+
path: str,
|
|
498
|
+
body: dict | None,
|
|
499
|
+
timeout: float,
|
|
500
|
+
what: str,
|
|
501
|
+
session_id: str | None = None,
|
|
502
|
+
limit: int | None = None,
|
|
503
|
+
) -> dict:
|
|
504
|
+
raw = json.dumps(body).encode("utf-8") if body is not None else b""
|
|
505
|
+
status, _, payload = self._forward(worker, method, path, JSON, raw, True, timeout, what, session_id, limit)
|
|
506
|
+
return self._answer(status, payload)
|
|
507
|
+
|
|
508
|
+
@staticmethod
|
|
509
|
+
def _answer(status: int, payload: bytes) -> dict:
|
|
510
|
+
"""A worker's control answer as this host's: its refusal re-raised as ours."""
|
|
511
|
+
answer = json.loads(payload or b"null")
|
|
512
|
+
if status == 200:
|
|
513
|
+
return answer if isinstance(answer, dict) else {"result": answer}
|
|
514
|
+
detail: dict[str, Any] = answer if isinstance(answer, dict) else {}
|
|
515
|
+
retry = {"retryAfter": detail["retryAfter"]} if "retryAfter" in detail else {}
|
|
516
|
+
fail(str(detail.get("error") or "host"), str(detail.get("message") or detail.get("detail") or answer), **retry)
|
|
517
|
+
|
|
518
|
+
def _forward(
|
|
519
|
+
self,
|
|
520
|
+
worker: Worker,
|
|
521
|
+
method: str,
|
|
522
|
+
path: str,
|
|
523
|
+
headers: dict[str, str],
|
|
524
|
+
raw: bytes,
|
|
525
|
+
control: bool,
|
|
526
|
+
timeout: float,
|
|
527
|
+
what: str,
|
|
528
|
+
session_id: str | None = None,
|
|
529
|
+
limit: int | None = None,
|
|
530
|
+
) -> tuple[int, list[tuple[str, str]], bytes]:
|
|
531
|
+
try:
|
|
532
|
+
return worker.request(method, path, headers, raw, control=control, timeout=timeout, limit=limit)
|
|
533
|
+
except WorkerGone as error:
|
|
534
|
+
raise self._unanswered(worker, error, what, timeout, session_id) from error
|
|
535
|
+
|
|
536
|
+
def _unanswered(
|
|
537
|
+
self, worker: Worker, error: WorkerGone, what: str, timeout: float, session_id: str | None
|
|
538
|
+
) -> ContractError:
|
|
539
|
+
if worker.alive() and worker.answers_health():
|
|
540
|
+
return self._refusal(worker, error, what, timeout, session_id)
|
|
541
|
+
self._recycle(worker)
|
|
542
|
+
return ContractError(
|
|
543
|
+
WORKER_EXITED,
|
|
544
|
+
f"the world's worker exited during {what}; retry: the session continues from what it committed",
|
|
545
|
+
)
|
|
546
|
+
|
|
547
|
+
def _recycle(self, worker: Worker) -> None:
|
|
548
|
+
worker.process.kill()
|
|
549
|
+
threading.Thread(target=self.recover, args=(worker,), name="world-worker-recovery", daemon=True).start()
|
|
550
|
+
|
|
551
|
+
def _refusal(
|
|
552
|
+
self, worker: Worker, error: WorkerGone, what: str, timeout: float, session_id: str | None
|
|
553
|
+
) -> ContractError:
|
|
554
|
+
if isinstance(error, WorkerTimeout):
|
|
555
|
+
self._mark_overdue(worker, session_id)
|
|
556
|
+
return ContractError(
|
|
557
|
+
CALL_TIMEOUT,
|
|
558
|
+
f"{what} did not finish within {timeout:g} s; it may still complete, and the session keeps serving",
|
|
559
|
+
)
|
|
560
|
+
return ContractError(
|
|
561
|
+
BUSY, f"the world's worker did not take {what}; retry after {RETRY_AFTER_S} s", retryAfter=RETRY_AFTER_S
|
|
562
|
+
)
|
|
563
|
+
|
|
564
|
+
def _mark_overdue(self, worker: Worker, session_id: str | None) -> None:
|
|
565
|
+
"""A call of ``session_id`` past its time may still hold the session's turn on ``worker``."""
|
|
566
|
+
if session_id is None:
|
|
567
|
+
return
|
|
568
|
+
with self._claims:
|
|
569
|
+
self._overdue[(id(worker), session_id)] = worker
|
|
570
|
+
|
|
571
|
+
def _unmark_overdue(self, key: tuple[int, str]) -> None:
|
|
572
|
+
with self._claims:
|
|
573
|
+
self._overdue.pop(key, None)
|
|
574
|
+
|
|
575
|
+
def _forget_overdue(self, gone: Callable[[Worker], bool]) -> None:
|
|
576
|
+
with self._claims:
|
|
577
|
+
self._overdue = {key: worker for key, worker in self._overdue.items() if not gone(worker)}
|