@withgateway/sdk 0.3.1 → 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -3
- package/dist/cli.js +2880 -2032
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +2982 -2135
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.d.mts +1 -1
- package/dist/files/index.d.ts +1 -1
- package/dist/files/index.js +180 -41
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +172 -35
- package/dist/files/index.mjs.map +1 -1
- package/dist/{index-CBqbB_0J.d.ts → index-DUHmS74p.d.ts} +1 -1
- package/dist/{index-D-AMnAMa.d.mts → index-mJhG3vZU.d.mts} +1 -1
- package/dist/index.d.mts +4 -4
- package/dist/index.d.ts +4 -4
- package/dist/index.js +1309 -540
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +1287 -532
- package/dist/index.mjs.map +1 -1
- package/dist/{media-CbcAfQvd.d.mts → media-BIApiPzS.d.mts} +21 -2
- package/dist/{media-CbcAfQvd.d.ts → media-BIApiPzS.d.ts} +21 -2
- package/dist/traces/index.d.mts +1 -1
- package/dist/traces/index.d.ts +1 -1
- package/dist/traces/index.js +177 -38
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +169 -32
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.d.mts +2 -2
- package/dist/tracing/index.d.ts +2 -2
- package/dist/tracing/index.js +69 -13
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +69 -13
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +249 -120
- package/dist/worlds/index.d.ts +249 -120
- package/dist/worlds/index.js +1240 -484
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +1231 -476
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
- package/runtime/gatewaysdk/environment/_bundle.py +2 -27
- package/runtime/gatewaysdk/environment/_materialize.py +49 -69
- package/runtime/gatewaysdk/environment/_world.py +122 -197
- package/runtime/gatewaysdk/environment/backing.py +137 -0
- package/runtime/gatewaysdk/environment/core.py +1 -1
- package/runtime/gatewaysdk/environment/legacy_sqlite.py +301 -0
- package/runtime/gatewaysdk/environment/reward_keys.py +25 -0
- package/runtime/gatewaysdk/environment/runtime.py +8 -13
- package/runtime/gatewaysdk/environment/schema/__init__.py +4 -4
- package/runtime/gatewaysdk/environment/schema/admin_routes.py +86 -0
- package/runtime/gatewaysdk/environment/schema/api.py +266 -514
- package/runtime/gatewaysdk/environment/schema/batch.py +42 -37
- package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +17 -5
- package/runtime/gatewaysdk/environment/schema/call_log.py +291 -0
- package/runtime/gatewaysdk/environment/schema/call_redaction.py +344 -0
- package/runtime/gatewaysdk/environment/schema/compiler.py +334 -158
- package/runtime/gatewaysdk/environment/schema/conform.py +8 -12
- package/runtime/gatewaysdk/environment/schema/connector.py +27 -62
- package/runtime/gatewaysdk/environment/schema/data.py +50 -207
- package/runtime/gatewaysdk/environment/schema/database/__init__.py +24 -0
- package/runtime/gatewaysdk/environment/schema/database/audit.py +121 -0
- package/runtime/gatewaysdk/environment/schema/database/base.py +995 -0
- package/runtime/gatewaysdk/environment/schema/database/budget.py +33 -0
- package/runtime/gatewaysdk/environment/schema/database/connections.py +101 -0
- package/runtime/gatewaysdk/environment/schema/database/engines.py +8 -0
- package/runtime/gatewaysdk/environment/schema/database/guard.py +208 -0
- package/runtime/gatewaysdk/environment/schema/database/ingest.py +154 -0
- package/runtime/gatewaysdk/environment/schema/database/matching.py +58 -0
- package/runtime/gatewaysdk/environment/schema/database/provenance.py +30 -0
- package/runtime/gatewaysdk/environment/schema/database/rows.py +53 -0
- package/runtime/gatewaysdk/environment/schema/database/runtime_rows.py +39 -0
- package/runtime/gatewaysdk/environment/schema/database/sqlite.py +367 -0
- package/runtime/gatewaysdk/environment/schema/domain_migration.py +236 -0
- package/runtime/gatewaysdk/environment/schema/envelope.py +169 -0
- package/runtime/gatewaysdk/environment/schema/extract.py +6 -13
- package/runtime/gatewaysdk/environment/schema/grading.py +45 -30
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +142 -12
- package/runtime/gatewaysdk/environment/schema/host.py +158 -250
- package/runtime/gatewaysdk/environment/schema/lanes.py +53 -0
- package/runtime/gatewaysdk/environment/schema/local.py +44 -71
- package/runtime/gatewaysdk/environment/schema/mapping.py +5 -25
- package/runtime/gatewaysdk/environment/schema/openapi_import.py +112 -73
- package/runtime/gatewaysdk/environment/schema/openapi_paging.py +147 -0
- package/runtime/gatewaysdk/environment/schema/openapi_payloads.py +288 -0
- package/runtime/gatewaysdk/environment/schema/openapi_routes.py +97 -102
- package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +113 -30
- package/runtime/gatewaysdk/environment/schema/platform.py +123 -171
- package/runtime/gatewaysdk/environment/schema/platform_import.py +2 -3
- package/runtime/gatewaysdk/environment/schema/platform_query.py +61 -5
- package/runtime/gatewaysdk/environment/schema/primary_key.py +61 -0
- package/runtime/gatewaysdk/environment/schema/proof.py +19 -12
- package/runtime/gatewaysdk/environment/schema/runtime.py +77 -160
- package/runtime/gatewaysdk/environment/schema/served_names.py +58 -0
- package/runtime/gatewaysdk/environment/schema/session_files.py +166 -0
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +2 -2
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +19 -11
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +29 -3
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +14 -14
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +12 -12
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +3 -2
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +15 -0
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/openapi-import.md +50 -21
- package/runtime/gatewaysdk/environment/schema/slack.py +4 -8
- package/runtime/gatewaysdk/environment/schema/snapshot.py +385 -129
- package/runtime/gatewaysdk/environment/schema/store_contract.py +8 -14
- package/runtime/gatewaysdk/environment/schema/tasks.py +140 -55
- package/runtime/gatewaysdk/environment/schema/tools_world.py +110 -87
- package/runtime/gatewaysdk/environment/schema/ui.py +24 -5
- package/runtime/gatewaysdk/environment/schema/ui_static/app.js +100 -130
- package/runtime/gatewaysdk/environment/schema/ui_static/index.html +1 -8
- package/runtime/gatewaysdk/environment/schema/validation.py +166 -40
- package/runtime/gatewaysdk/environment/schema/workers.py +142 -57
- package/runtime/gatewaysdk/environment/schema/world_sql.py +81 -0
- package/runtime/gatewaysdk/environment/schema/world_tests.py +71 -39
- package/runtime/gatewaysdk/environment/score.py +57 -0
- package/runtime/gatewaysdk/environment/secret_values.py +36 -0
- package/runtime/gatewaysdk/environment/world.py +42 -3
- package/runtime/gatewaysdk/environment/world_session_contract.py +32 -5
- package/skills/benchmarks-getting-started/SKILL.md +2 -2
- package/skills/connector-schema-authoring/SKILL.md +19 -11
- package/skills/connector-schema-authoring/connector-worlds.md +13 -4
- package/skills/connector-schema-authoring/world-schema.md +103 -51
- package/skills/world-data-ingestion/ROWS.md +29 -3
- package/skills/world-data-ingestion/SKILL.md +14 -14
- package/skills/worlds-getting-started/SKILL.md +12 -12
- package/skills/worlds-getting-started/references/cli-mcp-map.md +3 -2
- package/skills/worlds-getting-started/references/gotchas.md +15 -0
- package/skills/worlds-getting-started/references/openapi-import.md +50 -21
- package/runtime/gatewaysdk/environment/schema/store.py +0 -1013
|
@@ -23,13 +23,14 @@ import json
|
|
|
23
23
|
import sys
|
|
24
24
|
from dataclasses import dataclass, field
|
|
25
25
|
from pathlib import Path
|
|
26
|
-
from typing import IO, Any, Iterable, Iterator
|
|
26
|
+
from typing import IO, Any, Iterable, Iterator, NoReturn
|
|
27
27
|
|
|
28
28
|
from .cast import TEXT_CASTER, EntityCasters
|
|
29
|
-
from .compiler import Contract,
|
|
29
|
+
from .compiler import Contract, payload_value
|
|
30
30
|
from .csv_rows import csv_delimiter, is_delimited_name, iter_csv_records
|
|
31
|
+
from .primary_key import PrimaryKey
|
|
31
32
|
from .redaction import RowRedactor
|
|
32
|
-
from .
|
|
33
|
+
from .database import WorldDatabase
|
|
33
34
|
from .validation import ContractError, fail, validate
|
|
34
35
|
|
|
35
36
|
CHUNK_BYTES = 64 * 1024 * 1024
|
|
@@ -45,9 +46,6 @@ def encode_row(value: Any) -> str:
|
|
|
45
46
|
return json.dumps(value, ensure_ascii=False, separators=(",", ":"), allow_nan=False)
|
|
46
47
|
|
|
47
48
|
|
|
48
|
-
# ---------------------------------------------------------------- streaming rows
|
|
49
|
-
|
|
50
|
-
|
|
51
49
|
class _Reader:
|
|
52
50
|
"""A text stream with a growing buffer, for decoding JSON values one at a time.
|
|
53
51
|
|
|
@@ -256,7 +254,6 @@ def iter_file_rows(
|
|
|
256
254
|
seen[name] = seen.get(name, 0) + 1
|
|
257
255
|
yield name, row
|
|
258
256
|
return
|
|
259
|
-
# A document: rewind to its first character and stream it.
|
|
260
257
|
reader.pos = start
|
|
261
258
|
yield from _stream_document(reader, where, entity, seen)
|
|
262
259
|
|
|
@@ -337,9 +334,6 @@ def count_rows(
|
|
|
337
334
|
return total
|
|
338
335
|
|
|
339
336
|
|
|
340
|
-
# ---------------------------------------------------------------- chunks
|
|
341
|
-
|
|
342
|
-
|
|
343
337
|
@dataclass
|
|
344
338
|
class Chunk:
|
|
345
339
|
ordinal: int
|
|
@@ -350,7 +344,7 @@ class Chunk:
|
|
|
350
344
|
entity: str | None
|
|
351
345
|
content_bytes: int # the JSONL before gzip
|
|
352
346
|
content_sha256: str
|
|
353
|
-
first_row: int
|
|
347
|
+
first_row: int
|
|
354
348
|
|
|
355
349
|
def manifest_entry(self) -> dict:
|
|
356
350
|
return {
|
|
@@ -464,9 +458,6 @@ def iter_chunk_rows(path: str | Path, entity: str | None = None) -> Iterator[tup
|
|
|
464
458
|
fail("rows", f"Cannot read {path.name} as UTF-8 text", str(path))
|
|
465
459
|
|
|
466
460
|
|
|
467
|
-
# ---------------------------------------------------------------- refusals
|
|
468
|
-
|
|
469
|
-
|
|
470
461
|
def _field_of(error: ContractError) -> str | None:
|
|
471
462
|
path = error.detail.get("path") or ""
|
|
472
463
|
inside = path[1:] if path.startswith("/") and len(path) > 1 else None
|
|
@@ -489,14 +480,13 @@ def refusal(
|
|
|
489
480
|
them all, ``field`` the property refused (None for a row-level refusal)."""
|
|
490
481
|
key = None
|
|
491
482
|
if contract is not None and isinstance(row, dict) and entity in contract.entities:
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
key = {name: row[name] for name in fields}
|
|
483
|
+
primary = PrimaryKey.in_contract(contract, entity)
|
|
484
|
+
key = primary.key(row) if primary.missing(row) is None else None
|
|
495
485
|
pointer = "/" + (entity or "")
|
|
496
486
|
if index is not None:
|
|
497
487
|
pointer += f"/{index}"
|
|
498
488
|
if field_name:
|
|
499
|
-
pointer += f"/{field_name}"
|
|
489
|
+
pointer += field_name if field_name.startswith("/") else f"/{field_name}"
|
|
500
490
|
return {
|
|
501
491
|
"chunk": chunk,
|
|
502
492
|
"index": index,
|
|
@@ -523,9 +513,9 @@ def shape_refusal(contract: Contract, chunk: int, index: int, entity: str, row:
|
|
|
523
513
|
if not isinstance(row, dict):
|
|
524
514
|
return refusal(chunk, index, entity, row, None, "A row must be an object")
|
|
525
515
|
spec = contract.entities[entity]
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
516
|
+
missing = PrimaryKey.of(spec).missing(row)
|
|
517
|
+
if missing is not None:
|
|
518
|
+
return refusal(chunk, index, entity, row, missing, f"Row is missing primary key field {missing!r}", contract)
|
|
529
519
|
try:
|
|
530
520
|
validate(spec["schema"], row, location=f"{spec['location']}/schema")
|
|
531
521
|
except ContractError as error:
|
|
@@ -533,6 +523,26 @@ def shape_refusal(contract: Contract, chunk: int, index: int, entity: str, row:
|
|
|
533
523
|
return None
|
|
534
524
|
|
|
535
525
|
|
|
526
|
+
def raise_shape_refusal(
|
|
527
|
+
contract: Contract, first: dict, row: Any, summary: str, context: dict, unknown_at: str
|
|
528
|
+
) -> NoReturn:
|
|
529
|
+
"""The first :func:`shape_refusal` of a file as the ``ContractError`` the store
|
|
530
|
+
raises for it, under its own code (``entity``, ``rows``, ``primary_key``, or the
|
|
531
|
+
validator's), with ``summary`` before its message and ``context`` attached."""
|
|
532
|
+
spec = contract.entities.get(first["entity"])
|
|
533
|
+
if spec is None:
|
|
534
|
+
fail("entity", summary + first["detail"], unknown_at, **context)
|
|
535
|
+
if not isinstance(row, dict):
|
|
536
|
+
fail("rows", summary + first["detail"], first["pointer"], **context)
|
|
537
|
+
if PrimaryKey.of(spec).missing(row) is not None:
|
|
538
|
+
fail("primary_key", summary + first["detail"], first["pointer"], **context)
|
|
539
|
+
try:
|
|
540
|
+
validate(spec["schema"], row, location=f"{spec['location']}/schema")
|
|
541
|
+
except ContractError as error:
|
|
542
|
+
raise error.contextualize(message=summary + error.detail["message"], **context)
|
|
543
|
+
fail("rows", summary + first["detail"], first["pointer"], **context)
|
|
544
|
+
|
|
545
|
+
|
|
536
546
|
def write_report(records: Iterable[dict], destination: IO[str], *, cap: int = REPORT_CAP) -> int:
|
|
537
547
|
"""Refusal records as JSONL, at most ``cap`` of them followed by
|
|
538
548
|
``{"truncated": true, "total": N}`` when there were more. Returns the total."""
|
|
@@ -546,9 +556,6 @@ def write_report(records: Iterable[dict], destination: IO[str], *, cap: int = RE
|
|
|
546
556
|
return total
|
|
547
557
|
|
|
548
558
|
|
|
549
|
-
# ---------------------------------------------------------------- batches
|
|
550
|
-
|
|
551
|
-
|
|
552
559
|
@dataclass
|
|
553
560
|
class ChunkSpec:
|
|
554
561
|
ordinal: int
|
|
@@ -560,8 +567,8 @@ class ChunkSpec:
|
|
|
560
567
|
class _Staged:
|
|
561
568
|
"""The valid rows of a batch after the in-batch merge, keyed the way the store keys."""
|
|
562
569
|
|
|
563
|
-
rows: dict[str, dict[str, dict]] = field(default_factory=dict)
|
|
564
|
-
origin: dict[str, dict[str, tuple[int, int]]] = field(default_factory=dict)
|
|
570
|
+
rows: dict[str, dict[str, dict]] = field(default_factory=dict)
|
|
571
|
+
origin: dict[str, dict[str, tuple[int, int]]] = field(default_factory=dict)
|
|
565
572
|
refusals: list[dict] = field(default_factory=list)
|
|
566
573
|
read: int = 0
|
|
567
574
|
|
|
@@ -602,7 +609,7 @@ def _redacted(
|
|
|
602
609
|
|
|
603
610
|
def _stage(
|
|
604
611
|
contract: Contract,
|
|
605
|
-
store:
|
|
612
|
+
store: WorldDatabase,
|
|
606
613
|
chunks: Iterable[ChunkSpec],
|
|
607
614
|
merge: str,
|
|
608
615
|
staged: _Staged | None = None,
|
|
@@ -621,7 +628,7 @@ def _stage(
|
|
|
621
628
|
if refused is not None:
|
|
622
629
|
staged.refusals.append(refused)
|
|
623
630
|
continue
|
|
624
|
-
key = store.
|
|
631
|
+
key = store.key_of(entity, row)
|
|
625
632
|
held = staged.rows.setdefault(entity, {})
|
|
626
633
|
origin = staged.origin.setdefault(entity, {})
|
|
627
634
|
if key in held and merge == "fill":
|
|
@@ -635,7 +642,7 @@ def _stage(
|
|
|
635
642
|
return staged
|
|
636
643
|
|
|
637
644
|
|
|
638
|
-
def _reference_refusals(contract: Contract, store:
|
|
645
|
+
def _reference_refusals(contract: Contract, store: WorldDatabase, staged: _Staged, mode: str) -> list[dict]:
|
|
639
646
|
"""Every row whose relationship points at a key neither the parent snapshot
|
|
640
647
|
(minus replaced entities) nor the batch holds."""
|
|
641
648
|
refusals: list[dict] = []
|
|
@@ -645,8 +652,7 @@ def _reference_refusals(contract: Contract, store: SQLiteStore, staged: _Staged,
|
|
|
645
652
|
if target not in known:
|
|
646
653
|
held = set(staged.rows.get(target, {}))
|
|
647
654
|
if not (mode == "replace" and target in staged.rows):
|
|
648
|
-
|
|
649
|
-
held |= {row[0] for row in store._conn.execute(f"SELECT _key FROM {quote(target)}")}
|
|
655
|
+
held |= store.keys(target)
|
|
650
656
|
known[target] = held
|
|
651
657
|
return known[target]
|
|
652
658
|
|
|
@@ -657,7 +663,7 @@ def _reference_refusals(contract: Contract, store: SQLiteStore, staged: _Staged,
|
|
|
657
663
|
dropped = []
|
|
658
664
|
for key, row in rows.items():
|
|
659
665
|
for rel in relationships:
|
|
660
|
-
values = [row
|
|
666
|
+
values = [payload_value(row, name) for name in rel["fields"]]
|
|
661
667
|
if all(value is None for value in values):
|
|
662
668
|
continue
|
|
663
669
|
chunk, index = staged.origin[entity][key]
|
|
@@ -677,7 +683,7 @@ def _reference_refusals(contract: Contract, store: SQLiteStore, staged: _Staged,
|
|
|
677
683
|
break
|
|
678
684
|
target = rel["target"]
|
|
679
685
|
try:
|
|
680
|
-
wanted = store.
|
|
686
|
+
wanted = store.key_of(target, dict(zip(contract.entities[target]["primary_key"], values)))
|
|
681
687
|
except ContractError as error:
|
|
682
688
|
refusals.append(
|
|
683
689
|
refusal(chunk, index, entity, row, rel["fields"][0], error.detail["message"], contract)
|
|
@@ -704,7 +710,7 @@ def _reference_refusals(contract: Contract, store: SQLiteStore, staged: _Staged,
|
|
|
704
710
|
|
|
705
711
|
|
|
706
712
|
def _apply(
|
|
707
|
-
contract: Contract, store:
|
|
713
|
+
contract: Contract, store: WorldDatabase, staged: _Staged, mode: str, batch_id: str
|
|
708
714
|
) -> tuple[int, dict | None]:
|
|
709
715
|
"""Apply the staged rows as one batch; the count applied, or the store's refusal."""
|
|
710
716
|
from .data import seed_batch
|
|
@@ -779,7 +785,7 @@ def build_batch(
|
|
|
779
785
|
applied_chunks: list[int] = []
|
|
780
786
|
applied_digests: list[str] = []
|
|
781
787
|
counts: dict[str, int] = {}
|
|
782
|
-
with
|
|
788
|
+
with WorldDatabase.open(parent, contract, audit=False) as store:
|
|
783
789
|
if atomic:
|
|
784
790
|
staged = _stage(contract, store, chunks, merge, redactor=redactor)
|
|
785
791
|
refusals.extend(staged.refusals)
|
|
@@ -800,8 +806,7 @@ def build_batch(
|
|
|
800
806
|
found.append(refused)
|
|
801
807
|
refusals.extend(found)
|
|
802
808
|
if found and not dry_run:
|
|
803
|
-
# The store rolled the chunk's transaction back
|
|
804
|
-
# accept are gone with it, so the chunk is refused as a whole.
|
|
809
|
+
# The store rolled the chunk's transaction back, so the chunk is refused as a whole.
|
|
805
810
|
continue
|
|
806
811
|
if not dry_run:
|
|
807
812
|
applied_rows += applied
|
|
@@ -7,8 +7,18 @@ from pathlib import Path
|
|
|
7
7
|
|
|
8
8
|
SCHEMA_PACKAGE = Path(__file__).parent
|
|
9
9
|
ENVIRONMENT_PACKAGE = SCHEMA_PACKAGE.parent
|
|
10
|
-
|
|
11
|
-
|
|
10
|
+
SIBLING_MODULES = (
|
|
11
|
+
"_toml.py",
|
|
12
|
+
"backing.py",
|
|
13
|
+
"errors.py",
|
|
14
|
+
"legacy_sqlite.py",
|
|
15
|
+
"reward_keys.py",
|
|
16
|
+
"score.py",
|
|
17
|
+
"secret_values.py",
|
|
18
|
+
"world.py",
|
|
19
|
+
"world_session_contract.py",
|
|
20
|
+
)
|
|
21
|
+
SUBPACKAGES = ("database",)
|
|
12
22
|
|
|
13
23
|
|
|
14
24
|
def stage_runtime_package(server: Path) -> Path:
|
|
@@ -18,9 +28,11 @@ def stage_runtime_package(server: Path) -> Path:
|
|
|
18
28
|
for package_dir in ("gatewaysdk", "gatewaysdk/environment"):
|
|
19
29
|
(server / package_dir / "__init__.py").write_text("")
|
|
20
30
|
for name in SIBLING_MODULES:
|
|
21
|
-
shutil.copy(
|
|
22
|
-
ENVIRONMENT_PACKAGE / name, server / "gatewaysdk" / "environment" / name
|
|
23
|
-
)
|
|
31
|
+
shutil.copy(ENVIRONMENT_PACKAGE / name, server / "gatewaysdk" / "environment" / name)
|
|
24
32
|
for module in sorted(SCHEMA_PACKAGE.glob("*.py")):
|
|
25
33
|
shutil.copy(module, package / module.name)
|
|
34
|
+
for name in SUBPACKAGES:
|
|
35
|
+
(package / name).mkdir()
|
|
36
|
+
for module in sorted((SCHEMA_PACKAGE / name).glob("*.py")):
|
|
37
|
+
shutil.copy(module, package / name / module.name)
|
|
26
38
|
return package
|
|
@@ -0,0 +1,291 @@
|
|
|
1
|
+
"""The call log: every call one session answered, by every door, in order."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import collections
|
|
6
|
+
import os
|
|
7
|
+
import threading
|
|
8
|
+
from dataclasses import dataclass
|
|
9
|
+
from datetime import datetime, timezone
|
|
10
|
+
from enum import StrEnum
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
from .call_redaction import MAX_LOGGED_CHARS, MAX_LOGGED_VALUES, REDACTED, CallRedaction, RequestBudget, RequestTooLarge
|
|
14
|
+
from .connector import Operation, Spec, _dig
|
|
15
|
+
from .tasks import WITHHELD_FIELD
|
|
16
|
+
|
|
17
|
+
DEFAULT_CALL_LOG_MAX = 5000
|
|
18
|
+
DEFAULT_CALL_LOG_VALUES = 200_000
|
|
19
|
+
DEFAULT_CALL_LOG_CHARS = 8 << 20
|
|
20
|
+
WITHHELD_ROUTE = {
|
|
21
|
+
"args": REDACTED,
|
|
22
|
+
"query": {},
|
|
23
|
+
"body": None,
|
|
24
|
+
"selectors": [],
|
|
25
|
+
WITHHELD_FIELD: ["path", "args", "query", "body", "selectors"],
|
|
26
|
+
}
|
|
27
|
+
WITHHELD_TOOL = {"args": REDACTED, "query": {}, "body": None, "selectors": [], WITHHELD_FIELD: ["args", "selectors"]}
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _children(node: Any) -> Any:
|
|
31
|
+
if isinstance(node, dict):
|
|
32
|
+
return node.values()
|
|
33
|
+
if isinstance(node, (list, tuple)):
|
|
34
|
+
return node
|
|
35
|
+
return ()
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _strings_in(node: Any) -> list[str]:
|
|
39
|
+
"""Every string in a JSON value, however nested, once each, in document order."""
|
|
40
|
+
found: dict[str, None] = {}
|
|
41
|
+
_collect_strings(node, found)
|
|
42
|
+
return list(found)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _collect_strings(node: Any, found: dict[str, None]) -> None:
|
|
46
|
+
if isinstance(node, str) and node != REDACTED:
|
|
47
|
+
found[node] = None
|
|
48
|
+
for value in _children(node):
|
|
49
|
+
_collect_strings(value, found)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _counted(found: Any) -> int | None:
|
|
53
|
+
if isinstance(found, list):
|
|
54
|
+
return len(found)
|
|
55
|
+
if isinstance(found, dict):
|
|
56
|
+
return 1 if found else 0
|
|
57
|
+
return 0 if found is None else None
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _found(status: int, rows: int | None) -> bool | None:
|
|
61
|
+
if status >= 400:
|
|
62
|
+
return False
|
|
63
|
+
return None if rows is None else rows > 0
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
class Via(StrEnum):
|
|
67
|
+
"""The door a call came in by, set by the server from where it arrived and
|
|
68
|
+
never from a request header. It names the door, not the actor: a token
|
|
69
|
+
holder calling the ui origin's prefix without a browser is ``ui``."""
|
|
70
|
+
|
|
71
|
+
API = "api"
|
|
72
|
+
UI = "ui"
|
|
73
|
+
TOOL = "tool"
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
@dataclass(frozen=True)
|
|
77
|
+
class AnswerShape:
|
|
78
|
+
"""Where an answer carries its rows, as the route declares it: a list is
|
|
79
|
+
its rows; under the envelope key the route serves, a list; at the
|
|
80
|
+
operation's results path, a list, one object (one row) or nothing (none);
|
|
81
|
+
a ``single`` route's object is one row. Nothing answered is no rows. Any
|
|
82
|
+
other answer (a tool's object, a handler's own shape) carries rows nobody
|
|
83
|
+
declared: unknown, ``None``."""
|
|
84
|
+
|
|
85
|
+
envelope: str = ""
|
|
86
|
+
results: str = ""
|
|
87
|
+
single: bool = False
|
|
88
|
+
|
|
89
|
+
def rows(self, answer: Any) -> int | None:
|
|
90
|
+
if isinstance(answer, list):
|
|
91
|
+
return len(answer)
|
|
92
|
+
if answer is None:
|
|
93
|
+
return 0
|
|
94
|
+
if not isinstance(answer, dict):
|
|
95
|
+
return None
|
|
96
|
+
listed = _dig(answer, self.envelope) if self.envelope else None
|
|
97
|
+
if isinstance(listed, list):
|
|
98
|
+
return len(listed)
|
|
99
|
+
if self.results:
|
|
100
|
+
return _counted(_dig(answer, self.results))
|
|
101
|
+
if self.single:
|
|
102
|
+
return 1 if answer else 0
|
|
103
|
+
return None
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
TOOL_ANSWER = AnswerShape()
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
class CallLog:
|
|
110
|
+
"""The calls one session answered, in order: what the agent asked (a
|
|
111
|
+
route's operation, method, path, path args, query and body; a tool's name
|
|
112
|
+
and arguments), the door it came in by (``via``) and what came back
|
|
113
|
+
(status; ``rows``, the rows the answer carried as :class:`AnswerShape`
|
|
114
|
+
reads them; elapsed). ``selectors`` is every string the request carried
|
|
115
|
+
wherever it sat, and ``found`` is "answered with rows", so a grader's
|
|
116
|
+
scalar ``where`` can ask either without the vendor's request shape.
|
|
117
|
+
:class:`CallRedaction` decides what of a request is kept. A request too
|
|
118
|
+
large or too deep to keep names what it lacks in ``withheld``: a route call
|
|
119
|
+
keeps its request line (path, path arguments, query), which HTTP bounds,
|
|
120
|
+
and withholds its body (``["body", "selectors"]``); a tool call withholds its
|
|
121
|
+
arguments. Bounded at ``maximum`` entries (``WORLDS_CALL_LOG_MAX``, default
|
|
122
|
+
:data:`DEFAULT_CALL_LOG_MAX`; ``0`` keeps none) and at what its entries hold,
|
|
123
|
+
``max_values`` values and ``max_chars`` characters (the sum of their
|
|
124
|
+
:class:`RequestBudget` spends; about 60 MB of memory at most), the oldest
|
|
125
|
+
dropped first; ``total`` counts past the bound, so the first kept ``seq``
|
|
126
|
+
says how many are gone. Every session's ``state`` and grader state carry it
|
|
127
|
+
(:meth:`into_state`), ``serve()`` answers it at ``/__world/calls`` to its admin
|
|
128
|
+
token, a checkpoint carries it (:meth:`resume`), and a task's
|
|
129
|
+
``verify.py`` is handed it
|
|
130
|
+
as ``calls`` beside the rows, because a read-only investigation leaves no
|
|
131
|
+
row behind and its calls are the only evidence of what it did."""
|
|
132
|
+
|
|
133
|
+
def __init__(
|
|
134
|
+
self,
|
|
135
|
+
maximum: int | None = None,
|
|
136
|
+
redaction: CallRedaction | None = None,
|
|
137
|
+
max_values: int = DEFAULT_CALL_LOG_VALUES,
|
|
138
|
+
max_chars: int = DEFAULT_CALL_LOG_CHARS,
|
|
139
|
+
):
|
|
140
|
+
if maximum is None:
|
|
141
|
+
raw = os.environ.get("WORLDS_CALL_LOG_MAX")
|
|
142
|
+
maximum = int(raw) if raw else DEFAULT_CALL_LOG_MAX
|
|
143
|
+
self.maximum = max(0, int(maximum))
|
|
144
|
+
self.max_values = max_values
|
|
145
|
+
self.max_chars = max_chars
|
|
146
|
+
self.redaction = redaction or CallRedaction()
|
|
147
|
+
self.entries: collections.deque[dict] = collections.deque()
|
|
148
|
+
self._costs: collections.deque[tuple[int, int]] = collections.deque()
|
|
149
|
+
self.values = 0
|
|
150
|
+
self.chars = 0
|
|
151
|
+
self.total = 0
|
|
152
|
+
self._lock = threading.Lock()
|
|
153
|
+
|
|
154
|
+
@classmethod
|
|
155
|
+
def for_spec(cls, spec: Spec) -> CallLog:
|
|
156
|
+
return cls(redaction=CallRedaction.of(spec))
|
|
157
|
+
|
|
158
|
+
def record(
|
|
159
|
+
self,
|
|
160
|
+
operation: Operation,
|
|
161
|
+
method: str,
|
|
162
|
+
path: str,
|
|
163
|
+
args: dict,
|
|
164
|
+
query_all: dict,
|
|
165
|
+
body: Any,
|
|
166
|
+
status: int,
|
|
167
|
+
answer: Any,
|
|
168
|
+
elapsed_s: float,
|
|
169
|
+
via: Via | None,
|
|
170
|
+
*,
|
|
171
|
+
shape: AnswerShape,
|
|
172
|
+
) -> None:
|
|
173
|
+
if self.maximum == 0:
|
|
174
|
+
return
|
|
175
|
+
door = {} if via is None else {"via": str(via)}
|
|
176
|
+
heading = {"operation": operation.name, "method": method}
|
|
177
|
+
try:
|
|
178
|
+
request, budget = self._route_request(operation, path, args, query_all, body)
|
|
179
|
+
except (RecursionError, RequestTooLarge):
|
|
180
|
+
request, budget = self._route_line(operation, path, args, query_all)
|
|
181
|
+
self._append({**heading, **request, **door}, status, shape.rows(answer), elapsed_s, budget)
|
|
182
|
+
|
|
183
|
+
def _route_request(
|
|
184
|
+
self, operation: Operation, path: str, args: dict, query_all: dict, body: Any
|
|
185
|
+
) -> tuple[dict, RequestBudget]:
|
|
186
|
+
budget = RequestBudget()
|
|
187
|
+
line = self._request_line(operation, path, args, query_all, budget)
|
|
188
|
+
logged_body = self.redaction.value(body, budget) if isinstance(body, (dict, list)) and body else None
|
|
189
|
+
selectors = _strings_in([line["args"], line["query"], logged_body])
|
|
190
|
+
return {**line, "body": logged_body, "selectors": selectors}, budget
|
|
191
|
+
|
|
192
|
+
def _route_line(self, operation: Operation, path: str, args: dict, query_all: dict) -> tuple[dict, RequestBudget]:
|
|
193
|
+
budget = RequestBudget()
|
|
194
|
+
try:
|
|
195
|
+
line = self._request_line(operation, path, args, query_all, budget)
|
|
196
|
+
except (RecursionError, RequestTooLarge):
|
|
197
|
+
return {"path": operation.path, **WITHHELD_ROUTE}, RequestBudget()
|
|
198
|
+
selectors = _strings_in([line["args"], line["query"]])
|
|
199
|
+
return {**line, "body": None, "selectors": selectors, WITHHELD_FIELD: ["body", "selectors"]}, budget
|
|
200
|
+
|
|
201
|
+
def _request_line(
|
|
202
|
+
self, operation: Operation, path: str, args: dict, query_all: dict, budget: RequestBudget
|
|
203
|
+
) -> dict:
|
|
204
|
+
redaction = self.redaction
|
|
205
|
+
logged_path = redaction.path(operation, path, args)
|
|
206
|
+
budget.spend(1, len(logged_path))
|
|
207
|
+
return {
|
|
208
|
+
"path": logged_path,
|
|
209
|
+
"args": redaction.value(dict(args), budget),
|
|
210
|
+
"query": redaction.query(query_all, budget),
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
def record_tool(self, tool: str, args: Any, status: int, answer: Any, elapsed_s: float) -> None:
|
|
214
|
+
if self.maximum == 0:
|
|
215
|
+
return
|
|
216
|
+
heading = {"operation": tool, "method": None, "path": None}
|
|
217
|
+
try:
|
|
218
|
+
request, budget = self._tool_request(args)
|
|
219
|
+
except (RecursionError, RequestTooLarge):
|
|
220
|
+
request, budget = dict(WITHHELD_TOOL), RequestBudget()
|
|
221
|
+
self._append({**heading, **request, "via": str(Via.TOOL)}, status, TOOL_ANSWER.rows(answer), elapsed_s, budget)
|
|
222
|
+
|
|
223
|
+
def _tool_request(self, args: Any) -> tuple[dict, RequestBudget]:
|
|
224
|
+
budget = RequestBudget()
|
|
225
|
+
logged_args = self.redaction.value(args, budget)
|
|
226
|
+
return {"args": logged_args, "query": {}, "body": None, "selectors": _strings_in(logged_args)}, budget
|
|
227
|
+
|
|
228
|
+
def _append(self, request: dict, status: int, rows: int | None, elapsed_s: float, budget: RequestBudget) -> None:
|
|
229
|
+
rows = None if status >= 400 else rows
|
|
230
|
+
answered = {
|
|
231
|
+
"status": status,
|
|
232
|
+
"rows": rows,
|
|
233
|
+
"found": _found(status, rows),
|
|
234
|
+
"elapsed_ms": round(elapsed_s * 1000, 1),
|
|
235
|
+
}
|
|
236
|
+
at = datetime.now(timezone.utc).isoformat(timespec="milliseconds").replace("+00:00", "Z")
|
|
237
|
+
cost = budget.spent()
|
|
238
|
+
with self._lock:
|
|
239
|
+
self.total += 1
|
|
240
|
+
self._keep({"seq": self.total, "at": at, **request, **answered}, cost)
|
|
241
|
+
|
|
242
|
+
def _keep(self, entry: dict, cost: tuple[int, int]) -> None:
|
|
243
|
+
self.entries.append(entry)
|
|
244
|
+
self._costs.append(cost)
|
|
245
|
+
self.values += cost[0]
|
|
246
|
+
self.chars += cost[1]
|
|
247
|
+
while len(self.entries) > self.maximum or (len(self.entries) > 1 and self._over_budget()):
|
|
248
|
+
self.entries.popleft()
|
|
249
|
+
values, chars = self._costs.popleft()
|
|
250
|
+
self.values -= values
|
|
251
|
+
self.chars -= chars
|
|
252
|
+
|
|
253
|
+
def _over_budget(self) -> bool:
|
|
254
|
+
return self.values > self.max_values or self.chars > self.max_chars
|
|
255
|
+
|
|
256
|
+
def _held(self, call: dict) -> tuple[int, int]:
|
|
257
|
+
budget = RequestBudget()
|
|
258
|
+
try:
|
|
259
|
+
self.redaction.value([call.get(field) for field in ("path", "args", "query", "body")], budget)
|
|
260
|
+
except (RecursionError, RequestTooLarge):
|
|
261
|
+
return MAX_LOGGED_VALUES, MAX_LOGGED_CHARS
|
|
262
|
+
return budget.spent()
|
|
263
|
+
|
|
264
|
+
def describe(self) -> dict:
|
|
265
|
+
with self._lock:
|
|
266
|
+
return {"total": self.total, "kept": len(self.entries), "max": self.maximum, "calls": list(self.entries)}
|
|
267
|
+
|
|
268
|
+
def into_state(self, state: dict) -> dict:
|
|
269
|
+
"""``state`` with the calls beside its rows, as every session's ``state`` and grader state carry them."""
|
|
270
|
+
state["calls"] = self.describe()["calls"]
|
|
271
|
+
return state
|
|
272
|
+
|
|
273
|
+
def resume(self, calls: list[dict]) -> None:
|
|
274
|
+
"""Carry on from the calls a checkpoint kept: the same entries, numbered on
|
|
275
|
+
from the last of them, each charged what it holds. A log that keeps
|
|
276
|
+
nothing keeps none of them either."""
|
|
277
|
+
if self.maximum == 0:
|
|
278
|
+
return
|
|
279
|
+
costs = [self._held(call) for call in calls]
|
|
280
|
+
with self._lock:
|
|
281
|
+
for call, cost in zip(calls, costs):
|
|
282
|
+
self._keep(call, cost)
|
|
283
|
+
self.total = max([self.total, *(call["seq"] for call in calls)])
|
|
284
|
+
|
|
285
|
+
def clear(self) -> None:
|
|
286
|
+
with self._lock:
|
|
287
|
+
self.entries.clear()
|
|
288
|
+
self._costs.clear()
|
|
289
|
+
self.values = 0
|
|
290
|
+
self.chars = 0
|
|
291
|
+
self.total = 0
|