@withgateway/sdk 0.3.1 → 0.3.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (129) hide show
  1. package/README.md +4 -3
  2. package/dist/cli.js +2880 -2032
  3. package/dist/cli.js.map +1 -1
  4. package/dist/cli.mjs +2982 -2135
  5. package/dist/cli.mjs.map +1 -1
  6. package/dist/files/index.d.mts +1 -1
  7. package/dist/files/index.d.ts +1 -1
  8. package/dist/files/index.js +180 -41
  9. package/dist/files/index.js.map +1 -1
  10. package/dist/files/index.mjs +172 -35
  11. package/dist/files/index.mjs.map +1 -1
  12. package/dist/{index-CBqbB_0J.d.ts → index-DUHmS74p.d.ts} +1 -1
  13. package/dist/{index-D-AMnAMa.d.mts → index-mJhG3vZU.d.mts} +1 -1
  14. package/dist/index.d.mts +4 -4
  15. package/dist/index.d.ts +4 -4
  16. package/dist/index.js +1309 -540
  17. package/dist/index.js.map +1 -1
  18. package/dist/index.mjs +1287 -532
  19. package/dist/index.mjs.map +1 -1
  20. package/dist/{media-CbcAfQvd.d.mts → media-BIApiPzS.d.mts} +21 -2
  21. package/dist/{media-CbcAfQvd.d.ts → media-BIApiPzS.d.ts} +21 -2
  22. package/dist/traces/index.d.mts +1 -1
  23. package/dist/traces/index.d.ts +1 -1
  24. package/dist/traces/index.js +177 -38
  25. package/dist/traces/index.js.map +1 -1
  26. package/dist/traces/index.mjs +169 -32
  27. package/dist/traces/index.mjs.map +1 -1
  28. package/dist/tracing/index.d.mts +2 -2
  29. package/dist/tracing/index.d.ts +2 -2
  30. package/dist/tracing/index.js +69 -13
  31. package/dist/tracing/index.js.map +1 -1
  32. package/dist/tracing/index.mjs +69 -13
  33. package/dist/tracing/index.mjs.map +1 -1
  34. package/dist/worlds/index.d.mts +249 -120
  35. package/dist/worlds/index.d.ts +249 -120
  36. package/dist/worlds/index.js +1240 -484
  37. package/dist/worlds/index.js.map +1 -1
  38. package/dist/worlds/index.mjs +1231 -476
  39. package/dist/worlds/index.mjs.map +1 -1
  40. package/package.json +1 -1
  41. package/runtime/gatewaysdk/environment/_bundle.py +2 -27
  42. package/runtime/gatewaysdk/environment/_materialize.py +49 -69
  43. package/runtime/gatewaysdk/environment/_world.py +122 -197
  44. package/runtime/gatewaysdk/environment/backing.py +137 -0
  45. package/runtime/gatewaysdk/environment/core.py +1 -1
  46. package/runtime/gatewaysdk/environment/legacy_sqlite.py +301 -0
  47. package/runtime/gatewaysdk/environment/reward_keys.py +25 -0
  48. package/runtime/gatewaysdk/environment/runtime.py +8 -13
  49. package/runtime/gatewaysdk/environment/schema/__init__.py +4 -4
  50. package/runtime/gatewaysdk/environment/schema/admin_routes.py +86 -0
  51. package/runtime/gatewaysdk/environment/schema/api.py +266 -514
  52. package/runtime/gatewaysdk/environment/schema/batch.py +42 -37
  53. package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +17 -5
  54. package/runtime/gatewaysdk/environment/schema/call_log.py +291 -0
  55. package/runtime/gatewaysdk/environment/schema/call_redaction.py +344 -0
  56. package/runtime/gatewaysdk/environment/schema/compiler.py +334 -158
  57. package/runtime/gatewaysdk/environment/schema/conform.py +8 -12
  58. package/runtime/gatewaysdk/environment/schema/connector.py +27 -62
  59. package/runtime/gatewaysdk/environment/schema/data.py +50 -207
  60. package/runtime/gatewaysdk/environment/schema/database/__init__.py +24 -0
  61. package/runtime/gatewaysdk/environment/schema/database/audit.py +121 -0
  62. package/runtime/gatewaysdk/environment/schema/database/base.py +995 -0
  63. package/runtime/gatewaysdk/environment/schema/database/budget.py +33 -0
  64. package/runtime/gatewaysdk/environment/schema/database/connections.py +101 -0
  65. package/runtime/gatewaysdk/environment/schema/database/engines.py +8 -0
  66. package/runtime/gatewaysdk/environment/schema/database/guard.py +208 -0
  67. package/runtime/gatewaysdk/environment/schema/database/ingest.py +154 -0
  68. package/runtime/gatewaysdk/environment/schema/database/matching.py +58 -0
  69. package/runtime/gatewaysdk/environment/schema/database/provenance.py +30 -0
  70. package/runtime/gatewaysdk/environment/schema/database/rows.py +53 -0
  71. package/runtime/gatewaysdk/environment/schema/database/runtime_rows.py +39 -0
  72. package/runtime/gatewaysdk/environment/schema/database/sqlite.py +367 -0
  73. package/runtime/gatewaysdk/environment/schema/domain_migration.py +236 -0
  74. package/runtime/gatewaysdk/environment/schema/envelope.py +169 -0
  75. package/runtime/gatewaysdk/environment/schema/extract.py +6 -13
  76. package/runtime/gatewaysdk/environment/schema/grading.py +45 -30
  77. package/runtime/gatewaysdk/environment/schema/handler_errors.py +142 -12
  78. package/runtime/gatewaysdk/environment/schema/host.py +158 -250
  79. package/runtime/gatewaysdk/environment/schema/lanes.py +53 -0
  80. package/runtime/gatewaysdk/environment/schema/local.py +44 -71
  81. package/runtime/gatewaysdk/environment/schema/mapping.py +5 -25
  82. package/runtime/gatewaysdk/environment/schema/openapi_import.py +112 -73
  83. package/runtime/gatewaysdk/environment/schema/openapi_paging.py +147 -0
  84. package/runtime/gatewaysdk/environment/schema/openapi_payloads.py +288 -0
  85. package/runtime/gatewaysdk/environment/schema/openapi_routes.py +97 -102
  86. package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +113 -30
  87. package/runtime/gatewaysdk/environment/schema/platform.py +123 -171
  88. package/runtime/gatewaysdk/environment/schema/platform_import.py +2 -3
  89. package/runtime/gatewaysdk/environment/schema/platform_query.py +61 -5
  90. package/runtime/gatewaysdk/environment/schema/primary_key.py +61 -0
  91. package/runtime/gatewaysdk/environment/schema/proof.py +19 -12
  92. package/runtime/gatewaysdk/environment/schema/runtime.py +77 -160
  93. package/runtime/gatewaysdk/environment/schema/served_names.py +58 -0
  94. package/runtime/gatewaysdk/environment/schema/session_files.py +166 -0
  95. package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +2 -2
  96. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +19 -11
  97. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +29 -3
  98. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +14 -14
  99. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +12 -12
  100. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +3 -2
  101. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +15 -0
  102. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/openapi-import.md +50 -21
  103. package/runtime/gatewaysdk/environment/schema/slack.py +4 -8
  104. package/runtime/gatewaysdk/environment/schema/snapshot.py +385 -129
  105. package/runtime/gatewaysdk/environment/schema/store_contract.py +8 -14
  106. package/runtime/gatewaysdk/environment/schema/tasks.py +140 -55
  107. package/runtime/gatewaysdk/environment/schema/tools_world.py +110 -87
  108. package/runtime/gatewaysdk/environment/schema/ui.py +24 -5
  109. package/runtime/gatewaysdk/environment/schema/ui_static/app.js +100 -130
  110. package/runtime/gatewaysdk/environment/schema/ui_static/index.html +1 -8
  111. package/runtime/gatewaysdk/environment/schema/validation.py +166 -40
  112. package/runtime/gatewaysdk/environment/schema/workers.py +142 -57
  113. package/runtime/gatewaysdk/environment/schema/world_sql.py +81 -0
  114. package/runtime/gatewaysdk/environment/schema/world_tests.py +71 -39
  115. package/runtime/gatewaysdk/environment/score.py +57 -0
  116. package/runtime/gatewaysdk/environment/secret_values.py +36 -0
  117. package/runtime/gatewaysdk/environment/world.py +42 -3
  118. package/runtime/gatewaysdk/environment/world_session_contract.py +32 -5
  119. package/skills/benchmarks-getting-started/SKILL.md +2 -2
  120. package/skills/connector-schema-authoring/SKILL.md +19 -11
  121. package/skills/connector-schema-authoring/connector-worlds.md +13 -4
  122. package/skills/connector-schema-authoring/world-schema.md +103 -51
  123. package/skills/world-data-ingestion/ROWS.md +29 -3
  124. package/skills/world-data-ingestion/SKILL.md +14 -14
  125. package/skills/worlds-getting-started/SKILL.md +12 -12
  126. package/skills/worlds-getting-started/references/cli-mcp-map.md +3 -2
  127. package/skills/worlds-getting-started/references/gotchas.md +15 -0
  128. package/skills/worlds-getting-started/references/openapi-import.md +50 -21
  129. package/runtime/gatewaysdk/environment/schema/store.py +0 -1013
@@ -23,13 +23,14 @@ import json
23
23
  import sys
24
24
  from dataclasses import dataclass, field
25
25
  from pathlib import Path
26
- from typing import IO, Any, Iterable, Iterator
26
+ from typing import IO, Any, Iterable, Iterator, NoReturn
27
27
 
28
28
  from .cast import TEXT_CASTER, EntityCasters
29
- from .compiler import Contract, quote
29
+ from .compiler import Contract, payload_value
30
30
  from .csv_rows import csv_delimiter, is_delimited_name, iter_csv_records
31
+ from .primary_key import PrimaryKey
31
32
  from .redaction import RowRedactor
32
- from .store import SQLiteStore
33
+ from .database import WorldDatabase
33
34
  from .validation import ContractError, fail, validate
34
35
 
35
36
  CHUNK_BYTES = 64 * 1024 * 1024
@@ -45,9 +46,6 @@ def encode_row(value: Any) -> str:
45
46
  return json.dumps(value, ensure_ascii=False, separators=(",", ":"), allow_nan=False)
46
47
 
47
48
 
48
- # ---------------------------------------------------------------- streaming rows
49
-
50
-
51
49
  class _Reader:
52
50
  """A text stream with a growing buffer, for decoding JSON values one at a time.
53
51
 
@@ -256,7 +254,6 @@ def iter_file_rows(
256
254
  seen[name] = seen.get(name, 0) + 1
257
255
  yield name, row
258
256
  return
259
- # A document: rewind to its first character and stream it.
260
257
  reader.pos = start
261
258
  yield from _stream_document(reader, where, entity, seen)
262
259
 
@@ -337,9 +334,6 @@ def count_rows(
337
334
  return total
338
335
 
339
336
 
340
- # ---------------------------------------------------------------- chunks
341
-
342
-
343
337
  @dataclass
344
338
  class Chunk:
345
339
  ordinal: int
@@ -350,7 +344,7 @@ class Chunk:
350
344
  entity: str | None
351
345
  content_bytes: int # the JSONL before gzip
352
346
  content_sha256: str
353
- first_row: int # index of the chunk's first row in the input stream
347
+ first_row: int
354
348
 
355
349
  def manifest_entry(self) -> dict:
356
350
  return {
@@ -464,9 +458,6 @@ def iter_chunk_rows(path: str | Path, entity: str | None = None) -> Iterator[tup
464
458
  fail("rows", f"Cannot read {path.name} as UTF-8 text", str(path))
465
459
 
466
460
 
467
- # ---------------------------------------------------------------- refusals
468
-
469
-
470
461
  def _field_of(error: ContractError) -> str | None:
471
462
  path = error.detail.get("path") or ""
472
463
  inside = path[1:] if path.startswith("/") and len(path) > 1 else None
@@ -489,14 +480,13 @@ def refusal(
489
480
  them all, ``field`` the property refused (None for a row-level refusal)."""
490
481
  key = None
491
482
  if contract is not None and isinstance(row, dict) and entity in contract.entities:
492
- fields = contract.entities[entity]["primary_key"]
493
- if all(name in row for name in fields):
494
- key = {name: row[name] for name in fields}
483
+ primary = PrimaryKey.in_contract(contract, entity)
484
+ key = primary.key(row) if primary.missing(row) is None else None
495
485
  pointer = "/" + (entity or "")
496
486
  if index is not None:
497
487
  pointer += f"/{index}"
498
488
  if field_name:
499
- pointer += f"/{field_name}"
489
+ pointer += field_name if field_name.startswith("/") else f"/{field_name}"
500
490
  return {
501
491
  "chunk": chunk,
502
492
  "index": index,
@@ -523,9 +513,9 @@ def shape_refusal(contract: Contract, chunk: int, index: int, entity: str, row:
523
513
  if not isinstance(row, dict):
524
514
  return refusal(chunk, index, entity, row, None, "A row must be an object")
525
515
  spec = contract.entities[entity]
526
- for name in spec["primary_key"]:
527
- if name not in row:
528
- return refusal(chunk, index, entity, row, name, f"Row is missing primary key field {name!r}", contract)
516
+ missing = PrimaryKey.of(spec).missing(row)
517
+ if missing is not None:
518
+ return refusal(chunk, index, entity, row, missing, f"Row is missing primary key field {missing!r}", contract)
529
519
  try:
530
520
  validate(spec["schema"], row, location=f"{spec['location']}/schema")
531
521
  except ContractError as error:
@@ -533,6 +523,26 @@ def shape_refusal(contract: Contract, chunk: int, index: int, entity: str, row:
533
523
  return None
534
524
 
535
525
 
526
+ def raise_shape_refusal(
527
+ contract: Contract, first: dict, row: Any, summary: str, context: dict, unknown_at: str
528
+ ) -> NoReturn:
529
+ """The first :func:`shape_refusal` of a file as the ``ContractError`` the store
530
+ raises for it, under its own code (``entity``, ``rows``, ``primary_key``, or the
531
+ validator's), with ``summary`` before its message and ``context`` attached."""
532
+ spec = contract.entities.get(first["entity"])
533
+ if spec is None:
534
+ fail("entity", summary + first["detail"], unknown_at, **context)
535
+ if not isinstance(row, dict):
536
+ fail("rows", summary + first["detail"], first["pointer"], **context)
537
+ if PrimaryKey.of(spec).missing(row) is not None:
538
+ fail("primary_key", summary + first["detail"], first["pointer"], **context)
539
+ try:
540
+ validate(spec["schema"], row, location=f"{spec['location']}/schema")
541
+ except ContractError as error:
542
+ raise error.contextualize(message=summary + error.detail["message"], **context)
543
+ fail("rows", summary + first["detail"], first["pointer"], **context)
544
+
545
+
536
546
  def write_report(records: Iterable[dict], destination: IO[str], *, cap: int = REPORT_CAP) -> int:
537
547
  """Refusal records as JSONL, at most ``cap`` of them followed by
538
548
  ``{"truncated": true, "total": N}`` when there were more. Returns the total."""
@@ -546,9 +556,6 @@ def write_report(records: Iterable[dict], destination: IO[str], *, cap: int = RE
546
556
  return total
547
557
 
548
558
 
549
- # ---------------------------------------------------------------- batches
550
-
551
-
552
559
  @dataclass
553
560
  class ChunkSpec:
554
561
  ordinal: int
@@ -560,8 +567,8 @@ class ChunkSpec:
560
567
  class _Staged:
561
568
  """The valid rows of a batch after the in-batch merge, keyed the way the store keys."""
562
569
 
563
- rows: dict[str, dict[str, dict]] = field(default_factory=dict) # entity -> key -> row
564
- origin: dict[str, dict[str, tuple[int, int]]] = field(default_factory=dict) # entity -> key -> (chunk, index)
570
+ rows: dict[str, dict[str, dict]] = field(default_factory=dict)
571
+ origin: dict[str, dict[str, tuple[int, int]]] = field(default_factory=dict)
565
572
  refusals: list[dict] = field(default_factory=list)
566
573
  read: int = 0
567
574
 
@@ -602,7 +609,7 @@ def _redacted(
602
609
 
603
610
  def _stage(
604
611
  contract: Contract,
605
- store: SQLiteStore,
612
+ store: WorldDatabase,
606
613
  chunks: Iterable[ChunkSpec],
607
614
  merge: str,
608
615
  staged: _Staged | None = None,
@@ -621,7 +628,7 @@ def _stage(
621
628
  if refused is not None:
622
629
  staged.refusals.append(refused)
623
630
  continue
624
- key = store._key(entity, row)
631
+ key = store.key_of(entity, row)
625
632
  held = staged.rows.setdefault(entity, {})
626
633
  origin = staged.origin.setdefault(entity, {})
627
634
  if key in held and merge == "fill":
@@ -635,7 +642,7 @@ def _stage(
635
642
  return staged
636
643
 
637
644
 
638
- def _reference_refusals(contract: Contract, store: SQLiteStore, staged: _Staged, mode: str) -> list[dict]:
645
+ def _reference_refusals(contract: Contract, store: WorldDatabase, staged: _Staged, mode: str) -> list[dict]:
639
646
  """Every row whose relationship points at a key neither the parent snapshot
640
647
  (minus replaced entities) nor the batch holds."""
641
648
  refusals: list[dict] = []
@@ -645,8 +652,7 @@ def _reference_refusals(contract: Contract, store: SQLiteStore, staged: _Staged,
645
652
  if target not in known:
646
653
  held = set(staged.rows.get(target, {}))
647
654
  if not (mode == "replace" and target in staged.rows):
648
- with store._trusted():
649
- held |= {row[0] for row in store._conn.execute(f"SELECT _key FROM {quote(target)}")}
655
+ held |= store.keys(target)
650
656
  known[target] = held
651
657
  return known[target]
652
658
 
@@ -657,7 +663,7 @@ def _reference_refusals(contract: Contract, store: SQLiteStore, staged: _Staged,
657
663
  dropped = []
658
664
  for key, row in rows.items():
659
665
  for rel in relationships:
660
- values = [row.get(name) for name in rel["fields"]]
666
+ values = [payload_value(row, name) for name in rel["fields"]]
661
667
  if all(value is None for value in values):
662
668
  continue
663
669
  chunk, index = staged.origin[entity][key]
@@ -677,7 +683,7 @@ def _reference_refusals(contract: Contract, store: SQLiteStore, staged: _Staged,
677
683
  break
678
684
  target = rel["target"]
679
685
  try:
680
- wanted = store._key(target, dict(zip(contract.entities[target]["primary_key"], values)))
686
+ wanted = store.key_of(target, dict(zip(contract.entities[target]["primary_key"], values)))
681
687
  except ContractError as error:
682
688
  refusals.append(
683
689
  refusal(chunk, index, entity, row, rel["fields"][0], error.detail["message"], contract)
@@ -704,7 +710,7 @@ def _reference_refusals(contract: Contract, store: SQLiteStore, staged: _Staged,
704
710
 
705
711
 
706
712
  def _apply(
707
- contract: Contract, store: SQLiteStore, staged: _Staged, mode: str, batch_id: str
713
+ contract: Contract, store: WorldDatabase, staged: _Staged, mode: str, batch_id: str
708
714
  ) -> tuple[int, dict | None]:
709
715
  """Apply the staged rows as one batch; the count applied, or the store's refusal."""
710
716
  from .data import seed_batch
@@ -779,7 +785,7 @@ def build_batch(
779
785
  applied_chunks: list[int] = []
780
786
  applied_digests: list[str] = []
781
787
  counts: dict[str, int] = {}
782
- with SQLiteStore.open(parent, contract, audit=False) as store:
788
+ with WorldDatabase.open(parent, contract, audit=False) as store:
783
789
  if atomic:
784
790
  staged = _stage(contract, store, chunks, merge, redactor=redactor)
785
791
  refusals.extend(staged.refusals)
@@ -800,8 +806,7 @@ def build_batch(
800
806
  found.append(refused)
801
807
  refusals.extend(found)
802
808
  if found and not dry_run:
803
- # The store rolled the chunk's transaction back; the rows it did
804
- # accept are gone with it, so the chunk is refused as a whole.
809
+ # The store rolled the chunk's transaction back, so the chunk is refused as a whole.
805
810
  continue
806
811
  if not dry_run:
807
812
  applied_rows += applied
@@ -7,8 +7,18 @@ from pathlib import Path
7
7
 
8
8
  SCHEMA_PACKAGE = Path(__file__).parent
9
9
  ENVIRONMENT_PACKAGE = SCHEMA_PACKAGE.parent
10
- # Modules outside `schema/` that its modules import; copied beside them so `main.py` resolves them.
11
- SIBLING_MODULES = ("_toml.py", "world_session_contract.py")
10
+ SIBLING_MODULES = (
11
+ "_toml.py",
12
+ "backing.py",
13
+ "errors.py",
14
+ "legacy_sqlite.py",
15
+ "reward_keys.py",
16
+ "score.py",
17
+ "secret_values.py",
18
+ "world.py",
19
+ "world_session_contract.py",
20
+ )
21
+ SUBPACKAGES = ("database",)
12
22
 
13
23
 
14
24
  def stage_runtime_package(server: Path) -> Path:
@@ -18,9 +28,11 @@ def stage_runtime_package(server: Path) -> Path:
18
28
  for package_dir in ("gatewaysdk", "gatewaysdk/environment"):
19
29
  (server / package_dir / "__init__.py").write_text("")
20
30
  for name in SIBLING_MODULES:
21
- shutil.copy(
22
- ENVIRONMENT_PACKAGE / name, server / "gatewaysdk" / "environment" / name
23
- )
31
+ shutil.copy(ENVIRONMENT_PACKAGE / name, server / "gatewaysdk" / "environment" / name)
24
32
  for module in sorted(SCHEMA_PACKAGE.glob("*.py")):
25
33
  shutil.copy(module, package / module.name)
34
+ for name in SUBPACKAGES:
35
+ (package / name).mkdir()
36
+ for module in sorted((SCHEMA_PACKAGE / name).glob("*.py")):
37
+ shutil.copy(module, package / name / module.name)
26
38
  return package
@@ -0,0 +1,291 @@
1
+ """The call log: every call one session answered, by every door, in order."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import collections
6
+ import os
7
+ import threading
8
+ from dataclasses import dataclass
9
+ from datetime import datetime, timezone
10
+ from enum import StrEnum
11
+ from typing import Any
12
+
13
+ from .call_redaction import MAX_LOGGED_CHARS, MAX_LOGGED_VALUES, REDACTED, CallRedaction, RequestBudget, RequestTooLarge
14
+ from .connector import Operation, Spec, _dig
15
+ from .tasks import WITHHELD_FIELD
16
+
17
+ DEFAULT_CALL_LOG_MAX = 5000
18
+ DEFAULT_CALL_LOG_VALUES = 200_000
19
+ DEFAULT_CALL_LOG_CHARS = 8 << 20
20
+ WITHHELD_ROUTE = {
21
+ "args": REDACTED,
22
+ "query": {},
23
+ "body": None,
24
+ "selectors": [],
25
+ WITHHELD_FIELD: ["path", "args", "query", "body", "selectors"],
26
+ }
27
+ WITHHELD_TOOL = {"args": REDACTED, "query": {}, "body": None, "selectors": [], WITHHELD_FIELD: ["args", "selectors"]}
28
+
29
+
30
+ def _children(node: Any) -> Any:
31
+ if isinstance(node, dict):
32
+ return node.values()
33
+ if isinstance(node, (list, tuple)):
34
+ return node
35
+ return ()
36
+
37
+
38
+ def _strings_in(node: Any) -> list[str]:
39
+ """Every string in a JSON value, however nested, once each, in document order."""
40
+ found: dict[str, None] = {}
41
+ _collect_strings(node, found)
42
+ return list(found)
43
+
44
+
45
+ def _collect_strings(node: Any, found: dict[str, None]) -> None:
46
+ if isinstance(node, str) and node != REDACTED:
47
+ found[node] = None
48
+ for value in _children(node):
49
+ _collect_strings(value, found)
50
+
51
+
52
+ def _counted(found: Any) -> int | None:
53
+ if isinstance(found, list):
54
+ return len(found)
55
+ if isinstance(found, dict):
56
+ return 1 if found else 0
57
+ return 0 if found is None else None
58
+
59
+
60
+ def _found(status: int, rows: int | None) -> bool | None:
61
+ if status >= 400:
62
+ return False
63
+ return None if rows is None else rows > 0
64
+
65
+
66
+ class Via(StrEnum):
67
+ """The door a call came in by, set by the server from where it arrived and
68
+ never from a request header. It names the door, not the actor: a token
69
+ holder calling the ui origin's prefix without a browser is ``ui``."""
70
+
71
+ API = "api"
72
+ UI = "ui"
73
+ TOOL = "tool"
74
+
75
+
76
+ @dataclass(frozen=True)
77
+ class AnswerShape:
78
+ """Where an answer carries its rows, as the route declares it: a list is
79
+ its rows; under the envelope key the route serves, a list; at the
80
+ operation's results path, a list, one object (one row) or nothing (none);
81
+ a ``single`` route's object is one row. Nothing answered is no rows. Any
82
+ other answer (a tool's object, a handler's own shape) carries rows nobody
83
+ declared: unknown, ``None``."""
84
+
85
+ envelope: str = ""
86
+ results: str = ""
87
+ single: bool = False
88
+
89
+ def rows(self, answer: Any) -> int | None:
90
+ if isinstance(answer, list):
91
+ return len(answer)
92
+ if answer is None:
93
+ return 0
94
+ if not isinstance(answer, dict):
95
+ return None
96
+ listed = _dig(answer, self.envelope) if self.envelope else None
97
+ if isinstance(listed, list):
98
+ return len(listed)
99
+ if self.results:
100
+ return _counted(_dig(answer, self.results))
101
+ if self.single:
102
+ return 1 if answer else 0
103
+ return None
104
+
105
+
106
+ TOOL_ANSWER = AnswerShape()
107
+
108
+
109
+ class CallLog:
110
+ """The calls one session answered, in order: what the agent asked (a
111
+ route's operation, method, path, path args, query and body; a tool's name
112
+ and arguments), the door it came in by (``via``) and what came back
113
+ (status; ``rows``, the rows the answer carried as :class:`AnswerShape`
114
+ reads them; elapsed). ``selectors`` is every string the request carried
115
+ wherever it sat, and ``found`` is "answered with rows", so a grader's
116
+ scalar ``where`` can ask either without the vendor's request shape.
117
+ :class:`CallRedaction` decides what of a request is kept. A request too
118
+ large or too deep to keep names what it lacks in ``withheld``: a route call
119
+ keeps its request line (path, path arguments, query), which HTTP bounds,
120
+ and withholds its body (``["body", "selectors"]``); a tool call withholds its
121
+ arguments. Bounded at ``maximum`` entries (``WORLDS_CALL_LOG_MAX``, default
122
+ :data:`DEFAULT_CALL_LOG_MAX`; ``0`` keeps none) and at what its entries hold,
123
+ ``max_values`` values and ``max_chars`` characters (the sum of their
124
+ :class:`RequestBudget` spends; about 60 MB of memory at most), the oldest
125
+ dropped first; ``total`` counts past the bound, so the first kept ``seq``
126
+ says how many are gone. Every session's ``state`` and grader state carry it
127
+ (:meth:`into_state`), ``serve()`` answers it at ``/__world/calls`` to its admin
128
+ token, a checkpoint carries it (:meth:`resume`), and a task's
129
+ ``verify.py`` is handed it
130
+ as ``calls`` beside the rows, because a read-only investigation leaves no
131
+ row behind and its calls are the only evidence of what it did."""
132
+
133
+ def __init__(
134
+ self,
135
+ maximum: int | None = None,
136
+ redaction: CallRedaction | None = None,
137
+ max_values: int = DEFAULT_CALL_LOG_VALUES,
138
+ max_chars: int = DEFAULT_CALL_LOG_CHARS,
139
+ ):
140
+ if maximum is None:
141
+ raw = os.environ.get("WORLDS_CALL_LOG_MAX")
142
+ maximum = int(raw) if raw else DEFAULT_CALL_LOG_MAX
143
+ self.maximum = max(0, int(maximum))
144
+ self.max_values = max_values
145
+ self.max_chars = max_chars
146
+ self.redaction = redaction or CallRedaction()
147
+ self.entries: collections.deque[dict] = collections.deque()
148
+ self._costs: collections.deque[tuple[int, int]] = collections.deque()
149
+ self.values = 0
150
+ self.chars = 0
151
+ self.total = 0
152
+ self._lock = threading.Lock()
153
+
154
+ @classmethod
155
+ def for_spec(cls, spec: Spec) -> CallLog:
156
+ return cls(redaction=CallRedaction.of(spec))
157
+
158
+ def record(
159
+ self,
160
+ operation: Operation,
161
+ method: str,
162
+ path: str,
163
+ args: dict,
164
+ query_all: dict,
165
+ body: Any,
166
+ status: int,
167
+ answer: Any,
168
+ elapsed_s: float,
169
+ via: Via | None,
170
+ *,
171
+ shape: AnswerShape,
172
+ ) -> None:
173
+ if self.maximum == 0:
174
+ return
175
+ door = {} if via is None else {"via": str(via)}
176
+ heading = {"operation": operation.name, "method": method}
177
+ try:
178
+ request, budget = self._route_request(operation, path, args, query_all, body)
179
+ except (RecursionError, RequestTooLarge):
180
+ request, budget = self._route_line(operation, path, args, query_all)
181
+ self._append({**heading, **request, **door}, status, shape.rows(answer), elapsed_s, budget)
182
+
183
+ def _route_request(
184
+ self, operation: Operation, path: str, args: dict, query_all: dict, body: Any
185
+ ) -> tuple[dict, RequestBudget]:
186
+ budget = RequestBudget()
187
+ line = self._request_line(operation, path, args, query_all, budget)
188
+ logged_body = self.redaction.value(body, budget) if isinstance(body, (dict, list)) and body else None
189
+ selectors = _strings_in([line["args"], line["query"], logged_body])
190
+ return {**line, "body": logged_body, "selectors": selectors}, budget
191
+
192
+ def _route_line(self, operation: Operation, path: str, args: dict, query_all: dict) -> tuple[dict, RequestBudget]:
193
+ budget = RequestBudget()
194
+ try:
195
+ line = self._request_line(operation, path, args, query_all, budget)
196
+ except (RecursionError, RequestTooLarge):
197
+ return {"path": operation.path, **WITHHELD_ROUTE}, RequestBudget()
198
+ selectors = _strings_in([line["args"], line["query"]])
199
+ return {**line, "body": None, "selectors": selectors, WITHHELD_FIELD: ["body", "selectors"]}, budget
200
+
201
+ def _request_line(
202
+ self, operation: Operation, path: str, args: dict, query_all: dict, budget: RequestBudget
203
+ ) -> dict:
204
+ redaction = self.redaction
205
+ logged_path = redaction.path(operation, path, args)
206
+ budget.spend(1, len(logged_path))
207
+ return {
208
+ "path": logged_path,
209
+ "args": redaction.value(dict(args), budget),
210
+ "query": redaction.query(query_all, budget),
211
+ }
212
+
213
+ def record_tool(self, tool: str, args: Any, status: int, answer: Any, elapsed_s: float) -> None:
214
+ if self.maximum == 0:
215
+ return
216
+ heading = {"operation": tool, "method": None, "path": None}
217
+ try:
218
+ request, budget = self._tool_request(args)
219
+ except (RecursionError, RequestTooLarge):
220
+ request, budget = dict(WITHHELD_TOOL), RequestBudget()
221
+ self._append({**heading, **request, "via": str(Via.TOOL)}, status, TOOL_ANSWER.rows(answer), elapsed_s, budget)
222
+
223
+ def _tool_request(self, args: Any) -> tuple[dict, RequestBudget]:
224
+ budget = RequestBudget()
225
+ logged_args = self.redaction.value(args, budget)
226
+ return {"args": logged_args, "query": {}, "body": None, "selectors": _strings_in(logged_args)}, budget
227
+
228
+ def _append(self, request: dict, status: int, rows: int | None, elapsed_s: float, budget: RequestBudget) -> None:
229
+ rows = None if status >= 400 else rows
230
+ answered = {
231
+ "status": status,
232
+ "rows": rows,
233
+ "found": _found(status, rows),
234
+ "elapsed_ms": round(elapsed_s * 1000, 1),
235
+ }
236
+ at = datetime.now(timezone.utc).isoformat(timespec="milliseconds").replace("+00:00", "Z")
237
+ cost = budget.spent()
238
+ with self._lock:
239
+ self.total += 1
240
+ self._keep({"seq": self.total, "at": at, **request, **answered}, cost)
241
+
242
+ def _keep(self, entry: dict, cost: tuple[int, int]) -> None:
243
+ self.entries.append(entry)
244
+ self._costs.append(cost)
245
+ self.values += cost[0]
246
+ self.chars += cost[1]
247
+ while len(self.entries) > self.maximum or (len(self.entries) > 1 and self._over_budget()):
248
+ self.entries.popleft()
249
+ values, chars = self._costs.popleft()
250
+ self.values -= values
251
+ self.chars -= chars
252
+
253
+ def _over_budget(self) -> bool:
254
+ return self.values > self.max_values or self.chars > self.max_chars
255
+
256
+ def _held(self, call: dict) -> tuple[int, int]:
257
+ budget = RequestBudget()
258
+ try:
259
+ self.redaction.value([call.get(field) for field in ("path", "args", "query", "body")], budget)
260
+ except (RecursionError, RequestTooLarge):
261
+ return MAX_LOGGED_VALUES, MAX_LOGGED_CHARS
262
+ return budget.spent()
263
+
264
+ def describe(self) -> dict:
265
+ with self._lock:
266
+ return {"total": self.total, "kept": len(self.entries), "max": self.maximum, "calls": list(self.entries)}
267
+
268
+ def into_state(self, state: dict) -> dict:
269
+ """``state`` with the calls beside its rows, as every session's ``state`` and grader state carry them."""
270
+ state["calls"] = self.describe()["calls"]
271
+ return state
272
+
273
+ def resume(self, calls: list[dict]) -> None:
274
+ """Carry on from the calls a checkpoint kept: the same entries, numbered on
275
+ from the last of them, each charged what it holds. A log that keeps
276
+ nothing keeps none of them either."""
277
+ if self.maximum == 0:
278
+ return
279
+ costs = [self._held(call) for call in calls]
280
+ with self._lock:
281
+ for call, cost in zip(calls, costs):
282
+ self._keep(call, cost)
283
+ self.total = max([self.total, *(call["seq"] for call in calls)])
284
+
285
+ def clear(self) -> None:
286
+ with self._lock:
287
+ self.entries.clear()
288
+ self._costs.clear()
289
+ self.values = 0
290
+ self.chars = 0
291
+ self.total = 0