@withgateway/sdk 0.9.1 → 0.9.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agents/index.d.mts +12 -4
- package/dist/agents/index.d.ts +12 -4
- package/dist/agents/index.js +30 -0
- package/dist/agents/index.js.map +1 -1
- package/dist/agents/index.mjs +30 -0
- package/dist/agents/index.mjs.map +1 -1
- package/dist/cli.js +2291 -945
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +2290 -944
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs.map +1 -1
- package/dist/index.js +325 -184
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +324 -183
- package/dist/index.mjs.map +1 -1
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.js +1 -1
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +1 -1
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +3 -1
- package/dist/worlds/index.d.ts +3 -1
- package/dist/worlds/index.js +178 -67
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +177 -66
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
- package/runtime/gatewaysdk/environment/bundle_manifest.py +297 -9
- package/runtime/gatewaysdk/environment/factory_documents.py +670 -0
- package/runtime/gatewaysdk/environment/factory_rules.py +973 -0
- package/runtime/gatewaysdk/environment/factory_shapes.py +253 -0
- package/runtime/gatewaysdk/environment/factory_spine.py +1155 -0
- package/runtime/gatewaysdk/environment/factory_spine_shapes.py +480 -0
- package/runtime/gatewaysdk/environment/schema/admission.py +61 -59
- package/runtime/gatewaysdk/environment/schema/cast.py +133 -67
- package/runtime/gatewaysdk/environment/schema/compiler.py +16 -17
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +58 -4
- package/runtime/gatewaysdk/environment/schema/host.py +21 -2
- package/runtime/gatewaysdk/environment/schema/host_control.py +3 -1
- package/runtime/gatewaysdk/environment/schema/host_loops.py +27 -4
- package/runtime/gatewaysdk/environment/schema/host_metrics.py +69 -0
- package/runtime/gatewaysdk/environment/schema/host_requests.py +21 -0
- package/runtime/gatewaysdk/environment/schema/host_session.py +7 -0
- package/runtime/gatewaysdk/environment/schema/host_versions.py +4 -0
- package/runtime/gatewaysdk/environment/schema/host_workers.py +172 -50
- package/runtime/gatewaysdk/environment/schema/ingest.py +12 -3
- package/runtime/gatewaysdk/environment/schema/mapping.py +1 -1
- package/runtime/gatewaysdk/environment/schema/measured_tests.py +187 -0
- package/runtime/gatewaysdk/environment/schema/platform.py +57 -3
- package/runtime/gatewaysdk/environment/schema/platform_query.py +4 -3
- package/runtime/gatewaysdk/environment/schema/platform_templates.py +105 -17
- package/runtime/gatewaysdk/environment/schema/process_memory.py +76 -0
- package/runtime/gatewaysdk/environment/schema/redaction.py +13 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +14 -0
- package/runtime/gatewaysdk/environment/schema/scaffold.py +21 -4
- package/runtime/gatewaysdk/environment/schema/shape_catalog.py +493 -0
- package/runtime/gatewaysdk/environment/schema/shape_declarations.py +219 -0
- package/runtime/gatewaysdk/environment/schema/skill.py +1 -0
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +1 -0
- package/runtime/gatewaysdk/environment/schema/skills/factory-orchestrator/SKILL.md +147 -0
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +5 -1
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +4 -4
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +2 -2
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +1 -0
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/shape-catalog.md +59 -0
- package/runtime/gatewaysdk/environment/schema/validation.py +14 -0
- package/runtime/gatewaysdk/environment/schema/worker.py +7 -4
- package/runtime/gatewaysdk/environment/schema/worker_key.py +38 -0
- package/runtime/gatewaysdk/environment/schema/worker_peaks.py +38 -0
- package/runtime/gatewaysdk/environment/schema/worker_watch.py +50 -0
- package/runtime/gatewaysdk/environment/schema/workers.py +237 -68
- package/runtime/gatewaysdk/environment/schema/world_host.py +56 -62
- package/runtime/gatewaysdk/environment/schema/world_tests.py +3 -1
- package/skills/connector-schema-authoring/SKILL.md +1 -0
- package/skills/connector-schema-authoring/connector-worlds.md +1 -0
- package/skills/connector-schema-authoring/world-schema.md +7 -0
- package/skills/factory-orchestrator/SKILL.md +147 -0
- package/skills/world-data-ingestion/ROWS.md +5 -1
- package/skills/world-data-ingestion/SKILL.md +4 -4
- package/skills/worlds-getting-started/SKILL.md +2 -2
- package/skills/worlds-getting-started/references/cli-mcp-map.md +1 -0
- package/skills/worlds-getting-started/references/shape-catalog.md +59 -0
|
@@ -20,15 +20,15 @@ from ..world_session_contract import OPEN_DOOR_FEATURE, REQUEST_LOG_PAGE_MAX, WO
|
|
|
20
20
|
from .admission import RETRY_AFTER_S, Admission
|
|
21
21
|
from .cache_layout import SESSIONS_DIR, VERSIONS_DIR
|
|
22
22
|
from .door import VENDOR_MODE, session_mode
|
|
23
|
-
from .host_handoff import OUTCOMES
|
|
24
23
|
from .host_loops import swept
|
|
24
|
+
from .host_metrics import exposition
|
|
25
25
|
from .host_owners import PlatformOwners
|
|
26
26
|
from .host_requests import BrowserRelay, DependencyRef, parse_dependency_refs, restore_sha256, worker_snapshot
|
|
27
27
|
from .host_session import Export, Session
|
|
28
28
|
from .host_surfaces import SessionHost
|
|
29
29
|
from .host_transfers import check_object_store_url, download_verified, file_digest, upload_file
|
|
30
30
|
from .host_versions import DEFAULT_MAX_BUNDLE_MB, CachedVersion, DataKey, VersionCache
|
|
31
|
-
from .host_workers import CALL_TIMEOUT_S, EXPORT_TIMEOUT_S, LOST, UNKNOWN_SESSION, VersionWorkers
|
|
31
|
+
from .host_workers import CALL_TIMEOUT_S, EXPORT_TIMEOUT_S, LOST, OUT_OF_MEMORY_RETRY, UNKNOWN_SESSION, VersionWorkers
|
|
32
32
|
from .journal import WorldJournal
|
|
33
33
|
from .lanes import Lanes
|
|
34
34
|
from .request_ledger import HostLedger, LedgerLife, NoLedger
|
|
@@ -49,7 +49,11 @@ from .ui_interactions import RECORDER_ANSWER
|
|
|
49
49
|
from .tools_world import DependencyWorld
|
|
50
50
|
from .validation import fail
|
|
51
51
|
from .verbs import SESSION_VERBS
|
|
52
|
-
from .
|
|
52
|
+
from .worker_key import WorkerKey
|
|
53
|
+
from .process_memory import MemoryNeed
|
|
54
|
+
from .worker_peaks import PeakReport
|
|
55
|
+
from .worker_watch import WorkerMemoryWatch
|
|
56
|
+
from .workers import NO_WORKERS, Finishing, Worker, WorkerPool
|
|
53
57
|
|
|
54
58
|
DEFAULT_LEASE_S = 3600.0
|
|
55
59
|
DEFAULT_MAX_SESSIONS = 1500
|
|
@@ -101,6 +105,7 @@ class WorldHost:
|
|
|
101
105
|
self.advertise_url = advertise_url
|
|
102
106
|
self.workers = workers
|
|
103
107
|
self.placement = VersionWorkers(workers, self) if workers is not None else None
|
|
108
|
+
self.memory_watch = WorkerMemoryWatch(workers, self.tally) if workers is not None else None
|
|
104
109
|
self.run_handlers = run_handlers
|
|
105
110
|
self.pinned_version = pinned_version
|
|
106
111
|
self.limits = limits
|
|
@@ -108,6 +113,7 @@ class WorldHost:
|
|
|
108
113
|
self.draining = False
|
|
109
114
|
self.ledger: HostLedger = NoLedger()
|
|
110
115
|
self.finishing = Finishing()
|
|
116
|
+
self.peaks = PeakReport()
|
|
111
117
|
self.started_at = time.time()
|
|
112
118
|
self._lock = threading.Lock()
|
|
113
119
|
self._sessions: dict[str, Session] = {}
|
|
@@ -137,6 +143,8 @@ class WorldHost:
|
|
|
137
143
|
auth: str | None = None,
|
|
138
144
|
ledger_source: str | None = None,
|
|
139
145
|
parameters: dict[str, str] | None = None,
|
|
146
|
+
world_state: bool | None = None,
|
|
147
|
+
memory_peak_mb: int | None = None,
|
|
140
148
|
) -> dict:
|
|
141
149
|
"""Open ``session_id`` on ``version_id``, either on the bundled ``task``
|
|
142
150
|
or on ``task_spec``, a session task of its own (:mod:`tasks`); exactly
|
|
@@ -145,7 +153,10 @@ class WorldHost:
|
|
|
145
153
|
snapshot?}`` per alias the manifest declares, opened beside it. A
|
|
146
154
|
``pending`` open is a session moving here from another host: it serves
|
|
147
155
|
nothing until the platform's row names this host (:class:`SessionMove`).
|
|
148
|
-
``auth`` (``vendor`` or ``open``) overrides the world's ``[auth] mode``.
|
|
156
|
+
``auth`` (``vendor`` or ``open``) overrides the world's ``[auth] mode``.
|
|
157
|
+
A ``world_state`` session is served in its version's states lane (:class:`WorkerKey`); a retry that
|
|
158
|
+
names no lane (``None``, a platform from before lanes) gets the session already open on either.
|
|
159
|
+
``memory_peak_mb`` is what the platform measured the world to need: its worker is provisioned by it."""
|
|
149
160
|
if self.pinned_version is not None and version_id != self.pinned_version:
|
|
150
161
|
fail("host", f"this worker serves version {self.pinned_version} only, not {version_id}")
|
|
151
162
|
if (task is None) == (task_spec is None):
|
|
@@ -156,6 +167,8 @@ class WorldHost:
|
|
|
156
167
|
deps = parse_dependency_refs(dependencies)
|
|
157
168
|
knobs = parse_run_parameters(parameters)
|
|
158
169
|
life = LedgerLife.of(ledger_source)
|
|
170
|
+
key = WorkerKey.of(version_id, world_state is True)
|
|
171
|
+
need = MemoryNeed(memory_peak_mb).mb if memory_peak_mb else 0
|
|
159
172
|
self._replace_pending(session_id, restore, browser, pending)
|
|
160
173
|
with self._lock:
|
|
161
174
|
existing = self._sessions.get(session_id)
|
|
@@ -168,14 +181,17 @@ class WorldHost:
|
|
|
168
181
|
self._admit(session_id)
|
|
169
182
|
if existing is not None:
|
|
170
183
|
data = DataKey.of(version_id, snapshot).sha256
|
|
171
|
-
|
|
184
|
+
lane = WorkerKey(version_id, existing.key.states) if world_state is None else key
|
|
185
|
+
wanted = (lane, data, knobs, (task, spec.raw if spec is not None else None), token)
|
|
172
186
|
return self._reopened(existing, wanted, restore, pending, auth)
|
|
173
187
|
try:
|
|
174
188
|
session = self._open(
|
|
175
|
-
session_id,
|
|
189
|
+
session_id, key, bundle_url, task, spec, token, snapshot, browser, restore, deps, pending, auth, knobs,
|
|
190
|
+
need,
|
|
176
191
|
)
|
|
177
192
|
session.life = life
|
|
178
193
|
self._register(session)
|
|
194
|
+
self._refuse_poisoned(session)
|
|
179
195
|
described = self._describe(session)
|
|
180
196
|
except Exception:
|
|
181
197
|
self.tally("open_failures")
|
|
@@ -192,6 +208,14 @@ class WorldHost:
|
|
|
192
208
|
fail(UNKNOWN_SESSION, f"session {session_id} was closed while it opened")
|
|
193
209
|
return described
|
|
194
210
|
|
|
211
|
+
def _refuse_poisoned(self, session: Session) -> None:
|
|
212
|
+
"""A session registered on a worker poisoned since it was placed missed that worker's recovery: it is
|
|
213
|
+
closed here and its open is told to come back, never left on a worker about to stop."""
|
|
214
|
+
if session.worker is None or not session.worker.poisoned:
|
|
215
|
+
return
|
|
216
|
+
self.close(session.session_id, reason="replaced")
|
|
217
|
+
fail("busy", OUT_OF_MEMORY_RETRY, retryAfter=RETRY_AFTER_S)
|
|
218
|
+
|
|
195
219
|
def _replace_pending(self, session_id: str, restore: dict[str, str] | None, browser: Any, pending: bool) -> None:
|
|
196
220
|
"""A draining host takes no move, refused before anything here changes. A copy pending a move here that
|
|
197
221
|
a later open reuses gets a fresh lease; one from another export or browser is closed first."""
|
|
@@ -211,7 +235,7 @@ class WorldHost:
|
|
|
211
235
|
this host is another host's to open, and one opened from another export or auth is refused."""
|
|
212
236
|
if existing.move.leaving:
|
|
213
237
|
fail("draining", f"session {existing.session_id} is moving to another host; place it elsewhere")
|
|
214
|
-
identity = (existing.
|
|
238
|
+
identity = (existing.key, existing.snapshot_sha256, existing.parameters, existing.task_key, existing.token)
|
|
215
239
|
if identity != wanted or auth not in (None, existing.auth):
|
|
216
240
|
fail("host", f"session {existing.session_id} is already open as another version, data, task, token or auth")
|
|
217
241
|
if restore and restore_sha256(restore) != existing.restore_sha256:
|
|
@@ -241,7 +265,7 @@ class WorldHost:
|
|
|
241
265
|
def _open(
|
|
242
266
|
self,
|
|
243
267
|
session_id: str,
|
|
244
|
-
|
|
268
|
+
key: WorkerKey,
|
|
245
269
|
bundle_url: str,
|
|
246
270
|
task: str | None,
|
|
247
271
|
spec: TaskSpec | None,
|
|
@@ -253,7 +277,9 @@ class WorldHost:
|
|
|
253
277
|
pending: bool,
|
|
254
278
|
auth: str | None = None,
|
|
255
279
|
knobs: dict[str, str] | None = None,
|
|
280
|
+
need_mb: int = 0,
|
|
256
281
|
) -> Session:
|
|
282
|
+
version_id = key.version_id
|
|
257
283
|
cached = self.cache.ensure(version_id, *self._trusted_version(bundle_url, snapshot))
|
|
258
284
|
data = DataKey.of(version_id, snapshot).sha256
|
|
259
285
|
mode = session_mode(auth, token)
|
|
@@ -263,10 +289,11 @@ class WorldHost:
|
|
|
263
289
|
relay = BrowserRelay.parse(browser)
|
|
264
290
|
if cached.needs_worker and not self.run_handlers:
|
|
265
291
|
return self._open_on_worker(
|
|
266
|
-
cached, session_id, bundle_url, task, spec, token, relay, restore, deps, pending, mode, snapshot, knobs
|
|
292
|
+
cached, key, session_id, bundle_url, task, spec, token, relay, restore, deps, pending, mode, snapshot, knobs,
|
|
293
|
+
need_mb,
|
|
267
294
|
)
|
|
268
295
|
session_dir = self.sessions_root / session_id
|
|
269
|
-
with self._opens.turn(
|
|
296
|
+
with self._opens.turn(key.name):
|
|
270
297
|
runtime = WorldRuntime.open(
|
|
271
298
|
cached,
|
|
272
299
|
session_dir,
|
|
@@ -302,12 +329,14 @@ class WorldHost:
|
|
|
302
329
|
parameters=dict(knobs or {}),
|
|
303
330
|
pending=pending,
|
|
304
331
|
auth=mode,
|
|
332
|
+
world_state=key.states,
|
|
305
333
|
)
|
|
306
334
|
return session
|
|
307
335
|
|
|
308
336
|
def _open_on_worker(
|
|
309
337
|
self,
|
|
310
338
|
cached: CachedVersion,
|
|
339
|
+
key: WorkerKey,
|
|
311
340
|
session_id: str,
|
|
312
341
|
bundle_url: str,
|
|
313
342
|
task: str | None,
|
|
@@ -320,8 +349,9 @@ class WorldHost:
|
|
|
320
349
|
mode: str = VENDOR_MODE,
|
|
321
350
|
snapshot: dict[str, str] | None = None,
|
|
322
351
|
knobs: dict[str, str] | None = None,
|
|
352
|
+
need_mb: int = 0,
|
|
323
353
|
) -> Session:
|
|
324
|
-
"""Place the session on
|
|
354
|
+
"""Place the session on its key's worker (:class:`VersionWorkers`), which opens it from the bundle
|
|
325
355
|
this process cached; the supervisor keeps the ui's files and the browser tunnel, relays the ui's api
|
|
326
356
|
calls as they came (``via: ui``), and hands an export it fetched and verified to the worker as a local
|
|
327
357
|
file with its sha256, checked again there. No worker past the pool's memory budget: ``at_capacity``."""
|
|
@@ -342,9 +372,10 @@ class WorldHost:
|
|
|
342
372
|
"snapshot": worker_snapshot(snapshot),
|
|
343
373
|
"parameters": dict(knobs or {}),
|
|
344
374
|
"auth": mode,
|
|
375
|
+
"worldState": key.states,
|
|
345
376
|
}
|
|
346
377
|
fetch = functools.partial(self._restore_source, restore)
|
|
347
|
-
worker = self.placement.place(
|
|
378
|
+
worker = self.placement.place(key, session_id, body, fetch, self._opens, cached.world, need_mb)
|
|
348
379
|
session = Session(
|
|
349
380
|
session_id=session_id,
|
|
350
381
|
version_id=cached.version_id,
|
|
@@ -368,6 +399,7 @@ class WorldHost:
|
|
|
368
399
|
parameters=dict(knobs or {}),
|
|
369
400
|
pending=pending,
|
|
370
401
|
auth=mode,
|
|
402
|
+
world_state=key.states,
|
|
371
403
|
)
|
|
372
404
|
return session
|
|
373
405
|
|
|
@@ -676,6 +708,7 @@ class WorldHost:
|
|
|
676
708
|
return []
|
|
677
709
|
swept(self.placement.recover_dead, "recovered the dead worker of version")
|
|
678
710
|
swept(self.placement.recover_stuck, "recycled the stuck worker of version")
|
|
711
|
+
swept(self.memory_watch.sweep, "found near its memory cap the worker of version")
|
|
679
712
|
return self.workers.reap_idle()
|
|
680
713
|
|
|
681
714
|
def session_count(self) -> int:
|
|
@@ -690,6 +723,10 @@ class WorldHost:
|
|
|
690
723
|
"""Whether a call of the session still holds its turn on its worker (one past its time)."""
|
|
691
724
|
return self.placement is not None and session.worker is not None and self.placement.holds_turn(session)
|
|
692
725
|
|
|
726
|
+
def reported(self, stats: dict) -> None:
|
|
727
|
+
"""The platform took a heartbeat of ``stats``: the peaks it carried are its now."""
|
|
728
|
+
self.peaks.taken(stats.get("workerPeaks") or [])
|
|
729
|
+
|
|
693
730
|
def worker_count(self) -> int:
|
|
694
731
|
return len(self.workers._workers) if self.workers is not None else 0
|
|
695
732
|
|
|
@@ -725,7 +762,9 @@ class WorldHost:
|
|
|
725
762
|
"capacity": self.max_sessions,
|
|
726
763
|
"draining": self.draining,
|
|
727
764
|
"workers": self.worker_count(),
|
|
728
|
-
|
|
765
|
+
**(self.workers.stats() if self.workers is not None else NO_WORKERS),
|
|
766
|
+
"workerPeaks": self.peaks.pending(self.workers.peaks()) if self.workers is not None else [],
|
|
767
|
+
"workerMemory": self.memory_watch.readings() if self.memory_watch is not None else [],
|
|
729
768
|
"versions": self.cache.version_ids(),
|
|
730
769
|
"versionsInUse": versions_in_use,
|
|
731
770
|
"cloneMode": self.cache.clone_mode,
|
|
@@ -740,52 +779,7 @@ class WorldHost:
|
|
|
740
779
|
}
|
|
741
780
|
|
|
742
781
|
def metrics_text(self) -> str:
|
|
743
|
-
|
|
744
|
-
is the number the fleet scales on (sessions per host, not cpu)."""
|
|
745
|
-
stats = self.stats()
|
|
746
|
-
counters: dict[str, int] = stats["counters"]
|
|
747
|
-
lines = [
|
|
748
|
-
"# TYPE worlds_host_sessions gauge",
|
|
749
|
-
f"worlds_host_sessions {stats['sessions']}",
|
|
750
|
-
"# TYPE worlds_host_capacity gauge",
|
|
751
|
-
f"worlds_host_capacity {stats['capacity']}",
|
|
752
|
-
"# TYPE worlds_host_draining gauge",
|
|
753
|
-
f"worlds_host_draining {int(stats['draining'])}",
|
|
754
|
-
"# TYPE worlds_host_workers gauge",
|
|
755
|
-
f"worlds_host_workers {stats['workers']}",
|
|
756
|
-
"# TYPE worlds_host_versions_cached gauge",
|
|
757
|
-
f"worlds_host_versions_cached {len(stats['versions'])}",
|
|
758
|
-
"# TYPE worlds_host_uptime_seconds gauge",
|
|
759
|
-
f"worlds_host_uptime_seconds {stats['uptimeSeconds']:.0f}",
|
|
760
|
-
"# TYPE worlds_host_sessions_opened_total counter",
|
|
761
|
-
f"worlds_host_sessions_opened_total {counters.get('opened', 0)}",
|
|
762
|
-
"# TYPE worlds_host_open_failures_total counter",
|
|
763
|
-
f"worlds_host_open_failures_total {counters.get('open_failures', 0)}",
|
|
764
|
-
"# TYPE worlds_host_session_seconds_total counter",
|
|
765
|
-
f"worlds_host_session_seconds_total {stats['sessionSeconds']:.0f}",
|
|
766
|
-
"# TYPE worlds_host_exports_total counter",
|
|
767
|
-
f"worlds_host_exports_total {counters.get('exports', 0)}",
|
|
768
|
-
"# TYPE worlds_host_exports_uploaded_total counter",
|
|
769
|
-
f"worlds_host_exports_uploaded_total {counters.get('exports_uploaded', 0)}",
|
|
770
|
-
"# TYPE worlds_host_sessions_closed_total counter",
|
|
771
|
-
]
|
|
772
|
-
for reason in ("closed", "expired", "lost", "replaced", "moved"):
|
|
773
|
-
lines.append(
|
|
774
|
-
f'worlds_host_sessions_closed_total{{reason="{reason}"}} {counters.get(f"closed.{reason}", 0)}'
|
|
775
|
-
)
|
|
776
|
-
lines.append("# TYPE worlds_host_handoffs_total counter")
|
|
777
|
-
for outcome in OUTCOMES:
|
|
778
|
-
lines.append(f'worlds_host_handoffs_total{{outcome="{outcome}"}} {counters.get(f"handoffs.{outcome}", 0)}')
|
|
779
|
-
for name in ("workers_replaced", "workers_recycled", "sessions_recovered", "sessions_unrecovered"):
|
|
780
|
-
lines += [f"# TYPE worlds_host_{name}_total counter", f"worlds_host_{name}_total {counters.get(name, 0)}"]
|
|
781
|
-
lines.append("# TYPE worlds_host_requests_total counter")
|
|
782
|
-
for surface in ("api", "ui", "browser", "proxied"):
|
|
783
|
-
lines.append(f'worlds_host_requests_total{{surface="{surface}"}} {counters.get(f"requests.{surface}", 0)}')
|
|
784
|
-
lines += [
|
|
785
|
-
"# TYPE worlds_host_platform_bearer_requests_total counter",
|
|
786
|
-
f"worlds_host_platform_bearer_requests_total {counters.get('requests.platform_bearer', 0)}",
|
|
787
|
-
]
|
|
788
|
-
return "\n".join(lines) + "\n"
|
|
782
|
+
return exposition(self.stats())
|
|
789
783
|
|
|
790
784
|
def _dependency_worlds(self, cached: CachedVersion, deps: list[DependencyRef]) -> dict[str, DependencyWorld]:
|
|
791
785
|
"""The dependency versions a session opens beside ``cached``, cached the
|
|
@@ -869,10 +863,10 @@ class WorldHost:
|
|
|
869
863
|
"""One of the session's version's turns (:class:`Admission`) for a data-plane request, once its
|
|
870
864
|
move lets it in (:class:`SessionMove`): held while the session moves, counted against the
|
|
871
865
|
version's waiting bounds, served by a copy pending a move here once the platform names it."""
|
|
872
|
-
waiting = functools.partial(self.admission.waiting, session.
|
|
866
|
+
waiting = functools.partial(self.admission.waiting, session.key.name, session.session_id)
|
|
873
867
|
owner_now = functools.partial(self._owner_now, session.session_id)
|
|
874
868
|
with session.move.request(waiting, owner_now, self.advertise_url, hopped):
|
|
875
|
-
with self.admission.turn(session.
|
|
869
|
+
with self.admission.turn(session.key.name, session.session_id):
|
|
876
870
|
yield
|
|
877
871
|
|
|
878
872
|
def _owner_now(self, session_id: str) -> str | None:
|
|
@@ -894,7 +888,7 @@ class WorldHost:
|
|
|
894
888
|
with (
|
|
895
889
|
self.using(session_id, confirm=True, touch=touch) as session,
|
|
896
890
|
session.move.verb(),
|
|
897
|
-
self.admission.turn(session.
|
|
891
|
+
self.admission.turn(session.key.name, session.session_id),
|
|
898
892
|
):
|
|
899
893
|
if session.worker is not None:
|
|
900
894
|
assert self.placement is not None
|
|
@@ -147,6 +147,7 @@ class Report:
|
|
|
147
147
|
cases: list[CaseResult]
|
|
148
148
|
python: PythonResult
|
|
149
149
|
scenario: str | None = None
|
|
150
|
+
served: bool = False
|
|
150
151
|
|
|
151
152
|
@property
|
|
152
153
|
def summary(self) -> dict:
|
|
@@ -378,9 +379,10 @@ def run(world: str | Path, *, scenario: str | None = None) -> Report:
|
|
|
378
379
|
python = _run_python(root, python_files, session)
|
|
379
380
|
finally:
|
|
380
381
|
session.close()
|
|
382
|
+
served = bool(results) or python.tests > python.skipped
|
|
381
383
|
if not results and python.ran_nothing and declares_on_push_tests(root):
|
|
382
384
|
python = _unkept_promise(python)
|
|
383
|
-
return Report(str(root), results, python, scenario)
|
|
385
|
+
return Report(str(root), results, python, scenario, served)
|
|
384
386
|
|
|
385
387
|
|
|
386
388
|
def _unkept_promise(python: PythonResult) -> PythonResult:
|
|
@@ -106,6 +106,7 @@ deliverable can finish with synthetic fixtures and explicitly skipped live valid
|
|
|
106
106
|
- [ ] Enforce supported required fields, types, enums, key uniqueness, and reference integrity. Document vendor rules the dialect cannot express, normalization choices, and constraints the future mapper must enforce; structural validation is not vendor behavior.
|
|
107
107
|
- [ ] Real vendor documents compile as they come (Brex retrospective, 2026-09-14): `format: int64`/`double` on numbers is ignored, not refused (string formats are still validated); a schema-valued `additionalProperties` types a map (`Record<string, string>`); a type nested in itself (`ExpandableExpense` holding an `ExpandableExpense`) compiles, with the nested occurrence validated as its bare type and stored as JSON, never projected to a column. A contract whose inlined files would pass the 4 MB publication cap (Google Docs' recursive body) compiles each object/array definition once under `$defs` and references it instead; nothing to do in the template. A nested key or relationship field is a JSON Pointer, `/owner/user_id`, never `owner.user_id`; the compiler's refusal says so. A primary key may be nested too (`/protocolSection/identificationModule/nctId`) when every object on its path is required and non-null; lookups (`get`, `update`, `delete`) then pass it flat under the pointer itself. Field names keep the vendor's hyphens (`reference-count`); dots and slashes never belong to a name. A verifier's SQL double-quotes such a column (`"reference-count"`): unquoted, SQLite reads a subtraction.
|
|
108
108
|
- [ ] Every property verifier SQL will name has a column: scalars and compositions whose non-null branches are scalar (`anyOf [{$ref Status}, {type: null}]`, the nullable-enum shape) project; a composition that may hold an object or array stays JSON-only. A verifier that names a column the contract never projected fails the grade with the database engine's message and the statement; it never scores 0 quietly (double-quoted unknown names no longer fall back to string literals).
|
|
109
|
+
- [ ] Declare the shapes a writer would otherwise guess (`worlds describe --json` reads them into its shape catalog): the keys of an open JSON field defined by another entity's rows (`custom_fields`, e.g. a properties map keyed by a property-definitions entity), ids inside a row that come from another entity or a fixed vendor table (`ids`), dense vendor-minted numbers (`sequences`), and, on a handler route, where its answer keeps the records, the id, the count and the total (`[operations.answer]`). Enums and nested shapes need nothing beyond the JSON Schema itself. See `world-schema.md` and `connector-worlds.md`.
|
|
109
110
|
- [ ] `gateway worlds schema compile PATH` then `check PATH` pass. Never edit generated files; SDK templates contain authored inputs and generate bundle artifacts at `init` time. Changing the root `$id` after `init` is fine: `compile` re-stamps an empty `data/initial.json` onto the new domain; a journal that holds rows follows a change that only adds to its domain (a new entity, a new field that is not required) and is refused otherwise, with the fix named (re-import its rows, or restore the schema it was written against).
|
|
110
111
|
- [ ] Shared templates and committed fixtures hold synthetic data only. Tenant captures stay in their owning project; initial batches and refresh journals retain explicit capture/synthetic provenance.
|
|
111
112
|
|
|
@@ -254,6 +254,7 @@ least one `[[operations]]` entry.
|
|
|
254
254
|
| `[[operations]]` | `name`, `method`, `path`, `params`, `entity`, `results` | `path` may carry `{args}`; a `params` name keeps the vendor's spelling, dots included (`query.bibliographic`, `filter.overallStatus`), and may end in `[]` (`expand[]`, `status[]`) for a key the vendor repeats, which capture then sends repeated and the served route filters on any value of; `results` is the dotted path to the row array (empty when the body is the row). An operation without `entity` never seeds and answers 501 `not_modelled`. Routes are never tools: an agent calls them over HTTP (the api url and token `session open --surface api` prints), and a tool exists only where `x-gateway.tools` declares one with a handler. An `entity` (or an `explode` target) that `schema/world.json` does not declare is refused by `connector validate`, `schema compile`, `schema check` and the platform's describe (`gateway worlds describe`, local or platform) alike — `operation X lands on entity Y, which schema/world.json does not declare (declared: …)` at `/operations/<i>/entity` — and nothing is written; hand-edited TOML meets the same gate `route add` applies. |
|
|
255
255
|
| `[operations.ingest]` | `drop`, `explode`, `carry_args` | `drop` removes vendor fields the entity does not carry; each `[[operations.ingest.explode]]` (`field`, `entity`, `carry`, `take`) turns a nested array into child rows; `carry_args = { account_id = "id" }` stamps the capture's path argument or query parameter onto every row before projection (a statement only knows its account from the URL it was fetched under). `connector ingest --dry-run [--report FILE]` lists every violation of every capture and writes nothing. |
|
|
256
256
|
| `[operations.api]` | `filter`, `match`, `action`, `status`, `single`, `envelope`, `wrapper`, `page`, `body`, `defaults`, `generate`, `handler`, `sample`, `sample_headers`, `sample_body`, `not_found`, `not_found_status`, `cursor_absent_when_done` | `filter` maps a path argument or declared query parameter to the row field it selects on (case-insensitive, any depth; an absent parameter does not narrow, one sent more than once matches rows carrying any of its values); `match = { name = "contains" }` switches an argument from equality to substring or `prefix`, and a list field matches when any element does. `action` is `read` (default), `create`, `update`, `delete` or `handler`: writes persist through the contract, `body` maps row fields (dotted for nested) to request-body paths, `defaults` fixes constants, `generate` fills `uuid`, `int` or `now`; `update` and `delete` find their row through `filter` and answer the vendor's 404 otherwise; they, and a `single` read with a path argument, refuse (`finds its row by nothing`, 400) when neither `filter` nor a path argument naming a field selects a row, rather than act on some row; a singleton served under a scoped path answers through `action = "handler"`. `status` is the 2xx the vendor answers (201 for create, 204 for delete by default), `single` returns the row itself instead of an envelope, `envelope` and `page` override the world-wide setting per route. A dotted `envelope` nests the answer the way the vendor does (`message.items` answers `{message: {items, total-results, next-cursor}}`), and a single row is wrapped only by its own route's `envelope` (`message` answers `{message: row}`). `wrapper` is the rest of the vendor's envelope as a JSON object string (`'{"status": "ok", "message-type": "work-list"}'`), the answer laid onto it key by key; `schema init --from-openapi` writes it from the document. `handler` names a function `def name(ctx, args, query, body)` in the handlers module that returns a body, `(status, body)` or `(status, body, headers)`; `ctx` reads and writes the store (`rows`, `get`, `find`, `insert`, `update`, `delete`), keeps time (`now`, `elapsed`, `ladder` for status machines, frozen by `GATEWAY_WORLD_CLOCK`), mints ids (`uuid`) and shapes answers (`envelope`, `not_found`, `invalid`) and sees the request's `headers` and `query_all` (every value of every parameter, `{"expand[]": ["a", "b"]}`; the `query` argument keeps the first value only). `not_found` / `not_found_status` answer this route's missing row the vendor's own way (a 403 with its own message for one resource) and fall back to `[api]`; `cursor_absent_when_done` overrides `[api]` for this route. `sample = { ip = "203.0.113.5" }` gives the conformance run a value a validating route accepts when no seeded row can supply one; `sample_headers = { "Idempotency-Key" = "conform-1" }` and `sample_body = { reason = "conform" }` give it the headers and body a route demands (read by `connector conform` only; the served API never sends requests). |
|
|
257
|
+
| `[operations.answer]` | `records`, `id`, `fields`, `count`, `total`, `cursor`, `cursor_param` | What the route's answer holds where the served settings cannot say (a handler's own shape): JSON pointers to the records (a list or the one record), the count, the total and the next-page reference, the record's id key, the per-record fields a recorded answer is compared on, and the request parameter that takes the cursor. Each key overrides what `describe` derives from `[api]` (`envelope`, `hits_key`, `cursor_key`, paging) and the entity's key; read by the shape catalog only, never by the served route. |
|
|
257
258
|
| `[webhooks]` | `subscriptions`, `url`, `events`, `secret`, `enabled`, `scope`, `signing`, `signature_header` | Where the world's webhook subscriptions live and how the vendor signs what it sends (see Webhooks below). `subscriptions` names the entity agents register endpoints in through the vendor's own routes; `url`, `events`, `secret`, `enabled` and each `scope` value name its fields (checked against the contract at compile). `signing` is `svix`, `github`, `linear`, `stripe`, `hmac-sha256` (with `signature_header`) or `none`. |
|
|
258
259
|
|
|
259
260
|
### Scaffolding a route
|
|
@@ -115,6 +115,13 @@ The root's `x-gateway` requires:
|
|
|
115
115
|
Add `$defs`; the
|
|
116
116
|
[CRM fixture](../../samples/schema-worlds/crm/schema/world.json) supplies examples.
|
|
117
117
|
Entities require `schema`/`primary_key`; `unique`/`relationships` default to empty.
|
|
118
|
+
Three more members are optional and describe the rows rather than store them, so they never
|
|
119
|
+
change the domain hash: `custom_fields` (the keys of an object field, or the ids of an array
|
|
120
|
+
of `{id, value}` items, are rows of another entity: `{field, definitions, key, where?, pairs?}`),
|
|
121
|
+
`ids` (the values at a JSON pointer come from another entity's rows, `{pointer, entity, key,
|
|
122
|
+
where?}`, or a fixed table, `{pointer, values: [{id, label}]}`) and `sequences` (an integer
|
|
123
|
+
field numbered densely within `scope`: `{field, scope?, start?}`). `worlds describe --json`
|
|
124
|
+
reads them into its shape catalog ([world-shape-catalog](../../system-architecture/worlds/world-shape-catalog.md)).
|
|
118
125
|
Tools require `input`/`output`; `description` is optional. Unknown members reject.
|
|
119
126
|
Version labels imply no semantic-version compatibility.
|
|
120
127
|
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: factory-orchestrator
|
|
3
|
+
description: Co-author a world bundle's Data Factory with a person — its ontology, entity graph, rules, tests and spine — and, when your switches allow it, edit the bundle's worlds and their relationships and write their rows. Use when you are a bundle's factory orchestrator, or an outside agent joined to a factory's roster.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Data Factory: the orchestrator
|
|
7
|
+
|
|
8
|
+
## What you work on
|
|
9
|
+
|
|
10
|
+
A Data Factory belongs to one world bundle (a bundle of one world is allowed). It holds five documents. Each is a set of nodes, and each node has a stable `id`, a `name` and one `sentence`.
|
|
11
|
+
|
|
12
|
+
- **Ontology:** the things and events of this domain and how they relate. The kinds are whatever the person needs; there is no fixed vocabulary. A relation is `{type, to}`, with `type` a free word.
|
|
13
|
+
- **Entity graph:** notes on the bundle's entities: which world creates a record, cardinality, how fields merge. The links between worlds (identities and references) live in the bundle's `gateway-bundle.toml`, not here.
|
|
14
|
+
- **Rules:** what must hold. A rule is a sentence (guidance), code (`guard`, `derive`), or both.
|
|
15
|
+
- **Tests:** checks attached to nodes, marked `generate`, `verify` or both.
|
|
16
|
+
- **Spine:** the facts every writer must agree on: rosters, calendars, storylines, partitions and their quotas, reserved keys, sequence starts. Tables, facts and checks, checked at every commit.
|
|
17
|
+
|
|
18
|
+
Any node can point at another node, a spine row (`{kind: "row", node, key}`), a world, a schema field, a record or a drive file through `refs`.
|
|
19
|
+
|
|
20
|
+
The factory writes data. It never runs or simulates worlds, and there is no clock to advance: records carry their own dates.
|
|
21
|
+
|
|
22
|
+
## Your tools
|
|
23
|
+
|
|
24
|
+
Your switches decide which write tools you have. Call only what you were given. Reads name the bundle as `slug`, writes as `bundle`. Every result carries `head`, the head revision's id, and may carry `hook_context`: lines the factory's rules add for you (read them before your next step).
|
|
25
|
+
|
|
26
|
+
| Tool | Does |
|
|
27
|
+
|---|---|
|
|
28
|
+
| `get_factory` | The head revision and the documents (pass `doc` for one). |
|
|
29
|
+
| `list_factory_revisions`, `get_factory_revision`, `diff_factory_revisions` | History, the documents as they were, and the nodes added, changed and removed between two revisions. |
|
|
30
|
+
| `update_ontology`, `update_entity_graph`, `update_rules`, `update_tests`, `update_spine` | Commit nodes (`upsert`, `remove`) to one document, on a `base` revision. `update_spine` also takes row edits (`rows`). |
|
|
31
|
+
| `get_spine`, `get_spine_rows` | The spine: tables (without their rows, with `rowCount`), facts with their `resolved` values, and warnings; then one table's rows a page at a time (`offset`, `limit`, `q` to search). |
|
|
32
|
+
| `generate_spine` | Run tables' generators and commit their rows (`dryRun: true` shows a sample and writes nothing). Present with `update_spine` when "Writes spine" is on. |
|
|
33
|
+
| `accept_factory_unit` | Check a unit's quotas against the rows the run holds: `{partition, key}` → each quota's expected and actual count. |
|
|
34
|
+
| `get_factory_data` | What this run wrote per member: row counts per entity in its draft and in Day 0. `origin: true` adds where each entity's rows came from (`real:<source>`, `generated`, `propagated`, `authored`, `seed`); `pending: true` lists the run's pending references. |
|
|
35
|
+
| `describe_member` | Each member's contract as a write must fit it: every entity's fields with type, required, allowed values (`enum`) and `format`, its keys and relationships, and the identities and references that touch the member. Pass `world` (and `entity`) to narrow it. |
|
|
36
|
+
| `write_records`, `query_factory_data` | Write rows into the active run's drafts, for one member or several at once (`unit` names the partition unit the write is for); read one entity's rows: the run's draft, else Day 0, else main (`source` says which). Present only when "Writes data" is on. |
|
|
37
|
+
| `release_factory_data` | Release the drafts as each member's Day 0 state, and end the run. Present only when "Releases Day 0" is on. |
|
|
38
|
+
| World sandbox tools | `open_world_sandbox` … `commit_world_sandbox`, here, over MCP and with the CLI: edit a member world or the bundle's relationships, confined to the bundle and its members and signed as you. You cannot add or remove members; a commit that would is refused, and a person makes that change. Present only when "Edits worlds" is on. |
|
|
39
|
+
| `request_factory_upload`, `confirm_factory_upload` | Upload an export's rows into the active run, tagged `real:<source>`. Over MCP and the CLI, when "Uploads real data" is on; an upload that starts a run also needs "Starts runs". |
|
|
40
|
+
| `set_factory_enrichment` | Open or close one member entity's real fields to agents. Over MCP and the CLI, when "Opens real rows to agents" is on. |
|
|
41
|
+
| `start_factory_run` | Start a run when none is active. It is recorded on the factory's orchestrator and has no chat, and it is refused (`factory_run_needs_orchestrator`) until a person has set the factory up. Over MCP and the CLI, when "Starts runs" is on. |
|
|
42
|
+
|
|
43
|
+
These last three switches start off for every agent, and only a person turns them on. Use them only for the upload, the entity or the run the person asked for.
|
|
44
|
+
|
|
45
|
+
## Writing rules and tests
|
|
46
|
+
|
|
47
|
+
Rule code is Python, run by the factory's rule runner: no network, no files, no clock, the same answer every time.
|
|
48
|
+
|
|
49
|
+
```python
|
|
50
|
+
def guard(view, change):
|
|
51
|
+
payment = view.one("acme-billing", "payments", id=change["row"]["payment_id"])
|
|
52
|
+
if payment is None:
|
|
53
|
+
return "unknown payment " + str(change["row"]["payment_id"])
|
|
54
|
+
return True if change["row"]["amount"] <= payment["amount"] else "a refund never exceeds its payment"
|
|
55
|
+
|
|
56
|
+
def derive(view, change):
|
|
57
|
+
return [{"world": "acme-billing", "entity": "ledger", "row": {"payment_id": change["row"]["payment_id"]}}]
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
- **`view`:** `rows(world, entity, **where)`, `one(world, entity, **where)`, `spine(node_id)` (a spine fact as `{value}`, a table as `{columns, key, rows}`, resolved; any other id reads that ontology node's `properties`), `statements(about=None)`, and `rng(name)`, the only randomness.
|
|
61
|
+
- **`change`:** `{world, entity, op, row, before, at, by: {principal, unit}}`. A hook guard on a tool call sees `{tool, input, by}`.
|
|
62
|
+
- **A guard returns `True` or a reason string.** Anything else is an error.
|
|
63
|
+
- **Imports:** only `math`, `statistics`, `decimal`, `datetime`, `re`, `json`, `collections`, `itertools` and `zoneinfo`. `now()` and `today()` are refused; take dates from the change or the spine. `hash`, `id`, `samples`, setting or deleting an attribute, and a bare `except` are refused too: catch the exception you expect.
|
|
64
|
+
- **Conventions:** guard the transition (`before` → `row`), not every write. Take roles from the roster, refuse an unknown reference by name, and join on reserved keys, never on text.
|
|
65
|
+
- **`rule_unit` test spec:** `{"rule": "<rule id>", "cases": [{"name", "rows": {"<world>.<entity>": [...]}, "spine"?, "change", "expect": "ok"|"deny", "reason"?, "derived"?: [...]}]}`. Write a passing case, the failure the rule exists for, and an edge (month end, Day 0, a time-zone boundary).
|
|
66
|
+
- **`chronicle` test spec:** `{"each": {"about": ["<node>"]}, "require": [{"cause": {"about": ["<node>"]}} | {"at": true} | {"causeBefore": true} | {"link": {"world", "entity"}}], "cases"?: [...]}`.
|
|
67
|
+
- **A commit that changes a rule's code or its tests runs those tests first.** Its answer carries `ruleTests`: warnings, and mutants no test caught ("`<=` → `<` on line 3 and every case still passes"). Add the case that would catch it.
|
|
68
|
+
|
|
69
|
+
## Writing the spine
|
|
70
|
+
|
|
71
|
+
Put every number and name two writers must agree on in the spine, once, and point at it.
|
|
72
|
+
|
|
73
|
+
- **Tables:** `{kind: "table", columns: [{name, type, ref?}], key: "<column>", rows: [...]}`. Types: `string`, `integer`, `number`, `boolean`, `date`, `instant`, `json`, `ref`. Key cells are written literally and are unique. Change rows with `rows: [{table, upsert, remove}]` in `update_spine`; an `upsert` of a table node sends all its rows.
|
|
74
|
+
- **Facts:** `{kind: "fact", value}`. **Checks only:** `{kind: "check", checks: [...]}`. Any spine node may carry `checks` and a `group` ("roster", "calendar", "storyline") for the Spine tab.
|
|
75
|
+
- **Derived values, not copies.** A value (a fact's value, a cell, a check's field) is a literal or: `{from: {kind: "node", id}, path: "/volume"}`, `{from: {kind: "row", node, key}, path: "/publishedOn"}`, `{from: {kind: "node", id: "<table>"}, column: "tickets"}` (a list), `{formula: "a * b", with: {a: …, b: …}}`, or `{literal: …}` for an object that only looks like one. Formulas are Python expressions, with `largest_remainder(total, weights)`, `lookup(rows, column, **where)`, `count(rows, **where)`, `total(rows, column, **where)` and `working_day(date, holidays, weekend)`. When you must keep a written number, write `{copy: 3200, from: …}`: a copy that drifts from its source is a warning.
|
|
76
|
+
- **Checks:** `flow` (`start + created − closed = end`), `sum` (`parts` add up to `total`), `quota` (`quotas` sum to `total`, split from `weights` by largest remainder), `fact` (a history fact's `claims` equal its recomputed `value`, within `tolerance`), `steps` (each storyline row's `names` cell is a row ref that resolves, and business steps fall on working days). Arithmetic failures refuse the commit (`factory_spine_checks_failed`); off-day steps and drifted copies come back as `spineChecks.warnings`.
|
|
77
|
+
- **Generators:** a table's `generate` is Python `def generate(ctx)` returning its rows. `ctx.rng("<concept>")` is the only randomness, one stream per concept, so changing one roster never reshuffles another; `ctx.spine(id)` and `ctx.rows(id)` read the spine. Run it with `generate_spine`; bulk rows come from code, not from you row by row.
|
|
78
|
+
- **Partitions:** a table whose rows are units: `partition: {owner: "def owner(view, change): …", entities: [{world, entity}], quotas: [{column, world, entity, tolerance?}]}`. `owner` returns the unit key that owns `change["row"]`.
|
|
79
|
+
- **Reserved keys:** `reserves: [{column, world, entity, field}]` reserves each row's value of `column` in that member field (both ends of a cross-world join, such as an escalation's ticket and issue).
|
|
80
|
+
|
|
81
|
+
## Writing data
|
|
82
|
+
|
|
83
|
+
- **Rows land in this run's draft of a member world**, not in the world itself. No session or run sees them until a person releases them as Day 0. A run starts from the seed chosen when it was started: by default the Day 0 the last run released (else main), so it adds to it; or main, nothing at all, a named state or a snapshot.
|
|
84
|
+
- **Read the contract first.** `describe_member` names each field's type, whether it is required, the values it allows and its format, so a write fits the first time.
|
|
85
|
+
- **Write to the member version the bundle pins.** Every read and check of a member (`describe_member`, every write, the release) uses the version the bundle pinned, even after the member's main has moved on. A world sandbox opens main, which may differ; trust `describe_member`. When the person wants a newer contract, they move the pin by committing the bundle.
|
|
86
|
+
- **One write is a slice:** `rows: {"<world>": {"<entity>": [rows]}}`, for any members of the bundle. It lands in every member or in none.
|
|
87
|
+
- **Lay structure first, then write the text with `mode: "patch"`.** A patch row names a stored row by its primary key and sends only the fields to change; objects merge, anything else replaces, nothing is filled, and the merged row is checked against the contract and the references. A patch for a row no member holds is refused as `row_not_found`. A patched row still carries its identity, so a member that lacks the person gets a counterpart.
|
|
88
|
+
- **A reserved key needs no order.** A reference whose value the spine reserves for its target passes, and the release refuses while a reserved value has no row (`reservation_unfulfilled`).
|
|
89
|
+
- **Write a unit's rows as that unit.** Send `unit: {partition, key}`: a row of an entity a partition owns is refused (`partition_other_unit`) unless the partition's owner function names your unit, both for the row you send and for the stored row it would replace. When the unit is done, `accept_factory_unit` compares its quotas with what the run holds.
|
|
90
|
+
- **Write referenced records first, or in the same slice.** The bundle's `[[references]]` (in `gateway-bundle.toml`) say which field names a record in another member; `write_records` refuses a row whose reference neither the slice nor any member holds. When the record comes later in the run, send `deferReferences: true`: the row lands and its reference stays pending (`pending.deferred` in the answer) until a write lands the record. Every write re-checks the pending list, and the release refuses while any remain.
|
|
91
|
+
- **Rows from real data keep their fields.** An upload tags its rows `real:<source>`. Changing a field such a row holds is refused (`real_field_protected`), and so is replacing an entity that holds such rows (`real_rows_protected`), unless that entity is open to enrichment. You may fill a field the real row lacks (a patch that sends only that field), and write new rows that name real ones.
|
|
92
|
+
- **People are written once.** For each `[[identities]]`, a row you write carries its identity value to every other member the identity spans: the platform writes the counterpart there, unless that member holds the value already or your slice writes it. Values match by the identity's key, through each span's transforms (an email under `lower` matches in any case). The answer counts counterparts in `propagated`, and `unpropagated` names a span no value reaches through its transforms.
|
|
93
|
+
- **Names travel with the person.** A span's `carry` (in `gateway-bundle.toml`) names fields, such as a name or a title, that are the same in every member. A carried field you send reaches every counterpart: a new one gets it, one a member holds is patched, and a counterpart you write without it gets it. Only fields you send travel, never a filled or stored value. Two values for one person's carried field in one write is `identity_conflict`. A carry into a member's primary key refuses the write; ask a person to carry it into another field. A counterpart's other fields are filled from the contract; write them yourself when they matter.
|
|
94
|
+
- **`complete: true` fills what a row leaves out** from the member's contract: defaults, the first enum value, null where allowed, a value hashed from the row for every key and unique field, and a parent in the same world. Send every field you care about; a filled date is one fixed instant. It never goes with a patch.
|
|
95
|
+
- **Give every row its own dates.** Nothing runs to advance a clock.
|
|
96
|
+
- **Write a few hundred rows at a time**, at most 5,000 a call. A row replaces the row with the same key; `mode: "replace"` empties each entity the write sends rows for first; an entity named with no rows is refused.
|
|
97
|
+
- **Check what landed** with `get_factory_data` (row counts per entity) and `query_factory_data`.
|
|
98
|
+
- **Release only when the person says so.** `release_factory_data` makes each member's draft its Day 0 state and ends the run; a member the run never wrote is released at its main data. It refuses while any `[[references]]` row names a record no member holds, and lists each one.
|
|
99
|
+
|
|
100
|
+
## From a terminal
|
|
101
|
+
|
|
102
|
+
An agent with a factory-agent key (`fk_…`) stored by `gateway auth login` can do all of this with the `gateway` CLI. Each verb is the tool above that does the same work, under the same switch and the factory's rules:
|
|
103
|
+
|
|
104
|
+
- **Documents:** `gateway factory show|revisions|diff <bundle>`, and `gateway factory edit <bundle> --file edits.json --base <rev> --doc <ontology|graph|rules|tests|spine>`. Your edit must name its document with `--doc`.
|
|
105
|
+
- **Spine:** `gateway factory spine show <bundle>`, `spine rows <bundle> <table>`, `spine generate <bundle> <table>… --base <rev>` and `spine accept <bundle> <partition> <key>`.
|
|
106
|
+
- **Data:** `gateway factory describe|records|data|release <bundle>`, `gateway factory upload <bundle> <file> --world <world>`, `gateway factory enrichment <bundle> --world <world> --entity <entity> --on|--off` and `gateway factory run start <bundle>`.
|
|
107
|
+
- **Worlds:** `gateway worlds sandbox open <member>`, then `status`, `read`, `write`, `exec`, `check`, `commit`, `import` and `close` on that sandbox.
|
|
108
|
+
|
|
109
|
+
The key never reaches `gateway factory agents` (or its switches), `run stop`, `chat` or `sandbox list`. Those are a person's.
|
|
110
|
+
|
|
111
|
+
## How to work
|
|
112
|
+
|
|
113
|
+
1. **Read first.** Call `get_factory` and note `head`. Every write passes the latest `head` you saw as `base`.
|
|
114
|
+
2. **Agree before writing.** Restate what you understood in a sentence or two. When the request can be read two ways, ask.
|
|
115
|
+
3. **Write small.** Commit a few nodes at a time, with a `message` that says what changed. Reuse ids; never change an id to change a name.
|
|
116
|
+
4. **Write in sentences.** A node's `sentence` is what a person reads first. Make it true and specific.
|
|
117
|
+
5. **Point, don't copy.** Link to a world, a schema field or another node with `refs` instead of repeating it.
|
|
118
|
+
|
|
119
|
+
## When something pushes back
|
|
120
|
+
|
|
121
|
+
- **`factory_conflict`:** someone changed the same field. Re-read the head, re-apply your change to the new text and commit on the new base. If the two intents disagree, ask the person.
|
|
122
|
+
- **`factory_invalid`:** each issue names its node and field. Fix those and commit again; nothing was written.
|
|
123
|
+
- **`factory_tool_not_granted`:** that switch is off for you. Tell the person which switch the change needs. Never reach for another tool to make the same change.
|
|
124
|
+
- **A result that carries `changes`:** someone moved the head since your last call; the line says what. When it touches what you are doing, re-read before your next write.
|
|
125
|
+
- **`factory_spine_checks_failed`:** each issue names a spine node, the check and the numbers. Fix the values or the facts they read; nothing was written.
|
|
126
|
+
- **`factory_spine_generate_failed`:** a generator raised or returned rows that are not objects; the error names the table. Fix its code; nothing was written.
|
|
127
|
+
- **`factory_unit_unknown`:** the `unit` names no partition table or no row of it.
|
|
128
|
+
- **`factory_rule_tests_failed`:** a `block` test failed; each failure names its test and case. Fix the rule or the test; nothing was written.
|
|
129
|
+
- **`factory_hook_denied`:** a rule's guard refused the call; `rule` and `reason` say why. Do what the reason asks, or tell the person; never retry the same call unchanged.
|
|
130
|
+
- **`factory_records_refused`:** each refusal names a row (`world`, `entity`, `index`, and `propagated` when it is a counterpart the platform wrote) and why: `reference_broken` (write the referenced record first, fix the value, or send `deferReferences: true` when it comes later), `row_refused` (the world's contract), `row_not_found` (a patch for a row no member holds; write it without `patch` first), `identity_conflict` (two values for one person's carried field; send one), `real_field_protected` or `real_rows_protected` (real data; leave those fields as they are, or ask the person to enable enrichment), `partition_other_unit` (another unit owns that row; leave it to that unit) or `partition_owner_failed` (the partition's owner function raised; fix it or tell the person). Nothing was written; fix those rows and send the write again.
|
|
131
|
+
- **`factory_run_required`, `factory_run_released`, `factory_run_stopped`, `factory_budget_reached`:** no run is active, or it ended, so nothing can be written. Tell the person. Start a run yourself only when the person asked for one and "Starts runs" is on.
|
|
132
|
+
- **`factory_agent_key_mcp_only`, `factory_roster_people_only`:** that is a person's: the roster, switches, keys, stopping a run or the chat. Tell the person what you need.
|
|
133
|
+
- **`factory_run_busy`:** another write or the release held the run, or other writes kept moving the drafts while yours was built. Nothing of yours was kept in any member; send the same write again.
|
|
134
|
+
- **`factory_run_hold_ended`:** your write ran past the run's lock (three minutes). Nothing of yours was kept; send the same rows again as smaller slices.
|
|
135
|
+
- **`factory_draft_name_taken`:** a member already has a world state named like this run's draft. Tell the person; nothing was written.
|
|
136
|
+
- **`factory_records_failed`:** a member's import failed, and the message names it. Nothing of yours was kept; read the drafts with `get_factory_data` before writing again.
|
|
137
|
+
- **`factory_state_carrying`:** a person moved the bundle's pin, and a member's draft or Day 0 is being carried onto the new version. Nothing was written; wait a minute, then read or send the same call again. After three such answers for the same member, stop and tell the person.
|
|
138
|
+
- **`factory_state_not_carried`:** that carry failed or was refused. When the message says a write, a release or a run start queues it again, at most once every 10 minutes, send the same write again after that. When it says to start from main or empty, tell the person: this run's draft cannot follow the new pin.
|
|
139
|
+
- **`factory_state_off_pin`:** a member's state was built on a version past the one the bundle pins. Nothing was written; tell the person, who can commit the bundle to move the pin, or start a run from main or empty.
|
|
140
|
+
- **`factory_release_refused`:** each refusal names a member world, a row and the value no member holds (`pending` when the row was deferred, `reservation_unfulfilled` when the spine reserved it). Write the missing records or fix the rows, then release again; nothing was released.
|
|
141
|
+
- **`factory_release_conflict`:** a member's draft or Day 0 state changed outside this run. Check what the members hold, then release again; nothing was released.
|
|
142
|
+
- **`factory_rule_runner_unavailable`:** the rule runner could not start, so nothing that needs it ran. Tell the person; try again later.
|
|
143
|
+
|
|
144
|
+
## What never happens
|
|
145
|
+
|
|
146
|
+
- No document leaves the platform: never paste a whole document into a file, a world or a message outside this factory.
|
|
147
|
+
- Never invent a kind, a rule or a relationship the person did not ask for. Propose it, and wait.
|
|
@@ -260,7 +260,10 @@ A `.csv`/`.tsv` (optionally `.gz`) is read as a stream — memory is bounded at
|
|
|
260
260
|
split by `--delimiter` (one character, `\t` for a tab; default `,`, tab for a `.tsv`). The header
|
|
261
261
|
row names the fields; `--entity` says which entity they fill. Every cell is cast by that field's
|
|
262
262
|
schema type, the same rule in the TypeScript CLI (`worldData/cast.ts`) and the Python package
|
|
263
|
-
(`schema/cast.py`), pinned by `gatewaysdk/sdk/tests/conformance/csv_cast.json
|
|
263
|
+
(`schema/cast.py`), pinned by `gatewaysdk/sdk/tests/conformance/csv_cast.json`. The type is the one
|
|
264
|
+
`worlds describe` reports: the JSON type words the field admits through `$ref`s (into sibling
|
|
265
|
+
`schema/*.json` files too) and unions, so a field that is a reference to an object casts as JSON and
|
|
266
|
+
`anyOf [Count, null]` as `integer | null`:
|
|
264
267
|
|
|
265
268
|
| Field type | Cell | Value |
|
|
266
269
|
| --- | --- | --- |
|
|
@@ -270,6 +273,7 @@ schema type, the same rule in the TypeScript CLI (`worldData/cast.ts`) and the P
|
|
|
270
273
|
| `array` / `object` | JSON text of that shape (quote it in the CSV: `"[""a"",""b""]"`) | the parsed value |
|
|
271
274
|
| `string` | anything | the text as given |
|
|
272
275
|
| `integer \| string` (union) | each member tried in order, `string` last | the first that fits |
|
|
276
|
+
| `number \| boolean \| string \| null` (a union that admits string) | `00501`, `true` | the typed member that fits first (`501.0`, `true`): leading zeros and literal text do not survive; import JSON or JSONL to keep a value as text |
|
|
273
277
|
| any, empty cell | (nothing) | null when the type admits null; `""` for a required string; left out otherwise (a required one is then refused by the contract) |
|
|
274
278
|
|
|
275
279
|
A cell that fits none stays its text and the contract refuses the row as it refuses a JSON row
|
|
@@ -101,7 +101,7 @@ commit, with a warning); bulk rows reach the platform world through `worlds sand
|
|
|
101
101
|
|
|
102
102
|
```bash
|
|
103
103
|
gateway worlds describe <WORLD> # entities, keys, relationships, row counts, and how each field is stored
|
|
104
|
-
gateway worlds describe <WORLD> --json
|
|
104
|
+
gateway worlds describe <WORLD> --json [--entity e] # the document: fields[].redaction, .enum, .format, redact, catalog (enums, JSON shapes, custom fields, ids, sequences)
|
|
105
105
|
gateway worlds data show <WORLD> # {world, kind, entity_counts}
|
|
106
106
|
gateway worlds data show <WORLD> --entity users # the rows of one entity
|
|
107
107
|
gateway worlds data query <WORLD> users --where plan=pro --limit 50 # the rows you mean; a repeated --where key is IN
|
|
@@ -170,7 +170,7 @@ chunks the platform does not already hold. `--publish later` applies without a v
|
|
|
170
170
|
`gateway worlds data publish <slug>` folds every pending batch into one; `data batches <slug>` lists them.
|
|
171
171
|
A refusal record, everywhere (dry run, report, status, MCP):
|
|
172
172
|
`{"chunk", "index", "entity", "primaryKey", "field", "pointer", "detail"}`, `pointer` being
|
|
173
|
-
`/<entity>/<index>/<field>` inside the chunk.
|
|
173
|
+
`/<entity>/<index>/<field>` inside the chunk (`chunk`, `index` null when it spans chunks).
|
|
174
174
|
|
|
175
175
|
MCP: `import_world_data({ containerId | slug, rows, mode?, message? })`. `rows` is the JSON
|
|
176
176
|
object shape (`{entity: [rows]}`), at most 5,000 rows and 4 MiB per call: the inline lane,
|
|
@@ -495,6 +495,6 @@ handler or route edit keeps the imported data while an entity, field or key edit
|
|
|
495
495
|
|
|
496
496
|
- `worlds-getting-started`: the whole path, auth to runs, CLI with MCP twins.
|
|
497
497
|
- `connector-schema-authoring`: the data model the rows are checked against.
|
|
498
|
-
- `documentation/features/world-data-ingestion.md
|
|
499
|
-
- `
|
|
498
|
+
- `documentation/features/world-data-ingestion.md` (the contract, code pointers) and `connector-worlds.md` (captures, ingest, the vendor's API).
|
|
499
|
+
- `factory-orchestrator`: a bundle's factory run takes the same files at any point with `gateway factory upload <bundle> <file> --world <member>` (chunked, `--map ingest.toml`, `--per-chunk`, `--dry-run`, `--resume`, `--report`), checked across members and tagged `real:<source>`, so agents may not change those fields.
|
|
500
500
|
- Consumer page: https://docs.surfacearea.ai/worlds/data.
|
|
@@ -188,7 +188,7 @@ MCP: `validate_connector_template({ files, slug? })` compiles without writing; a
|
|
|
188
188
|
## 4. Data
|
|
189
189
|
|
|
190
190
|
```bash
|
|
191
|
-
gateway worlds describe ./x
|
|
191
|
+
gateway worlds describe ./x [--entity e] [--json] # entities, keys, row counts, [redact] per field; --json: the shape catalog
|
|
192
192
|
gateway worlds data show ./x # kind + entity counts
|
|
193
193
|
gateway worlds data query ./x users --where email=jo@acme.test --resolve # the rows you mean; plaintext into a hashed field
|
|
194
194
|
gateway worlds data import ./x rows.json # rows file: {entity: [rows]} | JSONL | .csv/.tsv --entity (cells cast by the schema; --delimiter)
|
|
@@ -204,7 +204,7 @@ gateway worlds data ingest ./x calls.jsonl --map ingest.toml # or a
|
|
|
204
204
|
shape-preserved (`[preserve]`), rounded, or dropped (`[drop]`, never on disk). A hashed field is still looked up by
|
|
205
205
|
its plaintext: the world's `[redact]` policy fixes the stored form (`sha256:` of `<slug>:<value>`), and the CLI, the
|
|
206
206
|
SDK (`storedForms` in `@withgateway/sdk/worlds`) and the served handlers all compute it from that one rule. A dropped
|
|
207
|
-
field is a documented miss, not an error.
|
|
207
|
+
field is a documented miss, not an error. Tags are capture's forms: other writes land as written (`--json` has the legend). `--json` also carries the **shape catalog** (enums at any depth, JSON shapes, custom fields, ids, sequences, each route's answer shape; MCP `describe_world`): `references/shape-catalog.md`.
|
|
208
208
|
|
|
209
209
|
Rows file contract, modes, upsert and refusals: `world-data-ingestion`. Captures: pace with `[capture] requests_per_minute`;
|
|
210
210
|
a vendor that allow-lists IPs is captured with `--remote` after `gateway bench secrets set --env x --key VENDOR_API_KEY`.
|
|
@@ -17,6 +17,7 @@ Reference for `worlds-getting-started`: every CLI step and its MCP twin, then ho
|
|
|
17
17
|
| `worlds migrate <slug> [--apply] [--force]` / `--all` | `migrate_world` |
|
|
18
18
|
| `bench push` | `env_write_file` + `env_commit`, or `create_environment_version` (one file) |
|
|
19
19
|
| `worlds data import [--state <name>]` | `import_world_data` (`worldState`: the rows land in that world state; main's data and versions do not move; the bulk lane is `request_data_upload` with `worldState`, refused with `publish: "later"`) |
|
|
20
|
+
| `worlds describe <world> [--entity <e>] [--json]` | `describe_world({ slug, ref?, entity? })` (`GET /api/public/worlds/{slug}?ref=&entity=`): entities with keys, row counts and each field's `[redact]` form, plus `catalog` (enumerations, JSON columns, custom fields, ids, sequences, the redaction legend, answer shapes per route; `references/shape-catalog.md`); an unknown entity is refused naming the entities |
|
|
20
21
|
| `worlds data sql <world> '<sql>' [--limit] [--scenario] [--json]` | `sql_world_data` (`POST /api/public/worlds/{slug}/data/sql`; ONE read-only statement over a schema world's rows, whatever its engine; a write, `ATTACH`, `PRAGMA` or a second statement is refused; at most 1000 rows. A db/schema.sql world is a SQLite file: `sqlite3` in a sandbox) |
|
|
21
22
|
| `worlds state list <world>` | `list_world_states({ slug })` (`GET /api/public/worlds/{slug}/states`): the world's states by name, each with its `bindings` and `dataSnapshotId` (null = the state opens on the version's own data). A world state is a named starting point: data plus knob values, opened on any version |
|
|
22
23
|
| `worlds state show <world> <name> [--ref <ref>]` | `get_world_state({ slug, name, ref? })` (`GET …/states/{name}?ref=`): how the state opens on that version (`opens`: `source` `version`\|`state`\|`carried`, `snapshot`, `bindings`, `fingerprint`), or why it does not (`refusal.code`: `state_knobs`, `state_not_carried`, `state_carrying`, `state_data_not_dispatched`); a world with no READY version answers `versionId`, `opens` and `refusal` null |
|