@arcforge/platform 2.0.219 → 2.0.244
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +27 -19
- package/src/build/blueprint/blueprint.ts +48 -64
- package/src/build/blueprint/cognet/abi.ts +3 -257
- package/src/build/blueprint/cognet/bundle-worker.ts +11 -0
- package/src/build/blueprint/cognet/bundle.ts +294 -89
- package/src/build/blueprint/cognet/cognet.ts +8 -7
- package/src/build/blueprint/cognet/config/config.ts +52 -0
- package/src/build/blueprint/cognet/config/index.ts +4 -0
- package/src/build/blueprint/cognet/config/literal.ts +83 -0
- package/src/build/blueprint/cognet/config/provider.ts +43 -0
- package/src/build/blueprint/cognet/config/shape.ts +183 -0
- package/src/build/blueprint/cognet/index.ts +4 -2
- package/src/build/blueprint/cognet/native/cargo.ts +31 -0
- package/src/build/blueprint/cognet/native/crate.ts +126 -0
- package/src/build/blueprint/cognet/native/index.ts +2 -0
- package/src/build/blueprint/cognet/native/library.ts +125 -0
- package/src/build/blueprint/cognet/native/manifest.ts +137 -0
- package/src/build/blueprint/cognet/python/index.ts +1 -0
- package/src/build/blueprint/cognet/python/project.ts +119 -0
- package/src/build/blueprint/cognet/schema/arrays.ts +23 -0
- package/src/build/blueprint/cognet/schema/authored.ts +273 -0
- package/src/build/blueprint/cognet/schema/compile.ts +78 -0
- package/src/build/blueprint/cognet/schema/components/external.ts +15 -0
- package/src/build/blueprint/cognet/schema/components/index.ts +3 -0
- package/src/build/blueprint/cognet/schema/components/lock.ts +58 -0
- package/src/build/blueprint/cognet/schema/components/model.ts +34 -0
- package/src/build/blueprint/cognet/schema/components/python.ts +15 -0
- package/src/build/blueprint/cognet/schema/components/python_source.py +69 -0
- package/src/build/blueprint/cognet/schema/components/rust.ts +33 -0
- package/src/build/blueprint/cognet/schema/components/rust_source/Cargo.lock +117 -0
- package/src/build/blueprint/cognet/schema/components/rust_source/Cargo.toml +11 -0
- package/src/build/blueprint/cognet/schema/components/rust_source/src/main.rs +104 -0
- package/src/build/blueprint/cognet/schema/components/source.ts +220 -0
- package/src/build/blueprint/cognet/schema/components/tags.ts +66 -0
- package/src/build/blueprint/cognet/schema/components/typescript.ts +157 -0
- package/src/build/blueprint/cognet/schema/index.ts +5 -0
- package/src/build/blueprint/cognet/schema/interfaces/index.ts +3 -0
- package/src/build/blueprint/cognet/schema/interfaces/source.ts +218 -0
- package/src/build/blueprint/cognet/schema/python.ts +260 -0
- package/src/build/blueprint/cognet/schema/rust.ts +316 -0
- package/src/build/blueprint/cognet/schema/schema.ts +248 -0
- package/src/build/blueprint/cognet/schema/typescript.ts +352 -0
- package/src/build/blueprint/index.ts +1 -1
- package/src/build/blueprint/modules/index.ts +7 -5
- package/src/build/blueprint/modules/meta.ts +47 -3
- package/src/build/blueprint/scan/config.ts +22 -16
- package/src/build/blueprint/scan/provider-globals.ts +46 -0
- package/src/build/blueprint/scan/providers.ts +25 -0
- package/src/build/blueprint/scan/tools.ts +21 -15
- package/src/build/extensions/edit.ts +66 -2
- package/src/build/extensions/extensions.ts +16 -3
- package/src/build/extensions/index.ts +1 -1
- package/src/build/project/bundle/agent.ts +26 -17
- package/src/build/project/bundle/source.ts +2 -2
- package/src/build/project/create/cognet.ts +84 -33
- package/src/build/project/create/profile.ts +13 -3
- package/src/build/project/index.ts +1 -1
- package/src/build/project/installer.ts +6 -5
- package/src/build/project/kinds.ts +4 -0
- package/src/build/project/manifest/env.ts +10 -0
- package/src/build/project/models/blobs/blobs.ts +57 -0
- package/src/build/project/models/blobs/index.ts +1 -0
- package/src/build/project/models/fetch.ts +46 -51
- package/src/build/project/models/index.ts +2 -1
- package/src/build/project/models/models.ts +12 -38
- package/src/build/project/models/onnx.ts +82 -0
- package/src/build/project/models/specifier.ts +9 -3
- package/src/build/project/models/storage/index-store.ts +27 -0
- package/src/build/project/models/storage/index.ts +1 -0
- package/src/build/project/models/store.ts +21 -16
- package/src/build/project/prepare.ts +82 -27
- package/src/build/project/project.ts +8 -14
- package/src/build/project/projects.ts +12 -1
- package/src/build/project/tree.ts +13 -3
- package/src/build/project/treecache.ts +8 -4
- package/src/build/project/typegen/axon-dts.ts +18 -0
- package/src/build/project/typegen/cognet-dts.ts +248 -59
- package/src/build/project/typegen/prompt-dts.ts +2 -3
- package/src/build/project/typegen/registry-dts.ts +2 -2
- package/src/build/project/typegen/tsconfig.ts +13 -13
- package/src/build/project/typegen/tui-dts.ts +3 -4
- package/src/build/project/typegen/workspace-scripts.ts +8 -7
- package/src/build/project/typegen/write.ts +15 -2
- package/src/build/project/watcher.ts +27 -0
- package/src/build/runtime/agent.ts +102 -63
- package/src/build/runtime/conformance.ts +25 -0
- package/src/build/runtime/escalations.ts +3 -8
- package/src/build/runtime/index.ts +2 -0
- package/src/build/runtime/instances.ts +7 -4
- package/src/build/runtime/progress.ts +6 -2
- package/src/build/runtime/reload-watch.ts +11 -8
- package/src/debug/README.md +398 -0
- package/src/debug/debug.ts +101 -0
- package/src/debug/events.ts +156 -0
- package/src/debug/graph.ts +94 -0
- package/src/debug/index.ts +5 -0
- package/src/debug/indexer.ts +286 -0
- package/src/debug/inference.ts +53 -0
- package/src/debug/log.ts +68 -0
- package/src/debug/overview.ts +78 -0
- package/src/debug/perf.ts +75 -0
- package/src/debug/resources.ts +125 -0
- package/src/debug/ring.ts +31 -0
- package/src/debug/sessions.ts +90 -0
- package/src/debug/signs.ts +119 -0
- package/src/debug/tally.ts +44 -0
- package/src/debug/types.ts +146 -0
- package/src/index.ts +9 -0
- package/src/link/agent-main.ts +36 -10
- package/src/link/index.ts +1 -1
- package/src/link/services.ts +88 -57
- package/src/platform.ts +17 -25
- package/src/services/cloud.ts +1 -4
- package/src/services/control/client.ts +1 -1
- package/src/services/control/server.ts +1 -1
- package/src/services/index.ts +1 -0
- package/src/services/ollama/daemon.ts +23 -0
- package/src/services/ollama/http.ts +7 -1
- package/src/services/ollama/ollama.ts +8 -14
- package/src/services/probe/ask.ts +139 -0
- package/src/services/probe/index.ts +3 -0
- package/src/services/providers.ts +18 -0
- package/src/services/resources/hardware.ts +15 -18
- package/src/services/resources/resources.ts +4 -5
- package/src/services/store/types.ts +3 -0
- package/src/services/running/index.ts +0 -1
- package/src/services/running/running.ts +0 -216
|
@@ -0,0 +1,398 @@
|
|
|
1
|
+
# axon debug
|
|
2
|
+
|
|
3
|
+
The tools an agent uses to see inside a running or recorded mind.
|
|
4
|
+
|
|
5
|
+
A person debugs with Mission Control. An agent debugs with this: a small set
|
|
6
|
+
of commands that answer, from a terminal, every question the debugger
|
|
7
|
+
answers on screen. Built for an agent first — exact, bounded, machine-read —
|
|
8
|
+
and usable by a person second.
|
|
9
|
+
|
|
10
|
+
This file is the specification. It is written before the code and kept
|
|
11
|
+
current with it.
|
|
12
|
+
|
|
13
|
+
## Why it exists
|
|
14
|
+
|
|
15
|
+
A good debugger for the person is half of the job. The agent working on the
|
|
16
|
+
code needs the same reach, or every change waits on a person booting the
|
|
17
|
+
game and describing what they see. With these commands an agent can find
|
|
18
|
+
the last run, see what failed, read any value at any tick, follow a value
|
|
19
|
+
back to its causes, and look at a frame with layers drawn over it.
|
|
20
|
+
|
|
21
|
+
## Principles
|
|
22
|
+
|
|
23
|
+
1. **One door.** Everything is under `axon debug`. Nothing is added to the
|
|
24
|
+
top-level CLI.
|
|
25
|
+
2. **A small bag, chosen.** Each command answers one kind of question. A
|
|
26
|
+
command is added when a question cannot be answered, never because the
|
|
27
|
+
data happens to be there. Too many tools is the same as none.
|
|
28
|
+
3. **JSON out, always bounded.** One JSON object on stdout. Lists stop at a
|
|
29
|
+
default limit and say so (`truncated`, `next`). No command can print a
|
|
30
|
+
500 MB log by accident.
|
|
31
|
+
4. **Recorded and running are the same.** A running session is read from
|
|
32
|
+
the same files as a finished one. Where a command cannot answer from
|
|
33
|
+
them (a heavy array, recorded as a digest) it fails and says why.
|
|
34
|
+
5. **Read-only by default.** Only `set` and `prune` change anything, and
|
|
35
|
+
`prune` does nothing without `--apply`.
|
|
36
|
+
6. **Never guess.** A value the record does not hold is reported as not
|
|
37
|
+
recorded, not as zero or empty.
|
|
38
|
+
|
|
39
|
+
## Status
|
|
40
|
+
|
|
41
|
+
Step 1 is built: `sessions`, `session`, `graph`, `errors`, `events`, `perf`,
|
|
42
|
+
`resources` and `prune`. `state`, `why`, `field` and `set` are specified below and not
|
|
43
|
+
built. `live` as a session name is not wired; a running session's files are
|
|
44
|
+
read the same way, so `latest` names it.
|
|
45
|
+
|
|
46
|
+
## What is recorded
|
|
47
|
+
|
|
48
|
+
Each agent keeps its data under `<agent>/.agent/data`, in four places with
|
|
49
|
+
four lifetimes.
|
|
50
|
+
|
|
51
|
+
| Data | Path | Holds | Kept |
|
|
52
|
+
|---|---|---|---|
|
|
53
|
+
| Session log | `sessions/<id>.jsonl` | What the mind declared, and what happened to it | For good |
|
|
54
|
+
| Telemetry | `telemetry/<id>/NNNNNN.axt` | Every tick, system run and component write | A window: 1 GB |
|
|
55
|
+
| Sensory | `sensory/<id>/NNNNNN.jsonl` | Dense stimuli as they arrived | A window: 1 GB |
|
|
56
|
+
| Media | `media/<id>/` | Frames sent by reference | A window: 1 GB |
|
|
57
|
+
|
|
58
|
+
The three windows are kept for the newest three sessions only. When a
|
|
59
|
+
session opens, the windows of every older session are removed; their logs
|
|
60
|
+
stay. So `perf`, `events` and `graph` answer for a recent session, and an
|
|
61
|
+
older one has its log alone: what it declared, its faults and its usage.
|
|
62
|
+
|
|
63
|
+
**The log** holds the build, the schema (`cognet:component:declare`,
|
|
64
|
+
`cognet:frame:declare`), the graph (`cognet:schedule:declare`), stimuli and
|
|
65
|
+
outputs that are not dense, kernel runs, usage and errors.
|
|
66
|
+
|
|
67
|
+
The log also holds a sample of what the agent's processes used, every
|
|
68
|
+
couple of seconds (`axon:usage`): memory, CPU and GPU by part, and CPU by
|
|
69
|
+
thread. It is in the log, so `resources` covers the whole session.
|
|
70
|
+
|
|
71
|
+
**Telemetry** holds what a mind does tick by tick (`TELEMETRY_EVENTS` in
|
|
72
|
+
`@arcforge/types`), in a binary format of its own:
|
|
73
|
+
|
|
74
|
+
| Event | Gives |
|
|
75
|
+
|---|---|
|
|
76
|
+
| `cognet:system:run` | One record per run: tick, start, duration, outcome, thread, and **what it actually read, by revision** |
|
|
77
|
+
| `cognet:component:add / update / remove` | Every logged write: tick, system, entity, component, revision, value |
|
|
78
|
+
| `cognet:tick:*`, `cognet:phase:*` | The clock |
|
|
79
|
+
|
|
80
|
+
Every event in either place carries the session's own `seq`, so the two are
|
|
81
|
+
one ordered stream.
|
|
82
|
+
|
|
83
|
+
Four limits follow, and the commands state them rather than hide them:
|
|
84
|
+
|
|
85
|
+
- **Telemetry is a window.** Terry writes about 1.9 MB of it a second
|
|
86
|
+
(measured 2026-10-02; half of that is one debug field, `shapes`), so
|
|
87
|
+
1 GB is the last nine minutes or so. Timings, writes and causes are
|
|
88
|
+
known for that stretch and not before it. `perf` and `session` say which
|
|
89
|
+
stretch they cover.
|
|
90
|
+
- **Resident components are not recorded.** `sight`, `features`, `held` and
|
|
91
|
+
the like are working state, too large to write down. They can be read on
|
|
92
|
+
a live session only.
|
|
93
|
+
- **An async run does not record its reads.** `why` stops at an async
|
|
94
|
+
system and says so.
|
|
95
|
+
- **A tick's time is its own.** The schedule opens and closes each tick, so
|
|
96
|
+
a tick's recorded duration is start to end: its systems, the workers it
|
|
97
|
+
waited for, and any lagged run it had to wait to land — not the async work
|
|
98
|
+
it started, which lands later. A tick that waited records on what
|
|
99
|
+
(`held`); `perf` reports that apart from the main thread's share. (A
|
|
100
|
+
runtime from before 2026-10-09 timed each wake instead.)
|
|
101
|
+
|
|
102
|
+
## Addressing
|
|
103
|
+
|
|
104
|
+
```
|
|
105
|
+
axon debug <command> [<session>] [flags] [--agent <dir>]
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
- `--agent` is the agent's directory. Default: the current one.
|
|
109
|
+
- `<session>` is a session id, any prefix of one that only one session has,
|
|
110
|
+
or `latest`. Default: `latest`.
|
|
111
|
+
- A system or component is named in full (`L0/world/map/map`) or by the end
|
|
112
|
+
of its name (`map`) when only one ends that way.
|
|
113
|
+
|
|
114
|
+
Every answer is one line of JSON with the same envelope:
|
|
115
|
+
|
|
116
|
+
```json
|
|
117
|
+
{ "ok": true, "session": "a64951ad-…", "data": { }, "truncated": false, "next": null }
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
A failure is `{ "ok": false, "code": "AX-…", "message": "…" }` with a
|
|
121
|
+
non-zero exit code.
|
|
122
|
+
|
|
123
|
+
## Shaping an answer
|
|
124
|
+
|
|
125
|
+
Every command that returns rows takes the same few flags, so each composes
|
|
126
|
+
the same way and an answer is only as large as the question.
|
|
127
|
+
|
|
128
|
+
| Flag | Does |
|
|
129
|
+
|---|---|
|
|
130
|
+
| `--where <key><op><value>` | Keeps rows that pass. Repeat to narrow further. `=` `!=` `>` `<` `>=` `<=`, and `~` for contains |
|
|
131
|
+
| `--search <text>` | Keeps rows that mention it anywhere, whatever the case |
|
|
132
|
+
| `--sort <key>` `[--desc]` | Orders by a key |
|
|
133
|
+
| `--limit <n>` `[--last]` | At most n rows: the first, or the last |
|
|
134
|
+
| `--fields <key,key>` | Gives back only these keys of each row |
|
|
135
|
+
|
|
136
|
+
A key is a path into a row: `bytes.total`, `data.system`,
|
|
137
|
+
`data.value.depth`. Numbers compare as numbers. A row without the key
|
|
138
|
+
fails every test but `!=`.
|
|
139
|
+
|
|
140
|
+
```
|
|
141
|
+
axon debug sessions --sort bytes.total --desc --limit 5 --fields id,seconds,bytes.total
|
|
142
|
+
axon debug perf --where "p95Ms>50" --where async=false --fields system,p95Ms,maxMs
|
|
143
|
+
axon debug events --component entity --where "data.value.depth>0.5" --last --limit 3
|
|
144
|
+
axon debug events --search "no fit" --fields type,data.system,data.error.message
|
|
145
|
+
```
|
|
146
|
+
|
|
147
|
+
The rows of each command: `sessions` its sessions; `graph` its systems and
|
|
148
|
+
components; `errors` its faults and troubles; `events` its events; `perf`
|
|
149
|
+
its systems; `resources` its threads.
|
|
150
|
+
|
|
151
|
+
On `events`, `--where` and `--search` are applied where the events are
|
|
152
|
+
read, before the limit — they narrow the whole session, not the first
|
|
153
|
+
fifty. On `sessions`, a test, a sort or `--last` reads every session, which
|
|
154
|
+
takes a few seconds with hundreds of them; without one, only as many as
|
|
155
|
+
are asked for are read.
|
|
156
|
+
|
|
157
|
+
The shaping is presentation, and lives with the CLI
|
|
158
|
+
(`apps/tui/cli/commands/shape.ts`).
|
|
159
|
+
|
|
160
|
+
## The commands
|
|
161
|
+
|
|
162
|
+
### Find
|
|
163
|
+
|
|
164
|
+
**`sessions`** — the recent sessions, newest first, twenty unless told
|
|
165
|
+
otherwise. Each row: id, started, last written, seconds, and
|
|
166
|
+
bytes on disk by kind. Read from headers and the filesystem alone, so it
|
|
167
|
+
costs nothing however large the sessions are.
|
|
168
|
+
|
|
169
|
+
**`session`** — one session on one screen. `signs` first: everything that
|
|
170
|
+
says it is not well, one line each (see Signs). Then the mind (systems,
|
|
171
|
+
components, target rate); tick rate against target and ticks over the
|
|
172
|
+
deadline; faults; troubles; the five costliest systems; memory, CPU and
|
|
173
|
+
the busiest threads; bytes on disk by kind.
|
|
174
|
+
The command to run first on any report of "something is wrong".
|
|
175
|
+
|
|
176
|
+
#### Signs
|
|
177
|
+
|
|
178
|
+
`session` reads these off what the other commands measure, so one command
|
|
179
|
+
says whether to look further and where. `checked` lists every sign looked
|
|
180
|
+
for: one missing from `found` was checked and was fine.
|
|
181
|
+
|
|
182
|
+
| Sign | Raised when | Look further with |
|
|
183
|
+
|---|---|---|
|
|
184
|
+
| `faults` | Any event ending `:failed` or `:error`, or a `build:warning` (a plugin that did not load) | `errors` |
|
|
185
|
+
| `failing` | A system had a run that did not complete | `errors`, `events --system` |
|
|
186
|
+
| `slow` | Under nine tenths of the mind's own tick rate | `perf` |
|
|
187
|
+
| `deadline` | More than one tick in fifty held the main thread past the deadline | `perf` |
|
|
188
|
+
| `late` | More than one tick in fifty ran past the deadline start to end — workers and lagged runs waited on included; names the systems ticks waited on, longest first | `perf` (`held`, `heldMs`) |
|
|
189
|
+
| `memory` | A part's memory still rising 10 MB a minute over the later half of the session — judged only once that half spans two minutes: a shorter stretch cannot tell growth from the collector's sawtooth | `resources` |
|
|
190
|
+
| `saturated` | The whole machine at 90% CPU or more on average; says what the agent's parts took and which processes took the rest | `resources` |
|
|
191
|
+
| `inference` | Model calls' durations add up to nine tenths or more of the time they were made in: calls waiting on each other | `inference` |
|
|
192
|
+
| `troubles` | A trouble is still open | `errors` |
|
|
193
|
+
| `quiet` | A stimulus channel heard at least twenty times fell silent for twenty of its own usual gaps (and at least 10s) before the session ended: a body lost or a cable dropped while the mind ran on | `events --search <channel>` |
|
|
194
|
+
|
|
195
|
+
The limits are one table at the top of `signs.ts`.
|
|
196
|
+
|
|
197
|
+
### Structure
|
|
198
|
+
|
|
199
|
+
**`graph`** — who reads and writes what. Read from the session's own
|
|
200
|
+
declaration, so it works with the agent stopped and an old recording is
|
|
201
|
+
explained by the graph it ran with.
|
|
202
|
+
|
|
203
|
+
- No arguments: every system in run order, with rate, thread, reads,
|
|
204
|
+
recalls and writes; every component with its flags.
|
|
205
|
+
- `--system <name>`: that system, with `--downstream` (everything its
|
|
206
|
+
writes can reach) or `--upstream` (everything its reads depend on).
|
|
207
|
+
- `--component <name>`: its writers and its readers.
|
|
208
|
+
- `--depth <n>` bounds a walk (default 6). Each system is reached once, by
|
|
209
|
+
the shortest way. A `recalls` edge crosses a tick, and is marked as one.
|
|
210
|
+
|
|
211
|
+
### Inspect a run
|
|
212
|
+
|
|
213
|
+
**`errors`** — everything that went wrong, in order. Faults: any event whose
|
|
214
|
+
type ends `:failed` or `:error`, and any system run that did not complete.
|
|
215
|
+
Troubles: each time a component named `trouble` was set or removed — the
|
|
216
|
+
convention by which a cognet reports a fault of its own.
|
|
217
|
+
|
|
218
|
+
**`events`** — the log, filtered.
|
|
219
|
+
`--type <prefix>`, `--system <name>`, `--component <name>`, `--entity <id>`,
|
|
220
|
+
`--from` / `--to` (ticks), `--from-ms` / `--to-ms`, `--after <seq>` (the
|
|
221
|
+
`next` of the answer before). Fifty unless told otherwise; `--last` gives
|
|
222
|
+
the last that match, read from the newest telemetry first. Long arrays and
|
|
223
|
+
strings are cut and say how long they were; `--full` gives them whole.
|
|
224
|
+
|
|
225
|
+
**`perf`** — the window it covers; ticks a second against target; each
|
|
226
|
+
tick's time start to end (`wall`) and how many ran past the deadline
|
|
227
|
+
(`late`), and its main-thread share and how many of those were over; each
|
|
228
|
+
system, costliest first: runs, mean, P95, worst, failures, threads, whether
|
|
229
|
+
it ran beside the tick, and how long ticks waited on it (`heldMs` over
|
|
230
|
+
`heldTicks`); and `held`, the systems ticks waited on, longest wait first.
|
|
231
|
+
P95 is to the edge of a bucket — an upper bound. `--system <name>` narrows.
|
|
232
|
+
|
|
233
|
+
**`resources`** — what the agent's processes used, over the whole session.
|
|
234
|
+
For each part (`brain`, and whatever supervises it): memory at first, now
|
|
235
|
+
and at most, how fast it rose after start-up and how fast over the later
|
|
236
|
+
half alone, and a short series of readings; CPU in cores; GPU; video
|
|
237
|
+
memory. `machine` is the whole machine beside them: how busy it was, and
|
|
238
|
+
the processes that kept it busy, the agent's or not — what says whose a
|
|
239
|
+
full machine is when the parts do not add up to it. Parts are `brain`,
|
|
240
|
+
`body` and `world` when it is in a lab, and the process supervising it
|
|
241
|
+
(`daemon`, or `terminal` when a command or the TUI hosts the supervisor).
|
|
242
|
+
Then each thread group — `main`, each `worker`, and the runtime's
|
|
243
|
+
own by name — with the cores it kept busy, busiest first. `--points <n>`
|
|
244
|
+
sets how many readings of a series come back (default 12).
|
|
245
|
+
|
|
246
|
+
**`inference`** — the calls the mind made to its models, over the whole
|
|
247
|
+
session. For each engine role: calls, calls a second, mean, median, p95 and
|
|
248
|
+
worst duration, how many failed, and `busy` — its durations added up over
|
|
249
|
+
the time covered. A duration runs from asking to answer, so it includes
|
|
250
|
+
waiting for the worker; one worker cannot be busier than 1, and a total
|
|
251
|
+
above that is calls waiting on each other. `behind` says it directly: of
|
|
252
|
+
the calls begun, how many found the worker free and how many found one,
|
|
253
|
+
two, three already under way.
|
|
254
|
+
|
|
255
|
+
**`state`** — the ECS at a moment.
|
|
256
|
+
|
|
257
|
+
- No component: the entities, and which components each holds.
|
|
258
|
+
- With a component: its value on each entity that holds it, with its
|
|
259
|
+
revision and when it was written. `--entity <id>` narrows.
|
|
260
|
+
- `--history`: that component's values over a range, one row per write.
|
|
261
|
+
|
|
262
|
+
**`why`** — the causes of a value. From a component at a moment: the run
|
|
263
|
+
that wrote it, the values that run read, and the runs that wrote those —
|
|
264
|
+
back to the stimuli that entered the mind. `--depth <n>` (default 3).
|
|
265
|
+
|
|
266
|
+
**`field`** — a field as numbers and as a picture.
|
|
267
|
+
Range, mean, share of cells measured, and where it lies in its frame.
|
|
268
|
+
`--png <path>` writes it; `--over frame` draws it over the frame of sight
|
|
269
|
+
it was computed from; `--with <component>,…` adds more layers. An agent
|
|
270
|
+
reads the PNG to see what the person sees.
|
|
271
|
+
|
|
272
|
+
### Control
|
|
273
|
+
|
|
274
|
+
**`set`** — write a component on a live session:
|
|
275
|
+
`set <component> --entity <id> --value <json>`. Later, a declared setting
|
|
276
|
+
of a system. Refused on a recorded session. `pause`, `resume` and
|
|
277
|
+
`step [n]` are flags of the same command, since they are the same kind of
|
|
278
|
+
act: taking hold of a running mind.
|
|
279
|
+
|
|
280
|
+
**`prune`** — what the retention rule removes, and removing it.
|
|
281
|
+
Every session is kept whole — log, telemetry, sensory, media, its index — for
|
|
282
|
+
three days after it was last written, and while an agent's sessions fit in
|
|
283
|
+
50GB; past that, the oldest go first. A conversation (a session a person or a
|
|
284
|
+
job typed to) keeps its log: that is what was said. A session written to in
|
|
285
|
+
the last five minutes is taken to be running and is left alone. The same rule
|
|
286
|
+
runs on its own each time a session opens; `--older-than <days>` and
|
|
287
|
+
`--budget <GB>` change it for one prune. Deletes only with `--apply`.
|
|
288
|
+
|
|
289
|
+
## What is built underneath
|
|
290
|
+
|
|
291
|
+
### The graph
|
|
292
|
+
|
|
293
|
+
`graph` reads the `cognet:schedule:declare` a session recorded.
|
|
294
|
+
|
|
295
|
+
Not built: a graph compiled at `axon prepare` and written beside the
|
|
296
|
+
bundle, so there is one before the agent has ever run and it matches the
|
|
297
|
+
source as it is now. That needs the bundle loaded at prepare to collect its
|
|
298
|
+
systems, which is its own piece of work.
|
|
299
|
+
|
|
300
|
+
### The index
|
|
301
|
+
|
|
302
|
+
A session is too large to read for every question. Its log reaches
|
|
303
|
+
hundreds of megabytes and its telemetry a gigabyte.
|
|
304
|
+
|
|
305
|
+
Each session gets `<data>/debug/<id>.json`, written by the first question
|
|
306
|
+
and extended by later ones:
|
|
307
|
+
|
|
308
|
+
- **The log** is scanned a line at a time, never loaded. Most lines are
|
|
309
|
+
read only as far as their type; faults and the declaration are parsed.
|
|
310
|
+
It only grows, so the index records the bytes it covers and the next
|
|
311
|
+
scan starts there.
|
|
312
|
+
- **Each telemetry segment** is summed once: its span of ticks and time,
|
|
313
|
+
every system's durations as a histogram, main-thread time per tick,
|
|
314
|
+
faults and troubles. A segment that has rolled closed never changes; the
|
|
315
|
+
open one is summed again while its size differs; one the ring has
|
|
316
|
+
deleted is dropped.
|
|
317
|
+
|
|
318
|
+
It is used only when it is of this very log: the same version, the same
|
|
319
|
+
session started at the same instant, and a log at least as long as what it
|
|
320
|
+
covers.
|
|
321
|
+
|
|
322
|
+
Building it for a full ring takes a few seconds; after that an answer
|
|
323
|
+
takes well under one.
|
|
324
|
+
|
|
325
|
+
### State at a moment
|
|
326
|
+
|
|
327
|
+
`state` and `why` need the ECS as it stood at a tick. Only the window can
|
|
328
|
+
be replayed, and a component last written before the window began has no
|
|
329
|
+
recorded value in it — that is reported as not recorded. Within the
|
|
330
|
+
window, the index will hold a checkpoint every so many ticks so a late
|
|
331
|
+
moment is a short replay.
|
|
332
|
+
|
|
333
|
+
### The live path
|
|
334
|
+
|
|
335
|
+
For `live`, logged data is read from the growing log exactly as for a
|
|
336
|
+
recording. Three things need the running process, over the agent's link:
|
|
337
|
+
|
|
338
|
+
- a heavy array's values, which the log holds only as a digest (`state`, `field`);
|
|
339
|
+
- a write (`set`);
|
|
340
|
+
- pause, resume and step.
|
|
341
|
+
|
|
342
|
+
What the link exposes for these today is not yet established; it is the
|
|
343
|
+
first thing step 3 settles.
|
|
344
|
+
|
|
345
|
+
## Shape of the code
|
|
346
|
+
|
|
347
|
+
The work lives in `@arcforge/platform`, here, and hangs off `Platform()` as
|
|
348
|
+
`platform.debug`. The CLI owns presentation only: arguments, the envelope,
|
|
349
|
+
exit codes (`apps/tui/cli/commands/debug.ts`).
|
|
350
|
+
|
|
351
|
+
```
|
|
352
|
+
Debug() the module's one door
|
|
353
|
+
├─ Sessions() finds and names sessions; sizes; prune
|
|
354
|
+
├─ Log() reads a session log a line at a time
|
|
355
|
+
├─ Ring() reads telemetry segments
|
|
356
|
+
├─ Index() sums a session once, and keeps it up to date
|
|
357
|
+
├─ Graph() a declaration, and walks over it
|
|
358
|
+
├─ Perf() timings, from the index
|
|
359
|
+
├─ Events() events from log and ring, filtered, in one order
|
|
360
|
+
├─ Resources() memory and its growth, CPU, GPU, threads
|
|
361
|
+
├─ Inference() model calls: how often, how long, how many waiting
|
|
362
|
+
├─ Signs() what says a session is not well, one line each
|
|
363
|
+
├─ Overview() the list of sessions, and one on one screen
|
|
364
|
+
│
|
|
365
|
+
├─ Replay() not built — the ECS at a moment; history; causes
|
|
366
|
+
├─ Fields() not built — a field's numbers, and its picture
|
|
367
|
+
└─ Live() not built — the running process
|
|
368
|
+
```
|
|
369
|
+
|
|
370
|
+
## Order of work
|
|
371
|
+
|
|
372
|
+
1. **Recordings, read-only.** Done. `sessions`, `session`, `graph`,
|
|
373
|
+
`errors`, `events`, `perf`, `prune`.
|
|
374
|
+
2. **Values.** Checkpoints; `state`, `why`, `field` with PNG.
|
|
375
|
+
3. **Live.** Resident components in `state` and `field`; `set`, pause and
|
|
376
|
+
step.
|
|
377
|
+
|
|
378
|
+
Each step ends with the commands documented here, tested, and a short note
|
|
379
|
+
in the repository's agent brief on when to reach for them.
|
|
380
|
+
|
|
381
|
+
## Not in scope
|
|
382
|
+
|
|
383
|
+
- Spawning agents, installing modules, placing an agent in a world. The
|
|
384
|
+
existing CLI does these.
|
|
385
|
+
- Anything the debugger does that is about looking — layouts, widgets,
|
|
386
|
+
timelines. `field --png` is the one picture.
|
|
387
|
+
- Editing code or configuration.
|
|
388
|
+
|
|
389
|
+
## Open
|
|
390
|
+
|
|
391
|
+
- **The compiled graph.** See The graph, above.
|
|
392
|
+
- **Agents by name.** `--agent` takes a directory. `@scope/name` is how the
|
|
393
|
+
rest of the CLI names an agent, and should work here.
|
|
394
|
+
- **Troubles as a convention.** `errors` knows a component named `trouble`.
|
|
395
|
+
If that is to hold for every cognet it belongs in the framework, not in
|
|
396
|
+
one cognet's contract.
|
|
397
|
+
- **Checkpoint spacing.** How often, against how much disk. To be measured
|
|
398
|
+
on a real session in step 2.
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
import { err } from "@arcforge/err"
|
|
2
|
+
import { Events, type Filter } from "./events"
|
|
3
|
+
import { Graph, type Reached } from "./graph"
|
|
4
|
+
import { Index } from "./indexer"
|
|
5
|
+
import { Log } from "./log"
|
|
6
|
+
import { Overview } from "./overview"
|
|
7
|
+
import { Inference } from "./inference"
|
|
8
|
+
import { Perf } from "./perf"
|
|
9
|
+
import { Resources } from "./resources"
|
|
10
|
+
import { Ring } from "./ring"
|
|
11
|
+
import { Sessions } from "./sessions"
|
|
12
|
+
import { Signs } from "./signs"
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Debug — what an agent uses to see inside a recorded or running mind.
|
|
16
|
+
* See README.md beside this file: it is the specification.
|
|
17
|
+
*
|
|
18
|
+
* Every verb takes the agent's directory and, where it is about one
|
|
19
|
+
* session, a reference to it: an id, a prefix of one, or `latest`.
|
|
20
|
+
*/
|
|
21
|
+
export function Debug() {
|
|
22
|
+
const sessions = Sessions()
|
|
23
|
+
const log = Log()
|
|
24
|
+
const ring = Ring()
|
|
25
|
+
const index = Index({ sessions: sessions, log: log, ring: ring })
|
|
26
|
+
const graph = Graph()
|
|
27
|
+
const perf = Perf()
|
|
28
|
+
const events = Events({ log: log, ring: ring })
|
|
29
|
+
const resources = Resources()
|
|
30
|
+
const inference = Inference()
|
|
31
|
+
const signs = Signs()
|
|
32
|
+
const overview = Overview({ sessions: sessions, index: index, perf: perf, events: events, resources: resources, inference: inference, signs: signs })
|
|
33
|
+
|
|
34
|
+
/** The graph a session's mind declared. */
|
|
35
|
+
async function declared(agent: string, ref: string) {
|
|
36
|
+
const session = sessions.resolve(agent, ref)
|
|
37
|
+
const summary = await index.of(agent, session)
|
|
38
|
+
if (!summary.log.declared) throw err("DEBUG_NO_GRAPH", { detail: `session ${session.id} declared no schedule — no cognet ran in it` })
|
|
39
|
+
return { session: session.id, declared: summary.log.declared }
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
return {
|
|
43
|
+
sessions: overview.rows,
|
|
44
|
+
session: overview.one,
|
|
45
|
+
|
|
46
|
+
graph: {
|
|
47
|
+
/** Every system in run order, and every component. */
|
|
48
|
+
async all(agent: string, ref: string) {
|
|
49
|
+
return declared(agent, ref)
|
|
50
|
+
},
|
|
51
|
+
|
|
52
|
+
/** One system, and with a direction, everything it reaches or depends on. */
|
|
53
|
+
async system(agent: string, ref: string, name: string, walk: { direction: "downstream" | "upstream"; depth: number } | null) {
|
|
54
|
+
const found = await declared(agent, ref)
|
|
55
|
+
const system = graph.system(found.declared, name)
|
|
56
|
+
const reached: { downstream?: Reached[]; upstream?: Reached[] } = walk ? { [walk.direction]: graph.walk(found.declared, system.name, walk.direction, walk.depth) } : {}
|
|
57
|
+
return { session: found.session, system: system, ...reached }
|
|
58
|
+
},
|
|
59
|
+
|
|
60
|
+
/** One component: its flags, its writers, its readers. */
|
|
61
|
+
async component(agent: string, ref: string, name: string) {
|
|
62
|
+
const found = await declared(agent, ref)
|
|
63
|
+
return { session: found.session, component: graph.component(found.declared, name) }
|
|
64
|
+
},
|
|
65
|
+
},
|
|
66
|
+
|
|
67
|
+
/** Everything that went wrong, in order. */
|
|
68
|
+
async errors(agent: string, ref: string) {
|
|
69
|
+
const session = sessions.resolve(agent, ref)
|
|
70
|
+
return { session: session.id, ...events.wrong(await index.of(agent, session)) }
|
|
71
|
+
},
|
|
72
|
+
|
|
73
|
+
/** The session's events, filtered. */
|
|
74
|
+
async events(agent: string, ref: string, filter: Filter) {
|
|
75
|
+
const session = sessions.resolve(agent, ref)
|
|
76
|
+
return { session: session.id, ...events.query(session, sessions.data(agent), await index.of(agent, session), filter) }
|
|
77
|
+
},
|
|
78
|
+
|
|
79
|
+
/** How long the mind took: its ticks, and each system. */
|
|
80
|
+
async perf(agent: string, ref: string, only?: string) {
|
|
81
|
+
const session = sessions.resolve(agent, ref)
|
|
82
|
+
return { session: session.id, ...perf.of(await index.of(agent, session), only) }
|
|
83
|
+
},
|
|
84
|
+
|
|
85
|
+
/** What the agent's processes used: memory and its growth, CPU, GPU, and which threads the CPU went to. */
|
|
86
|
+
async resources(agent: string, ref: string, points: number) {
|
|
87
|
+
const session = sessions.resolve(agent, ref)
|
|
88
|
+
return { session: session.id, ...resources.of(await index.of(agent, session), points) }
|
|
89
|
+
},
|
|
90
|
+
|
|
91
|
+
/** The calls the mind made to its models: how often, how long, and how many were waiting on each other. */
|
|
92
|
+
async inference(agent: string, ref: string) {
|
|
93
|
+
const session = sessions.resolve(agent, ref)
|
|
94
|
+
return { session: session.id, ...inference.of(await index.of(agent, session)) }
|
|
95
|
+
},
|
|
96
|
+
|
|
97
|
+
prune: sessions.prune,
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
export type DebugT = ReturnType<typeof Debug>
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
import type { LogT } from "./log"
|
|
2
|
+
import type { RingT } from "./ring"
|
|
3
|
+
import type { Event, Fault, Found, Summary, Trouble } from "./types"
|
|
4
|
+
|
|
5
|
+
export type Filter = {
|
|
6
|
+
/** Events whose type starts with this. */
|
|
7
|
+
type?: string
|
|
8
|
+
system?: string
|
|
9
|
+
component?: string
|
|
10
|
+
entity?: string
|
|
11
|
+
/** Bounds on the tick an event names; an event that names none is left out when either is set. */
|
|
12
|
+
fromTick?: number
|
|
13
|
+
toTick?: number
|
|
14
|
+
/** Bounds on when it happened, epoch ms. */
|
|
15
|
+
fromMs?: number
|
|
16
|
+
toMs?: number
|
|
17
|
+
/** Any further test of an event the caller has — applied before the limit, so it narrows the whole session. */
|
|
18
|
+
passes?: (event: Event) => boolean
|
|
19
|
+
/** Only events after this seq — the `next` of the answer before. */
|
|
20
|
+
after?: number
|
|
21
|
+
limit: number
|
|
22
|
+
/** The last `limit` that match, rather than the first. */
|
|
23
|
+
tail: boolean
|
|
24
|
+
/** Values as recorded, however long. */
|
|
25
|
+
full: boolean
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/** Arrays and strings longer than these are cut in an answer, unless it is asked for in full. */
|
|
29
|
+
const LONGEST = { array: 16, text: 400 }
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Events — a session's events, filtered, from both places they are kept:
|
|
33
|
+
* the log (what the mind declared, and what happened to it) and the
|
|
34
|
+
* telemetry ring (every tick, run and write, for as far back as it reaches).
|
|
35
|
+
*
|
|
36
|
+
* One order across both: every event carries the session's own `seq`.
|
|
37
|
+
*/
|
|
38
|
+
export function Events(opts: { log: LogT; ring: RingT }) {
|
|
39
|
+
const log = opts.log
|
|
40
|
+
const ring = opts.ring
|
|
41
|
+
|
|
42
|
+
return {
|
|
43
|
+
/** The events that pass `filter`, oldest first, and the seq to ask after for more (null when there are none). */
|
|
44
|
+
query(session: Found, data: string, summary: Summary, filter: Filter): { events: Event[]; next: number | null } {
|
|
45
|
+
const kept: Event[] = []
|
|
46
|
+
let more = false
|
|
47
|
+
const take = (event: Event): boolean => {
|
|
48
|
+
if (!match.passes(event, filter)) return true
|
|
49
|
+
if (filter.tail) {
|
|
50
|
+
kept.push(event)
|
|
51
|
+
if (kept.length > filter.limit) kept.shift()
|
|
52
|
+
return true
|
|
53
|
+
}
|
|
54
|
+
if (kept.length === filter.limit) { more = true; return false }
|
|
55
|
+
kept.push(event)
|
|
56
|
+
return true
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
// The log first, then the ring, then put in one order: the ring is the newer and larger half.
|
|
60
|
+
log.scan(session.file, 0, line => {
|
|
61
|
+
const head = log.head(line)
|
|
62
|
+
if (!head || head.type === "session:header") return
|
|
63
|
+
if (filter.type !== undefined && !head.type.startsWith(filter.type)) return
|
|
64
|
+
if (head.ms !== null && filter.toMs !== undefined && head.ms > filter.toMs) return false
|
|
65
|
+
if (head.ms !== null && filter.fromMs !== undefined && head.ms < filter.fromMs) return
|
|
66
|
+
const event = log.parse(line)
|
|
67
|
+
// A torn line is counted by the index, and is not an event.
|
|
68
|
+
return event ? take(event) : undefined
|
|
69
|
+
})
|
|
70
|
+
const fromLog = kept.splice(0)
|
|
71
|
+
const logMore = more
|
|
72
|
+
more = false
|
|
73
|
+
|
|
74
|
+
// The last of them are in the newest segments, so a tail reads those first and stops when it has enough.
|
|
75
|
+
const segments = ring.segments(data, session.id)
|
|
76
|
+
const fromRing: Event[] = []
|
|
77
|
+
for (const segment of filter.tail ? segments.reverse() : segments) {
|
|
78
|
+
const known = summary.segments[segment.name]
|
|
79
|
+
if (known && match.outside(known, filter)) continue
|
|
80
|
+
let stopped = false
|
|
81
|
+
for (const event of ring.read(segment.file, "array")) {
|
|
82
|
+
if (!take(event as Event)) { stopped = true; break }
|
|
83
|
+
}
|
|
84
|
+
if (filter.tail) {
|
|
85
|
+
fromRing.unshift(...kept.splice(0))
|
|
86
|
+
if (fromRing.length >= filter.limit) break
|
|
87
|
+
} else if (stopped) break
|
|
88
|
+
}
|
|
89
|
+
if (!filter.tail) fromRing.push(...kept.splice(0))
|
|
90
|
+
|
|
91
|
+
const merged = [...fromLog, ...fromRing].sort((a, b) => a.time.seq - b.time.seq)
|
|
92
|
+
const events = filter.tail ? merged.slice(-filter.limit) : merged.slice(0, filter.limit)
|
|
93
|
+
const cut = !filter.tail && (logMore || more || merged.length > filter.limit)
|
|
94
|
+
return { events: filter.full ? events : events.map(event => shorten.value(event) as Event), next: cut && events.length ? events[events.length - 1]!.time.seq : null }
|
|
95
|
+
},
|
|
96
|
+
|
|
97
|
+
/** Everything that went wrong, in order: faults from the log and the ring, and each trouble as it opened and closed. */
|
|
98
|
+
wrong(summary: Summary): { faults: Fault[]; troubles: Trouble[]; faultsTotal: number } {
|
|
99
|
+
const segments = Object.values(summary.segments)
|
|
100
|
+
return {
|
|
101
|
+
faults: [...summary.log.faults, ...segments.flatMap(segment => segment.faults)].sort((a, b) => a.seq - b.seq),
|
|
102
|
+
troubles: segments.flatMap(segment => segment.troubles).sort((a, b) => a.at - b.at),
|
|
103
|
+
faultsTotal: summary.log.faultsTotal + segments.reduce((sum, segment) => sum + segment.faultsTotal, 0),
|
|
104
|
+
}
|
|
105
|
+
},
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
export type EventsT = ReturnType<typeof Events>
|
|
110
|
+
|
|
111
|
+
const match = {
|
|
112
|
+
passes(event: Event, filter: Filter): boolean {
|
|
113
|
+
const data = (event.data ?? {}) as Record<string, unknown>
|
|
114
|
+
if (filter.after !== undefined && event.time.seq <= filter.after) return false
|
|
115
|
+
if (filter.type !== undefined && !event.type.startsWith(filter.type)) return false
|
|
116
|
+
if (filter.system !== undefined && !match.named(data.system, filter.system)) return false
|
|
117
|
+
if (filter.component !== undefined && !match.named(data.component, filter.component)) return false
|
|
118
|
+
if (filter.entity !== undefined && data.entity !== filter.entity) return false
|
|
119
|
+
if (filter.fromMs !== undefined && event.time.ms < filter.fromMs) return false
|
|
120
|
+
if (filter.toMs !== undefined && event.time.ms > filter.toMs) return false
|
|
121
|
+
if (filter.fromTick !== undefined || filter.toTick !== undefined) {
|
|
122
|
+
if (typeof data.tick !== "number") return false
|
|
123
|
+
if (filter.fromTick !== undefined && data.tick < filter.fromTick) return false
|
|
124
|
+
if (filter.toTick !== undefined && data.tick > filter.toTick) return false
|
|
125
|
+
}
|
|
126
|
+
return filter.passes === undefined || filter.passes(event)
|
|
127
|
+
},
|
|
128
|
+
|
|
129
|
+
/** A full name, or the end of one. */
|
|
130
|
+
named(value: unknown, wanted: string): boolean {
|
|
131
|
+
return typeof value === "string" && (value === wanted || value.endsWith(`/${wanted}`))
|
|
132
|
+
},
|
|
133
|
+
|
|
134
|
+
/** Whether a whole segment lies outside the filter's ticks or times, so it need not be read. */
|
|
135
|
+
outside(segment: { firstMs: number; lastMs: number; firstTick: number; lastTick: number }, filter: Filter): boolean {
|
|
136
|
+
if (filter.fromMs !== undefined && segment.lastMs < filter.fromMs) return true
|
|
137
|
+
if (filter.toMs !== undefined && segment.firstMs > filter.toMs) return true
|
|
138
|
+
if (filter.fromTick !== undefined && segment.lastTick < filter.fromTick) return true
|
|
139
|
+
if (filter.toTick !== undefined && segment.lastTick >= 0 && segment.firstTick > filter.toTick) return true
|
|
140
|
+
return false
|
|
141
|
+
},
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
const shorten = {
|
|
145
|
+
/** A value with its long arrays and strings cut, each saying how long it was. */
|
|
146
|
+
value(value: unknown): unknown {
|
|
147
|
+
if (typeof value === "string") return value.length > LONGEST.text ? `${value.slice(0, LONGEST.text)}… (${value.length} characters)` : value
|
|
148
|
+
if (Array.isArray(value)) {
|
|
149
|
+
if (value.length > LONGEST.array && value.every(entry => typeof entry === "number")) return { cut: `${value.length} numbers`, first: value.slice(0, 4) }
|
|
150
|
+
if (value.length > LONGEST.array) return [...value.slice(0, LONGEST.array).map(shorten.value), { cut: `${value.length - LONGEST.array} more` }]
|
|
151
|
+
return value.map(shorten.value)
|
|
152
|
+
}
|
|
153
|
+
if (value && typeof value === "object") return Object.fromEntries(Object.entries(value).map(([key, inner]) => [key, shorten.value(inner)]))
|
|
154
|
+
return value
|
|
155
|
+
},
|
|
156
|
+
}
|