@arcforge/platform 2.0.218 → 2.0.244

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (127) hide show
  1. package/package.json +27 -19
  2. package/src/build/blueprint/blueprint.ts +48 -64
  3. package/src/build/blueprint/cognet/abi.ts +3 -257
  4. package/src/build/blueprint/cognet/bundle-worker.ts +11 -0
  5. package/src/build/blueprint/cognet/bundle.ts +294 -89
  6. package/src/build/blueprint/cognet/cognet.ts +8 -7
  7. package/src/build/blueprint/cognet/config/config.ts +52 -0
  8. package/src/build/blueprint/cognet/config/index.ts +4 -0
  9. package/src/build/blueprint/cognet/config/literal.ts +83 -0
  10. package/src/build/blueprint/cognet/config/provider.ts +43 -0
  11. package/src/build/blueprint/cognet/config/shape.ts +183 -0
  12. package/src/build/blueprint/cognet/index.ts +4 -2
  13. package/src/build/blueprint/cognet/native/cargo.ts +31 -0
  14. package/src/build/blueprint/cognet/native/crate.ts +126 -0
  15. package/src/build/blueprint/cognet/native/index.ts +2 -0
  16. package/src/build/blueprint/cognet/native/library.ts +125 -0
  17. package/src/build/blueprint/cognet/native/manifest.ts +137 -0
  18. package/src/build/blueprint/cognet/python/index.ts +1 -0
  19. package/src/build/blueprint/cognet/python/project.ts +119 -0
  20. package/src/build/blueprint/cognet/schema/arrays.ts +23 -0
  21. package/src/build/blueprint/cognet/schema/authored.ts +273 -0
  22. package/src/build/blueprint/cognet/schema/compile.ts +78 -0
  23. package/src/build/blueprint/cognet/schema/components/external.ts +15 -0
  24. package/src/build/blueprint/cognet/schema/components/index.ts +3 -0
  25. package/src/build/blueprint/cognet/schema/components/lock.ts +58 -0
  26. package/src/build/blueprint/cognet/schema/components/model.ts +34 -0
  27. package/src/build/blueprint/cognet/schema/components/python.ts +15 -0
  28. package/src/build/blueprint/cognet/schema/components/python_source.py +69 -0
  29. package/src/build/blueprint/cognet/schema/components/rust.ts +33 -0
  30. package/src/build/blueprint/cognet/schema/components/rust_source/Cargo.lock +117 -0
  31. package/src/build/blueprint/cognet/schema/components/rust_source/Cargo.toml +11 -0
  32. package/src/build/blueprint/cognet/schema/components/rust_source/src/main.rs +104 -0
  33. package/src/build/blueprint/cognet/schema/components/source.ts +220 -0
  34. package/src/build/blueprint/cognet/schema/components/tags.ts +66 -0
  35. package/src/build/blueprint/cognet/schema/components/typescript.ts +157 -0
  36. package/src/build/blueprint/cognet/schema/index.ts +5 -0
  37. package/src/build/blueprint/cognet/schema/interfaces/index.ts +3 -0
  38. package/src/build/blueprint/cognet/schema/interfaces/source.ts +218 -0
  39. package/src/build/blueprint/cognet/schema/python.ts +260 -0
  40. package/src/build/blueprint/cognet/schema/rust.ts +316 -0
  41. package/src/build/blueprint/cognet/schema/schema.ts +248 -0
  42. package/src/build/blueprint/cognet/schema/typescript.ts +352 -0
  43. package/src/build/blueprint/index.ts +1 -1
  44. package/src/build/blueprint/modules/index.ts +7 -5
  45. package/src/build/blueprint/modules/meta.ts +47 -3
  46. package/src/build/blueprint/scan/config.ts +22 -16
  47. package/src/build/blueprint/scan/provider-globals.ts +46 -0
  48. package/src/build/blueprint/scan/providers.ts +25 -0
  49. package/src/build/blueprint/scan/tools.ts +21 -15
  50. package/src/build/extensions/edit.ts +66 -2
  51. package/src/build/extensions/extensions.ts +16 -3
  52. package/src/build/extensions/index.ts +1 -1
  53. package/src/build/project/bundle/agent.ts +26 -17
  54. package/src/build/project/bundle/source.ts +2 -2
  55. package/src/build/project/create/cognet.ts +84 -33
  56. package/src/build/project/create/profile.ts +13 -3
  57. package/src/build/project/index.ts +1 -1
  58. package/src/build/project/installer.ts +6 -5
  59. package/src/build/project/kinds.ts +4 -0
  60. package/src/build/project/manifest/env.ts +10 -0
  61. package/src/build/project/models/blobs/blobs.ts +57 -0
  62. package/src/build/project/models/blobs/index.ts +1 -0
  63. package/src/build/project/models/fetch.ts +46 -51
  64. package/src/build/project/models/index.ts +2 -1
  65. package/src/build/project/models/models.ts +12 -38
  66. package/src/build/project/models/onnx.ts +82 -0
  67. package/src/build/project/models/specifier.ts +9 -3
  68. package/src/build/project/models/storage/index-store.ts +27 -0
  69. package/src/build/project/models/storage/index.ts +1 -0
  70. package/src/build/project/models/store.ts +21 -16
  71. package/src/build/project/prepare.ts +82 -27
  72. package/src/build/project/project.ts +8 -14
  73. package/src/build/project/projects.ts +12 -1
  74. package/src/build/project/tree.ts +13 -3
  75. package/src/build/project/treecache.ts +8 -4
  76. package/src/build/project/typegen/axon-dts.ts +18 -0
  77. package/src/build/project/typegen/cognet-dts.ts +248 -59
  78. package/src/build/project/typegen/prompt-dts.ts +2 -3
  79. package/src/build/project/typegen/registry-dts.ts +2 -2
  80. package/src/build/project/typegen/tsconfig.ts +13 -13
  81. package/src/build/project/typegen/tui-dts.ts +3 -4
  82. package/src/build/project/typegen/workspace-scripts.ts +8 -7
  83. package/src/build/project/typegen/write.ts +15 -2
  84. package/src/build/project/watcher.ts +27 -0
  85. package/src/build/runtime/agent.ts +102 -63
  86. package/src/build/runtime/conformance.ts +25 -0
  87. package/src/build/runtime/escalations.ts +3 -8
  88. package/src/build/runtime/index.ts +2 -0
  89. package/src/build/runtime/instances.ts +7 -4
  90. package/src/build/runtime/progress.ts +6 -2
  91. package/src/build/runtime/reload-watch.ts +11 -8
  92. package/src/debug/README.md +398 -0
  93. package/src/debug/debug.ts +101 -0
  94. package/src/debug/events.ts +156 -0
  95. package/src/debug/graph.ts +94 -0
  96. package/src/debug/index.ts +5 -0
  97. package/src/debug/indexer.ts +286 -0
  98. package/src/debug/inference.ts +53 -0
  99. package/src/debug/log.ts +68 -0
  100. package/src/debug/overview.ts +78 -0
  101. package/src/debug/perf.ts +75 -0
  102. package/src/debug/resources.ts +125 -0
  103. package/src/debug/ring.ts +31 -0
  104. package/src/debug/sessions.ts +90 -0
  105. package/src/debug/signs.ts +119 -0
  106. package/src/debug/tally.ts +44 -0
  107. package/src/debug/types.ts +146 -0
  108. package/src/index.ts +9 -0
  109. package/src/link/agent-main.ts +36 -10
  110. package/src/link/index.ts +1 -1
  111. package/src/link/services.ts +88 -57
  112. package/src/platform.ts +17 -25
  113. package/src/services/cloud.ts +1 -4
  114. package/src/services/control/client.ts +1 -1
  115. package/src/services/control/server.ts +1 -1
  116. package/src/services/index.ts +1 -0
  117. package/src/services/ollama/daemon.ts +23 -0
  118. package/src/services/ollama/http.ts +7 -1
  119. package/src/services/ollama/ollama.ts +8 -14
  120. package/src/services/probe/ask.ts +139 -0
  121. package/src/services/probe/index.ts +3 -0
  122. package/src/services/providers.ts +18 -0
  123. package/src/services/resources/hardware.ts +15 -18
  124. package/src/services/resources/resources.ts +4 -5
  125. package/src/services/store/types.ts +3 -0
  126. package/src/services/running/index.ts +0 -1
  127. package/src/services/running/running.ts +0 -216
@@ -0,0 +1,94 @@
1
+ import { err } from "@arcforge/err"
2
+ import type { Declaration } from "./types"
3
+
4
+ type System = Declaration["systems"][number]
5
+
6
+ /** One step of a walk: a system reached, through which component, and how — read, run by it (`on`), or recalled, which crosses a tick. */
7
+ export type Reached = { system: string; through: string; edge: "reads" | "streams" | "runs" | "recalls"; depth: number }
8
+
9
+ /**
10
+ * Graph — who reads and writes what, and what follows from it.
11
+ *
12
+ * Answers from a mind's own declaration: the systems in run order, and for
13
+ * each the components it reads this tick, streams, is run by (`on`), recalls
14
+ * from the last, and writes. An edge runs from a system that writes a
15
+ * component to one that reads it, streams it, is run by it, or recalls it.
16
+ */
17
+ export function Graph() {
18
+ return {
19
+ /** A system by its full name, or by the end of it when only one ends that way. */
20
+ system(declared: Declaration, ref: string): System {
21
+ const name = names.find(declared.systems.map(system => system.name), ref, "system")
22
+ return declared.systems.find(system => system.name === name)!
23
+ },
24
+
25
+ /** A component: its flags, who writes it, who reads it, who streams it, whom it runs and who recalls it. */
26
+ component(declared: Declaration, ref: string) {
27
+ const name = names.find(declared.components.map(component => component.name), ref, "component")
28
+ const touching = (list: "reads" | "streams" | "recalls" | "writes") => declared.systems.filter(system => (system[list] ?? []).includes(name)).map(system => system.name)
29
+ const runs = declared.systems.filter(system => (system.on ?? []).includes(name)).map(system => system.name)
30
+ return { ...declared.components.find(component => component.name === name)!, writtenBy: touching("writes"), readBy: touching("reads"), streamedBy: touching("streams"), runs: runs, recalledBy: touching("recalls") }
31
+ },
32
+
33
+ /**
34
+ * Every system a change in `from` can reach (`downstream`), or every
35
+ * system it depends on (`upstream`), nearest first, each once — by
36
+ * the shortest way to it. `depth` bounds how far.
37
+ */
38
+ walk(declared: Declaration, from: string, direction: "downstream" | "upstream", depth: number): Reached[] {
39
+ const reached: Reached[] = []
40
+ const seen = new Set<string>([from])
41
+ let frontier = [from]
42
+ for (let level = 1; level <= depth && frontier.length > 0; level++) {
43
+ const next: string[] = []
44
+ for (const name of frontier) {
45
+ const system = declared.systems.find(candidate => candidate.name === name)!
46
+ for (const step of direction === "downstream" ? steps.after(declared, system) : steps.before(declared, system)) {
47
+ if (seen.has(step.system)) continue
48
+ seen.add(step.system)
49
+ reached.push({ ...step, depth: level })
50
+ next.push(step.system)
51
+ }
52
+ }
53
+ frontier = next
54
+ }
55
+ return reached
56
+ },
57
+ }
58
+ }
59
+
60
+ export type GraphT = ReturnType<typeof Graph>
61
+
62
+ const steps = {
63
+ /** The systems that take in what `system` writes. */
64
+ after(declared: Declaration, system: System): Array<Omit<Reached, "depth">> {
65
+ return system.writes.flatMap(component => declared.systems.flatMap(other => [
66
+ ...(other.reads.includes(component) ? [{ system: other.name, through: component, edge: "reads" as const }] : []),
67
+ ...((other.streams ?? []).includes(component) ? [{ system: other.name, through: component, edge: "streams" as const }] : []),
68
+ ...((other.on ?? []).includes(component) ? [{ system: other.name, through: component, edge: "runs" as const }] : []),
69
+ ...(other.recalls.includes(component) ? [{ system: other.name, through: component, edge: "recalls" as const }] : []),
70
+ ]))
71
+ },
72
+
73
+ /** The systems that write what `system` takes in. */
74
+ before(declared: Declaration, system: System): Array<Omit<Reached, "depth">> {
75
+ const taken = [
76
+ ...system.reads.map(component => ({ component: component, edge: "reads" as const })),
77
+ ...(system.streams ?? []).map(component => ({ component: component, edge: "streams" as const })),
78
+ ...(system.on ?? []).map(component => ({ component: component, edge: "runs" as const })),
79
+ ...system.recalls.map(component => ({ component: component, edge: "recalls" as const })),
80
+ ]
81
+ return taken.flatMap(input => declared.systems.filter(other => other.writes.includes(input.component)).map(other => ({ system: other.name, through: input.component, edge: input.edge })))
82
+ },
83
+ }
84
+
85
+ const names = {
86
+ /** The one name `ref` means: itself, or the only one ending in `/ref`. */
87
+ find(all: string[], ref: string, what: string): string {
88
+ if (all.includes(ref)) return ref
89
+ const ending = all.filter(name => name.endsWith(`/${ref}`))
90
+ if (ending.length === 1) return ending[0]!
91
+ if (ending.length === 0) throw err("DEBUG_UNKNOWN_NAME", { detail: `no ${what} is named "${ref}"` })
92
+ throw err("DEBUG_AMBIGUOUS", { detail: `"${ref}" names ${ending.length} ${what}s: ${ending.join(", ")}` })
93
+ },
94
+ }
@@ -0,0 +1,5 @@
1
+ // debug — what an agent uses to see inside a recorded or running mind.
2
+ // The specification is README.md, beside this.
3
+
4
+ export { Debug, type DebugT } from "./debug"
5
+ export type { Filter } from "./events"
@@ -0,0 +1,286 @@
1
+ import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from "node:fs"
2
+ import { dirname } from "node:path"
3
+ import { home, type Decoded } from "@arcforge/session"
4
+ import type { LogT } from "./log"
5
+ import type { RingT } from "./ring"
6
+ import type { SessionsT } from "./sessions"
7
+ import { tally } from "./tally"
8
+ import type { AgentUsage } from "@arcforge/types"
9
+ import type { Channel, Declaration, Event, Fault, Found, Inference, Segment, Summary, Usage } from "./types"
10
+
11
+ /** Bumped when what the index holds changes shape; an older one is rebuilt. */
12
+ const VERSION = 8
13
+ /** Faults kept in the index per source; the count is kept whole. */
14
+ const FAULTS_KEPT = 500
15
+
16
+ /**
17
+ * Index — what a session holds, summed once and kept beside it.
18
+ *
19
+ * A session is too large to read for every question, and most questions are
20
+ * about totals: how many errors, how long each system took. So the first
21
+ * question reads the session and writes down the answers; later ones read
22
+ * that.
23
+ *
24
+ * It is extended, never recomputed. The log only grows, so it is read on
25
+ * from the byte the index covers. A telemetry segment that has rolled
26
+ * closed never changes, so it is summed once; the open one is summed again
27
+ * while its size differs. A segment the ring has since deleted is dropped —
28
+ * the index says what is on disk now.
29
+ */
30
+ export function Index(opts: { sessions: SessionsT; log: LogT; ring: RingT }) {
31
+ const log = opts.log
32
+ const ring = opts.ring
33
+
34
+ return {
35
+ /** The session's summary, brought up to date with what is on disk. */
36
+ async of(agent: string, session: Found): Promise<Summary> {
37
+ const file = opts.sessions.index(agent, session.id)
38
+ const data = opts.sessions.data(agent)
39
+ const header = await home.data.sessions.header(data, session.id)
40
+ const summary = stored.read(file, session, header?.startedAt ?? null) ?? stored.empty(session.id, header?.agentId ?? null, header?.startedAt ?? null)
41
+
42
+ let changed = sum.log(summary, session.file, log)
43
+
44
+ const present = ring.segments(data, session.id)
45
+ for (const name of Object.keys(summary.segments)) {
46
+ if (!present.some(segment => segment.name === name)) { delete summary.segments[name]; changed = true }
47
+ }
48
+ const hz = summary.log.declared?.hz ?? null
49
+ for (const segment of present) {
50
+ if (summary.segments[segment.name]?.bytes === segment.bytes) continue
51
+ summary.segments[segment.name] = sum.segment(ring.read(segment.file, "typed"), segment.bytes, hz)
52
+ changed = true
53
+ }
54
+
55
+ if (changed) {
56
+ mkdirSync(dirname(file), { recursive: true })
57
+ writeFileSync(file, JSON.stringify(summary))
58
+ }
59
+ return summary
60
+ },
61
+ }
62
+ }
63
+
64
+ export type IndexT = ReturnType<typeof Index>
65
+
66
+ /** Log events the index reads whole; every other is only counted, by its type. */
67
+ const READ = new Set(["cognet:schedule:declare", "axon:usage", "cognet:thread:start", "kernel:transform:start", "kernel:transform:complete", "kernel:transform:failed", "kernel:transform:interrupted"])
68
+ /** Calls already in flight are counted up to this many; more is counted as this many. */
69
+ const BEHIND_MOST = 6
70
+
71
+ const stored = {
72
+ empty(id: string, agentId: string | null, startedAt: string | null): Summary {
73
+ return {
74
+ version: VERSION, id: id, agentId: agentId, startedAt: startedAt,
75
+ log: { covered: 0, events: 0, torn: 0, firstMs: null, lastMs: null, types: {}, faults: [], faultsTotal: 0, declared: null, usage: { at: [], parts: {}, threads: {}, workers: {}, machine: null }, inference: { roles: {}, behind: new Array<number>(BEHIND_MOST + 1).fill(0), open: {}, firstMs: null, lastMs: null }, channels: {} },
76
+ segments: {},
77
+ }
78
+ },
79
+
80
+ /**
81
+ * The index on disk, when it is of this very log: the same version, the
82
+ * same session begun at the same instant, and a log at least as long as
83
+ * what it covers. Anything else is another file's index and is not used.
84
+ */
85
+ read(file: string, session: Found, startedAt: string | null): Summary | null {
86
+ if (!existsSync(file)) return null
87
+ const found = JSON.parse(readFileSync(file, "utf-8")) as Summary
88
+ if (found.version !== VERSION || found.id !== session.id || found.startedAt !== startedAt) return null
89
+ if (statSync(session.file).size < found.log.covered) return null
90
+ return found
91
+ },
92
+ }
93
+
94
+ const sum = {
95
+ /** Reads the log on from where the summary stops. Whether anything was added. */
96
+ log(summary: Summary, file: string, log: LogT): boolean {
97
+ const into = summary.log
98
+ const end = log.scan(file, into.covered, line => {
99
+ const head = log.head(line)
100
+ if (!head) { into.torn++; return }
101
+ if (head.type === "session:header") return
102
+ into.events++
103
+ into.types[head.type] = (into.types[head.type] ?? 0) + 1
104
+ if (head.ms !== null) {
105
+ into.firstMs ??= head.ms
106
+ into.lastMs = head.ms
107
+ }
108
+ const wrong = sum.wrong(head.type)
109
+ const stimulus = head.type.startsWith("cognet:stimulus:")
110
+ if (!wrong && !stimulus && !READ.has(head.type)) return
111
+ const event = log.parse(line)
112
+ if (!event) { into.torn++; return }
113
+ if (stimulus) return sum.heard(into.channels, event)
114
+ if (head.type.startsWith("kernel:transform:")) sum.call(into.inference, event)
115
+ if (head.type === "axon:usage") {
116
+ sum.usage(into.usage, event.data as AgentUsage)
117
+ } else if (head.type === "cognet:thread:start") {
118
+ const started = event.data as { index: number; tid: number | null }
119
+ if (started.tid !== null) into.usage.workers[started.tid] = started.index
120
+ } else if (wrong) {
121
+ into.faultsTotal++
122
+ if (into.faults.length < FAULTS_KEPT) into.faults.push(sum.fault(event))
123
+ } else if (head.type === "cognet:schedule:declare") {
124
+ // A resumed session declares again; the newest is the mind that ran last.
125
+ into.declared = event.data as Declaration
126
+ }
127
+ })
128
+ if (end === into.covered) return false
129
+ into.covered = end
130
+ return true
131
+ },
132
+
133
+ /** One stimulus, counted against its channel. */
134
+ heard(into: Record<string, Channel>, event: Event): void {
135
+ const channel = (event.data as { channel?: unknown } | undefined)?.channel
136
+ if (typeof channel !== "string") throw new Error(`a ${event.type} at seq ${event.time.seq} names no channel`)
137
+ const heard = into[channel] ??= { count: 0, firstMs: event.time.ms, lastMs: event.time.ms }
138
+ heard.count++
139
+ heard.lastMs = event.time.ms
140
+ },
141
+
142
+ /** One end of a call to the model worker: begun, or ended however it ended. */
143
+ call(into: Inference, event: Event): void {
144
+ const span = (event.context as { spanId?: string } | undefined)?.spanId
145
+ const data = (event.data ?? {}) as { role?: string; durationMs?: number }
146
+ if (!span) return
147
+ into.firstMs ??= event.time.ms
148
+ into.lastMs = event.time.ms
149
+ if (event.type === "kernel:transform:start") {
150
+ into.behind[Math.min(BEHIND_MOST, Object.keys(into.open).length)]!++
151
+ into.open[span] = data.role ?? "unknown"
152
+ return
153
+ }
154
+ const role = data.role ?? into.open[span] ?? "unknown"
155
+ delete into.open[span]
156
+ const counted = into.roles[role] ??= { ...tally.start(), failed: 0 }
157
+ if (typeof data.durationMs === "number") tally.add(counted, data.durationMs)
158
+ if (event.type !== "kernel:transform:complete") counted.failed++
159
+ },
160
+
161
+ /** One sample of what the agent's processes used, added to the series. */
162
+ usage(into: Usage, sample: AgentUsage): void {
163
+ const position = into.at.length
164
+ into.at.push(sample.at)
165
+ for (const [name, part] of Object.entries(sample.parts)) {
166
+ // A part first seen now was absent for every sample before.
167
+ const series = into.parts[name] ??= { shared: part.shared, ram: new Array(position).fill(null), cpu: new Array(position).fill(null), gpu: new Array(position).fill(null), vram: new Array(position).fill(null) }
168
+ series.ram.push(part.ram)
169
+ series.cpu.push(part.cpu)
170
+ series.gpu.push(part.gpu)
171
+ series.vram.push(part.vram)
172
+ }
173
+ for (const [name, series] of Object.entries(into.parts)) {
174
+ if (!(name in sample.parts)) for (const list of [series.ram, series.cpu, series.gpu, series.vram]) list.push(null)
175
+ }
176
+ const machine = into.machine ??= { cores: sample.machine.cores, ramTotal: sample.machine.ram.total, cpu: [], busiest: {} }
177
+ machine.cpu.push(sample.machine.cpu)
178
+ // Processes of one name are one group, as threads are: sixteen browser processes are a browser.
179
+ const busy = new Map<string, { processes: number; cores: number }>()
180
+ for (const process of sample.machine.busiest ?? []) {
181
+ const group = busy.get(process.name) ?? { processes: 0, cores: 0 }
182
+ group.processes++
183
+ group.cores += process.cpu
184
+ busy.set(process.name, group)
185
+ }
186
+ for (const [name, group] of busy) {
187
+ const tallied = machine.busiest[name] ??= { threads: 0, samples: 0, totalCores: 0, maxCores: 0 }
188
+ tallied.threads = Math.max(tallied.threads, group.processes)
189
+ tallied.samples++
190
+ tallied.totalCores += group.cores
191
+ if (group.cores > tallied.maxCores) tallied.maxCores = group.cores
192
+ }
193
+
194
+ // Threads of one kind are one group: eight pool threads are a pool, and it is the pool that is busy or not.
195
+ const groups = new Map<string, { threads: number; cores: number; measured: boolean }>()
196
+ for (const thread of sample.threads) {
197
+ const worker = into.workers[thread.tid]
198
+ const name = thread.tid === thread.pid ? "main" : worker !== undefined ? `worker ${worker}` : thread.name.replace(/\s*\d+$/, "")
199
+ const group = groups.get(name) ?? { threads: 0, cores: 0, measured: false }
200
+ group.threads++
201
+ if (thread.cpu !== null) { group.cores += thread.cpu; group.measured = true }
202
+ groups.set(name, group)
203
+ }
204
+ for (const [name, group] of groups) {
205
+ const tallied = into.threads[name] ??= { threads: 0, samples: 0, totalCores: 0, maxCores: 0 }
206
+ tallied.threads = Math.max(tallied.threads, group.threads)
207
+ if (!group.measured) continue
208
+ tallied.samples++
209
+ tallied.totalCores += group.cores
210
+ if (group.cores > tallied.maxCores) tallied.maxCores = group.cores
211
+ }
212
+ },
213
+
214
+ /** One telemetry segment, summed. `hz` is the mind's own rate, which sets the deadline a tick is held to. */
215
+ segment(events: Decoded[], bytes: number, hz: number | null): Segment {
216
+ const into: Segment = {
217
+ bytes: bytes, events: events.length, firstMs: events[0]?.time.ms ?? 0, lastMs: events[events.length - 1]?.time.ms ?? 0,
218
+ firstTick: Infinity, lastTick: -Infinity, ticks: tally.start(), late: 0, held: {}, main: tally.start(), over: 0, systems: {}, faults: [], faultsTotal: 0, troubles: [],
219
+ }
220
+ const deadline = hz ? 1000 / hz : null
221
+ /** Main-thread time by tick, summed as its runs are met. */
222
+ const held = new Map<number, number>()
223
+ for (const event of events) {
224
+ const data = (event.data ?? {}) as Record<string, any>
225
+ if (typeof data.tick === "number") {
226
+ if (data.tick < into.firstTick) into.firstTick = data.tick
227
+ if (data.tick > into.lastTick) into.lastTick = data.tick
228
+ }
229
+ if (event.type === "cognet:tick:complete") {
230
+ tally.add(into.ticks, data.durationMs)
231
+ if (deadline !== null && data.durationMs > deadline) into.late++
232
+ for (const one of Array.isArray(data.held) ? data.held : []) {
233
+ if (typeof one?.system !== "string" || typeof one?.ms !== "number") continue
234
+ tally.add(into.held[one.system] ??= tally.start(), one.ms)
235
+ }
236
+ } else if (event.type === "cognet:system:run") {
237
+ const system = into.systems[data.system] ??= { ...tally.start(), failed: 0, async: false, threads: [] }
238
+ tally.add(system, data.durationMs)
239
+ if (data.async) system.async = true
240
+ if (typeof data.thread === "number" && !system.threads.includes(data.thread)) system.threads.push(data.thread)
241
+ if (data.outcome !== "complete") system.failed++
242
+ if (!data.async && typeof data.thread !== "number") held.set(data.tick, (held.get(data.tick) ?? 0) + data.durationMs)
243
+ } else if (sum.trouble(event.type, data.component)) {
244
+ const opened = event.type !== "cognet:component:remove"
245
+ into.troubles.push({ at: event.time.ms, tick: data.tick, entity: data.entity, opened: opened, ...(opened ? { what: data.value?.what, detail: data.value?.detail } : {}) })
246
+ }
247
+ if (sum.wrong(event.type) || (event.type === "cognet:system:run" && data.outcome !== "complete")) {
248
+ into.faultsTotal++
249
+ if (into.faults.length < FAULTS_KEPT) into.faults.push(sum.fault(event))
250
+ }
251
+ }
252
+ for (const ms of held.values()) {
253
+ tally.add(into.main, ms)
254
+ if (deadline !== null && ms > deadline) into.over++
255
+ }
256
+ if (into.firstTick === Infinity) { into.firstTick = -1; into.lastTick = -1 }
257
+ return into
258
+ },
259
+
260
+ /**
261
+ * Whether an event of this type says something went wrong. A build
262
+ * warning counts: a plugin that could not be imported is reported as
263
+ * one, and the agent then runs without it — Terry ran blind that way.
264
+ */
265
+ wrong(type: string): boolean {
266
+ return type.endsWith(":failed") || type.endsWith(":error") || type === "build:warning"
267
+ },
268
+
269
+ /** Whether an event writes or removes a trouble: a component named `trouble`, by the convention a cognet reports its own faults with. */
270
+ trouble(type: string, component: unknown): boolean {
271
+ if (!type.startsWith("cognet:component:") || typeof component !== "string") return false
272
+ return component === "trouble" || component.endsWith("/trouble")
273
+ },
274
+
275
+ fault(event: { type: string; time: { ms: number; seq: number }; data?: unknown }): Fault {
276
+ const data = (event.data ?? {}) as Record<string, any>
277
+ const message = data.error?.message ?? data.message
278
+ return {
279
+ at: event.time.ms, seq: event.time.seq, type: event.type,
280
+ ...(typeof data.tick === "number" ? { tick: data.tick } : {}),
281
+ ...(typeof data.system === "string" ? { system: data.system } : {}),
282
+ ...(typeof data.error?.code === "string" ? { code: data.error.code } : {}),
283
+ ...(typeof message === "string" ? { message: message } : {}),
284
+ }
285
+ },
286
+ }
@@ -0,0 +1,53 @@
1
+ import { tally } from "./tally"
2
+ import type { Summary } from "./types"
3
+
4
+ /**
5
+ * Inference — the calls a mind made to its models over a session: how
6
+ * often, how long each took, and how many were already under way when one
7
+ * began.
8
+ *
9
+ * A call's duration is from asking to answer, so it includes waiting for
10
+ * the worker. `busy` is each role's durations added up over the time
11
+ * covered: one worker cannot be busier than 1, so a total above that is
12
+ * calls waiting on each other — asking for more than there is.
13
+ *
14
+ * From the log, so it covers the whole session, not the telemetry window.
15
+ */
16
+ export function Inference() {
17
+ return {
18
+ of(summary: Summary) {
19
+ const counted = summary.log.inference
20
+ const seconds = counted.firstMs !== null && counted.lastMs !== null ? (counted.lastMs - counted.firstMs) / 1000 : 0
21
+ const roles = Object.entries(counted.roles)
22
+ .map(([role, calls]) => ({
23
+ role: role,
24
+ calls: calls.count,
25
+ failed: calls.failed,
26
+ perSecond: seconds > 0 ? round(calls.count / seconds, 2) : null,
27
+ meanMs: tally.mean(calls),
28
+ p50Ms: tally.within(calls, 0.5),
29
+ p95Ms: tally.within(calls, 0.95),
30
+ maxMs: calls.maxMs,
31
+ busy: seconds > 0 ? round(calls.totalMs / 1000 / seconds, 2) : null,
32
+ }))
33
+ .sort((a, b) => (b.busy ?? 0) - (a.busy ?? 0))
34
+ const begun = counted.behind.reduce((sum, count) => sum + count, 0)
35
+ return {
36
+ window: { fromMs: counted.firstMs, toMs: counted.lastMs, seconds: round(seconds, 1) },
37
+ calls: begun,
38
+ busy: seconds > 0 ? round(roles.reduce((sum, role) => sum + (role.busy ?? 0), 0), 2) : null,
39
+ // Of the calls begun, how many found the worker free, and how many found others ahead.
40
+ behind: counted.behind.map((count, already) => ({ already: already, calls: count })).filter(row => row.calls > 0),
41
+ unanswered: Object.keys(counted.open).length,
42
+ roles: roles,
43
+ }
44
+ },
45
+ }
46
+ }
47
+
48
+ export type InferenceT = ReturnType<typeof Inference>
49
+
50
+ function round(value: number, places: number): number {
51
+ const unit = 10 ** places
52
+ return Math.round(value * unit) / unit
53
+ }
@@ -0,0 +1,68 @@
1
+ import { closeSync, openSync, readSync } from "node:fs"
2
+ import type { Event } from "./types"
3
+
4
+ /** Bytes read at a time. */
5
+ const CHUNK = 4 * 1024 * 1024
6
+
7
+ /**
8
+ * Log — reads a session log a line at a time, from any byte offset.
9
+ *
10
+ * Never the whole file: a session log reaches hundreds of megabytes. A line
11
+ * is handed over as text with the offset it starts at; whether it is worth
12
+ * parsing is the caller's to decide, which is most of what makes a scan fast.
13
+ */
14
+ export function Log() {
15
+ return {
16
+ /**
17
+ * Every complete line from `from` on, in order. `visit` returns false
18
+ * to stop. Returns the offset after the last complete line — where
19
+ * the next scan starts; a line still being written is left for it.
20
+ */
21
+ scan(file: string, from: number, visit: (line: string, offset: number) => boolean | void): number {
22
+ const handle = openSync(file, "r")
23
+ try {
24
+ const chunk = Buffer.alloc(CHUNK)
25
+ let carry: Buffer = Buffer.alloc(0)
26
+ let at = from
27
+ let position = from
28
+ for (;;) {
29
+ const read = readSync(handle, chunk, 0, CHUNK, position)
30
+ if (read === 0) return at
31
+ position += read
32
+ const bytes = carry.length ? Buffer.concat([carry, chunk.subarray(0, read)]) : chunk.subarray(0, read)
33
+ let start = 0
34
+ for (;;) {
35
+ const end = bytes.indexOf(0x0a, start)
36
+ if (end < 0) break
37
+ if (end > start && visit(bytes.toString("utf-8", start, end), at) === false) return at
38
+ at += end + 1 - start
39
+ start = end + 1
40
+ }
41
+ carry = Buffer.from(bytes.subarray(start))
42
+ }
43
+ } finally {
44
+ closeSync(handle)
45
+ }
46
+ },
47
+
48
+ /** A line as an event, or null when it is torn — cut short by a crash mid-write. */
49
+ parse(line: string): Event | null {
50
+ try {
51
+ return JSON.parse(line) as Event
52
+ } catch (cause) {
53
+ if (cause instanceof SyntaxError) return null
54
+ throw cause
55
+ }
56
+ },
57
+
58
+ /** A line's type and time, read off its front without parsing it. Null for a line that is not an event. */
59
+ head(line: string): { type: string; ms: number | null } | null {
60
+ const type = /^\{"type":"([^"]+)"/.exec(line)
61
+ if (!type) return null
62
+ const ms = /"time":\{"ms":(\d+(?:\.\d+)?)/.exec(line.slice(0, 200))
63
+ return { type: type[1]!, ms: ms ? Number(ms[1]) : null }
64
+ },
65
+ }
66
+ }
67
+
68
+ export type LogT = ReturnType<typeof Log>
@@ -0,0 +1,78 @@
1
+ import { home } from "@arcforge/session"
2
+ import type { EventsT } from "./events"
3
+ import type { IndexT } from "./indexer"
4
+ import type { InferenceT } from "./inference"
5
+ import type { PerfT } from "./perf"
6
+ import type { ResourcesT } from "./resources"
7
+ import type { SessionsT } from "./sessions"
8
+ import type { SignsT } from "./signs"
9
+
10
+ /** Systems shown in a session's overview, and readings of each memory series. */
11
+ const COSTLIEST = 5
12
+ const POINTS = 8
13
+
14
+ /**
15
+ * Overview — sessions at a glance: the list of them, and one on one screen.
16
+ * What to run first when something is said to be wrong.
17
+ */
18
+ export function Overview(opts: { sessions: SessionsT; index: IndexT; perf: PerfT; events: EventsT; resources: ResourcesT; inference: InferenceT; signs: SignsT }) {
19
+ const sessions = opts.sessions
20
+
21
+ return {
22
+ /**
23
+ * The newest sessions. Read from each one's header and the
24
+ * filesystem alone, so listing costs nothing however large they
25
+ * are; what a session HOLDS is `one`.
26
+ */
27
+ async rows(agent: string, limit: number) {
28
+ const all = sessions.list(agent)
29
+ const data = sessions.data(agent)
30
+ const rows = []
31
+ for (const session of all.slice(0, limit)) {
32
+ const header = await home.data.sessions.header(data, session.id)
33
+ const startedAt = header ? Date.parse(header.startedAt) : null
34
+ rows.push({
35
+ id: session.id,
36
+ startedAt: header?.startedAt ?? null,
37
+ lastWrittenAt: new Date(session.modifiedAt).toISOString(),
38
+ seconds: startedAt === null ? null : Math.round((session.modifiedAt - startedAt) / 1000),
39
+ bytes: sessions.sizes(agent, session.id),
40
+ })
41
+ }
42
+ return { sessions: rows, total: all.length }
43
+ },
44
+
45
+ /** One session: how long, how it ran, what went wrong, what cost most, what it holds on disk. */
46
+ async one(agent: string, ref: string) {
47
+ const session = sessions.resolve(agent, ref)
48
+ const summary = await opts.index.of(agent, session)
49
+ const perf = opts.perf.of(summary)
50
+ const wrong = opts.events.wrong(summary)
51
+ const resources = opts.resources.of(summary, POINTS)
52
+ const inference = opts.inference.of(summary)
53
+ const faults = { total: wrong.faultsTotal, first: wrong.faults[0] ?? null }
54
+ const declared = summary.log.declared
55
+ return {
56
+ id: session.id,
57
+ // What says this session is not well. Nothing found is every check passing, and `checked` says which.
58
+ signs: opts.signs.of({ faults: faults, troubles: wrong.troubles, ticks: perf.ticks, held: perf.held, systems: perf.systems, parts: resources.parts, machine: resources.machine, inference: inference, senses: { channels: summary.log.channels, lastMs: summary.log.lastMs } }),
59
+ agentId: summary.agentId,
60
+ startedAt: summary.startedAt,
61
+ lastWrittenAt: new Date(session.modifiedAt).toISOString(),
62
+ seconds: summary.log.firstMs !== null && summary.log.lastMs !== null ? Math.round((summary.log.lastMs - summary.log.firstMs) / 1000) : null,
63
+ mind: declared ? { systems: declared.systems.length, components: declared.components.length, targetPerSecond: declared.hz ?? null } : null,
64
+ log: { events: summary.log.events, torn: summary.log.torn },
65
+ // Telemetry is a window: these are of the last stretch of the session, not all of it.
66
+ telemetry: { window: perf.window, ticks: perf.ticks },
67
+ faults: faults,
68
+ troubles: { opened: wrong.troubles.filter(trouble => trouble.opened).length },
69
+ costliest: perf.systems.slice(0, COSTLIEST),
70
+ // Over the whole session, from the usage samples in the log.
71
+ resources: { machine: resources.machine, parts: resources.parts, threads: resources.threads.slice(0, COSTLIEST) },
72
+ bytes: sessions.sizes(agent, session.id),
73
+ }
74
+ },
75
+ }
76
+ }
77
+
78
+ export type OverviewT = ReturnType<typeof Overview>
@@ -0,0 +1,75 @@
1
+ import { tally } from "./tally"
2
+ import type { Summary, SystemTally, Tally } from "./types"
3
+
4
+ /** A duration summed, as a command reports it. `p95Ms` is to the edge of a bucket — an upper bound. */
5
+ type Timing = { runs: number; meanMs: number; p95Ms: number; maxMs: number; totalMs: number }
6
+
7
+ /**
8
+ * Perf — how long the mind took, from the index.
9
+ *
10
+ * Over the telemetry still on disk, which is a window: the last stretch of
11
+ * the session, not all of it. A system that ran beside the tick is marked
12
+ * `async`; its time is wall time waiting, not time the tick was held up.
13
+ * What a tick WAS held up by is `heldMs`: the time ticks spent waiting on a
14
+ * system — a worker the rest of the tick needed, or a lagged run landing late.
15
+ */
16
+ export function Perf() {
17
+ return {
18
+ /** The window, the ticks, and every system, costliest first. `only` narrows to systems whose name ends with it. */
19
+ of(summary: Summary, only?: string) {
20
+ const segments = Object.values(summary.segments)
21
+ const ticks = tally.start()
22
+ const main = tally.start()
23
+ const systems: Record<string, SystemTally> = {}
24
+ const held: Record<string, Tally> = {}
25
+ let over = 0
26
+ let late = 0
27
+ for (const segment of segments) {
28
+ tally.merge(ticks, segment.ticks)
29
+ tally.merge(main, segment.main)
30
+ over += segment.over
31
+ late += segment.late
32
+ for (const [name, from] of Object.entries(segment.held)) tally.merge(held[name] ??= tally.start(), from)
33
+ for (const [name, from] of Object.entries(segment.systems)) {
34
+ const into = systems[name] ??= { ...tally.start(), failed: 0, async: false, threads: [] }
35
+ tally.merge(into, from)
36
+ into.failed += from.failed
37
+ into.async ||= from.async
38
+ for (const thread of from.threads) if (!into.threads.includes(thread)) into.threads.push(thread)
39
+ }
40
+ }
41
+ const fromMs = segments.length ? Math.min(...segments.map(segment => segment.firstMs)) : null
42
+ const toMs = segments.length ? Math.max(...segments.map(segment => segment.lastMs)) : null
43
+ const seconds = fromMs !== null && toMs !== null ? (toMs - fromMs) / 1000 : 0
44
+ const hz = summary.log.declared?.hz ?? null
45
+ return {
46
+ window: { fromMs: fromMs, toMs: toMs, seconds: Math.round(seconds * 10) / 10, segments: segments.length },
47
+ ticks: {
48
+ perSecond: seconds > 0 ? Math.round(ticks.count / seconds * 10) / 10 : 0,
49
+ targetPerSecond: hz,
50
+ targetMs: hz ? Math.round(1000 / hz * 10) / 10 : null,
51
+ // Each tick, start to end: what a tick past the deadline ran over in.
52
+ wall: timing(ticks),
53
+ late: late,
54
+ // The main thread's share of it.
55
+ main: timing(main),
56
+ overTarget: over,
57
+ },
58
+ systems: Object.entries(systems)
59
+ .filter(([name]) => only === undefined || name === only || name.endsWith(`/${only}`))
60
+ .sort((a, b) => b[1].totalMs - a[1].totalMs)
61
+ .map(([name, of]) => ({ system: name, ...timing(of), heldMs: Math.round(held[name]?.totalMs ?? 0), heldTicks: held[name]?.count ?? 0, failed: of.failed, async: of.async, threads: [...of.threads].sort((a, b) => a - b) })),
62
+ /** The systems ticks waited on, longest wait first. */
63
+ held: Object.entries(held)
64
+ .sort((a, b) => b[1].totalMs - a[1].totalMs)
65
+ .map(([name, of]) => ({ system: name, ticks: of.count, totalMs: Math.round(of.totalMs), meanMs: tally.mean(of), maxMs: Math.round(of.maxMs) })),
66
+ }
67
+ },
68
+ }
69
+ }
70
+
71
+ export type PerfT = ReturnType<typeof Perf>
72
+
73
+ function timing(of: Tally): Timing {
74
+ return { runs: of.count, meanMs: tally.mean(of), p95Ms: tally.within(of, 0.95), maxMs: Math.round(of.maxMs * 100) / 100, totalMs: Math.round(of.totalMs) }
75
+ }