@oakgreencc/cambium 0.0.6 → 0.0.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +166 -11
- package/otel/com.oakgreencc.otelcol.plist +33 -0
- package/otel/otelcol-aws.yaml +52 -0
- package/otel/otelcol.yaml +50 -0
- package/package.json +9 -2
- package/src/cli.mjs +18 -3
- package/src/config.mjs +4 -0
- package/src/graph/advance.mjs +19 -8
- package/src/graph-engine.mjs +289 -0
- package/src/graphs/task.mjs +587 -0
- package/src/issue-state.mjs +46 -0
- package/src/tick.mjs +97 -76
- package/src/tracing.mjs +522 -0
package/README.md
CHANGED
|
@@ -4,7 +4,7 @@ The scheduler daemon. Every tick it reads the `q:*` queues from GitHub, starts h
|
|
|
4
4
|
|
|
5
5
|
This is the **Task phase** MVP. Dispatch and the daily Release run are seams (see [Follow-ups](#follow-ups)).
|
|
6
6
|
|
|
7
|
-
|
|
7
|
+
Runtime dependencies: `zod` (the [graph module](#graph-module-preview)) and OpenTelemetry via `@oakgreencc/mycelium` (loaded only when [tracing](#tracing) is on); Node ≥ 22.13 (`node:sqlite`). GitHub access goes through [`graft`](../graft), which must be on `PATH`.
|
|
8
8
|
|
|
9
9
|
## Quick start
|
|
10
10
|
|
|
@@ -19,15 +19,16 @@ cambium status
|
|
|
19
19
|
|
|
20
20
|
## Commands
|
|
21
21
|
|
|
22
|
-
| Command
|
|
23
|
-
|
|
|
24
|
-
| `cambium start [--once]`
|
|
25
|
-
| `cambium status [--json]`
|
|
26
|
-
| `cambium pause`
|
|
27
|
-
| `cambium drain`
|
|
28
|
-
| `cambium resume`
|
|
29
|
-
| `cambium run task --issue <n> [--repo <owner/name>]`
|
|
30
|
-
| `cambium
|
|
22
|
+
| Command | What |
|
|
23
|
+
| ------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
24
|
+
| `cambium start [--once]` | Foreground daemon, one tick every `tickSeconds`. SIGTERM/SIGINT stop it after the current tick; workers keep running and are re-adopted on restart. One daemon per database. |
|
|
25
|
+
| `cambium status [--json]` | Mode, last tick, queue counts, workers, failed steps. Exit 1 when there is no heartbeat or it is older than 3 ticks. |
|
|
26
|
+
| `cambium pause` | Read and heartbeat only: no transitions, no starts. Running workers are left alone. |
|
|
27
|
+
| `cambium drain` | Reconcile, watch and sweep as usual; start nothing new. |
|
|
28
|
+
| `cambium resume` | Back to `running`. |
|
|
29
|
+
| `cambium run task --issue <n> [--repo <owner/name>]` | Claim one open `q:task` issue, run its worker in the foreground, then make the same transition the daemon would. |
|
|
30
|
+
| `cambium run <graph> --issue <n> [--repo <owner/name>] [--follow]` | `engine: "graph"` only (`task` is the one graph today). Advance one issue through the graph runtime in the foreground: a `q:task` issue without a lease starts with a foreground worker; anything else resumes where its marker rests. Stops at the first wait that is not ready; `--follow` re-reads and polls every `tickSeconds` until the graph ends. |
|
|
31
|
+
| `cambium install [--dry-run]` | Write the launchd agent (`KeepAlive`, `RunAtLoad`) for `start --config <this config>`; `--dry-run` prints it. It does not call `launchctl`; it prints the command. |
|
|
31
32
|
|
|
32
33
|
Every command takes `--config <file>`; the default is `$CAMBIUM_CONFIG`, then `~/.cambium/config.json`.
|
|
33
34
|
|
|
@@ -91,6 +92,7 @@ One JSON file. Everything but `repos` is optional; these are the defaults:
|
|
|
91
92
|
```json
|
|
92
93
|
{
|
|
93
94
|
"host": "<os.hostname()>",
|
|
95
|
+
"engine": "legacy",
|
|
94
96
|
"tickSeconds": 30,
|
|
95
97
|
"stepTimeoutSeconds": 20,
|
|
96
98
|
"repos": [{ "slug": "oakgreencc/monorepo", "path": "~/OGCC/monorepo", "baseBranch": "main" }],
|
|
@@ -115,6 +117,7 @@ One JSON file. Everything but `repos` is optional; these are the defaults:
|
|
|
115
117
|
|
|
116
118
|
- `repos[].path` is a local clone; worktrees are added from it and `graft` runs in it.
|
|
117
119
|
- `graft.app` is the App cambium acts as (`GRAFT_APP` for its own `graft` calls); `worker.graftApp` the one workers act as. Neither leaks into the other.
|
|
120
|
+
- `engine` picks how the Task phase runs: `"legacy"` (the stage tick, the default) or `"graph"` (the [`task` graph](#the-task-graph)). Both make the same moves; see [The task graph](#the-task-graph).
|
|
118
121
|
- Setting `paths.home` moves every default path under it.
|
|
119
122
|
- `release.time` is host-local `HH:MM`.
|
|
120
123
|
|
|
@@ -218,9 +221,62 @@ Written at the end of every tick to `paths.heartbeat` (default `~/.cambium/state
|
|
|
218
221
|
|
|
219
222
|
The mode lives in the SQLite file (`paths.db`, table `control`), so `cambium status --json` returns `{ "mode", "heartbeat" }` even when the daemon is down. The daemon's own log is `<paths.logs>/cambium.log` (one `.1` kept); launchd's stderr goes to `<paths.logs>/cambium.launchd.err.log`.
|
|
220
223
|
|
|
224
|
+
## The task graph
|
|
225
|
+
|
|
226
|
+
With `"engine": "graph"` the tick's Task-phase steps (Reconcile, Watch, Sweep, Schedule) drive the `task` graph (`src/graphs/task.mjs`) instead of the stage tick in `src/tick.mjs`. The default stays `"legacy"`; flipping it and deleting the legacy path is a follow-up.
|
|
227
|
+
|
|
228
|
+
```mermaid
|
|
229
|
+
flowchart TD
|
|
230
|
+
start_(["q:task"])
|
|
231
|
+
n0["start-worker"]
|
|
232
|
+
n1{{"worker-running"}}
|
|
233
|
+
n2{{"ci-running"}}
|
|
234
|
+
n3{{"merge-queue"}}
|
|
235
|
+
n4["close-out"]
|
|
236
|
+
n5["requeue"]
|
|
237
|
+
n6["to-human"]
|
|
238
|
+
end_(("end"))
|
|
239
|
+
start_ --> n0
|
|
240
|
+
n0 -->|"started"| n1
|
|
241
|
+
n0 -->|"budgetSpent"| n6
|
|
242
|
+
n0 --> n5
|
|
243
|
+
n1 -->|"issueGone"| end_
|
|
244
|
+
n1 -->|"submitted"| n2
|
|
245
|
+
n1 --> n5
|
|
246
|
+
n2 -->|"merged"| n4
|
|
247
|
+
n2 -->|"inMergeQueue"| n3
|
|
248
|
+
n2 -->|"stalled"| n6
|
|
249
|
+
n2 --> n5
|
|
250
|
+
n3 -->|"merged"| n4
|
|
251
|
+
n3 -->|"stalled"| n6
|
|
252
|
+
n3 --> n5
|
|
253
|
+
n5 -->|"attemptsLeft"| n0
|
|
254
|
+
n5 --> n6
|
|
255
|
+
n4 --> end_
|
|
256
|
+
n6 --> end_
|
|
257
|
+
```
|
|
258
|
+
|
|
259
|
+
Generated by `toMermaid(task)`; a test keeps this copy current. Boxes are step nodes, hexagons are waits.
|
|
260
|
+
|
|
261
|
+
| Node | Kind | Queue label | Holds `running` |
|
|
262
|
+
| ---------------- | ---- | ----------- | --------------- |
|
|
263
|
+
| `start-worker` | step | `q:task` | no |
|
|
264
|
+
| `worker-running` | wait | `q:task` | yes |
|
|
265
|
+
| `ci-running` | wait | none | no |
|
|
266
|
+
| `merge-queue` | wait | none | no |
|
|
267
|
+
| `close-out` | step | none | no |
|
|
268
|
+
| `requeue` | step | `q:task` | no |
|
|
269
|
+
| `to-human` | step | `q:human` | no |
|
|
270
|
+
|
|
271
|
+
- **Labels are a projection.** Each node declares its queue label and whether it holds `running`. When an issue lands on a node, the marker is written first, then the labels come from `move()` (`labels.mjs`) with that declaration.
|
|
272
|
+
- **Where a turn starts.** Reconcile polls `worker-running` for every tracked worker. Watch polls `ci-running` / `merge-queue` for agent PRs whose issue is in no queue; a merged PR goes to `close-out` wherever the issue is, as in the stage tick. Sweep sends a stale `running` lease to its node's `recover` (default: `requeue`). Schedule is the gate on the entry node: caps and priority pick the issues, each starts at `start-worker`.
|
|
273
|
+
- **Where a turn stops.** At a wait that is not ready, on entering a wait (it is polled next tick; `merge-queue` is polled on entry, so a stall is caught in the same pass), on reaching `start-worker` again (only Schedule admits it), or at the end.
|
|
274
|
+
- **Markers.** The graph writes marker v2 without `stage`: `node` is where the issue rests. `merge-queue` is the stage tick's `submitted` with `queued: true`, and a stage-era marker in that state resumes at `merge-queue`. A marker whose node the current graph version no longer has restarts at `start-worker` with a `graph-changed` history entry.
|
|
275
|
+
- **Same moves.** Every scenario in `test/tick.test.mjs` runs on both engines; the test compares their GitHub calls in order (markers semantically, not by rendered text), worker starts with their prompts, kills, teardowns and logs.
|
|
276
|
+
|
|
221
277
|
## Graph module (preview)
|
|
222
278
|
|
|
223
|
-
`@oakgreencc/cambium/graph` is the start of typed workflows: nodes with schemas, edges as data, and a loop that advances a position through them.
|
|
279
|
+
`@oakgreencc/cambium/graph` is the start of typed workflows: nodes with schemas, edges as data, and a loop that advances a position through them. The Task phase runs on it with `engine: "graph"` ([The task graph](#the-task-graph)).
|
|
224
280
|
|
|
225
281
|
```js
|
|
226
282
|
import { z } from 'zod'
|
|
@@ -258,6 +314,105 @@ const { transitions, reason, node, input } = await advance(g, { node: 'open-pr',
|
|
|
258
314
|
- **`validateGraph(graph)`** returns every problem as a string (`defineGraph` throws on any): unknown edge endpoints, nodes unreachable from `entry.node`, a node with outgoing edges whose last edge is conditional (no fallback), duplicate names, a non-integer `version`.
|
|
259
315
|
- **`advance(graph, { node, input }, ctx, { maxTransitions = 8, signal })`** runs step nodes and ready waits and returns `{ transitions, reason, node, input, error? }`: `reason` is `waiting` (a wait that is not ready), `end`, `limit`, `aborted` (`signal`, or `ctx.signal`, checked before each node) or `error`; `node`/`input` is where it stopped. Each transition is `{ from, to, edge, input, output, next }`. `ctx` is passed to `run`/`poll` untouched apart from `ctx.signal`; edges see `ctx.subject`.
|
|
260
316
|
- **`toMermaid(graph)`** renders a flowchart: boxes for steps, hexagons for waits, conditional edges labelled with `when`'s function name.
|
|
317
|
+
- **`opts.instrument(visit, step)`** (optional) wraps each node visit: `visit` is `{ node, kind: 'node' | 'wait', input }`; `step(extraCtx?)` runs the node (or poll) and picks the edge, returning the transition, or `null` for a wait that is not ready, and throws on a node error. `extraCtx` is merged into that node's `ctx`. Return what `step` returned. Without it, `advance` behaves exactly as before. [`advanceHooks`](#tracing) is the tracing implementation.
|
|
318
|
+
|
|
319
|
+
## Tracing
|
|
320
|
+
|
|
321
|
+
`src/tracing.mjs` is OpenTelemetry for cambium, configured only by the standard `OTEL_*` variables with `service.name=cambium`. It reuses mycelium's `createTracing` (same packages, same on/off rule): **on only when `OTEL_EXPORTER_OTLP_ENDPOINT` or `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` is set**, unless `OTEL_SDK_DISABLED=true` or `OTEL_TRACES_EXPORTER=none`; otherwise nothing is imported and every call is a no-op. Metrics follow the same rule for their signal (`OTEL_EXPORTER_OTLP_METRICS_ENDPOINT`, `OTEL_METRICS_EXPORTER`). With an endpoint but no collector, shutdown can take ~8 s; set `OTEL_EXPORTER_OTLP_TIMEOUT=2000` for the daemon.
|
|
322
|
+
|
|
323
|
+
Why not a shared package: cambium is published to npm, so it can only depend on published packages; a new publishable `@oakgreencc/telemetry` needs a protected `nx.json` release change. mycelium already publishes `createTracing`, so it gained a `serviceName` option and cambium depends on `@oakgreencc/mycelium` (which also gives it `parseTraceparent` for the intake trace).
|
|
324
|
+
|
|
325
|
+
| Span / metric | When | Parent |
|
|
326
|
+
| ------------------------------------ | -------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------- |
|
|
327
|
+
| `cambium.issue` | Once per issue, when the marker has no `traceparent`. Ended at once; its traceparent goes in the marker. | mycelium's intake span from the issue body, else none (a root) |
|
|
328
|
+
| `cambium.node` | Each step-node run | The marker's `traceparent` |
|
|
329
|
+
| `cambium.wait` | When a wait ends; `startTime` = the marker's `enteredAt`. No span stays open across ticks. | The marker's `traceparent` |
|
|
330
|
+
| `cambium.wait.not_ready` | Counter, +1 per not-ready poll (attributes graph, node, repo) | |
|
|
331
|
+
| `claude.session` / `.tool` / `.task` | From the worker's stream-json log, see below | The start-worker `cambium.node` span |
|
|
332
|
+
|
|
333
|
+
Node and wait spans carry `cambium.graph`, `cambium.graph.version`, `cambium.node`, `cambium.attempt`, `cambium.repo`, `cambium.issue.number`, `cambium.edge`, `cambium.outcome`.
|
|
334
|
+
|
|
335
|
+
```js
|
|
336
|
+
import { advanceHooks, createTelemetry } from './tracing.mjs' // inside cambium
|
|
337
|
+
|
|
338
|
+
const telemetry = await createTelemetry() // { env, exporter, metricReader, now } are for tests
|
|
339
|
+
try {
|
|
340
|
+
const hooks = advanceHooks(telemetry, {
|
|
341
|
+
graph,
|
|
342
|
+
graphVersion,
|
|
343
|
+
attempt,
|
|
344
|
+
traceparent,
|
|
345
|
+
enteredAt,
|
|
346
|
+
repo,
|
|
347
|
+
issue,
|
|
348
|
+
issueBody,
|
|
349
|
+
})
|
|
350
|
+
const r = await advance(g, at, ctx, { instrument: hooks.instrument })
|
|
351
|
+
marker.traceparent = hooks.traceparent // new when a cambium.issue root was started
|
|
352
|
+
} finally {
|
|
353
|
+
await telemetry.shutdown()
|
|
354
|
+
}
|
|
355
|
+
```
|
|
356
|
+
|
|
357
|
+
- `telemetry.nodeSpan(info, fn)`: runs `fn(run)` inside a `cambium.node` span. `run` is `{ span, issueTraceparent, traceparent, set({ edge, outcome }) }`: `issueTraceparent` is what the marker should hold; `traceparent` is this span's, to give a worker as `TRACEPARENT`. A throw sets ERROR and outcome `error` and is rethrown.
|
|
358
|
+
- `telemetry.waitSpan({ ...info, enteredAt, polls? })`: emits the `cambium.wait` span; returns the issue traceparent.
|
|
359
|
+
- `telemetry.notReady(info)`, `telemetry.issueTrace(info)` (the marker traceparent, starting a root if needed), `traceparentFor(spanContext)`, `flush()`, `shutdown()`.
|
|
360
|
+
- `advanceHooks(telemetry, base)` → `{ instrument, traceparent }`: step nodes run in `nodeSpan` and get **`ctx.traceparent`** (start-worker sets the worker's `TRACEPARENT` from it); `cambium.outcome` is the output's `outcome` string, else `ok`. A ready wait emits `waitSpan` (from `enteredAt`, or from the previous transition within one advance); a not-ready one bumps the counter.
|
|
361
|
+
|
|
362
|
+
### Worker spans from stream-json
|
|
363
|
+
|
|
364
|
+
Worker telemetry comes from the worker's own stdout log (`claude -p --output-format stream-json --verbose`), not from Claude Code's OTel export, so it works with telemetry off on the worker side:
|
|
365
|
+
|
|
366
|
+
```js
|
|
367
|
+
workerSpans(stream, { tracer, parent }) → { sessionId, complete, ok, start, end, tools, tasks, malformed, costUsd }
|
|
368
|
+
telemetry.workerSpans(stream, parent) // same, with cambium's tracer
|
|
369
|
+
```
|
|
370
|
+
|
|
371
|
+
`stream` is the log text, its lines, or parsed events; `parent` is the start-worker span's context or traceparent. Call it once the worker has exited; every span it makes is already ended.
|
|
372
|
+
|
|
373
|
+
- `claude.session`: first to last `timestamp`. Attributes from `result` and `system/init`: `claude.session_id`, `claude.model`, `claude_code.version`, `claude.cost_usd`, `claude.tokens.{input,output,cache_read,cache_creation}`, `claude.turns`, `claude.duration_ms`, `claude.terminal_reason`, `claude.result`, `claude.permission_denials`. No `result` (killed, truncated) or an error result → ERROR.
|
|
374
|
+
- `claude.tool`: each `tool_use` → `tool_result` by id (a subagent's tools nest under its Agent tool span); `claude.tool.name`, `claude.tool.outcome` (`ok` / `error` / `unfinished`), `claude.tool.command` (Bash, clipped to 200 chars). Unpaired → ends at the last timestamp, ERROR.
|
|
375
|
+
- `claude.task`: each background task, `task_started` → `task_notification` (or `task_updated.patch.end_time`), under the tool that started it. These events have no timestamp, so their times are those of the latest timestamped event.
|
|
376
|
+
- Malformed lines are counted (`malformed`) and skipped; `system/thinking_tokens`, `tool_progress`, hooks and rate-limit events are ignored.
|
|
377
|
+
|
|
378
|
+
### Does `claude -p` export traces that honour `TRACEPARENT`?
|
|
379
|
+
|
|
380
|
+
**Yes, opt-in and beta**; cambium does not depend on it (worker spans come from stream-json above). Per the [Claude Code monitoring docs](https://code.claude.com/docs/en/monitoring-usage) (checked with Claude Code 2.1.327; `claude --help` lists no telemetry flags, it is env-only):
|
|
381
|
+
|
|
382
|
+
- Traces need `CLAUDE_CODE_ENABLE_TELEMETRY=1` **and** `CLAUDE_CODE_ENHANCED_TELEMETRY_BETA=1` plus `OTEL_TRACES_EXPORTER=otlp` and the usual `OTEL_EXPORTER_OTLP_*` endpoint/protocol (use `OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf` against this collector). Spans: `claude_code.interaction` → `claude_code.llm_request`, `claude_code.tool` (→ `.execution`, `.blocked_on_user`).
|
|
383
|
+
- "In Agent SDK and non-interactive sessions started with `-p`, Claude Code also reads `TRACEPARENT` and `TRACESTATE` from its own environment when starting each interaction span": the interaction span becomes a child of the caller's span (`parent.source=env`). Interactive sessions ignore an inbound `TRACEPARENT`.
|
|
384
|
+
- Bash subprocesses it runs inherit a `TRACEPARENT` for the active tool span, so a `graft` call inside a worker could continue the trace.
|
|
385
|
+
- Prompts and tool inputs are redacted from spans unless `OTEL_LOG_USER_PROMPTS` / `OTEL_LOG_TOOL_DETAILS` / `OTEL_LOG_TOOL_CONTENT` are set.
|
|
386
|
+
|
|
387
|
+
So to also get Claude's own spans in the issue trace, add `CLAUDE_CODE_ENABLE_TELEMETRY=1`, `CLAUDE_CODE_ENHANCED_TELEMETRY_BETA=1`, `OTEL_TRACES_EXPORTER=otlp` and the endpoint to `worker.env`; start-worker already sets `TRACEPARENT`. Not verified against a live run here.
|
|
388
|
+
|
|
389
|
+
### Local collector
|
|
390
|
+
|
|
391
|
+
[`otel/`](otel/) holds an [otelcol-contrib](https://github.com/open-telemetry/opentelemetry-collector-releases/releases) config (OTLP/HTTP on `127.0.0.1:4318` → `~/.cambium/otel/{traces,metrics}.jsonl` + Jaeger) and a launchd agent, `com.oakgreencc.otelcol`. Nothing here loads it.
|
|
392
|
+
|
|
393
|
+
```sh
|
|
394
|
+
# 1. otelcol-contrib on PATH (darwin_arm64 release tarball; or darwin_amd64)
|
|
395
|
+
V=0.x.y # latest from the releases page
|
|
396
|
+
curl -fsSLO https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/v$V/otelcol-contrib_${V}_darwin_arm64.tar.gz
|
|
397
|
+
tar -xzf otelcol-contrib_${V}_darwin_arm64.tar.gz otelcol-contrib && mv otelcol-contrib /opt/homebrew/bin/
|
|
398
|
+
|
|
399
|
+
# 2. config + agent
|
|
400
|
+
mkdir -p ~/.cambium/otel
|
|
401
|
+
cp otel/otelcol.yaml ~/.cambium/otel/otelcol.yaml
|
|
402
|
+
otelcol-contrib validate --config ~/.cambium/otel/otelcol.yaml
|
|
403
|
+
cp otel/com.oakgreencc.otelcol.plist ~/Library/LaunchAgents/
|
|
404
|
+
launchctl bootstrap gui/$(id -u) ~/Library/LaunchAgents/com.oakgreencc.otelcol.plist
|
|
405
|
+
|
|
406
|
+
# 3. Jaeger UI on :16686, OTLP/gRPC on 14317 (its 4318 would clash with the collector)
|
|
407
|
+
docker run -d --name jaeger -p 16686:16686 -p 14317:4317 jaegertracing/jaeger:latest
|
|
408
|
+
|
|
409
|
+
# 4. point cambium (and mycelium) at it, e.g. in the cambium launchd plist's environment
|
|
410
|
+
OTEL_EXPORTER_OTLP_ENDPOINT=http://127.0.0.1:4318 OTEL_EXPORTER_OTLP_TIMEOUT=2000
|
|
411
|
+
```
|
|
412
|
+
|
|
413
|
+
Without Jaeger running the collector still writes the files; the Jaeger exporter just logs failures. Stop it with `launchctl bootout gui/$(id -u)/com.oakgreencc.otelcol`.
|
|
414
|
+
|
|
415
|
+
**AWS** is a collector-config change only: [`otel/otelcol-aws.yaml`](otel/otelcol-aws.yaml) keeps the same receiver and sends traces to X-Ray's OTLP endpoint (`https://xray.<region>.amazonaws.com/v1/traces`, `otlphttp` + `sigv4auth` with `service: xray`; needs X-Ray Transaction Search enabled) and metrics to Amazon Managed Service for Prometheus (`prometheusremotewrite` + `sigv4auth` with `service: aps`). Credentials come from the default AWS chain of the collector's process. cambium's env stays `OTEL_EXPORTER_OTLP_ENDPOINT=http://127.0.0.1:4318`.
|
|
261
416
|
|
|
262
417
|
## Lessons from the old Clock
|
|
263
418
|
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
|
2
|
+
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
|
3
|
+
<!--
|
|
4
|
+
The local OpenTelemetry collector as a launchd user agent. Not loaded by
|
|
5
|
+
anything in this repo: copy it to ~/Library/LaunchAgents and bootstrap it
|
|
6
|
+
yourself (see ../README.md#tracing). It reads
|
|
7
|
+
~/.cambium/otel/otelcol.yaml and logs to ~/.cambium/otel/otelcol.log.
|
|
8
|
+
-->
|
|
9
|
+
<plist version="1.0">
|
|
10
|
+
<dict>
|
|
11
|
+
<key>Label</key>
|
|
12
|
+
<string>com.oakgreencc.otelcol</string>
|
|
13
|
+
<key>ProgramArguments</key>
|
|
14
|
+
<array>
|
|
15
|
+
<string>/bin/sh</string>
|
|
16
|
+
<string>-c</string>
|
|
17
|
+
<string>exec otelcol-contrib --config "$HOME/.cambium/otel/otelcol.yaml" >>"$HOME/.cambium/otel/otelcol.log" 2>&1</string>
|
|
18
|
+
</array>
|
|
19
|
+
<key>EnvironmentVariables</key>
|
|
20
|
+
<dict>
|
|
21
|
+
<key>PATH</key>
|
|
22
|
+
<string>/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin</string>
|
|
23
|
+
</dict>
|
|
24
|
+
<key>RunAtLoad</key>
|
|
25
|
+
<true/>
|
|
26
|
+
<key>KeepAlive</key>
|
|
27
|
+
<true/>
|
|
28
|
+
<key>ThrottleInterval</key>
|
|
29
|
+
<integer>10</integer>
|
|
30
|
+
<key>ProcessType</key>
|
|
31
|
+
<string>Background</string>
|
|
32
|
+
</dict>
|
|
33
|
+
</plist>
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
# AWS variant: the same OTLP/HTTP receiver; traces go to AWS X-Ray's OTLP
|
|
2
|
+
# endpoint and metrics to Amazon Managed Service for Prometheus, both signed
|
|
3
|
+
# with SigV4 from the default AWS credential chain. cambium itself does not
|
|
4
|
+
# change: only the collector config does. Fill in REGION and WORKSPACE_ID.
|
|
5
|
+
#
|
|
6
|
+
# Prerequisites: X-Ray Transaction Search enabled in the account (the OTLP
|
|
7
|
+
# endpoint requires it); an AMP workspace; credentials allowed
|
|
8
|
+
# xray:PutTraceSegments (and the OTLP span ingest permission for Transaction
|
|
9
|
+
# Search) plus aps:RemoteWrite on the workspace.
|
|
10
|
+
extensions:
|
|
11
|
+
sigv4auth/xray:
|
|
12
|
+
region: REGION
|
|
13
|
+
service: xray
|
|
14
|
+
sigv4auth/aps:
|
|
15
|
+
region: REGION
|
|
16
|
+
service: aps
|
|
17
|
+
|
|
18
|
+
receivers:
|
|
19
|
+
otlp:
|
|
20
|
+
protocols:
|
|
21
|
+
http:
|
|
22
|
+
endpoint: 127.0.0.1:4318
|
|
23
|
+
|
|
24
|
+
processors:
|
|
25
|
+
batch: {}
|
|
26
|
+
memory_limiter:
|
|
27
|
+
check_interval: 1s
|
|
28
|
+
limit_mib: 256
|
|
29
|
+
|
|
30
|
+
exporters:
|
|
31
|
+
otlphttp/xray:
|
|
32
|
+
traces_endpoint: https://xray.REGION.amazonaws.com/v1/traces
|
|
33
|
+
auth:
|
|
34
|
+
authenticator: sigv4auth/xray
|
|
35
|
+
prometheusremotewrite/amp:
|
|
36
|
+
endpoint: https://aps-workspaces.REGION.amazonaws.com/workspaces/WORKSPACE_ID/api/v1/remote_write
|
|
37
|
+
auth:
|
|
38
|
+
authenticator: sigv4auth/aps
|
|
39
|
+
resource_to_telemetry_conversion:
|
|
40
|
+
enabled: true
|
|
41
|
+
|
|
42
|
+
service:
|
|
43
|
+
extensions: [sigv4auth/xray, sigv4auth/aps]
|
|
44
|
+
pipelines:
|
|
45
|
+
traces:
|
|
46
|
+
receivers: [otlp]
|
|
47
|
+
processors: [memory_limiter, batch]
|
|
48
|
+
exporters: [otlphttp/xray]
|
|
49
|
+
metrics:
|
|
50
|
+
receivers: [otlp]
|
|
51
|
+
processors: [memory_limiter, batch]
|
|
52
|
+
exporters: [prometheusremotewrite/amp]
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
# Local collector for cambium (and mycelium): OTLP/HTTP in on :4318,
|
|
2
|
+
# traces and metrics out to JSON-lines files, traces also to a local Jaeger.
|
|
3
|
+
# See ../README.md#tracing for install and the AWS variant (otelcol-aws.yaml).
|
|
4
|
+
receivers:
|
|
5
|
+
otlp:
|
|
6
|
+
protocols:
|
|
7
|
+
http:
|
|
8
|
+
endpoint: 127.0.0.1:4318
|
|
9
|
+
|
|
10
|
+
processors:
|
|
11
|
+
batch: {}
|
|
12
|
+
memory_limiter:
|
|
13
|
+
check_interval: 1s
|
|
14
|
+
limit_mib: 256
|
|
15
|
+
|
|
16
|
+
exporters:
|
|
17
|
+
file/traces:
|
|
18
|
+
path: ${env:HOME}/.cambium/otel/traces.jsonl
|
|
19
|
+
rotation:
|
|
20
|
+
max_megabytes: 50
|
|
21
|
+
max_backups: 5
|
|
22
|
+
file/metrics:
|
|
23
|
+
path: ${env:HOME}/.cambium/otel/metrics.jsonl
|
|
24
|
+
rotation:
|
|
25
|
+
max_megabytes: 50
|
|
26
|
+
max_backups: 5
|
|
27
|
+
# Jaeger takes OTLP natively. Its own OTLP/HTTP port (4318) would clash with
|
|
28
|
+
# this collector, so run it with OTLP/gRPC published on 14317 (see README).
|
|
29
|
+
otlp/jaeger:
|
|
30
|
+
endpoint: 127.0.0.1:14317
|
|
31
|
+
tls:
|
|
32
|
+
insecure: true
|
|
33
|
+
retry_on_failure:
|
|
34
|
+
enabled: false
|
|
35
|
+
sending_queue:
|
|
36
|
+
enabled: true
|
|
37
|
+
|
|
38
|
+
service:
|
|
39
|
+
telemetry:
|
|
40
|
+
logs:
|
|
41
|
+
level: warn
|
|
42
|
+
pipelines:
|
|
43
|
+
traces:
|
|
44
|
+
receivers: [otlp]
|
|
45
|
+
processors: [memory_limiter, batch]
|
|
46
|
+
exporters: [file/traces, otlp/jaeger]
|
|
47
|
+
metrics:
|
|
48
|
+
receivers: [otlp]
|
|
49
|
+
processors: [memory_limiter, batch]
|
|
50
|
+
exporters: [file/metrics]
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@oakgreencc/cambium",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.8",
|
|
4
4
|
"description": "The scheduler daemon: reads the q:* queues from GitHub every tick, runs headless claude workers in their own worktrees up to a cap, and makes every queue transition itself.",
|
|
5
5
|
"license": "AGPL-3.0-only",
|
|
6
6
|
"type": "module",
|
|
@@ -30,6 +30,7 @@
|
|
|
30
30
|
},
|
|
31
31
|
"files": [
|
|
32
32
|
"bin/",
|
|
33
|
+
"otel/",
|
|
33
34
|
"src/"
|
|
34
35
|
],
|
|
35
36
|
"engines": {
|
|
@@ -43,7 +44,13 @@
|
|
|
43
44
|
"name": "cambium"
|
|
44
45
|
},
|
|
45
46
|
"dependencies": {
|
|
46
|
-
"
|
|
47
|
+
"@opentelemetry/api": "^1.10.0",
|
|
48
|
+
"@opentelemetry/exporter-metrics-otlp-http": "^0.301.2",
|
|
49
|
+
"@opentelemetry/resources": "^3.1.1",
|
|
50
|
+
"@opentelemetry/sdk-metrics": "^3.1.1",
|
|
51
|
+
"@opentelemetry/sdk-trace": "^3.1.1",
|
|
52
|
+
"zod": "^4.6.7",
|
|
53
|
+
"@oakgreencc/mycelium": "0.0.0"
|
|
47
54
|
},
|
|
48
55
|
"scripts": {
|
|
49
56
|
"lint": "biome check .",
|
package/src/cli.mjs
CHANGED
|
@@ -13,6 +13,7 @@ import { acquireLock, readHeartbeat, realDeps, runDaemon } from './daemon.mjs'
|
|
|
13
13
|
import { plist, plistPath } from './launchd.mjs'
|
|
14
14
|
import { openStore } from './store.mjs'
|
|
15
15
|
import { finish, newCtx, startTask } from './tick.mjs'
|
|
16
|
+
import { runIssue } from './graph-engine.mjs'
|
|
16
17
|
|
|
17
18
|
export const VERSION = '0.0.0'
|
|
18
19
|
|
|
@@ -25,6 +26,9 @@ export const HELP = `cambium ${VERSION} — the scheduler daemon
|
|
|
25
26
|
cambium drain reconcile and watch, start no new workers
|
|
26
27
|
cambium run task --issue <n> [--repo <owner/name>]
|
|
27
28
|
claim one q:task issue, run its worker in the foreground, transition
|
|
29
|
+
cambium run <graph> --issue <n> [--repo <owner/name>] [--follow]
|
|
30
|
+
engine "graph": advance the issue in the foreground to the first wait
|
|
31
|
+
that is not ready; --follow keeps polling until the graph ends
|
|
28
32
|
cambium install [--dry-run] write the launchd agent (KeepAlive); --dry-run prints it
|
|
29
33
|
|
|
30
34
|
Common: --config <file> (default $CAMBIUM_CONFIG, then ~/.cambium/config.json)`
|
|
@@ -45,6 +49,7 @@ const has = (argv, name) => argv.includes(name)
|
|
|
45
49
|
* deps?: (config: import('./config.mjs').Config) => import('./tick.mjs').Deps,
|
|
46
50
|
* signal?: AbortSignal,
|
|
47
51
|
* now?: () => number,
|
|
52
|
+
* sleep?: (ms: number) => Promise<void>,
|
|
48
53
|
* }} IO
|
|
49
54
|
*/
|
|
50
55
|
|
|
@@ -141,15 +146,21 @@ function status(config, io, json) {
|
|
|
141
146
|
|
|
142
147
|
/**
|
|
143
148
|
* `run task --issue N`: one Task phase in the foreground, with the daemon's
|
|
144
|
-
* claim and transitions, so the daemon sees the same state afterwards.
|
|
149
|
+
* claim and transitions, so the daemon sees the same state afterwards. On
|
|
150
|
+
* `engine: "graph"`, `run <graph> --issue N [--follow]` runs the graph runtime
|
|
151
|
+
* in the foreground instead (graph-engine.mjs `runIssue`).
|
|
145
152
|
*
|
|
146
153
|
* @param {import('./config.mjs').Config} config @param {string[]} argv @param {IO} io
|
|
147
154
|
* @param {(config: import('./config.mjs').Config) => import('./tick.mjs').Deps} depsFor
|
|
148
155
|
*/
|
|
149
156
|
async function runOne(config, argv, io, depsFor) {
|
|
150
157
|
const n = Number(flag(argv, '--issue'))
|
|
151
|
-
|
|
152
|
-
|
|
158
|
+
const graph = argv[0]
|
|
159
|
+
if (!graph || graph.startsWith('-') || !Number.isInteger(n) || n <= 0)
|
|
160
|
+
return fail(io, 'usage: cambium run <graph> --issue <n> [--repo <owner/name>] [--follow]')
|
|
161
|
+
const follow = has(argv, '--follow')
|
|
162
|
+
if (config.engine === 'legacy' && (graph !== 'task' || follow))
|
|
163
|
+
return fail(io, `engine "legacy" runs only \`run task --issue <n>\`; set "engine": "graph" for graphs and --follow`)
|
|
153
164
|
const slug = flag(argv, '--repo') ?? (config.repos.length === 1 ? config.repos[0].slug : undefined)
|
|
154
165
|
if (!slug || !config.repos.some((r) => r.slug === slug))
|
|
155
166
|
return fail(
|
|
@@ -157,6 +168,10 @@ async function runOne(config, argv, io, depsFor) {
|
|
|
157
168
|
`--repo must name one of the configured repos: ${config.repos.map((r) => r.slug).join(', ') || '(none)'}`,
|
|
158
169
|
)
|
|
159
170
|
const deps = depsFor(config)
|
|
171
|
+
if (config.engine === 'graph') {
|
|
172
|
+
const r = await runIssue(deps, { graph, slug, number: n, follow, out: io.out, sleep: io.sleep, signal: io.signal })
|
|
173
|
+
return r.ok ? 0 : fail(io, /** @type {string} */ (r.error))
|
|
174
|
+
}
|
|
160
175
|
if (deps.store.worker(slug, n)) return fail(io, `${slug}#${n} already has a tracked worker`)
|
|
161
176
|
const snap = await deps.github.read(slug)
|
|
162
177
|
const issue = snap.issues.get(n)
|
package/src/config.mjs
CHANGED
|
@@ -12,6 +12,7 @@ import { join, resolve } from 'node:path'
|
|
|
12
12
|
* @typedef {{ cap: number, wallClockMinutes: number, maxTurns: number, maxAttempts: number }} PhaseCaps
|
|
13
13
|
* @typedef {{
|
|
14
14
|
* host: string,
|
|
15
|
+
* engine: 'legacy' | 'graph',
|
|
15
16
|
* tickSeconds: number,
|
|
16
17
|
* stepTimeoutSeconds: number,
|
|
17
18
|
* repos: Repo[],
|
|
@@ -32,6 +33,7 @@ export const DEFAULT_HOME = join(homedir(), '.cambium')
|
|
|
32
33
|
export function defaults(home = DEFAULT_HOME) {
|
|
33
34
|
return {
|
|
34
35
|
host: hostname(),
|
|
36
|
+
engine: 'legacy',
|
|
35
37
|
tickSeconds: 30,
|
|
36
38
|
stepTimeoutSeconds: 20,
|
|
37
39
|
repos: [],
|
|
@@ -86,6 +88,8 @@ export function resolveConfig(raw = {}) {
|
|
|
86
88
|
if (!r.path) throw new Error(`config: repo ${r.slug} needs "path" (a local clone)`)
|
|
87
89
|
return { slug: r.slug, path: untilde(r.path), baseBranch: r.baseBranch ?? 'main' }
|
|
88
90
|
})
|
|
91
|
+
if (config.engine !== 'legacy' && config.engine !== 'graph')
|
|
92
|
+
throw new Error(`config: engine must be "legacy" or "graph", got ${JSON.stringify(config.engine)}`)
|
|
89
93
|
if (!/^\d{2}:\d{2}$/.test(config.release.time)) throw new Error(`config: release.time must be HH:MM`)
|
|
90
94
|
return config
|
|
91
95
|
}
|
package/src/graph/advance.mjs
CHANGED
|
@@ -9,6 +9,10 @@ import { check, END, GraphNodeError } from './define.mjs'
|
|
|
9
9
|
* @typedef {'waiting' | 'end' | 'limit' | 'aborted' | 'error'} StopReason
|
|
10
10
|
* @typedef {{ transitions: Transition[], reason: StopReason, node: string, input: unknown, error?: GraphNodeError }} Advanced
|
|
11
11
|
* `node`/`input` is where it stopped: the held wait, END with the last output, the next node to run, or the failed node.
|
|
12
|
+
* @typedef {{ node: string, kind: 'node' | 'wait', input: unknown }} Visit
|
|
13
|
+
* @typedef {(visit: Visit, step: (extraCtx?: Record<string, unknown>) => Promise<Transition | null>) => Promise<Transition | null>} Instrument
|
|
14
|
+
* Wraps each node visit (tracing). `step` runs the node (or poll) and picks the edge: a Transition, or null for a
|
|
15
|
+
* wait that is not ready; it throws on a node error. `extraCtx` is merged into the node's ctx. Return what `step` returned.
|
|
12
16
|
*/
|
|
13
17
|
|
|
14
18
|
/** @param {unknown} err @param {string} node @param {'run' | 'poll' | 'edge'} phase */
|
|
@@ -37,7 +41,7 @@ async function call(node, phase, fn) {
|
|
|
37
41
|
* @param {import('./define.mjs').Graph} graph
|
|
38
42
|
* @param {Position} at
|
|
39
43
|
* @param {Record<string, unknown>} [ctx]
|
|
40
|
-
* @param {{ maxTransitions?: number, signal?: AbortSignal }} [opts]
|
|
44
|
+
* @param {{ maxTransitions?: number, signal?: AbortSignal, instrument?: Instrument }} [opts]
|
|
41
45
|
* @returns {Promise<Advanced>}
|
|
42
46
|
*/
|
|
43
47
|
export async function advance(graph, at, ctx = {}, opts = {}) {
|
|
@@ -56,22 +60,29 @@ export async function advance(graph, at, ctx = {}, opts = {}) {
|
|
|
56
60
|
if (transitions.length >= maxTransitions) return stop('limit')
|
|
57
61
|
const node = byName.get(name)
|
|
58
62
|
if (!node) return stop('error', new GraphNodeError(name, 'run', 'no such node in the graph'))
|
|
59
|
-
|
|
63
|
+
/** @type {(extraCtx?: Record<string, unknown>) => Promise<Transition | null>} */
|
|
64
|
+
const step = async (extraCtx) => {
|
|
65
|
+
const c = extraCtx ? { ...nodeCtx, ...extraCtx } : nodeCtx
|
|
60
66
|
const valid = await check(node.input, input, name, 'input')
|
|
61
67
|
let output
|
|
62
68
|
if (node.kind === 'wait') {
|
|
63
|
-
const polled = await call(name, 'poll', () => node.poll(valid,
|
|
69
|
+
const polled = await call(name, 'poll', () => node.poll(valid, c))
|
|
64
70
|
if (typeof polled?.ready !== 'boolean')
|
|
65
71
|
throw new GraphNodeError(name, 'poll', 'poll must return { ready: false } or { ready: true, output }')
|
|
66
|
-
if (!polled.ready) return
|
|
72
|
+
if (!polled.ready) return null
|
|
67
73
|
output = await check(node.output, polled.output, name, 'output')
|
|
68
74
|
} else {
|
|
69
|
-
output = await check(node.output, await call(name, 'run', () => node.run(valid,
|
|
75
|
+
output = await check(node.output, await call(name, 'run', () => node.run(valid, c)), name, 'output')
|
|
70
76
|
}
|
|
71
77
|
const { edge, to, next } = pick(graph, name, output, ctx.subject)
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
78
|
+
return { from: name, to, edge, input, output, next }
|
|
79
|
+
}
|
|
80
|
+
try {
|
|
81
|
+
const t = opts.instrument ? await opts.instrument({ node: name, kind: node.kind, input }, step) : await step()
|
|
82
|
+
if (!t) return stop('waiting')
|
|
83
|
+
transitions.push(t)
|
|
84
|
+
name = t.to
|
|
85
|
+
input = t.next
|
|
75
86
|
} catch (err) {
|
|
76
87
|
return stop('error', asNodeError(err, name, 'edge'))
|
|
77
88
|
}
|