loki-mode 10.6.8 → 10.6.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -14
- package/SKILL.md +2 -2
- package/VERSION +1 -1
- package/autonomy/lib/backlog.py +167 -36
- package/autonomy/lib/workspace.py +133 -39
- package/autonomy/lib/workspace_comment.py +91 -0
- package/autonomy/loki +92 -22
- package/bin/loki +3 -3
- package/completions/_loki +1 -0
- package/completions/loki.bash +1 -1
- package/dashboard/__init__.py +1 -1
- package/dashboard/api_start.py +26 -1
- package/docs/INSTALLATION.md +1 -1
- package/docs/TOOL-INTEGRATION.md +3 -3
- package/docs/WORKSPACES.md +11 -4
- package/docs/control-plane-container.md +8 -3
- package/docs/dashboard-guide.md +4 -0
- package/docs/exit-codes.md +1 -0
- package/docs/slack.md +12 -4
- package/docs/trackers.md +27 -0
- package/docs/v10/BOARD.md +70 -29
- package/docs/v10/CONTROL-PLANE.md +14 -7
- package/docs/v10/D51-PHASE-B.md +3 -1
- package/docs/v10/D61-SPEED.md +4 -0
- package/docs/v10/D63-CARDS.md +354 -0
- package/docs/v10/DECISIONS.md +44 -0
- package/docs/v10/E2E-D65.md +28 -0
- package/docs/v10/PROGRESS.md +63 -0
- package/loki-ts/dist/loki.js +584 -565
- package/mcp/__init__.py +1 -1
- package/mcp/tests/test_v10_tools.py +56 -0
- package/mcp/v10_tools.py +13 -3
- package/package.json +2 -1
- package/packages/control-plane/dist/server.js +76 -6
- package/packages/control-plane/drizzle/0001_group_id.sql +3 -0
- package/packages/control-plane/drizzle/meta/_journal.json +8 -1
- package/packages/control-plane/src/db/schema.ts +3 -0
- package/packages/control-plane/src/server/answer.ts +1 -1
- package/packages/control-plane/src/server/app.ts +6 -2
- package/packages/control-plane/src/server/auth.ts +51 -0
- package/packages/control-plane/src/server/runs.ts +4 -3
- package/packages/control-plane/src/server/serve.ts +11 -1
- package/packages/control-plane/src/shipper/discover.ts +29 -0
- package/packages/control-plane/ui/dist/assets/{index-53E32yrO.js → index-Jt31nrK-.js} +24 -24
- package/packages/control-plane/ui/dist/index.html +1 -1
- package/plugins/loki-mode/.claude-plugin/plugin.json +1 -1
- package/schemas/features.schema.json +71 -0
- package/schemas/loki-yaml.schema.json +123 -0
- package/schemas/manifest.schema.json +77 -0
- package/schemas/migration-plan.schema.json +109 -0
- package/schemas/outcome-contract.schema.json +73 -0
- package/schemas/seams.schema.json +90 -0
- package/schemas/status-result.schema.json +30 -0
- package/schemas/why-result.schema.json +51 -0
- package/web-app/dist/assets/{AdminPage-DN_pe8vo.js → AdminPage-OaSCF3M4.js} +1 -1
- package/web-app/dist/assets/{Avatar-BnQsr8z-.js → Avatar-xa3G7_vs.js} +1 -1
- package/web-app/dist/assets/{Badge-DzERiZn2.js → Badge-Ci_UcHij.js} +1 -1
- package/web-app/dist/assets/{Button-CaaoxQAt.js → Button-CFrrmbvL.js} +1 -1
- package/web-app/dist/assets/{CockpitPage-SXhnysJA.js → CockpitPage-6hyVl_QZ.js} +1 -1
- package/web-app/dist/assets/{ComparePage-krxTCFJY.js → ComparePage-B5H56KCG.js} +1 -1
- package/web-app/dist/assets/{ErrorBoundary-BL96ESMc.js → ErrorBoundary-CBbRHsYr.js} +1 -1
- package/web-app/dist/assets/{EvidenceReceiptPanel-PUiGSpkB.js → EvidenceReceiptPanel-D-Zoneum.js} +1 -1
- package/web-app/dist/assets/{GitHubIssuesPanel-C0hPs-P2.js → GitHubIssuesPanel-D7XPRsic.js} +1 -1
- package/web-app/dist/assets/{GitHubPRsPanel-CLvqNUlu.js → GitHubPRsPanel-CzHbBPAn.js} +1 -1
- package/web-app/dist/assets/{HomePage-X-sq_Rc9.js → HomePage-ClOx5qF1.js} +2 -2
- package/web-app/dist/assets/{LoginPage-DhQuwdua.js → LoginPage-3Yz_9j8E.js} +1 -1
- package/web-app/dist/assets/{MagicPage-CP2ODjfn.js → MagicPage-Dumn48gC.js} +1 -1
- package/web-app/dist/assets/{MetricsPage-C6SYiQES.js → MetricsPage-CYCYVG_8.js} +1 -1
- package/web-app/dist/assets/{NotFoundPage-CCDxxYc-.js → NotFoundPage-CV-x_1qH.js} +1 -1
- package/web-app/dist/assets/{ProjectPage-yDF--8FQ.js → ProjectPage-CeO7dnyM.js} +3 -3
- package/web-app/dist/assets/{ProjectsPage-BBLspdLn.js → ProjectsPage-BT9mz6iD.js} +1 -1
- package/web-app/dist/assets/{SettingsPage-C_sWbQRw.js → SettingsPage-BL8X9fAB.js} +1 -1
- package/web-app/dist/assets/{ShowcasePage-xCMuL0N8.js → ShowcasePage-8f3lnSpc.js} +1 -1
- package/web-app/dist/assets/{SystemSettingsPage-DnJKGCx7.js → SystemSettingsPage-BUNAAzh2.js} +1 -1
- package/web-app/dist/assets/{TeamsPage-BBXdhrSs.js → TeamsPage-Qj-VEZJz.js} +1 -1
- package/web-app/dist/assets/{TemplatesPage-AZ1Lr1rn.js → TemplatesPage-b8lKqe5q.js} +1 -1
- package/web-app/dist/assets/{TerminalOutput-CtIm9tXc.js → TerminalOutput-jOnfQKB4.js} +1 -1
- package/web-app/dist/assets/{activity-Dk1FK25Q.js → activity-w7GnyBNp.js} +1 -1
- package/web-app/dist/assets/{bell-dhqXyzP4.js → bell-2i4ejMOA.js} +1 -1
- package/web-app/dist/assets/{bot-CExHd4Kf.js → bot-jaPm4NIs.js} +1 -1
- package/web-app/dist/assets/{check-D51zaA1a.js → check-CvNtxeKM.js} +1 -1
- package/web-app/dist/assets/{chevron-left-DEwnQzH1.js → chevron-left-zzBjHAtX.js} +1 -1
- package/web-app/dist/assets/{circle-alert-C6U90Msk.js → circle-alert-CCU0IU2m.js} +1 -1
- package/web-app/dist/assets/{clock-6mFbMzzH.js → clock-BvZckMwN.js} +1 -1
- package/web-app/dist/assets/{cloud-BrZZtE7F.js → cloud-hrMWDdYp.js} +1 -1
- package/web-app/dist/assets/{code-xml-DTFzeepP.js → code-xml-v6l40T-K.js} +1 -1
- package/web-app/dist/assets/{database-B5rEiTre.js → database-l2TMUrQd.js} +1 -1
- package/web-app/dist/assets/{dollar-sign-bQs4rlS_.js → dollar-sign-C2V6OvpO.js} +1 -1
- package/web-app/dist/assets/{file-code-corner-Mo6e6qbf.js → file-code-corner-Bytb5JGA.js} +1 -1
- package/web-app/dist/assets/{file-plus-DzHZ0jpP.js → file-plus-BynCYQB8.js} +1 -1
- package/web-app/dist/assets/{globe-DYN9UhTa.js → globe-DifDdNMH.js} +1 -1
- package/web-app/dist/assets/{hammer-bB0hSRcJ.js → hammer-D7TsOJD6.js} +1 -1
- package/web-app/dist/assets/{index-CIH5CqZB.js → index-Zohfqaz4.js} +3 -3
- package/web-app/dist/assets/{layers-qhFkY8Nu.js → layers-Bk8c2YLS.js} +1 -1
- package/web-app/dist/assets/{loader-circle-BGBkaYH7.js → loader-circle-BrLdbMwP.js} +1 -1
- package/web-app/dist/assets/{lock-DIcsNFum.js → lock-COpmjMyJ.js} +1 -1
- package/web-app/dist/assets/{package-BdIIbZhS.js → package-BMzYdY7F.js} +1 -1
- package/web-app/dist/assets/{plus-Co1Bv8zT.js → plus-B9xpu1Pg.js} +1 -1
- package/web-app/dist/assets/{refresh-cw-CGF4ABWH.js → refresh-cw-C2Lzh-AA.js} +1 -1
- package/web-app/dist/assets/{rotate-ccw-BX10Q79h.js → rotate-ccw-2Yyy-uQd.js} +1 -1
- package/web-app/dist/assets/{scroll-text-CbXDlpGp.js → scroll-text-DiK2yhsT.js} +1 -1
- package/web-app/dist/assets/{server-CIhi6yUO.js → server-xlJBgleU.js} +1 -1
- package/web-app/dist/assets/{shield-alert-DJ5fB6Dg.js → shield-alert-D9w4yBCi.js} +1 -1
- package/web-app/dist/assets/{thumbs-up-BEft2T94.js → thumbs-up-CcWfvcE9.js} +1 -1
- package/web-app/dist/assets/{trash-2-CWGUPn7Y.js → trash-2-_hk_pEAp.js} +1 -1
- package/web-app/dist/assets/{trending-down-CjqstvxQ.js → trending-down-CDZT99tC.js} +1 -1
- package/web-app/dist/assets/{trending-up-CTyLjMzE.js → trending-up-A0oreTXc.js} +1 -1
- package/web-app/dist/assets/{upload-BmRx-uv4.js → upload-DfLwhGG2.js} +1 -1
- package/web-app/dist/assets/{usePolling-ChHkSJye.js → usePolling-B7gW2YY_.js} +1 -1
- package/web-app/dist/assets/{user-ZXh_f35z.js → user-C-y7_Oon.js} +1 -1
- package/web-app/dist/index.html +1 -1
package/README.md
CHANGED
|
@@ -11,7 +11,7 @@ An autonomous software factory that knows what it is supposed to deliver, and pr
|
|
|
11
11
|
|
|
12
12
|
Loki Mode is a free, source-available autonomous coding agent by [Autonomi](https://www.autonomi.dev/). Hand it a PRD, GitHub issue, OpenAPI doc or one-line brief. It derives a delivery contract (the acceptance criteria), builds against it, and ends with a signed Evidence Receipt that states what was proven and what was not. If the contract cannot be derived, it stops and asks one question instead of guessing.
|
|
13
13
|
|
|
14
|
-
Before a build counts as done, a review council selects reviewers from a specialist pool (`agents/types.json`, scored by `run.sh:FOCUS_KEYWORDS`). The bundled MCP server exposes 39 tools over stdio (`mcp/server.py`), including `loki_v10_run`, `loki_v10_status` and `loki_v10_verify` to start a Loki 10 run in the background, read its phase, verdict and cost, and verify its receipt.
|
|
14
|
+
Before a build counts as done, a review council selects reviewers from a specialist pool (`agents/types.json`, scored by `run.sh:FOCUS_KEYWORDS`). The bundled MCP server exposes 39 tools over stdio (`mcp/server.py`), including `loki_v10_run`, `loki_v10_status` and `loki_v10_verify` to start a Loki 10 run in the background, read its phase, verdict and cost, and verify its receipt (`BLOCKED` runs report their question). Add it with `claude mcp add loki-mode -- python3 -m mcp.server` from the install directory.
|
|
15
15
|
|
|
16
16
|
## Contents
|
|
17
17
|
|
|
@@ -153,7 +153,7 @@ The run cap is 900s (2700s with `--deep`, `DEFAULT_CAP_S` and `DEEP_CAP_S` in `t
|
|
|
153
153
|
|
|
154
154
|
A finished run prints a short summary (see [Quiet output](#quiet-output), which also shows an example) whose `NOT PROVEN` line is never empty by omission: deep checks deferred to the deep-verify pass are always listed there.
|
|
155
155
|
|
|
156
|
-
`loki status [run-id]` and `loki verify [run-id]` are built on the v10 path (bare `loki verify` follows the newest run, v10 or legacy). `loki dashboard` and `loki status` reach the v10 commands with `LOKI_ENGINE=v10`. Slack
|
|
156
|
+
`loki status [run-id]` and `loki verify [run-id]` are built on the v10 path (bare `loki verify` follows the newest run, v10 or legacy). `loki dashboard` and `loki status` reach the v10 commands with `LOKI_ENGINE=v10`. Two-way Slack (`loki slack serve`) is on by default and documented in [docs/slack.md](docs/slack.md); outbound notifications use `LOKI_SLACK_WEBHOOK_URL`.
|
|
157
157
|
|
|
158
158
|
## Run from Jira or Linear
|
|
159
159
|
|
|
@@ -167,14 +167,15 @@ loki linear:ENG-42 # or a linear.app issue URL
|
|
|
167
167
|
- Jira needs `JIRA_EMAIL` and `JIRA_API_TOKEN`, plus `JIRA_BASE_URL` (for example `https://acme.atlassian.net`) unless you pass the full browse URL. The ADF description is converted to plain text.
|
|
168
168
|
- Linear needs `LINEAR_API_KEY`.
|
|
169
169
|
- A missing variable stops the run before any work with an error naming it (exit code 2, like other intake errors). GitHub refs are unchanged.
|
|
170
|
+
- Self-hosted Jira: `<JIRA_BASE_URL>/browse/KEY` is accepted when its origin matches `JIRA_BASE_URL`. `LOKI_TRACKER_INTAKE=0` turns tracker intake off. Intake only, no sync. Details: [docs/trackers.md](docs/trackers.md).
|
|
170
171
|
|
|
171
|
-
## Two-way Slack
|
|
172
|
+
## Two-way Slack
|
|
172
173
|
|
|
173
|
-
With `
|
|
174
|
+
Two-way Slack is on by default. With `SLACK_BOT_TOKEN` and `SLACK_SIGNING_SECRET` in the environment, `loki slack serve --port N` (127.0.0.1 by default) lets you mention `@loki <issue ref or task>` in a Slack thread to start a run, and a BLOCKED question is answered in the same thread. With either variable missing it prints one line naming both and exits 2 without binding a port. Set `LOKI_SLACK_INBOUND=0` to disable it. See [docs/slack.md](docs/slack.md) for the scopes.
|
|
174
175
|
|
|
175
176
|
## Spec to contract
|
|
176
177
|
|
|
177
|
-
|
|
178
|
+
On by default; set `LOKI_CONTRACT=0` to turn it off. It does nothing unless `.loki/contract.json` exists. `loki contract <spec.md>` parses a spec or PRD into numbered acceptance criteria (AC-1, AC-2, up to 50) from checklist items (`- [ ]`) and from bullets and numbered lists under headings such as "Acceptance criteria", "Requirements" or "Must", prints them, and writes `.loki/contract.json` at the repo root. When a run seals with a contract present, each criterion is matched to changed files and checks by keyword overlap and recorded in an optional `contract` field on the receipt (`keyword_match` or `no_match`). Unmatched criteria are listed under NOT PROVEN in the receipt and the PR body (advisory only, the verdict is unchanged). Keyword overlap is a heuristic: `keyword_match` means a changed file plausibly relates to the criterion, not that the criterion is proven. The legacy PRD path is unchanged.
|
|
178
179
|
|
|
179
180
|
## Triggers without a cloud
|
|
180
181
|
|
|
@@ -276,19 +277,19 @@ Exit 0 when every required check passes; optional warnings (an absent provider C
|
|
|
276
277
|
|
|
277
278
|
Migration: `loki modernize heal <repo> --assess` was a legacy command for read-only analysis. The v10 path `loki modernize <repo> --to <target>` is not yet finished: only `--dry-run` works (inventory, dependency graph, unit clustering, cost, time and risk estimate). A real run stops after the estimate and prints `modernize: oracle capture and execution are not built yet; use --dry-run`. Targets `python3` and `java21` exist; the Java dependency graph is not wired in yet. Design: [docs/v10/MODERNIZE.md](docs/v10/MODERNIZE.md), user summary: [docs/v10/GUIDE-MODERNIZE.md](docs/v10/GUIDE-MODERNIZE.md).
|
|
278
279
|
|
|
279
|
-
## Control Plane v0
|
|
280
|
+
## Control Plane v0
|
|
280
281
|
|
|
281
|
-
A local control plane and UI for many runs,
|
|
282
|
+
A local control plane and UI for many runs, on by default. Turn it off with `LOKI_CONTROL=0`:
|
|
282
283
|
|
|
283
284
|
```bash
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
285
|
+
loki control serve # 127.0.0.1, default port 47821 (--port N, --db PATH)
|
|
286
|
+
loki control backfill . # ship ./.loki/runs to the control plane
|
|
287
|
+
loki control status # reachable? how many runs held?
|
|
287
288
|
```
|
|
288
289
|
|
|
289
|
-
The URL comes from `LOKI_CONTROL_URL`, else `http://127.0.0.1:${LOKI_CONTROL_PORT:-47821}`.
|
|
290
|
+
The URL comes from `LOKI_CONTROL_URL`, else `http://127.0.0.1:${LOKI_CONTROL_PORT:-47821}`. While `loki control serve` is running, runs on this machine ship to it automatically (it publishes `~/.loki/control/instance.json`, mode 0600, removed on exit); a run never starts the server. Set `LOKI_CONTROL_URL` to ship elsewhere. See [docs/v10/CONTROL-PLANE.md](docs/v10/CONTROL-PLANE.md).
|
|
290
291
|
|
|
291
|
-
Answering a BLOCKED run: when a run stops on a spec conflict, its run page shows the question and an answer box. Submitting posts to `POST /v1/runs/<source>/<run>/answer` (JSON `{"answer": "..."}`, up to 4000 characters, loopback only) and writes `~/.loki/control/answers/<source>/<run>.answer.txt` (override the directory with `LOKI_CONTROL_ANSWER_DIR`). The page prints the resume command, which
|
|
292
|
+
Answering a BLOCKED run: when a run stops on a spec conflict, its run page shows the question and an answer box. Submitting posts to `POST /v1/runs/<source>/<run>/answer` (JSON `{"answer": "..."}`, up to 4000 characters, loopback only) and writes `~/.loki/control/answers/<source>/<run>.answer.txt` (override the directory with `LOKI_CONTROL_ANSWER_DIR`). The page prints the resume command, `loki answer <run>`, which starts a fresh run carrying the task, the question and that file's text (or pass `--text "..."` yourself; with no run id it picks the newest BLOCKED run). A run that is not BLOCKED exits 2.
|
|
292
293
|
|
|
293
294
|
Set `LOKI_CONTROL_DEFAULT=1` (off by default) to make `loki dashboard`, `loki dashboard start` and `loki dashboard open` start the Control Plane instead of the old dashboard. The old dashboard is unchanged when the flag is unset.
|
|
294
295
|
|
|
@@ -552,7 +553,7 @@ Loki 10 engine:
|
|
|
552
553
|
| `loki status [--json]` | Current status |
|
|
553
554
|
| `loki verify [run-id]` | Re-check a sealed receipt (exit codes above) |
|
|
554
555
|
| `loki doctor [--json] [--airgap]` | Check environment and providers |
|
|
555
|
-
| `
|
|
556
|
+
| `loki control serve\|backfill\|status` | Control Plane server, ship existing runs, reachability check (on by default; `LOKI_CONTROL=0` turns it off) |
|
|
556
557
|
| `loki modernize <repo> --to <target> --dry-run` | Estimate only |
|
|
557
558
|
| `loki plan [PRD]` | Dry-run analysis: complexity, cost, execution plan |
|
|
558
559
|
| `loki version` | Show version |
|
|
@@ -645,7 +646,7 @@ loki enterprise status
|
|
|
645
646
|
| **Testing** | 8 automated quality gates | Test quality depends on AI assertions |
|
|
646
647
|
| **Providers** | Claude, Cline, Codex, Aider and opencode | Non-Claude providers are experimental and mostly sequential |
|
|
647
648
|
| **Dashboard** | Real-time single-machine monitoring | No multi-node clustering |
|
|
648
|
-
| **Loki 10** | The v10 run (`loki "<task>"`, `loki owner/repo#N`, `loki quick`), `status`, `verify`, `backlog` | `loki start <issue ref|issue URL|"task">` runs Loki 10, `loki start <file>` stays legacy; `loki modernize <repo> --to` runs `--dry-run` only; Slack is
|
|
649
|
+
| **Loki 10** | The v10 run (`loki "<task>"`, `loki owner/repo#N`, `loki quick`), `status`, `verify`, `backlog` | `loki start <issue ref|issue URL|"task">` runs Loki 10, `loki start <file>` stays legacy; `loki modernize <repo> --to` runs `--dry-run` only; two-way Slack is on by default (`loki slack serve`, `LOKI_SLACK_INBOUND=0` disables); Control Plane v0 is a preview |
|
|
649
650
|
|
|
650
651
|
> **What "autonomous" means:** the system runs RARV cycles without prompting. It does NOT access your cloud accounts, payment systems or external services unless you provide credentials. Human oversight is expected for deployment, API keys and critical decisions.
|
|
651
652
|
|
package/SKILL.md
CHANGED
|
@@ -3,7 +3,7 @@ name: loki-mode
|
|
|
3
3
|
description: An autonomous software factory that knows what it is supposed to deliver, and proves it did. Use when the user says Loki Mode or asks to build, fix or verify software autonomously.
|
|
4
4
|
---
|
|
5
5
|
|
|
6
|
-
# Loki Mode v10.6.
|
|
6
|
+
# Loki Mode v10.6.10
|
|
7
7
|
|
|
8
8
|
**Current entry points (Loki 10 engine):** `loki "<task>"`, `loki owner/repo#N` and `loki quick "<task>"` run the Loki 10 engine and end with a signed receipt (`loki verify`). `loki backlog owner/repo --all` runs many issues in parallel. Guide: `docs/v10/GUIDE.md`. The skill instructions below, and `loki start`, describe the legacy engine, which still ships and is being removed (`loki start owner/repo#N` still routes to it; prefer `loki owner/repo#N`).
|
|
9
9
|
|
|
@@ -473,4 +473,4 @@ See `CHANGELOG.md` entries [7.5.7], [7.5.8], [7.5.13] for the per-fix list and r
|
|
|
473
473
|
|
|
474
474
|
---
|
|
475
475
|
|
|
476
|
-
**v10.6.
|
|
476
|
+
**v10.6.10 | [Autonomi](https://www.autonomi.dev/) flagship product | core skill**
|
package/VERSION
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
10.6.
|
|
1
|
+
10.6.10
|
package/autonomy/lib/backlog.py
CHANGED
|
@@ -10,6 +10,15 @@ ladder: 0 VERIFIED, 1 FAILED, 3 BUDGET_STOP, 4 BLOCKED, 5 STALLED). A draft PR
|
|
|
10
10
|
from a failed run is still FAILED. Exit: 0 all ok, 3 only budget stops, 1 other
|
|
11
11
|
failures, 2 usage/config error.
|
|
12
12
|
|
|
13
|
+
Unit mode (D61 slice 10): --dag FILE [--group G] also runs the units of a
|
|
14
|
+
decomposer DAG (engine10 Dag JSON: units[] with id, items, writeSet, optional
|
|
15
|
+
deps; edges[] {from,to} mean "to" waits for "from"). Unit n runs on branch
|
|
16
|
+
loki/unit-<group>-<n> in worktree unit-<group>-<n>, shares the one slot pool
|
|
17
|
+
with issues, starts only after every parent passed, and is SKIPPED when a
|
|
18
|
+
parent did not. The launcher gets the unit task text plus --no-pr, and the env
|
|
19
|
+
LOKI_GROUP_ID, LOKI_UNIT_ID, LOKI_UNIT_N, LOKI_UNIT_DEPS, LOKI_UNIT_WRITE_SET.
|
|
20
|
+
Passing unit branches are kept for the integrator; issue mode is unchanged.
|
|
21
|
+
|
|
13
22
|
Test seam: LOKI_BACKLOG_LAUNCHER replaces the real launcher (called with the
|
|
14
23
|
issue ref, cwd = the issue worktree).
|
|
15
24
|
"""
|
|
@@ -27,6 +36,8 @@ import time
|
|
|
27
36
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
28
37
|
sys.path.insert(0, HERE)
|
|
29
38
|
import loki_yaml # noqa: E402
|
|
39
|
+
import workspace # noqa: E402
|
|
40
|
+
import worktree_prep # noqa: E402
|
|
30
41
|
|
|
31
42
|
REPO_ROOT = os.path.abspath(os.path.join(HERE, "..", ".."))
|
|
32
43
|
REPO_RE = re.compile(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$")
|
|
@@ -110,6 +121,72 @@ def parse_result(rc, text):
|
|
|
110
121
|
return False, "FAILED: " + (reason or "exit %d" % rc), usd
|
|
111
122
|
|
|
112
123
|
|
|
124
|
+
MAX_DAG_UNITS = 500
|
|
125
|
+
MAX_DAG_EDGES = 5000
|
|
126
|
+
UNIT_ID_RE = re.compile(r"^[A-Za-z0-9_][A-Za-z0-9_.:/-]{0,127}$")
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def load_dag(path, group):
|
|
130
|
+
"""-> (group, units) where units = [{n, id, task, deps(list of ids), write_set}] or raises ValueError."""
|
|
131
|
+
try:
|
|
132
|
+
with open(path) as f:
|
|
133
|
+
dag = json.load(f)
|
|
134
|
+
except (OSError, ValueError) as e:
|
|
135
|
+
raise ValueError("cannot read DAG %s: %s" % (path, e))
|
|
136
|
+
raw = dag.get("units") if isinstance(dag, dict) else None
|
|
137
|
+
if not isinstance(raw, list) or not raw:
|
|
138
|
+
raise ValueError("DAG has no units")
|
|
139
|
+
group = group or str(dag.get("group") or "g1")
|
|
140
|
+
if not re.match(r"^[A-Za-z0-9_][A-Za-z0-9_.-]*$", group) or ".." in group:
|
|
141
|
+
raise ValueError("group must match [A-Za-z0-9_][A-Za-z0-9_.-]* with no '..', got %r" % group)
|
|
142
|
+
if len(raw) > MAX_DAG_UNITS:
|
|
143
|
+
raise ValueError("DAG has %d units, the limit is %d" % (len(raw), MAX_DAG_UNITS))
|
|
144
|
+
ids = []
|
|
145
|
+
for u in raw:
|
|
146
|
+
uid = str(u.get("id", "")) if isinstance(u, dict) else ""
|
|
147
|
+
if not uid or uid in ids:
|
|
148
|
+
raise ValueError("DAG unit ids must be present and unique (got %r)" % uid)
|
|
149
|
+
if not UNIT_ID_RE.match(uid):
|
|
150
|
+
raise ValueError("DAG unit id %r is invalid (allowed: %s)" % (uid, UNIT_ID_RE.pattern))
|
|
151
|
+
ids.append(uid)
|
|
152
|
+
deps = {i: [] for i in ids}
|
|
153
|
+
for u in raw:
|
|
154
|
+
for d in u.get("deps") or []:
|
|
155
|
+
deps[str(u["id"])].append(str(d))
|
|
156
|
+
edges = dag.get("edges") or []
|
|
157
|
+
if not isinstance(edges, list) or len(edges) > MAX_DAG_EDGES:
|
|
158
|
+
raise ValueError("DAG edges must be a list of at most %d entries" % MAX_DAG_EDGES)
|
|
159
|
+
for e in edges:
|
|
160
|
+
if not isinstance(e, dict):
|
|
161
|
+
raise ValueError("DAG edge must be an object, got %r" % (e,))
|
|
162
|
+
src, dst = str(e.get("from")), str(e.get("to"))
|
|
163
|
+
if src not in deps or dst not in deps:
|
|
164
|
+
raise ValueError("DAG edge %s -> %s names an unknown unit id" % (src, dst))
|
|
165
|
+
if src not in deps[dst]:
|
|
166
|
+
deps[dst].append(src)
|
|
167
|
+
for i, ds in deps.items():
|
|
168
|
+
for d in ds:
|
|
169
|
+
if d not in deps or d == i:
|
|
170
|
+
raise ValueError("unit %s depends on unknown or itself: %s" % (i, d))
|
|
171
|
+
left = dict(deps)
|
|
172
|
+
while left:
|
|
173
|
+
free = [i for i, ds in left.items() if not [d for d in ds if d in left]]
|
|
174
|
+
if not free:
|
|
175
|
+
raise ValueError("DAG has a dependency cycle among: " + ", ".join(sorted(left)))
|
|
176
|
+
for i in free:
|
|
177
|
+
del left[i]
|
|
178
|
+
units = []
|
|
179
|
+
for n, u in enumerate(raw, 1):
|
|
180
|
+
uid = str(u["id"])
|
|
181
|
+
items = u.get("items") or []
|
|
182
|
+
task = "\n".join(str(x) for x in items) or str(u.get("task") or uid)
|
|
183
|
+
if task.lstrip().startswith("-"):
|
|
184
|
+
raise ValueError("unit %s task text starts with '-' and would be read as an option" % uid)
|
|
185
|
+
units.append({"n": n, "id": uid, "deps": deps[uid], "write_set": list(u.get("writeSet") or []),
|
|
186
|
+
"task": task})
|
|
187
|
+
return group, units
|
|
188
|
+
|
|
189
|
+
|
|
113
190
|
def main(argv):
|
|
114
191
|
ap = argparse.ArgumentParser(prog="loki backlog", add_help=True)
|
|
115
192
|
ap.add_argument("repo")
|
|
@@ -119,12 +196,20 @@ def main(argv):
|
|
|
119
196
|
sel.add_argument("--issues")
|
|
120
197
|
ap.add_argument("--concurrency", type=int)
|
|
121
198
|
ap.add_argument("--dry-run", action="store_true")
|
|
199
|
+
ap.add_argument("--dag", help="decomposer DAG JSON; its units run on loki/unit-<group>-<n>")
|
|
200
|
+
ap.add_argument("--group", help="unit group id (default: the DAG's group, else g1)")
|
|
122
201
|
a = ap.parse_args(argv)
|
|
123
202
|
|
|
124
203
|
if not REPO_RE.match(a.repo):
|
|
125
204
|
return die("repo must be owner/repo, got %r" % a.repo)
|
|
126
|
-
if not (a.all or a.label or a.issues):
|
|
127
|
-
return die("choose one of --all, --label X, --issues 1,2,3")
|
|
205
|
+
if not (a.all or a.label or a.issues or a.dag):
|
|
206
|
+
return die("choose one of --all, --label X, --issues 1,2,3, --dag FILE")
|
|
207
|
+
group, dag_units = None, []
|
|
208
|
+
if a.dag:
|
|
209
|
+
try:
|
|
210
|
+
group, dag_units = load_dag(a.dag, a.group)
|
|
211
|
+
except ValueError as e:
|
|
212
|
+
return die(str(e))
|
|
128
213
|
numbers = None
|
|
129
214
|
if a.issues:
|
|
130
215
|
if not re.match(r"^\d+(,\d+)*$", a.issues):
|
|
@@ -162,16 +247,19 @@ def main(argv):
|
|
|
162
247
|
|
|
163
248
|
try:
|
|
164
249
|
issues = ([{"number": n, "title": ""} for n in numbers] if numbers
|
|
165
|
-
else list_issues(a.repo, a.label, env))
|
|
250
|
+
else list_issues(a.repo, a.label, env) if (a.all or a.label) else [])
|
|
166
251
|
except RuntimeError as e:
|
|
167
252
|
return die(redact(str(e), token), 1)
|
|
168
253
|
|
|
169
|
-
say("backlog: %s concurrency %d, %d issue(s)" % (a.repo, conc, len(issues))
|
|
254
|
+
say("backlog: %s concurrency %d, %d issue(s)" % (a.repo, conc, len(issues))
|
|
255
|
+
+ (", %d unit(s) in group %s" % (len(dag_units), group) if dag_units else ""))
|
|
170
256
|
if a.dry_run:
|
|
171
257
|
for i in issues:
|
|
172
258
|
say("backlog: would run #%d %s" % (i["number"], i["title"]))
|
|
259
|
+
for u in dag_units:
|
|
260
|
+
say("backlog: would run unit %s-%d (%s) after: %s" % (group, u["n"], u["id"], ", ".join(u["deps"]) or "nothing"))
|
|
173
261
|
return 0
|
|
174
|
-
if not issues:
|
|
262
|
+
if not issues and not dag_units:
|
|
175
263
|
say("backlog: nothing to do")
|
|
176
264
|
return 0
|
|
177
265
|
|
|
@@ -191,11 +279,25 @@ def main(argv):
|
|
|
191
279
|
child_env["LOKI_BUDGET_LIMIT"] = str(cfg["budgets"]["per_run_usd"])
|
|
192
280
|
day_cap = cfg.get("budgets", {}).get("per_day_usd")
|
|
193
281
|
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
282
|
+
# One job table for issues and units: they share one slot pool.
|
|
283
|
+
spec = {}
|
|
284
|
+
for i in issues:
|
|
285
|
+
n = i["number"]
|
|
286
|
+
spec[n] = {"label": "#%d" % n, "branch": "loki/backlog-%d" % n, "wt": "issue-%d" % n,
|
|
287
|
+
"cmd": [launcher, "%s#%d" % (a.repo, n)], "env": {}, "deps": [], "unit": False}
|
|
288
|
+
key_of = {u["id"]: "unit-%s-%d" % (group, u["n"]) for u in dag_units}
|
|
289
|
+
for u in dag_units:
|
|
290
|
+
k = key_of[u["id"]]
|
|
291
|
+
spec[k] = {"label": k, "branch": "loki/" + k, "wt": k, "unit": True,
|
|
292
|
+
"cmd": [launcher, u["task"], "--no-pr"],
|
|
293
|
+
"deps": [key_of[d] for d in u["deps"]],
|
|
294
|
+
"env": {"LOKI_GROUP_ID": group, "LOKI_UNIT_ID": u["id"], "LOKI_UNIT_N": str(u["n"]),
|
|
295
|
+
"LOKI_UNIT_DEPS": ",".join(u["deps"]), "LOKI_UNIT_WRITE_SET": "\n".join(u["write_set"])}}
|
|
296
|
+
state = {k: {"status": "queued", "ok": None, "cost": None, "ran": False} for k in spec}
|
|
297
|
+
for k in state:
|
|
298
|
+
say("backlog: %s queued" % spec[k]["label"])
|
|
299
|
+
pending = list(spec)
|
|
300
|
+
running = {} # key -> (Popen, log path, worktree)
|
|
199
301
|
unmeasured = 0
|
|
200
302
|
|
|
201
303
|
def stop_children(*_):
|
|
@@ -208,9 +310,9 @@ def main(argv):
|
|
|
208
310
|
signal.signal(signal.SIGTERM, stop_children)
|
|
209
311
|
signal.signal(signal.SIGINT, stop_children)
|
|
210
312
|
|
|
211
|
-
def finish(
|
|
313
|
+
def finish(k):
|
|
212
314
|
nonlocal unmeasured
|
|
213
|
-
p, log, wt = running.pop(
|
|
315
|
+
p, log, wt = running.pop(k)
|
|
214
316
|
try:
|
|
215
317
|
with open(log, errors="replace") as f:
|
|
216
318
|
text = redact(f.read(), token)
|
|
@@ -223,51 +325,80 @@ def main(argv):
|
|
|
223
325
|
unmeasured += 1
|
|
224
326
|
else:
|
|
225
327
|
add_spend(usd)
|
|
226
|
-
state[
|
|
227
|
-
say("backlog:
|
|
328
|
+
state[k].update(status=status, ok=ok, cost=usd)
|
|
329
|
+
say("backlog: %s %s" % (spec[k]["label"], status))
|
|
228
330
|
if ok:
|
|
229
331
|
git("worktree", "remove", "--force", wt, cwd=top)
|
|
230
|
-
|
|
332
|
+
if not spec[k]["unit"]: # a passing unit branch is kept for the integrator
|
|
333
|
+
git("branch", "-D", spec[k]["branch"], cwd=top)
|
|
231
334
|
|
|
232
|
-
def launch(
|
|
233
|
-
|
|
335
|
+
def launch(k):
|
|
336
|
+
s = spec[k]
|
|
337
|
+
wt = os.path.join(wt_root, s["wt"])
|
|
234
338
|
if os.path.exists(wt):
|
|
235
339
|
git("worktree", "remove", "--force", wt, cwd=top)
|
|
236
|
-
|
|
237
|
-
|
|
340
|
+
if workspace.enabled():
|
|
341
|
+
# Shared worktree prep (D51-B05): per-base lock, clean start point, deps copied.
|
|
342
|
+
git("branch", "-D", s["branch"], cwd=top)
|
|
343
|
+
try:
|
|
344
|
+
worktree_prep.prepare_worktree(top, wt, s["branch"])
|
|
345
|
+
r = None
|
|
346
|
+
except (RuntimeError, subprocess.CalledProcessError) as e:
|
|
347
|
+
msg = getattr(e, "stderr", None) or str(e)
|
|
348
|
+
r = subprocess.CompletedProcess([], 1, "", msg)
|
|
349
|
+
else:
|
|
350
|
+
r = git("worktree", "add", "-B", s["branch"], wt, "HEAD", cwd=top)
|
|
351
|
+
if r is not None and r.returncode != 0:
|
|
238
352
|
err = (r.stderr.strip().splitlines() or ["git error"])[-1]
|
|
239
|
-
state[
|
|
240
|
-
say("backlog:
|
|
353
|
+
state[k].update(status="FAILED: worktree: " + redact(err, token), ok=False)
|
|
354
|
+
say("backlog: %s %s" % (s["label"], state[k]["status"]))
|
|
241
355
|
return
|
|
242
|
-
log = os.path.join(wt_root, "logs", "
|
|
243
|
-
cmd = [launcher, "%s#%d" % (a.repo, n)]
|
|
356
|
+
log = os.path.join(wt_root, "logs", s["wt"] + ".log")
|
|
244
357
|
with open(log, "w") as lf:
|
|
245
|
-
p = subprocess.Popen(cmd, cwd=wt, env=child_env, stdout=lf,
|
|
246
|
-
stdin=subprocess.DEVNULL, start_new_session=True)
|
|
247
|
-
running[
|
|
248
|
-
state[
|
|
249
|
-
say("backlog:
|
|
358
|
+
p = subprocess.Popen(s["cmd"], cwd=wt, env=dict(child_env, **s["env"]), stdout=lf,
|
|
359
|
+
stderr=subprocess.STDOUT, stdin=subprocess.DEVNULL, start_new_session=True)
|
|
360
|
+
running[k] = (p, log, wt)
|
|
361
|
+
state[k].update(status="running", ran=True)
|
|
362
|
+
say("backlog: %s running" % s["label"])
|
|
363
|
+
|
|
364
|
+
def next_ready():
|
|
365
|
+
"""First pending job whose parents all passed; skip jobs whose parent did not."""
|
|
366
|
+
for k in list(pending):
|
|
367
|
+
ds = spec[k]["deps"]
|
|
368
|
+
bad = [d for d in ds if state[d]["ok"] is False]
|
|
369
|
+
if bad:
|
|
370
|
+
pending.remove(k)
|
|
371
|
+
state[k].update(status="SKIPPED: dependency %s did not pass" % spec[bad[0]]["label"], ok=False)
|
|
372
|
+
say("backlog: %s %s" % (spec[k]["label"], state[k]["status"]))
|
|
373
|
+
return next_ready()
|
|
374
|
+
if all(state[d]["ok"] is True for d in ds):
|
|
375
|
+
pending.remove(k)
|
|
376
|
+
return k
|
|
377
|
+
return None
|
|
250
378
|
|
|
251
379
|
while pending or running:
|
|
252
|
-
for
|
|
253
|
-
finish(
|
|
380
|
+
for k in [k for k, (p, _l, _w) in running.items() if p.poll() is not None]:
|
|
381
|
+
finish(k)
|
|
254
382
|
while pending and len(running) < conc:
|
|
255
383
|
if day_cap and spent_today() >= day_cap:
|
|
256
|
-
for
|
|
257
|
-
state[
|
|
258
|
-
say("backlog:
|
|
384
|
+
for k in pending:
|
|
385
|
+
state[k].update(status="BUDGET_STOP: daily budget $%.2f reached ($%.2f spent)" % (day_cap, spent_today()), ok=False)
|
|
386
|
+
say("backlog: %s %s" % (spec[k]["label"], state[k]["status"]))
|
|
259
387
|
pending = []
|
|
260
388
|
break
|
|
261
|
-
|
|
389
|
+
k = next_ready()
|
|
390
|
+
if k is None:
|
|
391
|
+
break
|
|
392
|
+
launch(k)
|
|
262
393
|
if running:
|
|
263
394
|
time.sleep(0.2)
|
|
264
395
|
|
|
265
396
|
# ---- summary -----------------------------------------------------------
|
|
266
397
|
say("")
|
|
267
398
|
say("Summary: %s" % a.repo)
|
|
268
|
-
say("%-
|
|
399
|
+
say("%-14s %-10s %s" % ("ITEM", "COST", "RESULT"))
|
|
269
400
|
for n, s in state.items():
|
|
270
|
-
say("%-
|
|
401
|
+
say("%-14s %-10s %s" % (spec[n]["label"], ("$%.2f" % s["cost"] if s["cost"] is not None else "unmeasured" if s["ran"] else "-"), s["status"]))
|
|
271
402
|
measured = sum(s["cost"] or 0 for s in state.values())
|
|
272
403
|
say("Total measured cost: $%.2f%s" % (measured, " (%d run(s) unmeasured, not counted as $0)" % unmeasured if unmeasured else ""))
|
|
273
404
|
bad = [s for s in state.values() if not s["ok"]]
|