@really-knows-ai/foundry 2.3.1 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (170) hide show
  1. package/README.md +200 -198
  2. package/dist/.opencode/plugins/foundry-tools/appraiser-tools.js +28 -0
  3. package/dist/.opencode/plugins/foundry-tools/artefact-tools.js +58 -0
  4. package/dist/.opencode/plugins/foundry-tools/assay-tools.js +92 -0
  5. package/dist/.opencode/plugins/foundry-tools/attestation-tools.js +191 -0
  6. package/dist/.opencode/plugins/foundry-tools/config-create-tools.js +128 -0
  7. package/dist/.opencode/plugins/foundry-tools/config-law-tools.js +380 -0
  8. package/dist/.opencode/plugins/foundry-tools/config-tools.js +43 -0
  9. package/dist/.opencode/plugins/foundry-tools/feedback-tools.js +234 -0
  10. package/dist/.opencode/plugins/foundry-tools/git-helpers.js +354 -0
  11. package/dist/.opencode/plugins/foundry-tools/git-tools.js +181 -0
  12. package/dist/.opencode/plugins/foundry-tools/helpers.js +340 -0
  13. package/dist/.opencode/plugins/foundry-tools/history-tools.js +20 -0
  14. package/dist/.opencode/plugins/foundry-tools/memory-admin-tools.js +296 -0
  15. package/dist/.opencode/plugins/foundry-tools/memory-helpers.js +104 -0
  16. package/dist/.opencode/plugins/foundry-tools/memory-tools.js +286 -0
  17. package/dist/.opencode/plugins/foundry-tools/orchestrate-tool.js +159 -0
  18. package/dist/.opencode/plugins/foundry-tools/snapshot-tools.js +104 -0
  19. package/dist/.opencode/plugins/foundry-tools/stage-tools.js +186 -0
  20. package/dist/.opencode/plugins/foundry-tools/validate-tools.js +263 -0
  21. package/dist/.opencode/plugins/foundry-tools/workfile-tools.js +102 -0
  22. package/dist/.opencode/plugins/foundry.js +105 -0
  23. package/dist/CHANGELOG.md +490 -0
  24. package/dist/LICENSE +21 -0
  25. package/dist/README.md +278 -0
  26. package/dist/docs/README.md +59 -0
  27. package/dist/docs/architecture.md +434 -0
  28. package/dist/docs/concepts.md +396 -0
  29. package/dist/docs/getting-started.md +345 -0
  30. package/dist/docs/memory-maintenance.md +176 -0
  31. package/dist/docs/tools.md +1411 -0
  32. package/dist/docs/work-spec.md +283 -0
  33. package/dist/scripts/lib/artefacts.js +151 -0
  34. package/dist/scripts/lib/assay/loader.js +151 -0
  35. package/dist/scripts/lib/assay/parse-jsonl.js +102 -0
  36. package/dist/scripts/lib/assay/permissions.js +52 -0
  37. package/dist/scripts/lib/assay/run.js +219 -0
  38. package/dist/scripts/lib/assay/spawn-with-timeout.js +138 -0
  39. package/dist/scripts/lib/attestation/attest.js +111 -0
  40. package/dist/scripts/lib/attestation/canonical-json.js +109 -0
  41. package/dist/scripts/lib/attestation/hash.js +17 -0
  42. package/dist/scripts/lib/attestation/parse.js +14 -0
  43. package/dist/scripts/lib/attestation/payload.js +106 -0
  44. package/dist/scripts/lib/attestation/render.js +16 -0
  45. package/dist/scripts/lib/attestation/verify.js +15 -0
  46. package/dist/scripts/lib/branch-guard.js +72 -0
  47. package/dist/scripts/lib/config-creators/appraiser.js +9 -0
  48. package/dist/scripts/lib/config-creators/artefact-type.js +9 -0
  49. package/dist/scripts/lib/config-creators/cycle.js +11 -0
  50. package/dist/scripts/lib/config-creators/factory.js +49 -0
  51. package/dist/scripts/lib/config-creators/flow.js +11 -0
  52. package/dist/scripts/lib/config-validators/appraiser.js +49 -0
  53. package/dist/scripts/lib/config-validators/artefact-type.js +38 -0
  54. package/dist/scripts/lib/config-validators/cycle.js +131 -0
  55. package/dist/scripts/lib/config-validators/flow.js +57 -0
  56. package/dist/scripts/lib/config-validators/helpers.js +96 -0
  57. package/dist/scripts/lib/config-validators/law.js +96 -0
  58. package/dist/scripts/lib/config.js +393 -0
  59. package/dist/scripts/lib/failed-flow.js +131 -0
  60. package/dist/scripts/lib/feedback-store.js +249 -0
  61. package/dist/scripts/lib/feedback-transitions.js +105 -0
  62. package/dist/scripts/lib/finalize.js +70 -0
  63. package/dist/scripts/lib/foundational-guards.js +13 -0
  64. package/dist/scripts/lib/git-bridge.js +77 -0
  65. package/dist/scripts/lib/git-finish/work-finish.js +233 -0
  66. package/dist/scripts/lib/git-policy.js +101 -0
  67. package/dist/scripts/lib/guards.js +125 -0
  68. package/dist/scripts/lib/history.js +132 -0
  69. package/dist/scripts/lib/memory/admin/create-edge-type.js +91 -0
  70. package/dist/scripts/lib/memory/admin/create-entity-type.js +43 -0
  71. package/dist/scripts/lib/memory/admin/create-extractor.js +67 -0
  72. package/dist/scripts/lib/memory/admin/drop-edge-type.js +40 -0
  73. package/dist/scripts/lib/memory/admin/drop-entity-type.js +172 -0
  74. package/dist/scripts/lib/memory/admin/dump.js +47 -0
  75. package/dist/scripts/lib/memory/admin/helpers.js +31 -0
  76. package/dist/scripts/lib/memory/admin/init.js +170 -0
  77. package/dist/scripts/lib/memory/admin/live-store.js +76 -0
  78. package/dist/scripts/lib/memory/admin/reembed.js +285 -0
  79. package/dist/scripts/lib/memory/admin/rename-edge-type.js +54 -0
  80. package/dist/scripts/lib/memory/admin/rename-entity-type.js +151 -0
  81. package/dist/scripts/lib/memory/admin/reset.js +24 -0
  82. package/dist/scripts/lib/memory/admin/vacuum.js +9 -0
  83. package/dist/scripts/lib/memory/admin/validate.js +19 -0
  84. package/dist/scripts/lib/memory/config.js +149 -0
  85. package/dist/scripts/lib/memory/cozo.js +136 -0
  86. package/dist/scripts/lib/memory/drift.js +71 -0
  87. package/dist/scripts/lib/memory/embeddings.js +128 -0
  88. package/dist/scripts/lib/memory/frontmatter.js +75 -0
  89. package/dist/scripts/lib/memory/ndjson.js +84 -0
  90. package/dist/scripts/lib/memory/paths.js +25 -0
  91. package/dist/scripts/lib/memory/permissions.js +41 -0
  92. package/dist/scripts/lib/memory/prompt.js +109 -0
  93. package/dist/scripts/lib/memory/query.js +56 -0
  94. package/dist/scripts/lib/memory/reads.js +109 -0
  95. package/dist/scripts/lib/memory/schema.js +64 -0
  96. package/dist/scripts/lib/memory/search.js +73 -0
  97. package/dist/scripts/lib/memory/singleton.js +49 -0
  98. package/dist/scripts/lib/memory/store.js +162 -0
  99. package/dist/scripts/lib/memory/types.js +93 -0
  100. package/dist/scripts/lib/memory/validate.js +58 -0
  101. package/dist/scripts/lib/memory/writes.js +40 -0
  102. package/{scripts → dist/scripts}/lib/pending.js +7 -2
  103. package/dist/scripts/lib/secret.js +59 -0
  104. package/{scripts → dist/scripts}/lib/slug.js +3 -2
  105. package/dist/scripts/lib/snapshot/finish.js +103 -0
  106. package/dist/scripts/lib/snapshot/inspect.js +253 -0
  107. package/dist/scripts/lib/snapshot/render.js +55 -0
  108. package/dist/scripts/lib/sort-fs-check.js +121 -0
  109. package/dist/scripts/lib/sort-routing.js +101 -0
  110. package/{scripts → dist/scripts}/lib/stage-guard.js +12 -6
  111. package/{scripts → dist/scripts}/lib/state.js +4 -0
  112. package/dist/scripts/lib/token.js +57 -0
  113. package/dist/scripts/lib/tracing.js +59 -0
  114. package/dist/scripts/lib/ulid.js +100 -0
  115. package/dist/scripts/lib/validator-jsonl.js +162 -0
  116. package/{scripts → dist/scripts}/lib/workfile.js +38 -20
  117. package/dist/scripts/orchestrate-cycle.js +215 -0
  118. package/dist/scripts/orchestrate-phases.js +314 -0
  119. package/dist/scripts/orchestrate.js +163 -0
  120. package/dist/scripts/sort.js +278 -0
  121. package/{skills → dist/skills}/add-appraiser/SKILL.md +42 -6
  122. package/{skills → dist/skills}/add-artefact-type/SKILL.md +49 -21
  123. package/{skills → dist/skills}/add-cycle/SKILL.md +60 -14
  124. package/dist/skills/add-extractor/SKILL.md +133 -0
  125. package/{skills → dist/skills}/add-flow/SKILL.md +39 -7
  126. package/dist/skills/add-law/SKILL.md +191 -0
  127. package/dist/skills/add-memory-edge-type/SKILL.md +52 -0
  128. package/dist/skills/add-memory-entity-type/SKILL.md +74 -0
  129. package/{skills → dist/skills}/appraise/SKILL.md +62 -13
  130. package/dist/skills/assay/SKILL.md +72 -0
  131. package/dist/skills/change-embedding-model/SKILL.md +58 -0
  132. package/dist/skills/drop-memory-edge-type/SKILL.md +54 -0
  133. package/dist/skills/drop-memory-entity-type/SKILL.md +57 -0
  134. package/dist/skills/dry-run/SKILL.md +116 -0
  135. package/{skills → dist/skills}/flow/SKILL.md +15 -2
  136. package/dist/skills/forge/SKILL.md +121 -0
  137. package/dist/skills/human-appraise/SKILL.md +153 -0
  138. package/{skills → dist/skills}/init-foundry/SKILL.md +23 -4
  139. package/dist/skills/init-memory/SKILL.md +92 -0
  140. package/{skills → dist/skills}/orchestrate/SKILL.md +30 -4
  141. package/dist/skills/quench/SKILL.md +99 -0
  142. package/{skills → dist/skills}/refresh-agents/SKILL.md +1 -1
  143. package/dist/skills/rename-memory-edge-type/SKILL.md +50 -0
  144. package/dist/skills/rename-memory-entity-type/SKILL.md +51 -0
  145. package/dist/skills/reset-memory/SKILL.md +54 -0
  146. package/dist/skills/upgrade-foundry/SKILL.md +192 -0
  147. package/package.json +34 -17
  148. package/.opencode/plugins/foundry.js +0 -761
  149. package/CHANGELOG.md +0 -90
  150. package/docs/concepts.md +0 -59
  151. package/docs/getting-started.md +0 -78
  152. package/docs/work-spec.md +0 -193
  153. package/scripts/lib/artefacts.js +0 -124
  154. package/scripts/lib/config.js +0 -175
  155. package/scripts/lib/feedback-transitions.js +0 -25
  156. package/scripts/lib/feedback.js +0 -440
  157. package/scripts/lib/finalize.js +0 -41
  158. package/scripts/lib/history.js +0 -59
  159. package/scripts/lib/secret.js +0 -23
  160. package/scripts/lib/tags.js +0 -108
  161. package/scripts/lib/token.js +0 -26
  162. package/scripts/orchestrate.js +0 -418
  163. package/scripts/sort.js +0 -370
  164. package/scripts/validate-tags.js +0 -54
  165. package/skills/add-law/SKILL.md +0 -105
  166. package/skills/forge/SKILL.md +0 -88
  167. package/skills/human-appraise/SKILL.md +0 -82
  168. package/skills/quench/SKILL.md +0 -62
  169. package/skills/upgrade-foundry/SKILL.md +0 -216
  170. /package/{skills → dist/skills}/list-agents/SKILL.md +0 -0
@@ -0,0 +1,434 @@
1
+ # Architecture
2
+
3
+ This document provides a high-level overview of how Foundry works. For concept definitions, see [concepts.md](concepts.md). For the complete WORK.md specification, see [work-spec.md](work-spec.md).
4
+
5
+ ---
6
+
7
+ ## Design principles
8
+
9
+ Foundry's guiding rule is **if it can be deterministic, it will be**. Where a guarantee matters — routing, commits, state transitions, write invariants, feedback lifecycle — the logic lives in tested plugin code. This embodies several architectural commitments:
10
+
11
+ ### Everything is markdown
12
+
13
+ Flows, cycles, artefact types, laws, appraiser personalities, and skills are all markdown with YAML frontmatter. They are readable by humans, consumable by LLMs, diff-able in git, and stored directly in the repo.
14
+
15
+ ### Skills are the pipeline, tools are the machinery
16
+
17
+ Composition happens at the skill layer. `flow` reads a definition and invokes `orchestrate`. `orchestrate` calls `foundry_orchestrate` in a loop. The hard guarantees — routing, commits, state transitions, enforcement — live inside the plugin's custom tools and the libraries under `src/scripts/lib/`. Skills handle creative and subjective work; tools handle everything deterministic.
18
+
19
+ ### WORK.md as shared state
20
+
21
+ Inter-stage communication goes through `WORK.md`, `WORK.feedback.yaml`, and `WORK.history.yaml` via the `foundry_workfile_*`, `foundry_artefacts_*`, `foundry_feedback_*`, and `foundry_history_*` tools. This gives a complete audit trail, makes flows resumable after a crash, and lets any stage be re-run independently.
22
+
23
+ ### Cycles own their routing
24
+
25
+ A flow declares starting points; individual cycles declare `targets` and input contracts. The flow skill walks the resulting graph. Cycles stay composable across flows; the flow file stays declarative.
26
+
27
+ ### Feedback as structured state
28
+
29
+ Feedback lives in `WORK.feedback.yaml` with source tracking and tags. It remains human-readable and diff-able, whilst the plugin enforces lifecycle transitions as structured state. Feedback is append-only; the full transition history is preserved alongside each item. Every issue is raised, every decision is recorded, and every resolution is auditable.
30
+
31
+ ### Wont-fix requires approval
32
+
33
+ A forge sub-agent can decline subjective feedback with a justification, and an appraiser approves or rejects that decision on the next iteration. Validation and human feedback cannot be wont-fixed.
34
+
35
+ ### Humans can step in at known points
36
+
37
+ Human-in-the-loop gates are first-class stages. A cycle can declare `human-appraise: true` to run a human quality gate every iteration, or rely on `deadlock-appraise: true` (the default) to pull a human in only when LLM appraisers and forge ping-pong on the same items. Human feedback takes absolute priority and cannot be wont-fixed.
38
+
39
+ ### Multi-model diversity
40
+
41
+ Cycle definitions specify per-stage models; individual appraisers may override. Different models catch different issues; consolidation is a union. One appraiser flagging an issue is enough to raise it.
42
+
43
+ ### Input artefacts are read-only
44
+
45
+ When a cycle reads from another cycle's output, those files cannot be modified. This is enforced at the file-write stage. Downstream cycles cannot corrupt upstream work.
46
+
47
+ ### Glob patterns must not overlap
48
+
49
+ Two artefact types cannot have file patterns that match the same files. Hard-blocked at creation time; the file-ownership rule does not have a meaningful answer otherwise.
50
+
51
+ ### Flow memory is strictly opt-in and per-cycle
52
+
53
+ Memory is a separate, optional subsystem. Without `foundry/memory/`, the system runs with memory features disabled; prompt injection, tools, and vocabulary stay out. With memory initialised, a cycle accesses it by declaring a `memory: { read, write }` block in its frontmatter. A cycle can also declare `assay.extractors` to populate that scoped memory before forge starts. Extractor output is parsed and validated by plugin code before it becomes rows. The live Cozo database is gitignored and rebuildable from committed NDJSON; vocabulary (`entities/<type>.md`, `edges/<name>.md`) and row data (`foundry-memory/relations/*.ndjson`) are the durable source of truth. Destructive operations preview before they mutate.
54
+
55
+ ---
56
+
57
+ ## Enforcement model
58
+
59
+ Where a guarantee matters — routing, commits, state transitions, write invariants, feedback lifecycle — the logic lives in tested plugin code. Skills perform creative and subjective work; tools handle everything deterministic.
60
+
61
+ ### Tool-enforced guarantees
62
+
63
+ The following guarantees live in plugin code and are outside LLM control:
64
+
65
+ - **Stage-locked mutations.** Mutation tools (`foundry_feedback_*`, `foundry_artefacts_*`, `foundry_workfile_*`) require the caller's role to match the active stage. A forge sub-agent cannot add feedback; a quench sub-agent cannot register artefacts. Stage identity is verified through the token lifecycle (below).
66
+ - **Single-use tokens.** Every dispatched stage receives an HMAC-signed token minted by `foundry_orchestrate`. The sub-agent's first call must be `foundry_stage_begin({stage, cycle, token})` to redeem it. Replays, forgery, and cross-stage reuse all fail closed.
67
+ - **Commit-per-stage contract.** The orchestrator refuses to proceed if uncommitted changes to `WORK.md`, `WORK.feedback.yaml`, `WORK.history.yaml`, or `.foundry/` state files are present when a new stage is requested and history is non-empty. Every stage ends with a micro-commit, enforced by `foundry_orchestrate` after calling the stage's internal finalize step.
68
+ - **Write invariants.** The orchestrator's internal finalize step scans the git diff after `foundry_stage_end`. Files outside the stage's allowed patterns (artefact file-patterns for forge, tool-managed WORK files for evaluation stages) cause a hard stop with `{error: 'unexpected_files'}`.
69
+ - **Feedback state machine.** Transitions are source-based. Only legal state changes are accepted: `resolved` is terminal; quench cannot approve/reject a `wont-fix`; validation cannot be wont-fixed. See [work-spec.md](work-spec.md) for the full state machine.
70
+ - **Artefact-type glob uniqueness.** `add-artefact-type` refuses to create a type whose file patterns overlap with an existing type. The write-invariant enforcer relies on unambiguous file ownership.
71
+ - **Extractor validation.** Assay runs project-authored extractor commands, but plugin code owns JSONL parsing, vocabulary checks, per-extractor write scopes, per-cycle write scopes, and memory upserts. Invalid output or permission violations mark the workfile failed; they do not become artefact feedback.
72
+
73
+ ### Deterministic orchestration
74
+
75
+ The `orchestrate` skill is a thin driver around `foundry_orchestrate`. Its entire loop is:
76
+
77
+ ```text
78
+ call foundry_orchestrate({lastResult})
79
+ switch on action:
80
+ dispatch → dispatch the requested subagent → report back
81
+ human_appraise → run human-appraise inline → report back
82
+ done / blocked / violation → terminate the loop
83
+ ```
84
+
85
+ `foundry_orchestrate` owns:
86
+
87
+ - **Sort routing** — which stage runs next, based on feedback state, iteration depth, and deadlock detection.
88
+ - **Commit enforcement** — every stage ends with a micro-commit; the orchestrator refuses to proceed if dirty files are present.
89
+ - **History** — appends a record of every stage execution to `WORK.history.yaml`.
90
+ - **Finalize** — scans the git diff, registers matching artefacts, rejects unexpected writes.
91
+ - **Violation handling** — returns terminal envelopes (`unexpected_files`, `blocked`, `done`) that the `orchestrate` skill translates into user-facing outcomes.
92
+
93
+ Because the protocol lives in a plugin tool, the LLM cannot skip steps, reorder them, or silently drop a commit.
94
+
95
+ ---
96
+
97
+ ## Token lifecycle
98
+
99
+ Foundry uses HMAC-SHA256 tokens to gate stage execution. Tokens are single-use, cryptographically signed, and carry metadata about the stage they authorise.
100
+
101
+ ### Flow
102
+
103
+ 1. **Mint.** `foundry_orchestrate` creates a token when dispatching a stage. The token includes:
104
+ - `stage` — the stage identifier (e.g. `forge:write-haiku`)
105
+ - `cycle` — the cycle ID
106
+ - `nonce` — a random ULID
107
+ - `exp` — expiry timestamp (default: 1 hour from mint time)
108
+ - `hmac` — HMAC-SHA256 signature over `{stage, cycle, nonce, exp}` using the worktree's secret key
109
+ 2. **Dispatch.** The token is embedded in the sub-agent's dispatch prompt.
110
+ 3. **Redeem.** The sub-agent's **first** call is `foundry_stage_begin({stage, cycle, token})`. The tool verifies:
111
+ - The HMAC signature is valid.
112
+ - The nonce has not been used (checked against the pending store).
113
+ - The token has not expired.
114
+ - The claimed `stage` and `cycle` match the token's signed payload.
115
+ 4. **Activate.** On success, the stage is recorded in `.foundry/active-stage.json`. Mutation tools (`foundry_feedback_*`, `foundry_artefacts_*`, etc.) now check that their role matches the active stage.
116
+ 5. **End.** The sub-agent's **last** call is `foundry_stage_end({summary})`. This removes `.foundry/active-stage.json` and writes `.foundry/last-stage.json` for the orchestrator's finalize step.
117
+ 6. **Finalize.** The orchestrator's internal finalize step runs after `stage_end`, scanning the git diff and committing the stage.
118
+
119
+ ### Secret key
120
+
121
+ The HMAC key lives in `.foundry/.secret` (mode 0600, gitignored, one per worktree). It is generated on first boot and never transmitted. Because it is worktree-local, tokens cannot be replayed across clones or forks.
122
+
123
+ ### Pending store
124
+
125
+ The pending store (`src/scripts/lib/pending.js`) tracks consumed nonces to prevent replay attacks. It is an in-memory `Map` with automatic garbage collection: expired entries are swept periodically via the `gc()` method.
126
+
127
+ ---
128
+
129
+ ## Branch guards
130
+
131
+ Foundry partitions mutation across three branch namespaces. The plugin enforces this split at tool-call time, preventing config changes on work branches and flow data changes on config branches.
132
+
133
+ ### Namespace rules
134
+
135
+ | Namespace | Pattern | Owns | Created from | Finished by |
136
+ |-----------|---------|------|--------------|-------------|
137
+ | **config** | `config/<description>` | `foundry/` (schema and config) | `main` via `foundry_git_branch({ kind: "config", description })` | PR or direct merge to `main` |
138
+ | **work** | `work/<flowId>-<description>` | `WORK.md`, `WORK.feedback.yaml`, `WORK.history.yaml`, `foundry-memory/` (row data) | `main` via `foundry_git_branch({ kind: "work", flowId, description })` | `foundry_git_finish` (squash-merges to base, preserves forensic archive) |
139
+ | **dry-run** | `dry-run/<parentConfig>/<flowId>-<description>` | Same as `work/*` | `config/*` via `foundry_git_branch({ kind: "dry-run", flowId, description })` | `foundry_git_finish` (captures snapshot, force-deletes branch) |
140
+
141
+ ### Guard implementation
142
+
143
+ Every mutating tool composes one of three guards before its handler runs:
144
+
145
+ - **`requireOnConfigBranch`** — accepts only `config/<description>`. Used by schema-mutation tools (`foundry_config_create_*`, `foundry_memory_create_*`, `foundry_extractor_create`, memory admin tools). Dry-run is rejected by design — schema must change on a real config branch.
146
+ - **`requireOnFlowBranch`** — accepts `work/<flow>-<desc>` or `dry-run/<x>/<y>`. Used by flow-data tools (`foundry_orchestrate`, workfile/feedback/artefact-status/assay/appraisers families, `foundry_memory_put`/`_relate`/`_unrelate`).
147
+ - **`requireOnConfigOrFlowBranch`** — accepts any of the three namespaces. Used by read-only diagnostic tools.
148
+
149
+ Implementation: `src/scripts/lib/branch-guard.js`.
150
+
151
+ ### Forensic branches and snapshots
152
+
153
+ - **Work branches** are preserved as `archive/work/<flowId>-<description>-<hash>` when `foundry_git_finish` completes. The full stage micro-commit history, `WORK.*` files, and all intermediate artefact states remain intact. The signed squash commit on the base branch references the archive branch tip SHA in its attestation block.
154
+ - **Dry-run branches** are force-deleted after `foundry_git_finish` captures a snapshot to `.snapshots/<runId>/` on the parent `config/*` working tree. Each snapshot includes `README.md` (metadata), `work/WORK*` (workfile triple), `diff.patch` (full diff), and `trace.jsonl` (tool-call trace).
155
+
156
+ ---
157
+
158
+ ## Memory layout
159
+
160
+ Foundry's optional flow memory subsystem uses a two-tree split: configuration lives under `foundry/memory/` (committed alongside the rest of the foundry config), and row data lives under `foundry-memory/relations/` (a top-level sibling of `foundry/`) so it can be tracked or replaced independently. Assay extractors populate this row-data tree during an active assay stage; ordinary memory writes also end up in the same committed relations files.
161
+
162
+ ### Directory structure
163
+
164
+ ```
165
+ foundry/memory/ # config (committed)
166
+ ├── config.md # frontmatter: enabled, validation, embeddings
167
+ ├── schema.json # canonical, deterministic, derived from entity/edge files
168
+ ├── entities/<type>.md # prose brief per entity type (LLM-facing)
169
+ ├── edges/<name>.md # frontmatter (sources/targets) + prose brief
170
+ ├── extractors/<name>.md # extractor definitions (assay stage)
171
+ ├── memory.db # live Cozo 0.7 store (gitignored)
172
+ ├── memory.db-wal # WAL (gitignored)
173
+ └── memory.db-shm # shared memory (gitignored)
174
+
175
+ foundry-memory/ # row data (committed; top-level sibling of foundry/)
176
+ └── relations/<type>.ndjson # one line per row, source of truth for memory contents
177
+ ```
178
+
179
+ ### Data model
180
+
181
+ - **Entity** — `{ type, name, value }`, where `value` (≤ 4 KB) is free text describing intrinsic characteristics only. Relationships belong in edges.
182
+ - **Edge** — directed row `{ from_type, from_name, edge_type, to_type, to_name }`.
183
+ - **Entity type** — declared in `foundry/memory/entities/<type>.md`. The prose body is injected into the prompt of every cycle that reads or writes this type.
184
+ - **Edge type** — declared in `foundry/memory/edges/<name>.md`. Frontmatter declares `sources` and `targets` (list of entity types or `any`).
185
+
186
+ ### Schema and source of truth
187
+
188
+ - **`schema.json`** is a fully key-sorted derivation of the entity/edge files plus the active embedding configuration. It is regenerated by admin tools and serves as a diff-friendly artefact of the vocabulary, not a source of truth.
189
+ - **`relations/<type>.ndjson`** files are the durable source of truth for memory contents. The live `memory.db` is rebuildable from these files.
190
+ - **Self-healing reopen.** On store open, orphan relations left by drops/renames are reconciled (`::relations` filtered to `^(ent|edge)_[^:]+$`, HNSW indices dropped before `::remove`).
191
+
192
+ ### Per-cycle permissions
193
+
194
+ Cycles opt in to memory via frontmatter:
195
+
196
+ ```yaml
197
+ memory:
198
+ read: [class, method] # types this cycle can read
199
+ write: [method] # types this cycle can upsert
200
+ ```
201
+
202
+ - `read` types expose `foundry_memory_{get,list,neighbours,query,search}` (search requires embeddings).
203
+ - `write` types additionally expose `foundry_memory_{put,relate,unrelate}`.
204
+ - `read` and `write` are independent sets. A type listed in `write` only is writable but invisible to read tools. List a type in both sets for full access.
205
+ - Edge permissions are derived: an edge is readable if either endpoint type is in `read` or `write`, writable if either endpoint is in `write`.
206
+ - `foundry_memory_query` rejects Datalog that references `ent_*` / `edge_*` relations outside the read set.
207
+
208
+ ### Embeddings
209
+
210
+ Optional. When `embeddings.enabled: true` in `config.md`, entity values are embedded against an OpenAI-compatible endpoint (default: local Ollama `nomic-embed-text`, 768 dims) and stored in a typed `<F32; N>?` column with an HNSW index. `foundry_memory_search` exposes semantic nearest-neighbour search. `change-embedding-model` re-embeds all entities atomically (nothing is written on failure).
211
+
212
+ ### Operational guarantees
213
+
214
+ - **Deterministic scaffolding.** `foundry_memory_init` creates directories, config, schema, and gitignore entries in one call.
215
+ - **Preview-then-confirm for destructive ops.** Drop tools called without `confirm: true` return an impact report (row counts, affected edges).
216
+ - **Prompt-injection guard.** If memory is misconfigured or drifted, dispatch still succeeds with the base prompt (memory context is omitted, not fatal).
217
+
218
+ ---
219
+
220
+ ## Stage execution model
221
+
222
+ Every stage runs inside a token-gated lifecycle. The sub-agent must call `foundry_stage_begin`, do its work, then call `foundry_stage_end`. The orchestrator's finalize step then scans the disk, registers artefacts, and commits the stage.
223
+
224
+ ### Per-stage write rules
225
+
226
+ | Stage | May write |
227
+ |-------|-----------|
228
+ | `forge` | Files matching the output artefact type's `file-patterns`, plus `WORK.md` / `WORK.feedback.yaml` / `WORK.history.yaml` via tools |
229
+ | `quench` | `WORK.feedback.yaml` via feedback tools; `WORK.history.yaml` via stage finalization |
230
+ | `appraise` | `WORK.feedback.yaml` via feedback tools; `WORK.history.yaml` via stage finalization |
231
+ | `human-appraise` | `WORK.feedback.yaml` via feedback tools; `WORK.history.yaml` via stage finalization |
232
+ | `assay` | Flow memory via `foundry_assay_run` (not direct `foundry_memory_put`); marks the workfile failed on abort (no feedback writes) |
233
+
234
+ Input artefacts (files matching an input type's `file-patterns`) are read-only. Files outside any artefact type's patterns are read-only. Violations hard-stop the cycle with `{error: 'unexpected_files'}`.
235
+
236
+ ### Failed flow state
237
+
238
+ When an unrecoverable error occurs (e.g. assay extractor abort, invalid JSONL, or memory-sync failure), the orchestrator marks `WORK.md` frontmatter with `status: failed` and a `reason`. The flow is then locked:
239
+
240
+ - **Blocked tools.** All mutation tools refuse to run and return an error referencing the failure reason:
241
+ - **Lifecycle:** `foundry_stage_begin`, `foundry_orchestrate`, `foundry_workfile_create`, `foundry_artefacts_set_status`
242
+ - **Stage work:** `foundry_assay_run`, `foundry_validate_run`
243
+ - **Feedback writes:** `foundry_feedback_add`, `foundry_feedback_action`, `foundry_feedback_wontfix`, `foundry_feedback_resolve` (`foundry_feedback_list` remains callable)
244
+ - **Appraiser selection:** `foundry_appraisers_select`
245
+ - **Memory writes:** `foundry_memory_put`, `foundry_memory_relate`, `foundry_memory_unrelate`
246
+ - **Memory admin:** `foundry_memory_init`, `foundry_memory_reset`, `foundry_memory_vacuum`, `foundry_memory_change_embedding_model`, `foundry_memory_create_entity_type`, `foundry_memory_create_edge_type`, `foundry_memory_rename_entity_type`, `foundry_memory_rename_edge_type`, `foundry_memory_drop_entity_type`, `foundry_memory_drop_edge_type`, `foundry_extractor_create` (read-only `foundry_memory_validate` and `foundry_memory_dump` remain callable)
247
+ - **Config schema mutation:** `foundry_config_create_artefact_type`, `foundry_config_add_law`, `foundry_config_edit_law`, `foundry_config_create_appraiser`, `foundry_config_create_flow`, `foundry_config_create_cycle` (read-only `foundry_config_validate_*` and `foundry_config_read_law` remain callable)
248
+ - **Escape hatches.** `foundry_workfile_get` (to read the reason) and `foundry_workfile_delete({confirm: true})` (to abandon the cycle) remain callable. `foundry_git_finish` sits outside the failed-flow guard, allowing the user to exit the failed branch.
249
+ - **Recovery.** Read the reason via `foundry_workfile_get`, fix the root cause, then either call `foundry_stage_retry()` to clear the failed state and re-run the blocked stage, or abandon the cycle with `foundry_workfile_delete({confirm: true})` and start again.
250
+
251
+ All pipeline skills (`orchestrate`, `flow`, stage skills) check for this state at the top of their procedure and hand control back to the user immediately if found.
252
+
253
+ ---
254
+
255
+ ## Routing and feedback lifecycle
256
+
257
+ ### Sort routing
258
+
259
+ `src/scripts/sort.js` (exported as `runSort`) owns the routing engine. It reads `WORK.md`, `WORK.feedback.yaml`, and `WORK.history.yaml`, then decides which stage runs next based on:
260
+
261
+ - **Unresolved feedback.** If `quench` or `appraise` feedback exists in a non-terminal state (`open`, `actioned`, `wont-fix`), the next stage is usually `forge` or the originating evaluation stage.
262
+ - **Deadlock detection.** If the same feedback items ping-pong between forge and appraise for `deadlock-iterations` (default 5) iterations, sort marks them `deadlocked` and inserts a `human-appraise` stage (if `deadlock-appraise: true`, the default).
263
+ - **Iteration limits.** If `max-iterations` is exceeded, the cycle is marked `blocked` and control returns to the user.
264
+ - **Clean state.** If all feedback is resolved and no new validation or appraisal failures exist, the cycle is `done`.
265
+
266
+ ### Feedback state machine
267
+
268
+ Feedback items live in `WORK.feedback.yaml` with a full transition history. Each item has:
269
+
270
+ - `id` — a ULID.
271
+ - `source` — the stage that created it (e.g. `quench:check-syllables`, `appraise:pedantic`, `human-appraise:hitl`).
272
+ - `state` — current state (`open`, `actioned`, `wont-fix`, `resolved`, `rejected`, `deadlocked`).
273
+ - `history` — append-only log of state transitions with timestamps and metadata.
274
+
275
+ Transitions are **source-based**:
276
+
277
+ | Source stage | Forge can `wont-fix`? | Resolved by |
278
+ |--------------|------------------------|-------------|
279
+ | `quench` (CLI validation) | No — must `actioned` | the originating `quench` stage, or `human-appraise` override |
280
+ | `appraise` (subjective law) | Yes (with reason) | the originating `appraise` stage, or `human-appraise` override |
281
+ | `human-appraise` (user instruction) | No — must `actioned` | the originating `human-appraise` stage |
282
+
283
+ Implementation: `src/scripts/lib/feedback-transitions.js` and `src/scripts/lib/feedback-store.js`. See [work-spec.md](work-spec.md) for the full state machine table.
284
+
285
+ ---
286
+
287
+ ## Multi-model routing
288
+
289
+ Different stages can run on different models for cognitive diversity. Cycle definitions specify per-stage models; individual appraisers may override.
290
+
291
+ ### Configuration
292
+
293
+ - **Cycle-level.** Declare a `models` map in the cycle frontmatter:
294
+ ```yaml
295
+ models:
296
+ forge: anthropic/claude-opus-4.7
297
+ appraise: openai/gpt-5
298
+ ```
299
+ - **Appraiser-level.** Individual appraisers can declare a `model` field in their personality definition; this overrides the cycle-level appraise model on a per-appraiser basis.
300
+
301
+ ### Agent files
302
+
303
+ `refresh-agents` generates a `foundry-<slug>.md` agent file in `.opencode/agents/` for every model available in the session, where `<slug>` is the model ID with both `/` and `.` replaced by `-` (e.g. `anthropic-claude-opus-4-7.md`).
304
+
305
+ ### Dispatch behaviour
306
+
307
+ - **Non-appraise stages** (forge, quench, assay): if the cycle declares `models.<stage>`, the orchestrator dispatches to `foundry-<slug>` and hard-fails if `.opencode/agents/foundry-<slug>.md` is missing. If `models.<stage>` is not set, the stage is dispatched with the `general` subagent (session default).
308
+ - **Appraise stage**: each appraiser is dispatched independently by the `appraise` skill. If an appraiser has its own `model`, the skill dispatches to `foundry-<slug>` and hard-fails if that agent file is missing; otherwise the appraiser runs under the `general` subagent.
309
+
310
+ Implementation: `src/plugin/tools/helpers.js` (`buildCyclePromptExtras`) and `src/skills/appraise/SKILL.md`.
311
+
312
+ ---
313
+
314
+ ## Project layout
315
+
316
+ ### Package (this repo)
317
+
318
+ ```
319
+ @really-knows-ai/foundry
320
+ ├── src/
321
+ │ ├── plugin/
322
+ │ │ ├── foundry.js # plugin entrypoint: skills and custom tools
323
+ │ │ └── tools/ # tool registration + plugin helpers
324
+ │ ├── skills/ # shipped skill definitions
325
+ │ │ ├── flow/ # pipeline
326
+ │ │ ├── orchestrate/
327
+ │ │ ├── forge/
328
+ │ │ ├── quench/
329
+ │ │ ├── appraise/
330
+ │ │ ├── human-appraise/
331
+ │ │ ├── init-foundry/ # authoring
332
+ │ │ ├── add-artefact-type/
333
+ │ │ ├── add-law/
334
+ │ │ ├── add-appraiser/
335
+ │ │ ├── add-cycle/
336
+ │ │ ├── add-flow/
337
+ │ │ ├── add-extractor/
338
+ │ │ ├── list-agents/ # utility
339
+ │ │ ├── refresh-agents/
340
+ │ │ ├── upgrade-foundry/
341
+ │ │ ├── init-memory/ # memory
342
+ │ │ ├── add-memory-entity-type/
343
+ │ │ ├── add-memory-edge-type/
344
+ │ │ ├── rename-memory-entity-type/
345
+ │ │ ├── rename-memory-edge-type/
346
+ │ │ ├── drop-memory-entity-type/
347
+ │ │ ├── drop-memory-edge-type/
348
+ │ │ ├── reset-memory/
349
+ │ │ └── change-embedding-model/
350
+ │ └── scripts/
351
+ │ ├── lib/ # shared libraries (injectable I/O)
352
+ │ │ ├── workfile.js # WORK.md frontmatter
353
+ │ │ ├── artefacts.js # artefact table operations
354
+ │ │ ├── history.js # WORK.history.yaml operations
355
+ │ │ ├── feedback-store.js
356
+ │ │ ├── feedback-transitions.js
357
+ │ │ ├── finalize.js # stage finalization
358
+ │ │ ├── stage-guard.js
359
+ │ │ ├── branch-guard.js
360
+ │ │ ├── foundational-guards.js
361
+ │ │ ├── guards.js
362
+ │ │ ├── token.js
363
+ │ │ ├── secret.js
364
+ │ │ ├── pending.js
365
+ │ │ ├── state.js
366
+ │ │ ├── config.js # foundry/ config readers
367
+ │ │ ├── slug.js
368
+ │ │ ├── ulid.js
369
+ │ │ ├── tracing.js
370
+ │ │ ├── failed-flow.js
371
+ │ │ ├── git-bridge.js
372
+ │ │ ├── git-policy.js
373
+ │ │ ├── assay/
374
+ │ │ ├── config-creators/
375
+ │ │ ├── config-validators/
376
+ │ │ ├── snapshot/
377
+ │ │ └── memory/ # flow memory (Cozo 0.7)
378
+ │ ├── orchestrate.js # orchestration loop (exports runOrchestrate)
379
+ │ └── sort.js # routing engine (exports runSort)
380
+ ├── scripts/
381
+ │ └── build.js # builds src/ into dist/
382
+ ├── dist/
383
+ │ ├── .opencode/plugins/ # packaged plugin output
384
+ │ ├── skills/ # packaged skill output
385
+ │ └── scripts/ # packaged runtime libraries
386
+ ├── tests/ # node:test suite
387
+ ├── docs/ # concepts, getting-started, work-spec
388
+ ├── CHANGELOG.md
389
+ └── README.md
390
+ ```
391
+
392
+ ### User project (after `init-foundry`)
393
+
394
+ ```
395
+ your-project/
396
+ ├── foundry/
397
+ │ ├── flows/ # flow definitions
398
+ │ ├── cycles/ # cycle definitions
399
+ │ ├── artefacts/ # artefact type definitions
400
+ │ │ └── <type>/
401
+ │ │ ├── definition.md
402
+ │ │ ├── laws.md # optional
403
+ │ │ └── validation.md # optional
404
+ │ ├── laws/ # global laws
405
+ │ ├── appraisers/ # appraiser personalities
406
+ │ └── memory/ # optional flow memory config (init-memory)
407
+ │ ├── config.md
408
+ │ ├── schema.json
409
+ │ ├── entities/<type>.md
410
+ │ ├── edges/<name>.md
411
+ │ ├── extractors/<name>.md
412
+ │ └── memory.db* # gitignored
413
+ ├── foundry-memory/ # flow memory row data (top-level sibling)
414
+ │ └── relations/<type>.ndjson
415
+ ├── .foundry/ # runtime state (gitignored)
416
+ │ └── .secret # per-worktree HMAC key (mode 0600)
417
+ ├── .opencode/
418
+ │ └── agents/
419
+ │ └── foundry-*.md # generated by refresh-agents
420
+ ├── opencode.json
421
+ └── ...
422
+ ```
423
+
424
+ During a flow, a work branch also contains `WORK.md`, `WORK.feedback.yaml`, and `WORK.history.yaml` at the repo root. These are ephemeral work state; they are deleted before the squash-merge completes.
425
+
426
+ ---
427
+
428
+ ## Further reading
429
+
430
+ - [concepts.md](concepts.md) — every concept defined concisely.
431
+ - [work-spec.md](work-spec.md) — the full WORK.md + WORK.feedback.yaml + WORK.history.yaml spec.
432
+ - [memory-maintenance.md](memory-maintenance.md) — contributor notes on Cozo 0.7 and memory session lifecycle.
433
+ - [tools.md](tools.md) — complete reference for custom tools.
434
+ - [getting-started.md](getting-started.md) — end-to-end walkthrough.