mishkan-harness 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (186) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +205 -0
  3. package/bin/mishkan.js +221 -0
  4. package/docs/design/MISHKAN_agent_aliases.md +140 -0
  5. package/docs/design/MISHKAN_decisions.md +172 -0
  6. package/docs/design/MISHKAN_harness_design.md +820 -0
  7. package/docs/design/MISHKAN_ontology.md +87 -0
  8. package/docs/design/MISHKAN_token_optimisation.md +181 -0
  9. package/docs/engineer/README.md +37 -0
  10. package/docs/engineer/profile.example.md +79 -0
  11. package/docs/usage/01-installation.md +178 -0
  12. package/docs/usage/02-project-init.md +151 -0
  13. package/docs/usage/03-orchestration.md +218 -0
  14. package/docs/usage/04-memory-layer.md +201 -0
  15. package/docs/usage/05-selective-ingest.md +177 -0
  16. package/docs/usage/06-llm-providers.md +195 -0
  17. package/docs/usage/07-troubleshooting.md +316 -0
  18. package/docs/usage/08-glossary.md +154 -0
  19. package/docs/usage/09-workflows.md +123 -0
  20. package/docs/usage/README.md +77 -0
  21. package/package.json +43 -0
  22. package/payload/install/settings.hooks.json +47 -0
  23. package/payload/mishkan/AGENT_SPEC.md +154 -0
  24. package/payload/mishkan/agents/ahikam.md +58 -0
  25. package/payload/mishkan/agents/aholiab.md +68 -0
  26. package/payload/mishkan/agents/asaph.md +73 -0
  27. package/payload/mishkan/agents/baruch.md +88 -0
  28. package/payload/mishkan/agents/benaiah.md +76 -0
  29. package/payload/mishkan/agents/bezalel.md +83 -0
  30. package/payload/mishkan/agents/caleb.md +74 -0
  31. package/payload/mishkan/agents/deborah.md +63 -0
  32. package/payload/mishkan/agents/elasah.md +58 -0
  33. package/payload/mishkan/agents/eliashib.md +68 -0
  34. package/payload/mishkan/agents/ezra.md +69 -0
  35. package/payload/mishkan/agents/hanun.md +64 -0
  36. package/payload/mishkan/agents/hiram.md +68 -0
  37. package/payload/mishkan/agents/hizkiah.md +76 -0
  38. package/payload/mishkan/agents/huldah.md +59 -0
  39. package/payload/mishkan/agents/huram.md +66 -0
  40. package/payload/mishkan/agents/hushai.md +59 -0
  41. package/payload/mishkan/agents/igal.md +58 -0
  42. package/payload/mishkan/agents/ira.md +86 -0
  43. package/payload/mishkan/agents/jahaziel.md +71 -0
  44. package/payload/mishkan/agents/jakin.md +66 -0
  45. package/payload/mishkan/agents/jehonathan.md +62 -0
  46. package/payload/mishkan/agents/jehoshaphat.md +68 -0
  47. package/payload/mishkan/agents/joab.md +71 -0
  48. package/payload/mishkan/agents/joah.md +62 -0
  49. package/payload/mishkan/agents/maaseiah.md +61 -0
  50. package/payload/mishkan/agents/meremoth.md +65 -0
  51. package/payload/mishkan/agents/meshullam.md +67 -0
  52. package/payload/mishkan/agents/nathan.md +70 -0
  53. package/payload/mishkan/agents/nehemiah.md +93 -0
  54. package/payload/mishkan/agents/obed.md +60 -0
  55. package/payload/mishkan/agents/oholiab.md +67 -0
  56. package/payload/mishkan/agents/palal.md +63 -0
  57. package/payload/mishkan/agents/phinehas.md +73 -0
  58. package/payload/mishkan/agents/rehum.md +60 -0
  59. package/payload/mishkan/agents/salma.md +69 -0
  60. package/payload/mishkan/agents/seraiah.md +73 -0
  61. package/payload/mishkan/agents/shallum.md +66 -0
  62. package/payload/mishkan/agents/shaphan.md +64 -0
  63. package/payload/mishkan/agents/shemaiah.md +67 -0
  64. package/payload/mishkan/agents/shevna.md +58 -0
  65. package/payload/mishkan/agents/uriah.md +70 -0
  66. package/payload/mishkan/agents/zaccur.md +58 -0
  67. package/payload/mishkan/agents/zadok.md +67 -0
  68. package/payload/mishkan/agents/zerubbabel.md +69 -0
  69. package/payload/mishkan/cognee/.env.curated.example +61 -0
  70. package/payload/mishkan/cognee/.env.example +165 -0
  71. package/payload/mishkan/cognee/Dockerfile +50 -0
  72. package/payload/mishkan/cognee/README.md +129 -0
  73. package/payload/mishkan/cognee/docker-compose.curated-ui.yml +61 -0
  74. package/payload/mishkan/cognee/docker-compose.curated.yml +85 -0
  75. package/payload/mishkan/cognee/docker-compose.hardening.yml +16 -0
  76. package/payload/mishkan/cognee/docker-compose.selfhosted.yml +114 -0
  77. package/payload/mishkan/cognee/docker-compose.ui.yml +70 -0
  78. package/payload/mishkan/cognee/docker-compose.yml +71 -0
  79. package/payload/mishkan/cognee/ingest-curated.py +92 -0
  80. package/payload/mishkan/commands/dep-audit.md +24 -0
  81. package/payload/mishkan/commands/mishkan-init.md +25 -0
  82. package/payload/mishkan/commands/mishkan-resume.md +21 -0
  83. package/payload/mishkan/commands/promote.md +19 -0
  84. package/payload/mishkan/commands/sefer-pull.md +19 -0
  85. package/payload/mishkan/commands/sprint-close.md +21 -0
  86. package/payload/mishkan/config/curated-library.yaml +113 -0
  87. package/payload/mishkan/config/improvement-queries.md +29 -0
  88. package/payload/mishkan/config/model-routing.yaml +87 -0
  89. package/payload/mishkan/config/projects.yaml +38 -0
  90. package/payload/mishkan/evals/baruch/README.md +93 -0
  91. package/payload/mishkan/evals/baruch/fixtures/invalid/bad-outcome-enum.json +15 -0
  92. package/payload/mishkan/evals/baruch/fixtures/invalid/bad-sprint-pattern.json +15 -0
  93. package/payload/mishkan/evals/baruch/fixtures/invalid/bad-trigger-enum.json +15 -0
  94. package/payload/mishkan/evals/baruch/fixtures/invalid/malformed-json.json +7 -0
  95. package/payload/mishkan/evals/baruch/fixtures/invalid/missing-required-field.json +14 -0
  96. package/payload/mishkan/evals/baruch/fixtures/valid/blocked-vendor.json +15 -0
  97. package/payload/mishkan/evals/baruch/fixtures/valid/curated-shortcircuit.json +15 -0
  98. package/payload/mishkan/evals/baruch/fixtures/valid/partial-no-write.json +14 -0
  99. package/payload/mishkan/evals/baruch/fixtures/valid/resolved-cross-harness.json +15 -0
  100. package/payload/mishkan/evals/baruch/golden_case/expected.yaml +35 -0
  101. package/payload/mishkan/evals/baruch/golden_case/input.yaml +47 -0
  102. package/payload/mishkan/evals/baruch/golden_case/produced.json +15 -0
  103. package/payload/mishkan/evals/baruch/run.sh +129 -0
  104. package/payload/mishkan/hooks/model-route.py +96 -0
  105. package/payload/mishkan/hooks/post-tool-observe.sh +45 -0
  106. package/payload/mishkan/hooks/pre-tool-security.sh +150 -0
  107. package/payload/mishkan/hooks/session-start.sh +20 -0
  108. package/payload/mishkan/hooks/stop-reporter.sh +29 -0
  109. package/payload/mishkan/ontology.md +87 -0
  110. package/payload/mishkan/rules/backend/yasad.md +23 -0
  111. package/payload/mishkan/rules/common/dependencies.md +53 -0
  112. package/payload/mishkan/rules/common/quality.md +16 -0
  113. package/payload/mishkan/rules/common/security.md +20 -0
  114. package/payload/mishkan/rules/documentation/sefer.md +19 -0
  115. package/payload/mishkan/rules/frontend/panim.md +21 -0
  116. package/payload/mishkan/rules/infrastructure/migdal.md +22 -0
  117. package/payload/mishkan/scripts/dependency-audit.sh +171 -0
  118. package/payload/mishkan/scripts/ensure-curated-box.sh +66 -0
  119. package/payload/mishkan/scripts/mishkan-ingest.sh +92 -0
  120. package/payload/mishkan/scripts/observability-aggregate.sh +57 -0
  121. package/payload/mishkan/scripts/seed-curated-library.sh +62 -0
  122. package/payload/mishkan/scripts/sync-profile.sh +65 -0
  123. package/payload/mishkan/scripts/validate-research-log.sh +108 -0
  124. package/payload/mishkan/skills/asaph-a11y-seo-craft/SKILL.md +289 -0
  125. package/payload/mishkan/skills/baruch-research-reporting-craft/SKILL.md +460 -0
  126. package/payload/mishkan/skills/benaiah-devsecops-craft/SKILL.md +329 -0
  127. package/payload/mishkan/skills/bezalel-cto-craft/SKILL.md +391 -0
  128. package/payload/mishkan/skills/caleb-web-research-craft/SKILL.md +306 -0
  129. package/payload/mishkan/skills/cognee-promote/SKILL.md +40 -0
  130. package/payload/mishkan/skills/cognee-quickstart/SKILL.md +66 -0
  131. package/payload/mishkan/skills/context-compress/SKILL.md +36 -0
  132. package/payload/mishkan/skills/deborah-ux-craft/SKILL.md +295 -0
  133. package/payload/mishkan/skills/dependency-audit/SKILL.md +59 -0
  134. package/payload/mishkan/skills/dependency-vetting/SKILL.md +59 -0
  135. package/payload/mishkan/skills/documentation-craft/SKILL.md +468 -0
  136. package/payload/mishkan/skills/ezra-research-formulation-craft/SKILL.md +319 -0
  137. package/payload/mishkan/skills/hanun-observability-craft/SKILL.md +312 -0
  138. package/payload/mishkan/skills/hiram-ui-craft/SKILL.md +334 -0
  139. package/payload/mishkan/skills/hizkiah-implementation-craft/SKILL.md +701 -0
  140. package/payload/mishkan/skills/hushai-security-advisor-craft/SKILL.md +282 -0
  141. package/payload/mishkan/skills/ira-code-security-craft/SKILL.md +553 -0
  142. package/payload/mishkan/skills/jakin-intent-clarification-craft/SKILL.md +299 -0
  143. package/payload/mishkan/skills/jehonathan-publication-craft/SKILL.md +262 -0
  144. package/payload/mishkan/skills/joab-app-security-craft/SKILL.md +266 -0
  145. package/payload/mishkan/skills/meremoth-devops-craft/SKILL.md +298 -0
  146. package/payload/mishkan/skills/meshullam-infra-design-craft/SKILL.md +302 -0
  147. package/payload/mishkan/skills/mishkan-ingest/SKILL.md +65 -0
  148. package/payload/mishkan/skills/mishkan-init/SKILL.md +65 -0
  149. package/payload/mishkan/skills/nathan-architecture-craft/SKILL.md +547 -0
  150. package/payload/mishkan/skills/nehemiah-pm-craft/SKILL.md +484 -0
  151. package/payload/mishkan/skills/obed-asset-pipeline-craft/SKILL.md +286 -0
  152. package/payload/mishkan/skills/oholiab-design-system-craft/SKILL.md +334 -0
  153. package/payload/mishkan/skills/palal-systems-craft/SKILL.md +281 -0
  154. package/payload/mishkan/skills/qa-evaluation-craft/SKILL.md +406 -0
  155. package/payload/mishkan/skills/rehum-sre-advisor-craft/SKILL.md +228 -0
  156. package/payload/mishkan/skills/reporter-discipline-craft/SKILL.md +351 -0
  157. package/payload/mishkan/skills/research-pipeline/SKILL.md +55 -0
  158. package/payload/mishkan/skills/salma-frontend-implementation-craft/SKILL.md +369 -0
  159. package/payload/mishkan/skills/sefer-pull/SKILL.md +37 -0
  160. package/payload/mishkan/skills/shallum-database-craft/SKILL.md +347 -0
  161. package/payload/mishkan/skills/shaphan-summarisation-craft/SKILL.md +271 -0
  162. package/payload/mishkan/skills/shemaiah-evaluation-craft/SKILL.md +342 -0
  163. package/payload/mishkan/skills/sprint-report/SKILL.md +28 -0
  164. package/payload/mishkan/skills/team-lead-craft/SKILL.md +457 -0
  165. package/payload/mishkan/skills/zadok-contract-craft/SKILL.md +520 -0
  166. package/payload/mishkan/templates/case-node.schema.json +22 -0
  167. package/payload/mishkan/templates/mcp.json +22 -0
  168. package/payload/mishkan/templates/observability-log.schema.json +24 -0
  169. package/payload/mishkan/templates/project-CLAUDE.md +47 -0
  170. package/payload/mishkan/templates/research-log.schema.json +40 -0
  171. package/payload/mishkan/templates/settings.json +12 -0
  172. package/payload/mishkan/templates/settings.local.json +6 -0
  173. package/payload/mishkan/templates/sprint-state.schema.json +47 -0
  174. package/payload/mishkan/templates/team-report.schema.json +50 -0
  175. package/payload/mishkan/templates/user-CLAUDE.md +62 -0
  176. package/payload/mishkan/workflows/README.md +88 -0
  177. package/payload/mishkan/workflows/mishkan-architecture-panel.js +156 -0
  178. package/payload/mishkan/workflows/mishkan-codebase-audit.js +188 -0
  179. package/payload/mishkan/workflows/mishkan-deep-research.js +251 -0
  180. package/payload/mishkan/workflows/mishkan-init.js +156 -0
  181. package/payload/mishkan/workflows/mishkan-migration-wave.js +180 -0
  182. package/payload/mishkan/workflows/mishkan-release-readiness.js +163 -0
  183. package/payload/mishkan/workflows/mishkan-sprint-close.js +112 -0
  184. package/payload/user/CLAUDE.md +62 -0
  185. package/payload/user/rules/engineer-standards.md +66 -0
  186. package/payload/user/rules/y4nn-standards.md +167 -0
@@ -0,0 +1,93 @@
1
+ # Baruch eval
2
+
3
+ A minimal, deterministic eval for the terminal stage of the research
4
+ pipeline. It validates the schema enforcement around
5
+ `research-log.json` and asserts the semantic shape of a known-good
6
+ Baruch output for one scenario.
7
+
8
+ The eval **does not** run Baruch live (that would require an LLM
9
+ call). It is a contract eval: the schema, the validator, and a
10
+ golden output are exercised end-to-end so that regressions in any
11
+ of those layers are caught.
12
+
13
+ ## Layout
14
+
15
+ ```
16
+ evals/baruch/
17
+ ├── README.md ← this file
18
+ ├── run.sh ← runner (idempotent; exit 0 = all pass)
19
+ ├── fixtures/
20
+ │ ├── valid/ ← known-good logs; validator MUST accept
21
+ │ │ ├── resolved-cross-harness.json
22
+ │ │ ├── curated-shortcircuit.json
23
+ │ │ ├── partial-no-write.json
24
+ │ │ └── blocked-vendor.json
25
+ │ └── invalid/ ← each isolates one schema violation
26
+ │ ├── missing-required-field.json
27
+ │ ├── bad-trigger-enum.json
28
+ │ ├── bad-outcome-enum.json
29
+ │ ├── bad-sprint-pattern.json
30
+ │ └── malformed-json.json
31
+ └── golden_case/
32
+ ├── input.yaml ← what Baruch receives (upstream context)
33
+ ├── expected.yaml ← semantic assertions (jq filter: expected JSON)
34
+ └── produced.json ← reference Baruch output for this input
35
+ ```
36
+
37
+ ## What the runner checks
38
+
39
+ 1. **Every valid fixture validates.** Catches schema or validator
40
+ over-strictness regressions.
41
+ 2. **Every invalid fixture is rejected.** Catches validator
42
+ under-strictness regressions (missing required-field check,
43
+ missing enum check, etc.).
44
+ 3. **The golden case satisfies the schema and the semantic
45
+ assertions in `expected.yaml`.** Catches regressions in Baruch's
46
+ reasoning shape — fields being transcribed verbatim, the
47
+ cross-harness write rule, faithful `tools_invoked` list, etc.
48
+
49
+ ## Running
50
+
51
+ ```bash
52
+ ./run.sh
53
+ # 0 → all checks pass
54
+ # 1 → one or more checks failed
55
+ # 2 → environment problem (missing jq, validator, schema)
56
+ ```
57
+
58
+ ## Updating
59
+
60
+ - **A new valid scenario.** Drop a fully-populated, valid log into
61
+ `fixtures/valid/`. The runner picks it up automatically.
62
+ - **A new invalid scenario.** Drop a log with exactly one violation
63
+ into `fixtures/invalid/`. Isolating the violation is the
64
+ discipline — bundled invalid fixtures hide which check caught
65
+ what.
66
+ - **A new golden scenario.** Add a sibling under `golden_case/` and
67
+ extend the runner, or refactor `golden_case/` into multiple
68
+ sub-cases. Each golden case has the
69
+ `input.yaml` + `expected.yaml` + `produced.json` triplet.
70
+ - **Running Baruch live to refresh `produced.json`.** Re-run the
71
+ pipeline against `input.yaml`; copy the new output to
72
+ `produced.json`. Inspect the diff; if Baruch's reasoning shape
73
+ changed, update `expected.yaml` deliberately.
74
+
75
+ ## The expected.yaml format
76
+
77
+ ```
78
+ <jq filter>: <expected JSON value>
79
+ ```
80
+
81
+ - Lines starting with `#` are comments.
82
+ - Blank lines are ignored.
83
+ - The expected side is canonical JSON: `"string"`, `123`, `true`,
84
+ `false`, `null`, `[...]`, `{...}`. Bare words become strings.
85
+ - The filter is anything jq accepts. Single-quoted strings inside
86
+ the filter are fine (jq accepts them; they avoid escaping in YAML).
87
+
88
+ ## See also
89
+
90
+ - The schema: [`../../templates/research-log.schema.json`](../../templates/research-log.schema.json)
91
+ - The validator: [`../../scripts/validate-research-log.sh`](../../scripts/validate-research-log.sh)
92
+ - Baruch agent: [`../../agents/baruch.md`](../../agents/baruch.md)
93
+ - Baruch craft skill: [`../../skills/baruch-research-reporting-craft/SKILL.md`](../../skills/baruch-research-reporting-craft/SKILL.md)
@@ -0,0 +1,15 @@
1
+ {
2
+ "agent": "hizkiah",
3
+ "team": "yasad",
4
+ "sprint": "S2",
5
+ "trigger": "faced_problem",
6
+ "query_intent": "asyncpg recovery question",
7
+ "tools_invoked": ["jakin", "ezra", "caleb", "shaphan", "shemaiah", "baruch"],
8
+ "research_output_summary": "summary",
9
+ "applied_to_task": "T-12",
10
+ "outcome": "maybe",
11
+ "knowledge_graph_write": true,
12
+ "curated_library_match": false,
13
+ "cognee_node_id": "node_01HZ7K3X9Y00000000000000",
14
+ "timestamp": "2026-06-02T14:00:00Z"
15
+ }
@@ -0,0 +1,15 @@
1
+ {
2
+ "agent": "hizkiah",
3
+ "team": "yasad",
4
+ "sprint": "sprint-2",
5
+ "trigger": "faced_problem",
6
+ "query_intent": "asyncpg recovery question",
7
+ "tools_invoked": ["jakin", "ezra", "caleb", "shaphan", "shemaiah", "baruch"],
8
+ "research_output_summary": "summary",
9
+ "applied_to_task": "T-12",
10
+ "outcome": "resolved",
11
+ "knowledge_graph_write": true,
12
+ "curated_library_match": false,
13
+ "cognee_node_id": "node_01HZ7K3X9Y00000000000000",
14
+ "timestamp": "2026-06-02T14:00:00Z"
15
+ }
@@ -0,0 +1,15 @@
1
+ {
2
+ "agent": "hizkiah",
3
+ "team": "yasad",
4
+ "sprint": "S2",
5
+ "trigger": "curious",
6
+ "query_intent": "asyncpg recovery question",
7
+ "tools_invoked": ["jakin", "ezra", "caleb", "shaphan", "shemaiah", "baruch"],
8
+ "research_output_summary": "summary",
9
+ "applied_to_task": "T-12",
10
+ "outcome": "resolved",
11
+ "knowledge_graph_write": true,
12
+ "curated_library_match": false,
13
+ "cognee_node_id": "node_01HZ7K3X9Y00000000000000",
14
+ "timestamp": "2026-06-02T14:00:00Z"
15
+ }
@@ -0,0 +1,7 @@
1
+ {
2
+ "agent": "hizkiah",
3
+ "team": "yasad",
4
+ "sprint": "S2",
5
+ "trigger": "faced_problem"
6
+ "query_intent": "missing comma above"
7
+ }
@@ -0,0 +1,14 @@
1
+ {
2
+ "agent": "hizkiah",
3
+ "team": "yasad",
4
+ "sprint": "S2",
5
+ "trigger": "faced_problem",
6
+ "query_intent": "asyncpg recovery question",
7
+ "tools_invoked": ["jakin", "ezra", "caleb", "shaphan", "shemaiah", "baruch"],
8
+ "applied_to_task": "T-12",
9
+ "outcome": "resolved",
10
+ "knowledge_graph_write": true,
11
+ "curated_library_match": false,
12
+ "cognee_node_id": "node_01HZ7K3X9Y00000000000000",
13
+ "timestamp": "2026-06-02T14:00:00Z"
14
+ }
@@ -0,0 +1,15 @@
1
+ {
2
+ "agent": "joab",
3
+ "team": "mishmar",
4
+ "sprint": "S2",
5
+ "trigger": "requested",
6
+ "query_intent": "Whether VendorX OAuth2 honours PKCE S256 in production, since public docs describe staging only.",
7
+ "tools_invoked": ["jakin", "ezra", "caleb", "shaphan", "shemaiah", "baruch"],
8
+ "research_output_summary": "VendorX public OAuth2 docs describe staging behaviour and state production may differ. No authoritative source for production PKCE handling found; community reports conflict. Confidence: low. Cannot resolve without vendor confirmation.",
9
+ "applied_to_task": "T-24",
10
+ "outcome": "blocked",
11
+ "knowledge_graph_write": false,
12
+ "curated_library_match": false,
13
+ "cognee_node_id": null,
14
+ "timestamp": "2026-06-02T14:10:00Z"
15
+ }
@@ -0,0 +1,15 @@
1
+ {
2
+ "agent": "salma",
3
+ "team": "panim",
4
+ "sprint": "S2",
5
+ "trigger": "faced_problem",
6
+ "query_intent": "Whether TanStack Query v5 useQuery still exposes onSuccess and onError callbacks.",
7
+ "tools_invoked": ["jakin", "ezra", "baruch"],
8
+ "research_output_summary": "TanStack Query v5 removed onSuccess, onError, and onSettled from useQuery (Oct 2023). Migration: use useEffect keyed on data/error or a mutation observer pattern. Source: TanStack v5 migration guide.",
9
+ "applied_to_task": "T-19",
10
+ "outcome": "resolved",
11
+ "knowledge_graph_write": false,
12
+ "curated_library_match": true,
13
+ "cognee_node_id": null,
14
+ "timestamp": "2026-06-02T14:05:00Z"
15
+ }
@@ -0,0 +1,14 @@
1
+ {
2
+ "agent": "salma",
3
+ "team": "panim",
4
+ "sprint": "S2",
5
+ "trigger": "faced_problem",
6
+ "query_intent": "Whether Next.js 15 App Router supports streaming + Server Actions + dynamic imports in combination, and what the documented interaction is.",
7
+ "tools_invoked": ["jakin", "ezra", "caleb", "shaphan", "shemaiah", "baruch"],
8
+ "research_output_summary": "Streaming + Server Actions documented separately; dynamic imports under React 19 documented separately. Three-way combination not covered in v15 docs as of search date; vercel/examples #3104 shows it works but relies on cache-control choices. Confidence: medium. Gap: official documentation of combined behaviour.",
9
+ "applied_to_task": "T-22",
10
+ "outcome": "partial",
11
+ "knowledge_graph_write": false,
12
+ "curated_library_match": false,
13
+ "cognee_node_id": null
14
+ }
@@ -0,0 +1,15 @@
1
+ {
2
+ "agent": "hizkiah",
3
+ "team": "yasad",
4
+ "sprint": "S2",
5
+ "trigger": "faced_problem",
6
+ "query_intent": "How asyncpg handles connection loss mid-transaction; recovery semantics, observable error class, and whether the transaction is automatically retried.",
7
+ "tools_invoked": ["jakin", "ezra", "caleb", "shaphan", "shemaiah", "baruch"],
8
+ "research_output_summary": "asyncpg raises InterfaceError or ConnectionDoesNotExistError on network loss mid-transaction. The transaction is NOT auto-retried; the pool acquires a fresh connection on next use. Application must catch and decide. Sources: magicstack/asyncpg docs and issue #847. Confidence: high.",
9
+ "applied_to_task": "T-12",
10
+ "outcome": "resolved",
11
+ "knowledge_graph_write": true,
12
+ "curated_library_match": false,
13
+ "cognee_node_id": "node_01HZ7K3X9Y00000000000000",
14
+ "timestamp": "2026-06-02T14:00:00Z"
15
+ }
@@ -0,0 +1,35 @@
1
+ # Golden case expectations — semantic assertions on Baruch's output.
2
+ #
3
+ # Format: each non-blank, non-comment line is `<jq filter>: <expected JSON>`.
4
+ # The expected side is canonical JSON (string, number, true, false, array).
5
+ # Fields like timestamp and the exact cognee_node_id value are don't-care;
6
+ # the structural and semantic claims below must hold.
7
+
8
+ # Identity transcription (from input.yaml verbatim where applicable)
9
+ .agent: "hizkiah"
10
+ .team: "yasad"
11
+ .sprint: "S2"
12
+ .trigger: "faced_problem"
13
+ .applied_to_task: "T-12"
14
+
15
+ # Outcome carries Shemaiah's verdict verbatim
16
+ .outcome: "resolved"
17
+
18
+ # Cross-harness scope + resolved => cognee write
19
+ .knowledge_graph_write: true
20
+ .curated_library_match: false
21
+
22
+ # When a node was written, cognee_node_id is a non-empty string
23
+ .cognee_node_id | type: "string"
24
+ .cognee_node_id | length > 0: true
25
+
26
+ # Stages actually invoked are recorded faithfully (no over- or under-listing)
27
+ .tools_invoked | sort: ["baruch","caleb","ezra","jakin","shaphan","shemaiah"]
28
+
29
+ # query_intent carries Jakin's clarified intent (key phrase must survive)
30
+ .query_intent | contains("asyncpg"): true
31
+ .query_intent | contains("mid-transaction"): true
32
+
33
+ # research_output_summary preserves the source attribution from Shaphan
34
+ .research_output_summary | contains("InterfaceError"): true
35
+ .research_output_summary | contains("asyncpg"): true
@@ -0,0 +1,47 @@
1
+ # Golden case input — what Baruch receives as upstream context.
2
+ #
3
+ # Scenario: Hizkiah hit an asyncpg unknown; the pipeline ran all six
4
+ # stages; Shemaiah resolved with high confidence; the learning is
5
+ # cross-harness (every asyncpg-using project cares).
6
+ #
7
+ # This file documents the input Baruch would see. The eval does NOT
8
+ # run Baruch live (that would require an LLM call); it asserts that
9
+ # a Baruch output produced from this input passes the schema +
10
+ # matches the semantic expectations in expected.yaml.
11
+
12
+ calling_agent:
13
+ alias: hizkiah
14
+ team: yasad
15
+
16
+ sprint: S2
17
+ trigger: faced_problem
18
+ applied_to_task: T-12
19
+
20
+ jakin_clarified_intent: |
21
+ How asyncpg handles connection loss mid-transaction; recovery
22
+ semantics, observable error class, and whether the transaction
23
+ is automatically retried.
24
+
25
+ ezra_brief:
26
+ curated_library_match: false
27
+
28
+ shaphan_summary: |
29
+ asyncpg raises InterfaceError or ConnectionDoesNotExistError on
30
+ network loss mid-transaction. The transaction is NOT auto-retried;
31
+ the pool acquires a fresh connection on next use. Application must
32
+ catch and decide. Sources: magicstack/asyncpg docs and issue #847.
33
+ Confidence: high.
34
+
35
+ shemaiah_verdict:
36
+ outcome: resolved
37
+ confidence: high
38
+ curated_library_agreement: not_covered
39
+ cross_harness_promotion_recommended: true
40
+
41
+ stages_invoked_actually:
42
+ - jakin
43
+ - ezra
44
+ - caleb
45
+ - shaphan
46
+ - shemaiah
47
+ - baruch
@@ -0,0 +1,15 @@
1
+ {
2
+ "agent": "hizkiah",
3
+ "team": "yasad",
4
+ "sprint": "S2",
5
+ "trigger": "faced_problem",
6
+ "query_intent": "How asyncpg handles connection loss mid-transaction; recovery semantics, observable error class, and whether the transaction is automatically retried.",
7
+ "tools_invoked": ["jakin", "ezra", "caleb", "shaphan", "shemaiah", "baruch"],
8
+ "research_output_summary": "asyncpg raises InterfaceError or ConnectionDoesNotExistError on network loss mid-transaction. The transaction is NOT auto-retried; the pool acquires a fresh connection on next use. Application must catch and decide. Sources: magicstack/asyncpg docs and issue #847. Confidence: high.",
9
+ "applied_to_task": "T-12",
10
+ "outcome": "resolved",
11
+ "knowledge_graph_write": true,
12
+ "curated_library_match": false,
13
+ "cognee_node_id": "node_01HZ7K3X9Y00000000000000",
14
+ "timestamp": "2026-06-02T14:00:00Z"
15
+ }
@@ -0,0 +1,129 @@
1
+ #!/usr/bin/env bash
2
+ # evals/baruch/run.sh — exercise the Baruch contract end-to-end.
3
+ #
4
+ # Three checks, each independent:
5
+ # 1. Every fixture under fixtures/valid/ must validate (exit 0).
6
+ # 2. Every fixture under fixtures/invalid/ must fail validation (exit 1).
7
+ # 3. The golden case's produced.json must validate AND satisfy every
8
+ # jq assertion in golden_case/expected.yaml.
9
+ #
10
+ # Exit codes:
11
+ # 0 → all checks passed.
12
+ # 1 → one or more checks failed.
13
+ # 2 → environment problem (missing jq, missing validator, missing schema).
14
+ #
15
+ # This eval does NOT run Baruch live. It validates the contract and
16
+ # the golden output. To exercise Baruch's reasoning, swap produced.json
17
+ # for a fresh run's output and re-run.
18
+
19
+ set -uo pipefail
20
+
21
+ HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
22
+ VALIDATOR_CANDIDATES=(
23
+ "${MISHKAN_HOME:-$HOME/.claude/mishkan}/scripts/validate-research-log.sh"
24
+ "$HERE/../../scripts/validate-research-log.sh"
25
+ )
26
+ VALIDATOR=""
27
+ for c in "${VALIDATOR_CANDIDATES[@]}"; do
28
+ if [[ -x "$c" ]]; then VALIDATOR="$c"; break; fi
29
+ done
30
+ if [[ -z "$VALIDATOR" ]]; then
31
+ echo "error: validate-research-log.sh not found in any known location" >&2
32
+ printf ' searched: %s\n' "${VALIDATOR_CANDIDATES[@]}" >&2
33
+ exit 2
34
+ fi
35
+ if ! command -v jq >/dev/null 2>&1; then
36
+ echo "error: jq is required" >&2
37
+ exit 2
38
+ fi
39
+
40
+ PASS=0
41
+ FAIL=0
42
+ SKIP=0
43
+
44
+ pass() { printf " ✓ %s\n" "$1"; PASS=$((PASS+1)); }
45
+ fail() { printf " ✗ %s — %s\n" "$1" "$2"; FAIL=$((FAIL+1)); }
46
+
47
+ # --- Check 1: valid fixtures must validate -----------------------------
48
+ echo "[1/3] Valid fixtures must validate (expect exit 0)"
49
+ shopt -s nullglob
50
+ valid_count=0
51
+ for f in "$HERE/fixtures/valid"/*.json; do
52
+ valid_count=$((valid_count+1))
53
+ name="$(basename "$f")"
54
+ if out=$("$VALIDATOR" "$f" 2>&1); then
55
+ pass "$name"
56
+ else
57
+ fail "$name" "validator rejected a known-good fixture: $out"
58
+ fi
59
+ done
60
+ [[ $valid_count -eq 0 ]] && echo " (no fixtures found)" && SKIP=$((SKIP+1))
61
+
62
+ # --- Check 2: invalid fixtures must fail validation --------------------
63
+ echo "[2/3] Invalid fixtures must fail validation (expect exit 1)"
64
+ invalid_count=0
65
+ for f in "$HERE/fixtures/invalid"/*.json; do
66
+ invalid_count=$((invalid_count+1))
67
+ name="$(basename "$f")"
68
+ out=$("$VALIDATOR" "$f" 2>&1)
69
+ rc=$?
70
+ if [[ $rc -eq 1 ]]; then
71
+ pass "$name (rejected as expected)"
72
+ elif [[ $rc -eq 0 ]]; then
73
+ fail "$name" "validator accepted a known-bad fixture"
74
+ else
75
+ fail "$name" "validator returned $rc (expected 1)"
76
+ fi
77
+ done
78
+ [[ $invalid_count -eq 0 ]] && echo " (no fixtures found)" && SKIP=$((SKIP+1))
79
+
80
+ # --- Check 3: golden case — schema valid + semantic assertions ---------
81
+ echo "[3/3] Golden case must validate and satisfy semantic assertions"
82
+ GC="$HERE/golden_case/produced.json"
83
+ EXPECT="$HERE/golden_case/expected.yaml"
84
+
85
+ if [[ ! -r "$GC" ]]; then
86
+ fail "golden_case/produced.json" "missing or unreadable"
87
+ elif [[ ! -r "$EXPECT" ]]; then
88
+ fail "golden_case/expected.yaml" "missing or unreadable"
89
+ else
90
+ if out=$("$VALIDATOR" "$GC" 2>&1); then
91
+ pass "produced.json passes schema validation"
92
+ else
93
+ fail "produced.json" "did not pass schema validation: $out"
94
+ fi
95
+
96
+ # Each non-blank, non-comment line in expected.yaml is `<jq filter>: <expected>`.
97
+ # The expected side is YAML-ish JSON (string, number, true, false, JSON array).
98
+ while IFS= read -r raw; do
99
+ line="${raw%%#*}" # strip inline comments
100
+ line="$(echo "$line" | sed -e 's/[[:space:]]*$//')"
101
+ [[ -z "$line" ]] && continue
102
+ filter="$(echo "$line" | sed -E 's/:[[:space:]]*[^:]*$//')"
103
+ expected_raw="$(echo "$line" | sed -E 's/^.*:[[:space:]]*//')"
104
+ filter="${filter#\"}"; filter="${filter%\"}"
105
+
106
+ # Normalise expected_raw to canonical JSON for comparison.
107
+ if expected_json=$(echo "$expected_raw" | jq -c . 2>/dev/null); then
108
+ :
109
+ else
110
+ # bare word like `true` / `false` / unquoted string — wrap as JSON string
111
+ expected_json=$(jq -nc --arg v "$expected_raw" '$v')
112
+ fi
113
+
114
+ if actual_json=$(jq -c "$filter" "$GC" 2>/dev/null); then
115
+ if [[ "$actual_json" == "$expected_json" ]]; then
116
+ pass "assertion: $filter"
117
+ else
118
+ fail "assertion: $filter" "expected $expected_json, got $actual_json"
119
+ fi
120
+ else
121
+ fail "assertion: $filter" "jq filter errored"
122
+ fi
123
+ done < "$EXPECT"
124
+ fi
125
+
126
+ # --- Summary ----------------------------------------------------------
127
+ echo
128
+ echo "Summary: $PASS passed, $FAIL failed, $SKIP skipped sections"
129
+ [[ $FAIL -eq 0 ]]
@@ -0,0 +1,96 @@
1
+ #!/usr/bin/env python3
2
+ """MISHKAN PreToolUse hook — make config/model-routing.yaml authoritative.
3
+
4
+ Fires before the Task/Agent (subagent) tool runs. Reads the agent->tier map and
5
+ injects the resolved `model` into the tool input, so the YAML — not each agent's
6
+ frontmatter — is the single source of truth for which Claude tier a subagent
7
+ runs on. Frontmatter `model:` becomes the fallback used only when this hook is
8
+ absent.
9
+
10
+ Fail-open by contract: any parse/IO/format problem => emit nothing and exit 0,
11
+ so a broken config never blocks delegation. Only ever ADDS a `model` field.
12
+
13
+ stdin : Claude Code PreToolUse JSON {tool_name, tool_input, ...}
14
+ stdout : {"hookSpecificOutput": {"hookEventName": "PreToolUse",
15
+ "permissionDecision": "allow",
16
+ "updatedInput": <tool_input + model>}}
17
+ """
18
+ import json
19
+ import os
20
+ import sys
21
+
22
+ VALID = {"opus", "sonnet", "haiku"}
23
+ SUBAGENT_TOOLS = {"Task", "Agent"}
24
+ YAML = os.path.join(os.path.dirname(__file__), "..", "config", "model-routing.yaml")
25
+
26
+
27
+ def parse_routing(path):
28
+ """Minimal YAML reader for this file's shape (no external deps).
29
+
30
+ Captures `defaults.unlisted_agent` and the `agents:` name->tier map; strips
31
+ inline `# comments`. Returns (agents_map, default_tier).
32
+ """
33
+ agents, default = {}, "sonnet"
34
+ section = None
35
+ with open(path, "r") as fh:
36
+ for raw in fh:
37
+ line = raw.split("#", 1)[0].rstrip()
38
+ if not line.strip():
39
+ continue
40
+ if not line.startswith((" ", "\t")): # top-level key
41
+ section = line.strip().rstrip(":")
42
+ continue
43
+ key, _, val = line.strip().partition(":")
44
+ key, val = key.strip(), val.strip()
45
+ if not val:
46
+ continue
47
+ if section == "defaults" and key == "unlisted_agent" and val in VALID:
48
+ default = val
49
+ elif section == "agents" and val in VALID:
50
+ agents[key] = val
51
+ return agents, default
52
+
53
+
54
+ def main():
55
+ try:
56
+ payload = json.load(sys.stdin)
57
+ except Exception:
58
+ return # not parseable input -> do nothing
59
+
60
+ tool = payload.get("tool_name", "")
61
+ tin = payload.get("tool_input")
62
+ if tool not in SUBAGENT_TOOLS or not isinstance(tin, dict):
63
+ return
64
+
65
+ subagent = tin.get("subagent_type")
66
+ if not subagent:
67
+ return # generic/no-type agent call -> leave model to default mechanism
68
+
69
+ try:
70
+ agents, _default = parse_routing(YAML)
71
+ except Exception:
72
+ return # missing/broken routing file -> fail open
73
+
74
+ # Authoritative ONLY for agents the YAML explicitly lists (the MISHKAN fleet).
75
+ # Foreign agents (e.g. aiobi-ops, Explore) keep their own frontmatter model —
76
+ # never downgrade them via an unlisted fallback.
77
+ if subagent not in agents:
78
+ return
79
+ model = agents[subagent]
80
+ if model not in VALID:
81
+ return
82
+
83
+ updated = dict(tin)
84
+ updated["model"] = model
85
+ print(json.dumps({
86
+ "hookSpecificOutput": {
87
+ "hookEventName": "PreToolUse",
88
+ "permissionDecision": "allow",
89
+ "permissionDecisionReason": f"MISHKAN model-routing: {subagent} -> {model}",
90
+ "updatedInput": updated,
91
+ }
92
+ }))
93
+
94
+
95
+ if __name__ == "__main__":
96
+ main()
@@ -0,0 +1,45 @@
1
+ #!/usr/bin/env bash
2
+ # MISHKAN PostToolUse observability hook.
3
+ # Emits one append-only JSON line per tool call into the session log.
4
+ # Minimal: no parsing, no blocking. Token/cost enrichment happens later in the
5
+ # observability aggregation step (Phase 8) which reads these raw lines.
6
+ set -uo pipefail
7
+
8
+ LOG_DIR="${HOME}/.claude/mishkan/logs"
9
+ mkdir -p "$LOG_DIR"
10
+
11
+ INPUT="$(cat)"
12
+
13
+ if ! command -v jq >/dev/null 2>&1; then
14
+ # Fail open — never let observability break a tool call.
15
+ exit 0
16
+ fi
17
+
18
+ session="$(printf '%s' "$INPUT" | jq -r '.session_id // "unknown"' 2>/dev/null)"
19
+ tool="$(printf '%s' "$INPUT" | jq -r '.tool_name // "unknown"' 2>/dev/null)"
20
+ ts="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
21
+
22
+ # Outcome inferred from tool_response if present.
23
+ outcome="$(printf '%s' "$INPUT" | jq -r '
24
+ if (.tool_response.error? // empty) != "" then "blocked" else "completed" end
25
+ ' 2>/dev/null)"
26
+ [ -z "$outcome" ] && outcome="completed"
27
+
28
+ # Append a raw event line. agent/team/sprint/tokens are enriched later from
29
+ # session context; recorded here as null when the hook payload lacks them.
30
+ jq -nc \
31
+ --arg session "$session" \
32
+ --arg tool "$tool" \
33
+ --arg outcome "$outcome" \
34
+ --arg ts "$ts" \
35
+ '{
36
+ session: $session,
37
+ tool_calls: [$tool],
38
+ outcome: $outcome,
39
+ timestamp: $ts,
40
+ agent: null, team: null, sprint: null,
41
+ tokens_input: 0, tokens_cached: 0, tokens_output: 0,
42
+ cost: 0, cognee_writes: 0
43
+ }' >> "${LOG_DIR}/${session}.jsonl" 2>/dev/null
44
+
45
+ exit 0