@a5c-ai/krate 5.0.1-staging.f672fe79b

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (174) hide show
  1. package/Dockerfile +29 -0
  2. package/README.md +183 -0
  3. package/bin/krate-demo.mjs +23 -0
  4. package/bin/krate-server.mjs +14 -0
  5. package/dist/krate-controller-ui.json +2407 -0
  6. package/dist/krate-lifecycle.json +201 -0
  7. package/dist/krate-runtime-snapshot.json +2955 -0
  8. package/dist/krate-summary.json +687 -0
  9. package/docs/README.md +61 -0
  10. package/docs/agents/README.md +83 -0
  11. package/docs/agents/acceptance-test-matrix.md +193 -0
  12. package/docs/agents/agent-mux-adapter-contract.md +167 -0
  13. package/docs/agents/agent-mux-source-map.md +310 -0
  14. package/docs/agents/agent-run-memory-import-spec.md +256 -0
  15. package/docs/agents/agent-stack-management-spec.md +421 -0
  16. package/docs/agents/api-contract-spec.md +309 -0
  17. package/docs/agents/artifacts-writeback-spec.md +145 -0
  18. package/docs/agents/chart-packaging-spec.md +128 -0
  19. package/docs/agents/ci-orchestration-spec.md +140 -0
  20. package/docs/agents/context-assembly-spec.md +219 -0
  21. package/docs/agents/controller-reconciliation-spec.md +255 -0
  22. package/docs/agents/crd-schema-spec.md +315 -0
  23. package/docs/agents/decision-log-open-questions.md +169 -0
  24. package/docs/agents/developer-implementation-checklist.md +329 -0
  25. package/docs/agents/dispatching-design.md +262 -0
  26. package/docs/agents/glossary.md +66 -0
  27. package/docs/agents/implementation-blueprint.md +324 -0
  28. package/docs/agents/implementation-rollout-slices.md +251 -0
  29. package/docs/agents/memory-context-integration-spec.md +194 -0
  30. package/docs/agents/memory-ontology-schema-spec.md +253 -0
  31. package/docs/agents/memory-operations-runbook.md +121 -0
  32. package/docs/agents/mvp-vertical-slice-spec.md +146 -0
  33. package/docs/agents/observability-audit-spec.md +265 -0
  34. package/docs/agents/operator-runbook.md +174 -0
  35. package/docs/agents/org-memory-api-payload-examples.md +333 -0
  36. package/docs/agents/org-memory-controller-sequence-spec.md +181 -0
  37. package/docs/agents/org-memory-e2e-fixture-plan.md +161 -0
  38. package/docs/agents/org-memory-ui-implementation-map.md +114 -0
  39. package/docs/agents/org-memory-vertical-slice-spec.md +168 -0
  40. package/docs/agents/org-resource-model-delta-spec.md +111 -0
  41. package/docs/agents/org-route-resource-model-spec.md +183 -0
  42. package/docs/agents/org-scoping-namespace-spec.md +114 -0
  43. package/docs/agents/rbac-secrets-management-spec.md +406 -0
  44. package/docs/agents/repository-page-integration-spec.md +255 -0
  45. package/docs/agents/resource-contract-examples.md +808 -0
  46. package/docs/agents/resource-relationship-map.md +190 -0
  47. package/docs/agents/security-threat-model.md +188 -0
  48. package/docs/agents/shared-memory-company-brain-spec.md +358 -0
  49. package/docs/agents/storage-migration-spec.md +168 -0
  50. package/docs/agents/subagent-orchestration-spec.md +152 -0
  51. package/docs/agents/system-overview.md +88 -0
  52. package/docs/agents/tools-mcp-skills-spec.md +189 -0
  53. package/docs/agents/traceability-matrix.md +79 -0
  54. package/docs/agents/ui-flow-spec.md +211 -0
  55. package/docs/agents/ui-ux-system-spec.md +426 -0
  56. package/docs/agents/workspace-lifecycle-spec.md +166 -0
  57. package/docs/architecture-spec.md +78 -0
  58. package/docs/components/control-plane.md +78 -0
  59. package/docs/components/data-plane.md +69 -0
  60. package/docs/components/hooks-events.md +67 -0
  61. package/docs/components/identity-rbac-policy.md +73 -0
  62. package/docs/components/kubevela-oam.md +70 -0
  63. package/docs/components/operations-publishing.md +81 -0
  64. package/docs/components/runners-ci.md +66 -0
  65. package/docs/components/web-ui.md +94 -0
  66. package/docs/external/README.md +47 -0
  67. package/docs/external/bidirectional-sync-design.md +134 -0
  68. package/docs/external/cicd-interface.md +64 -0
  69. package/docs/external/external-backend-controllers.md +170 -0
  70. package/docs/external/external-backend-crds.md +234 -0
  71. package/docs/external/external-backend-ui-spec.md +151 -0
  72. package/docs/external/external-backend-ux-flows.md +115 -0
  73. package/docs/external/external-object-mapping.md +125 -0
  74. package/docs/external/git-forge-interface.md +68 -0
  75. package/docs/external/github-integration-design.md +151 -0
  76. package/docs/external/issue-tracking-interface.md +66 -0
  77. package/docs/external/provider-capability-manifests.md +204 -0
  78. package/docs/external/provider-catalog.md +139 -0
  79. package/docs/external/provider-rollout-testing.md +78 -0
  80. package/docs/external/research-results.md +48 -0
  81. package/docs/external/security-auth-permissions.md +81 -0
  82. package/docs/external/sync-state-machines.md +108 -0
  83. package/docs/external/unified-external-backend-model.md +107 -0
  84. package/docs/external/user-facing-changes.md +67 -0
  85. package/docs/gaps.md +161 -0
  86. package/docs/install.md +94 -0
  87. package/docs/krate-design.md +334 -0
  88. package/docs/local-minikube.md +55 -0
  89. package/docs/ontology/README.md +32 -0
  90. package/docs/ontology/bounded-contexts.md +29 -0
  91. package/docs/ontology/events-and-hooks.md +32 -0
  92. package/docs/ontology/oam-kubevela.md +32 -0
  93. package/docs/ontology/operations-and-release.md +25 -0
  94. package/docs/ontology/personas-and-actors.md +32 -0
  95. package/docs/ontology/policies-and-invariants.md +33 -0
  96. package/docs/ontology/problem-space.md +30 -0
  97. package/docs/ontology/resource-contracts.md +40 -0
  98. package/docs/ontology/resource-taxonomy.md +42 -0
  99. package/docs/ontology/runners-and-ci.md +29 -0
  100. package/docs/ontology/solution-space.md +24 -0
  101. package/docs/ontology/storage-and-data-boundaries.md +29 -0
  102. package/docs/ontology/validation-matrix.md +24 -0
  103. package/docs/ontology/web-ui-excellent-flows.md +32 -0
  104. package/docs/ontology/workflows.md +39 -0
  105. package/docs/ontology/world.md +35 -0
  106. package/docs/product-requirements.md +62 -0
  107. package/docs/roadmap-mvp.md +87 -0
  108. package/docs/system-requirements.md +90 -0
  109. package/docs/tests/README.md +53 -0
  110. package/docs/tests/agent-qa-plan.md +63 -0
  111. package/docs/tests/browser-ui-tests.md +62 -0
  112. package/docs/tests/ci-quality-gates.md +48 -0
  113. package/docs/tests/coverage-model.md +64 -0
  114. package/docs/tests/e2e-scenario-tests.md +53 -0
  115. package/docs/tests/fixtures-test-data.md +63 -0
  116. package/docs/tests/observability-reliability-tests.md +54 -0
  117. package/docs/tests/product-test-matrix.md +145 -0
  118. package/docs/tests/qa-adoption-roadmap.md +130 -0
  119. package/docs/tests/qa-automation-plan.md +101 -0
  120. package/docs/tests/security-compliance-tests.md +57 -0
  121. package/docs/tests/test-framework-tools.md +88 -0
  122. package/docs/tests/test-suite-layout.md +121 -0
  123. package/docs/tests/unit-integration-tests.md +48 -0
  124. package/docs/todo-kyverno +714 -0
  125. package/docs/user-stories.md +78 -0
  126. package/examples/minikube-demo.yaml +190 -0
  127. package/examples/oam-application.yaml +23 -0
  128. package/examples/policy-kyverno-pr-title.yaml +18 -0
  129. package/package.json +63 -0
  130. package/scripts/build.mjs +29 -0
  131. package/scripts/setup-minikube.mjs +65 -0
  132. package/scripts/smoke.mjs +37 -0
  133. package/scripts/validate-doc-coverage.mjs +152 -0
  134. package/scripts/validate-package.mjs +93 -0
  135. package/scripts/validate-ui.mjs +207 -0
  136. package/src/agent-approval-controller.js +123 -0
  137. package/src/agent-context-bundles.js +242 -0
  138. package/src/agent-dispatch-controller.js +86 -0
  139. package/src/agent-mux-client.js +280 -0
  140. package/src/agent-permission-review.js +162 -0
  141. package/src/agent-stack-controller.js +296 -0
  142. package/src/agent-trigger-controller.js +108 -0
  143. package/src/api-controller.js +206 -0
  144. package/src/argocd-gitops.js +43 -0
  145. package/src/auth.js +265 -0
  146. package/src/component-catalog.js +41 -0
  147. package/src/control-plane.js +136 -0
  148. package/src/controller-client.js +38 -0
  149. package/src/controller-ui.js +538 -0
  150. package/src/data-plane.js +178 -0
  151. package/src/gitea-backend.js +95 -0
  152. package/src/handoff.js +98 -0
  153. package/src/hooks-events.js +63 -0
  154. package/src/http-server.js +151 -0
  155. package/src/identity-policy.js +86 -0
  156. package/src/index.js +30 -0
  157. package/src/kubernetes-controller.js +812 -0
  158. package/src/kubernetes-resource-gateway.js +48 -0
  159. package/src/operations.js +112 -0
  160. package/src/resource-model.js +203 -0
  161. package/src/runners-ci.js +48 -0
  162. package/src/runtime.js +196 -0
  163. package/src/web-ui.js +40 -0
  164. package/tests/agent-approval-controller.test.js +173 -0
  165. package/tests/agent-context-bundles.test.js +278 -0
  166. package/tests/agent-dispatch-controller.test.js +176 -0
  167. package/tests/agent-mux-client.test.js +204 -0
  168. package/tests/agent-permission-review.test.js +209 -0
  169. package/tests/agent-resources.test.js +212 -0
  170. package/tests/agent-stack-controller.test.js +221 -0
  171. package/tests/agent-trigger-controller.test.js +211 -0
  172. package/tests/deployment.test.js +395 -0
  173. package/tests/e2e/lifecycle.test.js +117 -0
  174. package/tests/krate.test.js +727 -0
@@ -0,0 +1,63 @@
1
+ # Agent QA plan
2
+
3
+ ## Scope
4
+
5
+ Agent QA covers future agent orchestration functionality:
6
+
7
+ - agent stacks, tools, MCP servers, skills, subagents;
8
+ - triggers from CI, webhooks, issues, PRs, labels, mentions, schedules, and manual UI;
9
+ - Agent Mux run/session/chat integration;
10
+ - dispatches displayed as CI-like runs;
11
+ - context assembly, labels, memory, redaction, and snapshots;
12
+ - company brain memory and `.a5c` run imports;
13
+ - approvals, artifacts, write-back, and audit;
14
+ - org-scoped RBAC, secrets, config, service accounts, and runner placement.
15
+
16
+ ## Required suites
17
+
18
+ | Suite | Tests |
19
+ | --- | --- |
20
+ | Stack schema | stack/tool/MCP/skill/subagent resource validation and readiness conditions. |
21
+ | Permission review | RBAC, secret/config grants, memory grants, missing capability explanations. |
22
+ | Context assembly | prompt layers, source provenance, labels, redaction, digest snapshots. |
23
+ | Dispatch lifecycle | create run/attempt, Agent Mux handoff, event stream, cancel/resume/retry. |
24
+ | Trigger rules | dry-run, dedupe, coalesce, branch/source filters, trusted/untrusted refs. |
25
+ | Agent Mux adapter | launch payload, capability discovery, session binding, transcript events. |
26
+ | Memory | query, historical refs, tool access, snapshot reuse, stale warnings. |
27
+ | Run import | `MEMORY.md`, sessions, `.a5c` journals/tasks/artifacts, redaction, PR review. |
28
+ | Write-back | patch/comment/check/review artifacts, approval, idempotency, rollback. |
29
+ | UI | dispatch composer, run detail/chat, memory dashboard, imports, approvals. |
30
+
31
+ ## Critical negative tests
32
+
33
+ - stack references tool without required Secret grant;
34
+ - skill requires ConfigMap not granted;
35
+ - agent on fork tries to access trusted secrets;
36
+ - trigger label tries to grant permission;
37
+ - context label tries to hide instructions from preview;
38
+ - Agent Mux session ID belongs to another org/run;
39
+ - memory tool reads outside pinned snapshot;
40
+ - `.a5c` import contains secret-like content;
41
+ - write-back tries to mutate unapproved target;
42
+ - subagent requests parent-only capability.
43
+
44
+ ## Browser journeys
45
+
46
+ - manual dispatch from Code page with memory preview;
47
+ - failed CI repair from Runs page;
48
+ - issue mention dispatch with linked workspace/session;
49
+ - run detail chat/session with event timeline;
50
+ - memory import review and approval;
51
+ - permission wizard fixes missing secret/config/memory grant;
52
+ - trigger rule dry-run preview.
53
+
54
+ ## Done criteria
55
+
56
+ Agent functionality is not production-ready until:
57
+
58
+ - unit/integration/API tests cover resource and controller logic;
59
+ - browser tests cover the primary user journeys;
60
+ - cross-org and no-secret negative tests pass;
61
+ - Agent Mux fake/session tests pass;
62
+ - memory snapshot and import fixtures pass;
63
+ - audit/events can explain every dispatch and write-back.
@@ -0,0 +1,62 @@
1
+ # Browser and UI tests
2
+
3
+ ## Browser framework
4
+
5
+ Use Playwright for browser E2E once added. Browser tests should focus on route behavior, accessibility, and critical workflows rather than brittle visual snapshots.
6
+
7
+ ## Route smoke coverage
8
+
9
+ Required route smoke tests:
10
+
11
+ - `/orgs`;
12
+ - `/orgs/[org]`;
13
+ - `/orgs/[org]/repositories`;
14
+ - `/orgs/[org]/repositories/[repo]/code`;
15
+ - `/issues`, `/pull-requests`, `/runs`, `/hooks`, `/settings` under repo routes;
16
+ - `/orgs/[org]/deployments`;
17
+ - `/orgs/[org]/runs`;
18
+ - future `/orgs/[org]/agents/*` and `/orgs/[org]/agents/memory/*`.
19
+
20
+ Every route smoke asserts:
21
+
22
+ - org switcher visible;
23
+ - breadcrumbs include org;
24
+ - main heading exists;
25
+ - no server error;
26
+ - advanced YAML/resource panels are reachable where expected;
27
+ - unauthorized or missing resources show safe empty states.
28
+
29
+ ## Critical UI journeys
30
+
31
+ | Journey | Assertions |
32
+ | --- | --- |
33
+ | Org switch | route changes org, data changes, no cross-org leakage. |
34
+ | Repository navigation | tabs preserve org/repo and active page. |
35
+ | Create/apply resource | YAML/plan preview, server validation, status update. |
36
+ | Run debugging | run list, event stream, details, rerun affordance. |
37
+ | Agent dispatch | composer, memory preview, permission review, created run link. |
38
+ | Memory import review | generated diff, redaction status, validation status, approve/reject. |
39
+
40
+ ## Accessibility checks
41
+
42
+ Run automated checks on primary routes for:
43
+
44
+ - headings and landmarks;
45
+ - form labels;
46
+ - button/link names;
47
+ - keyboard navigation;
48
+ - focus management in dialogs/panels;
49
+ - color contrast for status indicators;
50
+ - reduced-motion behavior where relevant.
51
+
52
+ ## Visual regression
53
+
54
+ Use visual checks sparingly for stable layout contracts:
55
+
56
+ - app shell/sidebar/topbar;
57
+ - repository code layout;
58
+ - run detail layout;
59
+ - memory import review panel;
60
+ - empty/loading/error states.
61
+
62
+ Prefer semantic assertions for changing data-heavy pages.
@@ -0,0 +1,48 @@
1
+ # CI quality gates
2
+
3
+ ## Gate levels
4
+
5
+ | Gate | Trigger | Required checks |
6
+ | --- | --- | --- |
7
+ | PR fast gate | pull request | install, static/docs/package checks, unit/integration tests, UI validation. |
8
+ | PR browser gate | pull request when UI changes | browser route smoke, critical UI journeys impacted by change. |
9
+ | Merge gate | main/staging merge | full `npm run check`, package/chart validation, UI build. |
10
+ | Nightly gate | schedule | live-ish integration, browser full suite, security scans, performance smoke. |
11
+ | Release gate | tag/release | Docker build, Helm package, smoke install, upgrade/rollback plan, SBOM/signing if enabled. |
12
+ | Staging gate | deployment | real cluster smoke, webhooks, runners, Gitea, Argo/KubeVela, Agent Mux if enabled. |
13
+
14
+ ## Current required local gate
15
+
16
+ `npm run check` remains the all-up local gate:
17
+
18
+ ```text
19
+ build -> validate:docs -> test -> e2e -> package:check -> smoke -> ui:validate -> ui:build
20
+ ```
21
+
22
+ ## Future gate additions
23
+
24
+ - `test:browser` for Playwright route and journey tests.
25
+ - `test:coverage` for coverage reporting.
26
+ - `test:security` for dependency, secret, and auth/RBAC checks.
27
+ - `test:charts` for rendered chart validation.
28
+ - `test:agents` for agent/company-brain vertical slice.
29
+
30
+ ## CI artifact policy
31
+
32
+ CI should retain:
33
+
34
+ - test logs;
35
+ - browser traces/screenshots/videos on failure;
36
+ - coverage reports;
37
+ - rendered manifests;
38
+ - package validation report;
39
+ - memory import redaction/validation fixtures;
40
+ - smoke output;
41
+ - SBOM/signature artifacts when release gates run.
42
+
43
+ ## Flake policy
44
+
45
+ - A flaky test is a failing test until triaged.
46
+ - Retries may be used only to collect evidence, not to hide failures.
47
+ - Quarantined tests need owner, expiry, issue link, and reduced gate impact.
48
+ - CI should track test duration and failure signatures.
@@ -0,0 +1,64 @@
1
+ # Coverage model
2
+
3
+ ## Coverage dimensions
4
+
5
+ Krate coverage is multi-dimensional. Line coverage is useful but not sufficient.
6
+
7
+ | Dimension | Required coverage |
8
+ | --- | --- |
9
+ | Code coverage | statements, branches, functions, and critical path modules. |
10
+ | Resource coverage | every CRD/config kind and aggregated kind has schema and example tests. |
11
+ | Route coverage | every org/repo/agent/deployment route has render and authorization tests. |
12
+ | API coverage | every typed endpoint has success, validation failure, auth failure, and cross-org negative tests. |
13
+ | Controller coverage | reconcile create/update/delete, idempotency, retry, drift, finalizer, and status conditions. |
14
+ | UI coverage | primary journeys, disabled states, advanced YAML panels, accessibility, route guards. |
15
+ | Security coverage | auth, RBAC, Secret/ConfigMap grants, no-leak responses, audit records. |
16
+ | Release coverage | package files, chart templates, CRDs, examples, smoke install, Docker image. |
17
+ | Agent coverage | dispatch, context, memory, tools, triggers, sessions, imports, approvals, write-back. |
18
+
19
+ ## Initial thresholds
20
+
21
+ | Layer | Target |
22
+ | --- | --- |
23
+ | Pure `src` modules | 85% line, 75% branch once coverage tooling lands. |
24
+ | Controller/API critical paths | 90% path coverage by table-driven tests. |
25
+ | UI route smoke | 100% of primary org/repo/deployment/agent routes render. |
26
+ | Resource kinds | 100% listed in resource model, docs, package examples, and tests. |
27
+ | Security negative paths | 100% for cross-org, no-secret, untrusted fork, and missing grant cases. |
28
+
29
+ Thresholds should ratchet upward; do not block early docs-only work on coverage tooling that does not exist yet.
30
+
31
+ ## Traceability
32
+
33
+ Every feature should map:
34
+
35
+ ```text
36
+ requirement -> resource/API/UI/controller -> test file -> CI gate -> artifact/report
37
+ ```
38
+
39
+ The existing `docs/agents/traceability-matrix.md` is the model for agent features. Product-wide coverage should extend the same pattern into `docs/tests`.
40
+
41
+ ## Coverage reports
42
+
43
+ Reports should include:
44
+
45
+ - per-command status;
46
+ - code coverage when available;
47
+ - resource kind coverage;
48
+ - route/API coverage;
49
+ - browser scenario coverage;
50
+ - security negative coverage;
51
+ - flaky tests and retries;
52
+ - untested new files/resources.
53
+
54
+ ## Coverage exclusions
55
+
56
+ Allowed exclusions:
57
+
58
+ - generated files;
59
+ - static docs;
60
+ - vendored assets;
61
+ - intentionally unreachable defensive branches when documented;
62
+ - live-only integrations covered by staging/nightly gates.
63
+
64
+ Exclusions must be explicit and reviewed.
@@ -0,0 +1,53 @@
1
+ # E2E and scenario tests
2
+
3
+ ## Existing E2E baseline
4
+
5
+ Current E2E tests validate chart package surface and minikube dry-run command plans. This remains the first E2E layer because it is deterministic and does not require a live cluster.
6
+
7
+ ## Core forge scenarios
8
+
9
+ | Scenario | Steps | Assertions |
10
+ | --- | --- | --- |
11
+ | Create repository | org dashboard -> repositories -> create | repository resource exists, clone instructions render, namespace/org labels exist. |
12
+ | Pull request lifecycle | create PR -> review -> CI status -> merge | PR status, review state, pipeline/job link, policy gates. |
13
+ | CI run lifecycle | trigger pipeline -> jobs run -> logs/events -> rerun | pipeline/job statuses, runner pool, ServiceAccount, artifacts. |
14
+ | Webhook delivery | configure hook -> send test delivery -> replay failed delivery | signed payload, retry policy, delivery records. |
15
+ | Deployment promotion | repo change -> deployment page -> promote/rollback | OAM/Argo status, environment scoping, audit. |
16
+ | Org isolation | duplicate repo slug across orgs | no silent legacy route selection, cross-org API denial. |
17
+
18
+ ## Agent and memory scenarios
19
+
20
+ | Scenario | Steps | Assertions |
21
+ | --- | --- | --- |
22
+ | Manual agent dispatch | repo code -> dispatch agent -> run detail | dispatch run, attempt, Agent Mux session, context bundle. |
23
+ | Dispatch with memory | select memory source -> preview -> dispatch | memory snapshot commit, selected records, digests, redaction. |
24
+ | Historical memory | choose `refAt` -> dispatch -> retry | retry stays pinned, stale warning shown. |
25
+ | Import run memory | run detail -> import `.a5c` summary -> approve | redacted import, validation report, memory PR/commit. |
26
+ | Triggered repair | failed CI -> trigger rule -> dispatch | dedupe, permission review, run row beside pipeline. |
27
+ | Write-back approval | agent proposes patch/comment -> approve | artifact digest, approval audit, PR/comment update. |
28
+
29
+ ## Live cluster scenarios
30
+
31
+ Nightly/staging suites should eventually run against a real cluster with:
32
+
33
+ - Kubernetes API aggregation;
34
+ - Gitea smart HTTP/SSH;
35
+ - Postgres;
36
+ - object storage;
37
+ - NATS/webhook queue;
38
+ - Argo CD/KubeVela;
39
+ - ARC or runner abstraction;
40
+ - Agent Mux gateway/runtime when enabled.
41
+
42
+ ## E2E artifacts
43
+
44
+ E2E tests should collect:
45
+
46
+ - generated resources;
47
+ - API responses;
48
+ - event/watch logs;
49
+ - screenshots/traces for browser flows;
50
+ - Helm manifests;
51
+ - controller logs;
52
+ - redaction/validation reports;
53
+ - audit event excerpts.
@@ -0,0 +1,63 @@
1
+ # Fixtures and test data
2
+
3
+ ## Principles
4
+
5
+ - Fixtures are deterministic and committed to the repo.
6
+ - Fixtures must never contain real secrets, tokens, private keys, customer data, or personal data beyond synthetic examples.
7
+ - Secret-like synthetic values should be clearly marked and used only to test redaction.
8
+ - Every fixture has an owner and purpose.
9
+ - Fixtures should be small enough to understand in a test failure.
10
+
11
+ ## Core fixtures
12
+
13
+ | Fixture | Purpose |
14
+ | --- | --- |
15
+ | default org | simple org and namespace for current tests. |
16
+ | duplicate org repos | route ambiguity and cross-org denial. |
17
+ | repository with PR/issue/pipeline | core forge E2E path. |
18
+ | webhook delivery set | success, retry, replay, signature mismatch. |
19
+ | runner pool/job set | trusted/untrusted runner policy. |
20
+ | deployment/OAM set | environment, promotion, rollback. |
21
+ | company brain memory repo | graph/Markdown/frontmatter/search fixtures. |
22
+ | `.a5c` run fixture | Babysitter run import and redaction. |
23
+ | Agent Mux session fixture | session binding, transcript summary, events. |
24
+
25
+ ## Directory proposal
26
+
27
+ ```text
28
+ tests/fixtures/
29
+ orgs/
30
+ repositories/
31
+ resources/
32
+ webhooks/
33
+ runners/
34
+ deployments/
35
+ agents/
36
+ memory/
37
+ company-brain/
38
+ a5c-runs/
39
+ sessions/
40
+ browser/
41
+ ```
42
+
43
+ ## Redaction fixture requirements
44
+
45
+ Redaction fixtures should include synthetic values that look like:
46
+
47
+ - API keys;
48
+ - bearer tokens;
49
+ - private key headers;
50
+ - kubeconfig snippets;
51
+ - webhook signatures;
52
+ - high-entropy strings.
53
+
54
+ Tests assert these values do not appear in prompt previews, context bundles, memory imports, transcripts, artifacts, API responses, UI, or audit records.
55
+
56
+ ## Fixture review checklist
57
+
58
+ - No real credentials.
59
+ - No real customer data.
60
+ - Org labels and namespace fields included.
61
+ - Expected status conditions documented.
62
+ - Stable timestamps and IDs.
63
+ - Cross-platform paths where possible.
@@ -0,0 +1,54 @@
1
+ # Observability and reliability tests
2
+
3
+ ## Observability coverage
4
+
5
+ Required signals:
6
+
7
+ - API request latency and errors;
8
+ - controller reconcile counts, durations, retries, and failures;
9
+ - watch connection counts and reconnects;
10
+ - Git operation latency/errors;
11
+ - webhook queue depth and delivery status;
12
+ - runner queue/wait/runtime metrics;
13
+ - memory query latency and import validation status;
14
+ - Agent Mux session binding and event stream status;
15
+ - audit event counts by action/outcome.
16
+
17
+ ## Reliability tests
18
+
19
+ | Failure | Expected behavior |
20
+ | --- | --- |
21
+ | Kubernetes API temporary failure | retry with backoff, status condition, no duplicate side effects. |
22
+ | Gitea unavailable | repository status degraded, no data loss, UI warning. |
23
+ | Postgres unavailable | aggregated API degraded/read-only where possible. |
24
+ | object storage unavailable | artifact writes fail safely with retry. |
25
+ | webhook receiver fails | retry and replay available. |
26
+ | watch disconnects | UI reconnects and resumes from list state. |
27
+ | memory repo unavailable | required-memory dispatch blocks, optional memory warns. |
28
+ | Agent Mux unavailable | dispatch shows pending/failed handoff and retry/recover action. |
29
+ | redaction failure | memory import blocks and no content leaks. |
30
+
31
+ ## Chaos and load
32
+
33
+ Nightly/staging tests should eventually cover:
34
+
35
+ - burst webhook deliveries;
36
+ - many repository list queries;
37
+ - concurrent dispatches;
38
+ - runner pool exhaustion;
39
+ - memory grep/query bounds;
40
+ - large context truncation;
41
+ - controller restart during reconciliation;
42
+ - duplicate event delivery idempotency.
43
+
44
+ ## Audit assertions
45
+
46
+ Every mutating or denied action should emit audit with:
47
+
48
+ - org and namespace;
49
+ - actor;
50
+ - resource ref;
51
+ - action and outcome;
52
+ - source event/run/session when applicable;
53
+ - digest fields for artifacts/context/memory;
54
+ - no secret values.
@@ -0,0 +1,145 @@
1
+ # Product test matrix
2
+
3
+ ## Purpose
4
+
5
+ This matrix maps Krate product areas to required automated test coverage. It covers existing functionality and future agent/company-brain functionality so implementation work can add tests in the right layer instead of relying on one broad E2E path.
6
+
7
+ ## Matrix legend
8
+
9
+ | Mark | Meaning |
10
+ | --- | --- |
11
+ | Required | must exist before feature is considered complete. |
12
+ | Recommended | should exist when the feature reaches production or staging. |
13
+ | Nightly | acceptable in slower scheduled/live suites. |
14
+ | Future | planned once the underlying feature exists. |
15
+
16
+ ## Product-area coverage
17
+
18
+ | Product area | Unit | Integration/API | Browser/UI | E2E/scenario | Security | Package/install |
19
+ | --- | --- | --- | --- | --- | --- | --- |
20
+ | Resource model and schemas | Required | Required | Recommended | Recommended | Recommended | Required |
21
+ | Organization and namespace scoping | Required | Required | Required | Required | Required | Required |
22
+ | Repository data plane | Required | Required | Required | Required | Required | Required |
23
+ | Pull requests and reviews | Required | Required | Required | Required | Recommended | Recommended |
24
+ | Issues and inbox | Required | Required | Required | Recommended | Recommended | Recommended |
25
+ | Pipelines and jobs | Required | Required | Required | Required | Required | Required |
26
+ | Runner pools and job isolation | Required | Required | Recommended | Required | Required | Required |
27
+ | Webhook subscriptions and deliveries | Required | Required | Required | Required | Required | Recommended |
28
+ | Identity, auth, teams, invites | Required | Required | Required | Recommended | Required | Required |
29
+ | RBAC and policy | Required | Required | Recommended | Required | Required | Required |
30
+ | Secrets and config grants | Required | Required | Required | Required | Required | Recommended |
31
+ | Deployments and environments | Required | Required | Required | Required | Recommended | Required |
32
+ | Argo CD and KubeVela/OAM | Required | Required | Recommended | Nightly | Recommended | Required |
33
+ | Operations install/readiness | Required | Required | Required | Required | Required | Required |
34
+ | Web UI shell and navigation | Recommended | Recommended | Required | Required | Recommended | Recommended |
35
+ | Advanced YAML/resource panels | Required | Required | Required | Recommended | Required | Recommended |
36
+ | Agent stacks and capabilities | Future | Future | Future | Future | Future | Future |
37
+ | Agent dispatch and Agent Mux | Future | Future | Future | Future | Future | Future |
38
+ | Agent triggers | Future | Future | Future | Future | Future | Future |
39
+ | Agent workspaces and sessions | Future | Future | Future | Future | Future | Future |
40
+ | Company brain memory | Future | Future | Future | Future | Future | Future |
41
+ | `.a5c` run memory imports | Future | Future | Future | Future | Future | Future |
42
+ | Artifacts and write-back | Future | Future | Future | Future | Future | Future |
43
+ | Packaging and release | Required | Required | Recommended | Required | Required | Required |
44
+
45
+ ## Existing command mapping
46
+
47
+ | Command | Covers | Gaps |
48
+ | --- | --- | --- |
49
+ | `npm test` | unit/integration tests in `tests/*.test.js` | not yet split by subsystem; no coverage report. |
50
+ | `npm run e2e` | current deterministic package/minikube E2E tests | no browser automation or live cluster path. |
51
+ | `npm run validate:docs` | docs/source/ontology coverage | does not validate all `docs/tests` requirements yet. |
52
+ | `npm run package:check` | package/chart/example coverage | not yet aware of future agent/memory CRDs. |
53
+ | `npm run smoke` | runtime smoke | should expand as APIs/routes grow. |
54
+ | `npm run ui:validate` | static UI validation | not a browser test. |
55
+ | `npm run ui:build` | Next production build | not behavioral UI coverage. |
56
+ | `npm run check` | all current gates | should remain required as new gates are added. |
57
+
58
+ ## Future suite mapping
59
+
60
+ | Future suite | Product areas |
61
+ | --- | --- |
62
+ | `test:unit` | resource model, route helpers, redaction, context assembly, ref resolution, validators. |
63
+ | `test:integration` | API controller, controller fakes, memory import, Gitea/K8s/Agent Mux fakes. |
64
+ | `test:api` | org-scoped endpoints, stable errors, resource actions, watch filters. |
65
+ | `test:browser` | org navigation, repository flows, deployments, run detail, agent/memory flows. |
66
+ | `test:coverage` | coverage thresholds and untested-file reporting. |
67
+ | `test:security` | auth/RBAC/no-secret/cross-org/secret-grant checks. |
68
+ | `test:charts` | Helm render, CRD examples, kubeconform, APIService/RBAC. |
69
+ | `test:agents` | agent dispatch, context, memory, Agent Mux, imports, triggers. |
70
+ | `test:live` | real cluster/Gitea/Argo/KubeVela/NATS/ARC/Object storage. |
71
+
72
+ ## Required negative coverage
73
+
74
+ Every relevant product area must include negative tests for:
75
+
76
+ - missing or mismatched `organizationRef`;
77
+ - wrong namespace for org;
78
+ - missing RBAC permission;
79
+ - missing Secret/ConfigMap grant;
80
+ - untrusted fork or untrusted runner tries privileged action;
81
+ - cross-org resource reference;
82
+ - invalid or stale Git ref;
83
+ - invalid webhook signature;
84
+ - resource deleted while a controller is reconciling;
85
+ - secret-like value appears in input and must not appear in output;
86
+ - watch reconnect after disconnect;
87
+ - duplicate event delivery and idempotency.
88
+
89
+ ## Release readiness matrix
90
+
91
+ A release candidate is blocked if any of these are missing:
92
+
93
+ - package/chart validation;
94
+ - CRD/example coverage for every shipped kind;
95
+ - at least one install smoke path;
96
+ - auth/RBAC/no-secret tests;
97
+ - UI build and route smoke;
98
+ - repository/PR/CI core E2E;
99
+ - deployment/OAM smoke when deployment features ship;
100
+ - agent/company-brain vertical slice when agent features ship;
101
+ - documented known gaps and quarantined tests.
102
+
103
+ ## External backend coverage
104
+
105
+ External provider support adds required coverage for:
106
+
107
+ | Area | Required tests |
108
+ | --- | --- |
109
+ | Provider auth | GitHub App Secret metadata, installation access, no-token leak. |
110
+ | Webhooks | signature validation, dedupe, replay, enqueue, malformed payload. |
111
+ | Issue interface | issue/comment/label sync, PR-backed issue handling, conflicts. |
112
+ | CI/CD interface | workflow run/job/check sync, rerun/cancel permissions, lazy logs. |
113
+ | Git forge interface | repo/PR/ref/key/branch protection sync and drift. |
114
+ | Bidirectional writes | write intent, approval, provider failure, confirmation, conflict. |
115
+ | Rate limits | backoff, degraded status, resume. |
116
+ | Cross-org | provider binding and native object references cannot cross orgs. |
117
+
118
+ ## Pluggable provider contract tests
119
+
120
+ Each provider adapter should run a shared contract suite for every supported interface:
121
+
122
+ | Contract suite | Providers |
123
+ | --- | --- |
124
+ | Issue tracking contract | GitHub, GitLab, Bitbucket when enabled, Azure DevOps, Jira, Linear, Gitea, custom. |
125
+ | CI/CD contract | GitHub Actions, GitLab CI, Bitbucket Pipelines, Azure Pipelines, Buildkite, CircleCI, Jenkins, custom. |
126
+ | Git forge contract | GitHub, GitLab, Bitbucket, Azure Repos, Gitea, Gerrit, raw Git partial, custom. |
127
+ | Webhook contract | any provider with webhooks. |
128
+ | Write-intent contract | any provider with mutating operations. |
129
+ | Conflict contract | any bidirectional provider. |
130
+
131
+ Contract tests should use fake provider adapters first, then provider-specific fixtures and optional live tests.
132
+
133
+ ## External UX flow tests
134
+
135
+ Browser and E2E tests should cover:
136
+
137
+ - connect GitHub provider;
138
+ - connect Jira issue-only provider;
139
+ - combine GitHub forge with Buildkite CI;
140
+ - resolve a sync conflict;
141
+ - approve an agent-proposed external write;
142
+ - replay a dead-lettered webhook;
143
+ - show provider rate-limit degraded state.
144
+
145
+ These flows are specified in `docs/external/external-backend-ux-flows.md`.
@@ -0,0 +1,130 @@
1
+ # QA adoption roadmap
2
+
3
+ ## Purpose
4
+
5
+ This roadmap sequences QA automation work so Krate can improve coverage without blocking every product change on the final-state toolchain.
6
+
7
+ ## Stage 0: current baseline
8
+
9
+ Status: available now.
10
+
11
+ Required gates:
12
+
13
+ - `npm run validate:docs`;
14
+ - `npm test`;
15
+ - `npm run e2e`;
16
+ - `npm run package:check`;
17
+ - `npm run smoke`;
18
+ - `npm run ui:validate`;
19
+ - `npm run ui:build`;
20
+ - `npm run check` before release-like changes.
21
+
22
+ ## Stage 1: suite organization
23
+
24
+ Add:
25
+
26
+ - `tests/fixtures` with org/repository/resource fixtures;
27
+ - test helper modules for fake Kubernetes and API controller setup;
28
+ - metadata comments for owner/gate/area;
29
+ - docs check that `docs/tests` exists and is linked.
30
+
31
+ Exit criteria:
32
+
33
+ - existing tests still pass;
34
+ - new fixture policy is followed;
35
+ - no behavior change required.
36
+
37
+ ## Stage 2: browser route smoke
38
+
39
+ Add:
40
+
41
+ - Playwright dependency and config;
42
+ - route smoke for org dashboard, repositories, repo code/issues/runs/settings, deployments, and operations pages;
43
+ - screenshot/trace capture on failure;
44
+ - accessibility smoke on primary routes.
45
+
46
+ Exit criteria:
47
+
48
+ - browser gate runs in CI for UI changes;
49
+ - route failures show useful artifacts;
50
+ - no test relies on live external services.
51
+
52
+ ## Stage 3: coverage and API suites
53
+
54
+ Add:
55
+
56
+ - coverage command and reporting;
57
+ - split API/controller tests;
58
+ - stable error-code assertions;
59
+ - org mismatch tests;
60
+ - no-secret response assertions;
61
+ - watch filter tests.
62
+
63
+ Exit criteria:
64
+
65
+ - coverage report generated in CI;
66
+ - minimum thresholds set for critical modules;
67
+ - cross-org denial is tested for resource APIs.
68
+
69
+ ## Stage 4: security and package hardening
70
+
71
+ Add:
72
+
73
+ - dependency/secret/license checks;
74
+ - rendered chart schema validation;
75
+ - action/workflow lint;
76
+ - Docker build smoke;
77
+ - SBOM/signature plan for release.
78
+
79
+ Exit criteria:
80
+
81
+ - release gate publishes security/package artifacts;
82
+ - chart regressions fail before release.
83
+
84
+ ## Stage 5: agent/company-brain vertical slice
85
+
86
+ Add:
87
+
88
+ - org memory fixtures;
89
+ - fake Agent Mux;
90
+ - fake memory Git repo;
91
+ - dispatch with memory snapshot tests;
92
+ - summary-only `.a5c` import tests;
93
+ - cross-org memory denial tests;
94
+ - browser journey for memory preview/import review.
95
+
96
+ Exit criteria:
97
+
98
+ - `docs/agents/org-memory-vertical-slice-spec.md` acceptance paths are automated;
99
+ - no raw `.a5c` secret-like content leaks;
100
+ - retry uses pinned memory snapshot.
101
+
102
+ ## Stage 6: live/staging reliability
103
+
104
+ Add:
105
+
106
+ - live cluster smoke profiles;
107
+ - Gitea, NATS, Argo CD, KubeVela, ARC, object storage checks;
108
+ - controller restart/idempotency tests;
109
+ - performance smoke for API/UI;
110
+ - webhook burst and retry tests.
111
+
112
+ Exit criteria:
113
+
114
+ - staging gates prove install, core workflows, and rollback;
115
+ - failure artifacts are actionable.
116
+
117
+ ## Stage 7: continuous quality intelligence
118
+
119
+ Add:
120
+
121
+ - flaky test dashboard;
122
+ - coverage trend dashboard;
123
+ - failure signature clustering;
124
+ - ownership routing;
125
+ - QA metrics in release notes;
126
+ - automated gap reminders when new resources/routes lack tests.
127
+
128
+ Exit criteria:
129
+
130
+ - QA reports guide prioritization instead of only blocking merges.