@sun-asterisk/sungen 2.7.0-beta.1 → 3.0.0-beta.71
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/cli/commands/add.js +3 -3
- package/dist/cli/commands/add.js.map +1 -1
- package/dist/cli/commands/audit.d.ts +3 -0
- package/dist/cli/commands/audit.d.ts.map +1 -0
- package/dist/cli/commands/audit.js +134 -0
- package/dist/cli/commands/audit.js.map +1 -0
- package/dist/cli/commands/blindspot.d.ts +3 -0
- package/dist/cli/commands/blindspot.d.ts.map +1 -0
- package/dist/cli/commands/blindspot.js +58 -0
- package/dist/cli/commands/blindspot.js.map +1 -0
- package/dist/cli/commands/challenge.d.ts +3 -0
- package/dist/cli/commands/challenge.d.ts.map +1 -0
- package/dist/cli/commands/challenge.js +102 -0
- package/dist/cli/commands/challenge.js.map +1 -0
- package/dist/cli/commands/feedback.d.ts +3 -0
- package/dist/cli/commands/feedback.d.ts.map +1 -0
- package/dist/cli/commands/feedback.js +72 -0
- package/dist/cli/commands/feedback.js.map +1 -0
- package/dist/cli/commands/generate.d.ts.map +1 -1
- package/dist/cli/commands/generate.js +22 -0
- package/dist/cli/commands/generate.js.map +1 -1
- package/dist/cli/commands/ledger.d.ts +3 -0
- package/dist/cli/commands/ledger.d.ts.map +1 -0
- package/dist/cli/commands/ledger.js +71 -0
- package/dist/cli/commands/ledger.js.map +1 -0
- package/dist/cli/commands/manifest.d.ts +3 -0
- package/dist/cli/commands/manifest.d.ts.map +1 -0
- package/dist/cli/commands/manifest.js +101 -0
- package/dist/cli/commands/manifest.js.map +1 -0
- package/dist/cli/commands/script-check.d.ts +3 -0
- package/dist/cli/commands/script-check.d.ts.map +1 -0
- package/dist/cli/commands/script-check.js +97 -0
- package/dist/cli/commands/script-check.js.map +1 -0
- package/dist/cli/commands/trace.d.ts +3 -0
- package/dist/cli/commands/trace.d.ts.map +1 -0
- package/dist/cli/commands/trace.js +110 -0
- package/dist/cli/commands/trace.js.map +1 -0
- package/dist/cli/commands/update.d.ts.map +1 -1
- package/dist/cli/commands/update.js +22 -9
- package/dist/cli/commands/update.js.map +1 -1
- package/dist/cli/index.js +16 -0
- package/dist/cli/index.js.map +1 -1
- package/dist/generators/test-generator/adapters/playwright/templates/steps/actions/capture-variable.hbs +1 -0
- package/dist/generators/test-generator/adapters/playwright/templates/steps/assertions/all-contain-assertion.hbs +7 -0
- package/dist/generators/test-generator/patterns/capture-patterns.d.ts +16 -0
- package/dist/generators/test-generator/patterns/capture-patterns.d.ts.map +1 -0
- package/dist/generators/test-generator/patterns/capture-patterns.js +54 -0
- package/dist/generators/test-generator/patterns/capture-patterns.js.map +1 -0
- package/dist/generators/test-generator/patterns/index.d.ts.map +1 -1
- package/dist/generators/test-generator/patterns/index.js +2 -0
- package/dist/generators/test-generator/patterns/index.js.map +1 -1
- package/dist/generators/test-generator/step-mapper.d.ts.map +1 -1
- package/dist/generators/test-generator/step-mapper.js +1 -0
- package/dist/generators/test-generator/step-mapper.js.map +1 -1
- package/dist/generators/test-generator/utils/data-resolver.d.ts +5 -0
- package/dist/generators/test-generator/utils/data-resolver.d.ts.map +1 -1
- package/dist/generators/test-generator/utils/data-resolver.js +17 -0
- package/dist/generators/test-generator/utils/data-resolver.js.map +1 -1
- package/dist/harness/audit.d.ts +24 -0
- package/dist/harness/audit.d.ts.map +1 -0
- package/dist/harness/audit.js +115 -0
- package/dist/harness/audit.js.map +1 -0
- package/dist/harness/blindspot.d.ts +15 -0
- package/dist/harness/blindspot.d.ts.map +1 -0
- package/dist/harness/blindspot.js +85 -0
- package/dist/harness/blindspot.js.map +1 -0
- package/dist/harness/catalog/universal-viewpoints.yaml +114 -0
- package/dist/harness/challenge.d.ts +21 -0
- package/dist/harness/challenge.d.ts.map +1 -0
- package/dist/harness/challenge.js +151 -0
- package/dist/harness/challenge.js.map +1 -0
- package/dist/harness/feedback.d.ts +29 -0
- package/dist/harness/feedback.d.ts.map +1 -0
- package/dist/harness/feedback.js +106 -0
- package/dist/harness/feedback.js.map +1 -0
- package/dist/harness/intent.d.ts +11 -0
- package/dist/harness/intent.d.ts.map +1 -0
- package/dist/harness/intent.js +86 -0
- package/dist/harness/intent.js.map +1 -0
- package/dist/harness/ledger.d.ts +42 -0
- package/dist/harness/ledger.d.ts.map +1 -0
- package/dist/harness/ledger.js +171 -0
- package/dist/harness/ledger.js.map +1 -0
- package/dist/harness/manifest.d.ts +42 -0
- package/dist/harness/manifest.d.ts.map +1 -0
- package/dist/harness/manifest.js +209 -0
- package/dist/harness/manifest.js.map +1 -0
- package/dist/harness/parse.d.ts +22 -0
- package/dist/harness/parse.d.ts.map +1 -0
- package/dist/harness/parse.js +163 -0
- package/dist/harness/parse.js.map +1 -0
- package/dist/harness/script-check.d.ts +16 -0
- package/dist/harness/script-check.d.ts.map +1 -0
- package/dist/harness/script-check.js +169 -0
- package/dist/harness/script-check.js.map +1 -0
- package/dist/harness/secret-scan.d.ts +8 -0
- package/dist/harness/secret-scan.d.ts.map +1 -0
- package/dist/harness/secret-scan.js +88 -0
- package/dist/harness/secret-scan.js.map +1 -0
- package/dist/harness/sensors.d.ts +88 -0
- package/dist/harness/sensors.d.ts.map +1 -0
- package/dist/harness/sensors.js +232 -0
- package/dist/harness/sensors.js.map +1 -0
- package/dist/harness/trace.d.ts +31 -0
- package/dist/harness/trace.d.ts.map +1 -0
- package/dist/harness/trace.js +173 -0
- package/dist/harness/trace.js.map +1 -0
- package/dist/orchestrator/ai-rules-updater.d.ts +1 -0
- package/dist/orchestrator/ai-rules-updater.d.ts.map +1 -1
- package/dist/orchestrator/ai-rules-updater.js +55 -11
- package/dist/orchestrator/ai-rules-updater.js.map +1 -1
- package/dist/orchestrator/figma/spec-figma-renderer.d.ts +2 -2
- package/dist/orchestrator/figma/spec-figma-renderer.js +2 -2
- package/dist/orchestrator/figma/spec-figma-section-renderers.d.ts +1 -1
- package/dist/orchestrator/figma/spec-figma-section-renderers.js +1 -1
- package/dist/orchestrator/project-initializer.d.ts.map +1 -1
- package/dist/orchestrator/project-initializer.js +10 -6
- package/dist/orchestrator/project-initializer.js.map +1 -1
- package/dist/orchestrator/templates/ai-instructions/claude-agent-challenge.md +46 -0
- package/dist/orchestrator/templates/ai-instructions/claude-agent-discovery.md +32 -0
- package/dist/orchestrator/templates/ai-instructions/claude-agent-reviewer.md +37 -0
- package/dist/orchestrator/templates/ai-instructions/claude-cmd-add-flow.md +3 -3
- package/dist/orchestrator/templates/ai-instructions/claude-cmd-add-screen.md +5 -5
- package/dist/orchestrator/templates/ai-instructions/claude-cmd-create-test.md +36 -12
- package/dist/orchestrator/templates/ai-instructions/claude-cmd-design.md +12 -0
- package/dist/orchestrator/templates/ai-instructions/claude-cmd-feedback.md +36 -0
- package/dist/orchestrator/templates/ai-instructions/claude-cmd-review.md +27 -30
- package/dist/orchestrator/templates/ai-instructions/claude-cmd-run-test.md +4 -1
- package/dist/orchestrator/templates/ai-instructions/claude-config.md +1 -4
- package/dist/orchestrator/templates/ai-instructions/claude-skill-capture-mode-figma-mcp.md +82 -0
- package/dist/orchestrator/templates/ai-instructions/{github-skill-sungen-figma-source.md → claude-skill-capture-mode-figma-pat.md} +14 -48
- package/dist/orchestrator/templates/ai-instructions/claude-skill-capture-mode-live.md +60 -0
- package/dist/orchestrator/templates/ai-instructions/claude-skill-capture-mode-local.md +38 -0
- package/dist/orchestrator/templates/ai-instructions/claude-skill-capture.md +35 -0
- package/dist/orchestrator/templates/ai-instructions/claude-skill-harness-audit.md +84 -0
- package/dist/orchestrator/templates/ai-instructions/claude-skill-tc-generation.md +40 -1
- package/dist/orchestrator/templates/ai-instructions/copilot-cmd-add-flow.md +3 -3
- package/dist/orchestrator/templates/ai-instructions/copilot-cmd-add-screen.md +4 -4
- package/dist/orchestrator/templates/ai-instructions/copilot-cmd-create-test.md +18 -10
- package/dist/orchestrator/templates/ai-instructions/copilot-cmd-design.md +13 -0
- package/dist/orchestrator/templates/ai-instructions/copilot-cmd-feedback.md +24 -0
- package/dist/orchestrator/templates/ai-instructions/copilot-cmd-review.md +20 -30
- package/dist/orchestrator/templates/ai-instructions/copilot-cmd-run-test.md +2 -1
- package/dist/orchestrator/templates/ai-instructions/copilot-config.md +1 -4
- package/dist/orchestrator/templates/ai-instructions/github-skill-sungen-capture-mode-figma-mcp.md +82 -0
- package/{src/orchestrator/templates/ai-instructions/claude-skill-figma-source.md → dist/orchestrator/templates/ai-instructions/github-skill-sungen-capture-mode-figma-pat.md} +14 -48
- package/dist/orchestrator/templates/ai-instructions/github-skill-sungen-capture-mode-live.md +60 -0
- package/dist/orchestrator/templates/ai-instructions/github-skill-sungen-capture-mode-local.md +38 -0
- package/dist/orchestrator/templates/ai-instructions/github-skill-sungen-capture.md +35 -0
- package/dist/orchestrator/templates/ai-instructions/github-skill-sungen-harness-audit.md +84 -0
- package/dist/orchestrator/templates/ai-instructions/github-skill-sungen-selector-fix.md +1 -1
- package/dist/orchestrator/templates/ai-instructions/github-skill-sungen-tc-generation.md +40 -1
- package/dist/orchestrator/templates/specs-test-data.ts +9 -0
- package/dist/tools/figma/figma-auth.d.ts +5 -2
- package/dist/tools/figma/figma-auth.d.ts.map +1 -1
- package/dist/tools/figma/figma-auth.js +19 -9
- package/dist/tools/figma/figma-auth.js.map +1 -1
- package/docs/orchestration-spec.md +267 -0
- package/package.json +10 -6
- package/src/cli/commands/add.ts +3 -3
- package/src/cli/commands/audit.ts +92 -0
- package/src/cli/commands/blindspot.ts +48 -0
- package/src/cli/commands/challenge.ts +55 -0
- package/src/cli/commands/feedback.ts +65 -0
- package/src/cli/commands/generate.ts +19 -0
- package/src/cli/commands/ledger.ts +61 -0
- package/src/cli/commands/manifest.ts +55 -0
- package/src/cli/commands/script-check.ts +50 -0
- package/src/cli/commands/trace.ts +60 -0
- package/src/cli/commands/update.ts +30 -10
- package/src/cli/index.ts +16 -0
- package/src/generators/test-generator/adapters/playwright/templates/steps/actions/capture-variable.hbs +1 -0
- package/src/generators/test-generator/adapters/playwright/templates/steps/assertions/all-contain-assertion.hbs +7 -0
- package/src/generators/test-generator/patterns/capture-patterns.ts +59 -0
- package/src/generators/test-generator/patterns/index.ts +2 -0
- package/src/generators/test-generator/step-mapper.ts +1 -0
- package/src/generators/test-generator/utils/data-resolver.ts +20 -0
- package/src/harness/audit.ts +112 -0
- package/src/harness/blindspot.ts +51 -0
- package/src/harness/catalog/universal-viewpoints.yaml +114 -0
- package/src/harness/challenge.ts +131 -0
- package/src/harness/feedback.ts +84 -0
- package/src/harness/intent.ts +58 -0
- package/src/harness/ledger.ts +155 -0
- package/src/harness/manifest.ts +173 -0
- package/src/harness/parse.ts +145 -0
- package/src/harness/script-check.ts +149 -0
- package/src/harness/secret-scan.ts +51 -0
- package/src/harness/sensors.ts +279 -0
- package/src/harness/trace.ts +138 -0
- package/src/orchestrator/ai-rules-updater.ts +57 -10
- package/src/orchestrator/figma/spec-figma-renderer.ts +2 -2
- package/src/orchestrator/figma/spec-figma-section-renderers.ts +1 -1
- package/src/orchestrator/project-initializer.ts +10 -7
- package/src/orchestrator/templates/ai-instructions/claude-agent-challenge.md +46 -0
- package/src/orchestrator/templates/ai-instructions/claude-agent-discovery.md +32 -0
- package/src/orchestrator/templates/ai-instructions/claude-agent-reviewer.md +37 -0
- package/src/orchestrator/templates/ai-instructions/claude-cmd-add-flow.md +3 -3
- package/src/orchestrator/templates/ai-instructions/claude-cmd-add-screen.md +5 -5
- package/src/orchestrator/templates/ai-instructions/claude-cmd-create-test.md +36 -12
- package/src/orchestrator/templates/ai-instructions/claude-cmd-design.md +12 -0
- package/src/orchestrator/templates/ai-instructions/claude-cmd-feedback.md +36 -0
- package/src/orchestrator/templates/ai-instructions/claude-cmd-review.md +27 -30
- package/src/orchestrator/templates/ai-instructions/claude-cmd-run-test.md +4 -1
- package/src/orchestrator/templates/ai-instructions/claude-config.md +1 -4
- package/src/orchestrator/templates/ai-instructions/claude-skill-capture-mode-figma-mcp.md +82 -0
- package/{dist/orchestrator/templates/ai-instructions/copilot-skill-figma-source.md → src/orchestrator/templates/ai-instructions/claude-skill-capture-mode-figma-pat.md} +14 -48
- package/src/orchestrator/templates/ai-instructions/claude-skill-capture-mode-live.md +60 -0
- package/src/orchestrator/templates/ai-instructions/claude-skill-capture-mode-local.md +38 -0
- package/src/orchestrator/templates/ai-instructions/claude-skill-capture.md +35 -0
- package/src/orchestrator/templates/ai-instructions/claude-skill-harness-audit.md +84 -0
- package/src/orchestrator/templates/ai-instructions/claude-skill-tc-generation.md +40 -1
- package/src/orchestrator/templates/ai-instructions/copilot-cmd-add-flow.md +3 -3
- package/src/orchestrator/templates/ai-instructions/copilot-cmd-add-screen.md +4 -4
- package/src/orchestrator/templates/ai-instructions/copilot-cmd-create-test.md +18 -10
- package/src/orchestrator/templates/ai-instructions/copilot-cmd-design.md +13 -0
- package/src/orchestrator/templates/ai-instructions/copilot-cmd-feedback.md +24 -0
- package/src/orchestrator/templates/ai-instructions/copilot-cmd-review.md +20 -30
- package/src/orchestrator/templates/ai-instructions/copilot-cmd-run-test.md +2 -1
- package/src/orchestrator/templates/ai-instructions/copilot-config.md +1 -4
- package/src/orchestrator/templates/ai-instructions/github-skill-sungen-capture-mode-figma-mcp.md +82 -0
- package/{dist/orchestrator/templates/ai-instructions/claude-skill-figma-source.md → src/orchestrator/templates/ai-instructions/github-skill-sungen-capture-mode-figma-pat.md} +14 -48
- package/src/orchestrator/templates/ai-instructions/github-skill-sungen-capture-mode-live.md +60 -0
- package/src/orchestrator/templates/ai-instructions/github-skill-sungen-capture-mode-local.md +38 -0
- package/src/orchestrator/templates/ai-instructions/github-skill-sungen-capture.md +35 -0
- package/src/orchestrator/templates/ai-instructions/github-skill-sungen-harness-audit.md +84 -0
- package/src/orchestrator/templates/ai-instructions/github-skill-sungen-selector-fix.md +1 -1
- package/src/orchestrator/templates/ai-instructions/github-skill-sungen-tc-generation.md +40 -1
- package/src/orchestrator/templates/specs-test-data.ts +9 -0
- package/src/tools/figma/figma-auth.ts +20 -9
- package/dist/orchestrator/templates/ai-instructions/claude-skill-capture-figma.md +0 -142
- package/dist/orchestrator/templates/ai-instructions/claude-skill-capture-live.md +0 -112
- package/dist/orchestrator/templates/ai-instructions/claude-skill-capture-local.md +0 -73
- package/dist/orchestrator/templates/ai-instructions/github-skill-sungen-capture-figma.md +0 -142
- package/dist/orchestrator/templates/ai-instructions/github-skill-sungen-capture-live.md +0 -112
- package/dist/orchestrator/templates/ai-instructions/github-skill-sungen-capture-local.md +0 -73
- package/src/orchestrator/templates/ai-instructions/claude-skill-capture-figma.md +0 -142
- package/src/orchestrator/templates/ai-instructions/claude-skill-capture-live.md +0 -112
- package/src/orchestrator/templates/ai-instructions/claude-skill-capture-local.md +0 -73
- package/src/orchestrator/templates/ai-instructions/copilot-skill-figma-source.md +0 -151
- package/src/orchestrator/templates/ai-instructions/github-skill-sungen-capture-figma.md +0 -142
- package/src/orchestrator/templates/ai-instructions/github-skill-sungen-capture-live.md +0 -112
- package/src/orchestrator/templates/ai-instructions/github-skill-sungen-capture-local.md +0 -73
- package/src/orchestrator/templates/ai-instructions/github-skill-sungen-figma-source.md +0 -151
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
# Seed Universal Viewpoint Catalog (bundled, local — NOT a server).
|
|
2
|
+
# Role: REFERENCE for the Harness Coverage Gate (the "answer key" sensors check
|
|
3
|
+
# against). The AI still GENERATES the viewpoint-overview; this catalog only
|
|
4
|
+
# verifies that critical themes for a page-type were not missed.
|
|
5
|
+
#
|
|
6
|
+
# Each page-type lists must-cover themes. A theme is "covered" when the project's
|
|
7
|
+
# viewpoint-overview (or generated scenarios) contains one of its keywords.
|
|
8
|
+
# See docs/orchestration-spec.md §5.2 and reports/sungen_refactor_spec.md §9.
|
|
9
|
+
#
|
|
10
|
+
# `depth:` (optional, harness-roadmap P1) marks a theme as DATA-correctness:
|
|
11
|
+
# requires: data-assertion → scenarios on this theme must assert DATA (not just
|
|
12
|
+
# visibility) to count as "deep". Drives the depth gate.
|
|
13
|
+
# cross_screen: true → genuine depth needs another screen → use a flow
|
|
14
|
+
# (P5 `remember` / `see all … contain`); on one screen
|
|
15
|
+
# it should be @manual + deferred-to-flow, not shallow.
|
|
16
|
+
# keywords: → PRECISE data-noun keywords for depth matching (kept
|
|
17
|
+
# separate from coverage keywords to avoid matching
|
|
18
|
+
# navigation scenarios like "API list page").
|
|
19
|
+
# template: → the deep step the generator should emit by default.
|
|
20
|
+
# Themes with no `depth:` are visibility/navigation — landing/seeing IS the assertion.
|
|
21
|
+
|
|
22
|
+
page_types:
|
|
23
|
+
ecommerce-list:
|
|
24
|
+
detect_keywords: [cart, product, checkout, catalog, brand, category, "add to cart"]
|
|
25
|
+
must_cover:
|
|
26
|
+
- theme: list-data
|
|
27
|
+
keywords: [list, displayed, card, "product card", grid]
|
|
28
|
+
depth:
|
|
29
|
+
requires: data-assertion
|
|
30
|
+
cross_screen: false
|
|
31
|
+
keywords: ["product price", "product card", "every card", "all product", "product name", "displays a price", "card displays"]
|
|
32
|
+
template: "User see all [Product Price] contain {{currency}} (and name/image per card)"
|
|
33
|
+
- theme: product-detail-consistency
|
|
34
|
+
keywords: [consistent, consistency, match, "same product", "correct product"]
|
|
35
|
+
depth:
|
|
36
|
+
requires: data-assertion
|
|
37
|
+
cross_screen: true
|
|
38
|
+
keywords: ["product detail", "detail page", "same product name", "matching name", "consistency", "same price"]
|
|
39
|
+
template: "User remember [Product Name] text as {{v}} … User see [Detail Product Name] header with {{v}}"
|
|
40
|
+
- theme: cart-correctness
|
|
41
|
+
keywords: ["cart contains", quantity, "price", subtotal, "in cart", "cart content", "cart product"]
|
|
42
|
+
depth:
|
|
43
|
+
requires: data-assertion
|
|
44
|
+
cross_screen: true
|
|
45
|
+
keywords: ["cart product", "cart contains", "in cart", "cart item", "appears in the cart", "cart line", "subtotal", "quantity"]
|
|
46
|
+
template: "User remember [Product Name] text as {{v}} … User see all [Cart Product Name] contain {{v}}"
|
|
47
|
+
- theme: category-filter-correctness
|
|
48
|
+
keywords: ["belong to the selected category", "category result", "matching the selected category", "products belong to", "only products that belong to the selected category"]
|
|
49
|
+
depth:
|
|
50
|
+
requires: data-assertion
|
|
51
|
+
cross_screen: true
|
|
52
|
+
keywords: ["belong to the selected category", "category result", "products that belong", "matching the selected category"]
|
|
53
|
+
template: "User see all [Result Product Name] contain {{selected_category}}"
|
|
54
|
+
- theme: brand-filter-correctness
|
|
55
|
+
keywords: ["belong to the selected brand", "brand result", "only products ... brand", "products that belong to the selected brand", "all displayed products belong to"]
|
|
56
|
+
depth:
|
|
57
|
+
requires: data-assertion
|
|
58
|
+
cross_screen: true
|
|
59
|
+
keywords: ["belong to the selected brand", "brand result", "products that belong to the selected brand"]
|
|
60
|
+
template: "User see all [Result Product Name] contain {{selected_brand}}"
|
|
61
|
+
- theme: add-to-cart
|
|
62
|
+
keywords: ["add to cart", added, "added to cart"]
|
|
63
|
+
depth:
|
|
64
|
+
requires: data-assertion
|
|
65
|
+
cross_screen: false
|
|
66
|
+
keywords: ["add to cart", "added to cart", "added confirmation", "added message"]
|
|
67
|
+
template: "User see [Added Message] text contains {{added_message}} (not just the modal)"
|
|
68
|
+
|
|
69
|
+
form:
|
|
70
|
+
detect_keywords: [form, submit, field, input, validation]
|
|
71
|
+
must_cover:
|
|
72
|
+
- theme: required-validation
|
|
73
|
+
keywords: [required, empty, "must be", validation]
|
|
74
|
+
depth:
|
|
75
|
+
requires: data-assertion
|
|
76
|
+
cross_screen: false
|
|
77
|
+
keywords: ["required", "must be", "error message", "validation message"]
|
|
78
|
+
template: "User see [Field Error] message with {{error_text}}"
|
|
79
|
+
- theme: format-boundary
|
|
80
|
+
keywords: [format, invalid, boundary, length, range]
|
|
81
|
+
depth:
|
|
82
|
+
requires: data-assertion
|
|
83
|
+
cross_screen: false
|
|
84
|
+
keywords: ["invalid format", "boundary", "max length", "min length", "out of range"]
|
|
85
|
+
template: "User see [Field Error] message with {{error_text}}"
|
|
86
|
+
- theme: submit-success
|
|
87
|
+
keywords: [submit, success, saved, created]
|
|
88
|
+
depth:
|
|
89
|
+
requires: data-assertion
|
|
90
|
+
cross_screen: false
|
|
91
|
+
keywords: ["success message", "saved", "created", "confirmation"]
|
|
92
|
+
template: "User see [Success] message with {{success_text}}"
|
|
93
|
+
|
|
94
|
+
auth:
|
|
95
|
+
detect_keywords: [login, logout, password, signin, "sign in", credential]
|
|
96
|
+
must_cover:
|
|
97
|
+
- theme: valid-login
|
|
98
|
+
keywords: ["valid", login, success]
|
|
99
|
+
- theme: invalid-credential
|
|
100
|
+
keywords: ["invalid", "wrong password", error, incorrect]
|
|
101
|
+
depth:
|
|
102
|
+
requires: data-assertion
|
|
103
|
+
cross_screen: false
|
|
104
|
+
keywords: ["wrong password", "incorrect", "invalid credential", "error message"]
|
|
105
|
+
template: "User see [Login Error] message with {{error_text}}"
|
|
106
|
+
- theme: access-control
|
|
107
|
+
keywords: ["unauthorized", "redirect", "not logged in", permission]
|
|
108
|
+
|
|
109
|
+
# Universal themes worth checking on ANY page-type (low-weight reminders).
|
|
110
|
+
universal:
|
|
111
|
+
- theme: error-empty-state
|
|
112
|
+
keywords: ["empty", "no data", "no result", "failed", "error state"]
|
|
113
|
+
- theme: accessibility
|
|
114
|
+
keywords: ["keyboard", "tab order", "accessible", "aria", "focus"]
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Challenge / Exploration Harness (harness-roadmap P4) — Loop 2.
|
|
3
|
+
*
|
|
4
|
+
* Production mode (Loop 1: create-test → audit gate → repair) is deterministic by
|
|
5
|
+
* design: same spec + viewpoint → same official suite. That is the *feature* for
|
|
6
|
+
* delivery/CI, but it can feel like "a machine that always outputs the same thing".
|
|
7
|
+
*
|
|
8
|
+
* The Challenge Harness is the antidote: it does NOT regenerate the suite — it
|
|
9
|
+
* ATTACKS the existing one to surface what production missed. It is advisory
|
|
10
|
+
* (never auto-merges) and read-only.
|
|
11
|
+
*
|
|
12
|
+
* This module is the DETERMINISTIC spine — three structural critics:
|
|
13
|
+
* 1. Coverage — over-covered (low-value) areas + shallow gate themes.
|
|
14
|
+
* 2. Depth — titles that claim a collection but assert a single element.
|
|
15
|
+
* 3. Novelty — risk-based prompts the AI `sungen-challenge` agent expands into
|
|
16
|
+
* concrete novelty candidates (semantic — not deterministic here).
|
|
17
|
+
*
|
|
18
|
+
* The AI agent layer adds the semantic + novelty judgement on top of this spine.
|
|
19
|
+
*/
|
|
20
|
+
import * as path from 'path';
|
|
21
|
+
import { loadScenarios, ScenarioInfo } from './parse';
|
|
22
|
+
import { runAudit } from './audit';
|
|
23
|
+
|
|
24
|
+
export interface ChallengeFinding {
|
|
25
|
+
scenario?: string;
|
|
26
|
+
issue: string;
|
|
27
|
+
suggestion: string;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export interface ChallengeReport {
|
|
31
|
+
screen: string;
|
|
32
|
+
// Coverage critic
|
|
33
|
+
overCovered: { bucket: string; count: number; note: string }[];
|
|
34
|
+
shallowThemes: string[];
|
|
35
|
+
// Depth critic
|
|
36
|
+
collectionClaimSingular: ChallengeFinding[];
|
|
37
|
+
// Novelty critic (deterministic prompts → AI agent fills candidates)
|
|
38
|
+
noveltyPrompts: string[];
|
|
39
|
+
// Roll-up
|
|
40
|
+
explorationReadiness: string[];
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
// A collection claim = a PLURAL set noun (or an explicit quantifier) used with a
|
|
44
|
+
// DISPLAY verb. Plural-only avoids false-positives on single-item actions like
|
|
45
|
+
// "Adding A product ... shows the dialog" (which correctly asserts one item).
|
|
46
|
+
const PLURAL_NOUN = /\b(cards|items|products|rows|results|prices|entries|records)\b/i;
|
|
47
|
+
const QUANTIFIER = /\b(all|every|each)\b/i;
|
|
48
|
+
const DISPLAY_VERB = /\b(displays?|shows?|lists?|grid|contains?)\b/i;
|
|
49
|
+
|
|
50
|
+
/** Risk lenses the Novelty critic prompts the AI to explore (beyond the catalog). */
|
|
51
|
+
const NOVELTY_LENSES = [
|
|
52
|
+
'double-submit / rapid repeat of the primary action (duplicate side-effects?)',
|
|
53
|
+
'state after partial / slow load (assert against a not-yet-ready page)',
|
|
54
|
+
'boundary & unusual data (very long text, 0 / max quantity, special chars)',
|
|
55
|
+
'concurrency / back-button / refresh mid-flow',
|
|
56
|
+
'historical-incident mindset — what has broken on similar screens before?',
|
|
57
|
+
];
|
|
58
|
+
|
|
59
|
+
export function buildChallenge(screenDir: string, screenName: string): ChallengeReport {
|
|
60
|
+
const featurePath = path.join(screenDir, 'features', `${screenName}.feature`);
|
|
61
|
+
const scenarios: ScenarioInfo[] = loadScenarios(featurePath);
|
|
62
|
+
const audit = runAudit(screenDir, screenName);
|
|
63
|
+
|
|
64
|
+
// 1. Coverage critic — over-covered buckets (secondary >> business-core) + shallow themes.
|
|
65
|
+
const buckets = audit.balance.byBucket;
|
|
66
|
+
const core = buckets['business-core'] || 0;
|
|
67
|
+
const overCovered: ChallengeReport['overCovered'] = [];
|
|
68
|
+
for (const [bucket, count] of Object.entries(buckets)) {
|
|
69
|
+
if (['business-core', 'other'].includes(bucket)) continue;
|
|
70
|
+
if (core > 0 && count > core * 1.5) {
|
|
71
|
+
overCovered.push({ bucket, count, note: `${count} scenarios vs ${core} business-core — likely low-value expansion; trim toward correctness.` });
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
const shallowThemes = audit.gate.gaps.filter((g) => g.status === 'shallow').map((g) => g.theme);
|
|
75
|
+
|
|
76
|
+
// 2. Depth critic — title claims a collection but the assertion is singular (no "see all").
|
|
77
|
+
const collectionClaimSingular: ChallengeFinding[] = [];
|
|
78
|
+
for (const s of scenarios) {
|
|
79
|
+
if (s.manual || s.category === 'NAV') continue;
|
|
80
|
+
const claimsCollection = (PLURAL_NOUN.test(s.name) || QUANTIFIER.test(s.name)) && DISPLAY_VERB.test(s.name);
|
|
81
|
+
const assertsAll = /\bsee all\b/.test(s.haystack);
|
|
82
|
+
if (claimsCollection && !assertsAll) {
|
|
83
|
+
collectionClaimSingular.push({
|
|
84
|
+
scenario: s.name,
|
|
85
|
+
issue: 'Title implies a set (cards/items/all) but the assertion targets a single element.',
|
|
86
|
+
suggestion: 'Prove EVERY member: `Then User see all [<Card/Row>] contain {{...}}` instead of a single `see [X]`.',
|
|
87
|
+
});
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
// 3. Novelty critic — deterministic prompts; the AI agent expands these into candidates.
|
|
92
|
+
const noveltyPrompts = NOVELTY_LENSES.map((l) => `Find 1 non-obvious, valuable scenario via: ${l}`);
|
|
93
|
+
|
|
94
|
+
// Roll-up — exploration readiness signals (not a fake score).
|
|
95
|
+
const explorationReadiness: string[] = [];
|
|
96
|
+
if (collectionClaimSingular.length) explorationReadiness.push(`${collectionClaimSingular.length} title↔assertion gap(s) — deterministic depth critic flagged these; an AI Business-Depth critic should confirm + fix.`);
|
|
97
|
+
if (overCovered.length) explorationReadiness.push(`${overCovered.length} possibly over-covered area(s) — rebalance toward correctness.`);
|
|
98
|
+
if (shallowThemes.length) explorationReadiness.push(`Shallow themes: ${shallowThemes.join(', ')}.`);
|
|
99
|
+
explorationReadiness.push('Novelty candidates are NOT generated deterministically — run the `sungen-challenge` agent (Claude) or its inline criteria (Copilot) to propose them, then QA accept/reject (≤20% of official, no auto-merge).');
|
|
100
|
+
|
|
101
|
+
return { screen: screenName, overCovered, shallowThemes, collectionClaimSingular, noveltyPrompts, explorationReadiness };
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/** Render the Challenge Report as Markdown (advisory — not part of the official suite). */
|
|
105
|
+
export function renderChallengeMarkdown(r: ChallengeReport): string {
|
|
106
|
+
const lines: string[] = [];
|
|
107
|
+
lines.push(`# Challenge Report — ${r.screen}`, '');
|
|
108
|
+
lines.push('> Advisory (Loop 2 / exploration mode). Does NOT change the official suite — it attacks it to surface blind spots. QA decides what to adopt.', '');
|
|
109
|
+
|
|
110
|
+
lines.push('## Depth — title claims a collection but asserts a single element');
|
|
111
|
+
if (r.collectionClaimSingular.length) {
|
|
112
|
+
lines.push('| Scenario | Issue | Suggested |', '|---|---|---|');
|
|
113
|
+
for (const f of r.collectionClaimSingular) lines.push(`| ${f.scenario} | ${f.issue} | ${f.suggestion} |`);
|
|
114
|
+
} else lines.push('_none_');
|
|
115
|
+
lines.push('');
|
|
116
|
+
|
|
117
|
+
lines.push('## Coverage — possibly over-covered / shallow');
|
|
118
|
+
if (r.overCovered.length) for (const o of r.overCovered) lines.push(`- **${o.bucket}** — ${o.note}`);
|
|
119
|
+
if (r.shallowThemes.length) lines.push(`- Shallow themes: ${r.shallowThemes.join(', ')}`);
|
|
120
|
+
if (!r.overCovered.length && !r.shallowThemes.length) lines.push('_balanced_');
|
|
121
|
+
lines.push('');
|
|
122
|
+
|
|
123
|
+
lines.push('## Novelty — prompts for the AI critic (expand into candidates, ≤20% of official, no auto-merge)');
|
|
124
|
+
for (const p of r.noveltyPrompts) lines.push(`- ${p}`);
|
|
125
|
+
lines.push('');
|
|
126
|
+
|
|
127
|
+
lines.push('## Exploration readiness');
|
|
128
|
+
for (const e of r.explorationReadiness) lines.push(`- ${e}`);
|
|
129
|
+
lines.push('');
|
|
130
|
+
return lines.join('\n');
|
|
131
|
+
}
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Feedback — local-first capture of QA feedback. The most valuable use of
|
|
3
|
+
* feedback is closing the learning loop WITHIN a project (feed reuse/regenerate
|
|
4
|
+
* + improve the Guide), which needs no server. A future opt-in `feedback sync`
|
|
5
|
+
* can push anonymized metadata to a central API (Nấc 2) — not implemented here.
|
|
6
|
+
*
|
|
7
|
+
* Two purposes are kept distinct:
|
|
8
|
+
* - test-design: a viewpoint/scenario is wrong / missing / duplicate → knowledge
|
|
9
|
+
* - product: Sungen itself did X wrong → telemetry/issue
|
|
10
|
+
*
|
|
11
|
+
* Storage: .sungen/feedback/feedback.jsonl (append-only)
|
|
12
|
+
*/
|
|
13
|
+
import * as fs from 'fs';
|
|
14
|
+
import * as path from 'path';
|
|
15
|
+
|
|
16
|
+
export type FeedbackType = 'test-design' | 'product' | 'other';
|
|
17
|
+
export type FeedbackDecision = 'accept' | 'reject' | 'edit' | 'add' | 'none';
|
|
18
|
+
|
|
19
|
+
export interface FeedbackEntry {
|
|
20
|
+
ts: string;
|
|
21
|
+
type: FeedbackType;
|
|
22
|
+
screen?: string;
|
|
23
|
+
target?: string; // viewpoint id / scenario / command / artifact the feedback is about
|
|
24
|
+
decision?: FeedbackDecision;
|
|
25
|
+
message: string;
|
|
26
|
+
reason?: string;
|
|
27
|
+
source: string; // who (default: qa)
|
|
28
|
+
auditScore?: number; // snapshot of current audit score if available
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
function feedbackPath(): string {
|
|
32
|
+
return path.join(process.cwd(), '.sungen', 'feedback', 'feedback.jsonl');
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
function currentAuditScore(screen?: string): number | undefined {
|
|
36
|
+
if (!screen) return undefined;
|
|
37
|
+
const p = path.join(process.cwd(), '.sungen', 'reports', `${screen}-audit.json`);
|
|
38
|
+
if (!fs.existsSync(p)) return undefined;
|
|
39
|
+
try { return JSON.parse(fs.readFileSync(p, 'utf-8'))?.score?.overall; } catch { return undefined; }
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export function recordFeedback(entry: Omit<FeedbackEntry, 'ts' | 'source' | 'auditScore'> & { ts?: string; source?: string }): string {
|
|
43
|
+
const p = feedbackPath();
|
|
44
|
+
fs.mkdirSync(path.dirname(p), { recursive: true });
|
|
45
|
+
const full: FeedbackEntry = {
|
|
46
|
+
ts: entry.ts ?? new Date().toISOString(),
|
|
47
|
+
type: entry.type,
|
|
48
|
+
screen: entry.screen,
|
|
49
|
+
target: entry.target,
|
|
50
|
+
decision: entry.decision ?? 'none',
|
|
51
|
+
message: entry.message,
|
|
52
|
+
reason: entry.reason,
|
|
53
|
+
source: entry.source ?? 'qa',
|
|
54
|
+
auditScore: currentAuditScore(entry.screen),
|
|
55
|
+
};
|
|
56
|
+
fs.appendFileSync(p, JSON.stringify(full) + '\n', 'utf-8');
|
|
57
|
+
return p;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
export function readFeedback(): FeedbackEntry[] {
|
|
61
|
+
const p = feedbackPath();
|
|
62
|
+
if (!fs.existsSync(p)) return [];
|
|
63
|
+
return fs.readFileSync(p, 'utf-8').split('\n').filter(Boolean).map((l) => JSON.parse(l));
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
export interface FeedbackSummary {
|
|
67
|
+
total: number;
|
|
68
|
+
byType: Record<string, number>;
|
|
69
|
+
byDecision: Record<string, number>;
|
|
70
|
+
entries: FeedbackEntry[];
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
export function summarize(filter?: { screen?: string; type?: FeedbackType }): FeedbackSummary {
|
|
74
|
+
let entries = readFeedback();
|
|
75
|
+
if (filter?.screen) entries = entries.filter((e) => e.screen === filter.screen);
|
|
76
|
+
if (filter?.type) entries = entries.filter((e) => e.type === filter.type);
|
|
77
|
+
const byType: Record<string, number> = {};
|
|
78
|
+
const byDecision: Record<string, number> = {};
|
|
79
|
+
for (const e of entries) {
|
|
80
|
+
byType[e.type] = (byType[e.type] || 0) + 1;
|
|
81
|
+
byDecision[e.decision || 'none'] = (byDecision[e.decision || 'none'] || 0) + 1;
|
|
82
|
+
}
|
|
83
|
+
return { total: entries.length, byType, byDecision, entries };
|
|
84
|
+
}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Intent Profile (harness-roadmap P3) — lets the gate flex to project intent.
|
|
3
|
+
*
|
|
4
|
+
* Read from qa/context.md so a project can declare what it cares about:
|
|
5
|
+
*
|
|
6
|
+
* ## Intent
|
|
7
|
+
* focus: security # functional | e-commerce | security | smoke
|
|
8
|
+
* risk_tier: high # high | normal | low (reserved — future weighting)
|
|
9
|
+
* tier_scope: full # tier-1 | full (reserved)
|
|
10
|
+
*
|
|
11
|
+
* Keys may appear anywhere in context.md (a heading is optional). Unknown / missing
|
|
12
|
+
* values fall back to the safe default (functional), so behaviour is unchanged for
|
|
13
|
+
* projects that haven't declared an intent.
|
|
14
|
+
*/
|
|
15
|
+
import * as fs from 'fs';
|
|
16
|
+
import * as path from 'path';
|
|
17
|
+
|
|
18
|
+
export type IntentFocus = 'functional' | 'e-commerce' | 'security' | 'smoke';
|
|
19
|
+
|
|
20
|
+
export interface IntentProfile {
|
|
21
|
+
focus: IntentFocus;
|
|
22
|
+
riskTier: 'high' | 'normal' | 'low';
|
|
23
|
+
tierScope: 'tier-1' | 'full';
|
|
24
|
+
source: 'context.md' | 'default';
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
const DEFAULT_INTENT: IntentProfile = {
|
|
28
|
+
focus: 'functional', riskTier: 'normal', tierScope: 'full', source: 'default',
|
|
29
|
+
};
|
|
30
|
+
|
|
31
|
+
const FOCI: IntentFocus[] = ['functional', 'e-commerce', 'security', 'smoke'];
|
|
32
|
+
|
|
33
|
+
/** Resolve project root from a screen/flow dir (…/qa/screens/<name>). */
|
|
34
|
+
export function projectRootFromScreenDir(screenDir: string): string {
|
|
35
|
+
return path.resolve(screenDir, '..', '..', '..');
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export function readIntent(projectRoot: string): IntentProfile {
|
|
39
|
+
const ctx = path.join(projectRoot, 'qa', 'context.md');
|
|
40
|
+
if (!fs.existsSync(ctx)) return DEFAULT_INTENT;
|
|
41
|
+
let text: string;
|
|
42
|
+
try { text = fs.readFileSync(ctx, 'utf-8').toLowerCase(); } catch { return DEFAULT_INTENT; }
|
|
43
|
+
|
|
44
|
+
const grab = (key: string): string | undefined => {
|
|
45
|
+
const m = text.match(new RegExp(`(?:^|\\n)\\s*${key}\\s*:\\s*([a-z0-9-]+)`));
|
|
46
|
+
return m?.[1];
|
|
47
|
+
};
|
|
48
|
+
|
|
49
|
+
const focusRaw = grab('focus');
|
|
50
|
+
const focus = (FOCI.includes(focusRaw as IntentFocus) ? focusRaw : DEFAULT_INTENT.focus) as IntentFocus;
|
|
51
|
+
const risk = grab('risk_tier');
|
|
52
|
+
const riskTier = (['high', 'normal', 'low'].includes(risk as string) ? risk : DEFAULT_INTENT.riskTier) as IntentProfile['riskTier'];
|
|
53
|
+
const scope = grab('tier_scope');
|
|
54
|
+
const tierScope = (['tier-1', 'full'].includes(scope as string) ? scope : DEFAULT_INTENT.tierScope) as IntentProfile['tierScope'];
|
|
55
|
+
|
|
56
|
+
const found = focusRaw || risk || scope;
|
|
57
|
+
return { focus, riskTier, tierScope, source: found ? 'context.md' : 'default' };
|
|
58
|
+
}
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Usage Ledger — observability for AI resource spend during test design.
|
|
3
|
+
*
|
|
4
|
+
* The orchestrator records one event per step; the report aggregates them and
|
|
5
|
+
* derives efficiency metrics so you can SEE where cost goes (e.g. "50% of tokens
|
|
6
|
+
* spent in repair" → first-pass generation is weak; fix the Guide, not the count).
|
|
7
|
+
*
|
|
8
|
+
* Tokens are OPTIONAL: exact counts come from the orchestrator/harness when known;
|
|
9
|
+
* otherwise the ledger still reports time, repair rounds, and per-step structure.
|
|
10
|
+
* (Exact AI token capture is a harness-integration concern; the schema is ready.)
|
|
11
|
+
*
|
|
12
|
+
* Events: .sungen/ledger/<screen>.jsonl (append-only)
|
|
13
|
+
*/
|
|
14
|
+
import * as fs from 'fs';
|
|
15
|
+
import * as path from 'path';
|
|
16
|
+
|
|
17
|
+
export interface LedgerEvent {
|
|
18
|
+
ts: string;
|
|
19
|
+
step: string; // e.g. discovery | viewpoint | gherkin | audit | repair:1
|
|
20
|
+
runId?: string; // groups events of ONE create-test invocation (P2)
|
|
21
|
+
model?: string;
|
|
22
|
+
tokensIn?: number;
|
|
23
|
+
tokensOut?: number;
|
|
24
|
+
ms?: number;
|
|
25
|
+
note?: string;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
// The ledger is append-only across runs. When events carry no runId, a gap longer
|
|
29
|
+
// than this between consecutive timestamps marks a new run. Default report/trace
|
|
30
|
+
// scope to the LATEST run so multiple invocations don't get conflated.
|
|
31
|
+
const RUN_GAP_MS = 30 * 60 * 1000;
|
|
32
|
+
|
|
33
|
+
/** Split events into runs — by runId when present, else by timestamp gap. */
|
|
34
|
+
export function segmentRuns(events: LedgerEvent[]): LedgerEvent[][] {
|
|
35
|
+
if (events.length === 0) return [];
|
|
36
|
+
if (events.some((e) => e.runId)) {
|
|
37
|
+
const order: string[] = [];
|
|
38
|
+
const groups = new Map<string, LedgerEvent[]>();
|
|
39
|
+
for (const e of events) {
|
|
40
|
+
const key = e.runId ?? '__none__';
|
|
41
|
+
if (!groups.has(key)) { groups.set(key, []); order.push(key); }
|
|
42
|
+
groups.get(key)!.push(e);
|
|
43
|
+
}
|
|
44
|
+
return order.map((k) => groups.get(k)!);
|
|
45
|
+
}
|
|
46
|
+
const sorted = [...events].sort((a, b) => Date.parse(a.ts) - Date.parse(b.ts));
|
|
47
|
+
const runs: LedgerEvent[][] = [[sorted[0]]];
|
|
48
|
+
for (let i = 1; i < sorted.length; i++) {
|
|
49
|
+
const gap = Date.parse(sorted[i].ts) - Date.parse(sorted[i - 1].ts);
|
|
50
|
+
if (gap > RUN_GAP_MS) runs.push([]);
|
|
51
|
+
runs[runs.length - 1].push(sorted[i]);
|
|
52
|
+
}
|
|
53
|
+
return runs;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/** Events of the most recent run only. */
|
|
57
|
+
export function latestRunEvents(events: LedgerEvent[]): LedgerEvent[] {
|
|
58
|
+
const runs = segmentRuns(events);
|
|
59
|
+
return runs.length ? runs[runs.length - 1] : [];
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
function ledgerPath(screen: string): string {
|
|
63
|
+
return path.join(process.cwd(), '.sungen', 'ledger', `${screen}.jsonl`);
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
export function recordEvent(screen: string, ev: Omit<LedgerEvent, 'ts'> & { ts?: string }): string {
|
|
67
|
+
const p = ledgerPath(screen);
|
|
68
|
+
fs.mkdirSync(path.dirname(p), { recursive: true });
|
|
69
|
+
const full: LedgerEvent = { ts: ev.ts ?? new Date().toISOString(), ...ev };
|
|
70
|
+
fs.appendFileSync(p, JSON.stringify(full) + '\n', 'utf-8');
|
|
71
|
+
return p;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
export function readEvents(screen: string): LedgerEvent[] {
|
|
75
|
+
const p = ledgerPath(screen);
|
|
76
|
+
if (!fs.existsSync(p)) return [];
|
|
77
|
+
return fs.readFileSync(p, 'utf-8').split('\n').filter(Boolean).map((l) => JSON.parse(l));
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
export interface LedgerReport {
|
|
81
|
+
screen: string;
|
|
82
|
+
runs: number; // total runs detected in the ledger
|
|
83
|
+
runScope: 'latest' | 'all'; // which runs this report covers
|
|
84
|
+
events: number;
|
|
85
|
+
totalTokens: number;
|
|
86
|
+
totalMs: number;
|
|
87
|
+
byStep: Record<string, { tokens: number; ms: number; count: number }>;
|
|
88
|
+
repairRounds: number;
|
|
89
|
+
repairTokenPct: number; // share of tokens spent in repair (0..1)
|
|
90
|
+
coveredCriticalViewpoints: number | null;
|
|
91
|
+
tokensPerCoveredCritical: number | null;
|
|
92
|
+
scenarioCount: number | null;
|
|
93
|
+
tokensPerScenario: number | null;
|
|
94
|
+
verdicts: string[];
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
export function buildReport(screen: string, opts: { allRuns?: boolean } = {}): LedgerReport {
|
|
98
|
+
const allEvents = readEvents(screen);
|
|
99
|
+
const runCount = segmentRuns(allEvents).length;
|
|
100
|
+
const events = opts.allRuns ? allEvents : latestRunEvents(allEvents);
|
|
101
|
+
const byStep: LedgerReport['byStep'] = {};
|
|
102
|
+
let totalTokens = 0, totalMs = 0, repairTokens = 0;
|
|
103
|
+
const repairSteps = new Set<string>();
|
|
104
|
+
|
|
105
|
+
for (const e of events) {
|
|
106
|
+
const tok = (e.tokensIn ?? 0) + (e.tokensOut ?? 0);
|
|
107
|
+
const stepKey = e.step.replace(/:\d+$/, '');
|
|
108
|
+
byStep[stepKey] = byStep[stepKey] || { tokens: 0, ms: 0, count: 0 };
|
|
109
|
+
byStep[stepKey].tokens += tok;
|
|
110
|
+
byStep[stepKey].ms += e.ms ?? 0;
|
|
111
|
+
byStep[stepKey].count += 1;
|
|
112
|
+
totalTokens += tok;
|
|
113
|
+
totalMs += e.ms ?? 0;
|
|
114
|
+
if (/^repair/i.test(e.step)) { repairTokens += tok; repairSteps.add(e.step); }
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
// Pull audit signals if present
|
|
118
|
+
let coveredCritical: number | null = null;
|
|
119
|
+
let scenarioCount: number | null = null;
|
|
120
|
+
const auditPath = path.join(process.cwd(), '.sungen', 'reports', `${screen}-audit.json`);
|
|
121
|
+
if (fs.existsSync(auditPath)) {
|
|
122
|
+
try {
|
|
123
|
+
const a = JSON.parse(fs.readFileSync(auditPath, 'utf-8'));
|
|
124
|
+
coveredCritical = a?.gate?.themesCovered ?? null;
|
|
125
|
+
scenarioCount = a?.scenarioCount ?? null;
|
|
126
|
+
} catch { /* ignore */ }
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
const verdicts: string[] = [];
|
|
130
|
+
const repairPct = totalTokens ? repairTokens / totalTokens : 0;
|
|
131
|
+
if (totalTokens === 0) verdicts.push('No token data recorded — record steps with --tokens-in/--tokens-out for cost metrics (time & rounds still shown).');
|
|
132
|
+
if (repairPct > 0.4) verdicts.push(`High repair cost (${(repairPct * 100).toFixed(0)}% of tokens in repair) → first-pass generation is weak; improve the Guide/viewpoint step, not the count.`);
|
|
133
|
+
if (repairSteps.size >= 3) verdicts.push(`${repairSteps.size} repair rounds — near/over budget; check which sensor keeps failing.`);
|
|
134
|
+
if (coveredCritical != null && totalTokens && coveredCritical > 0 && totalTokens / coveredCritical > 30000) {
|
|
135
|
+
verdicts.push(`~${Math.round(totalTokens / coveredCritical / 1000)}k tokens per covered critical viewpoint — expensive; check for low-value expansion.`);
|
|
136
|
+
}
|
|
137
|
+
if (verdicts.length === 0) verdicts.push('Within expected efficiency.');
|
|
138
|
+
|
|
139
|
+
return {
|
|
140
|
+
screen,
|
|
141
|
+
runs: runCount,
|
|
142
|
+
runScope: opts.allRuns ? 'all' : 'latest',
|
|
143
|
+
events: events.length,
|
|
144
|
+
totalTokens,
|
|
145
|
+
totalMs,
|
|
146
|
+
byStep,
|
|
147
|
+
repairRounds: repairSteps.size,
|
|
148
|
+
repairTokenPct: Math.round(repairPct * 100) / 100,
|
|
149
|
+
coveredCriticalViewpoints: coveredCritical,
|
|
150
|
+
tokensPerCoveredCritical: coveredCritical && totalTokens ? Math.round(totalTokens / coveredCritical) : null,
|
|
151
|
+
scenarioCount,
|
|
152
|
+
tokensPerScenario: scenarioCount && totalTokens ? Math.round(totalTokens / scenarioCount) : null,
|
|
153
|
+
verdicts,
|
|
154
|
+
};
|
|
155
|
+
}
|