@agjs/tsforge 0.1.19 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (122) hide show
  1. package/package.json +6 -2
  2. package/scripts/browser-check.ts +41 -5
  3. package/scripts/build-rules-md.ts +78 -21
  4. package/scripts/cli-metrics.ts +10 -0
  5. package/scripts/sweep.ts +53 -23
  6. package/scripts/web-sweep.ts +292 -0
  7. package/src/browser/index.ts +3 -0
  8. package/src/browser/oracle.ts +215 -8
  9. package/src/cli.ts +22 -4
  10. package/src/config/index.ts +8 -0
  11. package/src/config/profiles.ts +150 -0
  12. package/src/config/tsforge-config.ts +64 -5
  13. package/src/detect-gate.ts +144 -13
  14. package/src/eval/eval.types.ts +9 -0
  15. package/src/eval/failure-class.ts +263 -0
  16. package/src/eval/index.ts +8 -0
  17. package/src/eval/metrics.ts +7 -0
  18. package/src/eval/parse-log.ts +105 -0
  19. package/src/eval/report.ts +19 -0
  20. package/src/eval/score.ts +10 -0
  21. package/src/loop/feedback/meta-rule-docs.ts +48 -0
  22. package/src/loop/feedback/rule-docs.ts +150 -0
  23. package/src/loop/loop.types.ts +4 -0
  24. package/src/loop/rule-docs.generated.json +131 -1
  25. package/src/loop/ttsr-defaults.ts +175 -4
  26. package/src/loop/turn.ts +3 -0
  27. package/src/meta-rules/registry.ts +32 -0
  28. package/src/meta-rules/rules/ci/no-github-context-in-shell.ts +40 -0
  29. package/src/meta-rules/rules/ci/no-pull-request-target-untrusted-checkout.ts +42 -0
  30. package/src/meta-rules/rules/ci/workflow-permissions-explicit.ts +49 -0
  31. package/src/meta-rules/rules/ci/workflow-permissions-least-privilege.ts +44 -0
  32. package/src/meta-rules/rules/config/next-image-remote-patterns-no-wildcards.ts +77 -0
  33. package/src/meta-rules/rules/config/next-instrumentation-present.ts +66 -0
  34. package/src/meta-rules/rules/config/next-proxy-over-middleware.ts +64 -0
  35. package/src/meta-rules/rules/config/tsconfig-recommended-flags.ts +75 -0
  36. package/src/meta-rules/rules/supply-chain/dependency-overrides-require-comment.ts +61 -0
  37. package/src/meta-rules/rules/supply-chain/fastify-security-plugins.ts +54 -0
  38. package/src/meta-rules/rules/supply-chain/lockfile-required.ts +51 -0
  39. package/src/meta-rules/rules/supply-chain/migrations-must-be-checked-in.ts +49 -0
  40. package/src/meta-rules/rules/supply-chain/no-git-or-tarball-dependencies.ts +70 -0
  41. package/src/meta-rules/rules/supply-chain/package-manager-field-required.ts +31 -0
  42. package/src/meta-rules/rules/supply-chain/production-must-not-use-drizzle-push.ts +75 -0
  43. package/src/meta-rules/rules/supply-chain/single-package-manager.ts +30 -0
  44. package/src/meta-rules/utils/lockfiles.ts +105 -0
  45. package/src/meta-rules/utils/workflow-yaml.ts +86 -0
  46. package/src/rule-packs/authorization/index.ts +26 -0
  47. package/src/rule-packs/authorization/rules/id-param-requires-object-authz.ts +87 -0
  48. package/src/rule-packs/authorization/rules/mutating-route-requires-authz.ts +116 -0
  49. package/src/rule-packs/authorization/rules/server-action-requires-authz.ts +101 -0
  50. package/src/rule-packs/authorization/utils.ts +285 -0
  51. package/src/rule-packs/boundary-utils.ts +13 -0
  52. package/src/rule-packs/code-flow/index.ts +4 -1
  53. package/src/rule-packs/code-flow/rules/no-throw-literal.ts +67 -0
  54. package/src/rule-packs/drizzle/index.ts +7 -0
  55. package/src/rule-packs/drizzle/rules/update-delete-account-scoped-must-filter-scope.ts +106 -0
  56. package/src/rule-packs/drizzle/rules/update-delete-must-have-where.ts +73 -0
  57. package/src/rule-packs/drizzle/utils.ts +133 -1
  58. package/src/rule-packs/fastify/index.ts +38 -0
  59. package/src/rule-packs/fastify/rules/error-handler-must-set-status.ts +78 -0
  60. package/src/rule-packs/fastify/rules/prefer-return-over-reply-send.ts +104 -0
  61. package/src/rule-packs/fastify/rules/require-fp-for-shared-plugins.ts +106 -0
  62. package/src/rule-packs/fastify/rules/require-plugin-name.ts +54 -0
  63. package/src/rule-packs/fastify/rules/require-response-schema.ts +62 -0
  64. package/src/rule-packs/fastify/rules/require-route-schema.ts +104 -0
  65. package/src/rule-packs/fastify/rules/test-inject-must-close-app.ts +44 -0
  66. package/src/rule-packs/fastify/utils/fastifyChain.ts +231 -0
  67. package/src/rule-packs/index.ts +10 -0
  68. package/src/rule-packs/jwt-cookies/index.ts +10 -0
  69. package/src/rule-packs/jwt-cookies/rules/auth-cookie-must-set-maxage-or-expires.ts +132 -0
  70. package/src/rule-packs/jwt-cookies/rules/auth-cookie-must-set-samesite.ts +151 -0
  71. package/src/rule-packs/jwt-cookies/rules/jwt-must-verify-not-decode.ts +124 -0
  72. package/src/rule-packs/module-boundaries/index.ts +3 -0
  73. package/src/rule-packs/module-boundaries/rules/no-react-in-services.ts +111 -0
  74. package/src/rule-packs/nextjs/index.ts +32 -0
  75. package/src/rule-packs/nextjs/rules/await-dynamic-request-apis.ts +65 -0
  76. package/src/rule-packs/nextjs/rules/error-boundary-require-use-client.ts +38 -0
  77. package/src/rule-packs/nextjs/rules/mutation-should-revalidate-cache.ts +152 -0
  78. package/src/rule-packs/nextjs/rules/no-html-img-element.ts +45 -0
  79. package/src/rule-packs/nextjs/rules/no-internal-api-fetch.ts +126 -0
  80. package/src/rule-packs/nextjs/rules/no-secret-props-to-client.ts +118 -0
  81. package/src/rule-packs/nextjs/rules/no-sensitive-next-public-env.ts +72 -0
  82. package/src/rule-packs/nextjs/rules/prefer-lazy-use-state-init.ts +85 -0
  83. package/src/rule-packs/nextjs/rules/server-action-requires-authz-and-validation.ts +178 -0
  84. package/src/rule-packs/nextjs/rules/server-only-modules-import-server-only.ts +87 -0
  85. package/src/rule-packs/nextjs/utils.ts +18 -0
  86. package/src/rule-packs/react-component-architecture/index.ts +18 -0
  87. package/src/rule-packs/react-component-architecture/rules/dangerous-html-requires-sanitize.ts +83 -0
  88. package/src/rule-packs/react-component-architecture/rules/no-anonymous-useEffect.ts +61 -0
  89. package/src/rule-packs/react-component-architecture/rules/no-component-invocation.ts +55 -0
  90. package/src/rule-packs/react-component-architecture/rules/no-derived-state-in-effect.ts +204 -0
  91. package/src/rule-packs/react-component-architecture/rules/no-nested-component.ts +152 -0
  92. package/src/rule-packs/react-component-architecture/rules/no-react-fc.ts +57 -0
  93. package/src/rule-packs/rule-catalog.types.ts +21 -0
  94. package/src/rule-packs/rule-metadata.ts +163 -0
  95. package/src/rule-packs/runtime-boundaries/index.ts +33 -0
  96. package/src/rule-packs/runtime-boundaries/rules/no-prototype-polluting-merge.ts +113 -0
  97. package/src/rule-packs/runtime-boundaries/rules/no-user-controlled-fetch-url.ts +69 -0
  98. package/src/rule-packs/runtime-boundaries/rules/no-user-controlled-redirect.ts +79 -0
  99. package/src/rule-packs/runtime-boundaries/rules/upload-must-set-limits.ts +126 -0
  100. package/src/rule-packs/runtime-boundaries/rules/webhook-must-verify-signature-before-parse.ts +87 -0
  101. package/src/rule-packs/security/index.ts +35 -0
  102. package/src/rule-packs/security/rules/catch-must-handle.ts +126 -0
  103. package/src/rule-packs/security/rules/no-auth-token-in-storage.ts +107 -0
  104. package/src/rule-packs/security/rules/no-child-process-exec.ts +72 -0
  105. package/src/rule-packs/security/rules/no-dynamic-regexp.ts +56 -0
  106. package/src/rule-packs/security/rules/no-inner-html-assignment.ts +42 -0
  107. package/src/rule-packs/security/rules/no-spawn-with-shell.ts +106 -0
  108. package/src/rule-packs/structured-logging/index.ts +6 -0
  109. package/src/rule-packs/structured-logging/rules/caught-error-log-requires-cause.ts +234 -0
  110. package/src/rule-packs/structured-logging/rules/logger-not-console.ts +146 -0
  111. package/src/rule-packs/test-conventions/index.ts +9 -0
  112. package/src/rule-packs/test-conventions/rules/fake-timers-must-be-restored.ts +143 -0
  113. package/src/rule-packs/test-conventions/rules/no-conditional-expect.ts +77 -0
  114. package/src/rule-packs/test-conventions/rules/no-real-network-in-unit-tests.ts +174 -0
  115. package/src/rule-packs/typescript-core/index.ts +30 -0
  116. package/src/rule-packs/typescript-core/rules/exported-functions-require-return-type.ts +74 -0
  117. package/src/rule-packs/typescript-core/rules/fetch-must-check-ok.ts +106 -0
  118. package/src/rule-packs/typescript-core/rules/json-parse-must-validate.ts +97 -0
  119. package/src/rule-packs/typescript-core/rules/no-unsafe-boundary-cast.ts +70 -0
  120. package/src/stack-detection/packs.ts +57 -0
  121. package/strict.type-aware.eslint.config.mjs +33 -0
  122. package/strict.web.eslint.config.mjs +32 -1
@@ -0,0 +1,263 @@
1
+ import type { ILoopEvent } from "../loop/loop.types";
2
+ import type { ErrorSet } from "../validate/validate.types";
3
+
4
+ /**
5
+ * Why a run failed — a structured reason, so every failed run maps to a possible
6
+ * harness intervention (the self-improving north-star). Derived purely from the
7
+ * event stream (+ an optional final gate error set), so the same classifier
8
+ * serves the live loop, the eval sweep, and the offline log analyzer.
9
+ */
10
+ export const FAILURE_CLASS = {
11
+ /** The run reached a green gate — no failure. */
12
+ none: "none",
13
+ /** Model emitted tool calls the parser couldn't read (repair L3 / salvage). */
14
+ toolMalformed: "tool-malformed",
15
+ /** Edits kept missing their target (missing-file / not-found / ambiguous). */
16
+ editReject: "edit-reject",
17
+ /** Hit the turn cap or the gate stalled with no decisive error class. */
18
+ noProgress: "no-progress",
19
+ /** Final gate red dominated by tsc type errors. */
20
+ typeError: "type-error",
21
+ /** Final gate red dominated by ESLint rule violations. */
22
+ lintRule: "lint-rule",
23
+ /** Imported a module that doesn't exist (TS2307 / "Cannot find module"). */
24
+ hallucinatedImport: "hallucinated-import",
25
+ /** Output degenerated into a repetition loop (StreamGuard fired). */
26
+ degeneration: "degeneration",
27
+ /** A per-call/timeout backstop tripped. */
28
+ timeout: "timeout",
29
+ /** A route rendered as an empty/phantom page. */
30
+ routePhantom: "route-phantom",
31
+ /** The built app failed to render / threw in the browser oracle. */
32
+ browserFail: "browser-fail",
33
+ /** The bundler/build step (vite) failed. */
34
+ buildFail: "build-fail",
35
+ /** Failed, but no signal was decisive. */
36
+ unknown: "unknown",
37
+ } as const;
38
+
39
+ export type FailureClass = (typeof FAILURE_CLASS)[keyof typeof FAILURE_CLASS];
40
+
41
+ /** Per-signal tallies behind a classification — kept for debugging/telemetry. */
42
+ export interface IFailureSignals {
43
+ repairs: number;
44
+ salvages: number;
45
+ editRejects: number;
46
+ degenerated: boolean;
47
+ tsErrors: number;
48
+ lintErrors: number;
49
+ missingModule: number;
50
+ browser: number;
51
+ build: number;
52
+ }
53
+
54
+ export interface IFailureSummary {
55
+ failureClass: FailureClass;
56
+ /** The dominant rule/code for type-error|lint-rule (e.g. "TS18048", "no-as"). */
57
+ detail?: string;
58
+ signals: IFailureSignals;
59
+ }
60
+
61
+ const TS_CODE = /^TS\d+$/;
62
+ const MISSING_MODULE = /cannot find module/i;
63
+ const DEGENERATE = /degenerat/i;
64
+ const TOOL_MALFORMED = /salvage|recovered|malformed|re-ask/i;
65
+ const REJECTED = /reject/i;
66
+ const BROWSER = /blank|did not render|did not mount|page error|uncaught/i;
67
+ const ROUTE = /route|phantom|stub/i;
68
+ const BUILD = /vite|esbuild|build failed|bundl/i;
69
+
70
+ /** The most frequently occurring string, or undefined for an empty list. */
71
+ function mostCommon(values: readonly string[]): string | undefined {
72
+ const counts = new Map<string, number>();
73
+ let best: string | undefined;
74
+ let bestN = 0;
75
+
76
+ for (const value of values) {
77
+ const n = (counts.get(value) ?? 0) + 1;
78
+
79
+ counts.set(value, n);
80
+
81
+ if (n > bestN) {
82
+ bestN = n;
83
+ best = value;
84
+ }
85
+ }
86
+
87
+ return best;
88
+ }
89
+
90
+ /** The final red gate's rules: prefer the explicit error set, else the rules
91
+ * carried on the last failing `validated` event. */
92
+ function finalRules(
93
+ events: readonly ILoopEvent[],
94
+ finalErrors?: ErrorSet
95
+ ): string[] {
96
+ if (finalErrors !== undefined) {
97
+ return finalErrors.flatMap((e) => (e.rule === undefined ? [] : [e.rule]));
98
+ }
99
+
100
+ let last: readonly string[] = [];
101
+
102
+ for (const event of events) {
103
+ if (event.kind === "validated" && event.passed === false && event.rules) {
104
+ last = event.rules;
105
+ }
106
+ }
107
+
108
+ return [...last];
109
+ }
110
+
111
+ /** Concatenated message/output text across the run — for keyword signals that
112
+ * aren't structured into a dedicated field (missing module, browser, build). */
113
+ function runText(
114
+ events: readonly ILoopEvent[],
115
+ finalErrors?: ErrorSet
116
+ ): string {
117
+ const parts: string[] = [];
118
+
119
+ for (const event of events) {
120
+ parts.push(event.message);
121
+
122
+ if (event.output !== undefined) {
123
+ parts.push(event.output);
124
+ }
125
+ }
126
+
127
+ for (const e of finalErrors ?? []) {
128
+ parts.push(e.message);
129
+ }
130
+
131
+ return parts.join("\n");
132
+ }
133
+
134
+ function gatherSignals(
135
+ events: readonly ILoopEvent[],
136
+ finalErrors?: ErrorSet
137
+ ): IFailureSignals {
138
+ const rules = finalRules(events, finalErrors);
139
+ const text = runText(events, finalErrors);
140
+ const missingModule =
141
+ rules.filter((r) => r === "TS2307").length +
142
+ (MISSING_MODULE.test(text) ? 1 : 0);
143
+
144
+ return {
145
+ repairs: events.filter((e) => e.kind === "repair").length,
146
+ salvages: events.filter(
147
+ (e) => e.kind === "tool" && TOOL_MALFORMED.test(e.message)
148
+ ).length,
149
+ editRejects: events.filter(
150
+ (e) => e.kind === "edit" && REJECTED.test(e.message)
151
+ ).length,
152
+ degenerated: events.some((e) => DEGENERATE.test(e.message)),
153
+ tsErrors: rules.filter((r) => TS_CODE.test(r) && r !== "TS2307").length,
154
+ lintErrors: rules.filter((r) => !TS_CODE.test(r)).length,
155
+ missingModule,
156
+ browser: BROWSER.test(text) ? 1 : 0,
157
+ build: BUILD.test(text) ? 1 : 0,
158
+ };
159
+ }
160
+
161
+ function finalStatusOf(
162
+ events: readonly ILoopEvent[]
163
+ ): "done" | "stuck" | "none" {
164
+ let status: "done" | "stuck" | "none" = "none";
165
+
166
+ for (const event of events) {
167
+ if (event.kind === "done") {
168
+ status = "done";
169
+ } else if (event.kind === "stuck") {
170
+ status = "stuck";
171
+ }
172
+ }
173
+
174
+ return status;
175
+ }
176
+
177
+ /** Pick the dominant gate-error class (type vs lint), with its commonest code. */
178
+ function classifyGateErrors(
179
+ events: readonly ILoopEvent[],
180
+ finalErrors: ErrorSet | undefined,
181
+ signals: IFailureSignals
182
+ ): IFailureSummary | undefined {
183
+ const rules = finalRules(events, finalErrors);
184
+
185
+ if (signals.tsErrors > 0 && signals.tsErrors >= signals.lintErrors) {
186
+ return {
187
+ failureClass: FAILURE_CLASS.typeError,
188
+ detail: mostCommon(rules.filter((r) => TS_CODE.test(r))),
189
+ signals,
190
+ };
191
+ }
192
+
193
+ if (signals.lintErrors > 0) {
194
+ return {
195
+ failureClass: FAILURE_CLASS.lintRule,
196
+ detail: mostCommon(rules.filter((r) => !TS_CODE.test(r))),
197
+ signals,
198
+ };
199
+ }
200
+
201
+ return undefined;
202
+ }
203
+
204
+ /** Behavioral fallback when no gate-error class dominates. */
205
+ function classifyBehavior(signals: IFailureSignals): FailureClass {
206
+ if (signals.degenerated) {
207
+ return FAILURE_CLASS.degeneration;
208
+ }
209
+
210
+ if (signals.salvages > 0 || signals.repairs > 0) {
211
+ return FAILURE_CLASS.toolMalformed;
212
+ }
213
+
214
+ if (signals.editRejects > 0) {
215
+ return FAILURE_CLASS.editReject;
216
+ }
217
+
218
+ return FAILURE_CLASS.noProgress;
219
+ }
220
+
221
+ /**
222
+ * Classify a run from its event stream. Pass the final gate `ErrorSet` when the
223
+ * caller has it (authoritative); otherwise the classifier falls back to the
224
+ * `rules` carried on the last failing `validated` event and keyword signals.
225
+ * A run that reached a green gate classifies as `none`.
226
+ */
227
+ export function classifyRun(
228
+ events: readonly ILoopEvent[],
229
+ finalErrors?: ErrorSet
230
+ ): IFailureSummary {
231
+ const signals = gatherSignals(events, finalErrors);
232
+
233
+ if (finalStatusOf(events) === "done") {
234
+ return { failureClass: FAILURE_CLASS.none, signals };
235
+ }
236
+
237
+ if (signals.missingModule > 0) {
238
+ return { failureClass: FAILURE_CLASS.hallucinatedImport, signals };
239
+ }
240
+
241
+ if (signals.browser > 0) {
242
+ const text = runText(events, finalErrors);
243
+
244
+ return {
245
+ failureClass: ROUTE.test(text)
246
+ ? FAILURE_CLASS.routePhantom
247
+ : FAILURE_CLASS.browserFail,
248
+ signals,
249
+ };
250
+ }
251
+
252
+ if (signals.build > 0 && signals.tsErrors === 0 && signals.lintErrors === 0) {
253
+ return { failureClass: FAILURE_CLASS.buildFail, signals };
254
+ }
255
+
256
+ const gate = classifyGateErrors(events, finalErrors, signals);
257
+
258
+ if (gate !== undefined) {
259
+ return gate;
260
+ }
261
+
262
+ return { failureClass: classifyBehavior(signals), signals };
263
+ }
package/src/eval/index.ts CHANGED
@@ -2,6 +2,14 @@ export * from "./eval.types";
2
2
  export { judge } from "./judge";
3
3
  export { summarize } from "./score";
4
4
  export { analyzeEvents, type IRunMetrics } from "./metrics";
5
+ export {
6
+ classifyRun,
7
+ FAILURE_CLASS,
8
+ type FailureClass,
9
+ type IFailureSummary,
10
+ type IFailureSignals,
11
+ } from "./failure-class";
12
+ export { parseEventLog } from "./parse-log";
5
13
  export {
6
14
  buildSweepReport,
7
15
  renderSweepReportMarkdown,
@@ -1,4 +1,5 @@
1
1
  import type { ILoopEvent } from "../loop/loop.types";
2
+ import { classifyRun, type FailureClass } from "./failure-class";
2
3
 
3
4
  /** Behavioral metrics distilled from a run's event stream — the signals the
4
5
  * local-model literature says predict outcomes (tokens-to-solution, repair
@@ -6,6 +7,10 @@ import type { ILoopEvent } from "../loop/loop.types";
6
7
  * the cli-metrics script. */
7
8
  export interface IRunMetrics {
8
9
  finalStatus: "done" | "stuck" | "none";
10
+ /** Structured reason the run failed (`none` when it reached green). The single
11
+ * source of truth for failure classification — the cli-metrics analyzer and
12
+ * the eval sweep both read this rather than re-deriving it. */
13
+ failureClass: FailureClass;
9
14
  /** Model turns (one per `cycle` event). */
10
15
  turns: number;
11
16
  /** Model calls (one per `usage` event). */
@@ -29,6 +34,7 @@ export interface IRunMetrics {
29
34
  function emptyMetrics(): IRunMetrics {
30
35
  return {
31
36
  finalStatus: "none",
37
+ failureClass: "none",
32
38
  turns: 0,
33
39
  modelCalls: 0,
34
40
  tokensOut: 0,
@@ -82,6 +88,7 @@ export function analyzeEvents(events: readonly ILoopEvent[]): IRunMetrics {
82
88
 
83
89
  m.filesCreated = created.size;
84
90
  m.avgTokensPerSecond = tpsCount > 0 ? Math.round(tpsSum / tpsCount) : 0;
91
+ m.failureClass = classifyRun(events).failureClass;
85
92
 
86
93
  return m;
87
94
  }
@@ -0,0 +1,105 @@
1
+ import type { ILoopEvent } from "../loop/loop.types";
2
+ import { isRecord } from "../lib/guards";
3
+
4
+ /** The known event kinds, as a runtime set, so a JSONL line can be validated
5
+ * into a typed ILoopEvent without an `as` cast. Keep in sync with ILoopEvent. */
6
+ const KNOWN_KINDS = new Set<string>([
7
+ "start",
8
+ "red",
9
+ "cycle",
10
+ "token",
11
+ "message",
12
+ "fix",
13
+ "edit",
14
+ "create",
15
+ "validated",
16
+ "done",
17
+ "stuck",
18
+ "run",
19
+ "tool",
20
+ "repair",
21
+ "timing",
22
+ "usage",
23
+ "ttsr",
24
+ ]);
25
+
26
+ function isKind(value: string): value is ILoopEvent["kind"] {
27
+ return KNOWN_KINDS.has(value);
28
+ }
29
+
30
+ function optionalString(value: unknown): string | undefined {
31
+ return typeof value === "string" ? value : undefined;
32
+ }
33
+
34
+ function stringArray(value: unknown): string[] | undefined {
35
+ if (!Array.isArray(value)) {
36
+ return undefined;
37
+ }
38
+
39
+ return value.filter((v): v is string => typeof v === "string");
40
+ }
41
+
42
+ /** Coerce one parsed JSONL record into an ILoopEvent, or null when it isn't one.
43
+ * Reads only the fields the failure classifier + metrics consume — enough to
44
+ * reconstruct a typed event stream from a `--log` file. */
45
+ function coerceEvent(record: unknown): ILoopEvent | null {
46
+ if (!isRecord(record)) {
47
+ return null;
48
+ }
49
+
50
+ const kind = record.kind;
51
+
52
+ if (typeof kind !== "string" || !isKind(kind)) {
53
+ return null;
54
+ }
55
+
56
+ const event: ILoopEvent = {
57
+ kind,
58
+ task: optionalString(record.task) ?? "",
59
+ message: optionalString(record.message) ?? "",
60
+ };
61
+ const output = optionalString(record.output);
62
+ const rules = stringArray(record.rules);
63
+
64
+ if (output !== undefined) {
65
+ event.output = output;
66
+ }
67
+
68
+ if (typeof record.passed === "boolean") {
69
+ event.passed = record.passed;
70
+ }
71
+
72
+ if (rules !== undefined) {
73
+ event.rules = rules;
74
+ }
75
+
76
+ return event;
77
+ }
78
+
79
+ /** Parse a `--log` JSONL transcript (one serialized event per line) into a typed
80
+ * event stream. Malformed lines and non-event records are skipped. */
81
+ export function parseEventLog(jsonl: string): ILoopEvent[] {
82
+ const events: ILoopEvent[] = [];
83
+
84
+ for (const line of jsonl.split("\n")) {
85
+ if (line.trim().length === 0) {
86
+ continue;
87
+ }
88
+
89
+ let parsed: unknown;
90
+
91
+ try {
92
+ parsed = JSON.parse(line);
93
+ } catch {
94
+ continue;
95
+ }
96
+
97
+ const event = coerceEvent(parsed);
98
+
99
+ if (event !== null) {
100
+ events.push(event);
101
+ }
102
+ }
103
+
104
+ return events;
105
+ }
@@ -164,5 +164,24 @@ export function renderSweepReportMarkdown(report: ISweepReport): string {
164
164
  ...rows,
165
165
  "",
166
166
  "`*` = significant at p < 0.05 (two-proportion z-test vs baseline).",
167
+ ...failureSection(report),
167
168
  ].join("\n");
168
169
  }
170
+
171
+ /** Format a variant's failure-class tally, e.g. "type-error×2, no-progress×1". */
172
+ function formatFailureClasses(classes: Record<string, number>): string {
173
+ return Object.entries(classes)
174
+ .sort(([, a], [, b]) => b - a)
175
+ .map(([cls, n]) => `${cls}×${String(n)}`)
176
+ .join(", ");
177
+ }
178
+
179
+ /** A "why failures happened" section — per-variant failure-class breakdown.
180
+ * Empty (no lines) when every run passed, so a clean sweep stays terse. */
181
+ function failureSection(report: ISweepReport): string[] {
182
+ const lines = report.variants
183
+ .filter((v) => Object.keys(v.failureClasses).length > 0)
184
+ .map((v) => `- **${v.label}**: ${formatFailureClasses(v.failureClasses)}`);
185
+
186
+ return lines.length === 0 ? [] : ["", "### Failure breakdown", ...lines];
187
+ }
package/src/eval/score.ts CHANGED
@@ -20,6 +20,15 @@ export function summarize(records: IRunRecord[]): IVariantSummary[] {
20
20
  const sum = (select: (r: IRunRecord) => number): number =>
21
21
  list.reduce((acc, r) => acc + select(r), 0);
22
22
  const scored = list.filter((r) => r.quality !== undefined);
23
+ const failureClasses: Record<string, number> = {};
24
+
25
+ for (const r of list) {
26
+ const fc = r.failureClass;
27
+
28
+ if (!r.passed && fc !== undefined && fc !== "none") {
29
+ failureClasses[fc] = (failureClasses[fc] ?? 0) + 1;
30
+ }
31
+ }
23
32
 
24
33
  summaries.push({
25
34
  label,
@@ -32,6 +41,7 @@ export function summarize(records: IRunRecord[]): IVariantSummary[] {
32
41
  scored.length > 0
33
42
  ? scored.reduce((acc, r) => acc + (r.quality ?? 0), 0) / scored.length
34
43
  : 0,
44
+ failureClasses,
35
45
  });
36
46
  }
37
47
 
@@ -11,6 +11,30 @@ export const META_RULE_DOCS: Record<string, string> = {
11
11
  "no-overlapping-libs":
12
12
  "Remove duplicate or conflicting library versions from the dependency tree; only one canonical version per library is allowed.",
13
13
 
14
+ "fastify-security-plugins":
15
+ "Add @fastify/helmet, @fastify/cors, and @fastify/rate-limit when using fastify in production.",
16
+
17
+ "lockfile-required":
18
+ "Commit the lockfile for your package manager (package-lock.json, yarn.lock, pnpm-lock.yaml, or bun.lockb) and keep it in sync with package.json.",
19
+
20
+ "single-package-manager":
21
+ "Remove extra lockfiles — use one package manager and delete lockfiles from other tools.",
22
+
23
+ "package-manager-field-required":
24
+ 'Add a "packageManager" field to package.json (e.g. "bun@1.3.14") so installs are reproducible across environments.',
25
+
26
+ "no-git-or-tarball-dependencies":
27
+ "Replace git+, git:, or HTTP tarball dependency URLs with registry versions from npm.",
28
+
29
+ "dependency-overrides-require-comment":
30
+ "Add a comment next to overrides/resolutions in package.json explaining why each override is needed.",
31
+
32
+ "production-must-not-use-drizzle-push":
33
+ "Replace `drizzle-kit push` in scripts and CI with checked-in SQL migrations and `drizzle-kit migrate`.",
34
+
35
+ "migrations-must-be-checked-in":
36
+ "Add a drizzle/ or migrations/ folder with generated SQL migration files when using Drizzle ORM.",
37
+
14
38
  // Source text
15
39
  "no-eslint-disable-comments":
16
40
  "Remove `// eslint-disable` comments — they hide warnings. Fix the underlying violation or refactor the code.",
@@ -25,6 +49,18 @@ export const META_RULE_DOCS: Record<string, string> = {
25
49
  "tsconfig-strict":
26
50
  "Enable all strict mode flags in tsconfig.json (strict: true or all strict flags individually).",
27
51
 
52
+ "tsconfig-recommended-flags":
53
+ "Enable useUnknownInCatchVariables, erasableSyntaxOnly, exactOptionalPropertyTypes, verbatimModuleSyntax, and noPropertyAccessFromIndexSignature in tsconfig.json compilerOptions.",
54
+
55
+ "next-proxy-over-middleware":
56
+ "Migrate middleware.ts to proxy.ts for Next.js 16 early request interception.",
57
+
58
+ "next-instrumentation-present":
59
+ "Add instrumentation.ts with registerOTel for OpenTelemetry tracing in Next.js apps.",
60
+
61
+ "next-image-remote-patterns-no-wildcards":
62
+ "Remove `**` hostname wildcards from next.config remotePatterns — allowlist specific image hostnames.",
63
+
28
64
  // Testing
29
65
  "test-sibling-required":
30
66
  "Add a test file for each source file; follow naming conventions (foo.ts → foo.test.ts or foo.spec.ts).",
@@ -38,4 +74,16 @@ export const META_RULE_DOCS: Record<string, string> = {
38
74
 
39
75
  "workflow-timeout-required":
40
76
  "Add a timeout-minutes setting to each GitHub Actions job to prevent hanging workflows.",
77
+
78
+ "workflow-permissions-explicit":
79
+ "Add a top-level permissions: block or job-level permissions to every GitHub Actions workflow.",
80
+
81
+ "workflow-permissions-least-privilege":
82
+ "Avoid workflow-level contents: write or id-token: write — scope write permissions to the job that needs them.",
83
+
84
+ "no-pull-request-target-untrusted-checkout":
85
+ "Do not combine pull_request_target with checkout of the PR head ref — use pull_request or checkout the base ref.",
86
+
87
+ "no-github-context-in-shell":
88
+ "Pass github.event values through env: instead of interpolating them directly in run: shell scripts.",
41
89
  };
@@ -154,6 +154,156 @@ const RULE_DOCS: Record<string, IRuleDoc> = {
154
154
  bad: "<button onClick={() => doThing(id)} />",
155
155
  good: "const onClickRow = useCallback(() => doThing(id), [id]); <button onClick={onClickRow} />",
156
156
  },
157
+ "tsforge/no-throw-literal": {
158
+ what: "Throw `Error` instances, not string or number literals.",
159
+ bad: "throw 'Unauthorized';",
160
+ good: "throw new Error('Unauthorized');",
161
+ },
162
+ "tsforge/no-react-fc": {
163
+ what: "Do not use React.FC — type props on the function parameter.",
164
+ bad: "const Button: React.FC<IButtonProps> = ({ onClick }) => <button onClick={onClick} />;",
165
+ good: "function Button({ onClick }: IButtonProps) { return <button onClick={onClick} />; }",
166
+ },
167
+ "tsforge/no-component-invocation": {
168
+ what: "Render components as JSX, not function calls.",
169
+ bad: "<div>{Header()}</div>",
170
+ good: "<div><Header /></div>",
171
+ },
172
+ "tsforge/no-nested-component": {
173
+ what: "Declare components at module scope, not inside another component.",
174
+ bad: "function App() { function Inner() { return <span />; } return <Inner />; }",
175
+ good: "function Inner() { return <span />; } function App() { return <Inner />; }",
176
+ },
177
+ "tsforge/dangerous-html-requires-sanitize": {
178
+ what: "Sanitize HTML before dangerouslySetInnerHTML — import DOMPurify.",
179
+ bad: "<div dangerouslySetInnerHTML={{ __html: rawHtml }} />",
180
+ good: "import DOMPurify from 'isomorphic-dompurify'; <div dangerouslySetInnerHTML={{ __html: DOMPurify.sanitize(rawHtml) }} />",
181
+ },
182
+ "tsforge/no-child-process-exec": {
183
+ what: "Do not use child_process.exec/execSync — shell execution enables command injection.",
184
+ bad: "import { exec } from 'child_process'; exec(`rm -rf ${dir}`);",
185
+ good: "import { execFile } from 'child_process'; execFile('rm', ['-rf', dir], callback);",
186
+ },
187
+ "tsforge/no-spawn-with-shell": {
188
+ what: "Do not pass `{ shell: true }` to spawn/spawnSync.",
189
+ bad: "spawn('sh', ['-c', cmd], { shell: true });",
190
+ good: "spawn('node', ['script.js', arg]);",
191
+ },
192
+ "tsforge/no-dynamic-regexp": {
193
+ what: "Do not build RegExp from runtime input — ReDoS risk.",
194
+ bad: "const re = new RegExp(userPattern);",
195
+ good: "const re = /^fixed-pattern$/;",
196
+ },
197
+ "tsforge/no-inner-html-assignment": {
198
+ what: "Do not assign to innerHTML — XSS risk in vanilla DOM code.",
199
+ bad: "el.innerHTML = userHtml;",
200
+ good: "el.textContent = userText;",
201
+ },
202
+ "tsforge/catch-must-handle": {
203
+ what: "Catch blocks must log, rethrow, or return a typed error — not silently mask failure.",
204
+ bad: "catch (e) { return null; }",
205
+ good: "catch (e) { logger.error(e); throw e; }",
206
+ },
207
+ "tsforge/no-react-in-services": {
208
+ what: "Service/data modules must not import React — keep business logic decoupled from UI.",
209
+ bad: "import { useMemo } from 'react'; // in src/services/users.ts",
210
+ good: "Move React hooks to components; keep services as plain TypeScript.",
211
+ },
212
+ "tsforge/no-anonymous-useEffect": {
213
+ what: "Pass a named function to useEffect for debuggable stack traces.",
214
+ bad: "useEffect(() => { sync(); }, [id]);",
215
+ good: "useEffect(function syncOnIdChange() { sync(); }, [id]);",
216
+ },
217
+ "tsforge/no-derived-state-in-effect": {
218
+ what: "Do not set local state inside useEffect when the value can be derived during render.",
219
+ bad: "useEffect(() => { setFullName(first + ' ' + last); }, [first, last]);",
220
+ good: "const fullName = `${first} ${last}`;",
221
+ },
222
+ "tsforge/no-internal-api-fetch": {
223
+ what: "Server Components must not fetch the app's own /api routes.",
224
+ bad: "await fetch('/api/users');",
225
+ good: "import { listUsers } from '@/services/users'; const users = await listUsers();",
226
+ },
227
+ "tsforge/await-dynamic-request-apis": {
228
+ what: "Await Next.js dynamic request APIs in Server Components.",
229
+ bad: "const jar = cookies();",
230
+ good: "const jar = await cookies();",
231
+ },
232
+ "tsforge/error-boundary-require-use-client": {
233
+ what: "error.tsx and global-error.tsx must be Client Components.",
234
+ bad: "export default function Error() { return <div />; }",
235
+ good: "'use client'; export default function Error() { return <div />; }",
236
+ },
237
+ "tsforge/no-html-img-element": {
238
+ what: "Prefer next/image over raw img elements.",
239
+ bad: "<img src='/hero.jpg' alt='hero' />",
240
+ good: "import Image from 'next/image'; <Image src='/hero.jpg' alt='hero' width={800} height={400} />",
241
+ },
242
+ "tsforge/no-sensitive-next-public-env": {
243
+ what: "NEXT_PUBLIC_* vars are exposed in the client bundle — never use for secrets.",
244
+ bad: "process.env.NEXT_PUBLIC_STRIPE_SECRET",
245
+ good: "process.env.STRIPE_SECRET_KEY // server-only, no NEXT_PUBLIC prefix",
246
+ },
247
+ "tsforge/prefer-lazy-use-state-init": {
248
+ what: "Use lazy useState when parsing localStorage on mount.",
249
+ bad: "useState(JSON.parse(localStorage.getItem('cfg') ?? '{}'))",
250
+ good: "useState(() => JSON.parse(localStorage.getItem('cfg') ?? '{}'))",
251
+ },
252
+ "tsforge/no-auth-token-in-storage": {
253
+ what: "Never store auth tokens in localStorage/sessionStorage.",
254
+ bad: "localStorage.setItem('auth_token', token);",
255
+ good: "Set an httpOnly session cookie on the server instead.",
256
+ },
257
+ "tsforge/fetch-must-check-ok": {
258
+ what: "Check response.ok before calling .json() on fetch results.",
259
+ bad: "const data = await fetch(url).then(r => r.json());",
260
+ good: "const res = await fetch(url); if (!res.ok) { throw new Error('fetch failed'); } const data = await res.json();",
261
+ },
262
+ "tsforge/json-parse-must-validate": {
263
+ what: "Parse external JSON through a schema library, not bare JSON.parse.",
264
+ bad: "const body = JSON.parse(raw);",
265
+ good: "const body = UserSchema.parse(JSON.parse(raw));",
266
+ },
267
+ "tsforge/no-unsafe-boundary-cast": {
268
+ what: "Do not cast untrusted parsed input with `as` — validate at the boundary.",
269
+ bad: "const user = (await req.json()) as IUser;",
270
+ good: "const user = UserSchema.parse(await req.json());",
271
+ },
272
+ "tsforge/no-user-controlled-redirect": {
273
+ what: "Redirect URLs must be string literals or allowlisted helpers — not user input.",
274
+ bad: "redirect(searchParams.get('next')!);",
275
+ good: "redirect('/dashboard');",
276
+ },
277
+ "tsforge/no-user-controlled-fetch-url": {
278
+ what: "fetch/axios URLs must be literals or pass through an allowlisted URL builder.",
279
+ bad: "await fetch(userSuppliedUrl);",
280
+ good: "await fetch('https://api.example.com/v1/status');",
281
+ },
282
+ "tsforge/no-prototype-polluting-merge": {
283
+ what: "Do not merge request body/query/params into objects wholesale.",
284
+ bad: "Object.assign(config, req.body);",
285
+ good: "const name = UserSchema.parse(req.body).name; config.name = name;",
286
+ },
287
+ "tsforge/server-only-modules-import-server-only": {
288
+ what: "Server-only modules importing DB/env must include `import 'server-only'`.",
289
+ bad: "import { db } from '@/lib/db'; // in lib/admin.ts",
290
+ good: "import 'server-only'; import { db } from '@/lib/db';",
291
+ },
292
+ "tsforge/server-action-requires-authz-and-validation": {
293
+ what: "Server actions must validate input and call authz before mutations.",
294
+ bad: "'use server'; export async function deleteUser(id: string) { await db.delete(users).where(eq(users.id, id)); }",
295
+ good: "'use server'; export async function deleteUser(raw: unknown) { const user = await requireUser(); const { id } = IdSchema.parse(raw); await authorize(user, id); ... }",
296
+ },
297
+ "tsforge/require-route-schema": {
298
+ what: "Fastify routes need a schema object with input validation.",
299
+ bad: "fastify.post('/users', async () => ({ ok: true }));",
300
+ good: "fastify.post('/users', { schema: { body: UserSchema } }, async () => ({ ok: true }));",
301
+ },
302
+ "tsforge/require-plugin-name": {
303
+ what: "fastify-plugin wrappers need a name option.",
304
+ bad: "export default fp(dbPlugin);",
305
+ good: "export default fp(dbPlugin, { name: 'db-connector', fastify: '5.x' });",
306
+ },
157
307
  };
158
308
 
159
309
  /**