@drkmttr/adhere 0.12.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +149 -51
- package/package.json +9 -9
- package/presets/effect/basics/{instrument-with-pipe.md → instrument-with-pipe/RULE.md} +1 -1
- package/presets/effect/config/{business-logic-depends-on-config-service.md → business-logic-depends-on-config-service/RULE.md} +3 -3
- package/presets/effect/config/{secrets-are-redacted.md → secrets-are-redacted/RULE.md} +3 -3
- package/presets/effect/config/{validate-with-schema.md → validate-with-schema/RULE.md} +4 -4
- package/skills/adhere/SKILL.md +206 -118
- package/skills/adhere-fix/SKILL.md +9 -8
- package/skills/adhere-setup/SKILL.md +244 -0
- package/src/cli.ts +49 -33
- package/src/config.ts +50 -3
- package/src/index.ts +2 -0
- package/src/init.ts +36 -25
- package/src/install.ts +37 -26
- package/src/main.ts +1 -1
- package/src/rules.ts +135 -56
- package/src/services/AdhereConfig.ts +2 -2
- package/src/services/Credentials.ts +4 -4
- package/src/services/Jev.http.ts +76 -10
- package/src/services/Jev.ts +39 -10
- package/src/workflows/audit.ts +23 -12
- package/src/workflows/format.ts +172 -35
- /package/presets/alchemy/apis/{match-remote-errors-by-tag.md → match-remote-errors-by-tag/RULE.md} +0 -0
- /package/presets/alchemy/apis/{schemas-at-trust-boundaries.md → schemas-at-trust-boundaries/RULE.md} +0 -0
- /package/presets/alchemy/data/{commit-migrations-then-deploy.md → commit-migrations-then-deploy/RULE.md} +0 -0
- /package/presets/alchemy/data/{hyperdrive-direct-origin.md → hyperdrive-direct-origin/RULE.md} +0 -0
- /package/presets/alchemy/data/{migrations-use-direct-url.md → migrations-use-direct-url/RULE.md} +0 -0
- /package/presets/alchemy/data/{never-disable-tls-verification.md → never-disable-tls-verification/RULE.md} +0 -0
- /package/presets/alchemy/data/{private-endpoint-at-runtime.md → private-endpoint-at-runtime/RULE.md} +0 -0
- /package/presets/alchemy/data/{single-migration-owner.md → single-migration-owner/RULE.md} +0 -0
- /package/presets/alchemy/durable/{idempotent-event-handlers.md → idempotent-event-handlers/RULE.md} +0 -0
- /package/presets/alchemy/durable/{io-inside-steps.md → io-inside-steps/RULE.md} +0 -0
- /package/presets/alchemy/durable/{no-self-triggering-bucket-writes.md → no-self-triggering-bucket-writes/RULE.md} +0 -0
- /package/presets/alchemy/durable/{stable-step-names.md → stable-step-names/RULE.md} +0 -0
- /package/presets/alchemy/durable/{webhook-state-is-truth.md → webhook-state-is-truth/RULE.md} +0 -0
- /package/presets/alchemy/providers/{diff-guards-unresolved-props.md → diff-guards-unresolved-props/RULE.md} +0 -0
- /package/presets/alchemy/providers/{idempotent-delete.md → idempotent-delete/RULE.md} +0 -0
- /package/presets/alchemy/providers/{lazy-layer-construction.md → lazy-layer-construction/RULE.md} +0 -0
- /package/presets/alchemy/providers/{reconcile-from-observed-state.md → reconcile-from-observed-state/RULE.md} +0 -0
- /package/presets/alchemy/resources/{anchor-paths-with-import-meta.md → anchor-paths-with-import-meta/RULE.md} +0 -0
- /package/presets/alchemy/resources/{idempotent-actions-and-seeds.md → idempotent-actions-and-seeds/RULE.md} +0 -0
- /package/presets/alchemy/resources/{previews-reference-shared-resources.md → previews-reference-shared-resources/RULE.md} +0 -0
- /package/presets/alchemy/resources/{protect-irreplaceable-data.md → protect-irreplaceable-data/RULE.md} +0 -0
- /package/presets/alchemy/resources/{remote-state-for-ci.md → remote-state-for-ci/RULE.md} +0 -0
- /package/presets/alchemy/resources/{stable-logical-ids.md → stable-logical-ids/RULE.md} +0 -0
- /package/presets/alchemy/resources/{stage-unique-physical-names.md → stage-unique-physical-names/RULE.md} +0 -0
- /package/presets/alchemy/runtime/{bind-tag-not-implementation.md → bind-tag-not-implementation/RULE.md} +0 -0
- /package/presets/alchemy/runtime/{deterministic-config-defaults.md → deterministic-config-defaults/RULE.md} +0 -0
- /package/presets/alchemy/runtime/{disposables-in-request-scope.md → disposables-in-request-scope/RULE.md} +0 -0
- /package/presets/alchemy/runtime/{lambda-no-post-response-work.md → lambda-no-post-response-work/RULE.md} +0 -0
- /package/presets/alchemy/runtime/{no-per-request-work-at-construction.md → no-per-request-work-at-construction/RULE.md} +0 -0
- /package/presets/alchemy/runtime/{resolve-config-and-bindings-at-construction.md → resolve-config-and-bindings-at-construction/RULE.md} +0 -0
- /package/presets/alchemy/secrets/{let-alchemy-mint-infra-secrets.md → let-alchemy-mint-infra-secrets/RULE.md} +0 -0
- /package/presets/alchemy/secrets/{none-in-browser-env.md → none-in-browser-env/RULE.md} +0 -0
- /package/presets/alchemy/secrets/{redacted-until-use.md → redacted-until-use/RULE.md} +0 -0
- /package/presets/alchemy/secrets/{unwrap-before-compare-or-send.md → unwrap-before-compare-or-send/RULE.md} +0 -0
- /package/presets/alchemy/security/{authorize-public-endpoints.md → authorize-public-endpoints/RULE.md} +0 -0
- /package/presets/alchemy/security/{github-secret-not-variable.md → github-secret-not-variable/RULE.md} +0 -0
- /package/presets/alchemy/security/{identity-from-verified-session.md → identity-from-verified-session/RULE.md} +0 -0
- /package/presets/alchemy/security/{keep-auth-protections-on.md → keep-auth-protections-on/RULE.md} +0 -0
- /package/presets/alchemy/security/{least-privilege-bindings.md → least-privilege-bindings/RULE.md} +0 -0
- /package/presets/alchemy/security/{least-privilege-ci-credentials.md → least-privilege-ci-credentials/RULE.md} +0 -0
- /package/presets/alchemy/security/{lookup-failure-is-not-anonymous.md → lookup-failure-is-not-anonymous/RULE.md} +0 -0
- /package/presets/effect/basics/{gen-for-sequencing.md → gen-for-sequencing/RULE.md} +0 -0
- /package/presets/effect/config/{tests-provide-values-directly.md → tests-provide-values-directly/RULE.md} +0 -0
- /package/presets/effect/data/{brand-meaningful-primitives.md → brand-meaningful-primitives/RULE.md} +0 -0
- /package/presets/effect/data/{variants-are-tagged-unions.md → variants-are-tagged-unions/RULE.md} +0 -0
- /package/presets/effect/errors/{catch-defects-at-boundaries-only.md → catch-defects-at-boundaries-only/RULE.md} +0 -0
- /package/presets/effect/errors/{defects-for-bugs.md → defects-for-bugs/RULE.md} +0 -0
- /package/presets/effect/errors/{wrap-external-errors.md → wrap-external-errors/RULE.md} +0 -0
- /package/presets/effect/services/{dependencies-through-layers.md → dependencies-through-layers/RULE.md} +0 -0
- /package/presets/effect/services/{fresh-layer-per-test.md → fresh-layer-per-test/RULE.md} +0 -0
- /package/presets/effect/services/{memoize-parameterized-layers.md → memoize-parameterized-layers/RULE.md} +0 -0
- /package/presets/effect/services/{no-mutable-state.md → no-mutable-state/RULE.md} +0 -0
- /package/presets/effect/services/{operations-have-no-requirements.md → operations-have-no-requirements/RULE.md} +0 -0
- /package/presets/effect/services/{test-layers-are-in-memory.md → test-layers-are-in-memory/RULE.md} +0 -0
- /package/presets/effect/testing/{test-clock-for-time.md → test-clock-for-time/RULE.md} +0 -0
- /package/presets/react/effects/{clean-up-what-they-start.md → clean-up-what-they-start/RULE.md} +0 -0
- /package/presets/react/effects/{event-logic-in-handlers.md → event-logic-in-handlers/RULE.md} +0 -0
- /package/presets/react/effects/{external-stores-use-sync-external-store.md → external-stores-use-sync-external-store/RULE.md} +0 -0
- /package/presets/react/effects/{fetches-ignore-stale-responses.md → fetches-ignore-stale-responses/RULE.md} +0 -0
- /package/presets/react/effects/{notify-parents-in-handlers.md → notify-parents-in-handlers/RULE.md} +0 -0
- /package/presets/react/server/{no-private-data-to-client.md → no-private-data-to-client/RULE.md} +0 -0
- /package/presets/react/server/{server-functions-authorize-and-validate.md → server-functions-authorize-and-validate/RULE.md} +0 -0
- /package/presets/react/state/{no-contradictory-state.md → no-contradictory-state/RULE.md} +0 -0
- /package/presets/react/state/{no-props-copied-into-state.md → no-props-copied-into-state/RULE.md} +0 -0
- /package/presets/react/state/{store-ids-not-copies.md → store-ids-not-copies/RULE.md} +0 -0
- /package/presets/security/{constant-time-secret-compare.md → constant-time-secret-compare/RULE.md} +0 -0
- /package/presets/security/{no-secrets-in-output.md → no-secrets-in-output/RULE.md} +0 -0
- /package/presets/security/{no-untrusted-input-in-commands-or-paths.md → no-untrusted-input-in-commands-or-paths/RULE.md} +0 -0
- /package/presets/security/{parameterized-queries.md → parameterized-queries/RULE.md} +0 -0
- /package/presets/typescript/async/{concurrent-independent-work.md → concurrent-independent-work/RULE.md} +0 -0
- /package/presets/typescript/async/{network-calls-have-timeouts.md → network-calls-have-timeouts/RULE.md} +0 -0
- /package/presets/typescript/design/{no-argument-mutation.md → no-argument-mutation/RULE.md} +0 -0
- /package/presets/typescript/errors/{no-swallowed-errors.md → no-swallowed-errors/RULE.md} +0 -0
- /package/presets/typescript/errors/{release-on-every-path.md → release-on-every-path/RULE.md} +0 -0
- /package/presets/typescript/testing/{assert-outcomes.md → assert-outcomes/RULE.md} +0 -0
- /package/presets/typescript/testing/{independent-tests.md → independent-tests/RULE.md} +0 -0
- /package/presets/typescript/types/{assertions-only-when-proven.md → assertions-only-when-proven/RULE.md} +0 -0
- /package/presets/typescript/types/{derive-dont-restate.md → derive-dont-restate/RULE.md} +0 -0
- /package/presets/typescript/types/{unrepresentable-invalid-states.md → unrepresentable-invalid-states/RULE.md} +0 -0
- /package/presets/typescript/types/{validate-external-data.md → validate-external-data/RULE.md} +0 -0
package/src/services/Jev.http.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { RuleId } from "#config.ts";
|
|
1
|
+
import type { AppendState, JevState, JudgedFile, RuleId } from "#config.ts";
|
|
2
2
|
import { sectionsOf } from "#excerpt.ts";
|
|
3
3
|
import { AdhereConfig } from "#services/AdhereConfig.ts";
|
|
4
4
|
import { Credentials } from "#services/Credentials.ts";
|
|
@@ -22,6 +22,7 @@ import {
|
|
|
22
22
|
namedPairs,
|
|
23
23
|
type Pair,
|
|
24
24
|
pairProbability,
|
|
25
|
+
requestGroups,
|
|
25
26
|
requestsOf,
|
|
26
27
|
type Rules,
|
|
27
28
|
sufficiencyKey,
|
|
@@ -38,9 +39,9 @@ import {
|
|
|
38
39
|
Schedule,
|
|
39
40
|
Schema,
|
|
40
41
|
} from "effect";
|
|
41
|
-
import { HttpClient, HttpClientError, HttpClientResponse } from "effect/
|
|
42
|
-
import * as HttpClientRequest from "effect/
|
|
43
|
-
import { RateLimiter } from "effect/
|
|
42
|
+
import { HttpClient, HttpClientError, HttpClientResponse } from "effect/http";
|
|
43
|
+
import * as HttpClientRequest from "effect/http/HttpClientRequest";
|
|
44
|
+
import { RateLimiter } from "effect/persistence";
|
|
44
45
|
|
|
45
46
|
const SYSTEM_ONE = "https://api.typesafe.ai/v1/systemone";
|
|
46
47
|
|
|
@@ -63,6 +64,35 @@ const LocateAnswers = Schema.Struct({
|
|
|
63
64
|
|
|
64
65
|
const refused = (message: string) => JevUnavailable.make({ message });
|
|
65
66
|
|
|
67
|
+
/** Bun's API, for a rule's hook. Read off `globalThis`, as presets.ts reads it: Vitest runs this on Node. */
|
|
68
|
+
const bun = (globalThis as { readonly Bun?: unknown }).Bun as Parameters<AppendState>[2];
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* A body as it goes out: when its rule has `appendState`, with what the hook
|
|
72
|
+
* returns spread over the body's state. A hook that fails refuses the run.
|
|
73
|
+
*/
|
|
74
|
+
const hooked = (
|
|
75
|
+
body: Body & { readonly state: JevState },
|
|
76
|
+
group: Rules,
|
|
77
|
+
file: JudgedFile,
|
|
78
|
+
): Effect.Effect<Body, JevUnavailable> => {
|
|
79
|
+
const [hook] = Object.entries(group).flatMap(([id, rule]) =>
|
|
80
|
+
rule.appendState === undefined ? [] : [[id, rule.appendState] as const],
|
|
81
|
+
);
|
|
82
|
+
if (hook === undefined) return Effect.succeed(body);
|
|
83
|
+
const [id, appendState] = hook;
|
|
84
|
+
return Effect.tryPromise({
|
|
85
|
+
try: async () => ({
|
|
86
|
+
...body,
|
|
87
|
+
state: { ...body.state, ...(await appendState(body.state, file, bun)) },
|
|
88
|
+
}),
|
|
89
|
+
catch: (cause) =>
|
|
90
|
+
refused(
|
|
91
|
+
`the appendState of ${id} failed: ${cause instanceof Error ? cause.message : String(cause)}`,
|
|
92
|
+
),
|
|
93
|
+
});
|
|
94
|
+
};
|
|
95
|
+
|
|
66
96
|
/** A failed attempt, for the log: the status and Ray ID when Jev's side answered, the cause otherwise. */
|
|
67
97
|
const attemptFailure = (
|
|
68
98
|
problem: HttpClientError.HttpClientError | RateLimiter.RateLimiterError | Cause.TimeoutError,
|
|
@@ -224,14 +254,44 @@ export const JevLive = Layer.effect(Jev)(
|
|
|
224
254
|
),
|
|
225
255
|
);
|
|
226
256
|
|
|
257
|
+
/**
|
|
258
|
+
* Every answer about a file's rules: a body for each group that
|
|
259
|
+
* `requestGroups` makes of them, each with its rule's hook run right
|
|
260
|
+
* before it goes out.
|
|
261
|
+
*/
|
|
262
|
+
const answersAbout = <A>(
|
|
263
|
+
kind: string,
|
|
264
|
+
rules: Rules,
|
|
265
|
+
bodyOf: (group: Rules) => Body & { readonly state: JevState },
|
|
266
|
+
file: JudgedFile,
|
|
267
|
+
Answers: Schema.Codec<
|
|
268
|
+
{ readonly answers: Readonly<Record<string, A>> },
|
|
269
|
+
unknown,
|
|
270
|
+
never,
|
|
271
|
+
never
|
|
272
|
+
>,
|
|
273
|
+
) =>
|
|
274
|
+
Effect.map(
|
|
275
|
+
Effect.forEach(requestGroups(rules), (group) =>
|
|
276
|
+
Effect.flatMap(hooked(bodyOf(group), group, file), (body) =>
|
|
277
|
+
answersTo(kind, body, Answers),
|
|
278
|
+
),
|
|
279
|
+
),
|
|
280
|
+
(groups) =>
|
|
281
|
+
groups.reduce<Record<string, A>>((answers, some) => ({ ...answers, ...some }), {}),
|
|
282
|
+
);
|
|
283
|
+
|
|
227
284
|
const judge = Effect.fn("Jev.judge")(function* (
|
|
228
285
|
lines: Lines,
|
|
229
286
|
rules: Rules,
|
|
230
|
-
sampled: ReadonlyArray<RuleId
|
|
287
|
+
sampled: ReadonlyArray<RuleId>,
|
|
288
|
+
file: JudgedFile,
|
|
231
289
|
) {
|
|
232
|
-
const answers = yield*
|
|
290
|
+
const answers = yield* answersAbout(
|
|
233
291
|
"judge",
|
|
234
|
-
|
|
292
|
+
rules,
|
|
293
|
+
(group) => judgeBody(config.model, lines, group, sampled),
|
|
294
|
+
file,
|
|
235
295
|
NoulAnswers,
|
|
236
296
|
);
|
|
237
297
|
/** Each rule's answer, read from the question it rode under. */
|
|
@@ -270,11 +330,17 @@ export const JevLive = Layer.effect(Jev)(
|
|
|
270
330
|
} satisfies Judged;
|
|
271
331
|
});
|
|
272
332
|
|
|
273
|
-
const locate = Effect.fn("Jev.locate")(function* (
|
|
333
|
+
const locate = Effect.fn("Jev.locate")(function* (
|
|
334
|
+
lines: Lines,
|
|
335
|
+
rules: Rules,
|
|
336
|
+
file: JudgedFile,
|
|
337
|
+
) {
|
|
274
338
|
const sections = sectionsOf(lines);
|
|
275
|
-
const answers = yield*
|
|
339
|
+
const answers = yield* answersAbout(
|
|
276
340
|
"locate",
|
|
277
|
-
|
|
341
|
+
rules,
|
|
342
|
+
(group) => locateBody(config.model, lines, group),
|
|
343
|
+
file,
|
|
278
344
|
LocateAnswers,
|
|
279
345
|
);
|
|
280
346
|
// A file of one section has every finding in it. An answer that names no section,
|
package/src/services/Jev.ts
CHANGED
|
@@ -1,4 +1,11 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import {
|
|
2
|
+
type Example,
|
|
3
|
+
examplesOf,
|
|
4
|
+
type JudgedFile,
|
|
5
|
+
type Rule,
|
|
6
|
+
type RuleId,
|
|
7
|
+
type Rules,
|
|
8
|
+
} from "#config.ts";
|
|
2
9
|
import { type Range, sectionsOf } from "#excerpt.ts";
|
|
3
10
|
import { Context, type Effect, Record, Schema } from "effect";
|
|
4
11
|
|
|
@@ -47,12 +54,13 @@ export class Jev extends Context.Service<
|
|
|
47
54
|
/**
|
|
48
55
|
* Each rule's probability that the file breaks it. A rule in `sampled`
|
|
49
56
|
* also carries the linter check's question, whose answer comes back
|
|
50
|
-
* under `linter`.
|
|
57
|
+
* under `linter`. `file` is what a rule's `appendState` gets.
|
|
51
58
|
*/
|
|
52
59
|
readonly judge: (
|
|
53
60
|
lines: Lines,
|
|
54
61
|
rules: Rules,
|
|
55
|
-
sampled
|
|
62
|
+
sampled: ReadonlyArray<RuleId>,
|
|
63
|
+
file: JudgedFile,
|
|
56
64
|
) => Effect.Effect<Judged, JevUnavailable | JevOverflow | JevBlocked>;
|
|
57
65
|
/**
|
|
58
66
|
* Per rule, the section of the file that most clearly breaks it, and
|
|
@@ -62,6 +70,7 @@ export class Jev extends Context.Service<
|
|
|
62
70
|
readonly locate: (
|
|
63
71
|
lines: Lines,
|
|
64
72
|
rules: Rules,
|
|
73
|
+
file: JudgedFile,
|
|
65
74
|
) => Effect.Effect<Record<RuleId, Located>, JevUnavailable | JevOverflow | JevBlocked>;
|
|
66
75
|
/**
|
|
67
76
|
* Per rule, the partner Jev names as impossible to follow in the same code,
|
|
@@ -468,33 +477,53 @@ export const requestsOf = (body: Body): ReadonlyArray<Body> => {
|
|
|
468
477
|
return requests.map(({ questions }) => ({ ...body, questions }));
|
|
469
478
|
};
|
|
470
479
|
|
|
480
|
+
/**
|
|
481
|
+
* The rules as requests carry them: every rule without `appendState` in one
|
|
482
|
+
* body, and each rule with it in a body of its own, so that what its hook
|
|
483
|
+
* puts in the state only its own questions read.
|
|
484
|
+
*/
|
|
485
|
+
export const requestGroups = (rules: Rules): ReadonlyArray<Rules> => {
|
|
486
|
+
const shared = Record.filter(rules, (rule) => rule.appendState === undefined);
|
|
487
|
+
return [
|
|
488
|
+
...(Record.isEmptyRecord(shared) ? [] : [shared]),
|
|
489
|
+
...Object.entries(rules).flatMap(([id, rule]) =>
|
|
490
|
+
rule.appendState === undefined ? [] : [{ [id]: rule }],
|
|
491
|
+
),
|
|
492
|
+
];
|
|
493
|
+
};
|
|
494
|
+
|
|
495
|
+
/** The judge requests for `rules` over a file: a body per group, split to fit, as `judge` sends them. */
|
|
496
|
+
const judgeRequestsOf = (lines: Lines, rules: Rules, sampled: ReadonlyArray<RuleId>) =>
|
|
497
|
+
requestGroups(rules).flatMap((group) => requestsOf(judgeBody("", lines, group, sampled)));
|
|
498
|
+
|
|
471
499
|
/**
|
|
472
500
|
* The requests judging `rules` over a file takes, and about how many input
|
|
473
|
-
* tokens they carry, by `tokensOf`: the judge
|
|
474
|
-
* sends
|
|
501
|
+
* tokens they carry, by `tokensOf`: the judge bodies, split to fit, as
|
|
502
|
+
* `judge` sends them, each request with the file again. What a rule's
|
|
503
|
+
* `appendState` adds is not known before it runs, so it is not counted.
|
|
475
504
|
*/
|
|
476
505
|
export const judgeLoad = (
|
|
477
506
|
lines: Lines,
|
|
478
507
|
rules: Rules,
|
|
479
508
|
sampled: ReadonlyArray<RuleId> = [],
|
|
480
509
|
): { readonly requests: number; readonly tokens: number } => {
|
|
481
|
-
const requests =
|
|
510
|
+
const requests = judgeRequestsOf(lines, rules, sampled);
|
|
482
511
|
return {
|
|
483
512
|
requests: requests.length,
|
|
484
513
|
tokens: requests.reduce((sum, request) => sum + tokensOf(request), 0),
|
|
485
514
|
};
|
|
486
515
|
};
|
|
487
516
|
|
|
488
|
-
/** The requests judging `rules` over a file takes: the judge
|
|
517
|
+
/** The requests judging `rules` over a file takes: the judge bodies, split to fit, as `judge` sends them. */
|
|
489
518
|
export const judgeRequests = (
|
|
490
519
|
lines: Lines,
|
|
491
520
|
rules: Rules,
|
|
492
521
|
sampled: ReadonlyArray<RuleId> = [],
|
|
493
|
-
): number =>
|
|
522
|
+
): number => judgeRequestsOf(lines, rules, sampled).length;
|
|
494
523
|
|
|
495
|
-
/** The requests locating `rules` in a file takes, split to fit as `locate` sends them. */
|
|
524
|
+
/** The requests locating `rules` in a file takes, a body per group split to fit, as `locate` sends them. */
|
|
496
525
|
export const locateRequests = (lines: Lines, rules: Rules): number =>
|
|
497
|
-
requestsOf(locateBody("", lines,
|
|
526
|
+
requestGroups(rules).flatMap((group) => requestsOf(locateBody("", lines, group))).length;
|
|
498
527
|
|
|
499
528
|
/** A choice keeps one criterion for "none", which leaves this many for partners. */
|
|
500
529
|
const PARTNERS_PER_QUESTION = CHOICE_LIMIT - 1;
|
package/src/workflows/audit.ts
CHANGED
|
@@ -41,12 +41,14 @@ export interface Finding {
|
|
|
41
41
|
/** The section's code as written, comments and all, without blank lines at either end. */
|
|
42
42
|
readonly excerpt: Excerpt;
|
|
43
43
|
/**
|
|
44
|
-
* Jev's probability that the file
|
|
45
|
-
*
|
|
46
|
-
* the report warns, and says how to add the context.
|
|
44
|
+
* Jev's probability that the file shows enough to check the rule. Below the
|
|
45
|
+
* sufficiency threshold, the report warns, and says how to add the context.
|
|
47
46
|
*/
|
|
48
|
-
readonly
|
|
47
|
+
readonly context?: number;
|
|
48
|
+
/** Jev's probability that the file breaks the rule. */
|
|
49
49
|
readonly probability: number;
|
|
50
|
+
/** The rule's threshold: lint reports the finding only when its probability is above it. */
|
|
51
|
+
readonly threshold: number;
|
|
50
52
|
/** An error fails the run; a warning does not, unless `--deny-warnings`. */
|
|
51
53
|
readonly level: Level;
|
|
52
54
|
}
|
|
@@ -109,12 +111,16 @@ const LAYOUT = "sections";
|
|
|
109
111
|
* laid out, and the questions asked for the rule, which carry the rule: its
|
|
110
112
|
* judge question, and its matchers' when it has any. An edited rule or
|
|
111
113
|
* matcher re-judges the rule, and a question asked differently re-judges
|
|
112
|
-
* every rule.
|
|
114
|
+
* every rule. So does an edited `appendState`, by its source; what the hook
|
|
115
|
+
* reads outside the file is not known until it runs, so a change there does not.
|
|
113
116
|
*/
|
|
114
117
|
const fingerprintOf = (model: string, rule: Rule) => {
|
|
115
118
|
const matchers = matcherQuestions("", rule).map(([, question]) => question);
|
|
116
119
|
const scope = matchers.length === 0 ? "" : `\u0000${JSON.stringify(matchers)}`;
|
|
117
|
-
|
|
120
|
+
const hook = rule.appendState === undefined ? "" : `\u0000${rule.appendState.toString()}`;
|
|
121
|
+
return sha256(
|
|
122
|
+
`${model}\u0000${LAYOUT}\u0000${JSON.stringify(judgeQuestion(rule))}${scope}${hook}`,
|
|
123
|
+
);
|
|
118
124
|
};
|
|
119
125
|
|
|
120
126
|
/** A section's lines as written, without the blank lines at either end. */
|
|
@@ -192,6 +198,8 @@ export interface AuditPlan {
|
|
|
192
198
|
readonly tokens: number;
|
|
193
199
|
/** The model's price per million input tokens, in dollars, when adhere knows it. */
|
|
194
200
|
readonly price?: number;
|
|
201
|
+
/** The run's threshold, which a rule's own, or an override's, replaces for that rule. */
|
|
202
|
+
readonly threshold: number;
|
|
195
203
|
/** Below this, a finding's sufficiency adds a warning to it. */
|
|
196
204
|
readonly sufficiencyThreshold: number;
|
|
197
205
|
}
|
|
@@ -459,6 +467,7 @@ export const planAudit = (
|
|
|
459
467
|
requests: loads.reduce((sum, load) => sum + load.requests, 0),
|
|
460
468
|
tokens: loads.reduce((sum, load) => sum + load.tokens, 0),
|
|
461
469
|
...(price === undefined ? {} : { price }),
|
|
470
|
+
threshold: config.threshold,
|
|
462
471
|
sufficiencyThreshold: config.sufficiencyThreshold,
|
|
463
472
|
};
|
|
464
473
|
});
|
|
@@ -509,9 +518,13 @@ export const executeAudit = (
|
|
|
509
518
|
deferred,
|
|
510
519
|
sampled,
|
|
511
520
|
}: FilePlan) {
|
|
521
|
+
// What a rule's appendState gets: the file as written, not as Jev reads it.
|
|
522
|
+
const asWritten = { path: file.path, contents: original.join("\n") };
|
|
512
523
|
const { probabilities, linter, matchers }: Judged = isEmpty(pending)
|
|
513
524
|
? { probabilities: {}, linter: {} }
|
|
514
|
-
: yield* jev
|
|
525
|
+
: yield* jev
|
|
526
|
+
.judge(file.lines, rulesOf(pending), Object.keys(sampled), asWritten)
|
|
527
|
+
.pipe(inFile(file));
|
|
515
528
|
if (!isEmpty(linter)) yield* record(file.path, sampled, linter);
|
|
516
529
|
const judged: Record<RuleId, Judgment> = { ...kept };
|
|
517
530
|
for (const [id, probability] of Object.entries(probabilities)) {
|
|
@@ -549,7 +562,7 @@ export const executeAudit = (
|
|
|
549
562
|
readonly blocked: Blocked | undefined;
|
|
550
563
|
} = isEmpty(flagged)
|
|
551
564
|
? { places: {}, blocked: undefined }
|
|
552
|
-
: yield* jev.locate(file.lines, rulesOf(flagged)).pipe(
|
|
565
|
+
: yield* jev.locate(file.lines, rulesOf(flagged), asWritten).pipe(
|
|
553
566
|
Effect.map((places) => ({ places, blocked: undefined })),
|
|
554
567
|
Effect.catchTag("JevBlocked", ({ ray }) =>
|
|
555
568
|
Effect.succeed({ places: {}, blocked: { file: file.path, ray } }),
|
|
@@ -581,11 +594,9 @@ export const executeAudit = (
|
|
|
581
594
|
section: judgment.section,
|
|
582
595
|
...(judgment.lines === undefined ? {} : { lines: judgment.lines }),
|
|
583
596
|
excerpt: excerptOf(original, judgment.section),
|
|
584
|
-
...(judgment.sufficiency
|
|
585
|
-
judgment.sufficiency < plan.sufficiencyThreshold
|
|
586
|
-
? { insufficiency: 1 - judgment.sufficiency }
|
|
587
|
-
: {}),
|
|
597
|
+
...(judgment.sufficiency === undefined ? {} : { context: judgment.sufficiency }),
|
|
588
598
|
probability: judgment.probability,
|
|
599
|
+
threshold: k.threshold,
|
|
589
600
|
level: k.rule.level ?? "error",
|
|
590
601
|
},
|
|
591
602
|
]
|
package/src/workflows/format.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { DEFAULT_SUFFICIENCY_THRESHOLD, DEFAULT_THRESHOLD } from "#config.ts";
|
|
1
2
|
import { type Kind, tokenize } from "#highlight.ts";
|
|
2
3
|
import { shownId } from "#rules.ts";
|
|
3
4
|
import type { AuditPlan, AuditResult, FileDone, Finding } from "#workflows/audit.ts";
|
|
@@ -11,11 +12,15 @@ type Role =
|
|
|
11
12
|
| "error"
|
|
12
13
|
| "warning"
|
|
13
14
|
| "probability"
|
|
15
|
+
| "low"
|
|
16
|
+
| "below"
|
|
14
17
|
| "description"
|
|
15
18
|
| "path"
|
|
16
19
|
| "lineNumber"
|
|
17
20
|
| "underline"
|
|
18
21
|
| "label"
|
|
22
|
+
| "legend"
|
|
23
|
+
| "quiet"
|
|
19
24
|
| Kind;
|
|
20
25
|
|
|
21
26
|
interface Span {
|
|
@@ -30,20 +35,28 @@ type Line = ReadonlyArray<Span>;
|
|
|
30
35
|
* palette (`38;5;N`) or truecolor (`38;2;R;G;B`), bold with a trailing `;1`,
|
|
31
36
|
* and a dim line number. Of the header, only the `×` and the rule are red, or
|
|
32
37
|
* a warning's `⚠` and rule amber, since a whole line of red is hard to read:
|
|
33
|
-
* the
|
|
34
|
-
*
|
|
35
|
-
*
|
|
36
|
-
*
|
|
38
|
+
* the description is a cool near-white, set here since a theme's own white
|
|
39
|
+
* can be gray. A score's three colors are one set, of like lightness, so
|
|
40
|
+
* they read as a scale, and none is the red of the `×`. Its labels, and the
|
|
41
|
+
* legend's meanings, are dim. The legend's words are bold in the terminal's
|
|
42
|
+
* own color, so they do not read as the hint's label. The code's are the
|
|
43
|
+
* terminal's own colors (`3N`), so the
|
|
44
|
+
* user's theme picks shades that read on its background; none is magenta,
|
|
45
|
+
* which would run into the underline.
|
|
37
46
|
*/
|
|
38
47
|
const THEME: Readonly<Record<Role, string>> = {
|
|
39
48
|
error: "38;2;164;20;71;1",
|
|
40
49
|
warning: "38;2;214;154;0;1",
|
|
41
|
-
probability: "38;
|
|
50
|
+
probability: "38;2;152;195;121",
|
|
51
|
+
low: "38;2;229;192;123",
|
|
52
|
+
below: "38;2;224;108;117",
|
|
42
53
|
description: "38;2;230;230;255",
|
|
43
54
|
path: "38;2;5;125;160;1",
|
|
44
55
|
lineNumber: "2",
|
|
45
56
|
underline: "38;2;255;0;175",
|
|
46
57
|
label: "38;2;242;205;205",
|
|
58
|
+
legend: "1",
|
|
59
|
+
quiet: "2",
|
|
47
60
|
comment: "2",
|
|
48
61
|
string: "32",
|
|
49
62
|
constant: "33",
|
|
@@ -90,15 +103,60 @@ const displayPath = (file: string, root: string | undefined): string =>
|
|
|
90
103
|
const counted = (count: number, one: string, many: string): string =>
|
|
91
104
|
`${count} ${count === 1 ? one : many}`;
|
|
92
105
|
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
106
|
+
/** A score as a header shows it, in hundredths, so two compare as they read. */
|
|
107
|
+
const hundredths = (value: number): number => Math.round(value * 100);
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* Where a confidence stops being far below a threshold and where it starts
|
|
111
|
+
* being well above it, in hundredths. Between the two it is near the
|
|
112
|
+
* threshold: within a quarter of the room above it, on either side, so the
|
|
113
|
+
* band narrows as the threshold rises and leaves room above for scores to be
|
|
114
|
+
* well clear of it. Scores that gather in the band, reported or not, turn on
|
|
115
|
+
* where the threshold sits, a sign it may be too low or too high.
|
|
116
|
+
*/
|
|
117
|
+
const bandsOf = (threshold: number): { readonly near: number; readonly clear: number } => {
|
|
118
|
+
const at = hundredths(threshold);
|
|
119
|
+
const margin = (100 - at) / 4;
|
|
120
|
+
return { near: Math.max(0, Math.ceil(at - margin)), clear: Math.ceil(at + margin) };
|
|
121
|
+
};
|
|
122
|
+
|
|
123
|
+
/**
|
|
124
|
+
* A score in one of three ranges set by its threshold: amber near it, red
|
|
125
|
+
* below that, the accent above. A confidence's threshold is its rule's, and
|
|
126
|
+
* a red one shows only in a run that prints every judgment. A context's is
|
|
127
|
+
* the sufficiency threshold, and a red one marks the finding to doubt first.
|
|
128
|
+
*/
|
|
129
|
+
const score = (value: number, threshold: number): Span => {
|
|
130
|
+
const { near, clear } = bandsOf(threshold);
|
|
131
|
+
const shown = hundredths(value);
|
|
132
|
+
return span(value.toFixed(2), shown < near ? "below" : shown < clear ? "low" : "probability");
|
|
133
|
+
};
|
|
134
|
+
|
|
135
|
+
const lacksContext = (finding: Finding, sufficiencyThreshold: number): boolean =>
|
|
136
|
+
finding.context !== undefined && finding.context < sufficiencyThreshold;
|
|
137
|
+
|
|
138
|
+
/**
|
|
139
|
+
* The rule and its description, as `vp lint` heads a diagnostic, then the
|
|
140
|
+
* scores on a line of their own, where they start in one place whatever the
|
|
141
|
+
* rule's length. Their labels are dim, so only the numbers carry color.
|
|
142
|
+
*/
|
|
143
|
+
const header = (finding: Finding, sufficiencyThreshold: number): ReadonlyArray<Line> => [
|
|
144
|
+
[
|
|
145
|
+
span(" "),
|
|
146
|
+
span(finding.level === "warning" ? "⚠" : "×", finding.level),
|
|
147
|
+
span(" "),
|
|
148
|
+
span(shownId({ id: finding.rule, preset: finding.preset }), finding.level),
|
|
149
|
+
span(": "),
|
|
150
|
+
span(finding.description, "description"),
|
|
151
|
+
],
|
|
152
|
+
[
|
|
153
|
+
span(" "),
|
|
154
|
+
span("confidence ", "quiet"),
|
|
155
|
+
score(finding.probability, finding.threshold),
|
|
156
|
+
...(finding.context === undefined
|
|
157
|
+
? []
|
|
158
|
+
: [span(" · context ", "quiet"), score(finding.context, sufficiencyThreshold)]),
|
|
159
|
+
],
|
|
102
160
|
];
|
|
103
161
|
|
|
104
162
|
/**
|
|
@@ -151,19 +209,14 @@ const excerpt = (finding: Finding, root: string | undefined): ReadonlyArray<Line
|
|
|
151
209
|
|
|
152
210
|
/**
|
|
153
211
|
* The warning on a finding whose file may not show enough to check its rule,
|
|
154
|
-
*
|
|
155
|
-
* code relies on as a note Jev reads, in the form the README asks for.
|
|
212
|
+
* which its context score in the header says how far, and help: how to add
|
|
213
|
+
* what the code relies on as a note Jev reads, in the form the README asks for.
|
|
156
214
|
*/
|
|
157
|
-
const insufficient = (finding: Finding): ReadonlyArray<Line> =>
|
|
158
|
-
finding
|
|
215
|
+
const insufficient = (finding: Finding, sufficiencyThreshold: number): ReadonlyArray<Line> =>
|
|
216
|
+
!lacksContext(finding, sufficiencyThreshold)
|
|
159
217
|
? []
|
|
160
218
|
: [
|
|
161
|
-
[
|
|
162
|
-
span(" warning: ", "warning"),
|
|
163
|
-
span(
|
|
164
|
-
`this file may not show enough to check this rule (Jev: ${finding.insufficiency.toFixed(2)} that it does not)`,
|
|
165
|
-
),
|
|
166
|
-
],
|
|
219
|
+
[span(" warning: ", "warning"), span("this file may not show enough to check this rule")],
|
|
167
220
|
...hanging(span(" help: ", "label"), [
|
|
168
221
|
[span("add what the code relies on outside this file as a note Jev reads:")],
|
|
169
222
|
highlighted("// @adhere <the fact>, and how you know it")[0] ?? [],
|
|
@@ -183,17 +236,91 @@ const hint = (finding: Finding): ReadonlyArray<Line> => {
|
|
|
183
236
|
|
|
184
237
|
/**
|
|
185
238
|
* One diagnostic, in the frame `vp lint` prints on a terminal: a header with
|
|
186
|
-
* the rule in red
|
|
187
|
-
* names in pink, a warning when the file may
|
|
188
|
-
* the code to write as the hint.
|
|
239
|
+
* the rule in red and its scores, the description, the section of the file
|
|
240
|
+
* Jev points at with the lines it names in pink, a warning when the file may
|
|
241
|
+
* not show enough to decide, and the code to write as the hint.
|
|
189
242
|
*/
|
|
190
|
-
const frame = (
|
|
191
|
-
|
|
243
|
+
const frame = (
|
|
244
|
+
finding: Finding,
|
|
245
|
+
root: string | undefined,
|
|
246
|
+
sufficiencyThreshold: number,
|
|
247
|
+
): ReadonlyArray<Line> => [
|
|
248
|
+
...header(finding, sufficiencyThreshold),
|
|
192
249
|
...excerpt(finding, root),
|
|
193
|
-
...insufficient(finding),
|
|
250
|
+
...insufficient(finding, sufficiencyThreshold),
|
|
194
251
|
...hint(finding),
|
|
195
252
|
];
|
|
196
253
|
|
|
254
|
+
/** A bound of a range: hundredths as a score shows them, and 0 and 1 without decimals. */
|
|
255
|
+
const bound = (value: number): string =>
|
|
256
|
+
value === 0 ? "0" : value === 100 ? "1" : (value / 100).toFixed(2);
|
|
257
|
+
|
|
258
|
+
/** The scores from one number of hundredths through another: `0–0.74`, or `0.85–1`. */
|
|
259
|
+
const range = (from: number, to: number): string => `${bound(from)}–${bound(to)}`;
|
|
260
|
+
|
|
261
|
+
/**
|
|
262
|
+
* A legend row's cells for a threshold: the scores far below it, the
|
|
263
|
+
* threshold, and the scores well above it, as in `0–0.74 < 0.8 < 0.85–1`.
|
|
264
|
+
* The scores between the two ranges are the ones near the threshold, so it
|
|
265
|
+
* and each `<` are in their color, and the ranges in theirs. A threshold too low to
|
|
266
|
+
* have scores far below it has no first range, and no `<` after one.
|
|
267
|
+
*/
|
|
268
|
+
const cellsOf = (threshold: number): ReadonlyArray<Span> => {
|
|
269
|
+
const { near, clear } = bandsOf(threshold);
|
|
270
|
+
return [
|
|
271
|
+
span(near > 0 ? range(0, near - 1) : "", "below"),
|
|
272
|
+
span(near > 0 ? "<" : "", "low"),
|
|
273
|
+
span(String(hundredths(threshold) / 100), "low"),
|
|
274
|
+
span("<", "low"),
|
|
275
|
+
span(range(clear, 100), "probability"),
|
|
276
|
+
];
|
|
277
|
+
};
|
|
278
|
+
|
|
279
|
+
/**
|
|
280
|
+
* What the scores under a header mean, once under the findings, as a table:
|
|
281
|
+
* a row for each score with its ranges around its threshold, then what the
|
|
282
|
+
* score is, dim. The cells stand in columns, so a color lines up down the
|
|
283
|
+
* rows. A confidence's threshold is the run's; a rule with a threshold of
|
|
284
|
+
* its own has its ranges around that. A context's is the sufficiency
|
|
285
|
+
* threshold.
|
|
286
|
+
*/
|
|
287
|
+
const legend = (
|
|
288
|
+
result: AuditResult,
|
|
289
|
+
threshold: number,
|
|
290
|
+
sufficiencyThreshold: number,
|
|
291
|
+
): ReadonlyArray<Line> => {
|
|
292
|
+
const rows = [
|
|
293
|
+
["confidence", cellsOf(threshold), "Jev's probability that the file breaks the rule"],
|
|
294
|
+
[
|
|
295
|
+
"context",
|
|
296
|
+
cellsOf(sufficiencyThreshold),
|
|
297
|
+
"its probability that the file shows enough to decide",
|
|
298
|
+
],
|
|
299
|
+
] as const;
|
|
300
|
+
const padding = (text: string, width: number): Span => span(" ".repeat(width - text.length));
|
|
301
|
+
const labels = Math.max(...rows.map(([label]) => label.length));
|
|
302
|
+
const widthOf = (column: number): number =>
|
|
303
|
+
Math.max(...rows.map(([, cells]) => cells[column]?.text.length ?? 0));
|
|
304
|
+
return result.findings.length === 0
|
|
305
|
+
? []
|
|
306
|
+
: [
|
|
307
|
+
...rows.map(([label, cells, meaning]) => [
|
|
308
|
+
span(" "),
|
|
309
|
+
span(label, "legend"),
|
|
310
|
+
padding(label, labels),
|
|
311
|
+
span(" "),
|
|
312
|
+
...cells.flatMap((cell, column) => [
|
|
313
|
+
span(" "),
|
|
314
|
+
cell,
|
|
315
|
+
padding(cell.text, widthOf(column)),
|
|
316
|
+
]),
|
|
317
|
+
span(" "),
|
|
318
|
+
span(meaning, "quiet"),
|
|
319
|
+
]),
|
|
320
|
+
[],
|
|
321
|
+
];
|
|
322
|
+
};
|
|
323
|
+
|
|
197
324
|
/** The findings counted by level: `1 error`, or `2 errors and 1 warning` once there are warnings. */
|
|
198
325
|
const found = (findings: ReadonlyArray<Finding>): string => {
|
|
199
326
|
const warnings = findings.filter((finding) => finding.level === "warning").length;
|
|
@@ -239,19 +366,29 @@ export interface RenderOptions {
|
|
|
239
366
|
/** Color the frame the way `vp lint` does on a terminal, and highlight the code. */
|
|
240
367
|
readonly color?: boolean;
|
|
241
368
|
readonly root?: string;
|
|
369
|
+
/** The run's threshold, for the legend: lint reports no finding at or below it. Defaults to 0.8. */
|
|
370
|
+
readonly threshold?: number;
|
|
371
|
+
/** Below this, a finding's context is low and it carries a warning. Defaults to 0.6. */
|
|
372
|
+
readonly sufficiencyThreshold?: number;
|
|
242
373
|
}
|
|
243
374
|
|
|
244
375
|
/**
|
|
245
376
|
* The report, in the default `vp lint` layout: each diagnostic followed by a
|
|
246
|
-
* blank line, then the
|
|
247
|
-
* terminal.
|
|
377
|
+
* blank line, then the legend for their scores, then the counts. Color is on
|
|
378
|
+
* when the caller is writing to a terminal.
|
|
248
379
|
*/
|
|
249
|
-
export const render = (result: AuditResult, options: RenderOptions = {}): ReadonlyArray<string> =>
|
|
250
|
-
|
|
251
|
-
|
|
380
|
+
export const render = (result: AuditResult, options: RenderOptions = {}): ReadonlyArray<string> => {
|
|
381
|
+
const sufficiencyThreshold = options.sufficiencyThreshold ?? DEFAULT_SUFFICIENCY_THRESHOLD;
|
|
382
|
+
return [
|
|
383
|
+
...result.findings.flatMap((finding) => [
|
|
384
|
+
...frame(finding, options.root, sufficiencyThreshold),
|
|
385
|
+
[],
|
|
386
|
+
]),
|
|
387
|
+
...legend(result, options.threshold ?? DEFAULT_THRESHOLD, sufficiencyThreshold),
|
|
252
388
|
...summary(result),
|
|
253
389
|
...blockedFiles(result, options.root),
|
|
254
390
|
].map((line) => serialize(line, options.color === true));
|
|
391
|
+
};
|
|
255
392
|
|
|
256
393
|
/**
|
|
257
394
|
* What a run will do, before it asks: the files and rules found, the checks
|
/package/presets/alchemy/apis/{match-remote-errors-by-tag.md → match-remote-errors-by-tag/RULE.md}
RENAMED
|
File without changes
|
/package/presets/alchemy/apis/{schemas-at-trust-boundaries.md → schemas-at-trust-boundaries/RULE.md}
RENAMED
|
File without changes
|
|
File without changes
|
/package/presets/alchemy/data/{hyperdrive-direct-origin.md → hyperdrive-direct-origin/RULE.md}
RENAMED
|
File without changes
|
/package/presets/alchemy/data/{migrations-use-direct-url.md → migrations-use-direct-url/RULE.md}
RENAMED
|
File without changes
|
|
File without changes
|
/package/presets/alchemy/data/{private-endpoint-at-runtime.md → private-endpoint-at-runtime/RULE.md}
RENAMED
|
File without changes
|
|
File without changes
|
/package/presets/alchemy/durable/{idempotent-event-handlers.md → idempotent-event-handlers/RULE.md}
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
/package/presets/alchemy/durable/{webhook-state-is-truth.md → webhook-state-is-truth/RULE.md}
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
/package/presets/alchemy/providers/{lazy-layer-construction.md → lazy-layer-construction/RULE.md}
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|