@autotelic/joggle 0.1.2 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/JOGGLE.md +16 -8
- package/README.md +5 -4
- package/dist/answer-store.js +284 -151
- package/dist/check.js +32 -4
- package/dist/config.js +22 -0
- package/dist/decision.js +3 -1
- package/dist/main.js +53 -8
- package/dist/tsgo.js +37 -15
- package/package.json +1 -1
package/JOGGLE.md
CHANGED
|
@@ -228,13 +228,15 @@ It is serialised with sorted keys, so the same candidate produces the same key
|
|
|
228
228
|
on every machine — which means **run from the repository root**. Evidence
|
|
229
229
|
carries root-relative paths; running from a subdirectory changes the key and
|
|
230
230
|
misses every cached verdict. Successful judgements are written to
|
|
231
|
-
`.joggle/answers
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
231
|
+
`.joggle/answers.jsonl`, one sorted, line-oriented answer per line; commit that
|
|
232
|
+
file and CI replays them with `--offline` and **no API key at all**. A new
|
|
233
|
+
answer is a new line, so two branches that judge different candidates merge with
|
|
234
|
+
git's built-in `merge=union`, and `joggle cache prune` keeps the store from
|
|
235
|
+
growing without a ceiling. A store large enough to make a whole-file rewrite or a
|
|
236
|
+
merge expensive can be sharded instead (`cache.layout: "sharded"`, or
|
|
237
|
+
`joggle cache migrate --layout sharded`); both layouts are read, and
|
|
238
|
+
`cache migrate` moves between them. The older `answers.json` object is read and
|
|
239
|
+
consolidated on the next write. The layouts and the deployment modes are in
|
|
238
240
|
[docs/artifacts.md](./docs/artifacts.md). Bump `policy.decisionVersion` when you
|
|
239
241
|
change a question's wording, and every cached answer for it is invalidated at
|
|
240
242
|
once instead of silently replayed against newer questions.
|
|
@@ -426,7 +428,7 @@ and two belong to the machine.
|
|
|
426
428
|
|
|
427
429
|
| artifact | lives in | committed | what it is |
|
|
428
430
|
| --- | --- | --- | --- |
|
|
429
|
-
| `answers/` | `<root>/.joggle/` | **yes** | one
|
|
431
|
+
| `answers.jsonl` (or `answers/` when sharded) | `<root>/.joggle/` | **yes** | one line per answer, sorted by key, keyed by the decision and the state it read. **This is the replay path**: CI replays it with no API key and no tokens. |
|
|
430
432
|
| `baseline/` | `<root>/.joggle/` | **yes** | the findings already accepted, one fragment per finding: the ratchet. |
|
|
431
433
|
| `wire.json` | `<machine cache>/joggle/<root>/` | no | every wire request's response, keyed by the request. A performance artifact, and far too large to commit: 11.5 megabytes on a 2,493-file repository. |
|
|
432
434
|
| `last-run.json` | `<machine cache>/joggle/<root>/` | no | a manifest and the last report. A performance artifact. |
|
|
@@ -447,6 +449,12 @@ joggle check # nothing changed: the previo
|
|
|
447
449
|
joggle cache prune --max-age 90 # the cache is regenerable; keep it small
|
|
448
450
|
```
|
|
449
451
|
|
|
452
|
+
`--update-baseline` refuses while any candidate was not judged
|
|
453
|
+
(`unreadable`, `no_evidence`, or past a budget): a baseline is the record of
|
|
454
|
+
what a person reviewed, and one written over unjudged candidates ratifies what
|
|
455
|
+
nobody looked at. `--allow-unjudged-baseline` is the explicit override; `jig
|
|
456
|
+
init` passes it, because a ledger born on day one has no review to protect yet.
|
|
457
|
+
|
|
450
458
|
The baseline is by identity, not by line: rule, kept symbol and cluster
|
|
451
459
|
membership. Code moves constantly, and a baseline that reports every edit as a
|
|
452
460
|
new finding is a baseline nobody reads. A rename or a new member does change the
|
package/README.md
CHANGED
|
@@ -46,10 +46,11 @@ among named options, a spectrum) with calibrated probabilities, and code keeps t
|
|
|
46
46
|
thresholds: a decisive answer prints, an unsure one is recorded as a notice, a no is
|
|
47
47
|
silent. Arithmetic, counts, spans and orderings all stay in code.
|
|
48
48
|
|
|
49
|
-
Verdicts are cached per question in `.joggle/answers
|
|
50
|
-
|
|
51
|
-
tokens. The
|
|
52
|
-
|
|
49
|
+
Verdicts are cached per question in `.joggle/answers.jsonl`, one sorted line per
|
|
50
|
+
answer, so commit that file and CI replays every judgement with no key and no
|
|
51
|
+
tokens. The lines are keyed and `merge=union` is enough for a branch merge, and
|
|
52
|
+
`joggle cache prune` keeps the store from growing without a ceiling. A large
|
|
53
|
+
store can be sharded instead (`cache.layout: "sharded"`); both are read. A yes/no is asked three times
|
|
53
54
|
and the **agreement** across the asks becomes its confidence, because one number on
|
|
54
55
|
its own has no margin. [How joggle uses TypeSafe](./docs/typesafe.md).
|
|
55
56
|
|
package/dist/answer-store.js
CHANGED
|
@@ -2,46 +2,68 @@ import { Clock, Effect, FileSystem, Option, Order, Path, Ref, Result, Schema, Sc
|
|
|
2
2
|
import { canonical } from "./canonical.js";
|
|
3
3
|
import { StoredAnswer } from "./plans.js";
|
|
4
4
|
/**
|
|
5
|
-
* The answer cache
|
|
5
|
+
* The answer cache: one logical ledger, two on-disk layouts.
|
|
6
6
|
*
|
|
7
|
-
*
|
|
8
|
-
* key
|
|
9
|
-
*
|
|
10
|
-
* judge different candidates write different shards -- or, when they collide on
|
|
11
|
-
* a shard, different lines. That is the whole design: a merge is a directory
|
|
12
|
-
* merge, and the residual collision is a same-line one, which the shipped merge
|
|
13
|
-
* driver (`joggle merge-answers`) resolves by key.
|
|
7
|
+
* An entry is a line of `<key>\t<unix-seconds>\t<json>`, sorted by key. The
|
|
8
|
+
* key (`answerKeyFor`) already content-addresses the decision, the atoms it
|
|
9
|
+
* read, the model and the decision version, so the line is the identity.
|
|
14
10
|
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
11
|
+
* **Single** (the default) is one file, `answers.jsonl`. It is the shape a
|
|
12
|
+
* reviewer can read: a verdict change is one changed line in one diff. Sharding
|
|
13
|
+
* bought merge granularity at a cost nobody was paying -- 130 shard files for 182
|
|
14
|
+
* entries is a zero-reviewability commit, and git cannot ship the merge driver
|
|
15
|
+
* that was supposed to make the shards safe.
|
|
20
16
|
*
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
*
|
|
17
|
+
* **Sharded** is `answers/<first two hex chars>.jsonl`, chosen explicitly when a
|
|
18
|
+
* store is large enough that a whole-file rewrite or a merge is a real cost. It
|
|
19
|
+
* uses git's built-in `merge=union`, which needs no per-machine registration.
|
|
20
|
+
*
|
|
21
|
+
* Both are read. The layout is decided by what is on disk, unless the caller
|
|
22
|
+
* names one; `joggle cache migrate --layout` rewrites in the other shape. The
|
|
23
|
+
* older `answers.json` object is read and consolidated into the single file on
|
|
24
|
+
* the first write.
|
|
24
25
|
*/
|
|
25
|
-
// How many leading hex characters name a shard. 256 files is enough granularity.
|
|
26
26
|
const SHARD_CHARS = 2;
|
|
27
27
|
/** The shard a key belongs to: its first two characters, lower-cased. */
|
|
28
28
|
export const shardOf = (key) => key.slice(0, SHARD_CHARS).toLowerCase().padEnd(SHARD_CHARS, "_");
|
|
29
29
|
/** The directory holding the shards, beside the single-file form. */
|
|
30
30
|
export const answersDir = (path, dir) => path.join(dir, "answers");
|
|
31
|
-
|
|
31
|
+
/** The one-file ledger. */
|
|
32
|
+
export const answersFile = (path, dir) => path.join(dir, "answers.jsonl");
|
|
33
|
+
/** The older pretty-printed object, still read. */
|
|
34
|
+
const answersObject = (path, dir) => path.join(dir, "answers.json");
|
|
32
35
|
/**
|
|
33
36
|
* The attributes joggle manages for the cache directory itself.
|
|
34
37
|
*
|
|
35
38
|
* Git reads a `.gitattributes` in a subdirectory and applies its patterns
|
|
36
|
-
* relative to that directory, so joggle
|
|
37
|
-
* without touching the repository's top-level file.
|
|
38
|
-
*
|
|
39
|
-
*
|
|
39
|
+
* relative to that directory, so joggle owns the treatment of its own files
|
|
40
|
+
* without touching the repository's top-level file.
|
|
41
|
+
*
|
|
42
|
+
* `merge=union` rather than a custom driver: git cannot ship a merge driver in
|
|
43
|
+
* `.gitattributes`, so `merge=joggle-answers` only worked on a machine where
|
|
44
|
+
* somebody registered it by hand -- and a fresh clone silently got the default
|
|
45
|
+
* merge instead. Union needs no registration and is correct for an append-only,
|
|
46
|
+
* key-sorted JSONL.
|
|
47
|
+
*
|
|
48
|
+
* No `-diff` and no `linguist-generated`: the ledger's justification is that a
|
|
49
|
+
* maintainer reviewed the verdicts, and those attributes made it a binary blob
|
|
50
|
+
* that GitHub collapses. A verdict change should be one visible line.
|
|
51
|
+
*/
|
|
52
|
+
const ATTRIBUTES = "answers.jsonl merge=union\n" + "answers/*.jsonl merge=union\n" + "answers.json merge=union\n";
|
|
53
|
+
/**
|
|
54
|
+
* Banners earlier versions wrote.
|
|
55
|
+
*
|
|
56
|
+
* Rewritten in place because each is byte-identical to what joggle generated, so
|
|
57
|
+
* this is a migration rather than an edit of a file a person may have changed.
|
|
58
|
+
* Any other content is left alone.
|
|
40
59
|
*/
|
|
41
|
-
const
|
|
42
|
-
"answers
|
|
43
|
-
|
|
44
|
-
|
|
60
|
+
const LEGACY_ATTRIBUTES = [
|
|
61
|
+
"answers/*.jsonl merge=joggle-answers linguist-generated -diff\n" +
|
|
62
|
+
"answers.json linguist-generated -diff\n",
|
|
63
|
+
"answers/*.jsonl merge=union\n" + "answers.json merge=union\n",
|
|
64
|
+
];
|
|
65
|
+
/** The shape of the older single object, read once and consolidated. */
|
|
66
|
+
const AnswersObject = Schema.Struct({
|
|
45
67
|
version: Schema.String,
|
|
46
68
|
entries: Schema.Record(Schema.String, StoredAnswer),
|
|
47
69
|
});
|
|
@@ -51,7 +73,7 @@ const decodeAnswer = Schema.fromJsonString(StoredAnswer);
|
|
|
51
73
|
* any of the three fields, so the split is unambiguous.
|
|
52
74
|
*/
|
|
53
75
|
export const renderEntry = (entry) => entry.key + "\t" + entry.writtenAt + "\t" + canonical(entry.answer);
|
|
54
|
-
/** Parse a
|
|
76
|
+
/** Parse a ledger file. A line that does not decode is skipped and COUNTED, never fatal. */
|
|
55
77
|
export const parseLines = (text) => {
|
|
56
78
|
const entries = [];
|
|
57
79
|
let skippedLines = 0;
|
|
@@ -75,13 +97,13 @@ export const parseLines = (text) => {
|
|
|
75
97
|
return { entries, skippedLines };
|
|
76
98
|
};
|
|
77
99
|
const byKey = Order.mapInput(Order.String, (entry) => entry.key);
|
|
78
|
-
/** A
|
|
100
|
+
/** A file body: entries sorted by key, newline-terminated. */
|
|
79
101
|
export const renderShard = (entries) => {
|
|
80
102
|
const sorted = [...entries].sort(byKey);
|
|
81
103
|
return sorted.length === 0 ? "" : sorted.map(renderEntry).join("\n") + "\n";
|
|
82
104
|
};
|
|
83
105
|
/**
|
|
84
|
-
* Union two sides of one
|
|
106
|
+
* Union two sides of one file by key.
|
|
85
107
|
*
|
|
86
108
|
* A key that only one side carries is kept. A key both sides carry with
|
|
87
109
|
* different content keeps the newer timestamp: for a content-addressed key the
|
|
@@ -100,7 +122,7 @@ export const mergeEntries = ({ ours, theirs, }) => {
|
|
|
100
122
|
return [...merged.values()];
|
|
101
123
|
};
|
|
102
124
|
/**
|
|
103
|
-
* A side of a merge that is not a
|
|
125
|
+
* A side of a merge that is not a ledger file at all.
|
|
104
126
|
*
|
|
105
127
|
* A tagged error rather than `Option.none`, because the reason is the whole
|
|
106
128
|
* value: "ours" or "theirs" is what the driver prints and what a person acts on.
|
|
@@ -110,7 +132,7 @@ export class NotAShard extends Schema.TaggedError()("joggle/NotAShard", {
|
|
|
110
132
|
}) {
|
|
111
133
|
}
|
|
112
134
|
/**
|
|
113
|
-
* A merge of one
|
|
135
|
+
* A merge of one file, failing when a non-empty side is not a ledger at all.
|
|
114
136
|
*
|
|
115
137
|
* Failing is how the driver declines: git then records an ordinary conflict,
|
|
116
138
|
* which is recoverable, instead of a union that silently dropped lines. Every
|
|
@@ -127,179 +149,290 @@ export const mergeShardText = (text) => Effect.gen(function* () {
|
|
|
127
149
|
}
|
|
128
150
|
return renderShard(mergeEntries({ ours: ourEntries, theirs: theirEntries }));
|
|
129
151
|
});
|
|
152
|
+
/* -------------------------------------------------------------------------- */
|
|
153
|
+
/* Reading and writing both layouts */
|
|
154
|
+
/* -------------------------------------------------------------------------- */
|
|
130
155
|
const readText = (fs, file) => fs.readFileString(file).pipe(Effect.orElseSucceed(() => ""));
|
|
131
156
|
const exists = (fs, file) => fs.exists(file).pipe(Effect.orElseSucceed(() => false));
|
|
132
157
|
const seconds = (millis) => Math.floor(millis / 1000);
|
|
133
|
-
const
|
|
134
|
-
const next = new Map(shards);
|
|
135
|
-
const inner = new Map(next.get(shardOf(entry.key)) ?? []);
|
|
136
|
-
inner.set(entry.key, entry);
|
|
137
|
-
next.set(shardOf(entry.key), inner);
|
|
138
|
-
return next;
|
|
139
|
-
};
|
|
140
|
-
const count = (shards) => {
|
|
141
|
-
let total = 0;
|
|
142
|
-
for (const entries of shards.values())
|
|
143
|
-
total += entries.size;
|
|
144
|
-
return total;
|
|
145
|
-
};
|
|
158
|
+
const bytesOf = (text) => new TextEncoder().encode(text).length;
|
|
146
159
|
/**
|
|
147
|
-
*
|
|
148
|
-
* dirty shards so a write is proportional to the change.
|
|
160
|
+
* Read every form into one map, newest timestamp winning a shared key.
|
|
149
161
|
*
|
|
150
|
-
*
|
|
151
|
-
*
|
|
152
|
-
*
|
|
153
|
-
* crash in between leaves both and the sharded side wins on the next read.
|
|
162
|
+
* The layouts are not mutually exclusive on disk: a repository that migrated
|
|
163
|
+
* may still carry the old file after a crash, and reading both is how that
|
|
164
|
+
* degrades to the union rather than to whichever the walk hit first.
|
|
154
165
|
*/
|
|
155
|
-
|
|
156
|
-
const single = singleFile(path, dir);
|
|
157
|
-
const migrating = yield* exists(fs, single);
|
|
166
|
+
const loadAll = (fs, path, dir) => Effect.gen(function* () {
|
|
158
167
|
const now = seconds(yield* Clock.currentTimeMillis);
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
const
|
|
162
|
-
|
|
168
|
+
const merged = new Map();
|
|
169
|
+
const absorb = (entry) => {
|
|
170
|
+
const existing = merged.get(entry.key);
|
|
171
|
+
if (existing === undefined || entry.writtenAt > existing.writtenAt)
|
|
172
|
+
merged.set(entry.key, entry);
|
|
173
|
+
};
|
|
174
|
+
let files = 0;
|
|
175
|
+
let bytes = 0;
|
|
176
|
+
let skippedLines = 0;
|
|
177
|
+
const older = answersObject(path, dir);
|
|
178
|
+
if (yield* exists(fs, older)) {
|
|
179
|
+
const text = yield* readText(fs, older);
|
|
180
|
+
const decoded = Result.getOrUndefined(SchemaParser.decodeResult(Schema.fromJsonString(AnswersObject))(text));
|
|
163
181
|
if (decoded !== undefined) {
|
|
182
|
+
files += 1;
|
|
183
|
+
bytes += bytesOf(text);
|
|
164
184
|
for (const [key, answer] of Object.entries(decoded.entries))
|
|
165
|
-
|
|
185
|
+
absorb({ key, writtenAt: now, answer });
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
const single = answersFile(path, dir);
|
|
189
|
+
const singleExists = yield* exists(fs, single);
|
|
190
|
+
if (singleExists) {
|
|
191
|
+
const text = yield* readText(fs, single);
|
|
192
|
+
const parsed = parseLines(text);
|
|
193
|
+
files += 1;
|
|
194
|
+
bytes += bytesOf(text);
|
|
195
|
+
skippedLines += parsed.skippedLines;
|
|
196
|
+
for (const entry of parsed.entries)
|
|
197
|
+
absorb(entry);
|
|
198
|
+
}
|
|
199
|
+
const target = answersDir(path, dir);
|
|
200
|
+
const names = yield* fs.readDirectory(target).pipe(Effect.orElseSucceed(() => []));
|
|
201
|
+
let shards = 0;
|
|
202
|
+
for (const name of names) {
|
|
203
|
+
if (!name.endsWith(".jsonl"))
|
|
204
|
+
continue;
|
|
205
|
+
const text = yield* readText(fs, path.join(target, name));
|
|
206
|
+
const parsed = parseLines(text);
|
|
207
|
+
if (text === "")
|
|
208
|
+
continue;
|
|
209
|
+
shards += 1;
|
|
210
|
+
files += 1;
|
|
211
|
+
bytes += bytesOf(text);
|
|
212
|
+
skippedLines += parsed.skippedLines;
|
|
213
|
+
for (const entry of parsed.entries)
|
|
214
|
+
absorb(entry);
|
|
215
|
+
}
|
|
216
|
+
// Sharded wins when both exist: it is the form a migration finishes by
|
|
217
|
+
// writing, so a half-finished single-to-sharded run reads as sharded.
|
|
218
|
+
return {
|
|
219
|
+
entries: merged,
|
|
220
|
+
layout: shards > 0 ? "sharded" : "single",
|
|
221
|
+
files,
|
|
222
|
+
bytes,
|
|
223
|
+
skippedLines,
|
|
224
|
+
};
|
|
225
|
+
});
|
|
226
|
+
const writeAttributes = (fs, path, dir) => Effect.gen(function* () {
|
|
227
|
+
const attributes = path.join(dir, ".gitattributes");
|
|
228
|
+
if (!(yield* exists(fs, attributes))) {
|
|
229
|
+
yield* fs.writeFileString(attributes, ATTRIBUTES).pipe(Effect.orElseSucceed(() => undefined));
|
|
230
|
+
return;
|
|
231
|
+
}
|
|
232
|
+
const current = yield* readText(fs, attributes);
|
|
233
|
+
if (LEGACY_ATTRIBUTES.includes(current)) {
|
|
234
|
+
yield* fs.writeFileString(attributes, ATTRIBUTES).pipe(Effect.orElseSucceed(() => undefined));
|
|
235
|
+
}
|
|
236
|
+
});
|
|
237
|
+
const removeQuietly = (fs, file) => fs.remove(file).pipe(Effect.orElseSucceed(() => undefined));
|
|
238
|
+
/**
|
|
239
|
+
* Write the whole ledger in one layout, and remove the other form.
|
|
240
|
+
*
|
|
241
|
+
* A full write is what migration and pruning do; the store's own `put` writes
|
|
242
|
+
* only what changed. Removing the other form first would lose it on a crash, so
|
|
243
|
+
* the new form is written first and the old one removed after.
|
|
244
|
+
*/
|
|
245
|
+
const writeAll = (fs, path, dir, layout, entries) => Effect.gen(function* () {
|
|
246
|
+
yield* writeAttributes(fs, path, dir);
|
|
247
|
+
const target = answersDir(path, dir);
|
|
248
|
+
if (layout === "single") {
|
|
249
|
+
yield* fs.makeDirectory(dir, { recursive: true }).pipe(Effect.orElseSucceed(() => undefined));
|
|
250
|
+
const file = answersFile(path, dir);
|
|
251
|
+
if (entries.size === 0) {
|
|
252
|
+
yield* removeQuietly(fs, file);
|
|
166
253
|
}
|
|
254
|
+
else {
|
|
255
|
+
yield* fs.writeFileString(file, renderShard(entries.values())).pipe(Effect.orElseSucceed(() => undefined));
|
|
256
|
+
}
|
|
257
|
+
const names = yield* fs.readDirectory(target).pipe(Effect.orElseSucceed(() => []));
|
|
258
|
+
for (const name of names) {
|
|
259
|
+
if (name.endsWith(".jsonl"))
|
|
260
|
+
yield* removeQuietly(fs, path.join(target, name));
|
|
261
|
+
}
|
|
262
|
+
yield* removeQuietly(fs, answersObject(path, dir));
|
|
263
|
+
return;
|
|
264
|
+
}
|
|
265
|
+
const grouped = new Map();
|
|
266
|
+
for (const entry of entries.values()) {
|
|
267
|
+
const shard = shardOf(entry.key);
|
|
268
|
+
const list = grouped.get(shard) ?? [];
|
|
269
|
+
list.push(entry);
|
|
270
|
+
grouped.set(shard, list);
|
|
271
|
+
}
|
|
272
|
+
yield* fs.makeDirectory(target, { recursive: true }).pipe(Effect.orElseSucceed(() => undefined));
|
|
273
|
+
for (const [shard, list] of grouped) {
|
|
274
|
+
yield* fs
|
|
275
|
+
.writeFileString(path.join(target, shard + ".jsonl"), renderShard(list))
|
|
276
|
+
.pipe(Effect.orElseSucceed(() => undefined));
|
|
167
277
|
}
|
|
168
|
-
const
|
|
169
|
-
for (const name of
|
|
278
|
+
const names = yield* fs.readDirectory(target).pipe(Effect.orElseSucceed(() => []));
|
|
279
|
+
for (const name of names) {
|
|
170
280
|
if (!name.endsWith(".jsonl"))
|
|
171
281
|
continue;
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
// both carry.
|
|
175
|
-
for (const entry of parseLines(text).entries)
|
|
176
|
-
loaded = add(loaded, entry);
|
|
282
|
+
if (!grouped.has(name.slice(0, -".jsonl".length)))
|
|
283
|
+
yield* removeQuietly(fs, path.join(target, name));
|
|
177
284
|
}
|
|
178
|
-
|
|
285
|
+
yield* removeQuietly(fs, answersFile(path, dir));
|
|
286
|
+
yield* removeQuietly(fs, answersObject(path, dir));
|
|
287
|
+
});
|
|
288
|
+
/**
|
|
289
|
+
* Open the store, and write in the layout the repository already uses unless the
|
|
290
|
+
* caller names one.
|
|
291
|
+
*
|
|
292
|
+
* One writer at a time: concurrent decisions finish together, and a single-file
|
|
293
|
+
* layout rewrites the whole ledger on a `put`, so letting two flushes interleave
|
|
294
|
+
* would lose one. A sharded flush writes only the shards whose keys changed.
|
|
295
|
+
*/
|
|
296
|
+
export const make = (fs, path, dir, options = {}) => Effect.gen(function* () {
|
|
297
|
+
const loaded = yield* loadAll(fs, path, dir);
|
|
298
|
+
const chosen = options.layout ?? loaded.layout;
|
|
299
|
+
const entries = yield* Ref.make(loaded.entries);
|
|
179
300
|
const dirty = yield* Ref.make(new Set());
|
|
180
|
-
const
|
|
301
|
+
const layoutRef = yield* Ref.make(chosen);
|
|
302
|
+
const fullWrite = yield* Ref.make(loaded.layout !== chosen);
|
|
181
303
|
const writer = yield* Semaphore.make(1);
|
|
182
304
|
const flush = writer.withPermits(1)(Effect.gen(function* () {
|
|
183
|
-
// Taken and cleared in one step: a put that lands while this flush is
|
|
184
|
-
// writing marks a fresh set, which its own flush then writes.
|
|
185
305
|
const changed = yield* Ref.getAndSet(dirty, new Set());
|
|
186
|
-
const current = yield* Ref.get(
|
|
187
|
-
const
|
|
188
|
-
const
|
|
306
|
+
const current = yield* Ref.get(entries);
|
|
307
|
+
const layout = yield* Ref.get(layoutRef);
|
|
308
|
+
const rewrite = yield* Ref.getAndSet(fullWrite, false);
|
|
309
|
+
if (rewrite || layout === "single") {
|
|
310
|
+
yield* writeAll(fs, path, dir, layout, current);
|
|
311
|
+
return;
|
|
312
|
+
}
|
|
313
|
+
yield* writeAttributes(fs, path, dir);
|
|
189
314
|
const target = answersDir(path, dir);
|
|
190
315
|
yield* fs.makeDirectory(target, { recursive: true }).pipe(Effect.orElseSucceed(() => undefined));
|
|
191
|
-
const
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
yield* fs
|
|
200
|
-
.writeFileString(path.join(target, shard + ".jsonl"), renderShard(entries.values()))
|
|
201
|
-
.pipe(Effect.orElseSucceed(() => undefined));
|
|
202
|
-
}
|
|
203
|
-
if (migrate) {
|
|
204
|
-
yield* fs.remove(single).pipe(Effect.orElseSucceed(() => undefined));
|
|
205
|
-
yield* Ref.set(needsMigration, false);
|
|
316
|
+
const shards = new Set([...changed].map(shardOf));
|
|
317
|
+
for (const shard of shards) {
|
|
318
|
+
const list = [...current.values()].filter((entry) => shardOf(entry.key) === shard);
|
|
319
|
+
const file = path.join(target, shard + ".jsonl");
|
|
320
|
+
if (list.length === 0)
|
|
321
|
+
yield* removeQuietly(fs, file);
|
|
322
|
+
else
|
|
323
|
+
yield* fs.writeFileString(file, renderShard(list)).pipe(Effect.orElseSucceed(() => undefined));
|
|
206
324
|
}
|
|
207
325
|
}));
|
|
208
326
|
return {
|
|
209
|
-
get: (key) => Effect.map(Ref.get(
|
|
210
|
-
const entry = current.get(shardOf(key))?.get(key);
|
|
211
|
-
return entry === undefined ? Option.none() : Option.some(entry.answer);
|
|
212
|
-
}),
|
|
327
|
+
get: (key) => Effect.map(Ref.get(entries), (current) => Option.fromUndefinedOr(current.get(key)?.answer)),
|
|
213
328
|
put: (key, answer) => Effect.gen(function* () {
|
|
214
329
|
const entry = { key, writtenAt: seconds(yield* Clock.currentTimeMillis), answer };
|
|
215
|
-
yield* Ref.update(
|
|
216
|
-
yield* Ref.update(dirty, (current) =>
|
|
217
|
-
const next = new Set(current);
|
|
218
|
-
next.add(shardOf(key));
|
|
219
|
-
return next;
|
|
220
|
-
});
|
|
330
|
+
yield* Ref.update(entries, (current) => new Map(current).set(key, entry));
|
|
331
|
+
yield* Ref.update(dirty, (current) => new Set(current).add(key));
|
|
221
332
|
yield* flush;
|
|
222
333
|
}),
|
|
223
|
-
size: Effect.map(Ref.get(
|
|
224
|
-
|
|
225
|
-
|
|
334
|
+
size: Effect.map(Ref.get(entries), (current) => current.size),
|
|
335
|
+
layout: chosen,
|
|
336
|
+
migrate: (layout) => Effect.gen(function* () {
|
|
337
|
+
const target = layout ?? (yield* Ref.get(layoutRef));
|
|
338
|
+
yield* Ref.set(layoutRef, target);
|
|
339
|
+
yield* Ref.set(fullWrite, true);
|
|
226
340
|
yield* flush;
|
|
227
|
-
return
|
|
341
|
+
return (yield* Ref.get(entries)).size;
|
|
228
342
|
}),
|
|
229
343
|
};
|
|
230
344
|
});
|
|
231
345
|
const sizeOf = (entry) => renderEntry(entry).length + 1;
|
|
232
346
|
/**
|
|
233
|
-
*
|
|
347
|
+
* Read the ledger and count what is in it.
|
|
348
|
+
*
|
|
349
|
+
* The keys are content hashes, so there is no rule name to group by: a verdict
|
|
350
|
+
* cannot be attributed to a question after the fact. What IS visible is the
|
|
351
|
+
* verdict distribution and the labels a classified answer chose, which is what a
|
|
352
|
+
* reviewer opening a batch of new lines actually wants.
|
|
353
|
+
*/
|
|
354
|
+
export const census = (fs, path, dir) => Effect.gen(function* () {
|
|
355
|
+
const loaded = yield* loadAll(fs, path, dir);
|
|
356
|
+
const kinds = new Map();
|
|
357
|
+
const labels = new Map();
|
|
358
|
+
let yes = 0;
|
|
359
|
+
let no = 0;
|
|
360
|
+
let oldest;
|
|
361
|
+
let newest;
|
|
362
|
+
for (const entry of loaded.entries.values()) {
|
|
363
|
+
oldest = oldest === undefined ? entry.writtenAt : Math.min(oldest, entry.writtenAt);
|
|
364
|
+
newest = newest === undefined ? entry.writtenAt : Math.max(newest, entry.writtenAt);
|
|
365
|
+
kinds.set(entry.answer.kind, (kinds.get(entry.answer.kind) ?? 0) + 1);
|
|
366
|
+
if (entry.answer.kind === "Probability") {
|
|
367
|
+
if (entry.answer.probability >= 0.5)
|
|
368
|
+
yes += 1;
|
|
369
|
+
else
|
|
370
|
+
no += 1;
|
|
371
|
+
}
|
|
372
|
+
else {
|
|
373
|
+
labels.set(entry.answer.label, (labels.get(entry.answer.label) ?? 0) + 1);
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
return {
|
|
377
|
+
entries: loaded.entries.size,
|
|
378
|
+
files: loaded.files,
|
|
379
|
+
layout: loaded.layout,
|
|
380
|
+
bytes: loaded.bytes,
|
|
381
|
+
skippedLines: loaded.skippedLines,
|
|
382
|
+
kinds: [...kinds.entries()]
|
|
383
|
+
.sort(([left], [right]) => left.localeCompare(right))
|
|
384
|
+
.map(([kind, count]) => ({ kind, count })),
|
|
385
|
+
verdicts: { yes, no },
|
|
386
|
+
labels: [...labels.entries()]
|
|
387
|
+
.sort(([left, leftCount], [right, rightCount]) => rightCount - leftCount || left.localeCompare(right))
|
|
388
|
+
.map(([label, count]) => ({ label, count })),
|
|
389
|
+
oldest,
|
|
390
|
+
newest,
|
|
391
|
+
};
|
|
392
|
+
});
|
|
393
|
+
/**
|
|
394
|
+
* Drop cache entries and rewrite only the file or shards that changed.
|
|
234
395
|
*
|
|
235
396
|
* The age cut is per entry, from the timestamp each line carries. The size cut
|
|
236
|
-
* is oldest-first across the whole
|
|
237
|
-
* than written empty
|
|
238
|
-
* number of files.
|
|
397
|
+
* is oldest-first across the whole ledger. An emptied file or shard is deleted
|
|
398
|
+
* rather than written empty.
|
|
239
399
|
*/
|
|
240
400
|
export const prune = (fs, path, dir, options = {}) => Effect.gen(function* () {
|
|
241
401
|
const now = options.now ?? seconds(yield* Clock.currentTimeMillis);
|
|
242
402
|
const cutoff = options.olderThanSeconds === undefined ? undefined : now - options.olderThanSeconds;
|
|
243
|
-
const
|
|
244
|
-
const
|
|
245
|
-
|
|
246
|
-
const
|
|
247
|
-
|
|
248
|
-
|
|
403
|
+
const loaded = yield* loadAll(fs, path, dir);
|
|
404
|
+
const kept = new Map();
|
|
405
|
+
let removed = 0;
|
|
406
|
+
for (const entry of loaded.entries.values()) {
|
|
407
|
+
if (cutoff !== undefined && entry.writtenAt < cutoff) {
|
|
408
|
+
removed += 1;
|
|
249
409
|
continue;
|
|
250
|
-
const shard = name.slice(0, -".jsonl".length);
|
|
251
|
-
const text = yield* readText(fs, path.join(target, name));
|
|
252
|
-
for (const entry of parseLines(text).entries) {
|
|
253
|
-
if (cutoff !== undefined && entry.writtenAt < cutoff)
|
|
254
|
-
dropped.push({ shard, entry });
|
|
255
|
-
else {
|
|
256
|
-
const kept = keptByShard.get(shard) ?? [];
|
|
257
|
-
kept.push(entry);
|
|
258
|
-
keptByShard.set(shard, kept);
|
|
259
|
-
}
|
|
260
410
|
}
|
|
411
|
+
kept.set(entry.key, entry);
|
|
261
412
|
}
|
|
262
413
|
if (options.maxBytes !== undefined) {
|
|
263
414
|
let total = 0;
|
|
264
|
-
for (const
|
|
265
|
-
|
|
266
|
-
total += sizeOf(entry);
|
|
415
|
+
for (const entry of kept.values())
|
|
416
|
+
total += sizeOf(entry);
|
|
267
417
|
if (total > options.maxBytes) {
|
|
268
418
|
// A total order: oldest first, ties by key, so the result does not
|
|
269
419
|
// depend on directory iteration.
|
|
270
|
-
const ordered = [...
|
|
271
|
-
ordered.sort(Order.mapInput(Order.Number, (entry) => entry.writtenAt));
|
|
272
|
-
const removedKeys = new Set();
|
|
420
|
+
const ordered = [...kept.values()].sort((left, right) => left.writtenAt - right.writtenAt || byKey(left, right));
|
|
273
421
|
for (const entry of ordered) {
|
|
274
422
|
if (total <= options.maxBytes)
|
|
275
423
|
break;
|
|
276
|
-
|
|
277
|
-
|
|
424
|
+
kept.delete(entry.key);
|
|
425
|
+
removed += 1;
|
|
278
426
|
total -= sizeOf(entry);
|
|
279
427
|
}
|
|
280
|
-
for (const [shard, entries] of keptByShard) {
|
|
281
|
-
keptByShard.set(shard, entries.filter((entry) => !removedKeys.has(entry.key)));
|
|
282
|
-
}
|
|
283
428
|
}
|
|
284
429
|
}
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
if (entries.length === 0) {
|
|
290
|
-
yield* fs.remove(file).pipe(Effect.orElseSucceed(() => undefined));
|
|
291
|
-
}
|
|
292
|
-
else {
|
|
293
|
-
yield* fs.writeFileString(file, renderShard(entries)).pipe(Effect.orElseSucceed(() => undefined));
|
|
294
|
-
}
|
|
295
|
-
}
|
|
296
|
-
let kept = 0;
|
|
430
|
+
// Writing only when something changed keeps a no-op prune from touching the
|
|
431
|
+
// ledger's bytes (and therefore its diff).
|
|
432
|
+
if (removed > 0)
|
|
433
|
+
yield* writeAll(fs, path, dir, loaded.layout, kept);
|
|
297
434
|
let bytes = 0;
|
|
298
|
-
for (const
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
bytes += sizeOf(entry);
|
|
302
|
-
}
|
|
303
|
-
}
|
|
304
|
-
return { removed: dropped.length, kept, bytes };
|
|
435
|
+
for (const entry of kept.values())
|
|
436
|
+
bytes += sizeOf(entry);
|
|
437
|
+
return { removed, kept: kept.size, bytes };
|
|
305
438
|
});
|
package/dist/check.js
CHANGED
|
@@ -20,6 +20,14 @@ import { Service as Tsgo } from "./tsgo.js";
|
|
|
20
20
|
import { applyDeclarationTypes, discoverFiles, loadWorkspace } from "./workspace.js";
|
|
21
21
|
import { contentMappersOf } from "./content-mappers.js";
|
|
22
22
|
import { looksLikeSource } from "./source.js";
|
|
23
|
+
/**
|
|
24
|
+
* The drop stages that mean nobody judged the candidate.
|
|
25
|
+
*
|
|
26
|
+
* `declined` is the model looking and saying no; `gated` is a judgement below the
|
|
27
|
+
* threshold. Both are answers. These three are not, and a baseline that lists their
|
|
28
|
+
* consequences is a record of something that never happened.
|
|
29
|
+
*/
|
|
30
|
+
const UNJUDGED_STAGES = new Set(["unreadable", "no_evidence", "budget"]);
|
|
23
31
|
const reasonOf = (error) => error.message;
|
|
24
32
|
const typecheckFindings = (output) => output.map((diagnostic) => finding({
|
|
25
33
|
ruleId: "joggle/typecheck",
|
|
@@ -187,25 +195,45 @@ export const checkRepository = Effect.fn("joggle.check")(function* (options) {
|
|
|
187
195
|
const hashOf = new Map(files.map((file) => [path.relative(options.cwd, file), shortHash(contents.get(file) ?? "")]));
|
|
188
196
|
const finish = (report) => Effect.gen(function* () {
|
|
189
197
|
let diagnostics = report.diagnostics;
|
|
190
|
-
|
|
198
|
+
const notes = [];
|
|
191
199
|
if (options.baselinePath !== undefined) {
|
|
192
200
|
const known = yield* readBaseline(fs, path, baselinePath(path, options.baselinePath));
|
|
193
201
|
if (known !== undefined) {
|
|
194
202
|
const before = diagnostics.length;
|
|
195
203
|
diagnostics = diagnostics.filter((entry) => entry.identity === undefined || !known.has(entry.identity));
|
|
196
|
-
|
|
204
|
+
notes.push({
|
|
197
205
|
ruleId: "joggle",
|
|
198
206
|
reason: `${before - diagnostics.length} finding(s) already in the baseline were not reported`,
|
|
199
|
-
};
|
|
207
|
+
});
|
|
200
208
|
}
|
|
201
209
|
}
|
|
210
|
+
// The census line the budget used to lack: a run that stopped judging says HOW MANY
|
|
211
|
+
// candidates nobody looked at, so it is one number to read rather than a count per
|
|
212
|
+
// rule to add up. `declined` is an answer and is not in here.
|
|
213
|
+
const unjudged = report.drops.filter((drop) => UNJUDGED_STAGES.has(drop.stage)).length;
|
|
214
|
+
if (unjudged > 0) {
|
|
215
|
+
notes.push({
|
|
216
|
+
ruleId: "joggle",
|
|
217
|
+
reason: `${unjudged} candidate(s) were not judged (unreadable, no evidence, or past the budget); a baseline written now would ratify what nobody looked at`,
|
|
218
|
+
});
|
|
219
|
+
}
|
|
202
220
|
if (options.updateBaselinePath !== undefined) {
|
|
221
|
+
if (unjudged > 0 && options.allowUnjudgedBaseline !== true) {
|
|
222
|
+
return yield* WorkspaceError.make({
|
|
223
|
+
path: options.updateBaselinePath,
|
|
224
|
+
operation: "baseline",
|
|
225
|
+
cause: new Error(`refusing to write ${options.updateBaselinePath}: ${unjudged} candidate(s) were not judged. ` +
|
|
226
|
+
"Pass --allow-unjudged-baseline to accept that, or judge the run first."),
|
|
227
|
+
});
|
|
228
|
+
}
|
|
203
229
|
const identities = report.diagnostics
|
|
204
230
|
.map((entry) => entry.identity)
|
|
205
231
|
.filter((identity) => identity !== undefined);
|
|
206
232
|
yield* writeBaseline(fs, path, baselinePath(path, options.updateBaselinePath), identities);
|
|
207
233
|
}
|
|
208
|
-
return
|
|
234
|
+
return notes.length === 0
|
|
235
|
+
? { ...report, diagnostics }
|
|
236
|
+
: { ...report, diagnostics, notes: [...report.notes, ...notes] };
|
|
209
237
|
});
|
|
210
238
|
const runCache = options.runCacheDir ?? options.cacheDir;
|
|
211
239
|
const stored = yield* readStored(fs, path, runCache);
|
package/dist/config.js
CHANGED
|
@@ -122,6 +122,28 @@ export const JoggleConfig = Schema.Struct({
|
|
|
122
122
|
* extensions it can parse, and says which declared ones it did not.
|
|
123
123
|
*/
|
|
124
124
|
runExternalCode: Schema.optionalKey(Schema.Boolean),
|
|
125
|
+
/**
|
|
126
|
+
* The run's judgement budget, in the file rather than in a shell history.
|
|
127
|
+
*
|
|
128
|
+
* `--max-tokens` was the only place to say it, so CI and a local run disagreed
|
|
129
|
+
* unless every invocation remembered the flag. This is the same number, read
|
|
130
|
+
* where the rest of the run's decisions live; an explicit flag still wins.
|
|
131
|
+
*/
|
|
132
|
+
decision: Schema.optionalKey(Schema.Struct({
|
|
133
|
+
/** Input tokens one run may spend on judgement. */
|
|
134
|
+
maxInputTokens: Schema.optionalKey(Schema.Finite),
|
|
135
|
+
})),
|
|
136
|
+
/**
|
|
137
|
+
* Where the committed ledger lives, and in what shape.
|
|
138
|
+
*
|
|
139
|
+
* One sorted `answers.jsonl` (the default) is what a reviewer can read: a
|
|
140
|
+
* verdict change is one line. `sharded` is the opt-in for a store large enough
|
|
141
|
+
* that a whole-file rewrite or a merge is a real cost; both are read either way,
|
|
142
|
+
* and `joggle cache migrate --layout` moves between them.
|
|
143
|
+
*/
|
|
144
|
+
cache: Schema.optionalKey(Schema.Struct({
|
|
145
|
+
layout: Schema.optionalKey(Schema.Literals(["single", "sharded"])),
|
|
146
|
+
})),
|
|
125
147
|
evidence: Schema.optionalKey(Schema.Struct({
|
|
126
148
|
/**
|
|
127
149
|
* The repository's architecture, in the repository's own words.
|
package/dist/decision.js
CHANGED
|
@@ -183,7 +183,9 @@ export const layer = (options) => Layer.effectContext(Effect.gen(function* () {
|
|
|
183
183
|
// merge driver resolves the residual case by key. A legacy single
|
|
184
184
|
// `answers.json` is read and migrated on the first write. See
|
|
185
185
|
// `docs/artifacts.md`.
|
|
186
|
-
const answers = yield* makeAnswerStore(fs, path, options.cacheDir
|
|
186
|
+
const answers = yield* makeAnswerStore(fs, path, options.cacheDir, {
|
|
187
|
+
layout: options.layout,
|
|
188
|
+
});
|
|
187
189
|
const planAnswers = {
|
|
188
190
|
get: (key) => Effect.map(answers.get(key), (stored) => Option.map(stored, answerOf)),
|
|
189
191
|
put: (key, answer) => answers.put(key, storedOf(answer)),
|
package/dist/main.js
CHANGED
|
@@ -5,7 +5,7 @@ import * as FetchHttpClient from "effect/http/FetchHttpClient";
|
|
|
5
5
|
import { NodeRuntime, NodeServices } from "@effect/platform-node";
|
|
6
6
|
import { checkRepository } from "./check.js";
|
|
7
7
|
import { layer as decisionLayer } from "./decision.js";
|
|
8
|
-
import { make as makeAnswerStore, mergeShardText, prune as pruneAnswers } from "./answer-store.js";
|
|
8
|
+
import { census as censusAnswers, make as makeAnswerStore, mergeShardText, prune as pruneAnswers, } from "./answer-store.js";
|
|
9
9
|
import { ask } from "./ask.js";
|
|
10
10
|
import { cacheDirFor, runCacheDirFor, shortHash } from "./state.js";
|
|
11
11
|
import { canonical } from "./canonical.js";
|
|
@@ -76,7 +76,7 @@ const checkCommand = Command.make("check", {
|
|
|
76
76
|
maxWarnings: Flag.Int("max-warnings").pipe(Flag.withDescription("Exit non-zero when warnings exceed this count (-1 disables)."), Flag.withDefault(-1)),
|
|
77
77
|
typecheck: Flag.Boolean("typecheck").pipe(Flag.withDescription("Include tsgo's own diagnostics as joggle/typecheck."), Flag.withDefault(false)),
|
|
78
78
|
types: Flag.Boolean("types").pipe(Flag.withDescription("Resolve declaration types through tsgo's trace (enables type-aware rules)."), Flag.withDefault(false)),
|
|
79
|
-
maxTokens: Flag.Int("max-tokens").pipe(Flag.withDescription("Input-token budget for judgement in this run."), Flag.
|
|
79
|
+
maxTokens: Flag.Int("max-tokens").pipe(Flag.withDescription("Input-token budget for judgement in this run (overrides decision.maxInputTokens)."), Flag.optional),
|
|
80
80
|
offline: Flag.Boolean("offline").pipe(Flag.withDescription("Answer only from the committed judgement cache; never call the model."), Flag.withDefault(false)),
|
|
81
81
|
noTsgo: Flag.Boolean("no-tsgo").pipe(Flag.withDescription("Never invoke tsgo; walk the paths on disk instead."), Flag.withDefault(false)),
|
|
82
82
|
runExternalCode: Flag.Boolean("run-external-code").pipe(Flag.withDescription("Allow tsc to execute content-mapper packages declared in tsconfig (TypeScript 7.1+). Off by default: a mapper is arbitrary code."), Flag.withDefault(false)),
|
|
@@ -87,6 +87,7 @@ const checkCommand = Command.make("check", {
|
|
|
87
87
|
prNumber: Flag.String("pr-number").pipe(Flag.withDescription("Scope to a specific pull request, by number or URL."), Flag.optional),
|
|
88
88
|
baseline: Flag.String("baseline").pipe(Flag.withDescription("Report only findings not already accepted in this baseline file."), Flag.optional),
|
|
89
89
|
updateBaseline: Flag.String("update-baseline").pipe(Flag.withDescription("Write the current findings to this file as the accepted baseline."), Flag.optional),
|
|
90
|
+
allowUnjudgedBaseline: Flag.Boolean("allow-unjudged-baseline").pipe(Flag.withDescription("Write the baseline even though some candidates were not judged. A baseline is what was reviewed; this is the explicit way to accept that some of it was not."), Flag.withDefault(false)),
|
|
90
91
|
cacheDir: Flag.String("cache-dir").pipe(Flag.withDescription("Where the answer cache lives: <cwd>/.joggle when it exists (an onboarded repository), otherwise the machine cache."), Flag.optional),
|
|
91
92
|
cwd: Flag.String("cwd").pipe(Flag.withDescription("Project root to analyse (default: the current directory)."), Flag.optional),
|
|
92
93
|
config: Flag.String("config").pipe(Flag.withDescription("Config file, relative to the project root (default joggle.config.json)."), Flag.optional),
|
|
@@ -143,7 +144,7 @@ const checkCommand = Command.make("check", {
|
|
|
143
144
|
rules,
|
|
144
145
|
typecheck: config.typecheck,
|
|
145
146
|
types: config.types ? "trace" : "off",
|
|
146
|
-
maxInputTokens: config.maxTokens,
|
|
147
|
+
maxInputTokens: Option.getOrUndefined(config.maxTokens) ?? effective.decision?.maxInputTokens,
|
|
147
148
|
useTsgo: !config.noTsgo,
|
|
148
149
|
runExternalCode: config.runExternalCode || effective.runExternalCode === true,
|
|
149
150
|
cacheDirExplicit: Option.isSome(config.cacheDir),
|
|
@@ -161,7 +162,14 @@ const checkCommand = Command.make("check", {
|
|
|
161
162
|
changedBase,
|
|
162
163
|
baselinePath: Option.getOrUndefined(config.baseline),
|
|
163
164
|
updateBaselinePath: Option.getOrUndefined(config.updateBaseline),
|
|
164
|
-
|
|
165
|
+
allowUnjudgedBaseline: config.allowUnjudgedBaseline,
|
|
166
|
+
}).pipe(Effect.provide(Layer.mergeAll(rulesLayer, decisionLayer({
|
|
167
|
+
cacheDir,
|
|
168
|
+
wireCacheDir: machineCache,
|
|
169
|
+
offline: config.offline,
|
|
170
|
+
apiKey,
|
|
171
|
+
layout: effective.cache?.layout,
|
|
172
|
+
}), tsgoLayer(cwd, {
|
|
165
173
|
runExternalCode: config.runExternalCode || effective.runExternalCode === true,
|
|
166
174
|
}))));
|
|
167
175
|
yield* write(render(report, config.format, { color: process.stdout.isTTY === true }));
|
|
@@ -473,6 +481,7 @@ const calibrateCommand = Command.make("calibrate", {
|
|
|
473
481
|
wireCacheDir: yield* runCacheDirFor(cwd),
|
|
474
482
|
offline: false,
|
|
475
483
|
apiKey,
|
|
484
|
+
layout: ruleSet.effective.cache?.layout,
|
|
476
485
|
onAnswer: (event) => recorded.push({ answers: event.answers }),
|
|
477
486
|
}),
|
|
478
487
|
// The trace and the node types both run tsgo; the calibrate command
|
|
@@ -550,19 +559,55 @@ const cachePrune = Command.make("prune", {
|
|
|
550
559
|
description: "Cap the committed cache, oldest answers first.",
|
|
551
560
|
},
|
|
552
561
|
]));
|
|
562
|
+
const cacheReport = Command.make("report", {
|
|
563
|
+
cwd: Flag.String("cwd").pipe(Flag.withDescription("Project root whose cache to report (default: the current directory)."), Flag.optional),
|
|
564
|
+
cacheDir: Flag.String("cache-dir").pipe(Flag.withDescription("Where the answer cache lives: <cwd>/.joggle when it exists, otherwise the machine cache."), Flag.optional),
|
|
565
|
+
}, (config) => Effect.gen(function* () {
|
|
566
|
+
const fs = yield* FileSystem.FileSystem;
|
|
567
|
+
const path = yield* Path.Path;
|
|
568
|
+
const cwd = path.resolve(Option.getOrUndefined(config.cwd) ?? process.cwd());
|
|
569
|
+
const dir = yield* cacheDirFor(cwd, Option.getOrUndefined(config.cacheDir));
|
|
570
|
+
const found = yield* censusAnswers(fs, path, dir);
|
|
571
|
+
const date = (seconds) => seconds === undefined ? "never" : new Date(seconds * 1000).toISOString().slice(0, 10);
|
|
572
|
+
const kinds = found.kinds.map((entry) => entry.kind + " " + String(entry.count)).join(", ");
|
|
573
|
+
const labels = found.labels
|
|
574
|
+
.slice(0, 8)
|
|
575
|
+
.map((entry) => entry.label + " " + String(entry.count))
|
|
576
|
+
.join(", ");
|
|
577
|
+
yield* write([
|
|
578
|
+
"joggle: " +
|
|
579
|
+
found.entries +
|
|
580
|
+
" entries in " +
|
|
581
|
+
found.files +
|
|
582
|
+
" file(s) (" +
|
|
583
|
+
found.layout +
|
|
584
|
+
"), " +
|
|
585
|
+
found.bytes +
|
|
586
|
+
" bytes",
|
|
587
|
+
" kinds " + (kinds === "" ? "none" : kinds),
|
|
588
|
+
" verdicts Probability " + found.verdicts.yes + " yes / " + found.verdicts.no + " no",
|
|
589
|
+
" labels " + (labels === "" ? "none" : labels),
|
|
590
|
+
" written " + date(found.oldest) + " .. " + date(found.newest),
|
|
591
|
+
...(found.skippedLines === 0 ? [] : [" skipped " + found.skippedLines + " unreadable line(s)"]),
|
|
592
|
+
].join("\n"));
|
|
593
|
+
})).pipe(Command.withDescription("Render the committed answer ledger for a reader: entry and file counts, layout, verdict distribution, labels, and the date range. A summary cannot be replayed; this is what to read when reviewing the diff."), Command.withShortDescription("Report on the answer cache."), Command.withExamples([
|
|
594
|
+
{ command: "joggle cache report", description: "Count what the committed ledger holds." },
|
|
595
|
+
]));
|
|
553
596
|
const cacheMigrate = Command.make("migrate", {
|
|
554
597
|
cwd: Flag.String("cwd").pipe(Flag.withDescription("Project root whose cache to migrate (default: the current directory)."), Flag.optional),
|
|
555
598
|
cacheDir: Flag.String("cache-dir").pipe(Flag.withDescription("Where the answer cache lives: <cwd>/.joggle when it exists, otherwise the machine cache."), Flag.optional),
|
|
599
|
+
layout: Flag.Literals("layout", ["single", "sharded"]).pipe(Flag.withDescription("How to write the ledger: one sorted answers.jsonl (default), or shards under answers/."), Flag.withDefault("single")),
|
|
556
600
|
}, (config) => Effect.gen(function* () {
|
|
557
601
|
const fs = yield* FileSystem.FileSystem;
|
|
558
602
|
const path = yield* Path.Path;
|
|
559
603
|
const cwd = path.resolve(Option.getOrUndefined(config.cwd) ?? process.cwd());
|
|
560
604
|
const dir = yield* cacheDirFor(cwd, Option.getOrUndefined(config.cacheDir));
|
|
561
605
|
const store = yield* makeAnswerStore(fs, path, dir);
|
|
562
|
-
const written = yield* store.migrate;
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
606
|
+
const written = yield* store.migrate(config.layout);
|
|
607
|
+
const target = config.layout === "single" ? path.join(dir, "answers.jsonl") : path.join(dir, "answers");
|
|
608
|
+
yield* write("joggle: wrote " + written + " answer(s) to " + target);
|
|
609
|
+
})).pipe(Command.withDescription("Rewrite the cache in one layout and remove the other: a single sorted answers.jsonl by default, or shards with --layout sharded. Idempotent."), Command.withShortDescription("Consolidate or shard the answer cache."));
|
|
610
|
+
const cacheCommand = Command.make("cache", {}, () => write("joggle cache: run `joggle cache report`, `prune`, or `migrate --layout single|sharded`; docs/artifacts.md has the reasoning.")).pipe(Command.withDescription("Work on the answer cache itself: prune it, migrate it, and see how it is laid out."), Command.withShortDescription("Prune the answer cache."), Command.withSubcommands([cachePrune, cacheMigrate, cacheReport]));
|
|
566
611
|
const cli = Command.make("joggle").pipe(Command.withDescription("Cross-file patterns and idioms for TypeScript, enforced like a linter: deterministic where it can prove, System One where it has to judge."), Command.withExamples([
|
|
567
612
|
{
|
|
568
613
|
command: "joggle check --pr",
|
package/dist/tsgo.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
import { Context, Effect, FileSystem, Layer, Order, Path, Stream } from "effect";
|
|
1
|
+
import { Context, Effect, FileSystem, Layer, Option, Order, Path, Stream } from "effect";
|
|
2
2
|
import { ChildProcess } from "effect/process";
|
|
3
|
-
import {
|
|
3
|
+
import { realpathSync } from "node:fs";
|
|
4
|
+
import { relative, resolve, sep } from "node:path";
|
|
4
5
|
import { TsgoError } from "./schema.js";
|
|
5
6
|
import { looksLikeSource } from "./source.js";
|
|
6
7
|
/**
|
|
@@ -29,17 +30,37 @@ const run = ({ binary, args, cwd, }) => Effect.scoped(Effect.gen(function* () {
|
|
|
29
30
|
operation: `tsc ${args.join(" ")}`,
|
|
30
31
|
detail: cause instanceof Error ? cause.message : String(cause),
|
|
31
32
|
})));
|
|
33
|
+
/** A path with every symlink in it resolved, or the lexical path when it does not exist. */
|
|
34
|
+
const realPathOf = (file) => {
|
|
35
|
+
// SAFETY: `realpathSync` is the only API that resolves symlinks, and it throws for a
|
|
36
|
+
// path that does not exist yet. The lexical path is the right fallback there: the file is
|
|
37
|
+
// absent either way, so `inRoot` rejects it on the same comparison.
|
|
38
|
+
try {
|
|
39
|
+
return realpathSync(file);
|
|
40
|
+
}
|
|
41
|
+
catch {
|
|
42
|
+
return resolve(file);
|
|
43
|
+
}
|
|
44
|
+
};
|
|
32
45
|
/**
|
|
33
|
-
*
|
|
46
|
+
* The compiler's file, expressed under the caller's root, or undefined when it is outside.
|
|
47
|
+
*
|
|
48
|
+
* The comparison happens on REAL paths because the compiler prints the path it resolved:
|
|
49
|
+
* invoked with a cwd of `/tmp/x`, where `/tmp` is a symlink to `/private/tmp`, tsgo lists
|
|
50
|
+
* `/private/tmp/x/src/a.ts`. A lexical `startsWith` on the caller's `/tmp/x` rejected every
|
|
51
|
+
* file, so the run reported "tsgo listed no source files" and analysed nothing -- silently,
|
|
52
|
+
* for anyone whose checkout lives under a symlinked path.
|
|
34
53
|
*
|
|
35
|
-
*
|
|
36
|
-
*
|
|
37
|
-
*
|
|
54
|
+
* The path handed back is in the caller's namespace (`resolve(cwd, relative(realRoot, file))`)
|
|
55
|
+
* so every later comparison -- git's changed files, `path.relative`, the report's locations --
|
|
56
|
+
* keeps using one spelling of the tree.
|
|
38
57
|
*/
|
|
39
|
-
const
|
|
40
|
-
const root =
|
|
41
|
-
const candidate =
|
|
42
|
-
|
|
58
|
+
const inRoot = ({ cwd, file }) => {
|
|
59
|
+
const root = realPathOf(cwd);
|
|
60
|
+
const candidate = realPathOf(file);
|
|
61
|
+
if (candidate === root || !candidate.startsWith(root + sep))
|
|
62
|
+
return Option.none();
|
|
63
|
+
return Option.some(resolve(cwd, relative(root, candidate)));
|
|
43
64
|
};
|
|
44
65
|
const parseDiagnostics = (output) => {
|
|
45
66
|
const found = [];
|
|
@@ -80,7 +101,7 @@ const parseDiagnostics = (output) => {
|
|
|
80
101
|
* cloned repository cannot make an agent execute code by declaring a mapper.
|
|
81
102
|
*/
|
|
82
103
|
export const compilerArgs = (args, runExternalCode) => (runExternalCode ? ["--runExternalCode", ...args] : args);
|
|
83
|
-
const make = (binary,
|
|
104
|
+
const make = (binary, runExternalCode) => {
|
|
84
105
|
const listFiles = Effect.fn("Tsgo.listFiles")(function* (cwd) {
|
|
85
106
|
const result = yield* run({ binary, args: compilerArgs(["--listFilesOnly"], runExternalCode), cwd });
|
|
86
107
|
const files = new Set();
|
|
@@ -88,9 +109,10 @@ const make = (binary, path, runExternalCode) => {
|
|
|
88
109
|
const file = raw.trim();
|
|
89
110
|
if (!looksLikeSource(file))
|
|
90
111
|
continue;
|
|
91
|
-
|
|
112
|
+
const rooted = inRoot({ cwd, file });
|
|
113
|
+
if (Option.isNone(rooted))
|
|
92
114
|
continue;
|
|
93
|
-
files.add(
|
|
115
|
+
files.add(rooted.value);
|
|
94
116
|
}
|
|
95
117
|
return [...files].sort(Order.String);
|
|
96
118
|
});
|
|
@@ -116,12 +138,12 @@ const make = (binary, path, runExternalCode) => {
|
|
|
116
138
|
* project root. Everything else about the adapter is a plain subprocess
|
|
117
139
|
* boundary: build args, run, parse, map failures to a typed error.
|
|
118
140
|
*/
|
|
119
|
-
export const layer = (binary, options = {}) => Layer.effect(Service, Effect.map(Path.Path, (
|
|
141
|
+
export const layer = (binary, options = {}) => Layer.effect(Service, Effect.map(Path.Path, () => make(binary, options.runExternalCode ?? false)));
|
|
120
142
|
/** Resolve the project's own compiler, falling back to PATH. */
|
|
121
143
|
export const layerFromConfig = (root, options = {}) => Layer.effect(Service, Effect.gen(function* () {
|
|
122
144
|
const fs = yield* FileSystem.FileSystem;
|
|
123
145
|
const path = yield* Path.Path;
|
|
124
146
|
const local = path.join(root, "node_modules", ".bin", "tsc");
|
|
125
147
|
const resolved = (yield* Effect.orElseSucceed(fs.exists(local), () => false)) ? local : "tsc";
|
|
126
|
-
return make(resolved,
|
|
148
|
+
return make(resolved, options.runExternalCode ?? false);
|
|
127
149
|
}));
|