@autotelic/joggle 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/JOGGLE.md +10 -8
- package/README.md +5 -4
- package/dist/answer-store.js +243 -197
- package/dist/config.js +11 -0
- package/dist/decision.js +3 -1
- package/dist/main.js +24 -7
- package/package.json +1 -1
package/JOGGLE.md
CHANGED
|
@@ -228,13 +228,15 @@ It is serialised with sorted keys, so the same candidate produces the same key
|
|
|
228
228
|
on every machine — which means **run from the repository root**. Evidence
|
|
229
229
|
carries root-relative paths; running from a subdirectory changes the key and
|
|
230
230
|
misses every cached verdict. Successful judgements are written to
|
|
231
|
-
`.joggle/answers
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
231
|
+
`.joggle/answers.jsonl`, one sorted, line-oriented answer per line; commit that
|
|
232
|
+
file and CI replays them with `--offline` and **no API key at all**. A new
|
|
233
|
+
answer is a new line, so two branches that judge different candidates merge with
|
|
234
|
+
git's built-in `merge=union`, and `joggle cache prune` keeps the store from
|
|
235
|
+
growing without a ceiling. A store large enough to make a whole-file rewrite or a
|
|
236
|
+
merge expensive can be sharded instead (`cache.layout: "sharded"`, or
|
|
237
|
+
`joggle cache migrate --layout sharded`); both layouts are read, and
|
|
238
|
+
`cache migrate` moves between them. The older `answers.json` object is read and
|
|
239
|
+
consolidated on the next write. The layouts and the deployment modes are in
|
|
238
240
|
[docs/artifacts.md](./docs/artifacts.md). Bump `policy.decisionVersion` when you
|
|
239
241
|
change a question's wording, and every cached answer for it is invalidated at
|
|
240
242
|
once instead of silently replayed against newer questions.
|
|
@@ -426,7 +428,7 @@ and two belong to the machine.
|
|
|
426
428
|
|
|
427
429
|
| artifact | lives in | committed | what it is |
|
|
428
430
|
| --- | --- | --- | --- |
|
|
429
|
-
| `answers/` | `<root>/.joggle/` | **yes** | one
|
|
431
|
+
| `answers.jsonl` (or `answers/` when sharded) | `<root>/.joggle/` | **yes** | one line per answer, sorted by key, keyed by the decision and the state it read. **This is the replay path**: CI replays it with no API key and no tokens. |
|
|
430
432
|
| `baseline/` | `<root>/.joggle/` | **yes** | the findings already accepted, one fragment per finding: the ratchet. |
|
|
431
433
|
| `wire.json` | `<machine cache>/joggle/<root>/` | no | every wire request's response, keyed by the request. A performance artifact, and far too large to commit: 11.5 megabytes on a 2,493-file repository. |
|
|
432
434
|
| `last-run.json` | `<machine cache>/joggle/<root>/` | no | a manifest and the last report. A performance artifact. |
|
package/README.md
CHANGED
|
@@ -46,10 +46,11 @@ among named options, a spectrum) with calibrated probabilities, and code keeps t
|
|
|
46
46
|
thresholds: a decisive answer prints, an unsure one is recorded as a notice, a no is
|
|
47
47
|
silent. Arithmetic, counts, spans and orderings all stay in code.
|
|
48
48
|
|
|
49
|
-
Verdicts are cached per question in `.joggle/answers
|
|
50
|
-
|
|
51
|
-
tokens. The
|
|
52
|
-
|
|
49
|
+
Verdicts are cached per question in `.joggle/answers.jsonl`, one sorted line per
|
|
50
|
+
answer, so commit that file and CI replays every judgement with no key and no
|
|
51
|
+
tokens. The lines are keyed and `merge=union` is enough for a branch merge, and
|
|
52
|
+
`joggle cache prune` keeps the store from growing without a ceiling. A large
|
|
53
|
+
store can be sharded instead (`cache.layout: "sharded"`); both are read. A yes/no is asked three times
|
|
53
54
|
and the **agreement** across the asks becomes its confidence, because one number on
|
|
54
55
|
its own has no margin. [How joggle uses TypeSafe](./docs/typesafe.md).
|
|
55
56
|
|
package/dist/answer-store.js
CHANGED
|
@@ -2,65 +2,68 @@ import { Clock, Effect, FileSystem, Option, Order, Path, Ref, Result, Schema, Sc
|
|
|
2
2
|
import { canonical } from "./canonical.js";
|
|
3
3
|
import { StoredAnswer } from "./plans.js";
|
|
4
4
|
/**
|
|
5
|
-
* The answer cache
|
|
5
|
+
* The answer cache: one logical ledger, two on-disk layouts.
|
|
6
6
|
*
|
|
7
|
-
*
|
|
8
|
-
* key
|
|
9
|
-
*
|
|
10
|
-
* judge different candidates write different shards -- or, when they collide on
|
|
11
|
-
* a shard, different lines. That is the whole design: a merge is a directory
|
|
12
|
-
* merge, and the residual collision is a same-line one, which the shipped merge
|
|
13
|
-
* driver (`joggle merge-answers`) resolves by key.
|
|
7
|
+
* An entry is a line of `<key>\t<unix-seconds>\t<json>`, sorted by key. The
|
|
8
|
+
* key (`answerKeyFor`) already content-addresses the decision, the atoms it
|
|
9
|
+
* read, the model and the decision version, so the line is the identity.
|
|
14
10
|
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
11
|
+
* **Single** (the default) is one file, `answers.jsonl`. It is the shape a
|
|
12
|
+
* reviewer can read: a verdict change is one changed line in one diff. Sharding
|
|
13
|
+
* bought merge granularity at a cost nobody was paying -- 130 shard files for 182
|
|
14
|
+
* entries is a zero-reviewability commit, and git cannot ship the merge driver
|
|
15
|
+
* that was supposed to make the shards safe.
|
|
20
16
|
*
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
*
|
|
17
|
+
* **Sharded** is `answers/<first two hex chars>.jsonl`, chosen explicitly when a
|
|
18
|
+
* store is large enough that a whole-file rewrite or a merge is a real cost. It
|
|
19
|
+
* uses git's built-in `merge=union`, which needs no per-machine registration.
|
|
20
|
+
*
|
|
21
|
+
* Both are read. The layout is decided by what is on disk, unless the caller
|
|
22
|
+
* names one; `joggle cache migrate --layout` rewrites in the other shape. The
|
|
23
|
+
* older `answers.json` object is read and consolidated into the single file on
|
|
24
|
+
* the first write.
|
|
24
25
|
*/
|
|
25
|
-
// How many leading hex characters name a shard. 256 files is enough granularity.
|
|
26
26
|
const SHARD_CHARS = 2;
|
|
27
27
|
/** The shard a key belongs to: its first two characters, lower-cased. */
|
|
28
28
|
export const shardOf = (key) => key.slice(0, SHARD_CHARS).toLowerCase().padEnd(SHARD_CHARS, "_");
|
|
29
29
|
/** The directory holding the shards, beside the single-file form. */
|
|
30
30
|
export const answersDir = (path, dir) => path.join(dir, "answers");
|
|
31
|
-
|
|
31
|
+
/** The one-file ledger. */
|
|
32
|
+
export const answersFile = (path, dir) => path.join(dir, "answers.jsonl");
|
|
33
|
+
/** The older pretty-printed object, still read. */
|
|
34
|
+
const answersObject = (path, dir) => path.join(dir, "answers.json");
|
|
32
35
|
/**
|
|
33
36
|
* The attributes joggle manages for the cache directory itself.
|
|
34
37
|
*
|
|
35
38
|
* Git reads a `.gitattributes` in a subdirectory and applies its patterns
|
|
36
|
-
* relative to that directory, so joggle
|
|
39
|
+
* relative to that directory, so joggle owns the treatment of its own files
|
|
37
40
|
* without touching the repository's top-level file.
|
|
38
41
|
*
|
|
39
42
|
* `merge=union` rather than a custom driver: git cannot ship a merge driver in
|
|
40
|
-
* `.gitattributes`, so `merge=joggle-answers` only
|
|
43
|
+
* `.gitattributes`, so `merge=joggle-answers` only worked on a machine where
|
|
41
44
|
* somebody registered it by hand -- and a fresh clone silently got the default
|
|
42
45
|
* merge instead. Union needs no registration and is correct for an append-only,
|
|
43
|
-
* key-sorted JSONL.
|
|
44
|
-
* that want newer-timestamp-wins on a same-key collision, but the committed
|
|
45
|
-
* shape no longer depends on it.
|
|
46
|
+
* key-sorted JSONL.
|
|
46
47
|
*
|
|
47
48
|
* No `-diff` and no `linguist-generated`: the ledger's justification is that a
|
|
48
49
|
* maintainer reviewed the verdicts, and those attributes made it a binary blob
|
|
49
50
|
* that GitHub collapses. A verdict change should be one visible line.
|
|
50
51
|
*/
|
|
51
|
-
const ATTRIBUTES = "answers/*.jsonl merge=union\n" +
|
|
52
|
-
"answers.json merge=union\n";
|
|
52
|
+
const ATTRIBUTES = "answers.jsonl merge=union\n" + "answers/*.jsonl merge=union\n" + "answers.json merge=union\n";
|
|
53
53
|
/**
|
|
54
|
-
*
|
|
54
|
+
* Banners earlier versions wrote.
|
|
55
55
|
*
|
|
56
|
-
* Rewritten in place because
|
|
56
|
+
* Rewritten in place because each is byte-identical to what joggle generated, so
|
|
57
57
|
* this is a migration rather than an edit of a file a person may have changed.
|
|
58
|
-
*
|
|
58
|
+
* Any other content is left alone.
|
|
59
59
|
*/
|
|
60
|
-
const LEGACY_ATTRIBUTES =
|
|
61
|
-
"answers
|
|
62
|
-
|
|
63
|
-
|
|
60
|
+
const LEGACY_ATTRIBUTES = [
|
|
61
|
+
"answers/*.jsonl merge=joggle-answers linguist-generated -diff\n" +
|
|
62
|
+
"answers.json linguist-generated -diff\n",
|
|
63
|
+
"answers/*.jsonl merge=union\n" + "answers.json merge=union\n",
|
|
64
|
+
];
|
|
65
|
+
/** The shape of the older single object, read once and consolidated. */
|
|
66
|
+
const AnswersObject = Schema.Struct({
|
|
64
67
|
version: Schema.String,
|
|
65
68
|
entries: Schema.Record(Schema.String, StoredAnswer),
|
|
66
69
|
});
|
|
@@ -70,7 +73,7 @@ const decodeAnswer = Schema.fromJsonString(StoredAnswer);
|
|
|
70
73
|
* any of the three fields, so the split is unambiguous.
|
|
71
74
|
*/
|
|
72
75
|
export const renderEntry = (entry) => entry.key + "\t" + entry.writtenAt + "\t" + canonical(entry.answer);
|
|
73
|
-
/** Parse a
|
|
76
|
+
/** Parse a ledger file. A line that does not decode is skipped and COUNTED, never fatal. */
|
|
74
77
|
export const parseLines = (text) => {
|
|
75
78
|
const entries = [];
|
|
76
79
|
let skippedLines = 0;
|
|
@@ -94,13 +97,13 @@ export const parseLines = (text) => {
|
|
|
94
97
|
return { entries, skippedLines };
|
|
95
98
|
};
|
|
96
99
|
const byKey = Order.mapInput(Order.String, (entry) => entry.key);
|
|
97
|
-
/** A
|
|
100
|
+
/** A file body: entries sorted by key, newline-terminated. */
|
|
98
101
|
export const renderShard = (entries) => {
|
|
99
102
|
const sorted = [...entries].sort(byKey);
|
|
100
103
|
return sorted.length === 0 ? "" : sorted.map(renderEntry).join("\n") + "\n";
|
|
101
104
|
};
|
|
102
105
|
/**
|
|
103
|
-
* Union two sides of one
|
|
106
|
+
* Union two sides of one file by key.
|
|
104
107
|
*
|
|
105
108
|
* A key that only one side carries is kept. A key both sides carry with
|
|
106
109
|
* different content keeps the newer timestamp: for a content-addressed key the
|
|
@@ -119,7 +122,7 @@ export const mergeEntries = ({ ours, theirs, }) => {
|
|
|
119
122
|
return [...merged.values()];
|
|
120
123
|
};
|
|
121
124
|
/**
|
|
122
|
-
* A side of a merge that is not a
|
|
125
|
+
* A side of a merge that is not a ledger file at all.
|
|
123
126
|
*
|
|
124
127
|
* A tagged error rather than `Option.none`, because the reason is the whole
|
|
125
128
|
* value: "ours" or "theirs" is what the driver prints and what a person acts on.
|
|
@@ -129,7 +132,7 @@ export class NotAShard extends Schema.TaggedError()("joggle/NotAShard", {
|
|
|
129
132
|
}) {
|
|
130
133
|
}
|
|
131
134
|
/**
|
|
132
|
-
* A merge of one
|
|
135
|
+
* A merge of one file, failing when a non-empty side is not a ledger at all.
|
|
133
136
|
*
|
|
134
137
|
* Failing is how the driver declines: git then records an ordinary conflict,
|
|
135
138
|
* which is recoverable, instead of a union that silently dropped lines. Every
|
|
@@ -146,116 +149,202 @@ export const mergeShardText = (text) => Effect.gen(function* () {
|
|
|
146
149
|
}
|
|
147
150
|
return renderShard(mergeEntries({ ours: ourEntries, theirs: theirEntries }));
|
|
148
151
|
});
|
|
152
|
+
/* -------------------------------------------------------------------------- */
|
|
153
|
+
/* Reading and writing both layouts */
|
|
154
|
+
/* -------------------------------------------------------------------------- */
|
|
149
155
|
const readText = (fs, file) => fs.readFileString(file).pipe(Effect.orElseSucceed(() => ""));
|
|
150
156
|
const exists = (fs, file) => fs.exists(file).pipe(Effect.orElseSucceed(() => false));
|
|
151
157
|
const seconds = (millis) => Math.floor(millis / 1000);
|
|
152
|
-
const
|
|
153
|
-
const next = new Map(shards);
|
|
154
|
-
const inner = new Map(next.get(shardOf(entry.key)) ?? []);
|
|
155
|
-
inner.set(entry.key, entry);
|
|
156
|
-
next.set(shardOf(entry.key), inner);
|
|
157
|
-
return next;
|
|
158
|
-
};
|
|
159
|
-
const count = (shards) => {
|
|
160
|
-
let total = 0;
|
|
161
|
-
for (const entries of shards.values())
|
|
162
|
-
total += entries.size;
|
|
163
|
-
return total;
|
|
164
|
-
};
|
|
158
|
+
const bytesOf = (text) => new TextEncoder().encode(text).length;
|
|
165
159
|
/**
|
|
166
|
-
*
|
|
167
|
-
* dirty shards so a write is proportional to the change.
|
|
160
|
+
* Read every form into one map, newest timestamp winning a shared key.
|
|
168
161
|
*
|
|
169
|
-
*
|
|
170
|
-
*
|
|
171
|
-
*
|
|
172
|
-
* crash in between leaves both and the sharded side wins on the next read.
|
|
162
|
+
* The layouts are not mutually exclusive on disk: a repository that migrated
|
|
163
|
+
* may still carry the old file after a crash, and reading both is how that
|
|
164
|
+
* degrades to the union rather than to whichever the walk hit first.
|
|
173
165
|
*/
|
|
174
|
-
|
|
175
|
-
const single = singleFile(path, dir);
|
|
176
|
-
const migrating = yield* exists(fs, single);
|
|
166
|
+
const loadAll = (fs, path, dir) => Effect.gen(function* () {
|
|
177
167
|
const now = seconds(yield* Clock.currentTimeMillis);
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
const
|
|
181
|
-
|
|
168
|
+
const merged = new Map();
|
|
169
|
+
const absorb = (entry) => {
|
|
170
|
+
const existing = merged.get(entry.key);
|
|
171
|
+
if (existing === undefined || entry.writtenAt > existing.writtenAt)
|
|
172
|
+
merged.set(entry.key, entry);
|
|
173
|
+
};
|
|
174
|
+
let files = 0;
|
|
175
|
+
let bytes = 0;
|
|
176
|
+
let skippedLines = 0;
|
|
177
|
+
const older = answersObject(path, dir);
|
|
178
|
+
if (yield* exists(fs, older)) {
|
|
179
|
+
const text = yield* readText(fs, older);
|
|
180
|
+
const decoded = Result.getOrUndefined(SchemaParser.decodeResult(Schema.fromJsonString(AnswersObject))(text));
|
|
182
181
|
if (decoded !== undefined) {
|
|
182
|
+
files += 1;
|
|
183
|
+
bytes += bytesOf(text);
|
|
183
184
|
for (const [key, answer] of Object.entries(decoded.entries))
|
|
184
|
-
|
|
185
|
+
absorb({ key, writtenAt: now, answer });
|
|
185
186
|
}
|
|
186
187
|
}
|
|
187
|
-
const
|
|
188
|
-
|
|
188
|
+
const single = answersFile(path, dir);
|
|
189
|
+
const singleExists = yield* exists(fs, single);
|
|
190
|
+
if (singleExists) {
|
|
191
|
+
const text = yield* readText(fs, single);
|
|
192
|
+
const parsed = parseLines(text);
|
|
193
|
+
files += 1;
|
|
194
|
+
bytes += bytesOf(text);
|
|
195
|
+
skippedLines += parsed.skippedLines;
|
|
196
|
+
for (const entry of parsed.entries)
|
|
197
|
+
absorb(entry);
|
|
198
|
+
}
|
|
199
|
+
const target = answersDir(path, dir);
|
|
200
|
+
const names = yield* fs.readDirectory(target).pipe(Effect.orElseSucceed(() => []));
|
|
201
|
+
let shards = 0;
|
|
202
|
+
for (const name of names) {
|
|
189
203
|
if (!name.endsWith(".jsonl"))
|
|
190
204
|
continue;
|
|
191
|
-
const text = yield* readText(fs, path.join(
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
205
|
+
const text = yield* readText(fs, path.join(target, name));
|
|
206
|
+
const parsed = parseLines(text);
|
|
207
|
+
if (text === "")
|
|
208
|
+
continue;
|
|
209
|
+
shards += 1;
|
|
210
|
+
files += 1;
|
|
211
|
+
bytes += bytesOf(text);
|
|
212
|
+
skippedLines += parsed.skippedLines;
|
|
213
|
+
for (const entry of parsed.entries)
|
|
214
|
+
absorb(entry);
|
|
215
|
+
}
|
|
216
|
+
// Sharded wins when both exist: it is the form a migration finishes by
|
|
217
|
+
// writing, so a half-finished single-to-sharded run reads as sharded.
|
|
218
|
+
return {
|
|
219
|
+
entries: merged,
|
|
220
|
+
layout: shards > 0 ? "sharded" : "single",
|
|
221
|
+
files,
|
|
222
|
+
bytes,
|
|
223
|
+
skippedLines,
|
|
224
|
+
};
|
|
225
|
+
});
|
|
226
|
+
const writeAttributes = (fs, path, dir) => Effect.gen(function* () {
|
|
227
|
+
const attributes = path.join(dir, ".gitattributes");
|
|
228
|
+
if (!(yield* exists(fs, attributes))) {
|
|
229
|
+
yield* fs.writeFileString(attributes, ATTRIBUTES).pipe(Effect.orElseSucceed(() => undefined));
|
|
230
|
+
return;
|
|
231
|
+
}
|
|
232
|
+
const current = yield* readText(fs, attributes);
|
|
233
|
+
if (LEGACY_ATTRIBUTES.includes(current)) {
|
|
234
|
+
yield* fs.writeFileString(attributes, ATTRIBUTES).pipe(Effect.orElseSucceed(() => undefined));
|
|
235
|
+
}
|
|
236
|
+
});
|
|
237
|
+
const removeQuietly = (fs, file) => fs.remove(file).pipe(Effect.orElseSucceed(() => undefined));
|
|
238
|
+
/**
|
|
239
|
+
* Write the whole ledger in one layout, and remove the other form.
|
|
240
|
+
*
|
|
241
|
+
* A full write is what migration and pruning do; the store's own `put` writes
|
|
242
|
+
* only what changed. Removing the other form first would lose it on a crash, so
|
|
243
|
+
* the new form is written first and the old one removed after.
|
|
244
|
+
*/
|
|
245
|
+
const writeAll = (fs, path, dir, layout, entries) => Effect.gen(function* () {
|
|
246
|
+
yield* writeAttributes(fs, path, dir);
|
|
247
|
+
const target = answersDir(path, dir);
|
|
248
|
+
if (layout === "single") {
|
|
249
|
+
yield* fs.makeDirectory(dir, { recursive: true }).pipe(Effect.orElseSucceed(() => undefined));
|
|
250
|
+
const file = answersFile(path, dir);
|
|
251
|
+
if (entries.size === 0) {
|
|
252
|
+
yield* removeQuietly(fs, file);
|
|
253
|
+
}
|
|
254
|
+
else {
|
|
255
|
+
yield* fs.writeFileString(file, renderShard(entries.values())).pipe(Effect.orElseSucceed(() => undefined));
|
|
256
|
+
}
|
|
257
|
+
const names = yield* fs.readDirectory(target).pipe(Effect.orElseSucceed(() => []));
|
|
258
|
+
for (const name of names) {
|
|
259
|
+
if (name.endsWith(".jsonl"))
|
|
260
|
+
yield* removeQuietly(fs, path.join(target, name));
|
|
261
|
+
}
|
|
262
|
+
yield* removeQuietly(fs, answersObject(path, dir));
|
|
263
|
+
return;
|
|
264
|
+
}
|
|
265
|
+
const grouped = new Map();
|
|
266
|
+
for (const entry of entries.values()) {
|
|
267
|
+
const shard = shardOf(entry.key);
|
|
268
|
+
const list = grouped.get(shard) ?? [];
|
|
269
|
+
list.push(entry);
|
|
270
|
+
grouped.set(shard, list);
|
|
271
|
+
}
|
|
272
|
+
yield* fs.makeDirectory(target, { recursive: true }).pipe(Effect.orElseSucceed(() => undefined));
|
|
273
|
+
for (const [shard, list] of grouped) {
|
|
274
|
+
yield* fs
|
|
275
|
+
.writeFileString(path.join(target, shard + ".jsonl"), renderShard(list))
|
|
276
|
+
.pipe(Effect.orElseSucceed(() => undefined));
|
|
196
277
|
}
|
|
197
|
-
const
|
|
278
|
+
const names = yield* fs.readDirectory(target).pipe(Effect.orElseSucceed(() => []));
|
|
279
|
+
for (const name of names) {
|
|
280
|
+
if (!name.endsWith(".jsonl"))
|
|
281
|
+
continue;
|
|
282
|
+
if (!grouped.has(name.slice(0, -".jsonl".length)))
|
|
283
|
+
yield* removeQuietly(fs, path.join(target, name));
|
|
284
|
+
}
|
|
285
|
+
yield* removeQuietly(fs, answersFile(path, dir));
|
|
286
|
+
yield* removeQuietly(fs, answersObject(path, dir));
|
|
287
|
+
});
|
|
288
|
+
/**
|
|
289
|
+
* Open the store, and write in the layout the repository already uses unless the
|
|
290
|
+
* caller names one.
|
|
291
|
+
*
|
|
292
|
+
* One writer at a time: concurrent decisions finish together, and a single-file
|
|
293
|
+
* layout rewrites the whole ledger on a `put`, so letting two flushes interleave
|
|
294
|
+
* would lose one. A sharded flush writes only the shards whose keys changed.
|
|
295
|
+
*/
|
|
296
|
+
export const make = (fs, path, dir, options = {}) => Effect.gen(function* () {
|
|
297
|
+
const loaded = yield* loadAll(fs, path, dir);
|
|
298
|
+
const chosen = options.layout ?? loaded.layout;
|
|
299
|
+
const entries = yield* Ref.make(loaded.entries);
|
|
198
300
|
const dirty = yield* Ref.make(new Set());
|
|
199
|
-
const
|
|
301
|
+
const layoutRef = yield* Ref.make(chosen);
|
|
302
|
+
const fullWrite = yield* Ref.make(loaded.layout !== chosen);
|
|
200
303
|
const writer = yield* Semaphore.make(1);
|
|
201
304
|
const flush = writer.withPermits(1)(Effect.gen(function* () {
|
|
202
|
-
// Taken and cleared in one step: a put that lands while this flush is
|
|
203
|
-
// writing marks a fresh set, which its own flush then writes.
|
|
204
305
|
const changed = yield* Ref.getAndSet(dirty, new Set());
|
|
205
|
-
const current = yield* Ref.get(
|
|
206
|
-
const
|
|
207
|
-
const
|
|
306
|
+
const current = yield* Ref.get(entries);
|
|
307
|
+
const layout = yield* Ref.get(layoutRef);
|
|
308
|
+
const rewrite = yield* Ref.getAndSet(fullWrite, false);
|
|
309
|
+
if (rewrite || layout === "single") {
|
|
310
|
+
yield* writeAll(fs, path, dir, layout, current);
|
|
311
|
+
return;
|
|
312
|
+
}
|
|
313
|
+
yield* writeAttributes(fs, path, dir);
|
|
208
314
|
const target = answersDir(path, dir);
|
|
209
315
|
yield* fs.makeDirectory(target, { recursive: true }).pipe(Effect.orElseSucceed(() => undefined));
|
|
210
|
-
const
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
yield* fs.writeFileString(
|
|
218
|
-
}
|
|
219
|
-
}
|
|
220
|
-
for (const shard of targets) {
|
|
221
|
-
const entries = current.get(shard);
|
|
222
|
-
if (entries === undefined)
|
|
223
|
-
continue;
|
|
224
|
-
yield* fs
|
|
225
|
-
.writeFileString(path.join(target, shard + ".jsonl"), renderShard(entries.values()))
|
|
226
|
-
.pipe(Effect.orElseSucceed(() => undefined));
|
|
227
|
-
}
|
|
228
|
-
if (migrate) {
|
|
229
|
-
yield* fs.remove(single).pipe(Effect.orElseSucceed(() => undefined));
|
|
230
|
-
yield* Ref.set(needsMigration, false);
|
|
316
|
+
const shards = new Set([...changed].map(shardOf));
|
|
317
|
+
for (const shard of shards) {
|
|
318
|
+
const list = [...current.values()].filter((entry) => shardOf(entry.key) === shard);
|
|
319
|
+
const file = path.join(target, shard + ".jsonl");
|
|
320
|
+
if (list.length === 0)
|
|
321
|
+
yield* removeQuietly(fs, file);
|
|
322
|
+
else
|
|
323
|
+
yield* fs.writeFileString(file, renderShard(list)).pipe(Effect.orElseSucceed(() => undefined));
|
|
231
324
|
}
|
|
232
325
|
}));
|
|
233
326
|
return {
|
|
234
|
-
get: (key) => Effect.map(Ref.get(
|
|
235
|
-
const entry = current.get(shardOf(key))?.get(key);
|
|
236
|
-
return entry === undefined ? Option.none() : Option.some(entry.answer);
|
|
237
|
-
}),
|
|
327
|
+
get: (key) => Effect.map(Ref.get(entries), (current) => Option.fromUndefinedOr(current.get(key)?.answer)),
|
|
238
328
|
put: (key, answer) => Effect.gen(function* () {
|
|
239
329
|
const entry = { key, writtenAt: seconds(yield* Clock.currentTimeMillis), answer };
|
|
240
|
-
yield* Ref.update(
|
|
241
|
-
yield* Ref.update(dirty, (current) =>
|
|
242
|
-
const next = new Set(current);
|
|
243
|
-
next.add(shardOf(key));
|
|
244
|
-
return next;
|
|
245
|
-
});
|
|
330
|
+
yield* Ref.update(entries, (current) => new Map(current).set(key, entry));
|
|
331
|
+
yield* Ref.update(dirty, (current) => new Set(current).add(key));
|
|
246
332
|
yield* flush;
|
|
247
333
|
}),
|
|
248
|
-
size: Effect.map(Ref.get(
|
|
249
|
-
|
|
250
|
-
|
|
334
|
+
size: Effect.map(Ref.get(entries), (current) => current.size),
|
|
335
|
+
layout: chosen,
|
|
336
|
+
migrate: (layout) => Effect.gen(function* () {
|
|
337
|
+
const target = layout ?? (yield* Ref.get(layoutRef));
|
|
338
|
+
yield* Ref.set(layoutRef, target);
|
|
339
|
+
yield* Ref.set(fullWrite, true);
|
|
251
340
|
yield* flush;
|
|
252
|
-
return
|
|
341
|
+
return (yield* Ref.get(entries)).size;
|
|
253
342
|
}),
|
|
254
343
|
};
|
|
255
344
|
});
|
|
256
345
|
const sizeOf = (entry) => renderEntry(entry).length + 1;
|
|
257
346
|
/**
|
|
258
|
-
* Read
|
|
347
|
+
* Read the ledger and count what is in it.
|
|
259
348
|
*
|
|
260
349
|
* The keys are content hashes, so there is no rule name to group by: a verdict
|
|
261
350
|
* cannot be attributed to a question after the fact. What IS visible is the
|
|
@@ -263,48 +352,33 @@ const sizeOf = (entry) => renderEntry(entry).length + 1;
|
|
|
263
352
|
* reviewer opening a batch of new lines actually wants.
|
|
264
353
|
*/
|
|
265
354
|
export const census = (fs, path, dir) => Effect.gen(function* () {
|
|
266
|
-
const
|
|
267
|
-
const names = yield* fs.readDirectory(target).pipe(Effect.orElseSucceed(() => []));
|
|
355
|
+
const loaded = yield* loadAll(fs, path, dir);
|
|
268
356
|
const kinds = new Map();
|
|
269
357
|
const labels = new Map();
|
|
270
|
-
let entries = 0;
|
|
271
|
-
let shards = 0;
|
|
272
|
-
let bytes = 0;
|
|
273
|
-
let skippedLines = 0;
|
|
274
358
|
let yes = 0;
|
|
275
359
|
let no = 0;
|
|
276
360
|
let oldest;
|
|
277
361
|
let newest;
|
|
278
|
-
for (const
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
newest = newest === undefined ? entry.writtenAt : Math.max(newest, entry.writtenAt);
|
|
291
|
-
kinds.set(entry.answer.kind, (kinds.get(entry.answer.kind) ?? 0) + 1);
|
|
292
|
-
if (entry.answer.kind === "Probability") {
|
|
293
|
-
if (entry.answer.probability >= 0.5)
|
|
294
|
-
yes += 1;
|
|
295
|
-
else
|
|
296
|
-
no += 1;
|
|
297
|
-
}
|
|
298
|
-
else {
|
|
299
|
-
labels.set(entry.answer.label, (labels.get(entry.answer.label) ?? 0) + 1);
|
|
300
|
-
}
|
|
362
|
+
for (const entry of loaded.entries.values()) {
|
|
363
|
+
oldest = oldest === undefined ? entry.writtenAt : Math.min(oldest, entry.writtenAt);
|
|
364
|
+
newest = newest === undefined ? entry.writtenAt : Math.max(newest, entry.writtenAt);
|
|
365
|
+
kinds.set(entry.answer.kind, (kinds.get(entry.answer.kind) ?? 0) + 1);
|
|
366
|
+
if (entry.answer.kind === "Probability") {
|
|
367
|
+
if (entry.answer.probability >= 0.5)
|
|
368
|
+
yes += 1;
|
|
369
|
+
else
|
|
370
|
+
no += 1;
|
|
371
|
+
}
|
|
372
|
+
else {
|
|
373
|
+
labels.set(entry.answer.label, (labels.get(entry.answer.label) ?? 0) + 1);
|
|
301
374
|
}
|
|
302
375
|
}
|
|
303
376
|
return {
|
|
304
|
-
entries,
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
377
|
+
entries: loaded.entries.size,
|
|
378
|
+
files: loaded.files,
|
|
379
|
+
layout: loaded.layout,
|
|
380
|
+
bytes: loaded.bytes,
|
|
381
|
+
skippedLines: loaded.skippedLines,
|
|
308
382
|
kinds: [...kinds.entries()]
|
|
309
383
|
.sort(([left], [right]) => left.localeCompare(right))
|
|
310
384
|
.map(([kind, count]) => ({ kind, count })),
|
|
@@ -317,76 +391,48 @@ export const census = (fs, path, dir) => Effect.gen(function* () {
|
|
|
317
391
|
};
|
|
318
392
|
});
|
|
319
393
|
/**
|
|
320
|
-
* Drop cache entries and rewrite only the shards that changed.
|
|
394
|
+
* Drop cache entries and rewrite only the file or shards that changed.
|
|
321
395
|
*
|
|
322
396
|
* The age cut is per entry, from the timestamp each line carries. The size cut
|
|
323
|
-
* is oldest-first across the whole
|
|
324
|
-
* than written empty
|
|
325
|
-
* number of files.
|
|
397
|
+
* is oldest-first across the whole ledger. An emptied file or shard is deleted
|
|
398
|
+
* rather than written empty.
|
|
326
399
|
*/
|
|
327
400
|
export const prune = (fs, path, dir, options = {}) => Effect.gen(function* () {
|
|
328
401
|
const now = options.now ?? seconds(yield* Clock.currentTimeMillis);
|
|
329
402
|
const cutoff = options.olderThanSeconds === undefined ? undefined : now - options.olderThanSeconds;
|
|
330
|
-
const
|
|
331
|
-
const
|
|
332
|
-
|
|
333
|
-
const
|
|
334
|
-
|
|
335
|
-
|
|
403
|
+
const loaded = yield* loadAll(fs, path, dir);
|
|
404
|
+
const kept = new Map();
|
|
405
|
+
let removed = 0;
|
|
406
|
+
for (const entry of loaded.entries.values()) {
|
|
407
|
+
if (cutoff !== undefined && entry.writtenAt < cutoff) {
|
|
408
|
+
removed += 1;
|
|
336
409
|
continue;
|
|
337
|
-
const shard = name.slice(0, -".jsonl".length);
|
|
338
|
-
const text = yield* readText(fs, path.join(target, name));
|
|
339
|
-
for (const entry of parseLines(text).entries) {
|
|
340
|
-
if (cutoff !== undefined && entry.writtenAt < cutoff)
|
|
341
|
-
dropped.push({ shard, entry });
|
|
342
|
-
else {
|
|
343
|
-
const kept = keptByShard.get(shard) ?? [];
|
|
344
|
-
kept.push(entry);
|
|
345
|
-
keptByShard.set(shard, kept);
|
|
346
|
-
}
|
|
347
410
|
}
|
|
411
|
+
kept.set(entry.key, entry);
|
|
348
412
|
}
|
|
349
413
|
if (options.maxBytes !== undefined) {
|
|
350
414
|
let total = 0;
|
|
351
|
-
for (const
|
|
352
|
-
|
|
353
|
-
total += sizeOf(entry);
|
|
415
|
+
for (const entry of kept.values())
|
|
416
|
+
total += sizeOf(entry);
|
|
354
417
|
if (total > options.maxBytes) {
|
|
355
418
|
// A total order: oldest first, ties by key, so the result does not
|
|
356
419
|
// depend on directory iteration.
|
|
357
|
-
const ordered = [...
|
|
358
|
-
ordered.sort(Order.mapInput(Order.Number, (entry) => entry.writtenAt));
|
|
359
|
-
const removedKeys = new Set();
|
|
420
|
+
const ordered = [...kept.values()].sort((left, right) => left.writtenAt - right.writtenAt || byKey(left, right));
|
|
360
421
|
for (const entry of ordered) {
|
|
361
422
|
if (total <= options.maxBytes)
|
|
362
423
|
break;
|
|
363
|
-
|
|
364
|
-
|
|
424
|
+
kept.delete(entry.key);
|
|
425
|
+
removed += 1;
|
|
365
426
|
total -= sizeOf(entry);
|
|
366
427
|
}
|
|
367
|
-
for (const [shard, entries] of keptByShard) {
|
|
368
|
-
keptByShard.set(shard, entries.filter((entry) => !removedKeys.has(entry.key)));
|
|
369
|
-
}
|
|
370
428
|
}
|
|
371
429
|
}
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
if (entries.length === 0) {
|
|
377
|
-
yield* fs.remove(file).pipe(Effect.orElseSucceed(() => undefined));
|
|
378
|
-
}
|
|
379
|
-
else {
|
|
380
|
-
yield* fs.writeFileString(file, renderShard(entries)).pipe(Effect.orElseSucceed(() => undefined));
|
|
381
|
-
}
|
|
382
|
-
}
|
|
383
|
-
let kept = 0;
|
|
430
|
+
// Writing only when something changed keeps a no-op prune from touching the
|
|
431
|
+
// ledger's bytes (and therefore its diff).
|
|
432
|
+
if (removed > 0)
|
|
433
|
+
yield* writeAll(fs, path, dir, loaded.layout, kept);
|
|
384
434
|
let bytes = 0;
|
|
385
|
-
for (const
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
bytes += sizeOf(entry);
|
|
389
|
-
}
|
|
390
|
-
}
|
|
391
|
-
return { removed: dropped.length, kept, bytes };
|
|
435
|
+
for (const entry of kept.values())
|
|
436
|
+
bytes += sizeOf(entry);
|
|
437
|
+
return { removed, kept: kept.size, bytes };
|
|
392
438
|
});
|
package/dist/config.js
CHANGED
|
@@ -133,6 +133,17 @@ export const JoggleConfig = Schema.Struct({
|
|
|
133
133
|
/** Input tokens one run may spend on judgement. */
|
|
134
134
|
maxInputTokens: Schema.optionalKey(Schema.Finite),
|
|
135
135
|
})),
|
|
136
|
+
/**
|
|
137
|
+
* Where the committed ledger lives, and in what shape.
|
|
138
|
+
*
|
|
139
|
+
* One sorted `answers.jsonl` (the default) is what a reviewer can read: a
|
|
140
|
+
* verdict change is one line. `sharded` is the opt-in for a store large enough
|
|
141
|
+
* that a whole-file rewrite or a merge is a real cost; both are read either way,
|
|
142
|
+
* and `joggle cache migrate --layout` moves between them.
|
|
143
|
+
*/
|
|
144
|
+
cache: Schema.optionalKey(Schema.Struct({
|
|
145
|
+
layout: Schema.optionalKey(Schema.Literals(["single", "sharded"])),
|
|
146
|
+
})),
|
|
136
147
|
evidence: Schema.optionalKey(Schema.Struct({
|
|
137
148
|
/**
|
|
138
149
|
* The repository's architecture, in the repository's own words.
|
package/dist/decision.js
CHANGED
|
@@ -183,7 +183,9 @@ export const layer = (options) => Layer.effectContext(Effect.gen(function* () {
|
|
|
183
183
|
// merge driver resolves the residual case by key. A legacy single
|
|
184
184
|
// `answers.json` is read and migrated on the first write. See
|
|
185
185
|
// `docs/artifacts.md`.
|
|
186
|
-
const answers = yield* makeAnswerStore(fs, path, options.cacheDir
|
|
186
|
+
const answers = yield* makeAnswerStore(fs, path, options.cacheDir, {
|
|
187
|
+
layout: options.layout,
|
|
188
|
+
});
|
|
187
189
|
const planAnswers = {
|
|
188
190
|
get: (key) => Effect.map(answers.get(key), (stored) => Option.map(stored, answerOf)),
|
|
189
191
|
put: (key, answer) => answers.put(key, storedOf(answer)),
|
package/dist/main.js
CHANGED
|
@@ -163,7 +163,13 @@ const checkCommand = Command.make("check", {
|
|
|
163
163
|
baselinePath: Option.getOrUndefined(config.baseline),
|
|
164
164
|
updateBaselinePath: Option.getOrUndefined(config.updateBaseline),
|
|
165
165
|
allowUnjudgedBaseline: config.allowUnjudgedBaseline,
|
|
166
|
-
}).pipe(Effect.provide(Layer.mergeAll(rulesLayer, decisionLayer({
|
|
166
|
+
}).pipe(Effect.provide(Layer.mergeAll(rulesLayer, decisionLayer({
|
|
167
|
+
cacheDir,
|
|
168
|
+
wireCacheDir: machineCache,
|
|
169
|
+
offline: config.offline,
|
|
170
|
+
apiKey,
|
|
171
|
+
layout: effective.cache?.layout,
|
|
172
|
+
}), tsgoLayer(cwd, {
|
|
167
173
|
runExternalCode: config.runExternalCode || effective.runExternalCode === true,
|
|
168
174
|
}))));
|
|
169
175
|
yield* write(render(report, config.format, { color: process.stdout.isTTY === true }));
|
|
@@ -475,6 +481,7 @@ const calibrateCommand = Command.make("calibrate", {
|
|
|
475
481
|
wireCacheDir: yield* runCacheDirFor(cwd),
|
|
476
482
|
offline: false,
|
|
477
483
|
apiKey,
|
|
484
|
+
layout: ruleSet.effective.cache?.layout,
|
|
478
485
|
onAnswer: (event) => recorded.push({ answers: event.answers }),
|
|
479
486
|
}),
|
|
480
487
|
// The trace and the node types both run tsgo; the calibrate command
|
|
@@ -568,29 +575,39 @@ const cacheReport = Command.make("report", {
|
|
|
568
575
|
.map((entry) => entry.label + " " + String(entry.count))
|
|
569
576
|
.join(", ");
|
|
570
577
|
yield* write([
|
|
571
|
-
"joggle: " +
|
|
578
|
+
"joggle: " +
|
|
579
|
+
found.entries +
|
|
580
|
+
" entries in " +
|
|
581
|
+
found.files +
|
|
582
|
+
" file(s) (" +
|
|
583
|
+
found.layout +
|
|
584
|
+
"), " +
|
|
585
|
+
found.bytes +
|
|
586
|
+
" bytes",
|
|
572
587
|
" kinds " + (kinds === "" ? "none" : kinds),
|
|
573
588
|
" verdicts Probability " + found.verdicts.yes + " yes / " + found.verdicts.no + " no",
|
|
574
589
|
" labels " + (labels === "" ? "none" : labels),
|
|
575
590
|
" written " + date(found.oldest) + " .. " + date(found.newest),
|
|
576
591
|
...(found.skippedLines === 0 ? [] : [" skipped " + found.skippedLines + " unreadable line(s)"]),
|
|
577
592
|
].join("\n"));
|
|
578
|
-
})).pipe(Command.withDescription("Render the committed answer ledger for a reader: entry and
|
|
593
|
+
})).pipe(Command.withDescription("Render the committed answer ledger for a reader: entry and file counts, layout, verdict distribution, labels, and the date range. A summary cannot be replayed; this is what to read when reviewing the diff."), Command.withShortDescription("Report on the answer cache."), Command.withExamples([
|
|
579
594
|
{ command: "joggle cache report", description: "Count what the committed ledger holds." },
|
|
580
595
|
]));
|
|
581
596
|
const cacheMigrate = Command.make("migrate", {
|
|
582
597
|
cwd: Flag.String("cwd").pipe(Flag.withDescription("Project root whose cache to migrate (default: the current directory)."), Flag.optional),
|
|
583
598
|
cacheDir: Flag.String("cache-dir").pipe(Flag.withDescription("Where the answer cache lives: <cwd>/.joggle when it exists, otherwise the machine cache."), Flag.optional),
|
|
599
|
+
layout: Flag.Literals("layout", ["single", "sharded"]).pipe(Flag.withDescription("How to write the ledger: one sorted answers.jsonl (default), or shards under answers/."), Flag.withDefault("single")),
|
|
584
600
|
}, (config) => Effect.gen(function* () {
|
|
585
601
|
const fs = yield* FileSystem.FileSystem;
|
|
586
602
|
const path = yield* Path.Path;
|
|
587
603
|
const cwd = path.resolve(Option.getOrUndefined(config.cwd) ?? process.cwd());
|
|
588
604
|
const dir = yield* cacheDirFor(cwd, Option.getOrUndefined(config.cacheDir));
|
|
589
605
|
const store = yield* makeAnswerStore(fs, path, dir);
|
|
590
|
-
const written = yield* store.migrate;
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
606
|
+
const written = yield* store.migrate(config.layout);
|
|
607
|
+
const target = config.layout === "single" ? path.join(dir, "answers.jsonl") : path.join(dir, "answers");
|
|
608
|
+
yield* write("joggle: wrote " + written + " answer(s) to " + target);
|
|
609
|
+
})).pipe(Command.withDescription("Rewrite the cache in one layout and remove the other: a single sorted answers.jsonl by default, or shards with --layout sharded. Idempotent."), Command.withShortDescription("Consolidate or shard the answer cache."));
|
|
610
|
+
const cacheCommand = Command.make("cache", {}, () => write("joggle cache: run `joggle cache report`, `prune`, or `migrate --layout single|sharded`; docs/artifacts.md has the reasoning.")).pipe(Command.withDescription("Work on the answer cache itself: prune it, migrate it, and see how it is laid out."), Command.withShortDescription("Prune the answer cache."), Command.withSubcommands([cachePrune, cacheMigrate, cacheReport]));
|
|
594
611
|
const cli = Command.make("joggle").pipe(Command.withDescription("Cross-file patterns and idioms for TypeScript, enforced like a linter: deterministic where it can prove, System One where it has to judge."), Command.withExamples([
|
|
595
612
|
{
|
|
596
613
|
command: "joggle check --pr",
|