@autotelic/joggle 0.8.2 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -300,7 +300,11 @@ const writeAll = (fs, path, dir, layout, entries) => Effect.gen(function* () {
300
300
  yield* removeQuietly(fs, file);
301
301
  }
302
302
  else {
303
- yield* fs.writeFileString(file, renderShard(entries.values())).pipe(Effect.orElseSucceed(() => undefined));
303
+ // Temp then rename: a killed run must leave a whole ledger or the previous one, never
304
+ // half a line. A truncated tail decodes as "skipped lines", which loses answers
305
+ // silently -- exactly the failure an interruption is most likely to cause.
306
+ yield* Effect.orElseSucceed(fs.writeFileString(file + ".tmp", renderShard(entries.values())), () => undefined);
307
+ yield* Effect.orElseSucceed(fs.rename(file + ".tmp", file), () => undefined);
304
308
  }
305
309
  const names = yield* fs.readDirectory(target).pipe(Effect.orElseSucceed(() => []));
306
310
  for (const name of names) {
@@ -319,9 +323,10 @@ const writeAll = (fs, path, dir, layout, entries) => Effect.gen(function* () {
319
323
  }
320
324
  yield* fs.makeDirectory(target, { recursive: true }).pipe(Effect.orElseSucceed(() => undefined));
321
325
  for (const [shard, list] of grouped) {
322
- yield* fs
323
- .writeFileString(path.join(target, shard + ".jsonl"), renderShard(list))
324
- .pipe(Effect.orElseSucceed(() => undefined));
326
+ // Same rule as the single layout: a shard appears whole or not at all.
327
+ const file = path.join(target, shard + ".jsonl");
328
+ yield* Effect.orElseSucceed(fs.writeFileString(file + ".tmp", renderShard(list)), () => undefined);
329
+ yield* Effect.orElseSucceed(fs.rename(file + ".tmp", file), () => undefined);
325
330
  }
326
331
  const names = yield* fs.readDirectory(target).pipe(Effect.orElseSucceed(() => []));
327
332
  for (const name of names) {
package/dist/check.js CHANGED
@@ -535,6 +535,9 @@ export const checkRepository = Effect.fn("joggle.check")(function* (options) {
535
535
  estimated += cost;
536
536
  allowed.push(chunk);
537
537
  }
538
+ // The denominator of the progress line: every question this run will resolve, replayed or
539
+ // judged. Set here because the plans are built after the judgement layer exists.
540
+ yield* decisionStats.setTotal(allowed.reduce((sum, chunk) => sum + chunk.plans.length, 0));
538
541
  // A chunk that fails is CUT AND RETRIED, not abandoned.
539
542
  //
540
543
  // The provider occasionally returns a distribution that does not sum to 1, and
package/dist/decision.js CHANGED
@@ -31,6 +31,40 @@ import { answerOf, memoize, PlanAnswers, storedOf } from "./plans.js";
31
31
  /** The run's judgement accounting, read once at the end of a run. */
32
32
  export class DecisionStats extends Context.Service()("@joggle/DecisionStats") {
33
33
  }
34
+ /** How often a long run says where it is. */
35
+ const PROGRESS_INTERVAL_MS = 10_000;
36
+ /** Tokens, compact enough to read in a moving line. */
37
+ const compactTokens = (value) => value >= 1_000_000
38
+ ? (value / 1_000_000).toFixed(1) + "M"
39
+ : value >= 1_000
40
+ ? Math.round(value / 1_000) + "k"
41
+ : String(value);
42
+ /** Elapsed time as a person reads it. */
43
+ const elapsedText = (milliseconds) => {
44
+ const seconds = Math.floor(milliseconds / 1000);
45
+ const minutes = Math.floor(seconds / 60);
46
+ return minutes > 0 ? minutes + "m" + String(seconds % 60).padStart(2, "0") + "s" : seconds + "s";
47
+ };
48
+ /**
49
+ * The progress line, as one testable string.
50
+ *
51
+ * Counts only, and the two units are named separately: "findings" and "findings no judgement
52
+ * reached" added into one number is the misread that made 7,787 unjudged candidates look like
53
+ * 5,315 pieces of debt. No denominator before the plans are known, because a wrong one is
54
+ * worse than none.
55
+ */
56
+ export const progressLine = (input) => "joggle: " +
57
+ String(input.requests) +
58
+ (input.total === undefined ? "" : "/" + String(input.total)) +
59
+ " candidate(s) resolved (" +
60
+ String(input.calls) +
61
+ " judged, " +
62
+ String(input.replayed) +
63
+ " replayed) · " +
64
+ compactTokens(input.inputTokens) +
65
+ (input.budget === undefined ? "" : "/" + compactTokens(input.budget)) +
66
+ " input tokens · " +
67
+ elapsedText(input.elapsedMs);
34
68
  const emptyStats = {
35
69
  requests: 0,
36
70
  replayed: 0,
@@ -118,6 +152,57 @@ export const layer = (options) => Layer.effectContext(Effect.gen(function* () {
118
152
  });
119
153
  const entries = yield* Ref.make(yield* load);
120
154
  const stats = yield* Ref.make(emptyStats);
155
+ const planned = yield* Ref.make(undefined);
156
+ const startedAt = Date.now();
157
+ let lastBeatAt = startedAt;
158
+ /*
159
+ * Where a long run says it is.
160
+ *
161
+ * A judged run can take many minutes, and before this it printed NOTHING until the
162
+ * report: stdout and stderr both zero bytes, indistinguishable from hung. The only
163
+ * observable signal was the answer cache growing, which is not something a person
164
+ * should have to tail.
165
+ *
166
+ * Counts only, on stderr, so every machine format on stdout stays clean; the two units
167
+ * are named separately because "findings" and "findings no judgement reached" being
168
+ * added into one number is exactly the misread this line exists to prevent.
169
+ */
170
+ const beat = Effect.gen(function* () {
171
+ const now = Date.now();
172
+ if (now - lastBeatAt < PROGRESS_INTERVAL_MS)
173
+ return;
174
+ lastBeatAt = now;
175
+ const current = yield* Ref.get(stats);
176
+ const total = yield* Ref.get(planned);
177
+ const elapsedMs = now - startedAt;
178
+ const budget = options.inputBudget ?? policy.decision.maxInputTokens;
179
+ process.stderr.write(progressLine({
180
+ requests: current.requests,
181
+ calls: current.calls,
182
+ replayed: current.replayed,
183
+ inputTokens: current.inputTokens,
184
+ total,
185
+ budget,
186
+ elapsedMs,
187
+ }) + "\n");
188
+ if (options.progressFile === undefined || options.readOnly === true)
189
+ return;
190
+ // Temp then rename: a poller reads a whole document or the previous one, never half.
191
+ const document = JSON.stringify({
192
+ phase: "judging",
193
+ candidatesTotal: total ?? null,
194
+ candidatesResolved: current.requests,
195
+ judged: current.calls,
196
+ replayed: current.replayed,
197
+ apiCalls: current.calls,
198
+ inputTokens: current.inputTokens,
199
+ inputBudget: budget,
200
+ elapsedMs,
201
+ updatedAt: new Date(now).toISOString(),
202
+ }, null, 2) + "\n";
203
+ yield* Effect.orElseSucceed(fs.writeFileString(options.progressFile + ".tmp", document), () => undefined);
204
+ yield* Effect.orElseSucceed(fs.rename(options.progressFile + ".tmp", options.progressFile), () => undefined);
205
+ });
121
206
  // Decisions run concurrently, so several calls can finish at once and each
122
207
  // one writes the whole cache. The permit keeps one writer in the file at a
123
208
  // time, so a full write cannot interleave with another.
@@ -168,7 +253,7 @@ export const layer = (options) => Layer.effectContext(Effect.gen(function* () {
168
253
  yield* Ref.update(entries, (current) => ({ ...current, [key]: response }));
169
254
  yield* flush;
170
255
  return response;
171
- }),
256
+ }).pipe(Effect.ensuring(beat)),
172
257
  });
173
258
  const providerModel = yield* TypeSafeDecisionModel.make({ model: policy.model }).pipe(Effect.provideService(TypeSafeClient.TypeSafeClient, client));
174
259
  // The per-question cache, above the wire.
@@ -212,5 +297,8 @@ export const layer = (options) => Layer.effectContext(Effect.gen(function* () {
212
297
  answers: decided.answers,
213
298
  }))),
214
299
  }));
215
- return Context.make(DecisionModel.DecisionModel, decisionModel).pipe(Context.add(DecisionStats, { read: Ref.get(stats) }), Context.add(PlanAnswers, planAnswers));
300
+ return Context.make(DecisionModel.DecisionModel, decisionModel).pipe(Context.add(DecisionStats, {
301
+ read: Ref.get(stats),
302
+ setTotal: (total) => Ref.set(planned, total),
303
+ }), Context.add(PlanAnswers, planAnswers));
216
304
  }));
package/dist/main.js CHANGED
@@ -185,6 +185,10 @@ const checkCommand = Command.make("check", {
185
185
  }).pipe(Effect.provide(Layer.mergeAll(rulesLayer, decisionLayer({
186
186
  cacheDir,
187
187
  wireCacheDir: machineCache,
188
+ // Machine-local, pollable, and never part of the committed ledger: it describes
189
+ // THIS run. A long judged run used to print nothing at all until the report.
190
+ progressFile: path.join(machineCache, "progress.json"),
191
+ inputBudget: Option.getOrUndefined(config.maxTokens) ?? effective.decision?.maxInputTokens,
188
192
  offline: config.offline,
189
193
  apiKey,
190
194
  layout: effective.cache?.layout,
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@autotelic/joggle",
3
- "version": "0.8.2",
3
+ "version": "0.9.0",
4
4
  "type": "module",
5
5
  "description": "Cross-file patterns and idioms for TypeScript — the layer above the linter and the typechecker.",
6
6
  "license": "MIT",