@awebai/oats 0.61.0 → 0.63.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/bin/oats.mjs +27 -12
- package/docs/capabilities.md +1 -1
- package/docs/desktop-cli-api.md +30 -2
- package/docs/execution-targets.md +89 -4
- package/docs/implementation.md +2 -1
- package/docs/named-test-runs.md +382 -0
- package/docs/official-catalog.md +1 -1
- package/docs/packages.md +5 -5
- package/docs/release-lane.md +13 -0
- package/docs/release-notes/v0.62.0.md +223 -0
- package/docs/release-notes/v0.63.0.md +135 -0
- package/docs/souls-and-instances.md +6 -1
- package/docs/upgrading.md +135 -11
- package/docs/workspaces.md +1 -1
- package/lib/core.mjs +73 -23
- package/lib/doctor-capture.mjs +35 -0
- package/lib/session-viewer.mjs +86 -7
- package/lib/tree-copy.mjs +188 -7
- package/package-catalog.json +1 -1
- package/package.json +1 -1
- package/packages/client/cli-locator.mjs +2 -2
- package/packages/record/README.md +87 -13
- package/packages/record/bin/capture-hook.mjs +8 -0
- package/packages/record/bin/capture.mjs +200 -139
- package/packages/record/bin/recall.mjs +8 -3
- package/packages/record/bin/setup.mjs +116 -225
- package/packages/record/bin/turn-record.mjs +2 -2
- package/packages/record/docs/capture-service.md +1527 -28
- package/packages/record/docs/capture-status.schema.json +429 -12
- package/packages/record/lib/capture-admission.mjs +82 -0
- package/packages/record/lib/capture-aw-unit.mjs +137 -0
- package/packages/record/lib/capture-aw.mjs +31 -12
- package/packages/record/lib/capture-budget.mjs +31 -0
- package/packages/record/lib/capture-cc.mjs +39 -6
- package/packages/record/lib/capture-cleanup-queue.mjs +135 -0
- package/packages/record/lib/capture-cleanup-work.mjs +77 -0
- package/packages/record/lib/capture-command-operation.mjs +37 -0
- package/packages/record/lib/capture-command-output.mjs +62 -0
- package/packages/record/lib/capture-command-worker.mjs +6 -0
- package/packages/record/lib/capture-command.mjs +65 -0
- package/packages/record/lib/capture-completion.mjs +152 -0
- package/packages/record/lib/capture-context.mjs +50 -0
- package/packages/record/lib/capture-default-roots.mjs +30 -0
- package/packages/record/lib/capture-discovery.mjs +114 -0
- package/packages/record/lib/capture-failures.mjs +45 -0
- package/packages/record/lib/capture-health-doctor.mjs +13 -0
- package/packages/record/lib/capture-health-installation.mjs +117 -0
- package/packages/record/lib/capture-health.mjs +116 -0
- package/packages/record/lib/capture-hook-ownership.mjs +58 -0
- package/packages/record/lib/capture-hook-runner.mjs +17 -0
- package/packages/record/lib/capture-hook.mjs +26 -0
- package/packages/record/lib/capture-index-inventory.mjs +205 -0
- package/packages/record/lib/capture-index-outcomes.mjs +113 -0
- package/packages/record/lib/capture-index-reset.mjs +110 -0
- package/packages/record/lib/capture-index-work.mjs +77 -0
- package/packages/record/lib/capture-issue-preparation.mjs +73 -0
- package/packages/record/lib/capture-issues.mjs +15 -2
- package/packages/record/lib/capture-lock.mjs +137 -17
- package/packages/record/lib/capture-native-unit.mjs +322 -0
- package/packages/record/lib/capture-pass.mjs +116 -0
- package/packages/record/lib/capture-recovery-snapshot.mjs +182 -0
- package/packages/record/lib/capture-retained-scratch.mjs +213 -0
- package/packages/record/lib/capture-retry.mjs +102 -0
- package/packages/record/lib/capture-run.mjs +125 -0
- package/packages/record/lib/capture-scratch.mjs +12 -3
- package/packages/record/lib/capture-service-definition.mjs +104 -0
- package/packages/record/lib/capture-service-install.mjs +362 -0
- package/packages/record/lib/capture-service-ownership.mjs +137 -0
- package/packages/record/lib/capture-service-runner.mjs +18 -0
- package/packages/record/lib/capture-small-snapshot.mjs +85 -0
- package/packages/record/lib/capture-source-discovery.mjs +319 -0
- package/packages/record/lib/capture-source-queue.mjs +199 -0
- package/packages/record/lib/capture-source-work.mjs +85 -0
- package/packages/record/lib/capture-state.mjs +16 -71
- package/packages/record/lib/capture-status.mjs +171 -0
- package/packages/record/lib/capture-supervisor.mjs +136 -0
- package/packages/record/lib/capture-watch.mjs +32 -0
- package/packages/record/lib/capture-work-session.mjs +241 -0
- package/packages/record/lib/capture-worker.mjs +74 -0
- package/packages/record/lib/derived-state.mjs +58 -17
- package/packages/record/lib/file-lines.mjs +22 -6
- package/packages/record/lib/formats.mjs +4 -3
- package/packages/record/lib/index-capture.mjs +97 -0
- package/packages/record/lib/index-db.mjs +121 -66
- package/packages/record/lib/index-reset.mjs +19 -0
- package/packages/record/lib/store.mjs +46 -14
- package/packages/record/lib/transcript-prefix.mjs +125 -0
- package/skills/oats-getting-started/SKILL.md +1 -1
- package/docs/macos-test-runs.md +0 -246
|
@@ -0,0 +1,241 @@
|
|
|
1
|
+
// Private owning worker session. Bootstrap has already registered this worker
|
|
2
|
+
// under the root lock and reaped its predecessor. Root selection and ordinary
|
|
3
|
+
// command routing remain separate; completion requires a fresh all-stage audit.
|
|
4
|
+
import { randomUUID } from "node:crypto";
|
|
5
|
+
import { RecordStore } from "./store.mjs";
|
|
6
|
+
import { RecordIndex } from "./index-db.mjs";
|
|
7
|
+
import { DerivedState } from "./derived-state.mjs";
|
|
8
|
+
import { CaptureSourceQueue } from "./capture-source-queue.mjs";
|
|
9
|
+
import { captureSourceWork } from "./capture-source-work.mjs";
|
|
10
|
+
import { CaptureIndexInventory } from "./capture-index-inventory.mjs";
|
|
11
|
+
import { CaptureIndexOutcomes } from "./capture-index-outcomes.mjs";
|
|
12
|
+
import { CaptureIndexReset } from "./capture-index-reset.mjs";
|
|
13
|
+
import { CaptureRun } from "./capture-run.mjs";
|
|
14
|
+
import { readCaptureStatus } from "./capture-status.mjs";
|
|
15
|
+
import { reconcileCaptureInterruption, selectCaptureWork } from "./capture-retry.mjs";
|
|
16
|
+
import { readCaptureAdmission } from "./capture-admission.mjs";
|
|
17
|
+
import { FAILURE_REASONS } from "./capture-failures.mjs";
|
|
18
|
+
import { CaptureCleanupQueue } from "./capture-cleanup-queue.mjs";
|
|
19
|
+
import { CaptureSourceDiscovery } from "./capture-source-discovery.mjs";
|
|
20
|
+
import { auditCaptureCompletion, auditCaptureSourceRetirement } from "./capture-completion.mjs";
|
|
21
|
+
import { CaptureIssuePreparation } from "./capture-issue-preparation.mjs";
|
|
22
|
+
|
|
23
|
+
export class CaptureWorkSession {
|
|
24
|
+
constructor(start, { owner, selection, trigger = "manual", now = Date.now, unit = {}, roots = null, indexEnabled = true, issues = null } = {}) {
|
|
25
|
+
if (typeof start?.root !== "string" || typeof start.workerNonce !== "string" || !start.workerNonce ||
|
|
26
|
+
typeof start.pass?.runId !== "string" || !start.pass.runId ||
|
|
27
|
+
typeof start.pass.finalize !== "boolean" || typeof start.pass.reset !== "boolean" || typeof now !== "function" ||
|
|
28
|
+
typeof indexEnabled !== "boolean")
|
|
29
|
+
throw new Error("capture work session requires a supervised pass");
|
|
30
|
+
this.start = start; this.now = now; this.unit = unit; this.indexEnabled = indexEnabled;
|
|
31
|
+
try {
|
|
32
|
+
start.budget.check();
|
|
33
|
+
this.store = new RecordStore(start.root, { owner });
|
|
34
|
+
this.state = new DerivedState(start.root);
|
|
35
|
+
// Append-only workers must not even open an existing index: construction
|
|
36
|
+
// can migrate/rebuild it before any scheduling decision is made.
|
|
37
|
+
this.index = indexEnabled ? new RecordIndex(this.store) : null;
|
|
38
|
+
const prior = readCaptureStatus(start.root);
|
|
39
|
+
if (prior?.lastRun?.id === start.pass.runId && prior.lastRun.outcome !== "running")
|
|
40
|
+
throw new Error("capture run identity already finished");
|
|
41
|
+
const previous = start.pass.previous?.outcome === "failed" ? start.pass.previous.reason : null;
|
|
42
|
+
// An observed child failure belongs to this pass. It cannot diagnose an
|
|
43
|
+
// older abandoned admission merely because this child reconciles it.
|
|
44
|
+
const reasonFor = id => id === start.pass.runId && FAILURE_REASONS.includes(previous) ? previous : "interrupted";
|
|
45
|
+
reconcileCaptureInterruption(this.state, { reason: reasonFor(readCaptureAdmission(this.state)?.runId), now: now() });
|
|
46
|
+
this.cleanup = new CaptureCleanupQueue(this.store, this.state);
|
|
47
|
+
this.cleanup.reconcile(start, { reason: reasonFor(this.cleanup.active()?.runId), now: now() });
|
|
48
|
+
this.queue = new CaptureSourceQueue(this.state, { owner, selection });
|
|
49
|
+
const retirement = this.state.get("capture-sweep-retirement", owner);
|
|
50
|
+
if (!start.pass.finalize && retirement?.sourceScope === this.queue.current().id &&
|
|
51
|
+
retirement.runId === prior?.lastRun?.id && prior.lastRun.outcome !== "running") {
|
|
52
|
+
this.state.transaction(() => {
|
|
53
|
+
this.queue.finish();
|
|
54
|
+
this.state.delete("capture-sweep-retirement", owner);
|
|
55
|
+
});
|
|
56
|
+
this.queue = new CaptureSourceQueue(this.state, { owner, selection });
|
|
57
|
+
}
|
|
58
|
+
if (roots !== null) {
|
|
59
|
+
this.sourceDiscovery = new CaptureSourceDiscovery(this.queue, roots);
|
|
60
|
+
this.sourceDiscovery.reconcile(start, { reason: reasonFor(this.sourceDiscovery.active()?.runId), now: now() });
|
|
61
|
+
}
|
|
62
|
+
if (issues !== null) this.issues = new CaptureIssuePreparation(this.state, this.queue,
|
|
63
|
+
{ ...issues, runId: start.pass.runId });
|
|
64
|
+
// Retain a reset's scope even if derived inventory tables were lost.
|
|
65
|
+
// Recreating membership may restart enumeration; it cannot erase reset
|
|
66
|
+
// admission or the index token needed to recover a committed operation.
|
|
67
|
+
if (indexEnabled) {
|
|
68
|
+
const previousSource = this.state.get("capture-index-source", owner);
|
|
69
|
+
const scope = this.state.get("capture-index-reset", "current")?.scope ??
|
|
70
|
+
(previousSource && previousSource.sourceScope !== this.queue.current().id ? randomUUID() :
|
|
71
|
+
this.state.get("index-inventory", "current")?.id ?? randomUUID());
|
|
72
|
+
this.openIndexScope(scope);
|
|
73
|
+
this.reset.reconcile(start, { reason: reasonFor(this.reset.current()?.attempt?.runId), now: now() });
|
|
74
|
+
}
|
|
75
|
+
const failures = [this.state.get("capture-run-failure", "last"),
|
|
76
|
+
this.state.get("capture-index-reset-failure", "last"), this.state.get("capture-cleanup-failure", "last"),
|
|
77
|
+
this.state.get("capture-discovery-failure", "last")].filter(Boolean);
|
|
78
|
+
const currentReason = start.pass.failure ?? failures.find(row => row.runId === start.pass.runId)?.reason ?? null;
|
|
79
|
+
const previousReason = failures.find(row => row.runId === prior?.lastRun?.id)?.reason ??
|
|
80
|
+
(prior?.lastRun?.id === start.pass.runId ? previous : null);
|
|
81
|
+
this.run = new CaptureRun(start.root, this.state, { id: start.pass.runId, trigger,
|
|
82
|
+
reclaimedLock: start.reclaimedLock ?? null, previousFailure: previousReason,
|
|
83
|
+
currentFailure: currentReason ? { id: start.pass.runId, reason: currentReason } : null });
|
|
84
|
+
} catch (error) { this.close(); throw error; }
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
openIndexScope(id) {
|
|
88
|
+
if (!this.indexEnabled) throw new Error("append-only capture cannot open index inventory");
|
|
89
|
+
this.inventory?.close();
|
|
90
|
+
this.inventory = new CaptureIndexInventory(this.state, { id });
|
|
91
|
+
this.outcomes = new CaptureIndexOutcomes(this.index, this.inventory);
|
|
92
|
+
this.reset = new CaptureIndexReset(this.index, this.inventory);
|
|
93
|
+
const binding = { sourceScope: this.queue.current().id, indexScope: id };
|
|
94
|
+
if (JSON.stringify(this.state.get("capture-index-source", this.queue.owner)) !== JSON.stringify(binding))
|
|
95
|
+
this.state.transaction(() => this.state.set("capture-index-source", this.queue.owner, binding));
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
sourceReady(input) {
|
|
99
|
+
if (!input) return false;
|
|
100
|
+
if (input.source === "aw") return true;
|
|
101
|
+
// Native recovery keys bind the exact input, not every source sharing a
|
|
102
|
+
// stream. A deleting allocation is a dependency, even if its directory is
|
|
103
|
+
// already gone: admitting cleanup-only capture would starve its due retry.
|
|
104
|
+
const claim = this.state.get("recovery-allocation", `${input.stream}:${input.path}`);
|
|
105
|
+
if (claim?.phase !== "deleting") return true;
|
|
106
|
+
if (claim.stream !== input.stream || !/^[a-f0-9]{32}$/.test(claim.id))
|
|
107
|
+
throw new Error("invalid native cleanup dependency");
|
|
108
|
+
return false;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
process(stage, allowRetry) {
|
|
112
|
+
if (stage !== "capture" && (stage !== "index" || !this.indexEnabled))
|
|
113
|
+
throw new Error("capture stage is not enabled");
|
|
114
|
+
const options = { ...this.unit, now: this.now(), allowRetry,
|
|
115
|
+
inputReady: input => this.sourceReady(input),
|
|
116
|
+
retryFilter: retry => retry.stage === stage && (stage !== "capture" || this.sourceReady(this.queue.forStream(retry.stream))) };
|
|
117
|
+
const result = stage === "capture"
|
|
118
|
+
? captureSourceWork(this.store, this.state, this.queue, this.run, this.start, { ...options, indexInventory: this.inventory ?? null })
|
|
119
|
+
: this.outcomes.advance(this.run, this.start, { ...options, memo: true });
|
|
120
|
+
if (result.resetRequired) this.reset.request(this.start);
|
|
121
|
+
// One independent bounded cleanup turn cannot change stream-stage rotation.
|
|
122
|
+
// A throw retains the already-committed source/index counts and its own
|
|
123
|
+
// admission, so a replacement can back off deletion and keep healthy work.
|
|
124
|
+
const cleanupWork = result.selected ? this.clean(false) : null;
|
|
125
|
+
return { stage, ...result, cleanupWork };
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
clean(retry) {
|
|
129
|
+
return this.cleanup.advance(this.run, this.start, { now: this.now(), retry,
|
|
130
|
+
maxRecords: Math.max(2, this.unit.maxRecords ?? 2000) });
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
collect() {
|
|
134
|
+
if (this.state.db.prepare("SELECT 1 AS found FROM state WHERE kind='source-queue-garbage' OR (? AND kind='index-inventory-garbage') LIMIT 1")
|
|
135
|
+
.get(Number(this.indexEnabled))?.found !== 1)
|
|
136
|
+
return null;
|
|
137
|
+
return this.state.transaction(() => {
|
|
138
|
+
const sources = this.queue.cleanup(), streams = this.inventory?.cleanup() ?? 0;
|
|
139
|
+
// Retiring obsolete enumeration is bookkeeping, not new capture/index
|
|
140
|
+
// progress. In particular it must not refresh every no-op pass's history.
|
|
141
|
+
return { stage: "inventory-cleanup", sources, streams };
|
|
142
|
+
});
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
step() {
|
|
146
|
+
const stopped = () => this.start.budget.sample() !== null;
|
|
147
|
+
if (stopped()) return { stopped: true };
|
|
148
|
+
if (this.reset?.current()) {
|
|
149
|
+
const result = this.reset.advance(this.run, this.start, { now: this.now() });
|
|
150
|
+
if (result.reset) { this.openIndexScope(result.scope); return { stage: "reset", ...result }; }
|
|
151
|
+
if (!this.start.pass.finalize && result.needsFreshWorker) return { resetRequired: true };
|
|
152
|
+
}
|
|
153
|
+
// A committed reset token may finish bounded publication above. Neither
|
|
154
|
+
// source/index work nor new atomic reset may use finalization allowance.
|
|
155
|
+
if (this.start.pass.finalize) return { finalize: true };
|
|
156
|
+
|
|
157
|
+
if (this.issues && !this.issues.complete())
|
|
158
|
+
return { stage: "issue-preparation", ...this.issues.advance(this.start.root, { stopped }) };
|
|
159
|
+
|
|
160
|
+
const sourceDiscovery = this.sourceDiscovery && !this.queue.current().discovered
|
|
161
|
+
? this.sourceDiscovery.advance({ stopped, limit: this.unit.discoveryLimit ?? 128,
|
|
162
|
+
runId: this.run.id, workerNonce: this.start.workerNonce, now: this.now() }) : null;
|
|
163
|
+
let discovery = null;
|
|
164
|
+
if (this.inventory && this.inventory.current().phase !== "complete") discovery = this.inventory.discover(this.index, { stopped });
|
|
165
|
+
if (stopped()) return { stopped: true, discovery, sourceDiscovery };
|
|
166
|
+
const blockedIndex = !this.indexEnabled || this.reset.current() !== null;
|
|
167
|
+
// Reuse the last durable admission's stage. Failed groups rotate too, and
|
|
168
|
+
// routing adds no separate per-record or per-unit scheduling transaction.
|
|
169
|
+
const last = this.state.get("capture-schedule", "position")?.stage;
|
|
170
|
+
for (const stage of last === "capture" ? ["index", "capture"] : ["capture", "index"]) {
|
|
171
|
+
if (stage === "index" && blockedIndex) continue;
|
|
172
|
+
const result = this.process(stage, false);
|
|
173
|
+
if (result.selected || result.resetRequired || result.stopped) return { ...result, discovery, sourceDiscovery };
|
|
174
|
+
}
|
|
175
|
+
if (stopped()) return { stopped: true, discovery, sourceDiscovery };
|
|
176
|
+
const cleanup = this.clean(false);
|
|
177
|
+
if (cleanup.selected || !cleanup.complete) return { stage: "cleanup", ...cleanup, discovery, sourceDiscovery };
|
|
178
|
+
if (stopped()) return { stopped: true, discovery, sourceDiscovery };
|
|
179
|
+
const collected = this.collect();
|
|
180
|
+
if (collected) return { ...collected, discovery, sourceDiscovery };
|
|
181
|
+
// Unknown eligible discovery can still contain healthy work. A refused
|
|
182
|
+
// root remains pending but cannot withhold unrelated due work indefinitely.
|
|
183
|
+
if ((this.sourceDiscovery ? this.sourceDiscovery.healthyPending() : !this.queue.current().discovered) ||
|
|
184
|
+
this.inventory && this.inventory.current().phase !== "complete")
|
|
185
|
+
return { waitingForDiscovery: true, discovery, sourceDiscovery };
|
|
186
|
+
const selection = selectCaptureWork(this.store, this.state, [], { runId: this.run.id, now: this.now(), stopped,
|
|
187
|
+
retryFilter: retry => retry.stage === "index" ? !blockedIndex :
|
|
188
|
+
retry.stage === "capture" && this.sourceReady(this.queue.forStream(retry.stream)) });
|
|
189
|
+
if (selection) return { ...this.process(selection.stage, true), discovery, sourceDiscovery };
|
|
190
|
+
const cleanupRetry = this.clean(true);
|
|
191
|
+
if (cleanupRetry.selected || !cleanupRetry.complete) return { stage: "cleanup", ...cleanupRetry, discovery, sourceDiscovery };
|
|
192
|
+
if (this.sourceDiscovery && !this.queue.current().discovered) {
|
|
193
|
+
if (sourceDiscovery?.source) return { waitingForDiscovery: true, discovery, sourceDiscovery };
|
|
194
|
+
const retried = this.sourceDiscovery.advance({ stopped, limit: this.unit.discoveryLimit ?? 128,
|
|
195
|
+
runId: this.run.id, workerNonce: this.start.workerNonce, now: this.now(), retry: true });
|
|
196
|
+
if (retried.source) return { waitingForDiscovery: true, discovery, sourceDiscovery: retried };
|
|
197
|
+
}
|
|
198
|
+
return { idle: true, discovery, sourceDiscovery };
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
// Yield/stop never infer completion. The owning worker calls complete() only
|
|
202
|
+
// when it wants a final all-stage decision; idle is merely a scheduling hint.
|
|
203
|
+
// An observed interruption ends the run as interrupted, finalization included.
|
|
204
|
+
yieldWorker({ resetRequired = false } = {}) {
|
|
205
|
+
if (this.start.pass.finalize || this.start.budget.reason === "interrupted")
|
|
206
|
+
return this.finish({ reason: this.start.budget.reason ?? "deadline" });
|
|
207
|
+
this.run.publish({ pending: true, peakRssBytes: this.start.budget.peakRssBytes });
|
|
208
|
+
return { outcome: "partial", reason: this.start.budget.reason ?? "deadline", continuation: true,
|
|
209
|
+
...(resetRequired ? { resetRequired: true } : {}) };
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
finish({ reason = "input" } = {}) {
|
|
213
|
+
const snapshot = this.run.finish({ outcome: reason === "interrupted" ? "failed" : "partial", reason,
|
|
214
|
+
pending: true, peakRssBytes: this.start.budget.peakRssBytes });
|
|
215
|
+
return { outcome: snapshot.lastRun.outcome, reason: snapshot.lastRun.reason, continuation: false };
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
complete(options) {
|
|
219
|
+
// The sampled reason (interrupted, rss, deadline) is kept; finalization
|
|
220
|
+
// without one reports its allowance.
|
|
221
|
+
const stopped = this.start.budget.sample();
|
|
222
|
+
if (this.start.pass.finalize || stopped !== null) return this.finish({ reason: stopped ?? "deadline" });
|
|
223
|
+
const audit = this.indexEnabled ? auditCaptureCompletion(this, options) : auditCaptureSourceRetirement(this, options);
|
|
224
|
+
if (audit.stopped || audit.needsWork) return this.yieldWorker();
|
|
225
|
+
// Normally these facts entered the run during replacement construction.
|
|
226
|
+
// Also retain a caught failure if an owning caller continues in this worker.
|
|
227
|
+
const failure = ["capture-run-failure", "capture-index-reset-failure", "capture-cleanup-failure", "capture-discovery-failure"]
|
|
228
|
+
.map(kind => this.state.get(kind, "last")).find(row => row?.runId === this.run.id);
|
|
229
|
+
const snapshot = this.run.finish({ outcome: failure ? "failed" : audit.complete ? "complete" : "partial",
|
|
230
|
+
reason: failure?.reason ?? (audit.complete ? null : "input"),
|
|
231
|
+
pending: audit.pending, peakRssBytes: this.start.budget.peakRssBytes });
|
|
232
|
+
return { outcome: snapshot.lastRun.outcome, reason: snapshot.lastRun.reason, continuation: false };
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
close() {
|
|
236
|
+
this.sourceDiscovery?.close();
|
|
237
|
+
this.inventory?.close();
|
|
238
|
+
try { this.index?.close(); } finally { this.state?.close(); }
|
|
239
|
+
this.index = null; this.state = null;
|
|
240
|
+
}
|
|
241
|
+
}
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
// Private worker bootstrap. Reconciliation is supplied by the owning entry;
|
|
2
|
+
// this handshake never captures or writes status on behalf of the operation.
|
|
3
|
+
import { randomBytes } from "node:crypto";
|
|
4
|
+
import { CaptureBudget, CaptureStop } from "./capture-budget.mjs";
|
|
5
|
+
import { readCaptureLockOwner } from "./capture-lock.mjs";
|
|
6
|
+
import { captureMonotonicMs, captureWorkerMessage, validWorkerResult } from "./capture-supervisor.mjs";
|
|
7
|
+
|
|
8
|
+
export async function runCaptureWorker(operation) {
|
|
9
|
+
if (!process.send || !process.connected) throw new Error("capture worker requires its supervisor");
|
|
10
|
+
// TERM/INT sent to the caller's whole process group (a terminal's Ctrl-C,
|
|
11
|
+
// timeout(1), a service manager's control group) reaches this worker too.
|
|
12
|
+
// Caught, it cannot end the worker mid-unit; the supervisor, which received
|
|
13
|
+
// the same signal, owns the stop request and the grace. A signal this worker
|
|
14
|
+
// observes itself also ends it as interrupted at its next check.
|
|
15
|
+
let signalled = false;
|
|
16
|
+
const onSignal = () => { signalled = true; };
|
|
17
|
+
process.on("SIGTERM", onSignal); process.on("SIGINT", onSignal);
|
|
18
|
+
try { return await superviseOperation(operation, () => signalled); }
|
|
19
|
+
finally { process.off("SIGTERM", onSignal); process.off("SIGINT", onSignal); }
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
async function superviseOperation(operation, signalled) {
|
|
23
|
+
const nonce = randomBytes(16).toString("hex"), parentPid = process.ppid;
|
|
24
|
+
const start = await new Promise((resolve, reject) => {
|
|
25
|
+
const disconnected = () => { cleanup(); reject(new Error("capture supervisor disconnected before start")); };
|
|
26
|
+
const receive = message => {
|
|
27
|
+
cleanup();
|
|
28
|
+
if (message?.type !== "start" || message.workerNonce !== nonce || message.parent?.pid !== parentPid ||
|
|
29
|
+
typeof message.root !== "string" || !Number.isFinite(message.drainAt) ||
|
|
30
|
+
!Number.isFinite(message.finalizeAt) || !Number.isFinite(message.hardAt) ||
|
|
31
|
+
message.drainAt > message.finalizeAt || message.finalizeAt > message.hardAt) {
|
|
32
|
+
reject(new Error("invalid capture worker start")); return;
|
|
33
|
+
}
|
|
34
|
+
resolve(message);
|
|
35
|
+
};
|
|
36
|
+
const cleanup = () => { process.off("message", receive); process.off("disconnect", disconnected); };
|
|
37
|
+
process.once("message", receive); process.once("disconnect", disconnected);
|
|
38
|
+
process.send({ type: "ready", pid: process.pid, nonce, startedAt: new Date().toISOString() }, error => {
|
|
39
|
+
if (error) { cleanup(); reject(new Error("capture worker could not announce readiness")); }
|
|
40
|
+
});
|
|
41
|
+
});
|
|
42
|
+
// The same owner read between bounded units also carries the supervisor's
|
|
43
|
+
// cooperative stop request; either way this worker stops as interrupted.
|
|
44
|
+
const owned = () => {
|
|
45
|
+
if (!process.connected || process.ppid !== parentPid) return null;
|
|
46
|
+
const owner = readCaptureLockOwner(start.root);
|
|
47
|
+
return owner?.pid === parentPid && owner.nonce === start.parent.nonce &&
|
|
48
|
+
owner.worker?.pid === process.pid && owner.worker.nonce === nonce ? owner : null;
|
|
49
|
+
};
|
|
50
|
+
const alive = () => { if (signalled()) return false; const owner = owned(); return owner !== null && owner.stop !== true; };
|
|
51
|
+
// Publication must already be durable before the first operation, including
|
|
52
|
+
// its first status write. A callback must check again between bounded units.
|
|
53
|
+
if (!owned()) throw new Error("capture worker lock publication is missing");
|
|
54
|
+
const budget = new CaptureBudget({ policy: start.policy, clock: captureMonotonicMs,
|
|
55
|
+
drainAt: start.drainAt, supervisorAlive: alive });
|
|
56
|
+
let result;
|
|
57
|
+
try {
|
|
58
|
+
result = await operation({ ...start, budget });
|
|
59
|
+
if (!validWorkerResult(result)) throw new Error("invalid capture worker result");
|
|
60
|
+
} catch (error) {
|
|
61
|
+
// The operation owns failure receipts. Bootstrap never invents completed
|
|
62
|
+
// progress or rewrites a receipt after an exception.
|
|
63
|
+
const reason = error instanceof CaptureStop ? error.reason : budget.reason === "interrupted" ? "interrupted" : "io";
|
|
64
|
+
// An operational failure (status persistence among them) names its cause
|
|
65
|
+
// to the supervising command only; the run receipt keeps its fixed message.
|
|
66
|
+
result = { outcome: "failed", reason, ...(reason === "io" ? { message: captureWorkerMessage(error) } : {}) };
|
|
67
|
+
}
|
|
68
|
+
if (process.connected) {
|
|
69
|
+
await new Promise(resolve => process.send({ type: "result", result }, () => resolve()));
|
|
70
|
+
process.disconnect();
|
|
71
|
+
}
|
|
72
|
+
process.exitCode = result.outcome === "failed" ? 1 : 0;
|
|
73
|
+
return result;
|
|
74
|
+
}
|
|
@@ -23,38 +23,79 @@ export function fileVersion(path) {
|
|
|
23
23
|
} catch (err) { if (err.code === "ENOENT") return null; throw err; }
|
|
24
24
|
}
|
|
25
25
|
export const sameVersion = (a, b) => JSON.stringify(a) === JSON.stringify(b);
|
|
26
|
+
function fileIdentity(path) {
|
|
27
|
+
try { const st = statSync(path, { bigint: true }); return { dev: String(st.dev), ino: String(st.ino) }; }
|
|
28
|
+
catch (err) { if (err.code === "ENOENT") return null; throw err; }
|
|
29
|
+
}
|
|
26
30
|
|
|
27
31
|
export class DerivedState {
|
|
28
32
|
constructor(root) {
|
|
29
33
|
const { DatabaseSync } = require("node:sqlite");
|
|
30
34
|
mkdirSync(join(root, "index"), { recursive: true });
|
|
31
|
-
this.
|
|
32
|
-
|
|
33
|
-
this.db.
|
|
35
|
+
this.path = join(root, "index", "capture.db");
|
|
36
|
+
const before = fileIdentity(this.path);
|
|
37
|
+
this.db = new DatabaseSync(this.path);
|
|
34
38
|
try {
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
39
|
+
this.db.exec("PRAGMA busy_timeout = 60000; PRAGMA cache_size = -2048; PRAGMA temp_store = FILE");
|
|
40
|
+
// A healthy accelerator needs no schema write transaction on every open.
|
|
41
|
+
// Repair rechecks under the write lock, including a lost ID lookup.
|
|
42
|
+
const present = name => this.db.prepare("SELECT 1 AS found FROM sqlite_master WHERE type='table' AND name=?").get(name)?.found === 1;
|
|
43
|
+
if (!present("state") || !present("journal_ids")) {
|
|
44
|
+
this.db.exec("BEGIN IMMEDIATE");
|
|
45
|
+
try {
|
|
46
|
+
const hadIds = this.db.prepare("SELECT 1 AS found FROM sqlite_master WHERE type='table' AND name='journal_ids'").get()?.found === 1;
|
|
47
|
+
this.db.exec(`CREATE TABLE IF NOT EXISTS state (kind TEXT, key TEXT, value TEXT NOT NULL, PRIMARY KEY(kind, key));
|
|
48
|
+
CREATE TABLE IF NOT EXISTS journal_ids (stream TEXT, id TEXT, PRIMARY KEY(stream, id));`);
|
|
49
|
+
// Recreating a lost lookup atomically revokes the old validation witness.
|
|
50
|
+
// A crash cannot qualify an empty replacement table with that witness.
|
|
51
|
+
if (!hadIds) this.db.exec("DELETE FROM state WHERE kind = 'journal'");
|
|
52
|
+
this.db.exec("COMMIT");
|
|
53
|
+
} catch (err) { this.db.exec("ROLLBACK"); throw err; }
|
|
54
|
+
}
|
|
55
|
+
// The file this connection writes, for callers that hold it across
|
|
56
|
+
// streams. SQLite does not expose the opened descriptor, so the path's
|
|
57
|
+
// identity binds the connection only when it is the same before the
|
|
58
|
+
// open and once the schema is on disk; otherwise it stays unbound and
|
|
59
|
+
// assertFile refuses.
|
|
60
|
+
const after = fileIdentity(this.path);
|
|
61
|
+
this.file = after && (!before || before.dev === after.dev && before.ino === after.ino) ? after : null;
|
|
62
|
+
this.depth = 0;
|
|
63
|
+
this.statements = new Map();
|
|
64
|
+
this.read = this.db.prepare("SELECT value FROM state WHERE kind = ? AND key = ?");
|
|
65
|
+
this.write = this.db.prepare("INSERT INTO state VALUES (?, ?, ?) ON CONFLICT(kind,key) DO UPDATE SET value=excluded.value WHERE state.value != excluded.value");
|
|
66
|
+
} catch (err) { this.db.close(); throw err; }
|
|
45
67
|
}
|
|
46
68
|
get(kind, key) { const row = this.read.get(kind, key); return row ? JSON.parse(row.value) : null; }
|
|
47
69
|
set(kind, key, value) { this.write.run(kind, key, JSON.stringify(value)); }
|
|
48
|
-
delete(kind, key) { this.
|
|
70
|
+
delete(kind, key) { this.statement("DELETE FROM state WHERE kind = ? AND key = ?").run(kind, key); }
|
|
71
|
+
// One prepared statement per SQL text for this connection's lifetime. Only
|
|
72
|
+
// for statements run to completion (run/get/all), never left iterating.
|
|
73
|
+
statement(sql) {
|
|
74
|
+
let prepared = this.statements.get(sql);
|
|
75
|
+
if (!prepared) { prepared = this.db.prepare(sql); this.statements.set(sql, prepared); }
|
|
76
|
+
return prepared;
|
|
77
|
+
}
|
|
78
|
+
// A connection held across many streams must not publish checkpoint proof
|
|
79
|
+
// into a capture.db that was replaced or removed after it opened.
|
|
80
|
+
assertFile() {
|
|
81
|
+
const now = fileIdentity(this.path);
|
|
82
|
+
if (!now || !this.file || now.dev !== this.file.dev || now.ino !== this.file.ino)
|
|
83
|
+
throw new Error(`capture.db changed during the pass; retry: ${this.path}`);
|
|
84
|
+
}
|
|
49
85
|
*entries(kind) {
|
|
50
86
|
for (const row of keysetRows(this.db, "SELECT key, value FROM state WHERE kind = ?", "key", [kind])) {
|
|
51
87
|
yield { key: row.key, value: JSON.parse(row.value) };
|
|
52
88
|
}
|
|
53
89
|
}
|
|
54
90
|
transaction(fn) {
|
|
55
|
-
this.
|
|
56
|
-
|
|
57
|
-
|
|
91
|
+
const nested = this.depth > 0, savepoint = `capture_${this.depth}`;
|
|
92
|
+
this.db.exec(nested ? `SAVEPOINT ${savepoint}` : "BEGIN IMMEDIATE");
|
|
93
|
+
this.depth++;
|
|
94
|
+
try { const result = fn(); this.db.exec(nested ? `RELEASE ${savepoint}` : "COMMIT"); return result; }
|
|
95
|
+
catch (err) {
|
|
96
|
+
this.db.exec(nested ? `ROLLBACK TO ${savepoint}; RELEASE ${savepoint}` : "ROLLBACK");
|
|
97
|
+
throw err;
|
|
98
|
+
} finally { this.depth--; }
|
|
58
99
|
}
|
|
59
100
|
close() { this.db.close(); }
|
|
60
101
|
}
|
|
@@ -8,8 +8,9 @@ import { readSync } from "node:fs";
|
|
|
8
8
|
// The bytes of `fd` from `start` up to `end` (default: the end of the file), in
|
|
9
9
|
// chunks of at most `size` bytes. Each chunk is a fresh buffer, so a caller may
|
|
10
10
|
// keep one past the next read.
|
|
11
|
-
export function* fileChunks(fd, { start = 0, end = Infinity, size = 1 << 20 } = {}) {
|
|
11
|
+
export function* fileChunks(fd, { start = 0, end = Infinity, size = 1 << 20, check = () => {} } = {}) {
|
|
12
12
|
for (let position = start; position < end; ) {
|
|
13
|
+
check();
|
|
13
14
|
const chunk = Buffer.allocUnsafe(Math.min(size, end - position));
|
|
14
15
|
const n = readSync(fd, chunk, 0, chunk.length, position);
|
|
15
16
|
if (n === 0) return;
|
|
@@ -22,17 +23,32 @@ export function* fileChunks(fd, { start = 0, end = Infinity, size = 1 << 20 } =
|
|
|
22
23
|
// then a final fragment without one, if the bytes end mid-line. A caller tells
|
|
23
24
|
// them apart by the last byte. Fragments are joined only at a newline, so a
|
|
24
25
|
// long line is copied once, not as each chunk arrives.
|
|
25
|
-
export
|
|
26
|
-
|
|
26
|
+
export class RecordSizeError extends Error {
|
|
27
|
+
constructor(limit, offset) {
|
|
28
|
+
super(`raw record exceeds ${limit} bytes at offset ${offset}`);
|
|
29
|
+
this.limit = limit; this.offset = offset;
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
// A line that starts before `limitFrom` (an offset into these chunks) was
|
|
34
|
+
// already captured and is not size-limited; the limit applies from there on.
|
|
35
|
+
export function* bufferLines(chunks, { maxBytes = Infinity, limitFrom = 0 } = {}) {
|
|
36
|
+
let pieces = [], size = 0, offset = 0;
|
|
37
|
+
const append = piece => {
|
|
38
|
+
// Enforce a bounded command's policy before joining or retaining the whole
|
|
39
|
+
// offending record. Public callers retain their unlimited default.
|
|
40
|
+
if (offset >= limitFrom && size + piece.length > maxBytes) throw new RecordSizeError(maxBytes, offset);
|
|
41
|
+
pieces.push(piece); size += piece.length;
|
|
42
|
+
};
|
|
27
43
|
for (const chunk of chunks) {
|
|
28
44
|
let from = 0;
|
|
29
45
|
for (let nl = chunk.indexOf(10, from); nl >= 0; nl = chunk.indexOf(10, from)) {
|
|
30
|
-
|
|
31
|
-
size += nl + 1 - from;
|
|
46
|
+
append(chunk.subarray(from, nl + 1));
|
|
32
47
|
yield pieces.length === 1 ? pieces[0] : Buffer.concat(pieces, size);
|
|
48
|
+
offset += size;
|
|
33
49
|
pieces = []; size = 0; from = nl + 1;
|
|
34
50
|
}
|
|
35
|
-
if (from < chunk.length)
|
|
51
|
+
if (from < chunk.length) append(chunk.subarray(from));
|
|
36
52
|
}
|
|
37
53
|
if (size > 0) yield pieces.length === 1 ? pieces[0] : Buffer.concat(pieces, size);
|
|
38
54
|
}
|
|
@@ -58,12 +58,13 @@ function* parsedLines(bytes) {
|
|
|
58
58
|
// Preserve the public inventory's per-directory localeCompare order and stable
|
|
59
59
|
// ties for ordinary capture too. Path metadata grows with directory size;
|
|
60
60
|
// source contents are streamed independently of this discovery inventory.
|
|
61
|
-
function
|
|
61
|
+
export function sortedSessionEntries(path, optional = false) {
|
|
62
62
|
let entries;
|
|
63
63
|
try { entries = readdirSync(path, { withFileTypes: true }); }
|
|
64
|
-
catch (err) { if (optional && err.code === "ENOENT") return; throw err; }
|
|
65
|
-
|
|
64
|
+
catch (err) { if (optional && err.code === "ENOENT") return []; throw err; }
|
|
65
|
+
return entries.sort((a, b) => a.name.localeCompare(b.name));
|
|
66
66
|
}
|
|
67
|
+
function* directoryEntries(path, optional = false) { yield* sortedSessionEntries(path, optional); }
|
|
67
68
|
function* iterateJsonlFiles(root, maxDepth, { strict = false } = {}, allowCaptured = false) {
|
|
68
69
|
const capturedPi = typeof root === "object" && root !== null ? root.capturedPi : undefined;
|
|
69
70
|
if (typeof root !== "string") {
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
// Internal bounded indexing units. The scheduler must visit the union of live
|
|
2
|
+
// streams and index_state streams (including deleted journals), and restart
|
|
3
|
+
// its index sweep when reset is returned. This helper alone cannot declare a
|
|
4
|
+
// root complete. Public synchronous update/rebuild keep their existing API.
|
|
5
|
+
import { closeSync, openSync, readSync } from "node:fs";
|
|
6
|
+
import { captureContext } from "./capture-context.mjs";
|
|
7
|
+
|
|
8
|
+
function cursorValid(store, stream, row, checkpoint) {
|
|
9
|
+
if (!row) return true;
|
|
10
|
+
if (!Number.isSafeInteger(row.bytes) || row.bytes < 0 ||
|
|
11
|
+
!Number.isSafeInteger(row.indexed_count) || row.indexed_count < 0 ||
|
|
12
|
+
row.indexed_count !== row.byte_count || row.epoch !== checkpoint.epoch ||
|
|
13
|
+
row.bytes > checkpoint.bytes || row.indexed_count > checkpoint.count) return false;
|
|
14
|
+
if (row.bytes > 0 && row.bytes < checkpoint.bytes) {
|
|
15
|
+
const fd = openSync(store.journalPath(stream), "r"), byte = Buffer.alloc(1);
|
|
16
|
+
try { return readSync(fd, byte, 0, 1, row.bytes - 1) === 1 && byte[0] === 10; }
|
|
17
|
+
finally { closeSync(fd); }
|
|
18
|
+
}
|
|
19
|
+
return true;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export function indexCaptureGroup(index, streams, {
|
|
23
|
+
maxRecords = 2000, maxBytes = 8 * 1024 * 1024, stopped = () => false, allowReset = false,
|
|
24
|
+
} = {}) {
|
|
25
|
+
if (typeof allowReset !== "boolean") throw new Error("invalid index reset admission");
|
|
26
|
+
const store = index.store, context = captureContext(store);
|
|
27
|
+
if (!context?.state.depth) throw new Error("bounded indexing requires an active capture work group");
|
|
28
|
+
const names = [...new Set(streams)].sort();
|
|
29
|
+
if (!names.length || names.length > 128 || names.some(name => !context.held.has(name)))
|
|
30
|
+
throw new Error("bounded indexing streams must belong to the active capture work group");
|
|
31
|
+
if (!Number.isSafeInteger(maxRecords) || maxRecords < names.length || !Number.isSafeInteger(maxBytes) || maxBytes < names.length)
|
|
32
|
+
throw new Error("bounded indexing must allow at least one work unit per stream");
|
|
33
|
+
const recordsEach = Math.floor(maxRecords / names.length), bytesEach = Math.floor(maxBytes / names.length);
|
|
34
|
+
const get = index.db.prepare(`SELECT s.indexed_count,b.bytes,b.indexed_count AS byte_count,b.epoch
|
|
35
|
+
FROM index_state s LEFT JOIN index_bytes b ON s.stream=b.stream WHERE s.stream=?`);
|
|
36
|
+
const prepared = [], work = { records: 0, bytes: 0, deletedIds: 0 };
|
|
37
|
+
let reset = false;
|
|
38
|
+
// Every group's locks were taken before capture.db's transaction. Validate
|
|
39
|
+
// journals before opening turns.db's transaction; never acquire another
|
|
40
|
+
// stream from inside either database transaction.
|
|
41
|
+
for (const stream of names) {
|
|
42
|
+
if (stopped()) { prepared.push({ stream, phase: "validation" }); continue; }
|
|
43
|
+
const checkpoint = store.validateJournal(stream, context.state, {
|
|
44
|
+
maxRecords: recordsEach, maxBytes: bytesEach, stopped, work,
|
|
45
|
+
});
|
|
46
|
+
const row = get.get(stream);
|
|
47
|
+
if (!cursorValid(store, stream, row?.indexed_count != null ? row : null, checkpoint)) reset = true;
|
|
48
|
+
prepared.push({ stream, checkpoint });
|
|
49
|
+
}
|
|
50
|
+
// Cross-stream tombstones and first-stream attribution make a per-stream
|
|
51
|
+
// row deletion unsound. Keep the reviewed atomic union reset. Publication
|
|
52
|
+
// of its cleared cursor tables is part of that same SQLite transaction.
|
|
53
|
+
if (reset && !allowReset) {
|
|
54
|
+
// The scheduler must end this work unit and reserve a fresh full worker
|
|
55
|
+
// allowance for atomic reset. Detection is not permission to start it in
|
|
56
|
+
// a nearly exhausted capture/index unit or the finalization reserve.
|
|
57
|
+
return { reset: false, resetRequired: true, turnsIndexed: 0, bytesIndexed: 0, validation: work,
|
|
58
|
+
streams: names.map(stream => ({ stream, phase: "reset" })) };
|
|
59
|
+
}
|
|
60
|
+
if (reset) index.reset();
|
|
61
|
+
const setCount = index.db.prepare("INSERT INTO index_state VALUES (?, ?) ON CONFLICT(stream) DO UPDATE SET indexed_count=excluded.indexed_count");
|
|
62
|
+
const setBytes = index.db.prepare("INSERT INTO index_bytes VALUES (?, ?, ?, ?) ON CONFLICT(stream) DO UPDATE SET bytes=excluded.bytes,indexed_count=excluded.indexed_count,epoch=excluded.epoch");
|
|
63
|
+
let transaction = false, turnsIndexed = 0, bytesIndexed = 0;
|
|
64
|
+
const begin = () => { if (!transaction) { index.db.exec("BEGIN IMMEDIATE"); transaction = true; } };
|
|
65
|
+
try {
|
|
66
|
+
const recordedOwner = index.db.prepare("SELECT value FROM meta WHERE key='owner'").get()?.value ?? null;
|
|
67
|
+
const owner = store.owner ?? recordedOwner;
|
|
68
|
+
if (owner && recordedOwner !== owner) {
|
|
69
|
+
begin(); index.db.prepare("INSERT INTO meta VALUES ('owner',?) ON CONFLICT(key) DO UPDATE SET value=excluded.value WHERE meta.value!=excluded.value").run(owner);
|
|
70
|
+
}
|
|
71
|
+
for (const item of prepared) {
|
|
72
|
+
const { stream, checkpoint } = item;
|
|
73
|
+
if (!checkpoint?.complete) { item.phase = "validation"; continue; }
|
|
74
|
+
const row = reset ? undefined : get.get(stream);
|
|
75
|
+
let position = row?.bytes ?? 0, count = row?.indexed_count ?? 0;
|
|
76
|
+
if (position === checkpoint.bytes) { item.phase = "complete"; continue; }
|
|
77
|
+
if (stopped()) { item.phase = "index"; continue; }
|
|
78
|
+
// Fix the validated end for this unit. Later growth is subsequent work.
|
|
79
|
+
const iterator = store.iterateJournal(stream, { start: position, end: checkpoint.bytes });
|
|
80
|
+
let records = 0, bytes = 0;
|
|
81
|
+
try {
|
|
82
|
+
while (records < recordsEach && bytes < bytesEach && !stopped()) {
|
|
83
|
+
const next = iterator.next(); if (next.done) break;
|
|
84
|
+
begin();
|
|
85
|
+
const entry = next.value;
|
|
86
|
+
if (entry.turn !== null) { index.addTurn(entry.turn, stream, owner); count++; turnsIndexed++; }
|
|
87
|
+
position = entry.end; records++; bytes += entry.bytes; bytesIndexed += entry.bytes;
|
|
88
|
+
}
|
|
89
|
+
} finally { iterator.return(); }
|
|
90
|
+
if (records) { setCount.run(stream, count); setBytes.run(stream, position, count, checkpoint.epoch); }
|
|
91
|
+
item.phase = position === checkpoint.bytes ? "complete" : "index";
|
|
92
|
+
}
|
|
93
|
+
if (transaction) index.db.exec("COMMIT");
|
|
94
|
+
} catch (err) { if (transaction) index.db.exec("ROLLBACK"); throw err; }
|
|
95
|
+
return { reset, turnsIndexed, bytesIndexed, validation: work,
|
|
96
|
+
streams: prepared.map(({ stream, phase }) => ({ stream, phase })) };
|
|
97
|
+
}
|