@stravica-ai/digital-operator-pa 0.1.64 → 0.1.65
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/engineering-pa-library-cli/README.md +179 -0
- package/dist/engineering-pa-library-cli/src/cli.mjs +185 -3
- package/dist/engineering-pa-library-cli/src/envelope.mjs +6 -1
- package/dist/engineering-pa-library-cli/src/migrate/apply.mjs +479 -0
- package/dist/engineering-pa-library-cli/src/migrate/kit.mjs +108 -0
- package/dist/engineering-pa-library-cli/src/migrate/plan.mjs +593 -0
- package/dist/engineering-pa-library-cli/src/migrate/verb.mjs +222 -0
- package/package.json +1 -1
|
@@ -0,0 +1,479 @@
|
|
|
1
|
+
// apply.mjs: `library migrate-mounts --apply`, the verified batch engine for
|
|
2
|
+
// the legacy-path migration (w-2026-10-16-dave-019, slice 3 Part B).
|
|
3
|
+
//
|
|
4
|
+
// One invocation moves one batch (or, with `--batches all`, batch after
|
|
5
|
+
// batch) of ONE tier's legacy rows to the mounted path, through the ingest as
|
|
6
|
+
// ordinary events, and proves every doc before the next step:
|
|
7
|
+
//
|
|
8
|
+
// plan re-plan the tier live (Part A's planner, with fetch); eligible
|
|
9
|
+
// rows in plan order are re-put, delete, take-legacy,
|
|
10
|
+
// take-mounted, and merge rows with a merged body on disk
|
|
11
|
+
// phase 1 compare-and-swap: fetch, re-read both blobs of every batch row;
|
|
12
|
+
// any change since the plan is `state-changed`
|
|
13
|
+
// phase 2 upserts (re-put, take-legacy: the legacy body; merge: the
|
|
14
|
+
// merged body) with deterministic event ids; an ack other than
|
|
15
|
+
// accepted or deduped is `rejected:<reason>`
|
|
16
|
+
// phase 3 wait until every target blob equals the posted body's blob
|
|
17
|
+
// (`not-applied` on timeout, `content-mismatch` on another blob)
|
|
18
|
+
// phase 4 deletes of the legacy path, only for rows whose mounted copy
|
|
19
|
+
// verified (phase 3, or phase 1 for delete and take-mounted)
|
|
20
|
+
// phase 5 wait until every deleted legacy path is absent
|
|
21
|
+
// phase 6 reader check: fast-forward the clone, then the real reader
|
|
22
|
+
// (`scanSourcesFor` over an empty home, `locateScannedFile`) must
|
|
23
|
+
// resolve the workspace id to `<clone>/<target>` with the body's
|
|
24
|
+
// sha256
|
|
25
|
+
//
|
|
26
|
+
// The first divergence stops the invocation: no further post of any kind,
|
|
27
|
+
// and a legacy copy is never deleted unless its mounted copy verified. Docs
|
|
28
|
+
// upserted but not yet deleted are safe (the reader's bridge serves the
|
|
29
|
+
// newer mounted copy) and the next run re-plans them. Cross-tier rows are
|
|
30
|
+
// never applied (Dave ruling 2026-10-19); they are listed for the relay.
|
|
31
|
+
//
|
|
32
|
+
// Seams (post, git, clock, sleep, hooks) are injected so tests drive a local
|
|
33
|
+
// ingest double and temp repos.
|
|
34
|
+
|
|
35
|
+
import { createHash } from 'node:crypto';
|
|
36
|
+
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmdirSync, writeFileSync } from 'node:fs';
|
|
37
|
+
import { tmpdir } from 'node:os';
|
|
38
|
+
import { join, resolve, sep } from 'node:path';
|
|
39
|
+
|
|
40
|
+
import { locateScannedFile, scanSourcesFor } from '../../../engineering-pa-core/src/knowledge/retrieval/links.mjs';
|
|
41
|
+
import { gitRemoteOf } from '../../../engineering-pa-core/src/knowledge/retrieval/mounts.mjs';
|
|
42
|
+
import { createMountMap } from '../../../engineering-pa-core/src/knowledge/retrieval/mount-map.mjs';
|
|
43
|
+
|
|
44
|
+
import { parseIngestAck } from '../ack.mjs';
|
|
45
|
+
import { buildEnvelope } from '../envelope.mjs';
|
|
46
|
+
import {
|
|
47
|
+
CROSS_TIER_COLUMNS,
|
|
48
|
+
LEGACY_PREFIX,
|
|
49
|
+
PlanError,
|
|
50
|
+
crossTierRows,
|
|
51
|
+
defaultGitRunner,
|
|
52
|
+
gitBlobSha,
|
|
53
|
+
lsTreeBlobs,
|
|
54
|
+
planTier,
|
|
55
|
+
resolvePlanRef,
|
|
56
|
+
tierLabel,
|
|
57
|
+
toCsv,
|
|
58
|
+
} from './plan.mjs';
|
|
59
|
+
|
|
60
|
+
export const APPLY_ACTIONS = new Set(['re-put', 'delete', 'take-legacy', 'take-mounted', 'merge']);
|
|
61
|
+
export const UPSERT_ACTIONS = new Set(['re-put', 'take-legacy', 'merge']);
|
|
62
|
+
export const DEFAULT_BATCH_SIZE = 25;
|
|
63
|
+
export const MAX_BATCH_SIZE = 100;
|
|
64
|
+
export const DEFAULT_WAIT_MS = 300000;
|
|
65
|
+
export const DEFAULT_POLL_MS = 10000;
|
|
66
|
+
|
|
67
|
+
// RFC 4122 URL namespace; the name carries `migrate-mounts` and the row.
|
|
68
|
+
const NAMESPACE_URL = '6ba7b811-9dad-11d1-80b4-00c04fd430c8';
|
|
69
|
+
|
|
70
|
+
export const ROLLBACK_NOTE = 'No rollback mode. Every pre-state copy is `git show <tierShaBefore>:<path>` in the tier. '
|
|
71
|
+
+ 'To restore a mounted copy, put that body at its workspace path with `library put`. Legacy copies are not '
|
|
72
|
+
+ 'restored: the mounted copy carries the content and is what the reader reads.';
|
|
73
|
+
|
|
74
|
+
/** A name-based (version 5, SHA-1) uuid, RFC 4122 layout, stdlib only. */
|
|
75
|
+
export function uuidV5(name, namespace = NAMESPACE_URL) {
|
|
76
|
+
const ns = Buffer.from(namespace.replace(/-/g, ''), 'hex');
|
|
77
|
+
const bytes = Buffer.from(createHash('sha1').update(ns).update(Buffer.from(String(name), 'utf8')).digest().subarray(0, 16));
|
|
78
|
+
bytes[6] = (bytes[6] & 0x0f) | 0x50;
|
|
79
|
+
bytes[8] = (bytes[8] & 0x3f) | 0x80;
|
|
80
|
+
const h = bytes.toString('hex');
|
|
81
|
+
return `${h.slice(0, 8)}-${h.slice(8, 12)}-${h.slice(12, 16)}-${h.slice(16, 20)}-${h.slice(20)}`;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* The deterministic event id for one migration event, over the tier, the
|
|
86
|
+
* event action (`upsert` or `delete`), the path, the body blob and `since`:
|
|
87
|
+
* the last commit that touched that path at the planned sha (empty when it
|
|
88
|
+
* never existed). A rerun after a crash, with the path untouched since,
|
|
89
|
+
* gives the same id and dedups at the ingest; once the path has changed
|
|
90
|
+
* (the worker wrote another body, a stale writer brought a legacy copy
|
|
91
|
+
* back), the corrective post gets a new id instead of being deduped
|
|
92
|
+
* forever.
|
|
93
|
+
*/
|
|
94
|
+
export function migrateEventId({ repoKey, action, path, blob, since = '' }) {
|
|
95
|
+
return uuidV5(['migrate-mounts', repoKey, action, path, blob, since].join('\n'));
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/** The last commit touching `path` at `sha`, or '' when none did. */
|
|
99
|
+
export function lastTouch(git, root, sha, path) {
|
|
100
|
+
return git(['log', '-1', '--format=%H', sha, '--', path], { cwd: root }).toString('utf8').trim();
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
const sha256 = (buf) => createHash('sha256').update(buf).digest('hex');
|
|
104
|
+
|
|
105
|
+
/** Where a merge row's merged body lives: `<merged-dir>/<tier>/<R>`. */
|
|
106
|
+
export function mergedBodyPath(mergedDir, tier, legacyPath) {
|
|
107
|
+
if (!mergedDir) return null;
|
|
108
|
+
const base = resolve(mergedDir);
|
|
109
|
+
const p = resolve(base, tier, ...legacyPath.slice(LEGACY_PREFIX.length).split('/'));
|
|
110
|
+
return p.startsWith(base + sep) ? p : null;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/** `YYYYMMDDTHHMMSSmmmZ` for batch ids and report names. */
|
|
114
|
+
export function utcStamp(date) {
|
|
115
|
+
return date.toISOString().replace(/[-:]/g, '').replace('.', '');
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
class Divergence extends Error {
|
|
119
|
+
constructor(code, path, phase, detail = '') {
|
|
120
|
+
super(`${code} at ${path}`);
|
|
121
|
+
this.code = code;
|
|
122
|
+
this.path = path;
|
|
123
|
+
this.phase = phase;
|
|
124
|
+
this.detail = detail;
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
function assertCleanClone(git, root) {
|
|
129
|
+
const dirty = git(['status', '--porcelain'], { cwd: root }).toString('utf8').trim();
|
|
130
|
+
if (dirty) throw new PlanError(`clone ${root} has uncommitted changes; --apply refuses a dirty clone`);
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/**
|
|
134
|
+
* Apply batches to one tier.
|
|
135
|
+
*
|
|
136
|
+
* @param {object} opts
|
|
137
|
+
* @param {string} opts.root the tier clone
|
|
138
|
+
* @param {string} opts.repoKey its registry key
|
|
139
|
+
* @param {object} opts.registry the registry (for the mount map)
|
|
140
|
+
* @param {string} opts.companyRoot the company clone (the reader's registry)
|
|
141
|
+
* @param {string} opts.out output dir (batch reports, divergence files)
|
|
142
|
+
* @param {string} opts.repoRoot operator repo root (provenance source path)
|
|
143
|
+
* @param {object} opts.producer envelope producer
|
|
144
|
+
* @param {(envelope: object) => Promise<{httpStatus: number, body: unknown}>} opts.post
|
|
145
|
+
* @param {string} [opts.mergedDir]
|
|
146
|
+
* @param {number} [opts.batchSize]
|
|
147
|
+
* @param {'1'|'all'} [opts.batches]
|
|
148
|
+
* @param {number} [opts.waitMs]
|
|
149
|
+
* @param {number} [opts.pollMs]
|
|
150
|
+
* @param {string} [opts.ref]
|
|
151
|
+
* @param {Function} [opts.git]
|
|
152
|
+
* @param {() => Date} [opts.clock]
|
|
153
|
+
* @param {(ms: number) => Promise<void>} [opts.sleep]
|
|
154
|
+
* @param {Function} [opts.remoteOf]
|
|
155
|
+
* @param {(repoKey: string) => object|null} [opts.targetTree] other tiers' trees (cross-tier rows)
|
|
156
|
+
* @param {object} [opts.hooks] test seam: `afterPlan({ batchIndex, plan })`
|
|
157
|
+
* @param {Function} [opts.log]
|
|
158
|
+
* @returns {Promise<{ exitCode: number, batches: object[] }>}
|
|
159
|
+
*/
|
|
160
|
+
export async function applyTier(opts) {
|
|
161
|
+
const {
|
|
162
|
+
root, repoKey, registry, git = defaultGitRunner, log = () => {},
|
|
163
|
+
} = opts;
|
|
164
|
+
const map = createMountMap(registry);
|
|
165
|
+
if (!map.mountsFor(repoKey).length) {
|
|
166
|
+
throw new PlanError(`${repoKey} has no knowledge mount in the registry; its legacy rows are owner decisions and --apply refuses the tier`);
|
|
167
|
+
}
|
|
168
|
+
assertCleanClone(git, root);
|
|
169
|
+
mkdirSync(resolve(opts.out, 'batches'), { recursive: true });
|
|
170
|
+
const batches = [];
|
|
171
|
+
for (let batchIndex = 1; ; batchIndex += 1) {
|
|
172
|
+
const report = await applyBatch({ ...opts, map, git, batchIndex });
|
|
173
|
+
batches.push(report);
|
|
174
|
+
log(batchSummaryLine(report));
|
|
175
|
+
if (report.crossTier.length) {
|
|
176
|
+
const merges = report.crossTier.filter((c) => c.crossKind === 'merge').length;
|
|
177
|
+
log(`${tierLabel(report.tier)} cross-tier: ${report.crossTier.length} row(s) not applied (${merges} merge), listed in ${report.crossTierCsv}`);
|
|
178
|
+
}
|
|
179
|
+
if (report.divergence) return { exitCode: 2, batches };
|
|
180
|
+
if (opts.batches !== 'all' || report.rows.length === 0) break;
|
|
181
|
+
}
|
|
182
|
+
return { exitCode: 0, batches };
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
/** `<tier> batch <id>: posted u upserts, d deletes; verified v of n; skipped s; divergence <none|code at path>` */
|
|
186
|
+
export function batchSummaryLine(r) {
|
|
187
|
+
const u = r.rows.filter((x) => x.upsertPosted).length;
|
|
188
|
+
const d = r.rows.filter((x) => x.deletePosted).length;
|
|
189
|
+
const v = r.rows.filter((x) => x.verified.mounted && x.verified.legacyGone && x.verified.reader).length;
|
|
190
|
+
const div = r.divergence ? `${r.divergence.code} at ${r.divergence.path}` : 'none';
|
|
191
|
+
return `${tierLabel(r.tier)} batch ${r.batchId}: posted ${u} upserts, ${d} deletes; verified ${v} of ${r.rows.length}; skipped ${r.skipped.length}; divergence ${div}`;
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
async function applyBatch(ctx) {
|
|
195
|
+
const {
|
|
196
|
+
root, repoKey, map, git, out, repoRoot, producer, post, mergedDir, companyRoot,
|
|
197
|
+
batchSize = DEFAULT_BATCH_SIZE, waitMs = DEFAULT_WAIT_MS, pollMs = DEFAULT_POLL_MS,
|
|
198
|
+
ref, clock = () => new Date(), sleep = (ms) => new Promise((r) => setTimeout(r, ms)),
|
|
199
|
+
remoteOf = gitRemoteOf, targetTree = null, hooks = {}, batchIndex,
|
|
200
|
+
} = ctx;
|
|
201
|
+
const tier = tierLabel(repoKey);
|
|
202
|
+
const started = clock();
|
|
203
|
+
let batchId = `${repoKey}-${utcStamp(started)}`;
|
|
204
|
+
if (existsSync(resolve(out, 'batches', `${batchId}.json`))) batchId += `-${batchIndex}`;
|
|
205
|
+
|
|
206
|
+
// ---- plan (live) -----------------------------------------------------------
|
|
207
|
+
const at = resolvePlanRef(git, root, { ref, fetch: true });
|
|
208
|
+
const plan = planTier({ root, repoKey, map, at, git, targetTree });
|
|
209
|
+
if (hooks.afterPlan) await hooks.afterPlan({ batchIndex, plan });
|
|
210
|
+
const eligible = [];
|
|
211
|
+
const skipped = [];
|
|
212
|
+
for (const row of plan.rows) {
|
|
213
|
+
if (!APPLY_ACTIONS.has(row.action)) {
|
|
214
|
+
skipped.push({ action: row.action, legacyPath: row.current, reason: row.reason });
|
|
215
|
+
continue;
|
|
216
|
+
}
|
|
217
|
+
if (row.action === 'merge') {
|
|
218
|
+
const p = mergedBodyPath(mergedDir, tier, row.current);
|
|
219
|
+
if (!p || !existsSync(p)) {
|
|
220
|
+
skipped.push({ action: 'merge', legacyPath: row.current, reason: p ? `no merged body at ${p}` : 'no --merged-dir; a merge row needs a merged body' });
|
|
221
|
+
continue;
|
|
222
|
+
}
|
|
223
|
+
row.mergedBody = readFileSync(p);
|
|
224
|
+
}
|
|
225
|
+
eligible.push(row);
|
|
226
|
+
}
|
|
227
|
+
const batch = eligible.slice(0, batchSize);
|
|
228
|
+
const crossTier = crossTierRows([{ ...plan, sha: at.sha, rows: plan.rows }]);
|
|
229
|
+
const crossTierCsv = resolve(out, 'cross-tier.csv');
|
|
230
|
+
|
|
231
|
+
const rows = batch.map((row) => ({
|
|
232
|
+
action: row.action,
|
|
233
|
+
legacyPath: row.current,
|
|
234
|
+
mountedPath: row.correct,
|
|
235
|
+
workspaceId: row.workspaceId,
|
|
236
|
+
preLegacyBlob: row.legacyBlob,
|
|
237
|
+
preMountedBlob: row.mountedBlob || null,
|
|
238
|
+
postedBlob: null,
|
|
239
|
+
mergedBodySha256: row.mergedBody ? sha256(row.mergedBody) : null,
|
|
240
|
+
upsertEventId: null,
|
|
241
|
+
deleteEventId: null,
|
|
242
|
+
upsertPosted: false,
|
|
243
|
+
deletePosted: false,
|
|
244
|
+
verified: { mounted: false, legacyGone: false, reader: false },
|
|
245
|
+
divergence: null,
|
|
246
|
+
}));
|
|
247
|
+
// Body to post (upserts) or expect (delete, take-mounted).
|
|
248
|
+
const expectBody = new Map();
|
|
249
|
+
for (const [i, row] of batch.entries()) {
|
|
250
|
+
if (row.action === 'merge') expectBody.set(i, row.mergedBody);
|
|
251
|
+
else if (UPSERT_ACTIONS.has(row.action)) expectBody.set(i, plan.bodies.get(row.legacyBlob));
|
|
252
|
+
else expectBody.set(i, plan.bodies.get(row.mountedBlob));
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
const report = {
|
|
256
|
+
batchId,
|
|
257
|
+
tier: repoKey,
|
|
258
|
+
ref: at.ref,
|
|
259
|
+
tierShaBefore: at.sha,
|
|
260
|
+
tierShaAfter: null,
|
|
261
|
+
startedAt: started.toISOString(),
|
|
262
|
+
finishedAt: null,
|
|
263
|
+
remainingEligible: eligible.length - batch.length,
|
|
264
|
+
rows,
|
|
265
|
+
skipped,
|
|
266
|
+
crossTier,
|
|
267
|
+
crossTierCsv,
|
|
268
|
+
divergence: null,
|
|
269
|
+
rollback: {
|
|
270
|
+
note: ROLLBACK_NOTE,
|
|
271
|
+
tierShaBefore: at.sha,
|
|
272
|
+
copies: rows.map((r) => ({
|
|
273
|
+
legacyPath: r.legacyPath,
|
|
274
|
+
legacyBlob: r.preLegacyBlob,
|
|
275
|
+
mountedPath: r.mountedPath,
|
|
276
|
+
mountedBlob: r.preMountedBlob,
|
|
277
|
+
workspaceId: r.workspaceId,
|
|
278
|
+
show: `git show ${at.sha}:${r.preMountedBlob ? r.mountedPath : r.legacyPath}`,
|
|
279
|
+
})),
|
|
280
|
+
},
|
|
281
|
+
};
|
|
282
|
+
|
|
283
|
+
const fetchTree = () => {
|
|
284
|
+
const now = resolvePlanRef(git, root, { ref: at.ref, fetch: true });
|
|
285
|
+
return { sha: now.sha, tree: lsTreeBlobs(git, root, now.sha) };
|
|
286
|
+
};
|
|
287
|
+
const deadlineFrom = () => clock().getTime() + waitMs;
|
|
288
|
+
|
|
289
|
+
try {
|
|
290
|
+
if (!batch.length) {
|
|
291
|
+
report.tierShaAfter = at.sha;
|
|
292
|
+
return report;
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
// ---- phase 1: compare-and-swap ------------------------------------------------
|
|
296
|
+
const p1 = fetchTree();
|
|
297
|
+
for (const [i, row] of batch.entries()) {
|
|
298
|
+
const legacyNow = p1.tree.get(row.current) ?? '';
|
|
299
|
+
const mountedNow = p1.tree.get(row.correct) ?? '';
|
|
300
|
+
if (legacyNow !== row.legacyBlob || mountedNow !== (row.mountedBlob || '')) {
|
|
301
|
+
throw Object.assign(new Divergence('state-changed', row.current, 1,
|
|
302
|
+
`planned legacy ${row.legacyBlob} mounted ${row.mountedBlob || '(absent)'}; now legacy ${legacyNow || '(absent)'} mounted ${mountedNow || '(absent)'}`), { row: i });
|
|
303
|
+
}
|
|
304
|
+
if (!UPSERT_ACTIONS.has(row.action)) rows[i].verified.mounted = true;
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
// ---- phase 2: upserts -----------------------------------------------------------
|
|
308
|
+
for (const [i, row] of batch.entries()) {
|
|
309
|
+
if (!UPSERT_ACTIONS.has(row.action)) continue;
|
|
310
|
+
const raw = expectBody.get(i) ?? Buffer.alloc(0);
|
|
311
|
+
const content = raw.toString('utf8');
|
|
312
|
+
const posted = Buffer.from(content, 'utf8');
|
|
313
|
+
if (!posted.equals(raw)) {
|
|
314
|
+
throw Object.assign(new Divergence('body-not-utf8', row.current, 2, 'the body does not round-trip through UTF-8; posting it would change its bytes'), { row: i });
|
|
315
|
+
}
|
|
316
|
+
expectBody.set(i, posted);
|
|
317
|
+
const postedBlob = gitBlobSha(posted);
|
|
318
|
+
rows[i].postedBlob = postedBlob;
|
|
319
|
+
const clientEventId = migrateEventId({ repoKey, action: 'upsert', path: row.correct, blob: postedBlob, since: lastTouch(git, root, at.sha, row.correct) });
|
|
320
|
+
rows[i].upsertEventId = clientEventId;
|
|
321
|
+
const envelope = buildEnvelope({
|
|
322
|
+
repoRoot,
|
|
323
|
+
sourcePath: join(repoRoot, ...row.workspaceId.split('/')),
|
|
324
|
+
content,
|
|
325
|
+
producer,
|
|
326
|
+
kind: 'knowledge.doc.upsert',
|
|
327
|
+
tier: repoKey,
|
|
328
|
+
sourceKind: 'manual',
|
|
329
|
+
payloadPath: row.correct,
|
|
330
|
+
clientEventId,
|
|
331
|
+
clock,
|
|
332
|
+
});
|
|
333
|
+
await postOne(post, envelope, row.correct, 2, i);
|
|
334
|
+
rows[i].upsertPosted = true;
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
// ---- phase 3: wait for the mounted copies ------------------------------------------
|
|
338
|
+
const pendingUp = new Set(batch.map((r, i) => i).filter((i) => UPSERT_ACTIONS.has(batch[i].action)));
|
|
339
|
+
if (pendingUp.size) {
|
|
340
|
+
const deadline = deadlineFrom();
|
|
341
|
+
for (;;) {
|
|
342
|
+
const { tree } = fetchTree();
|
|
343
|
+
for (const i of [...pendingUp]) {
|
|
344
|
+
const blob = tree.get(batch[i].correct) ?? '';
|
|
345
|
+
if (blob === rows[i].postedBlob) { rows[i].verified.mounted = true; pendingUp.delete(i); continue; }
|
|
346
|
+
if (blob !== (batch[i].mountedBlob || '')) {
|
|
347
|
+
throw Object.assign(new Divergence('content-mismatch', batch[i].correct, 3,
|
|
348
|
+
`expected blob ${rows[i].postedBlob}, tier holds ${blob || '(absent)'}`), { row: i });
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
if (!pendingUp.size) break;
|
|
352
|
+
if (clock().getTime() >= deadline) {
|
|
353
|
+
const i = [...pendingUp][0];
|
|
354
|
+
throw Object.assign(new Divergence('not-applied', batch[i].correct, 3,
|
|
355
|
+
`blob ${rows[i].postedBlob} not at ${at.ref}:${batch[i].correct} within ${waitMs} ms`), { row: i });
|
|
356
|
+
}
|
|
357
|
+
await sleep(pollMs);
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
// ---- phase 4: deletes, only behind a verified mounted copy -------------------------
|
|
362
|
+
const deleted = [];
|
|
363
|
+
for (const [i, row] of batch.entries()) {
|
|
364
|
+
if (!rows[i].verified.mounted) continue; // never delete a legacy copy whose mounted copy did not verify
|
|
365
|
+
const clientEventId = migrateEventId({ repoKey, action: 'delete', path: row.current, blob: row.legacyBlob, since: lastTouch(git, root, at.sha, row.current) });
|
|
366
|
+
rows[i].deleteEventId = clientEventId;
|
|
367
|
+
const envelope = buildEnvelope({
|
|
368
|
+
repoRoot,
|
|
369
|
+
sourcePath: join(repoRoot, ...row.workspaceId.split('/')),
|
|
370
|
+
content: undefined,
|
|
371
|
+
producer,
|
|
372
|
+
kind: 'knowledge.doc.delete',
|
|
373
|
+
tier: repoKey,
|
|
374
|
+
sourceKind: 'manual',
|
|
375
|
+
payloadPath: row.current,
|
|
376
|
+
clientEventId,
|
|
377
|
+
clock,
|
|
378
|
+
});
|
|
379
|
+
await postOne(post, envelope, row.current, 4, i);
|
|
380
|
+
rows[i].deletePosted = true;
|
|
381
|
+
deleted.push(i);
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
// ---- phase 5: wait for the legacy paths to go --------------------------------------
|
|
385
|
+
const pendingDel = new Set(deleted);
|
|
386
|
+
if (pendingDel.size) {
|
|
387
|
+
const deadline = deadlineFrom();
|
|
388
|
+
for (;;) {
|
|
389
|
+
const { sha, tree } = fetchTree();
|
|
390
|
+
for (const i of [...pendingDel]) {
|
|
391
|
+
const blob = tree.get(batch[i].current) ?? '';
|
|
392
|
+
if (!blob) { rows[i].verified.legacyGone = true; pendingDel.delete(i); continue; }
|
|
393
|
+
if (blob !== batch[i].legacyBlob) {
|
|
394
|
+
throw Object.assign(new Divergence('state-changed', batch[i].current, 5,
|
|
395
|
+
`legacy path rewritten to ${blob} while its delete was pending`), { row: i });
|
|
396
|
+
}
|
|
397
|
+
}
|
|
398
|
+
report.tierShaAfter = sha;
|
|
399
|
+
if (!pendingDel.size) break;
|
|
400
|
+
if (clock().getTime() >= deadline) {
|
|
401
|
+
const i = [...pendingDel][0];
|
|
402
|
+
throw Object.assign(new Divergence('not-applied', batch[i].current, 5,
|
|
403
|
+
`legacy path still at ${at.ref} after ${waitMs} ms`), { row: i });
|
|
404
|
+
}
|
|
405
|
+
await sleep(pollMs);
|
|
406
|
+
}
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
// ---- phase 6: the real reader -------------------------------------------------------
|
|
410
|
+
readerCheck({ git, root, ref: at.ref, companyRoot, remoteOf, batch, rows, expectBody });
|
|
411
|
+
if (!report.tierShaAfter) report.tierShaAfter = fetchTree().sha;
|
|
412
|
+
return report;
|
|
413
|
+
} catch (e) {
|
|
414
|
+
if (!(e instanceof Divergence)) throw e;
|
|
415
|
+
report.divergence = { code: e.code, path: e.path, phase: e.phase, detail: e.detail };
|
|
416
|
+
if (typeof e.row === 'number') rows[e.row].divergence = report.divergence;
|
|
417
|
+
try { report.tierShaAfter = resolvePlanRef(git, root, { ref: at.ref, fetch: false }).sha; } catch { /* keep null */ }
|
|
418
|
+
writeFileSync(resolve(out, `divergence-${batchId}.json`), `${JSON.stringify({ batchId, tier: repoKey, ...report.divergence }, null, 2)}\n`);
|
|
419
|
+
return report;
|
|
420
|
+
} finally {
|
|
421
|
+
report.finishedAt = clock().toISOString();
|
|
422
|
+
writeFileSync(resolve(out, 'batches', `${batchId}.json`), `${JSON.stringify(report, null, 2)}\n`);
|
|
423
|
+
writeFileSync(crossTierCsv, crossTierCsvText(crossTier));
|
|
424
|
+
}
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
async function postOne(post, envelope, path, phase, row) {
|
|
428
|
+
let res;
|
|
429
|
+
try {
|
|
430
|
+
res = await post(envelope);
|
|
431
|
+
} catch (e) {
|
|
432
|
+
throw Object.assign(new Divergence(`rejected:unreachable`, path, phase, e.message ?? String(e)), { row });
|
|
433
|
+
}
|
|
434
|
+
const ack = parseIngestAck(res);
|
|
435
|
+
if (ack.kind !== 'accepted' && ack.kind !== 'deduped') {
|
|
436
|
+
throw Object.assign(new Divergence(`rejected:${ack.reason}`, path, phase, ack.reason), { row });
|
|
437
|
+
}
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
/**
|
|
441
|
+
* Phase 6: fast-forward the clone to the ref, then ask the reader, over an
|
|
442
|
+
* empty home (so no home copy masks the tier), which file backs each
|
|
443
|
+
* workspace id. It must be `<clone>/<target>` with the expected sha256.
|
|
444
|
+
*/
|
|
445
|
+
function readerCheck({ git, root, ref, companyRoot, remoteOf, batch, rows, expectBody }) {
|
|
446
|
+
const dirty = git(['status', '--porcelain'], { cwd: root }).toString('utf8').trim();
|
|
447
|
+
if (dirty) throw new Divergence('reader:dirty-clone', root, 6, 'the clone has uncommitted changes; it was not fast-forwarded');
|
|
448
|
+
if (ref !== 'HEAD') {
|
|
449
|
+
try {
|
|
450
|
+
git(['merge', '--ff-only', '--quiet', ref], { cwd: root });
|
|
451
|
+
} catch (e) {
|
|
452
|
+
throw new Divergence('reader:not-fast-forward', root, 6, e.message);
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
const home = mkdtempSync(join(tmpdir(), 'migrate-mounts-reader-'));
|
|
456
|
+
try {
|
|
457
|
+
const extraRoots = [...new Set([root, companyRoot].filter(Boolean))];
|
|
458
|
+
const scans = scanSourcesFor(home, { extraRoots, remoteOf });
|
|
459
|
+
for (const [i, row] of batch.entries()) {
|
|
460
|
+
const expected = join(root, ...row.correct.split('/'));
|
|
461
|
+
const found = locateScannedFile(scans, row.workspaceId);
|
|
462
|
+
if (found !== expected) {
|
|
463
|
+
throw Object.assign(new Divergence('reader-mismatch', row.workspaceId, 6, `reader resolves ${found ?? '(nothing)'}, expected ${expected}`), { row: i });
|
|
464
|
+
}
|
|
465
|
+
const want = sha256(expectBody.get(i) ?? Buffer.alloc(0));
|
|
466
|
+
const got = sha256(readFileSync(found));
|
|
467
|
+
if (got !== want) {
|
|
468
|
+
throw Object.assign(new Divergence('reader-mismatch', row.workspaceId, 6, `sha256 ${got} at ${found}, expected ${want}`), { row: i });
|
|
469
|
+
}
|
|
470
|
+
rows[i].verified.reader = true;
|
|
471
|
+
}
|
|
472
|
+
} finally {
|
|
473
|
+
rmdirSync(home); // created empty above and never written to
|
|
474
|
+
}
|
|
475
|
+
}
|
|
476
|
+
|
|
477
|
+
function crossTierCsvText(cross) {
|
|
478
|
+
return toCsv(cross, CROSS_TIER_COLUMNS);
|
|
479
|
+
}
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
// kit.mjs: the merge kit for the legacy-path migration (slice 3 Part A).
|
|
2
|
+
//
|
|
3
|
+
// For every row whose two copies differ (`merge`, `take-legacy`,
|
|
4
|
+
// `take-mounted`) the kit holds both bodies, a unified diff from the
|
|
5
|
+
// mounted copy (a/) to the legacy copy (b/), and the home copy when it
|
|
6
|
+
// matches neither. A writer reads the kit and writes merged bodies to
|
|
7
|
+
// `<merged-dir>/<tier>/<R>`; nothing here writes outside `<out>/merge/`.
|
|
8
|
+
|
|
9
|
+
import { mkdirSync, writeFileSync } from 'node:fs';
|
|
10
|
+
import { dirname, join, resolve, sep } from 'node:path';
|
|
11
|
+
|
|
12
|
+
import { KIT_ACTIONS, LEGACY_PREFIX } from './plan.mjs';
|
|
13
|
+
|
|
14
|
+
/** Name the diff sides by path instead of blob id. */
|
|
15
|
+
export function labelDiff(diff, { mountedBlob, legacyBlob, mountedPath, legacyPath }) {
|
|
16
|
+
return diff
|
|
17
|
+
.replace(`diff --git a/${mountedBlob} b/${legacyBlob}`, `diff --git a/${mountedPath} b/${legacyPath}`)
|
|
18
|
+
.replace(`--- a/${mountedBlob}`, `--- a/${mountedPath}`)
|
|
19
|
+
.replace(`+++ b/${legacyBlob}`, `+++ b/${legacyPath}`);
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
const README = (rows) => `# Merge kit: legacy-path migration (w-2026-10-16-dave-019, slice 3)
|
|
23
|
+
|
|
24
|
+
Generated by \`library migrate-mounts --dry-run\`. One entry per row whose
|
|
25
|
+
two copies differ, under \`<tier>/<R>\`, where R is the path after
|
|
26
|
+
\`operator/knowledge/\`:
|
|
27
|
+
|
|
28
|
+
- \`<R>.legacy\`: the tier's legacy copy at \`operator/knowledge/<R>\`.
|
|
29
|
+
- \`<R>.mounted\`: the tier's copy at the mounted path (the plan's \`correct\`).
|
|
30
|
+
- \`<R>.diff\`: unified diff from the mounted copy (a/) to the legacy copy (b/).
|
|
31
|
+
- \`<R>.home\`: the home workspace copy, present only when it matches neither.
|
|
32
|
+
|
|
33
|
+
## The rule
|
|
34
|
+
|
|
35
|
+
Migrating a row is a merge of the two bodies, never a move or an overwrite,
|
|
36
|
+
and commit recency never decides it (the library's merge rule,
|
|
37
|
+
\`knowledge-memory/legacy-path-migration-merge.md\`).
|
|
38
|
+
|
|
39
|
+
- \`merge\` rows: keep the Librarian additions from the legacy copy and the
|
|
40
|
+
newer operator edits from the mounted copy, resolve every conflicting
|
|
41
|
+
passage on purpose, and write the merged body to
|
|
42
|
+
\`<merged-dir>/<tier>/<R>\`. A \`merge\` row without a merged body is
|
|
43
|
+
skipped by the apply.
|
|
44
|
+
- \`take-legacy\` rows: the mounted copy is an ancestor of the legacy copy, so
|
|
45
|
+
the legacy body is posted as is. Check the diff anyway.
|
|
46
|
+
- \`take-mounted\` rows: the legacy copy is an ancestor of the mounted copy, so
|
|
47
|
+
nothing is posted and only the legacy copy is deleted. Check the diff
|
|
48
|
+
anyway.
|
|
49
|
+
|
|
50
|
+
A \`.home\` file means the operator's local copy holds a third version; fold
|
|
51
|
+
anything it alone carries into the merged body.
|
|
52
|
+
|
|
53
|
+
## Cross-tier rows (Dave ruling 2026-10-19)
|
|
54
|
+
|
|
55
|
+
A legacy row under \`operator/knowledge/wsd/\` is wsd-tier content: its
|
|
56
|
+
target is the wsd tier at \`knowledge/R\`, R being the path after
|
|
57
|
+
\`operator/knowledge/wsd/\`. In the wsd tier it is an ordinary row (its kit
|
|
58
|
+
files, if any, are listed below with the other rows). In any other tier the
|
|
59
|
+
plan marks it \`cross-tier\`, the apply never writes it, and
|
|
60
|
+
\`cross-tier.csv\` (next to the plan CSVs) lists every such row with both
|
|
61
|
+
blobs: \`move\` (no wsd copy yet), \`identical\`, or \`merge\` (a wsd copy
|
|
62
|
+
that differs; the same merge rule applies, and the wsd tier owner writes and
|
|
63
|
+
applies the merged body from a wsd host). This kit holds no files for them.
|
|
64
|
+
|
|
65
|
+
## Rows (${rows.length})
|
|
66
|
+
|
|
67
|
+
| tier | R | action | home |
|
|
68
|
+
|---|---|---|---|
|
|
69
|
+
${rows.map((r) => `| ${r.tier} | \`${r.current.slice(LEGACY_PREFIX.length)}\` | ${r.action} | ${r.home} |`).join('\n')}
|
|
70
|
+
`;
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* Write the kit for every planned tier under `<out>/merge/`.
|
|
74
|
+
*
|
|
75
|
+
* @param {object} opts
|
|
76
|
+
* @param {string} opts.out
|
|
77
|
+
* @param {{ root: string, tier: string, rows: object[], bodies: Map, homes: Map }[]} opts.plans
|
|
78
|
+
* @param {Function} opts.git the runner (for `git diff <blob> <blob>`)
|
|
79
|
+
* @returns {{ dir: string, count: number }}
|
|
80
|
+
*/
|
|
81
|
+
export function writeMergeKit({ out, plans, git }) {
|
|
82
|
+
const dir = resolve(out, 'merge');
|
|
83
|
+
mkdirSync(dir, { recursive: true });
|
|
84
|
+
const kitRows = [];
|
|
85
|
+
for (const plan of plans) {
|
|
86
|
+
for (const row of plan.rows) {
|
|
87
|
+
if (!KIT_ACTIONS.has(row.action)) continue;
|
|
88
|
+
const rest = row.current.slice(LEGACY_PREFIX.length);
|
|
89
|
+
const base = resolve(dir, plan.tier, ...rest.split('/'));
|
|
90
|
+
if (!base.startsWith(dir + sep)) continue; // never write outside the kit
|
|
91
|
+
mkdirSync(dirname(base), { recursive: true });
|
|
92
|
+
writeFileSync(`${base}.legacy`, plan.bodies.get(row.legacyBlob) ?? Buffer.alloc(0));
|
|
93
|
+
writeFileSync(`${base}.mounted`, plan.bodies.get(row.mountedBlob) ?? Buffer.alloc(0));
|
|
94
|
+
const diff = git(['diff', '--no-color', '--no-ext-diff', row.mountedBlob, row.legacyBlob], { cwd: plan.root }).toString('utf8');
|
|
95
|
+
writeFileSync(`${base}.diff`, labelDiff(diff, {
|
|
96
|
+
mountedBlob: row.mountedBlob,
|
|
97
|
+
legacyBlob: row.legacyBlob,
|
|
98
|
+
mountedPath: row.correct,
|
|
99
|
+
legacyPath: row.current,
|
|
100
|
+
}));
|
|
101
|
+
const home = plan.homes.get(row.current);
|
|
102
|
+
if (row.home === 'differs' && home) writeFileSync(`${base}.home`, home);
|
|
103
|
+
kitRows.push(row);
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
writeFileSync(join(dir, 'README.md'), README(kitRows));
|
|
107
|
+
return { dir, count: kitRows.length };
|
|
108
|
+
}
|