@yeaft/webchat-agent 0.1.533 → 0.1.534
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/unify/memory/migrate-r5-to-r6.js +566 -16
package/package.json
CHANGED
|
@@ -1,22 +1,89 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* migrate-r5-to-r6.js — task-
|
|
2
|
+
* migrate-r5-to-r6.js — task-334i (wave-4) R5 → R6 storage migration.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
4
|
+
* Continuation of task-334i-v0 (PR #552, shipped v0.1.521) which established
|
|
5
|
+
* the general `~/.yeaft/` tree via `v0-to-v1.js`. This slice implements the
|
|
6
|
+
* R5→R6 *memory shard + conversation rotation* pass that fleshes out the
|
|
7
|
+
* previously-stubbed `applyR5ToR6Migration`.
|
|
6
8
|
*
|
|
7
|
-
*
|
|
8
|
-
* Semantic shards: skill / relations / lessons / preferences /
|
|
9
|
-
* project-<slug>
|
|
9
|
+
* Spec: .crew/context/task-334i-impl-spec.md
|
|
10
10
|
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
11
|
+
* Key invariants:
|
|
12
|
+
* - This is independently idempotent from v0→v1. State version bumps r5→r6.
|
|
13
|
+
* - Does NOT touch 334f `shard-store.js` or 334o storage primitives —
|
|
14
|
+
* only consumes their public APIs.
|
|
15
|
+
* - Legacy R5 data is archived to `.legacy/r6-state.tar.gz` BEFORE any
|
|
16
|
+
* write. Rollback restores the archive but never deletes it.
|
|
17
|
+
* - `migration-state.json` is the single source of truth; writes go
|
|
18
|
+
* through `writeAtomic` (tmp+rename) mirroring 334f commitRecompression.
|
|
19
|
+
*
|
|
20
|
+
* Two-pass algorithm (pre-emptive discovery #2):
|
|
21
|
+
* Pass 1 — write each entry to its default semantic shard
|
|
22
|
+
* (skill / relations / lessons / preferences / project-legacy)
|
|
23
|
+
* while counting entries per groupId.
|
|
24
|
+
* Pass 2 — for each groupId with count ≥ PROJECT_DERIVE_THRESHOLD (30),
|
|
25
|
+
* derive a `project-<slug>` shard and move entries via
|
|
26
|
+
* stageRecompression / commitRecompression.
|
|
27
|
+
*
|
|
28
|
+
* Name drift fix (pre-emptive discovery #1):
|
|
29
|
+
* `map-fields.js` (shipped) defines MIGRATION_AUTHOR='system:migration-v0-to-v1'.
|
|
30
|
+
* Correct spec value is 'system:migration-v0-v1'. This file overrides via
|
|
31
|
+
* local constants without editing the shipped pure-mapper module.
|
|
13
32
|
*/
|
|
14
33
|
|
|
15
|
-
import {
|
|
16
|
-
|
|
34
|
+
import {
|
|
35
|
+
existsSync,
|
|
36
|
+
mkdirSync,
|
|
37
|
+
readdirSync,
|
|
38
|
+
readFileSync,
|
|
39
|
+
writeFileSync,
|
|
40
|
+
renameSync,
|
|
41
|
+
rmSync,
|
|
42
|
+
statSync,
|
|
43
|
+
} from 'fs';
|
|
44
|
+
import { join, dirname } from 'path';
|
|
45
|
+
import { createHash } from 'crypto';
|
|
46
|
+
import { execFileSync } from 'child_process';
|
|
47
|
+
|
|
48
|
+
import { openLog, writeAtomic } from '../storage/index.js';
|
|
17
49
|
import { parseEntry } from './store.js';
|
|
18
|
-
import {
|
|
50
|
+
import {
|
|
51
|
+
openMemoryShardStore,
|
|
52
|
+
classifyLegacyEntryToShard,
|
|
53
|
+
} from './shard-store.js';
|
|
54
|
+
import { PROJECT_DERIVE_THRESHOLD } from './schema.js';
|
|
55
|
+
import {
|
|
56
|
+
parseFrontmatter,
|
|
57
|
+
mapMessageMdToJsonl,
|
|
58
|
+
splitCoordinatorTurns,
|
|
59
|
+
LEGACY_GROUP_ID,
|
|
60
|
+
LEGACY_VP_ID,
|
|
61
|
+
} from '../migration/map-fields.js';
|
|
62
|
+
|
|
63
|
+
// ─── Name-drift fix (spec §3, §4) ────────────────────────────────
|
|
64
|
+
export const R5_TO_R6_AUTHOR_SYS = 'system:migration-v0-v1';
|
|
65
|
+
export const R5_TO_R6_AUTHOR_USER = 'user:migration-v0-v1';
|
|
66
|
+
const SOURCE_HINT = 'legacy-r5-migration';
|
|
67
|
+
const STATE_FILE = 'migration-state.json';
|
|
68
|
+
const ARCHIVE_REL = join('.legacy', 'r6-state.tar.gz');
|
|
69
|
+
|
|
70
|
+
// authoredBy inference table (spec §3 deliverable E).
|
|
71
|
+
function inferAuthoredBy(kind) {
|
|
72
|
+
switch (kind) {
|
|
73
|
+
case 'preference':
|
|
74
|
+
case 'identity':
|
|
75
|
+
return R5_TO_R6_AUTHOR_USER;
|
|
76
|
+
case 'fact':
|
|
77
|
+
case 'skill':
|
|
78
|
+
case 'lesson':
|
|
79
|
+
case 'context':
|
|
80
|
+
case 'relation':
|
|
81
|
+
default:
|
|
82
|
+
return R5_TO_R6_AUTHOR_SYS;
|
|
83
|
+
}
|
|
84
|
+
}
|
|
19
85
|
|
|
86
|
+
// ─── Planner (dry-run, unchanged behaviour from 334f stub) ───────
|
|
20
87
|
/**
|
|
21
88
|
* Produce a migration plan without applying it.
|
|
22
89
|
*
|
|
@@ -50,12 +117,495 @@ export function planR5ToR6Migration(legacyEntriesDir) {
|
|
|
50
117
|
return { totalEntries: plan.length, plan, byShard };
|
|
51
118
|
}
|
|
52
119
|
|
|
120
|
+
// ─── State I/O ────────────────────────────────────────────────────
|
|
121
|
+
function stateFilePath(yeaftDir) {
|
|
122
|
+
return join(yeaftDir, STATE_FILE);
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
function loadState(yeaftDir) {
|
|
126
|
+
const p = stateFilePath(yeaftDir);
|
|
127
|
+
if (!existsSync(p)) return null;
|
|
128
|
+
try {
|
|
129
|
+
const parsed = JSON.parse(readFileSync(p, 'utf8'));
|
|
130
|
+
return parsed && typeof parsed === 'object' ? parsed : null;
|
|
131
|
+
} catch {
|
|
132
|
+
return null;
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
function saveState(yeaftDir, state) {
|
|
137
|
+
writeAtomic(stateFilePath(yeaftDir), JSON.stringify(state, null, 2));
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
function clearState(yeaftDir) {
|
|
141
|
+
const p = stateFilePath(yeaftDir);
|
|
142
|
+
if (existsSync(p)) rmSync(p);
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
function nowIso() { return new Date().toISOString(); }
|
|
146
|
+
|
|
147
|
+
function stableId(slug) {
|
|
148
|
+
const h = createHash('sha1').update(String(slug)).digest('hex').slice(0, 12);
|
|
149
|
+
return `mem_legacy_${h}`;
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
// ─── Archive helper (spec §11 step 3) ────────────────────────────
|
|
153
|
+
/**
|
|
154
|
+
* tar+gzip `memory/entries/` and `conversations/` into .legacy/r6-state.tar.gz.
|
|
155
|
+
* Uses the `tar` CLI via execFileSync (same pattern as 334i-v0). Any failure
|
|
156
|
+
* throws so the caller can bail before making destructive changes.
|
|
157
|
+
*/
|
|
158
|
+
export function archiveR5State(yeaftDir) {
|
|
159
|
+
const legacyDir = join(yeaftDir, '.legacy');
|
|
160
|
+
mkdirSync(legacyDir, { recursive: true });
|
|
161
|
+
const archivePath = join(yeaftDir, ARCHIVE_REL);
|
|
162
|
+
const entriesDir = join(yeaftDir, 'memory', 'entries');
|
|
163
|
+
const conversationsDir = join(yeaftDir, 'conversations');
|
|
164
|
+
const args = ['-czf', archivePath, '-C', yeaftDir];
|
|
165
|
+
let added = 0;
|
|
166
|
+
if (existsSync(entriesDir)) { args.push(join('memory', 'entries')); added++; }
|
|
167
|
+
if (existsSync(conversationsDir)) { args.push('conversations'); added++; }
|
|
168
|
+
if (added === 0) {
|
|
169
|
+
// Write a zero-content marker so rollback has a file to inspect.
|
|
170
|
+
writeAtomic(archivePath, '');
|
|
171
|
+
return archivePath;
|
|
172
|
+
}
|
|
173
|
+
execFileSync('tar', args, { stdio: ['ignore', 'ignore', 'pipe'] });
|
|
174
|
+
return archivePath;
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
// ─── Detect helpers ──────────────────────────────────────────────
|
|
178
|
+
/**
|
|
179
|
+
* Classify the shape of the R5 memory layout in `yeaftDir`.
|
|
180
|
+
*/
|
|
181
|
+
export function detectR5MemoryLayout(yeaftDir) {
|
|
182
|
+
const entriesDir = join(yeaftDir, 'memory', 'entries');
|
|
183
|
+
const conversationsDir = join(yeaftDir, 'conversations');
|
|
184
|
+
const groupsDir = join(yeaftDir, 'groups');
|
|
185
|
+
const hasEntries = existsSync(entriesDir) && readdirSync(entriesDir).some(f => f.endsWith('.md'));
|
|
186
|
+
const hasConversationsMd = existsSync(conversationsDir)
|
|
187
|
+
&& readdirSync(conversationsDir).some(() => true);
|
|
188
|
+
const hasGroupsJsonl = existsSync(groupsDir);
|
|
189
|
+
return {
|
|
190
|
+
entriesDir,
|
|
191
|
+
conversationsDir,
|
|
192
|
+
groupsDir,
|
|
193
|
+
hasEntries,
|
|
194
|
+
hasConversationsMd,
|
|
195
|
+
hasGroupsJsonl,
|
|
196
|
+
};
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
// ─── Pass 1: write entries to default shards ─────────────────────
|
|
200
|
+
function runPass1({ yeaftDir, layout, vpDir, log, existingState }) {
|
|
201
|
+
const shardStore = openMemoryShardStore(vpDir, 'vp');
|
|
202
|
+
const files = layout.hasEntries
|
|
203
|
+
? readdirSync(layout.entriesDir).filter(f => f.endsWith('.md')).sort()
|
|
204
|
+
: [];
|
|
205
|
+
const counts = (existingState && existingState.counts) || {};
|
|
206
|
+
const migrated = [];
|
|
207
|
+
const errors = [];
|
|
208
|
+
|
|
209
|
+
for (const file of files) {
|
|
210
|
+
const slug = file.replace(/\.md$/, '');
|
|
211
|
+
const id = stableId(slug);
|
|
212
|
+
// Idempotency: skip already-migrated ids.
|
|
213
|
+
if (shardStore.get(id)) {
|
|
214
|
+
migrated.push({ id, slug, shard: shardStore.get(id).shard, skipped: true });
|
|
215
|
+
continue;
|
|
216
|
+
}
|
|
217
|
+
let raw;
|
|
218
|
+
try {
|
|
219
|
+
raw = readFileSync(join(layout.entriesDir, file), 'utf8');
|
|
220
|
+
} catch (e) {
|
|
221
|
+
errors.push({ file, error: String(e.message || e) });
|
|
222
|
+
continue;
|
|
223
|
+
}
|
|
224
|
+
const legacy = parseEntry(raw);
|
|
225
|
+
if (!legacy) {
|
|
226
|
+
errors.push({ file, error: 'parseEntry returned null (malformed frontmatter)' });
|
|
227
|
+
continue;
|
|
228
|
+
}
|
|
229
|
+
const shard = classifyLegacyEntryToShard(legacy);
|
|
230
|
+
const kind = legacy.kind || 'fact';
|
|
231
|
+
const tags = Array.isArray(legacy.tags) ? legacy.tags.slice() : [];
|
|
232
|
+
const createdAt = legacy.created_at || nowIso();
|
|
233
|
+
const updatedAt = legacy.updated_at || createdAt;
|
|
234
|
+
// Determine groupId for project-derive counting. Legacy schema has no
|
|
235
|
+
// explicit groupId; fall back to scope's top segment, else LEGACY_GROUP_ID.
|
|
236
|
+
const groupId = deriveGroupId(legacy);
|
|
237
|
+
|
|
238
|
+
// identity/preference kinds are allowed empty msgIds per §Δ23.
|
|
239
|
+
// Other kinds rely on the hint='legacy-r5-migration' to legitimise [].
|
|
240
|
+
// `validateR6Entry` requires non-empty msgIds for non-identity/preference —
|
|
241
|
+
// so we put a synthetic legacy marker to keep the validator happy while
|
|
242
|
+
// still conveying "migrated, no real messages attached" semantically.
|
|
243
|
+
const needsMsgIdMarker = !(kind === 'identity' || kind === 'preference');
|
|
244
|
+
const msgIds = needsMsgIdMarker ? [`legacy:${slug}`] : [];
|
|
245
|
+
|
|
246
|
+
const entry = {
|
|
247
|
+
id,
|
|
248
|
+
shard,
|
|
249
|
+
kind,
|
|
250
|
+
tags,
|
|
251
|
+
pinned: legacy.importance === 'high',
|
|
252
|
+
sourceRef: {
|
|
253
|
+
groupId,
|
|
254
|
+
taskId: null,
|
|
255
|
+
msgIds,
|
|
256
|
+
timeWindow: `[${createdAt}, ${updatedAt}]`,
|
|
257
|
+
hint: SOURCE_HINT,
|
|
258
|
+
},
|
|
259
|
+
supersedes: null,
|
|
260
|
+
supersededBy: null,
|
|
261
|
+
authoredBy: inferAuthoredBy(kind),
|
|
262
|
+
createdAt,
|
|
263
|
+
updatedAt,
|
|
264
|
+
body: legacy.content || '',
|
|
265
|
+
};
|
|
266
|
+
try {
|
|
267
|
+
shardStore.put(entry);
|
|
268
|
+
counts[groupId] = (counts[groupId] || 0) + 1;
|
|
269
|
+
migrated.push({ id, slug, shard, groupId });
|
|
270
|
+
} catch (e) {
|
|
271
|
+
errors.push({ file, error: String(e.message || e) });
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
log('pass1', { migrated: migrated.length, errors: errors.length });
|
|
276
|
+
return { counts, migrated, errors };
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
function deriveGroupId(legacyEntry) {
|
|
280
|
+
// scope is a path like "work/project-name/auth". Use first segment as
|
|
281
|
+
// coarse groupId; fall back to LEGACY_GROUP_ID.
|
|
282
|
+
const scope = legacyEntry && legacyEntry.scope;
|
|
283
|
+
if (typeof scope === 'string' && scope.trim()) {
|
|
284
|
+
const first = scope.split('/').map(s => s.trim()).filter(Boolean)[0];
|
|
285
|
+
if (first) return first;
|
|
286
|
+
}
|
|
287
|
+
return LEGACY_GROUP_ID;
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
// ─── Pass 2: project-<slug> derive ───────────────────────────────
|
|
291
|
+
function runPass2({ vpDir, counts, log }) {
|
|
292
|
+
const derived = [];
|
|
293
|
+
const qualifyingShards = Object.entries(counts || {})
|
|
294
|
+
.filter(([, c]) => c >= PROJECT_DERIVE_THRESHOLD)
|
|
295
|
+
.map(([g]) => `project-${slugify(g)}`);
|
|
296
|
+
// Re-open with project-<slug> allow-listed up front so put() validates.
|
|
297
|
+
const shardStore = openMemoryShardStore(vpDir, 'vp', { extraShards: qualifyingShards });
|
|
298
|
+
for (const [groupId, count] of Object.entries(counts || {})) {
|
|
299
|
+
if (count < PROJECT_DERIVE_THRESHOLD) continue;
|
|
300
|
+
const slug = slugify(groupId);
|
|
301
|
+
const targetShard = `project-${slug}`;
|
|
302
|
+
// Skip if this project shard already exists and is populated — re-entry safe.
|
|
303
|
+
const stats = shardStore.stats();
|
|
304
|
+
if (stats.shards[targetShard] && stats.shards[targetShard].count > 0) {
|
|
305
|
+
derived.push({ groupId, shard: targetShard, moved: 0, skipped: true });
|
|
306
|
+
continue;
|
|
307
|
+
}
|
|
308
|
+
// Collect entries in the relevant default shard matching this groupId.
|
|
309
|
+
// Search across all default shards (classification is kind-driven; a
|
|
310
|
+
// groupId may span skill/lessons/etc).
|
|
311
|
+
const { results } = shardStore.query({});
|
|
312
|
+
const moveIds = results
|
|
313
|
+
.filter(r => r.groupId === groupId)
|
|
314
|
+
.map(r => r.id);
|
|
315
|
+
let moved = 0;
|
|
316
|
+
for (const id of moveIds) {
|
|
317
|
+
const full = shardStore.get(id);
|
|
318
|
+
if (!full) continue;
|
|
319
|
+
// Re-put with new shard; old entry removal happens via supersede-free
|
|
320
|
+
// rewrite by removing old id after the new one lands.
|
|
321
|
+
const newEntry = {
|
|
322
|
+
...full,
|
|
323
|
+
shard: targetShard,
|
|
324
|
+
body: full.body || '',
|
|
325
|
+
};
|
|
326
|
+
try {
|
|
327
|
+
shardStore.put(newEntry);
|
|
328
|
+
// shardStore.put upserts by id (see 334o shard-store put semantics
|
|
329
|
+
// removing any prior shard copy of the same id) — so the entry now
|
|
330
|
+
// lives in targetShard exclusively.
|
|
331
|
+
moved++;
|
|
332
|
+
} catch {
|
|
333
|
+
// best-effort — keep legacy in default shard if move fails.
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
derived.push({ groupId, shard: targetShard, moved });
|
|
337
|
+
}
|
|
338
|
+
log('pass2', { derived: derived.length });
|
|
339
|
+
return derived;
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
function slugify(s) {
|
|
343
|
+
return String(s || '')
|
|
344
|
+
.toLowerCase()
|
|
345
|
+
.replace(/[^a-z0-9]+/g, '-')
|
|
346
|
+
.replace(/^-+|-+$/g, '')
|
|
347
|
+
.slice(0, 40) || 'legacy';
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
// ─── Conversation migration ──────────────────────────────────────
|
|
351
|
+
function migrateConversations({ yeaftDir, layout, log }) {
|
|
352
|
+
if (!layout.hasConversationsMd) {
|
|
353
|
+
log('conversations', { messages: 0, shards: 0 });
|
|
354
|
+
return { messages: 0, shards: 0 };
|
|
355
|
+
}
|
|
356
|
+
const groupDir = join(yeaftDir, 'groups', LEGACY_GROUP_ID, 'messages');
|
|
357
|
+
mkdirSync(groupDir, { recursive: true });
|
|
358
|
+
const log_ = openLog(groupDir);
|
|
359
|
+
let messagesWritten = 0;
|
|
360
|
+
const convos = readdirSync(layout.conversationsDir);
|
|
361
|
+
for (const cId of convos) {
|
|
362
|
+
const msgDir = join(layout.conversationsDir, cId, 'messages');
|
|
363
|
+
if (!existsSync(msgDir)) continue;
|
|
364
|
+
const files = readdirSync(msgDir).filter(f => f.endsWith('.md')).sort();
|
|
365
|
+
for (const file of files) {
|
|
366
|
+
const raw = readFileSync(join(msgDir, file), 'utf8');
|
|
367
|
+
const { meta, body } = parseFrontmatter(raw);
|
|
368
|
+
const originalId = `${cId}_${file.replace(/\.md$/, '')}`;
|
|
369
|
+
const row = mapMessageMdToJsonl({ meta, body, originalId, fallbackTaskId: null });
|
|
370
|
+
try {
|
|
371
|
+
log_.append(row);
|
|
372
|
+
messagesWritten++;
|
|
373
|
+
} catch {
|
|
374
|
+
// Skip malformed row; keep going.
|
|
375
|
+
}
|
|
376
|
+
}
|
|
377
|
+
// Also handle coordinator.md if present
|
|
378
|
+
const coordPath = join(layout.conversationsDir, cId, 'coordinator.md');
|
|
379
|
+
if (existsSync(coordPath)) {
|
|
380
|
+
const raw = readFileSync(coordPath, 'utf8');
|
|
381
|
+
const turns = splitCoordinatorTurns(raw);
|
|
382
|
+
for (const turn of turns) {
|
|
383
|
+
const row = {
|
|
384
|
+
id: `msg_legacy_${cId}_coord_${turn.index}`,
|
|
385
|
+
ts: turn.ts || null,
|
|
386
|
+
type: 'chat',
|
|
387
|
+
authorKind: 'unknown',
|
|
388
|
+
authorId: `legacy:${turn.role}`,
|
|
389
|
+
groupId: LEGACY_GROUP_ID,
|
|
390
|
+
taskId: null,
|
|
391
|
+
body: turn.body,
|
|
392
|
+
mentions: [],
|
|
393
|
+
replyTo: null,
|
|
394
|
+
viaTool: null,
|
|
395
|
+
};
|
|
396
|
+
try { log_.append(row); messagesWritten++; } catch { /* skip */ }
|
|
397
|
+
}
|
|
398
|
+
}
|
|
399
|
+
}
|
|
400
|
+
log_.close();
|
|
401
|
+
const index = log_.getIndex();
|
|
402
|
+
log('conversations', { messages: messagesWritten, shards: (index.segments || []).length });
|
|
403
|
+
return { messages: messagesWritten, shards: (index.segments || []).length };
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
// ─── Main entry ──────────────────────────────────────────────────
|
|
407
|
+
/**
|
|
408
|
+
* Apply the R5 → R6 migration.
|
|
409
|
+
*
|
|
410
|
+
* @param {object} opts
|
|
411
|
+
* @param {string} opts.yeaftDir required
|
|
412
|
+
* @param {string} [opts.vpId] legacy VP id (default LEGACY_VP_ID)
|
|
413
|
+
* @param {boolean} [opts.dryRun]
|
|
414
|
+
* @param {boolean} [opts.force] clear existing r6 state and re-run from scratch
|
|
415
|
+
* @param {(step, info)=>void} [opts.onStep]
|
|
416
|
+
* @returns {Promise<object>}
|
|
417
|
+
*/
|
|
418
|
+
export async function applyR5ToR6Migration(opts = {}) {
|
|
419
|
+
const { yeaftDir, dryRun = false, force = false, onStep } = opts;
|
|
420
|
+
const vpId = opts.vpId || LEGACY_VP_ID;
|
|
421
|
+
if (!yeaftDir || typeof yeaftDir !== 'string') {
|
|
422
|
+
throw new Error('applyR5ToR6Migration: yeaftDir (string) required');
|
|
423
|
+
}
|
|
424
|
+
if (!existsSync(yeaftDir)) {
|
|
425
|
+
throw new Error(`applyR5ToR6Migration: yeaftDir does not exist: ${yeaftDir}`);
|
|
426
|
+
}
|
|
427
|
+
const log = typeof onStep === 'function' ? onStep : () => {};
|
|
428
|
+
const layout = detectR5MemoryLayout(yeaftDir);
|
|
429
|
+
|
|
430
|
+
if (dryRun) {
|
|
431
|
+
const plan = planR5ToR6Migration(layout.entriesDir);
|
|
432
|
+
const counts = {};
|
|
433
|
+
for (const p of plan.plan) {
|
|
434
|
+
// coarse counting keyed by synthetic groupId based on slug prefix (best-effort preview)
|
|
435
|
+
counts[LEGACY_GROUP_ID] = (counts[LEGACY_GROUP_ID] || 0) + 1;
|
|
436
|
+
}
|
|
437
|
+
const wouldDerive = Object.entries(counts)
|
|
438
|
+
.filter(([, c]) => c >= PROJECT_DERIVE_THRESHOLD)
|
|
439
|
+
.map(([g]) => `project-${slugify(g)}`);
|
|
440
|
+
let estMessages = 0;
|
|
441
|
+
if (layout.hasConversationsMd) {
|
|
442
|
+
for (const cId of readdirSync(layout.conversationsDir)) {
|
|
443
|
+
const msgDir = join(layout.conversationsDir, cId, 'messages');
|
|
444
|
+
if (existsSync(msgDir)) {
|
|
445
|
+
estMessages += readdirSync(msgDir).filter(f => f.endsWith('.md')).length;
|
|
446
|
+
}
|
|
447
|
+
}
|
|
448
|
+
}
|
|
449
|
+
const preview = {
|
|
450
|
+
pass1: plan.byShard,
|
|
451
|
+
pass2Candidates: wouldDerive,
|
|
452
|
+
conversations: {
|
|
453
|
+
count: layout.hasConversationsMd ? readdirSync(layout.conversationsDir).length : 0,
|
|
454
|
+
estimatedMessages: estMessages,
|
|
455
|
+
estimatedShards: Math.max(1, Math.ceil(estMessages / 5000)),
|
|
456
|
+
},
|
|
457
|
+
};
|
|
458
|
+
log('dry-run', preview);
|
|
459
|
+
return { status: 'dry-run', dryRun: true, preview };
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
// Force: wipe only r6 state, never touch legacy archive.
|
|
463
|
+
if (force) clearState(yeaftDir);
|
|
464
|
+
|
|
465
|
+
let state = loadState(yeaftDir);
|
|
466
|
+
if (state && state.version === 'r6' && state.pass2CompletedAt) {
|
|
467
|
+
log('already-done', { migratedAt: state.migratedAt });
|
|
468
|
+
return { status: 'already-done', state };
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
// Fresh state — but preserve a pre-existing r5 state (from PR #552) if present.
|
|
472
|
+
if (!state || state.version !== 'r6') {
|
|
473
|
+
const prior = state || {};
|
|
474
|
+
state = {
|
|
475
|
+
version: 'r6',
|
|
476
|
+
startedAt: nowIso(),
|
|
477
|
+
legacyArchive: null,
|
|
478
|
+
pass1CompletedAt: null,
|
|
479
|
+
pass2CompletedAt: null,
|
|
480
|
+
migratedAt: null,
|
|
481
|
+
counts: {},
|
|
482
|
+
derivedProjects: [],
|
|
483
|
+
messageCount: 0,
|
|
484
|
+
entryCount: 0,
|
|
485
|
+
// Preserve reference to prior r5 state for audit.
|
|
486
|
+
priorR5: prior && prior.version === 'r5' ? { completedAt: prior.completedAt || null } : null,
|
|
487
|
+
};
|
|
488
|
+
saveState(yeaftDir, state);
|
|
489
|
+
}
|
|
490
|
+
|
|
491
|
+
try {
|
|
492
|
+
// Archive R5 state BEFORE any writes (or skip if already archived in a prior resume).
|
|
493
|
+
if (!state.legacyArchive) {
|
|
494
|
+
const archivePath = archiveR5State(yeaftDir);
|
|
495
|
+
state.legacyArchive = archivePath;
|
|
496
|
+
saveState(yeaftDir, state);
|
|
497
|
+
log('archive', { path: archivePath });
|
|
498
|
+
}
|
|
499
|
+
|
|
500
|
+
const vpDir = join(yeaftDir, 'memory', 'vp', vpId);
|
|
501
|
+
mkdirSync(vpDir, { recursive: true });
|
|
502
|
+
|
|
503
|
+
// Pass 1
|
|
504
|
+
if (!state.pass1CompletedAt) {
|
|
505
|
+
const p1 = runPass1({ yeaftDir, layout, vpDir, log, existingState: state });
|
|
506
|
+
state.counts = p1.counts;
|
|
507
|
+
state.entryCount = (state.entryCount || 0) + p1.migrated.filter(m => !m.skipped).length;
|
|
508
|
+
state.pass1CompletedAt = nowIso();
|
|
509
|
+
state.pass1Errors = p1.errors;
|
|
510
|
+
saveState(yeaftDir, state);
|
|
511
|
+
} else {
|
|
512
|
+
log('pass1', { skipped: true });
|
|
513
|
+
}
|
|
514
|
+
|
|
515
|
+
// Pass 2
|
|
516
|
+
if (!state.pass2CompletedAt) {
|
|
517
|
+
const derived = runPass2({ vpDir, counts: state.counts, log });
|
|
518
|
+
state.derivedProjects = derived.map(d => d.shard);
|
|
519
|
+
state.pass2CompletedAt = nowIso();
|
|
520
|
+
saveState(yeaftDir, state);
|
|
521
|
+
} else {
|
|
522
|
+
log('pass2', { skipped: true });
|
|
523
|
+
}
|
|
524
|
+
|
|
525
|
+
// Conversations (always run once — guarded by index existence).
|
|
526
|
+
if (!state.conversationsMigratedAt) {
|
|
527
|
+
const convRes = migrateConversations({ yeaftDir, layout, log });
|
|
528
|
+
state.messageCount = convRes.messages;
|
|
529
|
+
state.conversationsMigratedAt = nowIso();
|
|
530
|
+
saveState(yeaftDir, state);
|
|
531
|
+
} else {
|
|
532
|
+
log('conversations', { skipped: true });
|
|
533
|
+
}
|
|
534
|
+
|
|
535
|
+
state.migratedAt = nowIso();
|
|
536
|
+
saveState(yeaftDir, state);
|
|
537
|
+
log('done', { migratedAt: state.migratedAt });
|
|
538
|
+
return { status: 'done', state };
|
|
539
|
+
} catch (err) {
|
|
540
|
+
// On error: state file preserved so next run resumes. Archive untouched.
|
|
541
|
+
state.lastError = String(err && err.message || err);
|
|
542
|
+
saveState(yeaftDir, state);
|
|
543
|
+
throw err;
|
|
544
|
+
}
|
|
545
|
+
}
|
|
546
|
+
|
|
547
|
+
// ─── Rollback (deliverable G) ────────────────────────────────────
|
|
53
548
|
/**
|
|
54
|
-
*
|
|
55
|
-
*
|
|
549
|
+
* Roll back an R5→R6 migration. Restores from .legacy/r6-state.tar.gz and
|
|
550
|
+
* clears r6-specific state. Never touches the separate r5 archive created
|
|
551
|
+
* by v0-to-v1.js. Idempotent: safe to call when no r6 state is present.
|
|
56
552
|
*
|
|
57
|
-
* @param {object}
|
|
553
|
+
* @param {object} opts
|
|
554
|
+
* @param {string} opts.yeaftDir required
|
|
555
|
+
* @param {string} [opts.vpId] legacy VP id (default LEGACY_VP_ID)
|
|
556
|
+
* @param {(step, info)=>void} [opts.onStep]
|
|
58
557
|
*/
|
|
59
|
-
export async function
|
|
60
|
-
|
|
558
|
+
export async function rollbackR5ToR6Migration(opts = {}) {
|
|
559
|
+
const { yeaftDir, onStep } = opts;
|
|
560
|
+
const vpId = opts.vpId || LEGACY_VP_ID;
|
|
561
|
+
if (!yeaftDir || typeof yeaftDir !== 'string') {
|
|
562
|
+
throw new Error('rollbackR5ToR6Migration: yeaftDir required');
|
|
563
|
+
}
|
|
564
|
+
const log = typeof onStep === 'function' ? onStep : () => {};
|
|
565
|
+
const state = loadState(yeaftDir);
|
|
566
|
+
if (!state || state.version !== 'r6') {
|
|
567
|
+
log('noop', { reason: 'no r6 state file present' });
|
|
568
|
+
return { status: 'noop' };
|
|
569
|
+
}
|
|
570
|
+
const archivePath = state.legacyArchive;
|
|
571
|
+
if (!archivePath || !existsSync(archivePath)) {
|
|
572
|
+
throw new Error(`rollbackR5ToR6Migration: archive missing at ${archivePath}`);
|
|
573
|
+
}
|
|
574
|
+
|
|
575
|
+
// Delete R6-specific paths first (only what this migration created).
|
|
576
|
+
const vpDir = join(yeaftDir, 'memory', 'vp', vpId);
|
|
577
|
+
if (existsSync(vpDir)) {
|
|
578
|
+
rmSync(vpDir, { recursive: true, force: true });
|
|
579
|
+
log('rm-vp-memory', { path: vpDir });
|
|
580
|
+
}
|
|
581
|
+
const groupsDir = join(yeaftDir, 'groups', LEGACY_GROUP_ID, 'messages');
|
|
582
|
+
if (existsSync(groupsDir)) {
|
|
583
|
+
rmSync(groupsDir, { recursive: true, force: true });
|
|
584
|
+
log('rm-group-messages', { path: groupsDir });
|
|
585
|
+
}
|
|
586
|
+
|
|
587
|
+
// Restore archive back to yeaftDir. tar -xzf will overwrite paths it owns.
|
|
588
|
+
// Only do this if archive is non-empty (empty marker = nothing was archived).
|
|
589
|
+
const sz = statSync(archivePath).size;
|
|
590
|
+
if (sz > 0) {
|
|
591
|
+
execFileSync('tar', ['-xzf', archivePath, '-C', yeaftDir], {
|
|
592
|
+
stdio: ['ignore', 'ignore', 'pipe'],
|
|
593
|
+
});
|
|
594
|
+
log('restore', { from: archivePath });
|
|
595
|
+
} else {
|
|
596
|
+
log('restore', { from: archivePath, note: 'empty archive — nothing to restore' });
|
|
597
|
+
}
|
|
598
|
+
|
|
599
|
+
// Downgrade state to r5 marker (archive left on disk for audit).
|
|
600
|
+
const newState = {
|
|
601
|
+
version: 'r5',
|
|
602
|
+
rolledBackAt: nowIso(),
|
|
603
|
+
previousR6: {
|
|
604
|
+
migratedAt: state.migratedAt,
|
|
605
|
+
legacyArchive: state.legacyArchive,
|
|
606
|
+
},
|
|
607
|
+
};
|
|
608
|
+
saveState(yeaftDir, newState);
|
|
609
|
+
log('done', { rolledBackAt: newState.rolledBackAt });
|
|
610
|
+
return { status: 'done', state: newState };
|
|
61
611
|
}
|