@aiwg/cli 2026.10.11 → 2026.10.12

package/dist/src/storage/cli.js

@@ -15,12 +15,10 @@
  * @issue #955
  */
 import { randomUUID } from 'crypto';
-import { existsSync } from 'fs';
-import { mkdir, readFile, writeFile, appendFile } from 'fs/promises';
-import { dirname, extname, join, resolve as resolvePath } from 'path';
-import { parseFrontmatter } from '../artifacts/index-builder.js';
-import { BACKEND_TYPES, FilesystemAdapter, ObsidianAdapter, LogseqAdapter, FortemiAdapter, SUBSYSTEM_KEYS, getLoadedConfig, initStorage, resolveStorage, resolveSubsystemRoot, storageConfigPath, } from './index.js';
-import { projectAiwgPath, resolveProjectAiwgDir } from '../config/project-artifacts.js';
+import { handleMigrate } from './cli-migrate.js';
+export { isTextCorpusEntry } from './migration-adapter.js';
+import { BACKEND_TYPES, SUBSYSTEM_KEYS, getLoadedConfig, initStorage, resolveStorage, resolveSubsystemRoot, storageConfigPath, } from './index.js';
+import { resolveProjectAiwgDir } from '../config/project-artifacts.js';
 export async function main(args, projectRoot = process.cwd()) {
     const subcommand = args[0];
     const subArgs = args.slice(1);
@@ -141,166 +139,6 @@ async function handleTest(projectRoot, args) {
             `  Subsystem: ${subsystem}\n  Probe path: ${probePath}`);
     }
 }
-/**
- * `aiwg storage migrate <subsystem> --from <spec> --to <spec> [--dry-run]`
- *
- * Spec format: `<type>:<location>` where type is one of fs/obsidian/
- * logseq/fortemi and location is:
- *   fs       → directory path (relative to projectRoot or absolute)
- *   obsidian → vault directory (folder via `--from-folder`/`--to-folder`)
- *   logseq   → graph directory
- *   fortemi  → MCP server name (default: 'fortemi')
- *
- * Examples:
- *   aiwg storage migrate memory --from fs:.aiwg/memory --to obsidian:~/vault --to-folder AIWG/memory
- *   aiwg storage migrate kb --from fs:.aiwg/kb --to fortemi:fortemi --dry-run
- *
- * Walks source.list(''), reads each entry, writes to destination. Tracks
- * progress in `.aiwg/.storage-cache/migrations/<id>.jsonl` so re-running
- * resumes from the last successfully-migrated entry.
- *
- * @issue #955
- */
-async function handleMigrate(projectRoot, args) {
-    const opts = parseMigrateArgs(args);
-    const source = buildAdapter(opts.subsystem, opts.from, projectRoot);
-    const destination = buildAdapter(opts.subsystem, opts.to, projectRoot);
-    if (locationsEqual(opts.from, opts.to, projectRoot)) {
-        throw new Error(`Refusing to migrate: source and destination resolve to the same location.\n  ${describeSpec(opts.from)}\n  ${describeSpec(opts.to)}`);
-    }
-    if (source.init)
-        await source.init();
-    if (!opts.dryRun && destination.init)
-        await destination.init();
-    console.log(`storage migrate (${opts.dryRun ? 'DRY RUN' : 'live'})`);
-    console.log(`  subsystem: ${opts.subsystem}`);
-    console.log(`  from:      ${describeSpec(opts.from)}`);
-    console.log(`  to:        ${describeSpec(opts.to)}`);
-    console.log('');
-    const entries = await source.list('');
-    if (entries.length === 0) {
-        console.log('No entries to migrate.');
-        return;
-    }
-    // Resume support: read the per-migration log; skip already-done paths
-    const migrationLogPath = projectAiwgPath(projectRoot, '.storage-cache', 'migrations', `${opts.subsystem}-${specSlug(opts.from)}-to-${specSlug(opts.to)}.jsonl`);
-    const completed = await readCompletedSet(migrationLogPath);
-    let copied = 0;
-    let skipped = 0;
-    let errored = 0;
-    let unsupported = 0;
-    for (const entry of entries) {
-        if (opts.textOnly && !isTextCorpusEntry(entry.path)) {
-            unsupported++;
-            console.log(`  · ${entry.path} (non-text attachment skipped)`);
-            continue;
-        }
-        if (completed.has(entry.path)) {
-            skipped++;
-            console.log(`  ✓ ${entry.path} (already migrated, skipped)`);
-            continue;
-        }
-        if (opts.dryRun) {
-            copied++;
-            console.log(`  → ${entry.path} (would copy)`);
-            continue;
-        }
-        try {
-            const content = await source.read(entry.path);
-            if (content === null) {
-                errored++;
-                console.log(`  ✗ ${entry.path} (read returned null)`);
-                continue;
-            }
-            await destination.write(entry.path, content, migrationMetadata(entry.path, content));
-            await recordCompletion(migrationLogPath, entry.path);
-            copied++;
-            console.log(`  ✓ ${entry.path}`);
-        }
-        catch (err) {
-            errored++;
-            console.log(`  ✗ ${entry.path} (${err.message})`);
-        }
-    }
-    if (source.close)
-        await source.close();
-    if (destination.close)
-        await destination.close();
-    console.log('');
-    console.log(`Summary: copied=${copied} skipped=${skipped} unsupported=${unsupported} ` +
-        `errored=${errored} total=${entries.length}`);
-    if (!opts.dryRun) {
-        console.log(`Migration log: ${migrationLogPath}`);
-    }
-    if (errored > 0) {
-        throw new Error(`${errored} entr${errored === 1 ? 'y' : 'ies'} failed to migrate.`);
-    }
-}
-function parseMigrateArgs(args) {
-    let subsystem;
-    let from;
-    let to;
-    let fromFolder;
-    let toFolder;
-    let dryRun = false;
-    let textOnly = false;
-    for (let i = 0; i < args.length; i++) {
-        const a = args[i];
-        if (a === '--from')
-            from = args[++i];
-        else if (a === '--to')
-            to = args[++i];
-        else if (a === '--from-folder')
-            fromFolder = args[++i];
-        else if (a === '--to-folder')
-            toFolder = args[++i];
-        else if (a === '--dry-run')
-            dryRun = true;
-        else if (a === '--text-only')
-            textOnly = true;
-        else if (!a.startsWith('--') && !subsystem)
-            subsystem = a;
-        else
-            throw new Error(`Unknown migrate flag: ${a}`);
-    }
-    if (!subsystem || !SUBSYSTEM_KEYS.includes(subsystem)) {
-        throw new Error(`Usage: aiwg storage migrate <subsystem> --from <type>:<location> --to <type>:<location> [--dry-run]\n  Valid subsystems: ${SUBSYSTEM_KEYS.join(', ')}`);
-    }
-    if (!from)
-        throw new Error('storage migrate: --from <type>:<location> is required');
-    if (!to)
-        throw new Error('storage migrate: --to <type>:<location> is required');
-    return {
-        subsystem: subsystem,
-        from: { ...parseSpec(from), ...(fromFolder ? { folder: fromFolder } : {}) },
-        to: { ...parseSpec(to), ...(toFolder ? { folder: toFolder } : {}) },
-        dryRun,
-        textOnly,
-    };
-}
-const TEXT_CORPUS_EXTENSIONS = new Set([
-    '.bib',
-    '.csv',
-    '.htm',
-    '.html',
-    '.json',
-    '.md',
-    '.ris',
-    '.txt',
-    '.xml',
-    '.yaml',
-    '.yml',
-]);
-export function isTextCorpusEntry(entryPath) {
-    return TEXT_CORPUS_EXTENSIONS.has(extname(entryPath).toLowerCase());
-}
-function migrationMetadata(entryPath, content) {
-    const extension = extname(entryPath).toLowerCase();
-    const contentType = extension === '.md' ? 'text/markdown' : 'text/plain';
-    if (extension !== '.md')
-        return { contentType };
-    return { contentType, frontmatter: parseFrontmatter(content).data };
-}
 async function handleImportCorpus(projectRoot, args) {
     let server = 'fortemi';
     let destination;
@@ -342,119 +180,6 @@ async function handleImportCorpus(projectRoot, args) {
         ...(dryRun ? ['--dry-run'] : []),
     ]);
 }
-function parseSpec(raw) {
-    const idx = raw.indexOf(':');
-    if (idx === -1) {
-        throw new Error(`Invalid backend spec "${raw}" — expected <type>:<location>`);
-    }
-    const type = raw.slice(0, idx);
-    const location = raw.slice(idx + 1);
-    if (!BACKEND_TYPES.includes(type)) {
-        throw new Error(`Unknown backend type "${type}" in spec "${raw}". Valid: ${BACKEND_TYPES.join(', ')}`);
-    }
-    if (location.length === 0) {
-        throw new Error(`Empty location in backend spec "${raw}"`);
-    }
-    return { type: type, location };
-}
-function buildAdapter(subsystem, spec, projectRoot) {
-    switch (spec.type) {
-        case 'fs': {
-            const root = expandFsLocation(spec.location, projectRoot);
-            return new FilesystemAdapter(root);
-        }
-        case 'obsidian':
-            return new ObsidianAdapter({
-                type: 'obsidian',
-                vault: expandFsLocation(spec.location, projectRoot),
-                ...(spec.folder ? { folder: spec.folder } : {}),
-                useCli: false,
-            });
-        case 'logseq':
-            return new LogseqAdapter({
-                type: 'logseq',
-                graph: expandFsLocation(spec.location, projectRoot),
-                useApi: false,
-            });
-        case 'fortemi':
-            return new FortemiAdapter({
-                subsystem,
-                config: { type: 'fortemi', mcpServer: spec.location },
-            });
-        case 'notion':
-        case 'anythingllm':
-        case 's3':
-        case 'webdav':
-            throw new Error(`storage migrate: backend "${spec.type}" not yet implemented. ` +
-                `See issues #959, #960, #962, #963 for tracking.`);
-        default: {
-            const _exhaustive = spec.type;
-            void _exhaustive;
-            throw new Error(`Unhandled backend type ${spec.type}`);
-        }
-    }
-}
-function expandFsLocation(loc, projectRoot) {
-    if (loc.startsWith('~/')) {
-        const { homedir } = require('os');
-        return join(homedir(), loc.slice(2));
-    }
-    if (loc === '~') {
-        const { homedir } = require('os');
-        return homedir();
-    }
-    if (loc.startsWith('/'))
-        return loc;
-    return resolvePath(projectRoot, loc);
-}
-function locationsEqual(a, b, projectRoot) {
-    if (a.type !== b.type)
-        return false;
-    if (a.type === 'fortemi')
-        return a.location === b.location;
-    // For file-shaped backends, compare resolved fs paths
-    return expandFsLocation(a.location, projectRoot) === expandFsLocation(b.location, projectRoot);
-}
-function describeSpec(spec) {
-    return spec.folder ? `${spec.type}:${spec.location} (folder=${spec.folder})` : `${spec.type}:${spec.location}`;
-}
-function specSlug(spec) {
-    return `${spec.type}-${spec.location.replace(/[^a-zA-Z0-9]/g, '_').slice(0, 32)}`;
-}
-async function readCompletedSet(logPath) {
-    if (!existsSync(logPath))
-        return new Set();
-    try {
-        const content = await readFile(logPath, 'utf-8');
-        const out = new Set();
-        for (const line of content.split('\n')) {
-            if (!line.trim())
-                continue;
-            try {
-                const obj = JSON.parse(line);
-                if (obj.path)
-                    out.add(obj.path);
-            }
-            catch {
-                /* ignore malformed line */
-            }
-        }
-        return out;
-    }
-    catch {
-        return new Set();
-    }
-}
-async function recordCompletion(logPath, path) {
-    await mkdir(dirname(logPath), { recursive: true });
-    const line = JSON.stringify({ path, ts: new Date().toISOString() }) + '\n';
-    if (existsSync(logPath)) {
-        await appendFile(logPath, line, 'utf-8');
-    }
-    else {
-        await writeFile(logPath, line, 'utf-8');
-    }
-}
 function describeBackendLocation(subsystem, config) {
     const b = config?.backends?.[subsystem];
     if (!b)

package/dist/src/storage/migration-adapter.js

@@ -0,0 +1,79 @@
+// Bridges text storage adapters to protocol records and single-record commits.
+import { extname } from 'node:path';
+import { randomUUID } from 'node:crypto';
+import { parseFrontmatter } from '../artifacts/index-builder.js';
+import { digestRecords, sha256 } from './migration-records.js';
+export function migrationEndpoint(adapter, identity) {
+    const readRecord = async (path) => {
+        const value = await adapter.read(path);
+        if (value === null)
+            return null;
+        const digest = sha256(value);
+        return { identity: { tenant: identity.tenant, subsystem: identity.subsystem, path }, value, digest, sourceRevision: digest };
+    };
+    const readAll = async () => {
+        const records = [];
+        for (const { path } of await adapter.list('')) {
+            const record = await readRecord(path);
+            if (record === null)
+                throw new Error(`source changed during enumeration: ${path} (read returned null)`);
+            records.push(record);
+        }
+        return records;
+    };
+    return {
+        identity, readRecord, readAll,
+        async snapshot() {
+            const records = await readAll();
+            return { id: randomUUID(), highWaterMark: digestRecords(records), records };
+        },
+        async commitBatch(mutations) {
+            // Baseline adapters provide single-path idempotent writes, not multi-path transactions.
+            if (mutations.length !== 1)
+                throw new Error('storage adapter copy requires single-record batches');
+            const mutation = mutations[0];
+            const path = mutation.record.identity.path;
+            if ((await readRecord(path))?.sourceRevision !== mutation.expectedRevision) {
+                throw new Error(`destination changed before commit: ${path}`);
+            }
+            if (mutation.operation === 'delete')
+                await adapter.delete(path);
+            else
+                await adapter.write(path, mutation.record.value, migrationMetadata(path, mutation.record.value));
+            const committed = await readRecord(path);
+            if (mutation.operation === 'delete' && committed !== null)
+                throw new Error(`delete not applied: ${path}`);
+            if (mutation.operation === 'upsert' && committed === null)
+                throw new Error(`write not applied: ${path}`);
+            return {
+                batchId: randomUUID(), committed: true, highWaterMark: mutation.record.sourceRevision,
+                recordReceipts: [{ identity: mutation.record.identity, digest: mutation.record.digest,
+                        sourceRevision: committed?.sourceRevision ?? mutation.record.sourceRevision }],
+            };
+        },
+    };
+}
+const TEXT_CORPUS_EXTENSIONS = new Set([
+    '.bib',
+    '.csv',
+    '.htm',
+    '.html',
+    '.json',
+    '.md',
+    '.ris',
+    '.txt',
+    '.xml',
+    '.yaml',
+    '.yml',
+]);
+export function isTextCorpusEntry(entryPath) {
+    return TEXT_CORPUS_EXTENSIONS.has(extname(entryPath).toLowerCase());
+}
+export function migrationMetadata(entryPath, content) {
+    const extension = extname(entryPath).toLowerCase();
+    const contentType = extension === '.md' ? 'text/markdown' : 'text/plain';
+    if (extension !== '.md')
+        return { contentType };
+    return { contentType, frontmatter: parseFrontmatter(content).data };
+}
+//# sourceMappingURL=migration-adapter.js.map

package/dist/src/storage/migration-copy.js

@@ -0,0 +1,108 @@
+// Plans and checkpoints copy-only migrations without changing backend routing.
+import { randomUUID } from 'node:crypto';
+import { STORAGE_MIGRATION_PROTOCOL, MigrationProtocolError, } from './migration-types.js';
+import { commitMigrationRecords, sha256, stableStringify } from './migration-records.js';
+export async function runMigrationCopy(request) {
+    const { source, destination } = request;
+    const prior = request.checkpoint;
+    if (prior && (prior.protocol !== STORAGE_MIGRATION_PROTOCOL || !prior.migrationId ||
+        stableStringify(prior.source) !== stableStringify(source.identity) ||
+        stableStringify(prior.destination) !== stableStringify(destination.identity))) {
+        throw new MigrationProtocolError('AIWG_MIGRATION_CHECKPOINT_MISMATCH', 'copy checkpoint does not match migration endpoints');
+    }
+    const latest = new Map(prior?.records.map(record => [record.identity.path, record]));
+    const sourceRecords = (await source.snapshot()).records;
+    // Preview must remain usable without connecting to the destination (e.g. corpus ingest).
+    const destinationRecords = request.dryRun ? [] : (await destination.snapshot()).records;
+    const sources = new Map(sourceRecords.map(record => [record.identity.path, record]));
+    const destinations = new Map(destinationRecords.map(record => [record.identity.path, record]));
+    const plan = [];
+    for (const record of sourceRecords) {
+        const path = record.identity.path;
+        const previous = latest.get(path);
+        const current = destinations.get(path);
+        const excluded = request.exclude?.(path);
+        const unchanged = previous && !previous.tombstone && previous.contentDigest === record.digest &&
+            previous.sourceRevision === record.sourceRevision &&
+            (request.dryRun || current?.sourceRevision === previous.destinationRevision);
+        plan.push({
+            path, sourceDigest: record.digest, destinationRevision: current?.sourceRevision,
+            action: excluded || unchanged ? 'skipped' : previous ? 'recopied' : 'copied',
+            reason: excluded ?? (unchanged ? 'already migrated, skipped' : previous ?
+                previous.contentDigest !== record.digest || previous.sourceRevision !== record.sourceRevision ? 'source changed since checkpoint' : 'destination changed since checkpoint' :
+                'not yet migrated'),
+        });
+    }
+    for (const record of destinationRecords) {
+        const path = record.identity.path;
+        if (sources.has(path))
+            continue;
+        const previous = latest.get(path);
+        const excluded = request.exclude?.(path);
+        const authorized = !excluded && previous !== undefined && !previous.tombstone &&
+            record.sourceRevision === previous.destinationRevision;
+        plan.push({
+            path, action: authorized ? 'deleted' : 'skipped', deleteAuthorized: authorized,
+            destinationRevision: record.sourceRevision,
+            reason: excluded ?? (authorized ? 'source removed; unchanged migration-owned destination' :
+                !previous || previous.tombstone ? 'delete not authorized: destination has no live migration receipt' :
+                    'delete not authorized: destination changed since checkpoint'),
+        });
+    }
+    plan.sort((a, b) => a.path.localeCompare(b.path));
+    const checkpoint = {
+        protocol: STORAGE_MIGRATION_PROTOCOL, migrationId: prior?.migrationId ?? randomUUID(),
+        source: source.identity, destination: destination.identity,
+        records: structuredClone(prior?.records ?? []), plan,
+        receipt: { copied: [], recopied: [], deleted: [], skipped: [], errored: [] },
+    };
+    if (!request.dryRun)
+        await request.save(structuredClone(checkpoint));
+    for (const item of plan) {
+        let action = item.action;
+        let reason = item.reason;
+        if (!request.dryRun && action !== 'skipped') {
+            try {
+                const sourceNow = await source.readRecord(item.path);
+                const destinationNow = await destination.readRecord(item.path);
+                if (destinationNow?.sourceRevision !== item.destinationRevision) {
+                    action = 'skipped';
+                    reason = 'destination changed after plan; retry to replan';
+                }
+                else if (item.action === 'deleted' && (!item.deleteAuthorized || sourceNow)) {
+                    action = 'skipped';
+                    reason = 'delete not authorized: source reappeared or plan denies deletion';
+                }
+                else if (item.action !== 'deleted' && sourceNow?.tombstone) {
+                    action = 'skipped';
+                    reason = 'delete not authorized: source tombstone is not a planned delete';
+                }
+                else if (item.action !== 'deleted' && sourceNow?.digest !== item.sourceDigest) {
+                    action = 'skipped';
+                    reason = 'source changed after plan; retry to replan';
+                }
+                else {
+                    const record = item.action === 'deleted' ? {
+                        identity: destinationNow.identity, sourceRevision: `deleted:${item.destinationRevision}`,
+                        digest: sha256(`deleted:${item.destinationRevision}`),
+                        tombstone: { deletedAt: new Date().toISOString(), reason },
+                    } : sourceNow;
+                    const committed = await commitMigrationRecords(`${checkpoint.migrationId}:${checkpoint.records.length}`, [record], () => item.destinationRevision, mutations => destination.commitBatch(mutations), () => new Date());
+                    checkpoint.records.push(...committed.records);
+                }
+            }
+            catch (error) {
+                action = 'errored';
+                reason = error instanceof Error ? error.message : String(error);
+            }
+        }
+        const receiptItem = { path: item.path, reason };
+        checkpoint.receipt[action].push(receiptItem);
+        // A failed checkpoint write stops the run: no later writes may outrun durable progress.
+        if (!request.dryRun)
+            await request.save(structuredClone(checkpoint));
+        await request.report?.(action, receiptItem);
+    }
+    return checkpoint;
+}
+//# sourceMappingURL=migration-copy.js.map