crewhaus 0.1.8 → 0.2.0

package/dist/spec-changelog.js

@@ -0,0 +1,237 @@
+/**
+ * Item 46 — auto-register spec versions with changelogs distilled from
+ * patch history. Shared by `crewhaus compile` (post-emission), `crewhaus
+ * optimize --write-back` (post-write-back) and `crewhaus spec put`:
+ *
+ *   - `autoRegisterSpecVersion` content-hashes the spec and, when the
+ *     local registry has no version with that content, `put`s the next
+ *     `vN` — so registry state tracks working files without a manual
+ *     `crewhaus spec put`. Recompiling an unchanged spec is a no-op.
+ *   - every auto/manual put appends a distilled entry to a per-spec
+ *     `CHANGELOG.md` stored BESIDE the registry manifest
+ *     (`.crewhaus/specs/<name>/CHANGELOG.md` — a NEW file; the registry's
+ *     own on-disk format is untouched): version, date, the field-level
+ *     YAML diff vs the previous version, and — when the content carries a
+ *     `formatWriteBackHeader` stamp — the optimizer's runId/mutator/score
+ *     plus the SpecPatch `rationale` from that run's `patch.json`.
+ *
+ * Kept in a side-effect-free module (nothing runs on import) so the logic
+ * is unit-testable — the CLI entry file runs an argv switch on import.
+ */
+import { createHash } from "node:crypto";
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { diffSpecYaml, parseWriteBackHeader, } from "@crewhaus/spec-patch";
+/** The per-spec changelog file, beside the registry's `manifest.json`. */
+export const CHANGELOG_FILE = "CHANGELOG.md";
+// The registry's own name floor (spec-registry NAME_REGEX). Spec names may
+// additionally contain spaces and ':' (the spec `safeName` grammar), so
+// auto-registration maps a display name onto this grammar deterministically.
+const REGISTRY_NAME_REGEX = /^[A-Za-z0-9_-][A-Za-z0-9_.-]*$/;
+// A write-back runId is read from a COMMENT in the spec being registered;
+// refuse to path-join anything that could escape the optimize root.
+const SAFE_RUN_ID_REGEX = /^[A-Za-z0-9_-]+$/;
+/**
+ * Map a spec's display name (spec `safeName`: letters, digits, spaces,
+ * `_ . - :`) onto the registry's stricter name grammar: runs of characters
+ * outside `[A-Za-z0-9_.-]` collapse to a single `-`, leading dots are
+ * stripped (the registry hides dot-dirs), and an empty result falls back to
+ * `"spec"`. Deterministic, so the same spec always lands in the same slot.
+ */
+export function registrySpecName(specName) {
+    const cleaned = specName.replace(/[^A-Za-z0-9_.-]+/g, "-").replace(/^\.+/, "");
+    const name = cleaned.length > 0 ? cleaned : "spec";
+    // Defensive: the transform above always satisfies the registry grammar,
+    // but verify so a future grammar drift fails loudly here, not deep in fs.
+    if (!REGISTRY_NAME_REGEX.test(name))
+        return "spec";
+    return name;
+}
+/** SHA-256 hex of the exact YAML text (the registry stores bytes verbatim). */
+export function contentHash(yaml) {
+    return createHash("sha256").update(yaml).digest("hex");
+}
+/**
+ * Next auto-version: `v<max+1>` over the existing `v<N>` versions (manifest
+ * order, numeric — `v10` beats `v2`). Manually-put versions that don't match
+ * `v<N>` (e.g. `2.0`, `release-1`) are ignored for numbering.
+ */
+export function nextVersion(existing) {
+    let max = 0;
+    for (const v of existing) {
+        const m = /^v(\d+)$/.exec(v);
+        if (m?.[1] === undefined)
+            continue;
+        const n = Number.parseInt(m[1], 10);
+        if (n > max)
+            max = n;
+    }
+    return `v${max + 1}`;
+}
+/**
+ * When the YAML carries a `formatWriteBackHeader` stamp, recover the run
+ * metadata and — if that run's `patch.json` is readable — its `rationale`.
+ * Returns `undefined` for content that was never written back. Never throws:
+ * provenance is best-effort garnish on the changelog, not a gate.
+ */
+export function extractOptimizeMetadata(opts) {
+    const header = parseWriteBackHeader(opts.yaml);
+    if (header === undefined)
+        return undefined;
+    let patchJsonPath = opts.patchJsonPath;
+    if (patchJsonPath === undefined &&
+        opts.optimizeRootDir !== undefined &&
+        SAFE_RUN_ID_REGEX.test(header.runId)) {
+        patchJsonPath = join(opts.optimizeRootDir, header.runId, "patch.json");
+    }
+    let rationale;
+    if (patchJsonPath !== undefined && existsSync(patchJsonPath)) {
+        try {
+            const parsed = JSON.parse(readFileSync(patchJsonPath, "utf-8"));
+            if (typeof parsed.rationale === "string" && parsed.rationale.length > 0) {
+                rationale = parsed.rationale;
+            }
+        }
+        catch {
+            // Unreadable/malformed patch.json — keep the header fields alone.
+        }
+    }
+    return { ...header, ...(rationale !== undefined ? { rationale } : {}) };
+}
+/**
+ * Render one changelog entry (pure): `## <version> — <date>` followed by
+ * the field-level diff bullets vs the previous version (or `initial
+ * version`) and, when present, the optimizer provenance + rationale.
+ */
+export function renderChangelogEntry(opts) {
+    const date = (opts.now ?? new Date()).toISOString().slice(0, 10);
+    const lines = [`## ${opts.version} — ${date}`, ""];
+    if (opts.previousYaml === undefined) {
+        lines.push("- initial version");
+    }
+    else {
+        let diff;
+        try {
+            diff = diffSpecYaml(opts.previousYaml, opts.yaml);
+        }
+        catch (err) {
+            lines.push(`- diff unavailable (${err.message})`);
+        }
+        if (diff !== undefined) {
+            if (diff.length === 0) {
+                lines.push("- no structural changes (comments/formatting only)");
+            }
+            else {
+                for (const d of diff)
+                    lines.push(formatDiffLine(d));
+            }
+        }
+    }
+    const meta = extractOptimizeMetadata(opts);
+    if (meta !== undefined) {
+        const details = [];
+        if (meta.mutator !== undefined)
+            details.push(`mutator ${meta.mutator}`);
+        if (meta.iterations !== undefined)
+            details.push(`${meta.iterations} iteration(s)`);
+        if (meta.scoreBefore !== undefined && meta.scoreAfter !== undefined) {
+            details.push(`score ${meta.scoreBefore.toFixed(3)} → ${meta.scoreAfter.toFixed(3)}`);
+        }
+        const suffix = details.length > 0 ? ` (${details.join(", ")})` : "";
+        lines.push(`- optimizer: runId ${meta.runId}${suffix}`);
+        if (meta.rationale !== undefined)
+            lines.push(`- rationale: ${meta.rationale}`);
+    }
+    lines.push("");
+    return lines.join("\n");
+}
+function formatDiffLine(d) {
+    switch (d.kind) {
+        case "added":
+            return `- added \`${d.path}\`: ${d.after ?? ""}`;
+        case "removed":
+            return `- removed \`${d.path}\`: ${d.before ?? ""}`;
+        case "changed":
+            return `- changed \`${d.path}\`: ${d.before ?? ""} → ${d.after ?? ""}`;
+    }
+}
+/**
+ * Insert an entry into a changelog text, NEWEST FIRST (pure): the new entry
+ * lands above the previous top `## ` heading, under the `# Changelog — …`
+ * title. `crewhaus spec log` can therefore print the file verbatim and read
+ * most-recent-first.
+ */
+export function insertNewestFirst(existing, name, entry) {
+    if (existing.trim() === "") {
+        return `# Changelog — ${name}\n\n${entry}`;
+    }
+    const firstHeading = /^## /m.exec(existing);
+    if (firstHeading === null) {
+        // A title-only (or hand-edited) file: append below whatever is there.
+        return `${existing.trimEnd()}\n\n${entry}`;
+    }
+    return `${existing.slice(0, firstHeading.index)}${entry}\n${existing.slice(firstHeading.index)}`;
+}
+/**
+ * Append (newest-first) one entry to `<registryRoot>/<name>/CHANGELOG.md`,
+ * creating the file with its title on first use. Returns the file path.
+ */
+export function appendChangelogEntry(opts) {
+    const dir = join(opts.registryRootDir, opts.name);
+    mkdirSync(dir, { recursive: true });
+    const path = join(dir, CHANGELOG_FILE);
+    const existing = existsSync(path) ? readFileSync(path, "utf-8") : "";
+    const entry = renderChangelogEntry(opts);
+    writeFileSync(path, insertNewestFirst(existing, opts.name, entry), { mode: 0o600 });
+    return path;
+}
+/**
+ * Content-hash gated auto-put: when some stored version already carries this
+ * exact content, report `unchanged` (with that version); otherwise `put` the
+ * next `vN` and append the distilled changelog entry. The previous-latest
+ * version (manifest order) is the diff baseline.
+ */
+export async function autoRegisterSpecVersion(opts) {
+    const name = registrySpecName(opts.specName);
+    const hash = contentHash(opts.yaml);
+    const manifest = await opts.registry.manifest(name);
+    // Scan ALL stored versions for this content — re-registering any past
+    // version (e.g. a rollback restored on disk) is also a no-op. Later
+    // matches win so the reported version is the most recent occurrence.
+    let matched;
+    for (const v of manifest.versions) {
+        try {
+            if (contentHash(await opts.registry.get(name, v)) === hash)
+                matched = v;
+        }
+        catch {
+            // A manifest-listed version whose file vanished: ignore for matching.
+        }
+    }
+    if (matched !== undefined) {
+        return { status: "unchanged", name, version: matched };
+    }
+    const previousVersion = manifest.versions[manifest.versions.length - 1];
+    let previousYaml;
+    if (previousVersion !== undefined) {
+        try {
+            previousYaml = await opts.registry.get(name, previousVersion);
+        }
+        catch {
+            // Baseline unavailable — the entry renders as diff-less.
+        }
+    }
+    const version = nextVersion(manifest.versions);
+    await opts.registry.put(name, version, opts.yaml);
+    appendChangelogEntry({
+        registryRootDir: opts.registryRootDir,
+        name,
+        version,
+        yaml: opts.yaml,
+        ...(previousYaml !== undefined ? { previousYaml } : {}),
+        ...(opts.optimizeRootDir !== undefined ? { optimizeRootDir: opts.optimizeRootDir } : {}),
+        ...(opts.patchJsonPath !== undefined ? { patchJsonPath: opts.patchJsonPath } : {}),
+        ...(opts.now !== undefined ? { now: opts.now } : {}),
+    });
+    return { status: "registered", name, version };
+}

package/dist/state-backup.d.ts

@@ -0,0 +1,223 @@
+import { type FeedbackRecord } from "./feedback";
+/** Manifest file name at the archive root (NOT restored into `.crewhaus`). */
+export declare const MANIFEST_FILENAME = "backup-manifest.json";
+/** Thrown for every operational failure (missing dir, tar failure, bad
+ *  archive). The CLI entry file catches it and routes the message through
+ *  `die()`; tests assert on `.message` without the process exiting. The
+ *  `name` stays a plain string so `StateRestoreRefusedError` can narrow it. */
+export declare class StateBackupError extends Error {
+    readonly name: string;
+}
+/** Thrown when `restore` (without --force/--merge) would overwrite an
+ *  existing non-empty `.crewhaus`. Carries a preformatted message listing
+ *  what is there and how to proceed. */
+export declare class StateRestoreRefusedError extends StateBackupError {
+    readonly stateDir: string;
+    readonly name = "StateRestoreRefusedError";
+    constructor(stateDir: string, entries: ReadonlyArray<{
+        name: string;
+        kind: "dir" | "file";
+        files: number;
+    }>);
+}
+export type SubdirStat = {
+    files: number;
+    bytes: number;
+};
+export type BackupManifest = {
+    schemaVersion: 1;
+    /** ISO 8601 creation timestamp. */
+    createdAt: string;
+    /** Absolute path of the `.crewhaus` dir that was archived. */
+    sourceDir: string;
+    /** CLI version that produced the backup. */
+    crewhausVersion: string;
+    totals: SubdirStat;
+    /** Per-top-level-subdir file counts + byte sizes; "." = state-dir root files. */
+    subdirs: Record<string, SubdirStat>;
+    /** false when any sqlite file could not be snapshotted consistently and was
+     *  byte-copied raw instead (a live writer may have torn the copy). */
+    sqliteConsistent: boolean;
+    sqlite: {
+        /** rel paths archived as a consistent read-only `serialize()` snapshot. */
+        snapshotted: string[];
+        /** rel paths byte-copied raw because the snapshot failed. */
+        copiedRaw: string[];
+        /** `-wal`/`-shm`/`-journal` sidecars omitted because their main db was
+         *  snapshotted (the snapshot folds WAL content in). */
+        foldedSidecars: string[];
+    };
+};
+/** One file as it lands in the archive (post-snapshot sizes). */
+export type ArchivedFile = {
+    relPath: string;
+    bytes: number;
+};
+/** Pure manifest assembly from the archived file list + backup metadata. */
+export declare function buildManifest(files: ReadonlyArray<ArchivedFile>, meta: {
+    createdAt: string;
+    sourceDir: string;
+    crewhausVersion: string;
+    sqliteConsistent: boolean;
+    sqlite: BackupManifest["sqlite"];
+}): BackupManifest;
+/** Narrow a parsed JSON value to a BackupManifest (tolerant structural check —
+ *  a hand-made or future-versioned manifest should degrade to a warning, not
+ *  a crash). */
+export declare function isBackupManifest(value: unknown): value is BackupManifest;
+/** Collapse a harness name / dir basename to a filename-safe label. */
+export declare function sanitizeBackupLabel(label: string): string;
+/** Default tarball name: `crewhaus-state-<label>-<ISO date>.tar.gz`. */
+export declare function defaultBackupFileName(label: string, now: Date): string;
+/** Split a comma-separated `--exclude` value into trimmed non-empty globs. */
+export declare function parseExcludeGlobs(raw: string | undefined): string[];
+/**
+ * Whether a state-dir-relative path (always "/"-separated) matches any
+ * exclude glob. A glob is matched against the full relative path; a bare
+ * pattern (no "/") additionally matches any single path segment, mirroring
+ * tar's basename `--exclude` ergonomics — so `sessions` drops the whole
+ * subdir and `*.sqlite` drops sqlite files anywhere in the tree.
+ */
+export declare function isExcluded(relPath: string, globs: ReadonlyArray<string>): boolean;
+/** Whether `bytes` starts with the 16-byte SQLite main-db magic header. */
+export declare function isSqliteHeader(bytes: Uint8Array): boolean;
+/** For a `-wal`/`-shm`/`-journal` sidecar path, the main db path it belongs
+ *  to; undefined for everything else. */
+export declare function sqliteSidecarBase(relPath: string): string | undefined;
+export type AdditiveMergePlan = {
+    /** Archive files absent locally — safe to copy in. */
+    copy: string[];
+    /** Archive files that already exist locally — skipped, never overwritten. */
+    skip: string[];
+};
+/** Plan a `--merge all` additive copy: only files that don't exist locally
+ *  are copied; existing local files always win. Pure via the injected
+ *  existence predicate. */
+export declare function planAdditiveMerge(archiveFiles: ReadonlyArray<string>, existsLocally: (relPath: string) => boolean): AdditiveMergePlan;
+export type FeedbackMergePlan = {
+    /** Folded records to append to the local feedback store. */
+    toImport: FeedbackRecord[];
+    /** Archived (session, turn) keys with no local record at all. */
+    added: number;
+    /** Keys where folding the archive in changed the local fold. */
+    updated: number;
+    /** Archived keys already fully represented locally — deduped away. */
+    unchanged: number;
+};
+/**
+ * Plan a `--merge feedback` fold using the exact dedupe semantics `crewhaus
+ * distill` applies: `mergeFeedback` collapses records to one per
+ * (sessionId, turnNumber), newest field wins, and a later comment-only
+ * record never erases an earlier rating. Local and archived records are
+ * each folded first, then each archived fold is folded AGAINST the local
+ * fold for its key — imported only when the archive actually contributes
+ * something (new key, or a fold that differs from the local one). Running
+ * the same merge twice is therefore a no-op.
+ */
+export declare function planFeedbackMerge(local: ReadonlyArray<FeedbackRecord>, archived: ReadonlyArray<FeedbackRecord>): FeedbackMergePlan;
+export type BackupOptions = {
+    /** The `.crewhaus` directory to archive. */
+    stateDir: string;
+    /** Absolute output tarball path. */
+    outFile: string;
+    excludeGlobs?: ReadonlyArray<string>;
+    /** Stamped into the manifest. */
+    crewhausVersion: string;
+    /** Test seam: clock. */
+    now?: () => Date;
+};
+export type BackupResult = {
+    manifest: BackupManifest;
+    outFile: string;
+    /** Files dropped by --exclude. */
+    excluded: string[];
+    /** Non-fatal warnings (skipped symlinks, raw-copied sqlite files). */
+    warnings: string[];
+};
+/**
+ * Archive `stateDir` to a gzipped tarball. Every regular file is staged as
+ * an exact byte copy (sqlite files as a consistent read-only snapshot — see
+ * the file header), a manifest is written at the archive root, and the
+ * system `tar` packs the staging dir. Source files are never modified.
+ */
+export declare function createStateBackup(opts: BackupOptions): Promise<BackupResult>;
+export type StateDirEntry = {
+    name: string;
+    kind: "dir" | "file";
+    files: number;
+};
+export type StateDirInspection = {
+    state: "absent";
+} | {
+    state: "empty";
+} | {
+    state: "occupied";
+    entries: StateDirEntry[];
+};
+/** What (if anything) currently occupies the target `.crewhaus` dir. */
+export declare function inspectStateDir(stateDir: string): StateDirInspection;
+export type RestoreOptions = {
+    archiveFile: string;
+    /** Directory that receives the `.crewhaus` dir (default: cwd — resolved by
+     *  the caller). */
+    intoDir: string;
+    /** Full replace: move an existing non-empty `.crewhaus` aside to
+     *  `.crewhaus.bak-<ts>` (never deleted) before restoring. */
+    force: boolean;
+};
+export type RestoreResult = {
+    stateDir: string;
+    /** Where the pre-existing state dir went (only set when one was moved). */
+    movedAsideTo?: string;
+    manifest?: BackupManifest;
+    filesRestored: number;
+    warnings: string[];
+};
+/**
+ * Full restore of a backup into `<intoDir>/.crewhaus`. Refuses (throws
+ * `StateRestoreRefusedError`) when the target is non-empty and `force` is
+ * not set. Extraction happens in a sibling temp dir which is renamed into
+ * place, so a half-extracted archive can never corrupt existing state, and
+ * restored file bytes are exactly the archived bytes (tar preserves them;
+ * nothing is rewritten).
+ */
+export declare function restoreStateArchive(opts: RestoreOptions): Promise<RestoreResult>;
+export type MergeFeedbackOptions = {
+    archiveFile: string;
+    intoDir: string;
+    /** Test seam: clock (names the imported jsonl file). */
+    now?: () => Date;
+};
+export type MergeFeedbackResult = FeedbackMergePlan & {
+    /** Distinct feedback records found in the archive (pre-fold). */
+    archivedRecords: number;
+    /** The `feedback/*.jsonl` file the imports were appended to (unset when
+     *  the archive contributed nothing new). */
+    wroteFile?: string;
+};
+/**
+ * `--merge feedback`: fold ONLY the archive's feedback records into the
+ * local store. Imported folds land as bare records in a new
+ * `feedback/restored-<ts>.jsonl` — the exact sink `crewhaus distill`
+ * already reads — so local session transcripts are never rewritten and the
+ * next distill sees local + imported records merged by `mergeFeedback`.
+ */
+export declare function mergeFeedbackFromArchive(opts: MergeFeedbackOptions): Promise<MergeFeedbackResult>;
+export type MergeAllOptions = {
+    archiveFile: string;
+    intoDir: string;
+};
+export type MergeAllResult = {
+    stateDir: string;
+    /** Archive files copied in (absent locally). */
+    copied: string[];
+    /** Archive files skipped (already exist locally — local always wins). */
+    skipped: string[];
+};
+/**
+ * `--merge all`: per-subdir additive copy. Only archive files that do NOT
+ * exist locally are written; an existing local file is never overwritten
+ * (the skips are reported). The archive manifest is metadata, not state,
+ * and is never copied.
+ */
+export declare function mergeAllFromArchive(opts: MergeAllOptions): Promise<MergeAllResult>;