@crewhaus/tool-lifecycle 0.7.0

package/dist/lib/knowledge.d.ts

@@ -0,0 +1,131 @@
+import { type Redactor, type SharedFragment, type SharedMemory } from "@crewhaus/harness-lifecycle";
+import { type Loaded } from "./result";
+export declare const NO_PII_PASS_REASON: string;
+/**
+ * The redactor this build can supply: the package's own credential masking
+ * and its strict post-mask rescan, with an identity PII pass.
+ *
+ * `IDENTITY_REDACTOR` from the package is deliberately NOT used — it never
+ * flags anything, so a memory holding an API key would be pushed verbatim.
+ */
+export declare function makeRedactor(): Redactor;
+/**
+ * The shared store's on-disk layout, as `applyPush` / `readSharedFragments`
+ * write and read it. Mirrored once here so the audit, the containment check
+ * and the post-push verification all name the same file rather than each
+ * spelling the rule out again.
+ */
+export declare const SHARED_MEMORIES_RELPATH = "memories.jsonl";
+export declare const SHARED_MANIFEST_RELPATH = "manifest.jsonl";
+export declare function sharedFragmentRelPath(frag: {
+    readonly kind: "grader" | "prompt";
+    readonly contentHash: string;
+}): string;
+export type Rejected = {
+    readonly at: string;
+    readonly reason: string;
+};
+export type SharedMemoryAudit = {
+    readonly memories: ReadonlyArray<SharedMemory>;
+    readonly rejected: ReadonlyArray<Rejected>;
+    readonly linesSeen: number;
+};
+/**
+ * Read the shared `memories.jsonl` through the package's own validator, and
+ * keep the rejects instead of dropping them.
+ *
+ * A missing file is an empty store and says so; an unreadable one is a
+ * failure and says THAT — a shared store that cannot be read is not a shared
+ * store with nothing in it.
+ */
+export declare function auditSharedMemories(sharedDirAbs: string): Loaded<SharedMemoryAudit>;
+export type SharedFragmentAudit = {
+    readonly fragments: ReadonlyArray<SharedFragment>;
+    readonly rejected: ReadonlyArray<Rejected>;
+};
+/**
+ * Fragments of one kind, with the files the package's validation refused.
+ *
+ * The valid set comes from `readSharedFragments`, so the accept/reject rule
+ * stays in one place; what is derived here is only WHICH candidate files did
+ * not survive it. Re-deriving the rule to explain each rejection would be a
+ * second copy of it, and the second copy is the one that goes stale.
+ */
+export declare function auditSharedFragments(sharedDirAbs: string, kind: "grader" | "prompt"): Loaded<SharedFragmentAudit>;
+/**
+ * The tag `applyPull` appends to every memory it lands
+ * (`shared:<source harness>`).
+ *
+ * It matters because the content hash covers text AND tags: a memory pulled
+ * from the shared store lands with one more tag than the record it came from,
+ * so hashing what is on disk does NOT reproduce the shared record's hash. The
+ * CLI computes the harness's hashes without accounting for it
+ * (`apps/cli/src/index.ts`, the `knowledge pull` branch), which is why a
+ * second pull re-imports everything and `shared.jsonl` grows on every run.
+ *
+ * `harnessMemoryHashes` below closes that by hashing a landed memory BOTH
+ * ways. The hashing rule itself is still the package's `memoryContentHash`;
+ * what is mirrored here is only the tag `applyPull` writes, and
+ * `index.test.ts` pins it — a second pull must bring in nothing.
+ */
+export declare const SHARED_TAG_PREFIX = "shared:";
+/**
+ * Every content hash the harness already holds, counting a pulled memory
+ * under its shared hash as well as its local one.
+ *
+ * The undo is EXACTLY `applyPull`'s do: it appends ONE provenance tag to the
+ * end of the record's tags, so exactly one trailing tag is taken back off.
+ * Stripping every `shared:`-prefixed tag instead — which is what this did —
+ * is a different function, and it is wrong for the record that already
+ * carried provenance of its own (one harness pulled a memory and pushed it
+ * on, so the shared record's own tags include a `shared:` tag). For that
+ * record neither form reproduced the shared hash, so the FIRST pull reported
+ * `verified: false` with the memory listed as missing although it had landed
+ * perfectly, and every later pull re-imported it — the unbounded growth this
+ * mirror exists to prevent, reintroduced by the mirror.
+ */
+export declare function harnessMemoryHashes(memories: ReadonlyArray<{
+    readonly text: string;
+    readonly tags: ReadonlyArray<string>;
+}>): Set<string>;
+/**
+ * Where `applyPull` lands each artifact inside a harness.
+ *
+ * Mirrored, not re-derived: the package chooses these paths and does not
+ * export them, and a tool that cannot name them can neither contain the
+ * writes (a symlink at one of them sends the file out of the workspace) nor
+ * check that they arrived. `index.test.ts` pins every one of them against a
+ * real pull, so a rename upstream fails a test here rather than silently
+ * turning the checks into no-ops.
+ */
+export declare const PULLED_MEMORIES_RELPATH = ".crewhaus/memories/shared.jsonl";
+export declare const PULLED_GRADER_PREFIX = "graders.shared-";
+export declare const PULLED_GRADER_SUFFIX = ".yaml";
+export declare function pulledFragmentRelPath(frag: {
+    readonly kind: "grader" | "prompt";
+    readonly contentHash: string;
+}): string;
+/**
+ * Content hashes of the grader fragments a previous pull already landed.
+ *
+ * `readHarnessGraders` looks only at `graders.yaml` / `.crewhaus/graders.yaml`
+ * — by design, those are the harness's OWN grader files — so a fragment
+ * `applyPull` wrote to `graders.shared-<hash>.yaml` is invisible to it, and
+ * every later pull counted that fragment as new and reported pulling it
+ * again. Matched by the filename shape and hashed with the package's own
+ * `fragmentContentHash`, so the accept rule stays the package's; only the
+ * landing name is mirrored.
+ */
+export declare function landedSharedGraderHashes(harnessDirAbs: string): Loaded<Set<string>>;
+export type HarnessMemoryAudit = {
+    readonly dirPresent: boolean;
+    readonly files: number;
+    readonly linesSeen: number;
+};
+/**
+ * How many lines the harness's memory files hold, so a store whose lines are
+ * mostly unparseable is not reported as a store with few memories. The parsed
+ * count comes from the package's `readHarnessMemories`; the difference
+ * between the two is the interesting number.
+ */
+export declare function auditHarnessMemoryFiles(harnessDirAbs: string): Loaded<HarnessMemoryAudit>;

package/dist/lib/knowledge.js

@@ -0,0 +1,283 @@
+/**
+ * `KnowledgeSync`'s half: accounting for what a shared store contains that
+ * could not be used, and wiring the redactor this build can honestly offer.
+ *
+ * The rules all belong to `@crewhaus/harness-lifecycle`: what a valid shared
+ * memory is (`validateSharedMemory`, including the re-hash that defeats
+ * dedupe-poisoning), what a credential looks like (`maskKnowledgeSecrets` /
+ * `looksLikeSecret`), what to push (`planPush`) and what to pull
+ * (`planPull`). None of it is restated here.
+ *
+ * Two things are added.
+ *
+ * FIRST, SKIPS ARE COUNTED. The package's readers drop an invalid shared
+ * record with a warning to stderr and return the rest, which is right for a
+ * CLI and wrong for a tool: a caller handed `{"memories": 12}` has no way to
+ * know that 40 more were rejected as forged. A shared store is untrusted
+ * input — the count of what it offered and could not be used is part of the
+ * answer, not noise.
+ *
+ * SECOND, THE REDACTION GAP IS NAMED. Production wires
+ * `buildKnowledgeRedactor` around `@crewhaus/pii-redactor`. That package is
+ * not a dependency here, so this build runs the credential half — masking,
+ * and the strict post-mask rescan that DROPS anything still secret-shaped —
+ * with an identity PII pass. Names, emails and phone numbers are therefore
+ * NOT redacted, and a push says so and requires the caller to acknowledge it
+ * rather than quietly shipping a weaker guarantee under the same name.
+ */
+import { readFileSync, readdirSync } from "node:fs";
+import * as path from "node:path";
+import { buildKnowledgeRedactor, fragmentContentHash, memoryContentHash, readSharedFragments, validateSharedMemory, } from "@crewhaus/harness-lifecycle";
+import { fail } from "./result";
+export const NO_PII_PASS_REASON = "@crewhaus/pii-redactor is not a dependency of @crewhaus/tool-lifecycle, so the PII pass of the " +
+    "knowledge redactor is an identity function here: credential-shaped tokens ARE masked and any " +
+    "artifact still secret-shaped afterwards is dropped, but names, emails, addresses and phone " +
+    "numbers are NOT removed. Pass allowWithoutPiiRedaction to push on those terms.";
+/**
+ * The redactor this build can supply: the package's own credential masking
+ * and its strict post-mask rescan, with an identity PII pass.
+ *
+ * `IDENTITY_REDACTOR` from the package is deliberately NOT used — it never
+ * flags anything, so a memory holding an API key would be pushed verbatim.
+ */
+export function makeRedactor() {
+    return buildKnowledgeRedactor(async (text) => text);
+}
+// ---------------------------------------------------------------------------
+// shared store, with the rejects counted
+// ---------------------------------------------------------------------------
+/**
+ * The shared store's on-disk layout, as `applyPush` / `readSharedFragments`
+ * write and read it. Mirrored once here so the audit, the containment check
+ * and the post-push verification all name the same file rather than each
+ * spelling the rule out again.
+ */
+export const SHARED_MEMORIES_RELPATH = "memories.jsonl";
+export const SHARED_MANIFEST_RELPATH = "manifest.jsonl";
+const SHARED_FRAGMENT_DIR = { grader: "graders", prompt: "prompts" };
+const SHARED_FRAGMENT_EXT = { grader: ".yaml", prompt: ".md" };
+export function sharedFragmentRelPath(frag) {
+    return `${SHARED_FRAGMENT_DIR[frag.kind]}/${frag.contentHash}${SHARED_FRAGMENT_EXT[frag.kind]}`;
+}
+/**
+ * Read the shared `memories.jsonl` through the package's own validator, and
+ * keep the rejects instead of dropping them.
+ *
+ * A missing file is an empty store and says so; an unreadable one is a
+ * failure and says THAT — a shared store that cannot be read is not a shared
+ * store with nothing in it.
+ */
+export function auditSharedMemories(sharedDirAbs) {
+    const file = path.join(sharedDirAbs, SHARED_MEMORIES_RELPATH);
+    let raw;
+    try {
+        raw = readFileSync(file, "utf8");
+    }
+    catch (err) {
+        const code = err.code;
+        if (code === "ENOENT")
+            return { ok: true, value: { memories: [], rejected: [], linesSeen: 0 } };
+        return fail("unreadable", `the shared memories file could not be read (${code ?? "unknown error"})`);
+    }
+    const memories = [];
+    const rejected = [];
+    let linesSeen = 0;
+    let lineNo = 0;
+    for (const line of raw.split("\n")) {
+        lineNo += 1;
+        if (line.trim() === "")
+            continue;
+        linesSeen += 1;
+        let parsed;
+        try {
+            parsed = JSON.parse(line);
+        }
+        catch {
+            rejected.push({ at: `memories.jsonl:${lineNo}`, reason: "not valid JSON" });
+            continue;
+        }
+        const verdict = validateSharedMemory(parsed);
+        if (verdict.ok)
+            memories.push(verdict.memory);
+        else
+            rejected.push({ at: `memories.jsonl:${lineNo}`, reason: verdict.reason });
+    }
+    return { ok: true, value: { memories, rejected, linesSeen } };
+}
+/**
+ * Fragments of one kind, with the files the package's validation refused.
+ *
+ * The valid set comes from `readSharedFragments`, so the accept/reject rule
+ * stays in one place; what is derived here is only WHICH candidate files did
+ * not survive it. Re-deriving the rule to explain each rejection would be a
+ * second copy of it, and the second copy is the one that goes stale.
+ */
+export function auditSharedFragments(sharedDirAbs, kind) {
+    const dir = path.join(sharedDirAbs, SHARED_FRAGMENT_DIR[kind]);
+    const ext = SHARED_FRAGMENT_EXT[kind];
+    let names;
+    try {
+        names = readdirSync(dir).sort();
+    }
+    catch (err) {
+        const code = err.code;
+        if (code === "ENOENT")
+            return { ok: true, value: { fragments: [], rejected: [] } };
+        return fail("unreadable", `the shared ${kind} directory could not be listed (${code ?? "unknown error"})`);
+    }
+    let fragments;
+    try {
+        fragments = readSharedFragments(sharedDirAbs, kind);
+    }
+    catch (err) {
+        return fail("unreadable", `the shared ${kind} fragments could not be read: ${err.message}`);
+    }
+    const accepted = new Set(fragments.map((f) => f.filename));
+    const rejected = names
+        .filter((n) => n.endsWith(ext) && !accepted.has(n))
+        .map((n) => ({
+        at: `${SHARED_FRAGMENT_DIR[kind]}/${n}`,
+        reason: "refused by @crewhaus/harness-lifecycle's fragment validation — the filename's claimed content hash does not match the body's, or the body is over the size cap",
+    }));
+    return { ok: true, value: { fragments, rejected } };
+}
+// ---------------------------------------------------------------------------
+// harness side
+// ---------------------------------------------------------------------------
+/**
+ * The tag `applyPull` appends to every memory it lands
+ * (`shared:<source harness>`).
+ *
+ * It matters because the content hash covers text AND tags: a memory pulled
+ * from the shared store lands with one more tag than the record it came from,
+ * so hashing what is on disk does NOT reproduce the shared record's hash. The
+ * CLI computes the harness's hashes without accounting for it
+ * (`apps/cli/src/index.ts`, the `knowledge pull` branch), which is why a
+ * second pull re-imports everything and `shared.jsonl` grows on every run.
+ *
+ * `harnessMemoryHashes` below closes that by hashing a landed memory BOTH
+ * ways. The hashing rule itself is still the package's `memoryContentHash`;
+ * what is mirrored here is only the tag `applyPull` writes, and
+ * `index.test.ts` pins it — a second pull must bring in nothing.
+ */
+export const SHARED_TAG_PREFIX = "shared:";
+/**
+ * Every content hash the harness already holds, counting a pulled memory
+ * under its shared hash as well as its local one.
+ *
+ * The undo is EXACTLY `applyPull`'s do: it appends ONE provenance tag to the
+ * end of the record's tags, so exactly one trailing tag is taken back off.
+ * Stripping every `shared:`-prefixed tag instead — which is what this did —
+ * is a different function, and it is wrong for the record that already
+ * carried provenance of its own (one harness pulled a memory and pushed it
+ * on, so the shared record's own tags include a `shared:` tag). For that
+ * record neither form reproduced the shared hash, so the FIRST pull reported
+ * `verified: false` with the memory listed as missing although it had landed
+ * perfectly, and every later pull re-imported it — the unbounded growth this
+ * mirror exists to prevent, reintroduced by the mirror.
+ */
+export function harnessMemoryHashes(memories) {
+    const hashes = new Set();
+    for (const memory of memories) {
+        hashes.add(memoryContentHash(memory.text, memory.tags));
+        const last = memory.tags[memory.tags.length - 1];
+        if (last?.startsWith(SHARED_TAG_PREFIX) === true) {
+            hashes.add(memoryContentHash(memory.text, memory.tags.slice(0, -1)));
+        }
+    }
+    return hashes;
+}
+/**
+ * Where `applyPull` lands each artifact inside a harness.
+ *
+ * Mirrored, not re-derived: the package chooses these paths and does not
+ * export them, and a tool that cannot name them can neither contain the
+ * writes (a symlink at one of them sends the file out of the workspace) nor
+ * check that they arrived. `index.test.ts` pins every one of them against a
+ * real pull, so a rename upstream fails a test here rather than silently
+ * turning the checks into no-ops.
+ */
+export const PULLED_MEMORIES_RELPATH = ".crewhaus/memories/shared.jsonl";
+const PULLED_PROMPT_RELDIR = ".crewhaus/prompts";
+export const PULLED_GRADER_PREFIX = "graders.shared-";
+export const PULLED_GRADER_SUFFIX = ".yaml";
+export function pulledFragmentRelPath(frag) {
+    const short = frag.contentHash.slice(0, 12);
+    return frag.kind === "prompt"
+        ? `${PULLED_PROMPT_RELDIR}/shared-${short}.md`
+        : `${PULLED_GRADER_PREFIX}${short}${PULLED_GRADER_SUFFIX}`;
+}
+/**
+ * Content hashes of the grader fragments a previous pull already landed.
+ *
+ * `readHarnessGraders` looks only at `graders.yaml` / `.crewhaus/graders.yaml`
+ * — by design, those are the harness's OWN grader files — so a fragment
+ * `applyPull` wrote to `graders.shared-<hash>.yaml` is invisible to it, and
+ * every later pull counted that fragment as new and reported pulling it
+ * again. Matched by the filename shape and hashed with the package's own
+ * `fragmentContentHash`, so the accept rule stays the package's; only the
+ * landing name is mirrored.
+ */
+export function landedSharedGraderHashes(harnessDirAbs) {
+    let names;
+    try {
+        names = readdirSync(harnessDirAbs).sort();
+    }
+    catch (err) {
+        const code = err.code;
+        return fail("unreadable", `the harness directory could not be listed (${code ?? "unknown error"})`);
+    }
+    const hashes = new Set();
+    for (const name of names) {
+        if (!name.startsWith(PULLED_GRADER_PREFIX) || !name.endsWith(PULLED_GRADER_SUFFIX))
+            continue;
+        let contents;
+        try {
+            contents = readFileSync(path.join(harnessDirAbs, name), "utf8");
+        }
+        catch (err) {
+            // An unreadable landed fragment is not an absent one: reporting it as
+            // absent would make the next pull re-import it and call it new.
+            return fail("unreadable", `the pulled grader fragment ${name} could not be read (${err.code ?? "unknown error"})`);
+        }
+        hashes.add(fragmentContentHash(contents));
+    }
+    return { ok: true, value: hashes };
+}
+/**
+ * How many lines the harness's memory files hold, so a store whose lines are
+ * mostly unparseable is not reported as a store with few memories. The parsed
+ * count comes from the package's `readHarnessMemories`; the difference
+ * between the two is the interesting number.
+ */
+export function auditHarnessMemoryFiles(harnessDirAbs) {
+    const dir = path.join(harnessDirAbs, ".crewhaus", "memories");
+    let names;
+    try {
+        names = readdirSync(dir).sort();
+    }
+    catch (err) {
+        const code = err.code;
+        if (code === "ENOENT")
+            return { ok: true, value: { dirPresent: false, files: 0, linesSeen: 0 } };
+        return fail("unreadable", `the harness memories directory could not be listed (${code ?? "unknown error"})`);
+    }
+    let files = 0;
+    let linesSeen = 0;
+    for (const name of names) {
+        if (!name.endsWith(".jsonl"))
+            continue;
+        files += 1;
+        let raw;
+        try {
+            raw = readFileSync(path.join(dir, name), "utf8");
+        }
+        catch (err) {
+            return fail("unreadable", `the harness memory file ${name} could not be read (${err.code ?? "unknown error"})`);
+        }
+        for (const line of raw.split("\n"))
+            if (line.trim() !== "")
+                linesSeen += 1;
+    }
+    return { ok: true, value: { dirPresent: true, files, linesSeen } };
+}

package/dist/lib/migrate.d.ts

@@ -0,0 +1,106 @@
+import { type Loaded } from "./result";
+/**
+ * The two directories under `.crewhaus` that ARE the harness's record store:
+ * the pair `runRetentionExport` carries, and the same pair
+ * `openHarnessRecordStore` enumerates and a sweep deletes from. Mirrored here
+ * because `@crewhaus/harness-lifecycle` keeps the relpaths private.
+ */
+export declare const COVERED_STORE_DIRS: readonly ["sessions", "audit"];
+export declare const RECEIPT_FILENAME = "store-migration.json";
+/**
+ * The file `runRetentionExport` writes into its output directory on every
+ * REAL run, on top of the copies themselves.
+ *
+ * Mirrored here (the package does not export the name) because it is a
+ * destination this tool writes but does not copy: leaving it out of the
+ * written set made the conflict check miss it, so a destination that already
+ * held a `manifest.json` — a previous `crewhaus retention export`, say — had
+ * it overwritten without `overwrite` being asked for and without the preview
+ * ever naming it. `index.test.ts` pins the coupling: a real migration must
+ * produce this file at this name.
+ */
+export declare const EXPORT_MANIFEST_FILENAME = "manifest.json";
+export declare const RECORD_SHAPE_MIGRATION_UNAVAILABLE: string;
+export type StoreVersion = {
+    readonly state: "known";
+    readonly memoriesSchemaVersion: number;
+}
+/** No `.crewhaus/meta.json`: nothing has stamped a version here yet. */
+ | {
+    readonly state: "unstamped";
+} | {
+    readonly state: "unknown";
+    readonly reason: string;
+};
+/**
+ * Read `<harnessDir>/.crewhaus/meta.json`, the file `crewhaus migrate
+ * memories` stamps (`{ memories: { schemaVersion, migratedAt } }`).
+ *
+ * An unreadable or malformed meta file is `unknown`, never version 1: a
+ * migration that guesses the version it is migrating FROM is a migration that
+ * rewrites records twice.
+ */
+export declare function readStoreVersion(harnessDirAbs: string): StoreVersion;
+export type Uncovered = {
+    readonly entry: string;
+    readonly reason: string;
+};
+/**
+ * Everything directly under `.crewhaus` that the export path does not copy.
+ *
+ * Returned so the result can name it. A caller who migrates a store and is
+ * told only "42 sessions, 3 audit days" will believe the memories came too.
+ */
+export declare function uncoveredStateEntries(harnessDirAbs: string): Loaded<Uncovered[]>;
+export type MigrationReceiptFile = {
+    readonly rel: string;
+    readonly bytes: number;
+    readonly sha256: string;
+};
+export type MigrationReceipt = {
+    readonly writtenAt: string;
+    readonly sourceRoot: string;
+    readonly destinationRoot: string;
+    readonly storeVersion: StoreVersion;
+    readonly since?: string;
+    readonly sessions: ReadonlyArray<string>;
+    readonly auditDays: ReadonlyArray<string>;
+    readonly chainTailCopied: boolean;
+    readonly files: ReadonlyArray<MigrationReceiptFile>;
+    readonly verified: boolean;
+};
+export type DestinationState = {
+    readonly exists: boolean;
+    /** Files already present that a migration would write over. */
+    readonly conflicts: ReadonlyArray<string>;
+    readonly receipt?: MigrationReceipt;
+    /** Set when a receipt is present but could not be trusted. */
+    readonly receiptProblem?: string;
+};
+/**
+ * What is already at the destination, and whether a previous migration put it
+ * there. A receipt naming a DIFFERENT source is the case that matters: two
+ * harnesses' stores merged into one directory cannot be told apart
+ * afterwards, and the audit chains cannot both be valid.
+ */
+export declare function inspectDestination(destAbs: string, plannedRelPaths: ReadonlyArray<string>): Loaded<DestinationState>;
+/** The destination-relative paths a migration of this selection would write. */
+export declare function plannedPaths(opts: {
+    readonly sessions: ReadonlyArray<string>;
+    readonly auditDays: ReadonlyArray<string>;
+    readonly chainTail: boolean;
+    /** Session ids that have a sibling event log at the source. */
+    readonly withEventLog: ReadonlySet<string>;
+}): string[];
+/**
+ * Every destination-relative path a REAL migration writes: the copies, plus
+ * the export's own `manifest.json` and this tool's receipt.
+ *
+ * Distinct from {@link plannedPaths} on purpose. `plannedPaths` is the set
+ * that is copied from a source and can therefore be hashed on both sides;
+ * this is the set that is WRITTEN, which is what the conflict check and the
+ * containment check have to be asked about. Reporting the first as the second
+ * is how a destination file nobody copied gets destroyed quietly.
+ */
+export declare function writtenPaths(plannedRelPaths: ReadonlyArray<string>): string[];
+export declare function writeReceipt(destAbs: string, receipt: MigrationReceipt): Loaded<string>;