@deepwatch/dsh-tools 0.1.0

package/lib/library-generations.d.ts

@@ -0,0 +1,107 @@
+/**
+ * The Library index, and the only thing allowed to replace it.
+ *
+ * The host used to build its index once per process. Records written after
+ * start were invisible until somebody restarted the application, and the
+ * surface's rebuild control could not reach the host's index at all — so it
+ * said "Search again", which was honest and useless. This is the capability
+ * that makes it a real one.
+ *
+ * Four properties, and each exists because the obvious implementation gets it
+ * wrong.
+ *
+ * **A rebuild is an explicit operation.** Not a flag on a search. A search that
+ * might re-read the corpus is a search whose cost nobody can predict, and it
+ * leaves a caller no way to say "answer from what you have".
+ *
+ * **One rebuild at a time, and callers join it.** Two people pressing Refresh
+ * must not read the directory twice. A second caller waits on the first
+ * rebuild and receives its outcome, which is also why a repeated request id
+ * returns the recorded answer rather than starting anything.
+ *
+ * **Cancellation is by reference count.** A caller that stops waiting has
+ * withdrawn, not cancelled — the work may still be wanted by somebody else.
+ * The rebuild is abandoned when the last waiter leaves, and only then.
+ *
+ * **The swap is atomic and conditional.** The new index is built beside the
+ * one in service and installed only when it is complete and healthy. A failed
+ * or abandoned rebuild leaves the previous generation searchable, which is the
+ * difference between a refresh that did not work and a Library that broke.
+ *
+ * @module @deepwatch/dsh-tools/library-generations
+ */
+import type { IndexableRecord, LibraryIndex } from '@deepwatch/dsh-library';
+import type { LibraryIndexState } from '@deepwatch/dsh-contracts/query/wire';
+/** One built index, and what is true about it. */
+export interface IndexGeneration {
+    /** Increments only when a healthy rebuild was swapped into service. */
+    readonly generation: number;
+    readonly startedAt: string;
+    readonly completedAt: string | null;
+    readonly sourceCount: number;
+    readonly recordCount: number;
+    readonly indexState: LibraryIndexState;
+    /** By filename, never by path. */
+    readonly skipped: readonly string[];
+}
+/** What a refresh did. Every one of these leaves a searchable Library. */
+export type RefreshOutcome = {
+    readonly kind: 'refreshed';
+    readonly index: IndexGeneration;
+} | {
+    readonly kind: 'cancelled';
+    readonly index: IndexGeneration;
+} | {
+    readonly kind: 'failed';
+    readonly reason: string;
+    readonly index: IndexGeneration;
+};
+/** What the service needs to build an index and to timestamp what it built. */
+export interface GenerationsConfig {
+    readonly roots: readonly string[];
+    /** Injected in tests so a generation record is comparable across runs. */
+    readonly now?: () => string;
+    /** Injected in tests to exercise failure and slowness without a filesystem. */
+    readonly build?: (roots: readonly string[], signal: AbortSignal) => Promise<{
+        readonly index: LibraryIndex | null;
+        readonly skipped: readonly string[];
+        readonly sourceCount: number;
+    }>;
+}
+/** The index in service, and the one operation that may replace it. */
+export declare class LibraryGenerations {
+    #private;
+    constructor(config: GenerationsConfig);
+    /**
+     * Index one record the Host made, now, without a rebuild.
+     *
+     * The failure this closes: the evaluation produced 76 tool actions and a
+     * Library with nothing in it, because nothing indexed a receipt until
+     * somebody pressed Refresh — and Refresh read roots that did not contain
+     * them yet.
+     */
+    addLive(record: IndexableRecord): void;
+    /** How many live records are being carried across rebuilds. */
+    liveCount(): number;
+    /**
+     * The index searches answer from, built on first use.
+     *
+     * Lazy rather than eager: a profile that never opens the Library should not
+     * pay for reading its evidence roots at boot.
+     */
+    index(): LibraryIndex;
+    /** What is in service. Building it first if nothing is. */
+    generation(): IndexGeneration;
+    /**
+     * Read the roots again and, if that succeeds, put the result into service.
+     *
+     * `requestId` makes the operation idempotent: a caller that retries after a
+     * dropped connection gets the answer its first attempt produced rather than
+     * a second read of the corpus. `signal` is that caller's withdrawal, not a
+     * cancellation of the work — see the note on reference counting above.
+     */
+    refresh(requestId: string, signal: AbortSignal): Promise<RefreshOutcome>;
+    /** Whether a rebuild is running. Reported to a surface as bounded progress. */
+    rebuilding(): boolean;
+}
+//# sourceMappingURL=library-generations.d.ts.map

package/lib/library-generations.js

@@ -0,0 +1,227 @@
+/**
+ * The Library index, and the only thing allowed to replace it.
+ *
+ * The host used to build its index once per process. Records written after
+ * start were invisible until somebody restarted the application, and the
+ * surface's rebuild control could not reach the host's index at all — so it
+ * said "Search again", which was honest and useless. This is the capability
+ * that makes it a real one.
+ *
+ * Four properties, and each exists because the obvious implementation gets it
+ * wrong.
+ *
+ * **A rebuild is an explicit operation.** Not a flag on a search. A search that
+ * might re-read the corpus is a search whose cost nobody can predict, and it
+ * leaves a caller no way to say "answer from what you have".
+ *
+ * **One rebuild at a time, and callers join it.** Two people pressing Refresh
+ * must not read the directory twice. A second caller waits on the first
+ * rebuild and receives its outcome, which is also why a repeated request id
+ * returns the recorded answer rather than starting anything.
+ *
+ * **Cancellation is by reference count.** A caller that stops waiting has
+ * withdrawn, not cancelled — the work may still be wanted by somebody else.
+ * The rebuild is abandoned when the last waiter leaves, and only then.
+ *
+ * **The swap is atomic and conditional.** The new index is built beside the
+ * one in service and installed only when it is complete and healthy. A failed
+ * or abandoned rebuild leaves the previous generation searchable, which is the
+ * difference between a refresh that did not work and a Library that broke.
+ *
+ * @module @deepwatch/dsh-tools/library-generations
+ */
+import { buildIndex, buildIndexCancellable } from './library-search.js';
+import { wireIndexState } from './read-plane.js';
+/** How many settled request ids are remembered for idempotency. */
+const REMEMBERED_REQUESTS = 64;
+/** The index in service, and the one operation that may replace it. */
+export class LibraryGenerations {
+    #config;
+    #current = null;
+    #inFlight = null;
+    #settled = new Map();
+    #nextGeneration = 1;
+    constructor(config) {
+        this.#config = config;
+    }
+    /**
+     * Records the Host produced while running, kept across rebuilds.
+     *
+     * An execution receipt is minted by this Host, in memory, seconds ago; a
+     * rebuild reads the evidence roots on disk. Without this, indexing a receipt
+     * and then pressing Refresh would lose it, and the Library would go from
+     * "searchable" to "empty" for no reason a person could see. Re-applied after
+     * every build, so Refresh converges on the same answer rather than racing
+     * whatever arrived while it ran.
+     */
+    #live = new Map();
+    /**
+     * Index one record the Host made, now, without a rebuild.
+     *
+     * The failure this closes: the evaluation produced 76 tool actions and a
+     * Library with nothing in it, because nothing indexed a receipt until
+     * somebody pressed Refresh — and Refresh read roots that did not contain
+     * them yet.
+     */
+    addLive(record) {
+        this.#live.set(record.recordId, record);
+        this.index().add(record);
+    }
+    /** How many live records are being carried across rebuilds. */
+    liveCount() {
+        return this.#live.size;
+    }
+    /** Re-apply the live records to a freshly built index. */
+    #applyLive(index) {
+        for (const record of this.#live.values())
+            index.add(record);
+    }
+    /** The clock, injectable so a test can assert on a generation record. */
+    #now() {
+        return this.#config.now?.() ?? new Date().toISOString();
+    }
+    /**
+     * The index searches answer from, built on first use.
+     *
+     * Lazy rather than eager: a profile that never opens the Library should not
+     * pay for reading its evidence roots at boot.
+     */
+    index() {
+        if (this.#current === null) {
+            const startedAt = this.#now();
+            const built = buildIndex(this.#config.roots);
+            this.#applyLive(built.index);
+            this.#current = {
+                index: built.index,
+                meta: this.#describe(this.#nextGeneration, startedAt, built.index, built.skipped),
+            };
+            this.#nextGeneration += 1;
+        }
+        return this.#current.index;
+    }
+    /** What is in service. Building it first if nothing is. */
+    generation() {
+        this.index();
+        // `index()` above installs it, so this cannot be null. Reading through a
+        // local keeps that obvious to the compiler as well as to a reader.
+        const current = this.#current;
+        if (current === null)
+            throw new Error('library: the index did not install');
+        return current.meta;
+    }
+    #describe(generation, startedAt, index, skipped) {
+        return {
+            generation,
+            startedAt,
+            completedAt: this.#now(),
+            sourceCount: this.#config.roots.length,
+            recordCount: index.size,
+            // The index's own word for its condition, mapped to the wire's. A
+            // rebuild that read nothing is empty, not ready: an index nobody has
+            // filled is not a complete answer to anything.
+            indexState: wireIndexState(index.health, index.size),
+            skipped,
+        };
+    }
+    /**
+     * Read the roots again and, if that succeeds, put the result into service.
+     *
+     * `requestId` makes the operation idempotent: a caller that retries after a
+     * dropped connection gets the answer its first attempt produced rather than
+     * a second read of the corpus. `signal` is that caller's withdrawal, not a
+     * cancellation of the work — see the note on reference counting above.
+     */
+    async refresh(requestId, signal) {
+        const remembered = this.#settled.get(requestId);
+        if (remembered !== undefined)
+            return remembered;
+        const flight = this.#inFlight ?? this.#start();
+        flight.waiters += 1;
+        const withdraw = () => {
+            flight.waiters -= 1;
+            // The last waiter leaving is what ends the work. While anybody is still
+            // waiting, one caller's deadline is not everybody's.
+            if (flight.waiters <= 0)
+                flight.controller.abort();
+        };
+        if (signal.aborted)
+            withdraw();
+        else
+            signal.addEventListener('abort', withdraw, { once: true });
+        try {
+            const outcome = await flight.promise;
+            this.#remember(requestId, outcome);
+            return outcome;
+        }
+        finally {
+            if (!signal.aborted)
+                signal.removeEventListener('abort', withdraw);
+        }
+    }
+    /** Whether a rebuild is running. Reported to a surface as bounded progress. */
+    rebuilding() {
+        return this.#inFlight !== null;
+    }
+    #remember(requestId, outcome) {
+        this.#settled.set(requestId, outcome);
+        // Bounded: a long-lived host must not accumulate one entry per refresh
+        // anybody has ever asked for. Oldest first, which is insertion order.
+        while (this.#settled.size > REMEMBERED_REQUESTS) {
+            const oldest = this.#settled.keys().next();
+            if (oldest.done === true)
+                break;
+            this.#settled.delete(oldest.value);
+        }
+    }
+    #start() {
+        const controller = new AbortController();
+        const startedAt = this.#now();
+        const build = this.#config.build ?? buildIndexCancellable;
+        // The generation in service before this ran, read now rather than after
+        // the await. Every outcome below reports it, so a caller always learns
+        // what it can still search.
+        const previous = this.generation();
+        // Declared before the body so the body can compare against it, and
+        // assigned immediately after. `run` reads it only in its `finally`, which
+        // cannot be reached before the first await.
+        let flight = null;
+        const run = async () => {
+            try {
+                const built = await build(this.#config.roots, controller.signal);
+                if (built.index === null)
+                    return { kind: 'cancelled', index: previous };
+                // Before the generation is described, so its counts include them and a
+                // surface comparing generations is comparing the whole answer. A
+                // rebuild reads the roots; the receipts this Host minted while running
+                // are not there yet, and dropping them would take a searchable Library
+                // back to empty for no reason a person could see.
+                this.#applyLive(built.index);
+                const meta = this.#describe(this.#nextGeneration, startedAt, built.index, built.skipped);
+                this.#nextGeneration += 1;
+                // The swap. One assignment, after the new index is complete, so no
+                // search can ever observe a half-built one.
+                this.#current = { index: built.index, meta };
+                return { kind: 'refreshed', index: meta };
+            }
+            catch (cause) {
+                return {
+                    kind: 'failed',
+                    // The message and nothing else: a stack or an errno string would
+                    // name the host's directories.
+                    reason: cause instanceof Error ? cause.message : 'the rebuild failed',
+                    index: previous,
+                };
+            }
+            finally {
+                // Only if this is still the current flight. A later refresh that has
+                // already started must not be cleared by an earlier one finishing.
+                if (this.#inFlight === flight)
+                    this.#inFlight = null;
+            }
+        };
+        flight = { controller, waiters: 0, promise: run() };
+        this.#inFlight = flight;
+        return flight;
+    }
+}
+//# sourceMappingURL=library-generations.js.map

package/lib/library-search.d.ts

@@ -0,0 +1,143 @@
+/**
+ * Library search, as a tool the host owns.
+ *
+ * The index has to live somewhere that can read the evidence store, and the
+ * browser cannot: a client plugin receives no config, `ctx.remote` is an event
+ * bus rather than a query client, and the boot graph carries no data. The host
+ * can read the store, so the host holds the index and the agent reaches it the
+ * way it reaches everything else in Watch — as a tool.
+ *
+ * It is the same `LibraryIndex` the client surface uses. One implementation,
+ * one set of semantics, one place where "every term must match" is decided;
+ * two would drift within a release and disagree about what the library
+ * contains.
+ *
+ * Three things it will not do.
+ *
+ * It reads only inside the roots it was configured with. `isWithinRoots`
+ * refuses traversal rather than normalising it, because normalising an attempt
+ * to escape produces a path that works.
+ *
+ * It returns no verdict. A search result is a pointer to a record, and whether
+ * that record's claim is true is `watch_verify`'s question. A tool that
+ * answered both would let a search become an assertion.
+ *
+ * And it never rebuilds silently. A stale or corrupt index is reported as such
+ * in the result, because a search that quietly returns less than it should is
+ * worse than one that says it is behind.
+ *
+ * @module @deepwatch/dsh-tools/library-search
+ */
+import type { Context } from '@deepseek-ai/cordis';
+import { LibraryIndex } from '@deepwatch/dsh-library';
+import type { IndexableRecord } from '@deepwatch/dsh-library';
+/** Where the index is allowed to read from. */
+export interface LibrarySearchConfig {
+    /**
+     * Directories holding evidence records.
+     *
+     * Nothing outside these is read, whatever a filename claims. An empty list
+     * means the tool has nothing to index and says so rather than falling back
+     * to somewhere convenient.
+     */
+    readonly roots: readonly string[];
+    /**
+     * Who owns the index, when somebody does.
+     *
+     * The tool used to hold its own, which was fine while it was the only
+     * reader. It is not: the read plane answers the same corpus for the Library
+     * surface, and a refresh asked for by either has to be the same refresh.
+     * Two caches would drift inside one release and disagree about what the
+     * library contains — a person searching the UI and the agent searching the
+     * tool getting different answers to the same question.
+     */
+    readonly generations?: LibraryGenerationsLike;
+}
+/**
+ * The part of `LibraryGenerations` this module needs.
+ *
+ * Structural rather than an import of the class, because the class imports the
+ * builders from here and a direct edge would close a cycle between two modules
+ * that are one concern.
+ */
+export interface LibraryGenerationsLike {
+    index(): LibraryIndex;
+    refresh(requestId: string, signal: AbortSignal): Promise<unknown>;
+}
+/**
+ * Read one JSON file into a record, or nothing.
+ *
+ * Deliberately total: a malformed file, an unreadable one, or one holding
+ * something that is not a record yields no record rather than throwing. One bad
+ * file must not stop the whole library being searchable — a corpus is exactly
+ * where a single malformed entry is most likely and least excusable as a
+ * failure mode.
+ */
+/**
+ * Every string in a record, at any depth.
+ *
+ * Reading only the top level looked reasonable and was wrong: a citation's
+ * text, a check's detail and a revision's transcript all live one or two
+ * levels down, so a phrase plainly present in the file returned nothing.
+ *
+ * Bounded on purpose. A record is data, not a program, so a deeply nested or
+ * self-referential one costs a fixed amount of work rather than a stack
+ * overflow — and a malformed file is exactly where that would show up.
+ */
+export declare function gatherText(value: unknown, depth?: number, seen?: Set<unknown>): string[];
+/**
+ * The digest of a file's bytes.
+ *
+ * Bytes, not the decoded string. A file is what is on disk, and a record whose
+ * identity depended on how it happened to be decoded would change identity for
+ * a byte-order mark nobody typed.
+ */
+export declare function contentDigest(bytes: Uint8Array | string): string;
+export declare function recordFromFile(path: string, raw: string | Uint8Array): IndexableRecord | null;
+/**
+ * Read every record under the configured roots.
+ *
+ * Returns what it managed to read plus what it refused, because a caller that
+ * cannot tell "there is nothing here" from "I was not allowed to look" cannot
+ * report either honestly.
+ */
+export declare function collectRecords(roots: readonly string[]): {
+    readonly records: readonly IndexableRecord[];
+    readonly skipped: readonly string[];
+};
+/** Build a fresh index over the roots. Cheap enough to do on demand. */
+export declare function buildIndex(roots: readonly string[]): {
+    readonly index: LibraryIndex;
+    readonly skipped: readonly string[];
+};
+/**
+ * Build an index, yielding often enough that a caller can stop it.
+ *
+ * The synchronous builder above is what a tool call uses: it is one pass over
+ * a directory and returning a promise would buy nothing. A refresh is
+ * different — it is a person waiting, it has a deadline, and it has to be
+ * abandonable — so this one checks the signal between files and hands the
+ * event loop back so the check can actually fire.
+ *
+ * It builds into a *new* index. Nothing in service is touched until the caller
+ * decides to swap, which is what makes a failed or abandoned rebuild leave the
+ * previous generation exactly as it was.
+ */
+export declare function buildIndexCancellable(roots: readonly string[], signal: AbortSignal): Promise<{
+    readonly index: LibraryIndex | null;
+    readonly skipped: readonly string[];
+    readonly sourceCount: number;
+}>;
+/**
+ * Register the search tool, and hand back the index it holds.
+ *
+ * The accessor is returned rather than the index itself so the read plane sees
+ * a rebuild the moment it happens, without either side holding a reference to
+ * an object the other has replaced. One index answers both the agent's tool
+ * and the person's surface: two would drift inside a release and disagree
+ * about what the library contains.
+ */
+export declare function applyLibrarySearch(ctx: Context, config: LibrarySearchConfig): {
+    readonly index: () => LibraryIndex;
+};
+//# sourceMappingURL=library-search.d.ts.map