@deepwatch/dsh-library 0.1.0

package/lib/search.d.ts

@@ -0,0 +1,129 @@
+/**
+ * Library search: finding the source, and saying how it was found.
+ *
+ * Search here has one unusual requirement. It has to report *which retrieval
+ * path produced a result*, because the two paths make different promises. A
+ * lexical hit means those characters are in that source at that moment. A
+ * semantic hit means something in that source was near the query in an
+ * embedding space, which is a much weaker claim and occasionally a wrong one.
+ *
+ * A product that merged them into one ranked list with one relevance number
+ * would be a product where "it found nothing" and "the embedding model was not
+ * installed" look identical, and where a paraphrase match and an exact quote
+ * look equally certain. So {@link searchPlan} states the path before anything
+ * runs, and every hit carries the path that produced it.
+ *
+ * Facets are computed from the results rather than from a fixed taxonomy. A
+ * facet that offers "Arabic (0)" on a library with no Arabic in it is a filter
+ * that teaches people the filter is broken.
+ *
+ * @module @deepwatch/dsh-library/search
+ */
+import type { TemporalRange } from '@deepwatch/dsh-contracts';
+import type { Source, SourceKind, IndexState } from './sources.js';
+/** How a result was retrieved. */
+export type RetrievalPath = 'lexical' | 'semantic' | 'both';
+/** What the engine can actually do here. */
+export interface SearchCapabilities {
+    /** Substring and token matching over extracted text. */
+    readonly lexical: boolean;
+    /** Embedding retrieval. Requires a bound embeddings role. */
+    readonly semantic: boolean;
+}
+/** What search will do, decided before it runs. */
+export interface SearchPlan {
+    readonly path: RetrievalPath | 'none';
+    /** One sentence for the results header. Always populated. */
+    readonly explanation: string;
+    /** What is missing, and what to do about it. Empty when nothing is. */
+    readonly degradedBecause: string;
+    readonly fix: string;
+}
+/**
+ * Decide the retrieval path.
+ *
+ * Semantic-only is a real state and is reported as one rather than silently
+ * treated as "search works". A library where exact-phrase search is
+ * unavailable behaves very differently from one where it is not, and a user
+ * searching for an error code needs to know which they are in.
+ */
+export declare function searchPlan(capabilities: SearchCapabilities): SearchPlan;
+/** One hit inside one source. */
+export interface SearchHit {
+    readonly sourceId: string;
+    readonly sourceRevisionId: string;
+    /** Where in the source, when the modality has a clock. */
+    readonly range: TemporalRange | null;
+    /** The matched text, verbatim and in its original script. */
+    readonly text: string;
+    /** Which path produced this hit. */
+    readonly path: RetrievalPath;
+    /**
+     * Score, in the producing path's own units.
+     *
+     * Deliberately not normalized across paths. A lexical rank and a cosine
+     * similarity are not comparable, and putting them on one 0–1 scale would
+     * manufacture a comparison that does not exist.
+     */
+    readonly score: number;
+    /** Evidence this hit resolves to, when the engine minted any. */
+    readonly evidenceIds: readonly string[];
+}
+/** A result: one source, and the hits inside it. */
+export interface SearchResult {
+    readonly sourceId: string;
+    readonly title: string;
+    readonly kind: SourceKind;
+    readonly hits: readonly SearchHit[];
+    /** Whether the hits are against the source's current revision. */
+    readonly current: boolean;
+}
+/** One facet value and how many results carry it. */
+export interface FacetValue {
+    readonly value: string;
+    readonly count: number;
+}
+/** The facets computed from a result set. */
+export interface Facets {
+    readonly kind: readonly FacetValue[];
+    readonly indexState: readonly FacetValue[];
+    readonly collection: readonly FacetValue[];
+    readonly script: readonly FacetValue[];
+    readonly path: readonly FacetValue[];
+}
+/** What a search was narrowed to. */
+export interface SearchFilters {
+    readonly kinds?: readonly SourceKind[];
+    readonly collections?: readonly string[];
+    readonly indexStates?: readonly IndexState[];
+    readonly scripts?: readonly string[];
+    /** Only hits from the current revision of each source. */
+    readonly currentOnly?: boolean;
+}
+/**
+ * Compute facets over a result set.
+ *
+ * Only values that actually occur. A facet list generated from the schema
+ * rather than from the results offers filters that return nothing, and a
+ * filter that returns nothing is indistinguishable from a broken one.
+ */
+export declare function facetsFor(results: readonly SearchResult[], sources: readonly Source[]): Facets;
+/** Apply filters to a result set. Pure; the engine does the retrieval. */
+export declare function applyFilters(results: readonly SearchResult[], sources: readonly Source[], filters: SearchFilters): readonly SearchResult[];
+/**
+ * Order results for display.
+ *
+ * Within a source, hits are ordered by path and then by time — not by score
+ * across paths, because the scores are not comparable. Across sources, the
+ * source with the strongest lexical evidence leads, because an exact match is
+ * the strongest claim search can make.
+ */
+export declare function rankResults(results: readonly SearchResult[]): readonly SearchResult[];
+/**
+ * One line above the results, stating what was searched and how.
+ *
+ * Always says the path. "12 results" alone invites the reading that the library
+ * was searched thoroughly, which may not be true.
+ */
+export declare function describeSearch(plan: SearchPlan, results: readonly SearchResult[]): string;
+//# sourceMappingURL=search.d.ts.map

package/lib/search.js

@@ -0,0 +1,180 @@
+/**
+ * Library search: finding the source, and saying how it was found.
+ *
+ * Search here has one unusual requirement. It has to report *which retrieval
+ * path produced a result*, because the two paths make different promises. A
+ * lexical hit means those characters are in that source at that moment. A
+ * semantic hit means something in that source was near the query in an
+ * embedding space, which is a much weaker claim and occasionally a wrong one.
+ *
+ * A product that merged them into one ranked list with one relevance number
+ * would be a product where "it found nothing" and "the embedding model was not
+ * installed" look identical, and where a paraphrase match and an exact quote
+ * look equally certain. So {@link searchPlan} states the path before anything
+ * runs, and every hit carries the path that produced it.
+ *
+ * Facets are computed from the results rather than from a fixed taxonomy. A
+ * facet that offers "Arabic (0)" on a library with no Arabic in it is a filter
+ * that teaches people the filter is broken.
+ *
+ * @module @deepwatch/dsh-library/search
+ */
+/**
+ * Decide the retrieval path.
+ *
+ * Semantic-only is a real state and is reported as one rather than silently
+ * treated as "search works". A library where exact-phrase search is
+ * unavailable behaves very differently from one where it is not, and a user
+ * searching for an error code needs to know which they are in.
+ */
+export function searchPlan(capabilities) {
+    if (capabilities.lexical && capabilities.semantic) {
+        return {
+            path: 'both',
+            explanation: 'Hybrid search: exact matches and meaning-based matches, marked separately.',
+            degradedBecause: '',
+            fix: '',
+        };
+    }
+    if (capabilities.lexical) {
+        return {
+            path: 'lexical',
+            explanation: 'Exact matching only. A paraphrase of what was said will not be found.',
+            degradedBecause: 'No embeddings role is bound, so semantic retrieval is unavailable.',
+            fix: 'Bind an embeddings role in Settings to search by meaning as well.',
+        };
+    }
+    if (capabilities.semantic) {
+        return {
+            path: 'semantic',
+            explanation: 'Meaning-based matching only. An exact phrase may rank below a paraphrase.',
+            degradedBecause: 'The lexical index is unavailable.',
+            fix: 'Re-index the library to restore exact matching.',
+        };
+    }
+    return {
+        path: 'none',
+        explanation: 'Search is unavailable.',
+        degradedBecause: 'Neither the lexical index nor an embeddings role is available.',
+        fix: 'Index a source, or bind an embeddings role in Settings.',
+    };
+}
+/** Count values, dropping the empty ones. */
+function tally(values) {
+    const counts = new Map();
+    for (const value of values)
+        counts.set(value, (counts.get(value) ?? 0) + 1);
+    return [...counts.entries()]
+        .map(([value, count]) => ({ value, count }))
+        .sort((left, right) => {
+        const byCount = right.count - left.count;
+        return byCount !== 0 ? byCount : left.value.localeCompare(right.value);
+    });
+}
+/**
+ * Compute facets over a result set.
+ *
+ * Only values that actually occur. A facet list generated from the schema
+ * rather than from the results offers filters that return nothing, and a
+ * filter that returns nothing is indistinguishable from a broken one.
+ */
+export function facetsFor(results, sources) {
+    const byId = new Map(sources.map(source => [source.sourceId, source]));
+    const kinds = [];
+    const indexStates = [];
+    const collections = [];
+    const scripts = [];
+    const paths = [];
+    for (const result of results) {
+        kinds.push(result.kind);
+        const source = byId.get(result.sourceId);
+        if (source !== undefined) {
+            for (const collection of source.collections)
+                collections.push(collection);
+            for (const revision of source.revisions) {
+                if (!result.hits.some(hit => hit.sourceRevisionId === revision.sourceRevisionId))
+                    continue;
+                indexStates.push(revision.indexState);
+                for (const script of revision.scripts)
+                    scripts.push(script);
+            }
+        }
+        for (const hit of result.hits)
+            paths.push(hit.path);
+    }
+    return {
+        kind: tally(kinds),
+        indexState: tally(indexStates),
+        collection: tally(collections),
+        script: tally(scripts),
+        path: tally(paths),
+    };
+}
+/** Apply filters to a result set. Pure; the engine does the retrieval. */
+export function applyFilters(results, sources, filters) {
+    const byId = new Map(sources.map(source => [source.sourceId, source]));
+    return results.filter(result => {
+        if (filters.kinds !== undefined && !filters.kinds.includes(result.kind))
+            return false;
+        if (filters.currentOnly === true && !result.current)
+            return false;
+        const source = byId.get(result.sourceId);
+        if (filters.collections !== undefined) {
+            if (source === undefined)
+                return false;
+            if (!filters.collections.some(collection => source.collections.includes(collection)))
+                return false;
+        }
+        if (filters.indexStates !== undefined) {
+            if (source === undefined)
+                return false;
+            const states = source.revisions
+                .filter(revision => result.hits.some(hit => hit.sourceRevisionId === revision.sourceRevisionId))
+                .map(revision => revision.indexState);
+            if (!states.some(state => filters.indexStates?.includes(state) === true))
+                return false;
+        }
+        if (filters.scripts !== undefined) {
+            if (source === undefined)
+                return false;
+            const present = new Set(source.revisions.flatMap(revision => revision.scripts));
+            if (!filters.scripts.some(script => present.has(script)))
+                return false;
+        }
+        return true;
+    });
+}
+/**
+ * Order results for display.
+ *
+ * Within a source, hits are ordered by path and then by time — not by score
+ * across paths, because the scores are not comparable. Across sources, the
+ * source with the strongest lexical evidence leads, because an exact match is
+ * the strongest claim search can make.
+ */
+export function rankResults(results) {
+    const lexicalWeight = (result) => result.hits.filter(hit => hit.path === 'lexical' || hit.path === 'both').length;
+    return [...results].sort((left, right) => {
+        const byLexical = lexicalWeight(right) - lexicalWeight(left);
+        if (byLexical !== 0)
+            return byLexical;
+        const byHits = right.hits.length - left.hits.length;
+        if (byHits !== 0)
+            return byHits;
+        return left.sourceId.localeCompare(right.sourceId);
+    });
+}
+/**
+ * One line above the results, stating what was searched and how.
+ *
+ * Always says the path. "12 results" alone invites the reading that the library
+ * was searched thoroughly, which may not be true.
+ */
+export function describeSearch(plan, results) {
+    const hits = results.reduce((total, result) => total + result.hits.length, 0);
+    const count = `${String(hits)} hit(s) in ${String(results.length)} source(s)`;
+    return plan.degradedBecause === ''
+        ? `${count} · ${plan.explanation}`
+        : `${count} · ${plan.explanation} ${plan.degradedBecause}`;
+}
+//# sourceMappingURL=search.js.map

package/lib/sources.d.ts

@@ -0,0 +1,146 @@
+/**
+ * The Library: sources, their revisions, and what is still true about them.
+ *
+ * The Library is not memory. That separation is the first thing this module
+ * exists to hold: memory is what the system believes, and the Library is what
+ * it has *seen*. Conflating them produces the worst version of both — a
+ * knowledge base you cannot cite and an evidence store that argues with you.
+ * So nothing here has a scope, a confidence or a status; those are memory's
+ * vocabulary. A source has revisions, and evidence is addressed to one of them.
+ *
+ * The second thing it holds is the revision rule, which is the whole reason
+ * evidence ids are worth anything:
+ *
+ * > A source that changed is a different source revision, and evidence stays
+ * > addressed to the revision it was taken from — forever.
+ *
+ * When a page is re-indexed, the evidence from last week does not become
+ * wrong, and it does not become unreachable. It becomes *stale*: it still
+ * opens, still resolves to the same frame at the same millisecond, and now
+ * carries a note saying the source has moved on. Deleting it would destroy the
+ * receipt; silently re-pointing it at the new revision would be worse, because
+ * the citation would then be to something nobody observed.
+ *
+ * @module @deepwatch/dsh-library/sources
+ */
+import type { EvidenceRecord, Freshness, TemporalRange } from '@deepwatch/dsh-contracts';
+import type { ScriptTag } from '@deepwatch/dsh-technology';
+/** What kind of thing a source is. */
+export type SourceKind = 'video' | 'audio' | 'page' | 'stream' | 'document' | 'screen_capture';
+/** Where an index is in its life. */
+export type IndexState = 
+/** Known to the Library, nothing extracted. */
+'not_indexed' | 'indexing'
+/** Extracted and searchable. */
+ | 'indexed'
+/** Indexed, but against an older revision than the current one. */
+ | 'stale' | 'failed';
+/** One immutable version of a source. */
+export interface SourceRevision {
+    readonly sourceRevisionId: string;
+    readonly sourceId: string;
+    /** Monotonic within a source. Revision 1 is the first thing observed. */
+    readonly revision: number;
+    /** Digest of the bytes, which is what makes "changed" a fact. */
+    readonly contentDigest: string;
+    readonly observedAt: string;
+    readonly durationMs: number | null;
+    readonly indexState: IndexState;
+    /** Why indexing failed, when it did. */
+    readonly indexError: string | null;
+    /** Scripts detected in this revision's text. Structural, not measured. */
+    readonly scripts: readonly ScriptTag[];
+}
+/** A source, with every revision the Library holds. */
+export interface Source {
+    readonly sourceId: string;
+    readonly kind: SourceKind;
+    /** What it is called. Presentation only; nothing resolves from it. */
+    readonly title: string;
+    /** URL or path. */
+    readonly locator: string;
+    readonly revisions: readonly SourceRevision[];
+    /** Collections this source belongs to. */
+    readonly collections: readonly string[];
+    /** Entities extracted from it, when the engine extracts any. */
+    readonly entities: readonly string[];
+}
+/** A named group of sources. Curation, not classification. */
+export interface Collection {
+    readonly collectionId: string;
+    readonly name: string;
+    readonly sourceIds: readonly string[];
+}
+/**
+ * The current revision of a source.
+ *
+ * Highest revision number, not most recently observed — a re-index of an old
+ * revision must not become "current" because it happened last.
+ */
+export declare function currentRevision(source: Source): SourceRevision | null;
+/** Find one revision by id, wherever it sits in the history. */
+export declare function findRevision(source: Source, sourceRevisionId: string): SourceRevision | null;
+/**
+ * Whether a source revision is still the one a fresh observation would produce.
+ *
+ * Separated from freshness below because they are different questions: this is
+ * about the *source*, and freshness is about one piece of evidence taken from
+ * it. A source can be current while a specific observation from it is stale,
+ * when the observation covers a range the new revision no longer contains.
+ */
+export declare function isCurrentRevision(source: Source, sourceRevisionId: string): boolean;
+/**
+ * Freshness of one evidence record, given what the Library now holds.
+ *
+ * The rules, in order:
+ *
+ * - Evidence whose source the Library does not hold is `unavailable`. Not
+ *   `expired` — nobody knows whether it expired; it simply cannot be checked.
+ * - Evidence against the current revision keeps whatever freshness the engine
+ *   assigned it, including `gap`. Freshness is not the Library's to upgrade.
+ * - Evidence against a superseded revision is `stale`. It still resolves; it
+ *   no longer describes the source.
+ *
+ * Note what this function never returns: `current` for something it was not
+ * already told was current. A Library that could promote evidence to fresh
+ * would be a Library that re-validates by assertion.
+ */
+export declare function freshnessOf(evidence: Pick<EvidenceRecord, 'sourceRevisionId' | 'freshness'>, sources: readonly Source[]): Freshness;
+/**
+ * Whether an evidence id can still be opened.
+ *
+ * Always true when the Library holds its revision, whatever the freshness. The
+ * function exists to make that a stated guarantee rather than an accident of
+ * whichever query happens to run: a stale citation that stopped opening would
+ * turn every old receipt into a dead link.
+ */
+export declare function isAddressable(evidence: Pick<EvidenceRecord, 'sourceRevisionId'>, sources: readonly Source[]): boolean;
+/** A place in a source that a citation resolves to. */
+export interface EvidenceLocation {
+    readonly sourceId: string;
+    readonly sourceRevisionId: string;
+    readonly revision: number;
+    readonly range: TemporalRange | null;
+    readonly freshness: Freshness;
+    /** Whether the source has moved on since this was observed. */
+    readonly supersededBy: string | null;
+}
+/**
+ * Resolve an evidence record to a place in the Library.
+ *
+ * Returns null only when the revision is not held. Everything else resolves,
+ * including evidence from four revisions ago — with `supersededBy` naming what
+ * replaced it, so the surface can offer "look at the same moment in the
+ * current revision" without silently doing it.
+ */
+export declare function locate(evidence: Pick<EvidenceRecord, 'sourceRevisionId' | 'temporalRange' | 'freshness'>, sources: readonly Source[]): EvidenceLocation | null;
+/**
+ * Record a new revision of a source.
+ *
+ * Old revisions are kept, and their index state is marked `stale` rather than
+ * removed. That is the mechanism behind every "old evidence still opens"
+ * guarantee above — there is no code path that discards a revision, so there is
+ * no code path that could orphan a citation.
+ */
+export declare function withRevision(source: Source, revision: SourceRevision): Source;
+//# sourceMappingURL=sources.d.ts.map

package/lib/sources.js

@@ -0,0 +1,135 @@
+/**
+ * The Library: sources, their revisions, and what is still true about them.
+ *
+ * The Library is not memory. That separation is the first thing this module
+ * exists to hold: memory is what the system believes, and the Library is what
+ * it has *seen*. Conflating them produces the worst version of both — a
+ * knowledge base you cannot cite and an evidence store that argues with you.
+ * So nothing here has a scope, a confidence or a status; those are memory's
+ * vocabulary. A source has revisions, and evidence is addressed to one of them.
+ *
+ * The second thing it holds is the revision rule, which is the whole reason
+ * evidence ids are worth anything:
+ *
+ * > A source that changed is a different source revision, and evidence stays
+ * > addressed to the revision it was taken from — forever.
+ *
+ * When a page is re-indexed, the evidence from last week does not become
+ * wrong, and it does not become unreachable. It becomes *stale*: it still
+ * opens, still resolves to the same frame at the same millisecond, and now
+ * carries a note saying the source has moved on. Deleting it would destroy the
+ * receipt; silently re-pointing it at the new revision would be worse, because
+ * the citation would then be to something nobody observed.
+ *
+ * @module @deepwatch/dsh-library/sources
+ */
+/**
+ * The current revision of a source.
+ *
+ * Highest revision number, not most recently observed — a re-index of an old
+ * revision must not become "current" because it happened last.
+ */
+export function currentRevision(source) {
+    let best = null;
+    for (const revision of source.revisions) {
+        if (best === null || revision.revision > best.revision)
+            best = revision;
+    }
+    return best;
+}
+/** Find one revision by id, wherever it sits in the history. */
+export function findRevision(source, sourceRevisionId) {
+    return source.revisions.find(revision => revision.sourceRevisionId === sourceRevisionId) ?? null;
+}
+/**
+ * Whether a source revision is still the one a fresh observation would produce.
+ *
+ * Separated from freshness below because they are different questions: this is
+ * about the *source*, and freshness is about one piece of evidence taken from
+ * it. A source can be current while a specific observation from it is stale,
+ * when the observation covers a range the new revision no longer contains.
+ */
+export function isCurrentRevision(source, sourceRevisionId) {
+    return currentRevision(source)?.sourceRevisionId === sourceRevisionId;
+}
+/**
+ * Freshness of one evidence record, given what the Library now holds.
+ *
+ * The rules, in order:
+ *
+ * - Evidence whose source the Library does not hold is `unavailable`. Not
+ *   `expired` — nobody knows whether it expired; it simply cannot be checked.
+ * - Evidence against the current revision keeps whatever freshness the engine
+ *   assigned it, including `gap`. Freshness is not the Library's to upgrade.
+ * - Evidence against a superseded revision is `stale`. It still resolves; it
+ *   no longer describes the source.
+ *
+ * Note what this function never returns: `current` for something it was not
+ * already told was current. A Library that could promote evidence to fresh
+ * would be a Library that re-validates by assertion.
+ */
+export function freshnessOf(evidence, sources) {
+    const owner = sources.find(source => source.revisions.some(revision => revision.sourceRevisionId === evidence.sourceRevisionId));
+    if (owner === undefined)
+        return 'unavailable';
+    if (!isCurrentRevision(owner, evidence.sourceRevisionId))
+        return 'stale';
+    return evidence.freshness;
+}
+/**
+ * Whether an evidence id can still be opened.
+ *
+ * Always true when the Library holds its revision, whatever the freshness. The
+ * function exists to make that a stated guarantee rather than an accident of
+ * whichever query happens to run: a stale citation that stopped opening would
+ * turn every old receipt into a dead link.
+ */
+export function isAddressable(evidence, sources) {
+    return sources.some(source => source.revisions.some(revision => revision.sourceRevisionId === evidence.sourceRevisionId));
+}
+/**
+ * Resolve an evidence record to a place in the Library.
+ *
+ * Returns null only when the revision is not held. Everything else resolves,
+ * including evidence from four revisions ago — with `supersededBy` naming what
+ * replaced it, so the surface can offer "look at the same moment in the
+ * current revision" without silently doing it.
+ */
+export function locate(evidence, sources) {
+    for (const source of sources) {
+        const revision = findRevision(source, evidence.sourceRevisionId);
+        if (revision === null)
+            continue;
+        const current = currentRevision(source);
+        return {
+            sourceId: source.sourceId,
+            sourceRevisionId: revision.sourceRevisionId,
+            revision: revision.revision,
+            range: evidence.temporalRange,
+            freshness: freshnessOf(evidence, sources),
+            supersededBy: current === null || current.sourceRevisionId === revision.sourceRevisionId
+                ? null
+                : current.sourceRevisionId,
+        };
+    }
+    return null;
+}
+/**
+ * Record a new revision of a source.
+ *
+ * Old revisions are kept, and their index state is marked `stale` rather than
+ * removed. That is the mechanism behind every "old evidence still opens"
+ * guarantee above — there is no code path that discards a revision, so there is
+ * no code path that could orphan a citation.
+ */
+export function withRevision(source, revision) {
+    const existing = source.revisions.filter(entry => entry.sourceRevisionId !== revision.sourceRevisionId);
+    const superseded = existing.map(entry => entry.revision < revision.revision && entry.indexState === 'indexed'
+        ? { ...entry, indexState: 'stale' }
+        : entry);
+    return {
+        ...source,
+        revisions: [...superseded, revision].sort((left, right) => left.revision - right.revision),
+    };
+}
+//# sourceMappingURL=sources.js.map