@retinue/agentkit 0.1.0

package/dist/knowledge/retrieval.d.ts

@@ -0,0 +1,146 @@
+/**
+ * Hybrid retrieval (#136).
+ *
+ * Semantic search misses what it was never trained on — a product code, an error number, a campaign
+ * identifier. An embedding of `ERR-4021` is an embedding of a string that looks like other strings. Keyword
+ * search misses everything phrased differently from the document. Neither is sufficient, which is why docs/05
+ * specifies both.
+ *
+ * **Fusion is reciprocal rank fusion, not weighted score addition**, and the reason is that the two scores are
+ * not comparable. A cosine similarity is bounded and roughly linear in relevance; `ts_rank_cd` is unbounded and
+ * corpus-dependent. Adding them with weights means choosing a constant that is wrong for some corpus, and the
+ * failure is silent — one signal quietly dominates and the hybrid is the worse of the two. RRF uses only the
+ * *rank*, so it needs no calibration and cannot be dominated:
+ *
+ *     score(d) = Σ over signals of 1 / (K + rank(d))
+ *
+ * A document ranked first by one signal and absent from the other still beats one ranked fifth by both, which
+ * is the behaviour that makes hybrid better than either — the exact-term hit surfaces even though the semantic
+ * signal never saw it.
+ *
+ * **`K = 60`** is the value from Cormack, Clarke and Buettcher's original TREC work and the one every
+ * implementation since has used. It is large relative to the ranks that matter, which flattens the difference
+ * between rank 1 and rank 2 and lets *agreement between signals* outweigh a single signal's confidence.
+ *
+ * **The empty answer is a distinct shape.** `RetrievalOutcome` is a union, so "found nothing" has no success
+ * shape to hide in: a caller cannot accidentally treat an empty hit list as a weak answer, and the model is
+ * told there is nothing rather than handed the least-bad match — which it would cite. AC-4 is a type here, not
+ * a convention.
+ */
+import type { TenantId } from "../core/ids.js";
+import type { KeywordIndex, KnowledgeChunk, KnowledgeSourceType, VectorIndex } from "../persistence/index.js";
+import type { EmbeddingProvider } from "./index.js";
+/** The rank-fusion constant. See the note above on why 60 and why rank rather than score. */
+export declare const RRF_K = 60;
+/** How many candidates each signal contributes before fusion. */
+export declare const DEFAULT_CANDIDATES = 20;
+/**
+ * Below this fused score, relative to the best result, nothing is returned.
+ *
+ * The floor that makes AC-4 real. Without one, every query returns *something* — the least-bad match — and a
+ * model handed the least-bad match cites it. 0.4 of the best score is loose enough not to discard a genuine
+ * second-best answer and tight enough to reject a corpus that simply has nothing to say.
+ */
+export declare const DEFAULT_RELEVANCE_FLOOR = 0.4;
+export type RetrievalMode = "semantic" | "keyword" | "hybrid";
+/** What a citation needs, derived from a hit so there is one shape rather than each caller's own (AC-6). */
+export type SourceReference = {
+    readonly sourceType: KnowledgeSourceType;
+    readonly sourceId: string;
+    readonly chunkIndex: number;
+    readonly chunkId: string;
+    /** The heading path, when the chunker found one. What makes a citation point at a place, not a document. */
+    readonly locator?: string;
+};
+export type RetrievalHit = {
+    readonly chunk: KnowledgeChunk;
+    /** The fused score, 0–1 relative to the best hit. Not comparable across queries, only within one. */
+    readonly score: number;
+    /** Which signals found it, for explaining a result and for measuring the fusion. */
+    readonly signals: readonly RetrievalMode[];
+    readonly reference: SourceReference;
+};
+/**
+ * Why retrieval returned nothing.
+ *
+ * Separate values because the assistant's sentence differs: "nothing in your documents mentions that" is a
+ * different answer from "you have no indexed documents", and telling a user the first when the second is true
+ * sends them looking for content they never uploaded.
+ */
+export declare const NO_RESULT_REASONS: readonly ["nothing-indexed", "no-match", "below-threshold", "no-access"];
+export type NoResultReason = (typeof NO_RESULT_REASONS)[number];
+export type RetrievalOutcome = {
+    readonly found: true;
+    readonly hits: readonly RetrievalHit[];
+    readonly mode: RetrievalMode;
+} | {
+    readonly found: false;
+    readonly reason: NoResultReason;
+    readonly message: string;
+    readonly mode: RetrievalMode;
+};
+/**
+ * Reorders a fused candidate set — AC-5.
+ *
+ * A port, and **switchable**, because a reranker's value is a claim that has to be provable. A cross-encoder is
+ * materially more expensive than the retrieval it reorders, so "we rerank" without a measured contribution is
+ * a cost nobody justified. Absent means fusion order stands, which is the honest default.
+ */
+export interface Reranker {
+    readonly id: string;
+    rerank(input: {
+        readonly query: string;
+        readonly candidates: readonly RetrievalHit[];
+        readonly limit: number;
+    }): Promise<readonly RetrievalHit[]>;
+}
+export type RetrieverDeps = {
+    readonly vector: VectorIndex;
+    readonly keyword: KeywordIndex;
+    readonly embeddings: EmbeddingProvider;
+    readonly reranker?: Reranker;
+    readonly candidates?: number;
+    /** Relative: how much worse than the best hit is still worth returning. */
+    readonly relevanceFloor?: number;
+    /**
+     * Absolute: how similar a vector hit must be to be a candidate at all.
+     *
+     * Two floors because they answer different questions, and one cannot do both — a *relative* floor can never
+     * reject a uniformly poor result set, because something is always the best of it. This is what makes AC-4
+     * possible: without it the vector index returns every chunk it is asked for (0.5 is orthogonal, not
+     * "no match"), the best of them normalises to 1.0, and every query finds something.
+     */
+    readonly semanticFloor?: number;
+};
+export type RetrieveInput = {
+    readonly query: string;
+    /**
+     * The subjects this caller may read.
+     *
+     * Required, and passed to *both* signals. A filter applied to one and not the other is a filter that does not
+     * exist — the unfiltered signal would surface the chunk and fusion would rank it.
+     */
+    readonly authSubjects: readonly string[];
+    readonly limit: number;
+    readonly sourceTypes?: readonly KnowledgeSourceType[];
+    /** Defaults to `hybrid`. The other two exist so the hybrid claim can be measured against them. */
+    readonly mode?: RetrievalMode;
+};
+export declare const createRetriever: (deps: RetrieverDeps) => {
+    rerankerId: string | null;
+    retrieve(context: {
+        readonly tenantId: TenantId;
+    }, input: RetrieveInput): Promise<RetrievalOutcome>;
+};
+export type Retriever = ReturnType<typeof createRetriever>;
+/**
+ * A reranker that promotes chunks matching the query's rare exact terms.
+ *
+ * Deliberately simple and deliberately *not* a model: this exists so AC-5's "switchable and measured" has
+ * something real to switch on, and so the measurement harness has a baseline that is better than nothing and
+ * obviously cheaper than a cross-encoder. It promotes a candidate containing a query term verbatim, which is
+ * the signal fusion is weakest on — a rank-based fusion cannot know that one of the two signals matched an
+ * exact identifier rather than a common word.
+ */
+export declare const createExactTermReranker: () => Reranker;
+//# sourceMappingURL=retrieval.d.ts.map

package/dist/knowledge/retrieval.js

@@ -0,0 +1,170 @@
+/**
+ * Hybrid retrieval (#136).
+ *
+ * Semantic search misses what it was never trained on — a product code, an error number, a campaign
+ * identifier. An embedding of `ERR-4021` is an embedding of a string that looks like other strings. Keyword
+ * search misses everything phrased differently from the document. Neither is sufficient, which is why docs/05
+ * specifies both.
+ *
+ * **Fusion is reciprocal rank fusion, not weighted score addition**, and the reason is that the two scores are
+ * not comparable. A cosine similarity is bounded and roughly linear in relevance; `ts_rank_cd` is unbounded and
+ * corpus-dependent. Adding them with weights means choosing a constant that is wrong for some corpus, and the
+ * failure is silent — one signal quietly dominates and the hybrid is the worse of the two. RRF uses only the
+ * *rank*, so it needs no calibration and cannot be dominated:
+ *
+ *     score(d) = Σ over signals of 1 / (K + rank(d))
+ *
+ * A document ranked first by one signal and absent from the other still beats one ranked fifth by both, which
+ * is the behaviour that makes hybrid better than either — the exact-term hit surfaces even though the semantic
+ * signal never saw it.
+ *
+ * **`K = 60`** is the value from Cormack, Clarke and Buettcher's original TREC work and the one every
+ * implementation since has used. It is large relative to the ranks that matter, which flattens the difference
+ * between rank 1 and rank 2 and lets *agreement between signals* outweigh a single signal's confidence.
+ *
+ * **The empty answer is a distinct shape.** `RetrievalOutcome` is a union, so "found nothing" has no success
+ * shape to hide in: a caller cannot accidentally treat an empty hit list as a weak answer, and the model is
+ * told there is nothing rather than handed the least-bad match — which it would cite. AC-4 is a type here, not
+ * a convention.
+ */
+import { SEMANTIC_RELEVANCE_FLOOR } from "../persistence/index.js";
+/** The rank-fusion constant. See the note above on why 60 and why rank rather than score. */
+export const RRF_K = 60;
+/** How many candidates each signal contributes before fusion. */
+export const DEFAULT_CANDIDATES = 20;
+/**
+ * Below this fused score, relative to the best result, nothing is returned.
+ *
+ * The floor that makes AC-4 real. Without one, every query returns *something* — the least-bad match — and a
+ * model handed the least-bad match cites it. 0.4 of the best score is loose enough not to discard a genuine
+ * second-best answer and tight enough to reject a corpus that simply has nothing to say.
+ */
+export const DEFAULT_RELEVANCE_FLOOR = 0.4;
+/**
+ * Why retrieval returned nothing.
+ *
+ * Separate values because the assistant's sentence differs: "nothing in your documents mentions that" is a
+ * different answer from "you have no indexed documents", and telling a user the first when the second is true
+ * sends them looking for content they never uploaded.
+ */
+export const NO_RESULT_REASONS = ["nothing-indexed", "no-match", "below-threshold", "no-access"];
+const referenceFor = (chunk) => ({
+    sourceType: chunk.sourceType,
+    sourceId: chunk.sourceId,
+    chunkIndex: chunk.chunkIndex,
+    chunkId: chunk.id,
+    ...(chunk.locator === undefined ? {} : { locator: chunk.locator }),
+});
+const NO_RESULT_MESSAGES = {
+    "nothing-indexed": "There is no indexed material to search yet.",
+    "no-match": "Nothing in the available material matches that.",
+    "below-threshold": "Nothing in the available material is a close enough match to rely on.",
+    "no-access": "There is no material you have access to that matches that.",
+};
+export const createRetriever = (deps) => {
+    const candidateCount = deps.candidates ?? DEFAULT_CANDIDATES;
+    const floor = deps.relevanceFloor ?? DEFAULT_RELEVANCE_FLOOR;
+    const semanticFloor = deps.semanticFloor ?? SEMANTIC_RELEVANCE_FLOOR;
+    return {
+        rerankerId: deps.reranker?.id ?? null,
+        async retrieve(context, input) {
+            const mode = input.mode ?? "hybrid";
+            // Checked before either index is asked. An empty subject list is not a query with no results — it is a
+            // caller with no access, and the two want different sentences.
+            if (input.authSubjects.length === 0)
+                return { found: false, reason: "no-access", message: NO_RESULT_MESSAGES["no-access"], mode };
+            if (input.query.trim() === "")
+                return { found: false, reason: "no-match", message: NO_RESULT_MESSAGES["no-match"], mode };
+            const scope = {
+                tenantId: context.tenantId,
+                authSubjects: input.authSubjects,
+                limit: candidateCount,
+                ...(input.sourceTypes === undefined ? {} : { sourceTypes: input.sourceTypes }),
+            };
+            const semantic = mode === "keyword"
+                ? []
+                : await (async () => {
+                    const [embedding] = await deps.embeddings.embed([input.query]);
+                    // The absolute floor goes to the index, not applied afterwards: a chunk that shares nothing with
+                    // the query must not be a candidate, because fusion would rank it first in a result set where
+                    // nothing is good.
+                    return deps.vector.search({ ...scope, embedding: embedding ?? [], minScore: semanticFloor });
+                })();
+            const lexical = mode === "semantic" ? [] : await deps.keyword.search({ ...scope, query: input.query });
+            if (semantic.length === 0 && lexical.length === 0)
+                return { found: false, reason: "no-match", message: NO_RESULT_MESSAGES["no-match"], mode };
+            // RRF. Rank, not score: see the note at the top on why adding two incomparable scales fails silently.
+            const fused = new Map();
+            const contribute = (hits, signal) => {
+                hits.forEach((hit, rank) => {
+                    const existing = fused.get(hit.chunk.id);
+                    const increment = 1 / (RRF_K + rank + 1);
+                    if (existing === undefined)
+                        fused.set(hit.chunk.id, { chunk: hit.chunk, score: increment, signals: new Set([signal]) });
+                    else {
+                        existing.score += increment;
+                        existing.signals.add(signal);
+                    }
+                });
+            };
+            contribute(semantic, "semantic");
+            contribute(lexical, "keyword");
+            const ordered = [...fused.values()].sort((a, b) => b.score !== a.score ? b.score - a.score : a.chunk.id.localeCompare(b.chunk.id));
+            const best = ordered[0]?.score ?? 0;
+            const candidates = ordered.map((entry) => ({
+                chunk: entry.chunk,
+                // Normalised against the best fused score, so the floor means the same thing whatever the corpus.
+                score: best === 0 ? 0 : entry.score / best,
+                signals: [...entry.signals],
+                reference: referenceFor(entry.chunk),
+            }));
+            const relevant = candidates.filter((hit) => hit.score >= floor);
+            if (relevant.length === 0)
+                // Something was found and none of it was good enough. A distinct reason from "no match", because the
+                // useful answer differs: one says rephrase, the other says there is nothing there.
+                return { found: false, reason: "below-threshold", message: NO_RESULT_MESSAGES["below-threshold"], mode };
+            const hits = deps.reranker === undefined
+                ? relevant.slice(0, input.limit)
+                : await deps.reranker.rerank({ query: input.query, candidates: relevant, limit: input.limit });
+            // A reranker that returned nothing is a reranker that broke; fusion order is a better answer than an
+            // empty one, and a silent empty result here would look like AC-4 working.
+            return hits.length === 0
+                ? { found: true, hits: relevant.slice(0, input.limit), mode }
+                : { found: true, hits: hits.slice(0, input.limit), mode };
+        },
+    };
+};
+/**
+ * A reranker that promotes chunks matching the query's rare exact terms.
+ *
+ * Deliberately simple and deliberately *not* a model: this exists so AC-5's "switchable and measured" has
+ * something real to switch on, and so the measurement harness has a baseline that is better than nothing and
+ * obviously cheaper than a cross-encoder. It promotes a candidate containing a query term verbatim, which is
+ * the signal fusion is weakest on — a rank-based fusion cannot know that one of the two signals matched an
+ * exact identifier rather than a common word.
+ */
+export const createExactTermReranker = () => ({
+    id: "exact-term",
+    async rerank({ query, candidates, limit }) {
+        const terms = (query.toLowerCase().match(/[a-z0-9]+(?:-[a-z0-9]+)*/g) ?? []).filter(
+        // Only terms that look like identifiers: containing a digit or a hyphen. A common word appearing
+        // verbatim is not evidence of anything, and boosting on it would just re-rank by word frequency.
+        (t) => /[0-9]/.test(t) || t.includes("-"));
+        if (terms.length === 0)
+            return candidates.slice(0, limit);
+        const boosted = candidates.map((hit) => {
+            const content = hit.chunk.content.toLowerCase();
+            const matched = terms.filter((t) => content.includes(t)).length;
+            return { hit, matched };
+        });
+        return boosted
+            .sort((a, b) => b.matched !== a.matched
+            ? b.matched - a.matched
+            : // Fusion order within a tier, so reranking only ever *promotes* on evidence and never reshuffles
+                // arbitrarily.
+                b.hit.score - a.hit.score)
+            .map((b) => b.hit)
+            .slice(0, limit);
+    },
+});
+//# sourceMappingURL=retrieval.js.map

package/dist/loadtest/harness.d.ts

@@ -0,0 +1,168 @@
+/**
+ * The load and soak driver — AC-1, AC-2, AC-3, AC-4.
+ *
+ * Drives the **real durable path**: a real Postgres schema, a real BullMQ queue on a real Redis, several real
+ * `WorkerRuntime` instances competing for work, real atomic claims, real leases, real checkpoints and a real
+ * event log. The only synthetic part is the agent engine, and that is not a shortcut — see `scenario.ts`.
+ *
+ * What this is *not* is a request against a deployed HTTP endpoint. #144 asks for that, and there is no deployed
+ * instance yet; the layers this cannot reach are the GraphQL server and the process boundary between host and
+ * worker. Everything below them is exercised for real. That boundary is stated in the report the harness writes,
+ * not left for someone to infer from a green tick.
+ *
+ * Structured so the driver returns **data** and the judging happens in `metrics.ts` and `injection.ts`. A
+ * harness that both ran the load and decided whether it passed would be a harness whose verdict nobody could
+ * test, and the verdict is the part most likely to be quietly wrong.
+ */
+import { type JobConsumer } from "../worker/main.js";
+import type { ConversationId, RunId, TenantId } from "../core/ids.js";
+import type { JobDispatcher } from "../runtime/index.js";
+import type { SqlExecutor } from "../adapters/postgres/sql.js";
+import { type EffectLedger, type TrafficShape } from "./scenario.js";
+import { detectGrowth, type LoadStep, type ResourceSample } from "./metrics.js";
+export type HarnessConfig = {
+    readonly tenantId: TenantId;
+    /** Worker runtimes competing for the same queue. Several, or a claim race is never exercised. */
+    readonly workers: number;
+    readonly concurrency: number;
+    /** Short, so a killed worker's lease expires inside a test rather than inside a coffee break. */
+    readonly leaseMs: number;
+    readonly reapEveryMs: number;
+    readonly traffic: TrafficShape;
+    readonly seed: number;
+    /**
+     * The queue's bound. Past it, admission is refused.
+     *
+     * The mechanism AC-4 asks for. Without a bound, "overload" means an unbounded in-memory backlog, which is the
+     * failure mode rather than the test.
+     */
+    readonly maxQueueDepth: number;
+    readonly sampleEveryMs: number;
+};
+export declare const DEFAULT_HARNESS: Omit<HarnessConfig, "tenantId">;
+/**
+ * An in-process queue with a **bounded** depth and an explicit refusal.
+ *
+ * Used where the point is backpressure rather than Redis. Deliberately not BullMQ for the overload case: BullMQ's
+ * depth is Redis memory, so "bounded queueing" there is a Redis configuration question and the assertion would
+ * be about Redis. Here the bound is ours and the refusal is the platform's.
+ */
+export declare class QueueFull extends Error {
+    readonly depth: number;
+    readonly code = "resource-exhausted";
+    readonly retryable = true;
+    constructor(depth: number);
+}
+export type BoundedQueue = {
+    readonly dispatcher: JobDispatcher;
+    /**
+     * A **fresh consumer per worker**, not one shared object.
+     *
+     * `JobConsumer.stop()` has no handle to say *which* worker is stopping, so a shared consumer's `stop` had to
+     * clear every registration — which meant killing one worker silently stopped all three. The worker-kill
+     * injection then hung forever waiting for a drain that could never happen, produced no output at all, and it
+     * took a process listing to find. Each worker gets its own view, closing over its own slot.
+     */
+    consumerFor(): JobConsumer;
+    depth(): number;
+    peakDepth(): number;
+    refused(): number;
+    /** Jobs handed to a worker more than once. A pass that needed retries is not a clean first-attempt pass. */
+    retried(): number;
+    drained(): Promise<void>;
+};
+export declare const createBoundedQueue: (maxDepth: number, perWorkerConcurrency?: number) => BoundedQueue;
+export type WorkerHandle = {
+    readonly id: string;
+    /** Stop without draining — the worker-kill injector. */
+    kill(): Promise<void>;
+    shutdown(): Promise<void>;
+};
+export type Harness = {
+    readonly effects: EffectLedger;
+    readonly queue: BoundedQueue;
+    readonly workers: readonly WorkerHandle[];
+    /** Admit and enqueue one run. Throws `QueueFull` when the queue is at its bound. */
+    admit(input: {
+        conversationId: ConversationId;
+        runId: RunId;
+    }): Promise<void>;
+    /** Runs in a terminal state, from the store — not from the harness's own bookkeeping. */
+    terminalCount(): Promise<number>;
+    /** Every run status and its count, so a paused run is never miscounted as a failure. */
+    statusCounts(): Promise<Readonly<Record<string, number>>>;
+    /**
+     * Approve everything waiting, and re-enqueue it.
+     *
+     * A load test that left approvals pending would report them as lost work, which is the opposite of the truth:
+     * a run waiting for a human is the platform holding state correctly, and it is the longest-lived state it has.
+     * Deciding them here is also the only way the resume path is exercised under load.
+     *
+     * Returns how long each run waited, so approval wait time is measured rather than assumed.
+     */
+    approvePending(): Promise<readonly number[]>;
+    /**
+     * Wait until nothing with this id prefix is still in flight, then report what happened.
+     *
+     * Necessary because "the queue is empty" is not "the work is done": the queue empties when the last job reaches
+     * a worker. Without settling, a step's slowest runs are still executing when it is measured, and they count as
+     * failures — my first staircase reported `mode: errors` at 20/s for exactly that reason, which would have sent
+     * an operator hunting a failure that was really my clock.
+     *
+     * A run still non-terminal when the timeout expires *does* count as stuck. That is the honest line: waiting
+     * forever would hide a genuine hang, and not waiting at all reports one that is not there.
+     */
+    settle(input: {
+        readonly idPrefix: string;
+        readonly timeoutMs: number;
+    }): Promise<{
+        readonly completed: number;
+        readonly failed: number;
+        readonly stuck: number;
+        readonly stuckByStatus?: Readonly<Record<string, number>>;
+        readonly approvalWaitsMs: readonly number[];
+    }>;
+    /**
+     * End-to-end latency per finished run, in ms, from the store's own timestamps.
+     *
+     * The number that matters. Admission latency is a property of the enqueue and stays flat under any backlog;
+     * what a user experiences is admission to terminal, and only the store knows both ends.
+     */
+    terminalLatencies(idPrefix: string): Promise<readonly number[]>;
+    admittedCount(): number;
+    sample(): ResourceSample;
+    stop(): Promise<void>;
+};
+/**
+ * Build the harness over an already-migrated schema.
+ *
+ * Takes a `SqlExecutor` rather than a URL so the caller owns the connection: the database-unavailable injector
+ * needs to stop the server underneath a live pool, and a harness that owned the pool would reconnect on its own
+ * schedule and the injector could not control the window.
+ */
+export declare const createHarness: (input: {
+    readonly sql: SqlExecutor;
+    readonly config: HarnessConfig;
+    readonly now?: () => number;
+    readonly sleep?: (ms: number) => Promise<void>;
+}) => Promise<Harness>;
+/**
+ * One step of the load staircase.
+ *
+ * Offers a fixed rate for a fixed duration, then waits for the queue to drain before measuring. Measuring
+ * without draining would attribute a step's backlog to the *next* step and the staircase would show capacity
+ * that is really borrowed from earlier.
+ */
+export declare const runLoadStep: (input: {
+    readonly harness: Harness;
+    readonly offeredPerSecond: number;
+    readonly durationMs: number;
+    readonly tenantId: TenantId;
+    readonly startIndex: number;
+    /** How long to wait for a step's work to finish before calling the remainder stuck. */
+    readonly settleTimeoutMs?: number;
+    readonly now?: () => number;
+    readonly sleep?: (ms: number) => Promise<void>;
+}) => Promise<LoadStep>;
+export { detectGrowth };
+//# sourceMappingURL=harness.d.ts.map