@adhd/backlog 0.1.9 → 1.0.0

package/store/graph-backlog-store.d.ts

@@ -1,36 +1,99 @@
-import { GraphBackend } from '@adhd/sox-graph-store';
+import { IEmbedDrainOptions, IEmbedDrainResult } from '../write/embed-drain.js';
+import { GraphBackend, TypePolicy } from '@adhd/sox-graph-store';
 import { StoreAdapter } from '@adhd/sox-store-adapter';
 
 export interface GraphBacklogStore {
-    /** Store-adapter handle — ONLY for the CAS transaction wrapper (mutate-metadata.ts / ids.ts). */
+    /**
+     * Store-adapter handle — reached directly by the write layer's CAS
+     * transaction wrapper (`write/tx.ts`'s `executeWriteTransaction`), the
+     * vocabulary guard, and close. All other reads/writes go through `graph`.
+     */
     readonly adapter: StoreAdapter;
     /** All non-CAS reads/writes go through this. */
     readonly graph: GraphBackend;
+    /**
+     * The SAME `TypePolicy` instance `graph` was constructed with. The write
+     * layer's `IWriteStoreHandle` (write/tx.ts) requires one, and it must be
+     * the store's own — importing a second copy at the call site is exactly how
+     * the production/ETL/test divergence documented in store/type-policy.ts
+     * came about.
+     *
+     * `OPEN_TYPE_POLICY` — the single policy, installed on the graph backend and
+     * reported here as one value, so a caller and the backend can never disagree
+     * about what is permitted. The application layer's own node kinds
+     * (`project`/`component`/`issue`/`citation`/`audit`/…) and rels
+     * (`owns_project`/`has_status`/`has_citation`/…) are outside
+     * `@adhd/sox-graph-store`'s closed default vocabulary, so a closed policy
+     * rejects every write this package makes.
+     */
+    readonly typePolicy: TypePolicy;
+    /**
+     * Await every embed the write layer scheduled against this store's adapter
+     * (`write/embed-drain.ts`'s per-adapter registry), bounded — RAG-SPEC.md
+     * §2.2's durability backstop for a short-lived process. `closeGraphBacklogStore`
+     * calls this BEFORE closing the adapter, so a fire-and-forget embed that has
+     * not yet settled gets its chance to land rather than dying on a closed
+     * connection. Exposed on the store (rather than only inside close) so a host
+     * that wants to drain without closing can, and so tests can override the bound.
+     */
+    flushEmbeds(opts?: IEmbedDrainOptions): Promise<IEmbedDrainResult>;
 }
 /**
- * @param busyTimeoutMs SQLite `busy_timeout` (ms) — how long a blocked
+ * @param busyTimeoutMs `busy_timeout` (ms) — how long a blocked
  *   `.transaction(fn, { mode: 'immediate' })` waits for a contended lock
- *   before throwing `SQLITE_BUSY` (DEBT-BACKLOG-CONCURRENCY-BUSY-RETRY-001).
- *   BUG-SOXGRAPH-002 moved busy_timeout ownership OUT of graph-store into the
- *   adapters (SqliteAdapter hardcodes 3000 at connect; graph-store's
- *   `PRAGMAS` no longer touches it), and `AdapterConfig` exposes no
- *   busy_timeout field — so the caller's value is routed through the
- *   adapter's own `pragmaSet('busy_timeout', N)` surface, which both
- *   adapters honor (verified: read-back works on turso and sqlite). Callers
+ *   before it gives up (DEBT-BACKLOG-CONCURRENCY-BUSY-RETRY-001).
+ *   BUG-SOXGRAPH-002 puts busy_timeout ownership in the adapter layer, and
+ *   `AdapterConfig` exposes no busy_timeout field — so the caller's value is
+ *   routed through the adapter's own `pragmaSet('busy_timeout', N)` surface,
+ *   which every adapter honors (verified by read-back). Callers
  *   reading from `BacklogConfig` should pass `env.config.db.busyTimeoutMs`;
  *   the default here (5000) matches that config field's own default, for
  *   callers (tests, ad-hoc scripts) that open a store directly without going
  *   through `buildBacklogEnv`.
  */
 export declare function openGraphBacklogStore(dbPath: string, busyTimeoutMs?: number): Promise<GraphBacklogStore>;
-export declare function closeGraphBacklogStore(store: GraphBacklogStore): Promise<void>;
+/**
+ * Async — drains the adapter's in-flight embeds, records any that are still
+ * unsettled as durable `embedding_failed` audit rows, THEN closes the adapter.
+ *
+ * This is RAG-SPEC.md §2.2's durability backstop for a short-lived process: a
+ * write verb schedules its embed fire-and-forget by default, and without this
+ * drain a CLI that exits immediately after `create` would close the adapter
+ * out from under that embed — its `upsertVector`/audit write would hit a
+ * closed connection and die as a log line. The drain runs BEFORE `close()`, so
+ * the post-close state is never exercised on the normal path; anything the
+ * drain cannot settle within its bound is recorded as `embedding_failed`
+ * WHILE THE CONNECTION IS STILL OPEN, so the loss is durable rather than
+ * silent.
+ *
+ * `adapter.close()` runs unconditionally (in a `finally`): a failed drain, or
+ * a failed recording, must still close the adapter — a leaked connection is
+ * never an acceptable outcome of a close call. `opts` is the drain bound
+ * (default `DEFAULT_EMBED_DRAIN_TIMEOUT_MS`); it exists so a caller can tune
+ * the wait and so tests can bound it deterministically.
+ *
+ * Return type widened from `Promise<void>` to `Promise<IEmbedDrainResult>` —
+ * non-breaking, since `Promise<T>` is assignable where `Promise<void>` is
+ * expected and every existing caller `await`s or ignores the result.
+ */
+export declare function closeGraphBacklogStore(store: GraphBacklogStore, opts?: IEmbedDrainOptions): Promise<IEmbedDrainResult>;
 /**
  * Best-effort close for teardown finally-paths (cli.ts / server.ts).
- * closeGraphBacklogStore may throw only in the extreme edge where the
- * driver's own db.close() fails (every checkpoint/verify failure is already
- * caught and logged inside the adapter — turso-adapter close()). On that edge
- * this logs and returns so a close failure can never mask a command/transport
- * error or turn a successful command into a failed exit. The adapter's own
- * logs are the durable record; the client never rethrows here.
+ *
+ * Unlike the pre-drain version, this is no longer a silent discard: a store
+ * close can now carry an embed-durability outcome, and swallowing it would
+ * recreate exactly the "died unrecorded" defect this wave fixes. So:
+ *
+ * - `closeGraphBacklogStore` throwing (the extreme edge where the driver's own
+ *   `db.close()` fails) still logs and returns — a close failure must never
+ *   mask a command/transport error or turn a successful command into a failed
+ *   exit.
+ * - An **unrecorded** embed death (its `embedding_failed` audit row could not
+ *   be written, so the outcome is not durably recorded anywhere) logs loudly
+ *   AND sets `process.exitCode = 1`: the subject write is durable but the
+ *   vector is missing and nothing recorded why. Only this case changes the
+ *   exit code — a **recorded** failure (the `embedding_failed` row IS durable)
+ *   warns but exits 0, because the subject write genuinely succeeded and the
+ *   failure is recorded. Filing must never depend on RAG succeeding.
  */
 export declare function closeGraphBacklogStoreSafe(store: GraphBacklogStore | undefined): Promise<void>;

package/store/immediate-retry.d.ts

@@ -1,19 +1,30 @@
 /**
  * immediate-retry.ts — bounded, jittered exponential-backoff retry wrapper
  * around `adapter.transaction(fn, { mode: 'immediate' })` (DEBT-BACKLOG-
- * CONCURRENCY-BUSY-RETRY-001). `mutate-metadata.ts` / `ids.ts` are the ONLY
- * two write paths that call `.immediate()` directly (DESIGN.md §3/§4.3) and
- * both funnel through this wrapper — retrying ONLY busy-shaped errors, using
- * store-adapter's portable `isBusyError`/`isConcurrentConflict` duck-type
- * checks (the same helpers the adapter's own retry loop uses) so BOTH
- * adapter error shapes are covered: the legacy SQLite adapter's
- * `SQLITE_BUSY`/`SQLITE_BUSY_SNAPSHOT` codes and the turso driver's
- * `GenericFailure` with a
- * "database is locked"/"database is busy" message. Any other thrown error
- * (including `NotFoundError`, `ClaimContentionError`) propagates immediately,
- * unretried — and the semantic `'held'` claim-contention RESULT (claim.ts) is
- * a normal RETURN VALUE, never an exception, so it is never touched by this
- * wrapper either.
+ * CONCURRENCY-BUSY-RETRY-001). The write layer's `executeWriteTransaction`
+ * (`write/tx.ts`) and `graph-backlog-store.ts`'s schema apply are the write
+ * paths that reach the `immediate` mode directly, and both funnel through
+ * this wrapper.
+ *
+ * It retries ONLY busy-shaped errors, and it does not classify them itself:
+ * `@adhd/sox-store-adapter`'s `isBusyError`/`isConcurrentConflict` duck-type
+ * checks are the single portable definition of "this write lost a lock race"
+ * — the same helpers the adapter's own retry loop uses. Keeping the
+ * classification behind that seam is the point: every driver-specific error
+ * shape lives on the adapter's side of it, and nothing here needs to know
+ * which substrate raised the error.
+ *
+ * This wrapper retries a THROWN busy/locked error; it neither sets nor
+ * introspects a `busy_timeout` PRAGMA. The busy-timeout budget is a separate,
+ * adapter-owned concern applied at connect time (`graph-backlog-store.ts`
+ * applies the caller's value via `adapter.pragmaSet('busy_timeout', N)` after
+ * the factory's init; a substrate may apply its own equivalent). Nothing here
+ * reads a PRAGMA or classifies an error by one.
+ *
+ * Any other thrown error (including `NotFoundError`, `ClaimContentionError`)
+ * propagates immediately, unretried — and the semantic `'held'`
+ * claim-contention RESULT (claim.ts) is a normal RETURN VALUE, never an
+ * exception, so it is never touched by this wrapper either.
  */
 export interface ImmediateRetryOpts {
     /** Total attempts (first try + retries). Default 5. */

package/store/type-policy.d.ts

@@ -0,0 +1,4 @@
+import { TypePolicy } from '@adhd/sox-graph-store';
+
+/** Unconditionally permissive — see this file's own doc comment for why. */
+export declare const OPEN_TYPE_POLICY: TypePolicy;

package/store/vocabulary-guard.d.ts

@@ -0,0 +1,52 @@
+import { StoreAdapter } from '@adhd/sox-store-adapter';
+
+/** The node kind every backlog ITEM is stored under. The read path filters `kind:'issue'`, so a store holding items under any other kind reads as empty. */
+export declare const ITEM_NODE_KIND = "issue";
+/**
+ * Every node kind this build's write layer composes — the authoritative list
+ * is `write/tx.ts`'s `IWriteNodeTxInput.kind` doc comment (SPEC.md §3): the
+ * catalog kinds, the resolved-only kinds, and every subject kind. A store
+ * whose live nodes are entirely outside this set was written by a build
+ * speaking a different vocabulary.
+ */
+export declare const RECOGNIZED_NODE_KINDS: ReadonlySet<string>;
+/** One row of the live-node kind histogram. */
+export interface IObservedKind {
+    kind: string;
+    count: number;
+}
+/**
+ * A store whose live-node vocabulary this build does not recognize. Carries
+ * the observed histogram and the expected kind set as structured fields so a
+ * caller (e.g. the `doctor` CLI diagnostic) can render them without parsing
+ * the message.
+ */
+export declare class StoreVocabularyMismatchError extends Error {
+    readonly observed: readonly IObservedKind[];
+    readonly recognized: readonly string[];
+    constructor(observed: readonly IObservedKind[], recognized: readonly string[]);
+}
+/**
+ * Read the store's live-node kind histogram. A pure read — it never writes,
+ * so it is safe to run against any store, including the live one.
+ */
+export declare function inspectStoreVocabulary(adapter: StoreAdapter): Promise<{
+    total: number;
+    observed: IObservedKind[];
+}>;
+/**
+ * Assert the store's live nodes speak a vocabulary this build understands.
+ *
+ * Passes (no throw) when the store is empty, or when at least one live node
+ * is of a recognized kind. Throws {@link StoreVocabularyMismatchError} when
+ * the store holds live nodes but none of a recognized kind.
+ *
+ * BOUNDED BY CONSTRUCTION: this runs on every write verb (`api.ts`'s
+ * `writeHandle`) and every query (`api.ts`'s `queryHandle` → `query.ts`), so it
+ * must not scan the store per call. The criterion is answered by a single
+ * `LIMIT 1` existence probe ({@link hasRecognizedLiveNode}); only when that
+ * finds no recognized node — the empty-store and foreign-store cases — does the
+ * full histogram run, and there it is either trivial (no rows) or the payload
+ * of the error we are about to throw.
+ */
+export declare function assertRecognizedStoreVocabulary(adapter: StoreAdapter): Promise<void>;

package/write/audit.d.ts

@@ -0,0 +1,36 @@
+import { TypePolicy } from '@adhd/sox-graph-store';
+import { AdapterTransaction } from '@adhd/sox-store-adapter';
+
+export interface IWriteAuditInput {
+    tx: AdapterTransaction;
+    typePolicy: TypePolicy;
+    /** The node the `audits` edge points FROM — any kind, per `audits`' declared `source_kind: '*'` sentinel (§2). */
+    subjectRowid: number;
+    subjectUid: string;
+    subjectKind: string;
+    /** The acting identity — the verb's own `by` (or the resolved `agent` catalog name), never the raw unresolved input. */
+    actor: string;
+    /** e.g. `'created'`, `'claimed'`, `'released'`, `'renewed'`, `'reclaimed-stale'`, `'transitioned'`, `'moved'`, `'related'`, `'deleted'`. */
+    action: string;
+    from?: string;
+    to?: string;
+    note?: string;
+    /** Defaults to `nowISO()`. Accepted explicitly so a verb that already computed `now` for its subject write reuses the identical timestamp rather than a microsecond-later second call. */
+    at?: string;
+}
+export interface IAuditWriteResult {
+    rowid: number;
+    uid: string;
+    sha: string;
+}
+/**
+ * Write one `audit` node + its `audits` edge, against the SAME `tx` the
+ * caller's own subject write already opened. `sha` is computed HERE,
+ * unconditionally — "automatic, cannot be forgotten" (§4a) — as `sha256` over
+ * the canonical (sorted-key) JSON serialization of the audit's own recorded
+ * fields (DATA_MODEL.md §10 point 2: "the same convention SPEC.md §4a
+ * already states for the audit `sha`," applied identically to how
+ * `transition.sha` is computed — never hashing external content, unlike a
+ * `citation`'s `sha`, §8.5).
+ */
+export declare function writeAudit(input: IWriteAuditInput): Promise<IAuditWriteResult>;

package/write/bootstrap.d.ts

@@ -0,0 +1,123 @@
+import { IEmbeddingBackend } from './tx.js';
+import { BacklogConfig } from '../env.js';
+import { StoreAdapter } from '@adhd/sox-store-adapter';
+import { GraphBackend } from '@adhd/sox-graph-store';
+import { AsyncVectorBackend } from '@adhd/sox-vector-store';
+import { StoreSearchBackend } from '@adhd/sox-hybrid-search';
+
+/**
+ * The two handle members `api.ts`'s `writeHandle`/`queryHandle` need, in the
+ * exact shapes their consumers require:
+ *  - `search` matches `query/query.ts`'s `IQueryStoreHandle['search']`
+ *    (mandatory `embedQuery` + `spacePopulated`) — a strict subset that also
+ *    satisfies `write/create-issue.ts`'s `IDuplicateScanHandle['search']`
+ *    (optional `embedQuery`), since a mandatory field trivially satisfies an
+ *    optional one.
+ *  - `embedding` matches `write/tx.ts`'s `IWriteStoreHandle['embedding']`
+ *    (`IEmbeddingBackend`) exactly.
+ *
+ * Both are OPTIONAL and, per BUG-045's binding rule (quoted in `api.ts`'s
+ * former `queryHandle` comment), absence is the only honest degrade: never a
+ * stub whose methods run but return empty/wrong results. `deriveMembers`
+ * below returns `{}` — both members absent — whenever the real embedding
+ * model genuinely cannot be reached, so `scanForDuplicates` reports "scan
+ * unavailable" (zero candidates, write proceeds) and `filter.semantic`/
+ * `view:'similar'` throw `InvalidArgumentError('semantic', ...)` exactly as
+ * they did before this module existed — never a silently-empty "search".
+ */
+export interface SemanticStoreMembers {
+    readonly search?: {
+        readonly backend: StoreSearchBackend;
+        embedQuery(text: string): Promise<Float32Array>;
+        /**
+         * Bounded existence probe of the REAL vector table: `true` iff at least
+         * one vector exists under this member's own space `modelId`. Backed by
+         * `@adhd/sox-vector-store`'s `hasVectors` (`SELECT 1 … LIMIT 1`), so it
+         * never reads the embedding column or scans past the first row. This is
+         * per-query truth read straight from the write layer's durable vectors —
+         * never a process-lifetime latch — so it is correct across process
+         * boundaries (a second process's `create` is visible here; ADR-0012).
+         */
+        spacePopulated(): Promise<boolean>;
+    };
+    readonly embedding?: IEmbeddingBackend;
+}
+/**
+ * §2.5 (RAG-SPEC.md, carried over from `store/semantic-search.ts`'s own
+ * identically-named class) — a returned vector whose dimension does not
+ * match the resolved model. Permanent and non-retryable by construction: the
+ * dimensional contract is structural (the vector column's type encodes it),
+ * so a mismatch means the provider is not the model the space was built for.
+ * Truncating or padding to fit would produce plausible-looking, permanently
+ * wrong neighbours — so this throws instead. Defined locally (not imported)
+ * because the module it came from is on the deletion list — see this file's
+ * header comment.
+ */
+export declare class PermanentEmbeddingDimensionError extends Error {
+    readonly operation: string;
+    readonly modelId: string;
+    readonly expected: number;
+    readonly actual: number;
+    constructor(operation: string, modelId: string, expected: number, actual: number);
+}
+/**
+ * Bounded existence probe of a REAL vector table: `true` iff at least one
+ * vector exists under `modelId`. This is the readiness source the text-routing
+ * decision (`query/query.ts`'s `resolveTextInput`) consults — per query, read
+ * from the durable vector table the write layer's own `embedding` member writes
+ * to.
+ *
+ * Deliberately NOT a flag latched at boot: a process that opens a store while
+ * its vector space is still empty (a fresh store, or one populated by another
+ * process) must still route `text:` to the semantic ranker the moment real
+ * vectors exist. Reading the table itself makes that true across process
+ * boundaries (ADR-0012: concurrent processes share the store).
+ *
+ * Delegates to `@adhd/sox-vector-store`'s `hasVectors` — a bounded
+ * `SELECT 1 … LIMIT 1` that projects no embedding column and stops at the
+ * first row, so the probe is O(1) in the size of the space and never
+ * materializes an embedding BLOB. This replaced an `iter`-first-row probe
+ * (BUG e19bc9d0): `AsyncVectorBackend.iter` is a FULL corpus scan on the Turso
+ * backend — its `db.all`-backed query materializes every row *and every
+ * embedding BLOB* before the first yield — so the old probe read the entire
+ * vector table on every bare-`text:` query. Returning on the first yielded row
+ * short-circuited nothing.
+ *
+ * The capability is additive, NOT on the pinned `AsyncVectorBackend` contract,
+ * so a backend predating it (or a structural test double) narrows to
+ * `undefined`; absence degrades to `false` — the honest conservative answer,
+ * and a backend that cannot answer cheaply must not fall back to the unbounded
+ * scan this probe exists to avoid. An absent table (a space never
+ * `ensureSpace`d) is likewise "empty", not an error (`hasVectors` tolerates it).
+ */
+export declare function isVectorSpacePopulated(vectorBackend: AsyncVectorBackend, modelId: string): Promise<boolean>;
+/**
+ * Derives the `search`/`embedding` members for a store's already-open
+ * `adapter`/`graph` pair, memoized per `adapter` instance. `cfg` is
+ * `BacklogConfig['embedding']` (`env.ts`) — the caller passes
+ * `ctx.env.config.embedding` verbatim.
+ *
+ * **The cache holds only a successful, member-ful derive.** `deriveMembers`
+ * never throws on a soft failure — it returns `{}` (both members absent) from
+ * five paths: `embedding` disabled, a non-Turso adapter (`nativeVectors !==
+ * true`), the optional packages unresolvable, `createEmbeddingProvider`
+ * throwing, or `openTursoVectorStore`/`ensureSpace` throwing. Several of those
+ * are TRANSIENT (a provider briefly unreachable, a store-open that hit a lock),
+ * so a member-less result is evicted rather than cached: the next semantic verb
+ * re-derives and recovers without a process restart. A hard rejection is
+ * likewise evicted, so it is surfaced to the current caller and retried later
+ * rather than replayed forever. Only a member-ful success is retained, which is
+ * where the expensive cold load actually is.
+ *
+ * (The `cfg.enabled === false` case is member-less and so is not retained
+ * either — but it does no I/O at all: `deriveMembers` returns `{}` on its first
+ * line. Re-deriving it per verb costs a `WeakMap` miss and a branch, which is
+ * cheaper than carrying a second "is it safe to cache?" signal through
+ * `deriveMembers`. Stated here as the deliberate choice.)
+ *
+ * @param log where a failed opt-in is reported (never thrown — mirrors
+ *   `store/semantic-search.ts`'s own former `enableSemanticSearchFromConfig`
+ *   "best-effort, never fatal" contract). Defaults to `console.error`; a host
+ *   with a structured logger should pass its own sink.
+ */
+export declare function bootstrapSemanticStoreMembers(adapter: StoreAdapter, graph: GraphBackend, cfg: BacklogConfig['embedding'], log?: (message: string) => void): Promise<SemanticStoreMembers>;