@loomwork/knowledge 0.0.0-stage → 0.1.1

package/src/prompt.ts

@@ -0,0 +1,56 @@
+import { UNTRUSTED_CONTENT_SYSTEM_NOTE } from "@loomwork/llm";
+import type { LlmMessageInput, LlmRequestInput } from "@loomwork/llm";
+import type { Passage } from "./types";
+
+/**
+ * Retrieved passages as model messages: one `user` message per passage, each still
+ * carrying the taint it was ingested with.
+ *
+ * This shape is the whole safety argument, and it is the shape almost nobody writes first.
+ * The natural thing — join the passages into one string under a "here is the relevant
+ * documentation" header, and put it in the SYSTEM prompt — is refused by `guardRequest`
+ * with an `UntrustedContentError`, because the system prompt is the operator's voice and a
+ * corpus anyone can edit is not the operator. Two properties make this function the fix
+ * rather than a wrapper around the same mistake:
+ *
+ *  - ONE message per passage, never a join. Taint is attached to a VALUE, so joining means
+ *    unwrapping first, and provenance stops travelling exactly where it starts mattering.
+ *  - the `user` role, which is the only position `guardRequest` fences rather than refuses.
+ *
+ * `guardRequest` then labels each fenced block with the passage's origin before any
+ * provider sees it, so the model is told which passages are data and where each came from.
+ */
+export function passageMessages(passages: readonly Passage[]): LlmMessageInput[] {
+  return passages.map((passage) => ({ role: "user" as const, content: passage.text }));
+}
+
+export interface GroundedRequestOptions {
+  /** The operator's instructions. Trusted content — a plain string, never a passage. */
+  instructions: string;
+  /** The question, appended after the passages so it reads last. */
+  question: string;
+  passages: readonly Passage[];
+  maxTokens?: number;
+  temperature?: number;
+}
+
+/**
+ * A complete grounded-answer request: operator instructions plus the untrusted-content
+ * note in `system`, the retrieved passages as fenced `user` messages, the question last.
+ *
+ * `instructions` is typed `string` rather than `LlmContent` on purpose. The system slot is
+ * where a passage must never land, and a signature that accepts a tainted value there
+ * would move the refusal from compile time to run time — the check exists either way, but
+ * only one of the two tells you before you ship.
+ */
+export function groundedRequest(opts: GroundedRequestOptions): LlmRequestInput {
+  return {
+    system: `${opts.instructions} ${UNTRUSTED_CONTENT_SYSTEM_NOTE}`,
+    messages: [
+      ...passageMessages(opts.passages),
+      { role: "user" as const, content: opts.question },
+    ],
+    ...(opts.maxTokens !== undefined && { maxTokens: opts.maxTokens }),
+    ...(opts.temperature !== undefined && { temperature: opts.temperature }),
+  };
+}

package/src/types.ts

@@ -0,0 +1,97 @@
+import type { TaintedValue } from "@loomwork/classify";
+
+/**
+ * A source document, as the application already has it: an id it can name the document
+ * by again tomorrow, its text, and whatever metadata it wants carried onto every passage.
+ *
+ * There is deliberately no `source`, `url`, `author`, `section` or `updatedAt` here. Those
+ * are a vocabulary, they differ per corpus, and a framework that named them would be
+ * asking every app to translate into its dialect — put them in `meta`, which travels
+ * verbatim and is indexed like any other content.
+ */
+export interface KnowledgeDoc {
+  /** Stable across revisions. This is what a re-ingest supersedes and what a citation names. */
+  id: string;
+  /** Optional label, carried onto every passage so a citation can read as a title. */
+  title?: string;
+  text: string;
+  /** Carried onto every passage verbatim. Must be JSON — it is stored in a MemoryEntry. */
+  meta?: Record<string, unknown>;
+}
+
+/**
+ * One retrieved (or just-ingested) piece of a document.
+ *
+ * `text` is TAINTED (ADR 0085) and stays tainted all the way into the model request. A
+ * corpus is edited by whoever can edit the source, so a passage is third-party content by
+ * definition — the same class as an MCP tool result — and `guardRequest` refuses it in
+ * every instruction position. Tainting at INGEST rather than at the prompt is what makes
+ * that hold on every path: the wrapper is plain JSON, so it survives the store, the
+ * recorded retrieval, and a replay that rebuilds the prompt from the log.
+ */
+export interface Passage {
+  docId: string;
+  title?: string;
+  /** 0-based position within the document, in source order. */
+  index: number;
+  text: TaintedValue<string>;
+  meta?: Record<string, unknown>;
+}
+
+/**
+ * Splits a document into passage texts, in source order.
+ *
+ * Injected, never inferred. How a document splits is a property of the corpus — a runbook
+ * splits on paragraphs, a legal contract on numbered clauses, a chat export on turns — and
+ * the retrieval quality of a knowledge base is mostly decided here. {@link packParagraphs}
+ * is a default worth having, not a default worth hiding.
+ */
+export type Chunker = (doc: KnowledgeDoc) => string[];
+
+/**
+ * The manifest row for one document: what a corpus knows about a document without
+ * reading its passages.
+ *
+ * `chunkCount` is what makes removal possible at all — passage ids are derived
+ * (`docId#0…docId#n`), so the count IS the id list, in one small row instead of n.
+ *
+ * `fingerprint` is over the document's content, so re-ingesting an unchanged document is
+ * free: that is what turns {@link KnowledgeBase.sync} from "rebuild the corpus" into "write
+ * the documents that actually changed".
+ *
+ * `committed` is the crash-safety flag, and the reason ingest writes a manifest row TWICE.
+ * The dangerous interleaving is not a lost write, it is a manifest that claims a revision
+ * whose passages were never appended: the next sync would compare fingerprints, see a
+ * match, and skip the document forever — a corpus silently missing a document that every
+ * status view calls present. Writing `committed: false` first, and flipping it only after
+ * the passages land, makes the crash window converge on a re-run instead.
+ */
+export interface DocRecord {
+  docId: string;
+  chunkCount: number;
+  fingerprint: string;
+  committed: boolean;
+  /**
+   * WHO manages this document — the writer that may revise or delete it. `undefined` is a
+   * source of its own (the corpus's default writer), not a wildcard.
+   *
+   * It exists because `sync` is a destructive reconcile over everything the manifest holds,
+   * and a corpus is not always written by one thing. An agent that writes down what it
+   * learned from feedback puts documents in the same scope as an authored corpus — and the
+   * nightly sync of a wiki export would then delete every one of them, because they are not
+   * in the set the wiki produced. Stamping each document with its writer makes that
+   * impossible instead of documented: a `sync` only ever reconciles its own documents.
+   */
+  source?: string;
+  /**
+   * Generation of this manifest ROW, not a version of the document.
+   *
+   * Manifest rows are only ever APPENDED — `docId#0`, `docId#1`, … — and the highest one
+   * wins, because the alternative (delete the old row, append the new one) has a crash
+   * window in which the document has no manifest row at all. That window is the worst of
+   * the three: passages nothing counts are passages nothing can remove. Superseded rows
+   * are pruned after the new one is durable, so a crash costs a stale row and never a
+   * missing one.
+   */
+  revision: number;
+}