@loomwork/knowledge 0.0.0-stage → 0.1.0

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;AAAA,iDAAmE;AAA1D,8GAAA,aAAa,OAAA;AAAE,kHAAA,iBAAiB,OAAA;AAUzC,iCAAyC;AAAhC,uGAAA,cAAc,OAAA;AACvB,yCAAsF;AAA7E,2GAAA,cAAc,OAAA;AAAE,2GAAA,cAAc,OAAA;AAAE,oHAAA,uBAAuB,OAAA;AAEhE,mCAA4D;AAAnD,yGAAA,eAAe,OAAA;AAAE,yGAAA,eAAe,OAAA"}

package/dist/prompt.d.ts

@@ -0,0 +1,41 @@
+import type { LlmMessageInput, LlmRequestInput } from "@loomwork/llm";
+import type { Passage } from "./types";
+/**
+ * Retrieved passages as model messages: one `user` message per passage, each still
+ * carrying the taint it was ingested with.
+ *
+ * This shape is the whole safety argument, and it is the shape almost nobody writes first.
+ * The natural thing — join the passages into one string under a "here is the relevant
+ * documentation" header, and put it in the SYSTEM prompt — is refused by `guardRequest`
+ * with an `UntrustedContentError`, because the system prompt is the operator's voice and a
+ * corpus anyone can edit is not the operator. Two properties make this function the fix
+ * rather than a wrapper around the same mistake:
+ *
+ *  - ONE message per passage, never a join. Taint is attached to a VALUE, so joining means
+ *    unwrapping first, and provenance stops travelling exactly where it starts mattering.
+ *  - the `user` role, which is the only position `guardRequest` fences rather than refuses.
+ *
+ * `guardRequest` then labels each fenced block with the passage's origin before any
+ * provider sees it, so the model is told which passages are data and where each came from.
+ */
+export declare function passageMessages(passages: readonly Passage[]): LlmMessageInput[];
+export interface GroundedRequestOptions {
+    /** The operator's instructions. Trusted content — a plain string, never a passage. */
+    instructions: string;
+    /** The question, appended after the passages so it reads last. */
+    question: string;
+    passages: readonly Passage[];
+    maxTokens?: number;
+    temperature?: number;
+}
+/**
+ * A complete grounded-answer request: operator instructions plus the untrusted-content
+ * note in `system`, the retrieved passages as fenced `user` messages, the question last.
+ *
+ * `instructions` is typed `string` rather than `LlmContent` on purpose. The system slot is
+ * where a passage must never land, and a signature that accepts a tainted value there
+ * would move the refusal from compile time to run time — the check exists either way, but
+ * only one of the two tells you before you ship.
+ */
+export declare function groundedRequest(opts: GroundedRequestOptions): LlmRequestInput;
+//# sourceMappingURL=prompt.d.ts.map

package/dist/prompt.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"prompt.d.ts","sourceRoot":"","sources":["../src/prompt.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACtE,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAEvC;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAgB,eAAe,CAAC,QAAQ,EAAE,SAAS,OAAO,EAAE,GAAG,eAAe,EAAE,CAE/E;AAED,MAAM,WAAW,sBAAsB;IACrC,sFAAsF;IACtF,YAAY,EAAE,MAAM,CAAC;IACrB,kEAAkE;IAClE,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,SAAS,OAAO,EAAE,CAAC;IAC7B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;;;;;;;GAQG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,sBAAsB,GAAG,eAAe,CAU7E"}

package/dist/prompt.js

@@ -0,0 +1,47 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.passageMessages = passageMessages;
+exports.groundedRequest = groundedRequest;
+const llm_1 = require("@loomwork/llm");
+/**
+ * Retrieved passages as model messages: one `user` message per passage, each still
+ * carrying the taint it was ingested with.
+ *
+ * This shape is the whole safety argument, and it is the shape almost nobody writes first.
+ * The natural thing — join the passages into one string under a "here is the relevant
+ * documentation" header, and put it in the SYSTEM prompt — is refused by `guardRequest`
+ * with an `UntrustedContentError`, because the system prompt is the operator's voice and a
+ * corpus anyone can edit is not the operator. Two properties make this function the fix
+ * rather than a wrapper around the same mistake:
+ *
+ *  - ONE message per passage, never a join. Taint is attached to a VALUE, so joining means
+ *    unwrapping first, and provenance stops travelling exactly where it starts mattering.
+ *  - the `user` role, which is the only position `guardRequest` fences rather than refuses.
+ *
+ * `guardRequest` then labels each fenced block with the passage's origin before any
+ * provider sees it, so the model is told which passages are data and where each came from.
+ */
+function passageMessages(passages) {
+    return passages.map((passage) => ({ role: "user", content: passage.text }));
+}
+/**
+ * A complete grounded-answer request: operator instructions plus the untrusted-content
+ * note in `system`, the retrieved passages as fenced `user` messages, the question last.
+ *
+ * `instructions` is typed `string` rather than `LlmContent` on purpose. The system slot is
+ * where a passage must never land, and a signature that accepts a tainted value there
+ * would move the refusal from compile time to run time — the check exists either way, but
+ * only one of the two tells you before you ship.
+ */
+function groundedRequest(opts) {
+    return {
+        system: `${opts.instructions} ${llm_1.UNTRUSTED_CONTENT_SYSTEM_NOTE}`,
+        messages: [
+            ...passageMessages(opts.passages),
+            { role: "user", content: opts.question },
+        ],
+        ...(opts.maxTokens !== undefined && { maxTokens: opts.maxTokens }),
+        ...(opts.temperature !== undefined && { temperature: opts.temperature }),
+    };
+}
+//# sourceMappingURL=prompt.js.map

package/dist/prompt.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"prompt.js","sourceRoot":"","sources":["../src/prompt.ts"],"names":[],"mappings":";;AAsBA,0CAEC;AAqBD,0CAUC;AAvDD,uCAA8D;AAI9D;;;;;;;;;;;;;;;;;GAiBG;AACH,SAAgB,eAAe,CAAC,QAA4B;IAC1D,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,OAAO,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;AACvF,CAAC;AAYD;;;;;;;;GAQG;AACH,SAAgB,eAAe,CAAC,IAA4B;IAC1D,OAAO;QACL,MAAM,EAAE,GAAG,IAAI,CAAC,YAAY,IAAI,mCAA6B,EAAE;QAC/D,QAAQ,EAAE;YACR,GAAG,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC;YACjC,EAAE,IAAI,EAAE,MAAe,EAAE,OAAO,EAAE,IAAI,CAAC,QAAQ,EAAE;SAClD;QACD,GAAG,CAAC,IAAI,CAAC,SAAS,KAAK,SAAS,IAAI,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC;QAClE,GAAG,CAAC,IAAI,CAAC,WAAW,KAAK,SAAS,IAAI,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC;KACzE,CAAC;AACJ,CAAC"}

package/dist/types.d.ts

@@ -0,0 +1,94 @@
+import type { TaintedValue } from "@loomwork/classify";
+/**
+ * A source document, as the application already has it: an id it can name the document
+ * by again tomorrow, its text, and whatever metadata it wants carried onto every passage.
+ *
+ * There is deliberately no `source`, `url`, `author`, `section` or `updatedAt` here. Those
+ * are a vocabulary, they differ per corpus, and a framework that named them would be
+ * asking every app to translate into its dialect — put them in `meta`, which travels
+ * verbatim and is indexed like any other content.
+ */
+export interface KnowledgeDoc {
+    /** Stable across revisions. This is what a re-ingest supersedes and what a citation names. */
+    id: string;
+    /** Optional label, carried onto every passage so a citation can read as a title. */
+    title?: string;
+    text: string;
+    /** Carried onto every passage verbatim. Must be JSON — it is stored in a MemoryEntry. */
+    meta?: Record<string, unknown>;
+}
+/**
+ * One retrieved (or just-ingested) piece of a document.
+ *
+ * `text` is TAINTED (ADR 0085) and stays tainted all the way into the model request. A
+ * corpus is edited by whoever can edit the source, so a passage is third-party content by
+ * definition — the same class as an MCP tool result — and `guardRequest` refuses it in
+ * every instruction position. Tainting at INGEST rather than at the prompt is what makes
+ * that hold on every path: the wrapper is plain JSON, so it survives the store, the
+ * recorded retrieval, and a replay that rebuilds the prompt from the log.
+ */
+export interface Passage {
+    docId: string;
+    title?: string;
+    /** 0-based position within the document, in source order. */
+    index: number;
+    text: TaintedValue<string>;
+    meta?: Record<string, unknown>;
+}
+/**
+ * Splits a document into passage texts, in source order.
+ *
+ * Injected, never inferred. How a document splits is a property of the corpus — a runbook
+ * splits on paragraphs, a legal contract on numbered clauses, a chat export on turns — and
+ * the retrieval quality of a knowledge base is mostly decided here. {@link packParagraphs}
+ * is a default worth having, not a default worth hiding.
+ */
+export type Chunker = (doc: KnowledgeDoc) => string[];
+/**
+ * The manifest row for one document: what a corpus knows about a document without
+ * reading its passages.
+ *
+ * `chunkCount` is what makes removal possible at all — passage ids are derived
+ * (`docId#0…docId#n`), so the count IS the id list, in one small row instead of n.
+ *
+ * `fingerprint` is over the document's content, so re-ingesting an unchanged document is
+ * free: that is what turns {@link KnowledgeBase.sync} from "rebuild the corpus" into "write
+ * the documents that actually changed".
+ *
+ * `committed` is the crash-safety flag, and the reason ingest writes a manifest row TWICE.
+ * The dangerous interleaving is not a lost write, it is a manifest that claims a revision
+ * whose passages were never appended: the next sync would compare fingerprints, see a
+ * match, and skip the document forever — a corpus silently missing a document that every
+ * status view calls present. Writing `committed: false` first, and flipping it only after
+ * the passages land, makes the crash window converge on a re-run instead.
+ */
+export interface DocRecord {
+    docId: string;
+    chunkCount: number;
+    fingerprint: string;
+    committed: boolean;
+    /**
+     * WHO manages this document — the writer that may revise or delete it. `undefined` is a
+     * source of its own (the corpus's default writer), not a wildcard.
+     *
+     * It exists because `sync` is a destructive reconcile over everything the manifest holds,
+     * and a corpus is not always written by one thing. An agent that writes down what it
+     * learned from feedback puts documents in the same scope as an authored corpus — and the
+     * nightly sync of a wiki export would then delete every one of them, because they are not
+     * in the set the wiki produced. Stamping each document with its writer makes that
+     * impossible instead of documented: a `sync` only ever reconciles its own documents.
+     */
+    source?: string;
+    /**
+     * Generation of this manifest ROW, not a version of the document.
+     *
+     * Manifest rows are only ever APPENDED — `docId#0`, `docId#1`, … — and the highest one
+     * wins, because the alternative (delete the old row, append the new one) has a crash
+     * window in which the document has no manifest row at all. That window is the worst of
+     * the three: passages nothing counts are passages nothing can remove. Superseded rows
+     * are pruned after the new one is durable, so a crash costs a stale row and never a
+     * missing one.
+     */
+    revision: number;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAEvD;;;;;;;;GAQG;AACH,MAAM,WAAW,YAAY;IAC3B,8FAA8F;IAC9F,EAAE,EAAE,MAAM,CAAC;IACX,oFAAoF;IACpF,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,yFAAyF;IACzF,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAChC;AAED;;;;;;;;;GASG;AACH,MAAM,WAAW,OAAO;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,6DAA6D;IAC7D,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,YAAY,CAAC,MAAM,CAAC,CAAC;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAChC;AAED;;;;;;;GAOG;AACH,MAAM,MAAM,OAAO,GAAG,CAAC,GAAG,EAAE,YAAY,KAAK,MAAM,EAAE,CAAC;AAEtD;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,WAAW,SAAS;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,OAAO,CAAC;IACnB;;;;;;;;;;OAUG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;;;;;;;OASG;IACH,QAAQ,EAAE,MAAM,CAAC;CAClB"}

package/dist/types.js

@@ -0,0 +1,3 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=types.js.map

package/dist/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":""}

package/package.json

@@ -1,6 +1,60 @@
 {
   "name": "@loomwork/knowledge",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
+  "version": "0.1.0",
+  "description": "Document corpora over memory — ingest, revise, retrieve with provenance",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "require": "./dist/index.js",
+      "default": "./dist/index.js"
+    }
+  },
+  "dependencies": {
+    "@loomwork/classify": "^0.1.0",
+    "@loomwork/core": "^0.1.0",
+    "@loomwork/llm": "^0.1.0",
+    "@loomwork/memory": "^0.1.0",
+    "@loomwork/verify": "^0.1.0"
+  },
+  "devDependencies": {
+    "@types/node": "^22.0.0",
+    "typescript": "^5.6.3",
+    "vitest": "^2.1.8",
+    "@loomwork/plugin-sqlite": "0.1.0"
+  },
+  "license": "Apache-2.0",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/chhetrisushil/loom.git",
+    "directory": "packages/knowledge"
+  },
+  "files": [
+    "dist",
+    "src",
+    "!src/**/*.test.ts",
+    "!src/**/*.test.tsx",
+    "!src/**/__tests__",
+    "!src/**/__fixtures__",
+    "!src/**/__snapshots__",
+    "!src/**/__golden__",
+    "LICENSE",
+    "NOTICE"
+  ],
+  "publishConfig": {
+    "access": "public",
+    "tag": "next"
+  },
+  "engines": {
+    "node": ">=20.0.0"
+  },
+  "scripts": {
+    "build": "tsc --build",
+    "dev": "tsc --build --watch",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "clean": "rm -rf dist tsconfig.tsbuildinfo",
+    "typecheck:tests": "tsc -p tsconfig.test.json"
+  }
 }