@openshain/core 0.4.0 → 0.5.0

package/dist/config/schema.d.ts

@@ -11,6 +11,7 @@ export declare const ConfigFileSchema: z.ZodObject<{
             en: "en";
             ja: "ja";
         }>>;
+        timezone: z.ZodOptional<z.ZodString>;
     }, z.core.$strict>;
     principal: z.ZodObject<{
         id: z.ZodString;
@@ -26,6 +27,7 @@ export declare const ConfigFileSchema: z.ZodObject<{
         api_key_env: z.ZodString;
         base_url: z.ZodOptional<z.ZodURL>;
         options: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+        context_tokens: z.ZodOptional<z.ZodInt>;
     }, z.core.$strict>>;
     tools: z.ZodDefault<z.ZodArray<z.ZodObject<{
         provider: z.ZodOptional<z.ZodString>;
@@ -36,6 +38,7 @@ export declare const ConfigFileSchema: z.ZodObject<{
         max_model_calls: z.ZodDefault<z.ZodInt>;
         max_tool_calls: z.ZodDefault<z.ZodInt>;
         max_output_tokens: z.ZodDefault<z.ZodInt>;
+        compact_at_input_tokens: z.ZodOptional<z.ZodInt>;
     }, z.core.$strict>>;
     debug: z.ZodPrefault<z.ZodObject<{
         persist_raw: z.ZodDefault<z.ZodBoolean>;
@@ -57,12 +60,15 @@ export interface ModelConfig {
     apiKeyEnv: string;
     baseUrl: string | undefined;
     options: Record<string, unknown> | undefined;
+    /** How much this model takes as input, when the person wrote it. */
+    contextTokens: number | undefined;
 }
 export interface Config {
     version: 1;
     company: {
         name: string;
         language: Language;
+        timezone: string;
     };
     principal: {
         id: string;
@@ -79,6 +85,8 @@ export interface Config {
         maxModelCalls: number;
         maxToolCalls: number;
         maxOutputTokens: number;
+        /** Where the conversation is summarized, when the person wrote it. 0 never summarizes. */
+        compactAtInputTokens: number | undefined;
     };
     debug: {
         persistRaw: boolean;

package/dist/config/schema.js

@@ -1,4 +1,5 @@
 import { z } from "zod";
+import { hostTimezone, isTimezone } from "../time.js";
 const identifier = z
     .string()
     .regex(/^[a-z][a-z0-9_-]*$/, "use lowercase letters, digits, _ or -, starting with a letter");
@@ -35,6 +36,16 @@ export const ConfigFileSchema = z.strictObject({
     company: z.strictObject({
         name: z.string().min(1).max(200),
         language: z.enum(LANGUAGES).default("ja"),
+        // The company's own clock decides every business date, so a workspace answers the same
+        // whether it runs on a laptop in Tokyo or in a container set to UTC.
+        // No default in the schema: it would bake the machine that generated it into the published
+        // JSON Schema. The fallback is applied when the file is turned into a Config.
+        timezone: z
+            .string()
+            .min(1)
+            .max(100)
+            .refine(isTimezone, "not a timezone name, such as Asia/Tokyo")
+            .optional(),
     }),
     principal: z.strictObject({ id: identifier, name: z.string().min(1).max(200) }),
     profession: z.strictObject({ id: identifier, instructions: z.string().min(1).max(100_000) }),
@@ -55,6 +66,9 @@ export const ConfigFileSchema = z.strictObject({
         }, "base_url must use https unless it points at this machine (localhost, 127.0.0.0/8, ::1)")
             .optional(),
         options: z.record(z.string(), z.unknown()).optional(),
+        // How much this model takes as input. Written by the person: openshain does not guess a
+        // length from a model's name, and an OpenAI-compatible endpoint may serve anything.
+        context_tokens: z.int().positive().optional(),
     })
         .optional(),
     tools: z.array(toolProviderRef).default([{ provider: "standard" }]),
@@ -63,6 +77,14 @@ export const ConfigFileSchema = z.strictObject({
         max_model_calls: z.int().positive().default(30),
         max_tool_calls: z.int().positive().default(100),
         max_output_tokens: z.int().positive().default(16000),
+        // What the conversation may reach before it is summarized. Written when the default does
+        // not suit the model; 0 turns compaction off. Below 50000 a conversation is summarized so
+        // often that it loses more than it saves.
+        compact_at_input_tokens: z
+            .int()
+            .nonnegative()
+            .refine((value) => value === 0 || value >= 50_000, "write 0 to never compact, or at least 50000")
+            .optional(),
     })
         .prefault({}),
     debug: z.strictObject({ persist_raw: z.boolean().default(false) }).prefault({}),
@@ -70,7 +92,11 @@ export const ConfigFileSchema = z.strictObject({
 export function toConfig(file) {
     return {
         version: file.version,
-        company: { name: file.company.name, language: file.company.language },
+        company: {
+            name: file.company.name,
+            language: file.company.language,
+            timezone: file.company.timezone ?? hostTimezone(),
+        },
         principal: { id: file.principal.id, name: file.principal.name },
         profession: { id: file.profession.id, instructions: file.profession.instructions },
         ...(file.model && {
@@ -80,6 +106,7 @@ export function toConfig(file) {
                 apiKeyEnv: file.model.api_key_env,
                 baseUrl: file.model.base_url,
                 options: file.model.options,
+                contextTokens: file.model.context_tokens,
             },
         }),
         tools: file.tools.map(toToolProviderRef),
@@ -87,6 +114,7 @@ export function toConfig(file) {
             maxModelCalls: file.limits.max_model_calls,
             maxToolCalls: file.limits.max_tool_calls,
             maxOutputTokens: file.limits.max_output_tokens,
+            compactAtInputTokens: file.limits.compact_at_input_tokens,
         },
         debug: { persistRaw: file.debug.persist_raw },
     };

package/dist/errors.d.ts

@@ -1,4 +1,4 @@
-export declare const ERROR_CODES: readonly ["auth", "network", "rate_limit", "invalid_response", "config", "corrupt_log", "invalid_transition", "duplicate_tool", "invalid_id", "invalid_tool", "invalid_path", "lock_held", "not_found", "reserved_path", "outside_workspace", "concurrent_write", "invalid_event"];
+export declare const ERROR_CODES: readonly ["auth", "network", "rate_limit", "too_large", "invalid_response", "config", "corrupt_log", "invalid_transition", "duplicate_tool", "invalid_id", "invalid_tool", "invalid_path", "lock_held", "not_found", "reserved_path", "outside_workspace", "concurrent_write", "invalid_event"];
 export type ErrorCode = (typeof ERROR_CODES)[number];
 export declare class OpenshainError extends Error {
     readonly name = "OpenshainError";
@@ -8,3 +8,8 @@ export declare class OpenshainError extends Error {
     });
 }
 export declare function isOpenshainError(value: unknown): value is OpenshainError;
+/**
+ * A request the model refused for its size. Both APIs answer 400 for it, and the only thing that
+ * separates it from a wrong setting is what the message says, so the words are matched loosely.
+ */
+export declare function isTooLarge(message: string): boolean;

package/dist/errors.js

@@ -2,6 +2,8 @@ export const ERROR_CODES = [
     "auth",
     "network",
     "rate_limit",
+    // The request did not fit the model: the conversation has to get shorter before it can run.
+    "too_large",
     "invalid_response",
     "config",
     "corrupt_log",
@@ -28,3 +30,10 @@ export class OpenshainError extends Error {
 export function isOpenshainError(value) {
     return value instanceof OpenshainError;
 }
+/**
+ * A request the model refused for its size. Both APIs answer 400 for it, and the only thing that
+ * separates it from a wrong setting is what the message says, so the words are matched loosely.
+ */
+export function isTooLarge(message) {
+    return /too long|too large|context[ _-]?length|maximum context|context window|reduce the length/i.test(message);
+}

package/dist/index.d.ts

@@ -2,16 +2,23 @@ export { AUTHORITY_DIR_NAME, type Authority, type AuthorityRequest, DECISION_KIN
 export { CONFIG_FILE_NAME, loadConfig, type ParseConfigOptions, parseConfig, } from "./config/load.ts";
 export type { Config, ModelConfig, ToolProviderRef } from "./config/schema.ts";
 export { LANGUAGES, type Language } from "./config/schema.ts";
-export { ERROR_CODES, type ErrorCode, isOpenshainError, OpenshainError } from "./errors.ts";
+export { ERROR_CODES, type ErrorCode, isOpenshainError, isTooLarge, OpenshainError, } from "./errors.ts";
 export { type EventId, newEventId, newWorkId, parseEventId, parseWorkId, type WorkId, } from "./ids.ts";
+export { buildIndex, hashKnowledgeInput, INDEX_FORMAT_VERSION, type IndexState, type IndexUnit, type KnowledgeIndex, type Manifest, readIndex, serializeIndex, writeIndex, } from "./knowledge/build.ts";
+export { type Checked, checkKnowledge, hasKnowledge, KNOWLEDGE_DIR_NAME, } from "./knowledge/check.ts";
+export { type LoadedRule as LoadedKnowledgeRule, type Rule as KnowledgeRule, RuleSchema as KnowledgeRuleSchema, RulesFileSchema, type Scope as KnowledgeScope, ScopeSchema as KnowledgeScopeSchema, type Source as KnowledgeSource, type SourceFrontMatter, SourceFrontMatterSchema, } from "./knowledge/schema.ts";
+export { type Hit, inEffect, MIN_QUERY_LENGTH, type SearchOptions, search, } from "./knowledge/search.ts";
+export { knowledgePath, readKnowledgeFile, writeKnowledgeFile, } from "./knowledge/store.ts";
 export type { ModelDescription, ModelMessage, ModelProvider, ModelRequest, ModelResponse, UserPart, } from "./model/types.ts";
 export { type CallOptions, type CreateRuntimeOptions, createRuntime, createToolCaller, createToolRegistry, MAX_TOOL_TEXT_CHARS, type PendingApprovalResult, REVIEW_DIR_NAME, type Runtime, type RuntimeProviders, type ToolSummary, } from "./runtime.ts";
 export { jsonSchemas, type SchemaName } from "./schemas.ts";
+export { businessDate, companyTime, hostTimezone, isTimezone } from "./time.ts";
 export { ASK_USER, RUNTIME_PROVIDER_ID } from "./tool/ask-user.ts";
+export { MAX_READ_BYTES, MAX_WRITE_BYTES, readWorkspaceText, readWorkspaceTextIfAny, writeWorkspaceText, } from "./tool/files.ts";
 export { loadToolModule } from "./tool/load-module.ts";
 export { RESERVED_PATHS, resolveWorkspacePath } from "./tool/paths.ts";
 export { type HiddenTool, type RegisteredTool, type RegisterOptions, ToolRegistry, } from "./tool/registry.ts";
-export { ASK_USER_TOOL_NAME, type JsonSchema, RESERVED_TOOL_NAMES, TOOL_NAME_PATTERN, type ToolCall, type ToolContext, type ToolDefinition, type ToolEffect, type ToolProvider, type ToolResult, } from "./tool/types.ts";
+export { ASK_USER_TOOL_NAME, type JsonSchema, type Observation, RESERVED_TOOL_NAMES, TOOL_NAME_PATTERN, type ToolCall, type ToolContext, type ToolDefinition, type ToolEffect, type ToolProvider, type ToolResult, } from "./tool/types.ts";
 export { compileInputValidator, type InputValidation } from "./tool/validate.ts";
 export { uuidv7 } from "./uuid.ts";
 export { verifyArtifact } from "./work/artifacts.ts";
@@ -19,6 +26,6 @@ export { EVENTS_FILE_NAME, EventLog, type NewEvent } from "./work/event-log.ts";
 export { type AnyEvent, type Artifact, type AssistantPart, canonical, type Event, type EventFile, EventFileSchema, type EventPayloads, type EventType, eventFromFile, eventToFile, isKnownEventType, type ModelUsage, parsePayloadFile, payloadFileSchemas, type ReviewPackage, type StopReason, TOOL_REJECTION_CODES, type ToolContent, type ToolRejectionCode, type UnknownEvent, } from "./work/events.ts";
 export { countToolCalls, type FailureReason, type HistoryCall, type PendingApproval, type PendingQuestion, pendingApprovals, pendingQuestions, type WorkHistory, workHistory, } from "./work/history.ts";
 export { acquireLock, LOCK_FILE_NAME, type Lock } from "./work/lock.ts";
-export { buildProjection, type Projection, type ProjectionInput } from "./work/projection.ts";
+export { buildProjection, type Projection, type ProjectionInput, RECENT_MESSAGES, } from "./work/projection.ts";
 export { type CreateWorkInput, type ListResult, WORK_DIR_NAME, WORK_FILE_NAME, type WorkHandle, WorkStore, } from "./work/store.ts";
 export { isTerminal, reduceWork, SESSION_WORK_TYPE, transition, WORK_STATUSES, type Work, type WorkFile, WorkFileSchema, WorkStatus, workToFile, } from "./work/work.ts";

package/dist/index.js

@@ -2,11 +2,18 @@
 export { AUTHORITY_DIR_NAME, DECISION_KINDS, DECISIONS_DIR_NAME, DELEGATIONS_FILE_NAME, DecisionFileSchema, DelegationsFileSchema, evaluate, loadAuthority, matchGlob, OPEN_AUTHORITY, POLICY_FILE_NAME, PolicyFileSchema, writeDecision, } from "./authority/policy.js";
 export { CONFIG_FILE_NAME, loadConfig, parseConfig, } from "./config/load.js";
 export { LANGUAGES } from "./config/schema.js";
-export { ERROR_CODES, isOpenshainError, OpenshainError } from "./errors.js";
+export { ERROR_CODES, isOpenshainError, isTooLarge, OpenshainError, } from "./errors.js";
 export { newEventId, newWorkId, parseEventId, parseWorkId, } from "./ids.js";
+export { buildIndex, hashKnowledgeInput, INDEX_FORMAT_VERSION, readIndex, serializeIndex, writeIndex, } from "./knowledge/build.js";
+export { checkKnowledge, hasKnowledge, KNOWLEDGE_DIR_NAME, } from "./knowledge/check.js";
+export { RuleSchema as KnowledgeRuleSchema, RulesFileSchema, ScopeSchema as KnowledgeScopeSchema, SourceFrontMatterSchema, } from "./knowledge/schema.js";
+export { inEffect, MIN_QUERY_LENGTH, search, } from "./knowledge/search.js";
+export { knowledgePath, readKnowledgeFile, writeKnowledgeFile, } from "./knowledge/store.js";
 export { createRuntime, createToolCaller, createToolRegistry, MAX_TOOL_TEXT_CHARS, REVIEW_DIR_NAME, } from "./runtime.js";
 export { jsonSchemas } from "./schemas.js";
+export { businessDate, companyTime, hostTimezone, isTimezone } from "./time.js";
 export { ASK_USER, RUNTIME_PROVIDER_ID } from "./tool/ask-user.js";
+export { MAX_READ_BYTES, MAX_WRITE_BYTES, readWorkspaceText, readWorkspaceTextIfAny, writeWorkspaceText, } from "./tool/files.js";
 export { loadToolModule } from "./tool/load-module.js";
 export { RESERVED_PATHS, resolveWorkspacePath } from "./tool/paths.js";
 export { ToolRegistry, } from "./tool/registry.js";
@@ -18,6 +25,6 @@ export { EVENTS_FILE_NAME, EventLog } from "./work/event-log.js";
 export { canonical, EventFileSchema, eventFromFile, eventToFile, isKnownEventType, parsePayloadFile, payloadFileSchemas, TOOL_REJECTION_CODES, } from "./work/events.js";
 export { countToolCalls, pendingApprovals, pendingQuestions, workHistory, } from "./work/history.js";
 export { acquireLock, LOCK_FILE_NAME } from "./work/lock.js";
-export { buildProjection } from "./work/projection.js";
+export { buildProjection, RECENT_MESSAGES, } from "./work/projection.js";
 export { WORK_DIR_NAME, WORK_FILE_NAME, WorkStore, } from "./work/store.js";
 export { isTerminal, reduceWork, SESSION_WORK_TYPE, transition, WORK_STATUSES, WorkFileSchema, WorkStatus, workToFile, } from "./work/work.js";

package/dist/knowledge/build.d.ts

@@ -0,0 +1,102 @@
+import type { Checked } from "./check.ts";
+import type { Scope } from "./schema.ts";
+/**
+ * Turning what a person wrote into the index the runtime serves. The index is a build artifact:
+ * the same input always makes the same bytes, and the manifest says which input it came from and
+ * what the index itself hashes to, so a rewritten index is not served.
+ */
+/** Raised when the index is read by a runtime that indexes differently than the one that wrote it. */
+export declare const INDEX_FORMAT_VERSION = 1;
+/** One thing the search can return: a rule, or one section of a source. */
+export interface IndexUnit {
+    /** `rule:<id>` or `source:<id>#<heading>`; unique in the index. */
+    key: string;
+    kind: "rule" | "source";
+    /** The id a person wrote, which citations name. */
+    ref: string;
+    heading: string;
+    /** What the unit says: the statement of a rule, or the text of a section. */
+    text: string;
+    scope: Scope | null;
+    /** The professions a rule is for, or null for every profession. A source is for all of them. */
+    professions: string[] | null;
+    expertise: string;
+    from: string;
+    to: string | null;
+    /** For a rule, the source it cites. */
+    source?: {
+        id: string;
+        section?: string;
+    };
+    /** For a source, where it came from. */
+    provenance?: {
+        publisher: string;
+        title: string;
+        version?: string;
+        retrieved_at: string;
+    };
+}
+export interface KnowledgeIndex {
+    format: number;
+    units: IndexUnit[];
+    /**
+     * The groups of characters to the units that contain them. Two-character groups answer a
+     * question too short to have three.
+     */
+    postings: {
+        pairs: Record<string, number[]>;
+        triples: Record<string, number[]>;
+    };
+    /** How many distinct three-character grams each unit has, for the length correction. */
+    sizes: number[];
+}
+export interface Manifest {
+    format: number;
+    input_sha256: string;
+    index_sha256: string;
+    units: number;
+    rules: number;
+    sources: number;
+    built_at: string;
+}
+/** Text as the index compares it: full width and half width alike, one case, no spaces. */
+export declare function normalize(text: string): string;
+/** The distinct groups of `n` characters in the text. */
+export declare function grams(text: string, n: number): Set<string>;
+/**
+ * The index of a checked set. A rule that another rule replaces is closed the day before the
+ * newer one starts, so the two are never in effect together.
+ */
+export declare function buildIndex(checked: Checked): KnowledgeIndex;
+/** The same bytes for the same index: keys in a fixed order, two spaces, a newline at the end. */
+export declare function serializeIndex(index: KnowledgeIndex): string;
+/**
+ * The hash of what a person wrote, read from the files themselves rather than from what was
+ * parsed out of them. The runtime recomputes this before it trusts an index.
+ */
+export declare function hashKnowledgeInput(workspaceRoot: string): Promise<string>;
+/**
+ * Writes the index and then the manifest, each through a temporary file. The manifest lands last
+ * and is the mark that the index beside it is whole: a reader that finds a manifest finds an
+ * index that was fully written.
+ */
+export declare function writeIndex(workspaceRoot: string, index: KnowledgeIndex, input: {
+    hash: string;
+    rules: number;
+    sources: number;
+}, now?: Date): Promise<Manifest>;
+/** What the runtime reads before it serves knowledge, or a reason not to serve any. */
+export type IndexState = {
+    ok: true;
+    index: KnowledgeIndex;
+    manifest: Manifest;
+} | {
+    ok: false;
+    reason: string;
+};
+/**
+ * Reads the index only if it is the one the manifest describes and the manifest describes the
+ * files that are there now. An index rewritten on its own, a manifest rewritten on its own, and
+ * an index built by another version of the runtime all come back as a reason not to serve it.
+ */
+export declare function readIndex(workspaceRoot: string): Promise<IndexState>;

package/dist/knowledge/build.js

@@ -0,0 +1,279 @@
+import { createHash } from "node:crypto";
+import { readdir } from "node:fs/promises";
+import { join } from "node:path";
+import { readWorkspaceTextIfAny } from "../tool/files.js";
+import { KNOWLEDGE_DIR_NAME } from "./check.js";
+import { readKnowledgeFile, writeKnowledgeFile } from "./store.js";
+/**
+ * Turning what a person wrote into the index the runtime serves. The index is a build artifact:
+ * the same input always makes the same bytes, and the manifest says which input it came from and
+ * what the index itself hashes to, so a rewritten index is not served.
+ */
+/** Raised when the index is read by a runtime that indexes differently than the one that wrote it. */
+export const INDEX_FORMAT_VERSION = 1;
+const BUILD_DIR = "build";
+const INDEX_FILE = "index.json";
+const MANIFEST_FILE = "manifest.json";
+/** Text as the index compares it: full width and half width alike, one case, no spaces. */
+export function normalize(text) {
+    return text.normalize("NFKC").toLowerCase().replace(/\s+/gu, "");
+}
+/** The distinct groups of `n` characters in the text. */
+export function grams(text, n) {
+    const chars = [...normalize(text)];
+    const out = new Set();
+    for (let i = 0; i + n <= chars.length; i++)
+        out.add(chars.slice(i, i + n).join(""));
+    return out;
+}
+/**
+ * The index of a checked set. A rule that another rule replaces is closed the day before the
+ * newer one starts, so the two are never in effect together.
+ */
+export function buildIndex(checked) {
+    const closed = closeSuperseded(checked.rules);
+    const units = [
+        ...closed.map(ruleUnit),
+        ...checked.sources.flatMap((source) => sectionUnits(source)),
+    ].sort((a, b) => (a.key < b.key ? -1 : a.key > b.key ? 1 : 0));
+    const postings = { pairs: {}, triples: {} };
+    const sizes = [];
+    for (const [at, unit] of units.entries()) {
+        // What a unit is matched on: its own words, and its heading when that is not the words
+        // themselves. The id is left out; it would only make a unit look longer than it reads.
+        // Aliases are already part of a rule's text, and nothing else bridges words that share no
+        // characters with it.
+        const matter = unit.heading === unit.text ? unit.text : `${unit.text} ${unit.heading}`;
+        add(postings.pairs, grams(matter, 2), at);
+        add(postings.triples, grams(matter, 3), at);
+        sizes.push(grams(matter, 3).size);
+    }
+    return {
+        format: INDEX_FORMAT_VERSION,
+        units,
+        postings: { pairs: settled(postings.pairs), triples: settled(postings.triples) },
+        sizes,
+    };
+}
+/** Notes that the unit at `at` holds each of these groups. */
+function add(postings, of, at) {
+    for (const gram of of) {
+        const units = postings[gram];
+        if (units)
+            units.push(at);
+        else
+            postings[gram] = [at];
+    }
+}
+/** The same postings in a fixed order, so that the same input writes the same bytes. */
+function settled(postings) {
+    return Object.fromEntries(Object.entries(postings)
+        .sort(([a], [b]) => (a < b ? -1 : 1))
+        .map(([gram, units]) => [gram, [...units].sort((x, y) => x - y)]));
+}
+/** A rule replaced by another ends the day before that one begins. */
+function closeSuperseded(rules) {
+    const replacedBy = new Map();
+    for (const rule of rules)
+        if (rule.supersedes)
+            replacedBy.set(rule.supersedes, rule);
+    return rules.map((rule) => {
+        const next = replacedBy.get(rule.id);
+        if (!next)
+            return rule;
+        const end = dayBefore(next.effective_from);
+        return {
+            ...rule,
+            effective_to: rule.effective_to === null ? end : minDay(rule.effective_to, end),
+        };
+    });
+}
+function dayBefore(day) {
+    const at = new Date(`${day}T00:00:00Z`);
+    at.setUTCDate(at.getUTCDate() - 1);
+    return at.toISOString().slice(0, 10);
+}
+const minDay = (a, b) => (a < b ? a : b);
+function ruleUnit(rule) {
+    return {
+        key: `rule:${rule.id}`,
+        kind: "rule",
+        ref: rule.id,
+        heading: rule.statement,
+        text: [rule.statement, ...(rule.aliases ?? [])].join(" "),
+        scope: rule.scope ?? null,
+        professions: rule.applies_to?.profession ?? null,
+        expertise: rule.expertise,
+        from: rule.effective_from,
+        to: rule.effective_to,
+        source: {
+            id: rule.source.id,
+            ...(rule.source.section !== undefined && { section: rule.source.section }),
+        },
+    };
+}
+/** A source becomes one unit per heading; a source with no heading becomes one unit. */
+function sectionUnits(source) {
+    const provenance = {
+        publisher: source.publisher,
+        title: source.title,
+        ...(source.version !== undefined && { version: source.version }),
+        retrieved_at: source.retrieved_at,
+    };
+    const common = {
+        kind: "source",
+        ref: source.id,
+        scope: source.scope ?? null,
+        professions: null,
+        expertise: source.expertise,
+        from: source.effective_from,
+        to: source.effective_to,
+        provenance,
+    };
+    const sections = split(source.body);
+    if (sections.length === 0) {
+        return [{ ...common, key: `source:${source.id}#`, heading: source.title, text: source.body }];
+    }
+    // Two sections of one document may carry the same heading. A key names one unit, so the
+    // second one of a name says which it is.
+    const seen = new Map();
+    return sections.map((section) => {
+        const nth = (seen.get(section.heading) ?? 0) + 1;
+        seen.set(section.heading, nth);
+        return {
+            ...common,
+            key: `source:${source.id}#${section.heading}${nth === 1 ? "" : ` (${nth})`}`,
+            heading: section.heading,
+            text: section.text,
+        };
+    });
+}
+/** The body cut at its markdown headings. Text before the first heading joins the first section. */
+function split(body) {
+    const lines = body.split("\n");
+    const sections = [];
+    for (const line of lines) {
+        const heading = /^(#{1,6})\s+(.*)$/.exec(line);
+        if (heading)
+            sections.push({ heading: heading[2], text: [] });
+        else
+            sections.at(-1)?.text.push(line);
+    }
+    return sections.map((section) => ({
+        heading: section.heading,
+        text: section.text.join("\n").trim(),
+    }));
+}
+/** The same bytes for the same index: keys in a fixed order, two spaces, a newline at the end. */
+export function serializeIndex(index) {
+    const units = index.units.map((unit) => ordered(unit));
+    const body = {
+        format: index.format,
+        postings: index.postings,
+        sizes: index.sizes,
+        units,
+    };
+    return `${JSON.stringify(body, null, 2)}\n`;
+}
+/** An object with its keys in code point order, all the way down. */
+function ordered(value) {
+    const out = {};
+    for (const key of Object.keys(value).sort()) {
+        const inner = value[key];
+        out[key] =
+            inner && typeof inner === "object" && !Array.isArray(inner)
+                ? ordered(inner)
+                : inner;
+    }
+    return out;
+}
+const sha256 = (text) => createHash("sha256").update(text).digest("hex");
+/**
+ * The hash of what a person wrote, read from the files themselves rather than from what was
+ * parsed out of them. The runtime recomputes this before it trusts an index.
+ */
+export async function hashKnowledgeInput(workspaceRoot) {
+    const dir = join(workspaceRoot, KNOWLEDGE_DIR_NAME);
+    const parts = [];
+    for (const [sub, extension] of [
+        ["rules", ".yaml"],
+        ["sources", ".md"],
+    ]) {
+        let names;
+        try {
+            names = (await readdir(join(dir, sub))).filter((n) => n.endsWith(extension)).sort();
+        }
+        catch {
+            continue;
+        }
+        for (const name of names) {
+            // The same guarded read the checks use, so hashing and checking see one set of files: a
+            // file too large to read, or a link that leads out, is refused here as it is there.
+            const text = await readWorkspaceTextIfAny(dir, join(sub, name));
+            parts.push(`${sub}/${name}\n${text === undefined ? "unreadable" : sha256(text)}`);
+        }
+    }
+    return sha256(parts.join("\n"));
+}
+/**
+ * Writes the index and then the manifest, each through a temporary file. The manifest lands last
+ * and is the mark that the index beside it is whole: a reader that finds a manifest finds an
+ * index that was fully written.
+ */
+export async function writeIndex(workspaceRoot, index, input, now = new Date()) {
+    const serialized = serializeIndex(index);
+    const manifest = {
+        format: INDEX_FORMAT_VERSION,
+        input_sha256: input.hash,
+        index_sha256: sha256(serialized),
+        units: index.units.length,
+        rules: input.rules,
+        sources: input.sources,
+        built_at: now.toISOString(),
+    };
+    await writeKnowledgeFile(workspaceRoot, [BUILD_DIR, INDEX_FILE], serialized);
+    await writeKnowledgeFile(workspaceRoot, [BUILD_DIR, MANIFEST_FILE], `${JSON.stringify(manifest, null, 2)}\n`);
+    return manifest;
+}
+/**
+ * Reads the index only if it is the one the manifest describes and the manifest describes the
+ * files that are there now. An index rewritten on its own, a manifest rewritten on its own, and
+ * an index built by another version of the runtime all come back as a reason not to serve it.
+ */
+export async function readIndex(workspaceRoot) {
+    const stale = {
+        ok: false,
+        reason: "the index does not match the files it was built from; run `openshain knowledge build`",
+    };
+    let manifest;
+    let serialized;
+    try {
+        // Read nothing before knowing its size: these two files are as writable as any other in the
+        // folder, and an index of a company's knowledge is far below this.
+        const manifestText = await readKnowledgeFile(workspaceRoot, [BUILD_DIR, MANIFEST_FILE]);
+        const indexText = await readKnowledgeFile(workspaceRoot, [BUILD_DIR, INDEX_FILE]);
+        if (manifestText === undefined || indexText === undefined)
+            throw new Error("no index");
+        manifest = JSON.parse(manifestText);
+        serialized = indexText;
+    }
+    catch {
+        return { ok: false, reason: "there is no index; run `openshain knowledge build`" };
+    }
+    if (manifest.format !== INDEX_FORMAT_VERSION) {
+        return {
+            ok: false,
+            reason: "the index was built by another version of openshain; run `openshain knowledge build`",
+        };
+    }
+    if (sha256(serialized) !== manifest.index_sha256)
+        return stale;
+    if ((await hashKnowledgeInput(workspaceRoot)) !== manifest.input_sha256)
+        return stale;
+    try {
+        return { ok: true, index: JSON.parse(serialized), manifest };
+    }
+    catch {
+        return stale;
+    }
+}

package/dist/knowledge/check.d.ts

@@ -0,0 +1,30 @@
+import { type LoadedRule, type Source } from "./schema.ts";
+/**
+ * Reading and checking what a person wrote under `knowledge/`. Every problem is collected, never
+ * thrown at the first one: a person fixing a set of files should see all of it in one pass.
+ */
+export declare const KNOWLEDGE_DIR_NAME = "knowledge";
+/** How much a build reads in total, and how many files it reads, whatever is in the folder. */
+export declare const MAX_KNOWLEDGE_FILES = 2000;
+export declare const MAX_KNOWLEDGE_BYTES: number;
+export interface Checked {
+    rules: LoadedRule[];
+    sources: Source[];
+    /** Everything wrong with what was read, each as `file:line:col field: message` or `file: message`. */
+    problems: string[];
+}
+/** True when the workspace has a `knowledge/` directory to read at all. */
+export declare function hasKnowledge(workspaceRoot: string): Promise<boolean>;
+/**
+ * Reads `knowledge/rules/*.yaml` and `knowledge/sources/*.md` and checks them against each other.
+ * The rules that come back are the ones a build would index; when `problems` is not empty, nothing
+ * should be written.
+ */
+export declare function checkKnowledge(workspaceRoot: string, options?: {
+    /**
+     * Rules to check as though they were already written, with the file they would go in. A
+     * command that adds one asks this first, so that a rule which does not hold up is never
+     * written at all.
+     */
+    adding?: LoadedRule[];
+}): Promise<Checked>;