@openshain/core 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config/schema.d.ts +8 -0
- package/dist/config/schema.js +29 -1
- package/dist/errors.d.ts +6 -1
- package/dist/errors.js +9 -0
- package/dist/index.d.ts +10 -3
- package/dist/index.js +9 -2
- package/dist/knowledge/build.d.ts +102 -0
- package/dist/knowledge/build.js +279 -0
- package/dist/knowledge/check.d.ts +30 -0
- package/dist/knowledge/check.js +262 -0
- package/dist/knowledge/schema.d.ts +93 -0
- package/dist/knowledge/schema.js +76 -0
- package/dist/knowledge/search.d.ts +26 -0
- package/dist/knowledge/search.js +54 -0
- package/dist/knowledge/store.d.ts +10 -0
- package/dist/knowledge/store.js +69 -0
- package/dist/runtime.d.ts +3 -1
- package/dist/runtime.js +5 -7
- package/dist/schemas.d.ts +1 -1
- package/dist/schemas.js +3 -0
- package/dist/time.d.ts +14 -0
- package/dist/time.js +50 -0
- package/dist/tool/files.d.ts +21 -0
- package/dist/tool/files.js +69 -0
- package/dist/tool/paths.d.ts +1 -1
- package/dist/tool/paths.js +9 -1
- package/dist/tool/types.d.ts +13 -5
- package/dist/work/events.d.ts +23 -3
- package/dist/work/events.js +30 -3
- package/dist/work/projection.d.ts +9 -0
- package/dist/work/projection.js +54 -2
- package/package.json +1 -1
- package/src/config/schema.ts +42 -3
- package/src/errors.ts +12 -0
- package/src/index.ts +63 -2
- package/src/knowledge/build.ts +349 -0
- package/src/knowledge/check.ts +314 -0
- package/src/knowledge/schema.ts +105 -0
- package/src/knowledge/search.ts +74 -0
- package/src/knowledge/store.ts +82 -0
- package/src/runtime.ts +8 -8
- package/src/schemas.ts +14 -1
- package/src/time.ts +54 -0
- package/src/tool/files.ts +79 -0
- package/src/tool/paths.ts +9 -1
- package/src/tool/types.ts +14 -2
- package/src/work/events.ts +39 -4
- package/src/work/projection.ts +57 -2
|
@@ -0,0 +1,314 @@
|
|
|
1
|
+
import { readdir } from "node:fs/promises";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import { parseYamlFile } from "../config/yaml.ts";
|
|
4
|
+
import { isOpenshainError } from "../errors.ts";
|
|
5
|
+
import { readWorkspaceTextIfAny } from "../tool/files.ts";
|
|
6
|
+
import { resolveWorkspacePath } from "../tool/paths.ts";
|
|
7
|
+
import {
|
|
8
|
+
type LoadedRule,
|
|
9
|
+
RulesFileSchema,
|
|
10
|
+
type Scope,
|
|
11
|
+
type Source,
|
|
12
|
+
SourceFrontMatterSchema,
|
|
13
|
+
} from "./schema.ts";
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Reading and checking what a person wrote under `knowledge/`. Every problem is collected, never
|
|
17
|
+
* thrown at the first one: a person fixing a set of files should see all of it in one pass.
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
export const KNOWLEDGE_DIR_NAME = "knowledge";
|
|
21
|
+
|
|
22
|
+
/** How much a build reads in total, and how many files it reads, whatever is in the folder. */
|
|
23
|
+
export const MAX_KNOWLEDGE_FILES = 2000;
|
|
24
|
+
export const MAX_KNOWLEDGE_BYTES = 64 * 1024 * 1024;
|
|
25
|
+
const RULES_DIR = "rules";
|
|
26
|
+
const SOURCES_DIR = "sources";
|
|
27
|
+
|
|
28
|
+
export interface Checked {
|
|
29
|
+
rules: LoadedRule[];
|
|
30
|
+
sources: Source[];
|
|
31
|
+
/** Everything wrong with what was read, each as `file:line:col field: message` or `file: message`. */
|
|
32
|
+
problems: string[];
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** True when the workspace has a `knowledge/` directory to read at all. */
|
|
36
|
+
export async function hasKnowledge(workspaceRoot: string): Promise<boolean> {
|
|
37
|
+
try {
|
|
38
|
+
await readdir(join(workspaceRoot, KNOWLEDGE_DIR_NAME));
|
|
39
|
+
return true;
|
|
40
|
+
} catch {
|
|
41
|
+
return false;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Reads `knowledge/rules/*.yaml` and `knowledge/sources/*.md` and checks them against each other.
|
|
47
|
+
* The rules that come back are the ones a build would index; when `problems` is not empty, nothing
|
|
48
|
+
* should be written.
|
|
49
|
+
*/
|
|
50
|
+
export async function checkKnowledge(
|
|
51
|
+
workspaceRoot: string,
|
|
52
|
+
options: {
|
|
53
|
+
/**
|
|
54
|
+
* Rules to check as though they were already written, with the file they would go in. A
|
|
55
|
+
* command that adds one asks this first, so that a rule which does not hold up is never
|
|
56
|
+
* written at all.
|
|
57
|
+
*/
|
|
58
|
+
adding?: LoadedRule[];
|
|
59
|
+
} = {},
|
|
60
|
+
): Promise<Checked> {
|
|
61
|
+
const dir = join(workspaceRoot, KNOWLEDGE_DIR_NAME);
|
|
62
|
+
const problems: string[] = [];
|
|
63
|
+
const budget = { files: MAX_KNOWLEDGE_FILES, bytes: MAX_KNOWLEDGE_BYTES };
|
|
64
|
+
const rules = [...(await readRules(dir, problems, budget)), ...(options.adding ?? [])];
|
|
65
|
+
const sources = await readSources(dir, problems, budget);
|
|
66
|
+
|
|
67
|
+
await checkPaths(workspaceRoot, sources, problems);
|
|
68
|
+
crossCheck(rules, sources, problems);
|
|
69
|
+
return { rules, sources, problems: problems.sort() };
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/** Files of a directory in code point order; an unreadable directory is simply empty. */
|
|
73
|
+
async function filesOf(dir: string, extension: string): Promise<string[]> {
|
|
74
|
+
try {
|
|
75
|
+
return (await readdir(dir))
|
|
76
|
+
.filter((name) => name.endsWith(extension) && !name.startsWith("."))
|
|
77
|
+
.sort();
|
|
78
|
+
} catch {
|
|
79
|
+
return [];
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/** What a read may still take. A build reads a bounded amount, whatever the folder holds. */
|
|
84
|
+
interface Budget {
|
|
85
|
+
files: number;
|
|
86
|
+
bytes: number;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/** The text of a file, or a reason it was not read. Spends the budget as it goes. */
|
|
90
|
+
async function within(
|
|
91
|
+
dir: string,
|
|
92
|
+
relative: string,
|
|
93
|
+
file: string,
|
|
94
|
+
budget: Budget,
|
|
95
|
+
problems: string[],
|
|
96
|
+
): Promise<string | undefined> {
|
|
97
|
+
if (budget.files <= 0 || budget.bytes <= 0) {
|
|
98
|
+
problems.push(`${file}: not read; a build reads at most ${MAX_KNOWLEDGE_FILES} files`);
|
|
99
|
+
return undefined;
|
|
100
|
+
}
|
|
101
|
+
budget.files -= 1;
|
|
102
|
+
const text = await readWorkspaceTextIfAny(dir, relative);
|
|
103
|
+
if (text === undefined) {
|
|
104
|
+
problems.push(`${file}: cannot read the file`);
|
|
105
|
+
return undefined;
|
|
106
|
+
}
|
|
107
|
+
budget.bytes -= Buffer.byteLength(text, "utf8");
|
|
108
|
+
return text;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
async function readRules(dir: string, problems: string[], budget: Budget): Promise<LoadedRule[]> {
|
|
112
|
+
const rules: LoadedRule[] = [];
|
|
113
|
+
for (const name of await filesOf(join(dir, RULES_DIR), ".yaml")) {
|
|
114
|
+
const file = `${KNOWLEDGE_DIR_NAME}/${RULES_DIR}/${name}`;
|
|
115
|
+
const text = await within(dir, join(RULES_DIR, name), file, budget, problems);
|
|
116
|
+
if (text === undefined) continue;
|
|
117
|
+
try {
|
|
118
|
+
const { data } = parseYamlFile(text, RulesFileSchema, file);
|
|
119
|
+
for (const rule of data.rules) rules.push({ ...rule, file });
|
|
120
|
+
} catch (err) {
|
|
121
|
+
problems.push(...messageOf(err, file));
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
return rules;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
async function readSources(dir: string, problems: string[], budget: Budget): Promise<Source[]> {
|
|
128
|
+
const sources: Source[] = [];
|
|
129
|
+
for (const name of await filesOf(join(dir, SOURCES_DIR), ".md")) {
|
|
130
|
+
const file = `${KNOWLEDGE_DIR_NAME}/${SOURCES_DIR}/${name}`;
|
|
131
|
+
const text = await within(dir, join(SOURCES_DIR, name), file, budget, problems);
|
|
132
|
+
if (text === undefined) continue;
|
|
133
|
+
const split = frontMatter(text);
|
|
134
|
+
if (!split) {
|
|
135
|
+
problems.push(`${file}: the file must start with front matter between --- lines`);
|
|
136
|
+
continue;
|
|
137
|
+
}
|
|
138
|
+
try {
|
|
139
|
+
const { data } = parseYamlFile(split.head, SourceFrontMatterSchema, file);
|
|
140
|
+
sources.push({ ...data, body: split.body, file });
|
|
141
|
+
} catch (err) {
|
|
142
|
+
problems.push(...messageOf(err, file));
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
return sources;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/** The YAML between the opening `---` and the next one, and everything after it. */
|
|
149
|
+
function frontMatter(text: string): { head: string; body: string } | undefined {
|
|
150
|
+
const lines = text.split("\n");
|
|
151
|
+
if (lines[0]?.trim() !== "---") return undefined;
|
|
152
|
+
const end = lines.indexOf("---", 1);
|
|
153
|
+
if (end < 0) return undefined;
|
|
154
|
+
return {
|
|
155
|
+
head: lines.slice(1, end).join("\n"),
|
|
156
|
+
body: lines
|
|
157
|
+
.slice(end + 1)
|
|
158
|
+
.join("\n")
|
|
159
|
+
.trim(),
|
|
160
|
+
};
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
function messageOf(err: unknown, file: string): string[] {
|
|
164
|
+
if (isOpenshainError(err)) return err.message.split("\n");
|
|
165
|
+
return [`${file}: ${err instanceof Error ? err.message : String(err)}`];
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
/**
|
|
169
|
+
* A source that names a file names it inside the company folder. The guard the tools run under
|
|
170
|
+
* answers this, so a source cannot pull `authority/`, a principal's record or anything outside
|
|
171
|
+
* the folder into the index.
|
|
172
|
+
*/
|
|
173
|
+
async function checkPaths(
|
|
174
|
+
workspaceRoot: string,
|
|
175
|
+
sources: Source[],
|
|
176
|
+
problems: string[],
|
|
177
|
+
): Promise<void> {
|
|
178
|
+
for (const source of sources) {
|
|
179
|
+
if (source.path === undefined) continue;
|
|
180
|
+
try {
|
|
181
|
+
await resolveWorkspacePath(workspaceRoot, source.path);
|
|
182
|
+
} catch (err) {
|
|
183
|
+
problems.push(
|
|
184
|
+
`${source.file}: ${source.id} points at ${source.path}, which the runtime does not read (${err instanceof Error ? err.message : String(err)})`,
|
|
185
|
+
);
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
/** The checks that need more than one file: references, scope, effective days. */
|
|
191
|
+
function crossCheck(rules: LoadedRule[], sources: Source[], problems: string[]): void {
|
|
192
|
+
const byId = new Map(sources.map((source) => [source.id, source]));
|
|
193
|
+
for (const source of duplicates(sources)) {
|
|
194
|
+
problems.push(`${source.file}: source ${source.id} is defined more than once`);
|
|
195
|
+
}
|
|
196
|
+
for (const rule of duplicates(rules)) {
|
|
197
|
+
problems.push(`${rule.file}: rule ${rule.id} is defined more than once`);
|
|
198
|
+
}
|
|
199
|
+
for (const item of [...sources, ...rules]) endsAfterItStarts(item, problems);
|
|
200
|
+
for (const rule of rules) againstItsSource(rule, byId.get(rule.source.id), problems);
|
|
201
|
+
overlaps(rules, problems);
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
/** A day of the calendar comes before another; a rule or a source that ends first says nothing. */
|
|
205
|
+
function endsAfterItStarts(
|
|
206
|
+
item: { id: string; file: string; effective_from: string; effective_to: string | null },
|
|
207
|
+
problems: string[],
|
|
208
|
+
): void {
|
|
209
|
+
if (item.effective_to !== null && item.effective_to < item.effective_from) {
|
|
210
|
+
problems.push(`${item.file}: ${item.id} ends before it starts`);
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
/**
|
|
215
|
+
* A rule stands on the source it cites, so it may not exist without it, outlive it, begin before
|
|
216
|
+
* it, or be read by people who may not read it.
|
|
217
|
+
*/
|
|
218
|
+
function againstItsSource(rule: LoadedRule, source: Source | undefined, problems: string[]): void {
|
|
219
|
+
if (!source) {
|
|
220
|
+
problems.push(`${rule.file}: ${rule.id} cites ${rule.source.id}, which no source declares`);
|
|
221
|
+
return;
|
|
222
|
+
}
|
|
223
|
+
if (rule.effective_from < source.effective_from) {
|
|
224
|
+
problems.push(
|
|
225
|
+
`${rule.file}: ${rule.id} starts before ${source.id}, the source it cites, is in effect`,
|
|
226
|
+
);
|
|
227
|
+
}
|
|
228
|
+
if (source.effective_to !== null && (rule.effective_to ?? FOREVER) > source.effective_to) {
|
|
229
|
+
problems.push(
|
|
230
|
+
`${rule.file}: ${rule.id} outlives ${source.id}, the source it cites; close it on ${source.effective_to} or cite a newer source`,
|
|
231
|
+
);
|
|
232
|
+
}
|
|
233
|
+
if (!covers(source.scope, rule.scope)) {
|
|
234
|
+
problems.push(
|
|
235
|
+
`${rule.file}: ${rule.id} may be read by more people than ${source.id}, the source it cites; its citation would name a source they cannot read`,
|
|
236
|
+
);
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
/** A day later than any a person would write, for a rule or a source with no end. */
|
|
241
|
+
const FOREVER = "9999-12-31";
|
|
242
|
+
|
|
243
|
+
/** The items whose id was already taken by an earlier item. */
|
|
244
|
+
function duplicates<T extends { id: string }>(items: T[]): T[] {
|
|
245
|
+
const seen = new Set<string>();
|
|
246
|
+
const again: T[] = [];
|
|
247
|
+
for (const item of items) {
|
|
248
|
+
if (seen.has(item.id)) again.push(item);
|
|
249
|
+
else seen.add(item.id);
|
|
250
|
+
}
|
|
251
|
+
return again;
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
/** The people a scope names, and how it names them. Company-wide names everyone. */
|
|
255
|
+
function named(scope: Scope | undefined): {
|
|
256
|
+
kind: "everyone" | "principals" | "roles";
|
|
257
|
+
who: string[];
|
|
258
|
+
} {
|
|
259
|
+
if (scope === undefined || "visibility" in scope) return { kind: "everyone", who: [] };
|
|
260
|
+
if ("principals" in scope) return { kind: "principals", who: scope.principals };
|
|
261
|
+
return { kind: "roles", who: scope.roles };
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
/** Whether everyone who may read `inner` may also read `outer`. */
|
|
265
|
+
function covers(outer: Scope | undefined, inner: Scope | undefined): boolean {
|
|
266
|
+
const wider = named(outer);
|
|
267
|
+
const narrower = named(inner);
|
|
268
|
+
if (wider.kind === "everyone") return true;
|
|
269
|
+
if (narrower.kind === "everyone") return false;
|
|
270
|
+
// Two lists of different kinds cannot be compared without knowing who holds which role.
|
|
271
|
+
if (wider.kind !== narrower.kind) return false;
|
|
272
|
+
const allowed = new Set(wider.who);
|
|
273
|
+
return narrower.who.every((name) => allowed.has(name));
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
/**
|
|
277
|
+
* Two rules drawn from the very same passage, both in effect, are two answers to one question
|
|
278
|
+
* unless one says it replaces the other or they are for different people or professions.
|
|
279
|
+
*
|
|
280
|
+
* The passage, not the document: one policy document backs many rules — receipts over one
|
|
281
|
+
* amount, an approval over another — and that is how a company writes. Only rules that name the
|
|
282
|
+
* same section of the same source are compared, so ordinary writing is never refused.
|
|
283
|
+
*/
|
|
284
|
+
function overlaps(rules: LoadedRule[], problems: string[]): void {
|
|
285
|
+
const replaced = new Set(rules.map((rule) => rule.supersedes).filter(Boolean));
|
|
286
|
+
for (const [i, rule] of rules.entries()) {
|
|
287
|
+
for (const other of rules.slice(i + 1)) {
|
|
288
|
+
if (rule.source.id !== other.source.id) continue;
|
|
289
|
+
if (rule.source.section === undefined || rule.source.section !== other.source.section) {
|
|
290
|
+
continue;
|
|
291
|
+
}
|
|
292
|
+
if (replaced.has(rule.id) || replaced.has(other.id)) continue;
|
|
293
|
+
if (!inEffectTogether(rule, other)) continue;
|
|
294
|
+
if (disjoint(rule, other)) continue;
|
|
295
|
+
problems.push(
|
|
296
|
+
`${other.file}: ${other.id} and ${rule.id} both cite ${rule.source.id} の ${rule.source.section} and are in effect at the same time; close one with effective_to, or say supersedes`,
|
|
297
|
+
);
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
function inEffectTogether(a: LoadedRule, b: LoadedRule): boolean {
|
|
303
|
+
const end = (rule: LoadedRule) => rule.effective_to ?? FOREVER;
|
|
304
|
+
return a.effective_from <= end(b) && b.effective_from <= end(a);
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
/** Rules that no one person and no one profession sees together do not contradict each other. */
|
|
308
|
+
function disjoint(a: LoadedRule, b: LoadedRule): boolean {
|
|
309
|
+
const professions = (rule: LoadedRule) => rule.applies_to?.profession;
|
|
310
|
+
const pa = professions(a);
|
|
311
|
+
const pb = professions(b);
|
|
312
|
+
if (pa && pb && !pa.some((name) => pb.includes(name))) return true;
|
|
313
|
+
return !covers(a.scope, b.scope) && !covers(b.scope, a.scope);
|
|
314
|
+
}
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* What a person writes under `knowledge/`: the company's own rules, and the sources behind them.
|
|
5
|
+
* The shapes only; what makes a set of them consistent is in `check.ts`.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
/** An id a person writes: lowercase, and readable as a path of meaning (`expenses.receipt-required`). */
|
|
9
|
+
const knowledgeId = z
|
|
10
|
+
.string()
|
|
11
|
+
.min(1)
|
|
12
|
+
.max(200)
|
|
13
|
+
.regex(
|
|
14
|
+
/^[a-z0-9][a-z0-9._-]*$/,
|
|
15
|
+
"use lowercase letters, digits, . - or _, starting with a letter or a digit",
|
|
16
|
+
);
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* A date as the company writes it. The runtime compares these as strings, so the form is fixed,
|
|
20
|
+
* and it must be a day that exists: the index closes a superseded rule the day before the next
|
|
21
|
+
* one starts, and `2026-13-45` would end that arithmetic in an error rather than a refusal.
|
|
22
|
+
*/
|
|
23
|
+
const day = z
|
|
24
|
+
.string()
|
|
25
|
+
.regex(/^\d{4}-\d{2}-\d{2}$/, "write a date as YYYY-MM-DD")
|
|
26
|
+
.refine(isDay, "that day does not exist");
|
|
27
|
+
|
|
28
|
+
function isDay(text: string): boolean {
|
|
29
|
+
const at = new Date(`${text}T00:00:00Z`);
|
|
30
|
+
return !Number.isNaN(at.getTime()) && at.toISOString().startsWith(text);
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Who may read it. A company with one principal may leave it out; with two or more it is written,
|
|
35
|
+
* because from then on leaving it out would mean deciding by accident.
|
|
36
|
+
*/
|
|
37
|
+
export const ScopeSchema = z.union([
|
|
38
|
+
z.strictObject({ visibility: z.literal("company") }),
|
|
39
|
+
z.strictObject({ principals: z.array(knowledgeId).min(1) }),
|
|
40
|
+
z.strictObject({ roles: z.array(knowledgeId).min(1) }),
|
|
41
|
+
]);
|
|
42
|
+
export type Scope = z.infer<typeof ScopeSchema>;
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* The professional domain a piece of knowledge belongs to. `none` is the only value core knows;
|
|
46
|
+
* the rest is a string a profession pack or the company defines. Core compares it and holds no
|
|
47
|
+
* list of qualifications (docs/design/core.md, spec/professional-boundary.md).
|
|
48
|
+
*/
|
|
49
|
+
const expertise = z
|
|
50
|
+
.string()
|
|
51
|
+
.min(1)
|
|
52
|
+
.max(40)
|
|
53
|
+
.regex(/^[a-z][a-z0-9-]*$/, "use lowercase letters, digits and -");
|
|
54
|
+
|
|
55
|
+
export const RuleSchema = z.strictObject({
|
|
56
|
+
id: knowledgeId,
|
|
57
|
+
statement: z.string().min(10).max(240),
|
|
58
|
+
aliases: z.array(z.string().min(1).max(80)).max(20).optional(),
|
|
59
|
+
applies_to: z.strictObject({ profession: z.array(knowledgeId).min(1) }).optional(),
|
|
60
|
+
scope: ScopeSchema.optional(),
|
|
61
|
+
effective_from: day,
|
|
62
|
+
// Written even when there is no end, so that an open-ended rule is a decision, not an omission.
|
|
63
|
+
effective_to: day.nullable(),
|
|
64
|
+
supersedes: knowledgeId.optional(),
|
|
65
|
+
expertise,
|
|
66
|
+
source: z.strictObject({ id: knowledgeId, section: z.string().min(1).max(200).optional() }),
|
|
67
|
+
});
|
|
68
|
+
export type Rule = z.infer<typeof RuleSchema>;
|
|
69
|
+
|
|
70
|
+
export const RulesFileSchema = z.strictObject({
|
|
71
|
+
version: z.literal(1),
|
|
72
|
+
rules: z.array(RuleSchema).min(1),
|
|
73
|
+
});
|
|
74
|
+
|
|
75
|
+
/** The front matter of a source document. The body below it is the citation itself. */
|
|
76
|
+
export const SourceFrontMatterSchema = z
|
|
77
|
+
.strictObject({
|
|
78
|
+
id: knowledgeId,
|
|
79
|
+
title: z.string().min(1).max(200),
|
|
80
|
+
publisher: z.string().min(1).max(200),
|
|
81
|
+
url: z.url().optional(),
|
|
82
|
+
path: z.string().min(1).max(1000).optional(),
|
|
83
|
+
retrieved_at: day,
|
|
84
|
+
version: z.string().min(1).max(80).optional(),
|
|
85
|
+
effective_from: day,
|
|
86
|
+
effective_to: day.nullable(),
|
|
87
|
+
scope: ScopeSchema.optional(),
|
|
88
|
+
expertise,
|
|
89
|
+
})
|
|
90
|
+
.refine(
|
|
91
|
+
(source) => source.url !== undefined || source.path !== undefined,
|
|
92
|
+
"say where it came from: url or path",
|
|
93
|
+
);
|
|
94
|
+
export type SourceFrontMatter = z.infer<typeof SourceFrontMatterSchema>;
|
|
95
|
+
|
|
96
|
+
/** A source as it was read: its front matter, the body, and the file it came from. */
|
|
97
|
+
export interface Source extends SourceFrontMatter {
|
|
98
|
+
body: string;
|
|
99
|
+
file: string;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/** A rule as it was read, with the file it came from for the messages. */
|
|
103
|
+
export interface LoadedRule extends Rule {
|
|
104
|
+
file: string;
|
|
105
|
+
}
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import { grams, type IndexUnit, type KnowledgeIndex, normalize } from "./build.ts";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Finding a unit by the characters it shares with a question. There is no embedding and no
|
|
5
|
+
* outside search engine: the index is groups of two and three characters, and the score is how
|
|
6
|
+
* much of the question a unit accounts for.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
export interface Hit {
|
|
10
|
+
unit: IndexUnit;
|
|
11
|
+
/** Between 0 and 1. The share of the question's grams the unit holds, corrected for length. */
|
|
12
|
+
score: number;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
/** The shortest question the index can answer. One character matches almost everything. */
|
|
16
|
+
export const MIN_QUERY_LENGTH = 2;
|
|
17
|
+
|
|
18
|
+
export interface SearchOptions {
|
|
19
|
+
/** How many hits to return. */
|
|
20
|
+
limit?: number;
|
|
21
|
+
/** Only these units are searched. Everything else is invisible, count included. */
|
|
22
|
+
allowed?: (unit: IndexUnit) => boolean;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* The best units for a question, most fitting first. Two hits with the same score keep the order
|
|
27
|
+
* of their keys, so the same index and question always answer the same way.
|
|
28
|
+
*/
|
|
29
|
+
export function search(index: KnowledgeIndex, query: string, options: SearchOptions = {}): Hit[] {
|
|
30
|
+
const text = normalize(query);
|
|
31
|
+
if (text.length < MIN_QUERY_LENGTH) return [];
|
|
32
|
+
// A question of two characters has no group of three, and two-character words are ordinary in
|
|
33
|
+
// Japanese business writing, so the shorter grams answer it.
|
|
34
|
+
const n = text.length < 3 ? 2 : 3;
|
|
35
|
+
const wanted = grams(query, n);
|
|
36
|
+
if (wanted.size === 0) return [];
|
|
37
|
+
const postings = n === 2 ? index.postings.pairs : index.postings.triples;
|
|
38
|
+
|
|
39
|
+
const allowed = options.allowed ?? (() => true);
|
|
40
|
+
// Narrow before scoring: what a person may not read costs nothing to rank, and the time a
|
|
41
|
+
// search takes then says nothing about how much of it there is.
|
|
42
|
+
const visible = new Set<number>();
|
|
43
|
+
for (const [at, unit] of index.units.entries()) if (allowed(unit)) visible.add(at);
|
|
44
|
+
|
|
45
|
+
const matched = new Map<number, number>();
|
|
46
|
+
for (const gram of wanted) {
|
|
47
|
+
for (const at of postings[gram] ?? []) {
|
|
48
|
+
if (visible.has(at)) matched.set(at, (matched.get(at) ?? 0) + 1);
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
const hits: Hit[] = [];
|
|
53
|
+
for (const [at, count] of matched) {
|
|
54
|
+
const unit = index.units[at];
|
|
55
|
+
if (!unit) continue;
|
|
56
|
+
hits.push({ unit, score: (count / wanted.size) * lengthCorrection(index.sizes[at] ?? 0) });
|
|
57
|
+
}
|
|
58
|
+
hits.sort((a, b) => b.score - a.score || (a.unit.key < b.unit.key ? -1 : 1));
|
|
59
|
+
return hits.slice(0, options.limit ?? 5);
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* A long document holds more groups of characters and so matches more questions by size alone.
|
|
64
|
+
* The correction keeps a short rule from losing to a page of prose that happens to contain the
|
|
65
|
+
* same words; it lowers a long unit's score without ever ruling it out.
|
|
66
|
+
*/
|
|
67
|
+
function lengthCorrection(size: number): number {
|
|
68
|
+
return 1 / (1 + Math.log10(1 + size / 40));
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/** Whether a unit is in effect on a day. */
|
|
72
|
+
export function inEffect(unit: IndexUnit, day: string): boolean {
|
|
73
|
+
return unit.from <= day && (unit.to === null || day <= unit.to);
|
|
74
|
+
}
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
import { constants } from "node:fs";
|
|
2
|
+
import { mkdir, open, realpath, rename } from "node:fs/promises";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
import { OpenshainError } from "../errors.ts";
|
|
5
|
+
import { KNOWLEDGE_DIR_NAME } from "./check.ts";
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Touching the files under `knowledge/`. The tools cannot reach them — it is a reserved path, so
|
|
9
|
+
* that what the index filters cannot be read around — which leaves the runtime to read and write
|
|
10
|
+
* them itself, without the guard the tools go through. These functions are that guard: the
|
|
11
|
+
* directory must be the one inside the company folder, and a link must never carry a write.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
/** How large a file under `knowledge/` may be for the runtime to read it whole. */
|
|
15
|
+
const MAX_BYTES = 64 * 1024 * 1024;
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* The directory the path names inside `knowledge/`, once it is known to be that directory. A
|
|
19
|
+
* link left in the folder would otherwise send a write anywhere the person can write.
|
|
20
|
+
*/
|
|
21
|
+
async function directory(workspaceRoot: string, parts: string[]): Promise<string> {
|
|
22
|
+
const root = await realpath(workspaceRoot);
|
|
23
|
+
const dir = join(root, KNOWLEDGE_DIR_NAME, ...parts);
|
|
24
|
+
await mkdir(dir, { recursive: true });
|
|
25
|
+
if ((await realpath(dir)) !== dir) {
|
|
26
|
+
throw new OpenshainError(
|
|
27
|
+
"invalid_path",
|
|
28
|
+
`${[KNOWLEDGE_DIR_NAME, ...parts].join("/")} leads out of the company folder; nothing was written`,
|
|
29
|
+
);
|
|
30
|
+
}
|
|
31
|
+
return dir;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Writes a file under `knowledge/` through a temporary file renamed into place, so a reader
|
|
36
|
+
* never sees half of one. The temporary name is easy to guess, so it is opened without following
|
|
37
|
+
* a link.
|
|
38
|
+
*/
|
|
39
|
+
export async function writeKnowledgeFile(
|
|
40
|
+
workspaceRoot: string,
|
|
41
|
+
parts: string[],
|
|
42
|
+
text: string,
|
|
43
|
+
): Promise<void> {
|
|
44
|
+
const name = parts.at(-1) as string;
|
|
45
|
+
const dir = await directory(workspaceRoot, parts.slice(0, -1));
|
|
46
|
+
const path = join(dir, name);
|
|
47
|
+
const temporary = `${path}.writing`;
|
|
48
|
+
const flags =
|
|
49
|
+
constants.O_WRONLY | constants.O_CREAT | constants.O_TRUNC | (constants.O_NOFOLLOW ?? 0);
|
|
50
|
+
const handle = await open(temporary, flags, 0o644);
|
|
51
|
+
try {
|
|
52
|
+
await handle.writeFile(text, "utf8");
|
|
53
|
+
} finally {
|
|
54
|
+
await handle.close();
|
|
55
|
+
}
|
|
56
|
+
await rename(temporary, path);
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/** Reads a file under `knowledge/`, or nothing when it is missing, a link, or too large. */
|
|
60
|
+
export async function readKnowledgeFile(
|
|
61
|
+
workspaceRoot: string,
|
|
62
|
+
parts: string[],
|
|
63
|
+
): Promise<string | undefined> {
|
|
64
|
+
const path = join(workspaceRoot, KNOWLEDGE_DIR_NAME, ...parts);
|
|
65
|
+
try {
|
|
66
|
+
const handle = await open(path, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0));
|
|
67
|
+
try {
|
|
68
|
+
const { size } = await handle.stat();
|
|
69
|
+
if (size > MAX_BYTES) return undefined;
|
|
70
|
+
return await handle.readFile("utf8");
|
|
71
|
+
} finally {
|
|
72
|
+
await handle.close();
|
|
73
|
+
}
|
|
74
|
+
} catch {
|
|
75
|
+
return undefined;
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/** Where a file under `knowledge/` is, for a message a person reads. */
|
|
80
|
+
export function knowledgePath(parts: string[]): string {
|
|
81
|
+
return [KNOWLEDGE_DIR_NAME, ...parts].join("/");
|
|
82
|
+
}
|
package/src/runtime.ts
CHANGED
|
@@ -11,6 +11,7 @@ import { loadConfig } from "./config/load.ts";
|
|
|
11
11
|
import type { Config, ModelConfig } from "./config/schema.ts";
|
|
12
12
|
import { isOpenshainError, OpenshainError } from "./errors.ts";
|
|
13
13
|
import type { ModelProvider } from "./model/types.ts";
|
|
14
|
+
import { businessDate } from "./time.ts";
|
|
14
15
|
import { loadToolModule } from "./tool/load-module.ts";
|
|
15
16
|
import type { HiddenTool } from "./tool/registry.ts";
|
|
16
17
|
import { type RegisteredTool, ToolRegistry } from "./tool/registry.ts";
|
|
@@ -24,7 +25,9 @@ export interface RuntimeProviders {
|
|
|
24
25
|
/** Model providers by the id used in openshain.yaml. */
|
|
25
26
|
models: Record<string, (model: ModelConfig) => ModelProvider>;
|
|
26
27
|
/** Tool providers by the id used in openshain.yaml. Modules are loaded from the config directly. */
|
|
27
|
-
|
|
28
|
+
/** By the provider id used in openshain.yaml. The workspace is given, since what a provider
|
|
29
|
+
* offers can depend on what is in it. */
|
|
30
|
+
tools: Record<string, (workspaceRoot: string) => ToolProvider>;
|
|
28
31
|
}
|
|
29
32
|
|
|
30
33
|
export interface CreateRuntimeOptions {
|
|
@@ -150,7 +153,7 @@ export async function createToolRegistry(
|
|
|
150
153
|
`unknown tool provider "${entry.provider}"; known providers: ${Object.keys(tools).join(", ")}`,
|
|
151
154
|
);
|
|
152
155
|
}
|
|
153
|
-
await registry.register(factory(), registerOptions);
|
|
156
|
+
await registry.register(factory(workspaceRoot), registerOptions);
|
|
154
157
|
} else {
|
|
155
158
|
await registry.register(await loadToolModule(workspaceRoot, entry.module), registerOptions);
|
|
156
159
|
}
|
|
@@ -215,7 +218,7 @@ async function callTool(input: {
|
|
|
215
218
|
principal: config.principal.id,
|
|
216
219
|
profession: config.profession.id,
|
|
217
220
|
workType: (await work.current()).type,
|
|
218
|
-
businessDate: businessDate(),
|
|
221
|
+
businessDate: businessDate(config.company.timezone),
|
|
219
222
|
});
|
|
220
223
|
if (judged.kind === "deny") return reject("denied", judged.reason);
|
|
221
224
|
if (judged.kind === "approval_required" || judged.kind === "review_required") {
|
|
@@ -282,6 +285,8 @@ async function callTool(input: {
|
|
|
282
285
|
result = await tool.provider.call(call, {
|
|
283
286
|
workId: work.id,
|
|
284
287
|
principalId: config.principal.id,
|
|
288
|
+
profession: config.profession.id,
|
|
289
|
+
businessDate: businessDate(config.company.timezone),
|
|
285
290
|
workspaceRoot,
|
|
286
291
|
});
|
|
287
292
|
} catch (err) {
|
|
@@ -370,11 +375,6 @@ function pathOf(input: unknown): string | undefined {
|
|
|
370
375
|
}
|
|
371
376
|
|
|
372
377
|
/** Today's date on this machine's clock, YYYY-MM-DD. */
|
|
373
|
-
function businessDate(): string {
|
|
374
|
-
const now = new Date();
|
|
375
|
-
const pad = (n: number) => String(n).padStart(2, "0");
|
|
376
|
-
return `${now.getFullYear()}-${pad(now.getMonth() + 1)}-${pad(now.getDate())}`;
|
|
377
|
-
}
|
|
378
378
|
|
|
379
379
|
function isRejectionCode(code: string): code is ToolRejectionCode {
|
|
380
380
|
return (TOOL_REJECTION_CODES as readonly string[]).includes(code);
|
package/src/schemas.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
2
|
import { DelegationsFileSchema, PolicyFileSchema } from "./authority/policy.ts";
|
|
3
3
|
import { ConfigFileSchema } from "./config/schema.ts";
|
|
4
|
+
import { RulesFileSchema, SourceFrontMatterSchema } from "./knowledge/schema.ts";
|
|
4
5
|
import type { JsonSchema } from "./tool/types.ts";
|
|
5
6
|
import { EventFileSchema, payloadFileSchemas } from "./work/events.ts";
|
|
6
7
|
import { WorkFileSchema } from "./work/work.ts";
|
|
@@ -10,7 +11,9 @@ export type SchemaName =
|
|
|
10
11
|
| "events.v1"
|
|
11
12
|
| "work.v1"
|
|
12
13
|
| "authority-policy.v1"
|
|
13
|
-
| "authority-delegations.v1"
|
|
14
|
+
| "authority-delegations.v1"
|
|
15
|
+
| "knowledge-rules.v1"
|
|
16
|
+
| "knowledge-source.v1";
|
|
14
17
|
|
|
15
18
|
/**
|
|
16
19
|
* The JSON Schemas (draft 2020-12) of the files openshain reads and writes, derived from the zod
|
|
@@ -41,6 +44,16 @@ export function jsonSchemas(): Record<SchemaName, JsonSchema> {
|
|
|
41
44
|
"authority/delegations.yaml",
|
|
42
45
|
"Who the agent may act for, as which profession, and when.",
|
|
43
46
|
),
|
|
47
|
+
"knowledge-rules.v1": describe(
|
|
48
|
+
RulesFileSchema,
|
|
49
|
+
"knowledge/rules/*.yaml",
|
|
50
|
+
"The company's own rules, each with the source behind it and the days it is in effect.",
|
|
51
|
+
),
|
|
52
|
+
"knowledge-source.v1": describe(
|
|
53
|
+
SourceFrontMatterSchema,
|
|
54
|
+
"knowledge/sources/*.md (front matter)",
|
|
55
|
+
"Where a cited document came from, when it applies, and who may read it.",
|
|
56
|
+
),
|
|
44
57
|
};
|
|
45
58
|
}
|
|
46
59
|
|