@openshain/core 0.4.1 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config/schema.d.ts +6 -0
- package/dist/config/schema.js +13 -0
- package/dist/errors.d.ts +6 -1
- package/dist/errors.js +9 -0
- package/dist/index.d.ts +9 -3
- package/dist/index.js +8 -2
- package/dist/knowledge/build.d.ts +102 -0
- package/dist/knowledge/build.js +279 -0
- package/dist/knowledge/check.d.ts +30 -0
- package/dist/knowledge/check.js +262 -0
- package/dist/knowledge/schema.d.ts +93 -0
- package/dist/knowledge/schema.js +76 -0
- package/dist/knowledge/search.d.ts +26 -0
- package/dist/knowledge/search.js +54 -0
- package/dist/knowledge/store.d.ts +10 -0
- package/dist/knowledge/store.js +69 -0
- package/dist/runtime.d.ts +3 -1
- package/dist/runtime.js +3 -1
- package/dist/schemas.d.ts +1 -1
- package/dist/schemas.js +3 -0
- package/dist/tool/files.d.ts +21 -0
- package/dist/tool/files.js +69 -0
- package/dist/tool/paths.d.ts +1 -1
- package/dist/tool/paths.js +9 -1
- package/dist/tool/types.d.ts +13 -5
- package/dist/work/events.d.ts +23 -3
- package/dist/work/events.js +30 -3
- package/dist/work/projection.d.ts +9 -0
- package/dist/work/projection.js +54 -2
- package/package.json +1 -1
- package/src/config/schema.ts +25 -1
- package/src/errors.ts +12 -0
- package/src/index.ts +62 -2
- package/src/knowledge/build.ts +349 -0
- package/src/knowledge/check.ts +314 -0
- package/src/knowledge/schema.ts +105 -0
- package/src/knowledge/search.ts +74 -0
- package/src/knowledge/store.ts +82 -0
- package/src/runtime.ts +6 -2
- package/src/schemas.ts +14 -1
- package/src/tool/files.ts +79 -0
- package/src/tool/paths.ts +9 -1
- package/src/tool/types.ts +14 -2
- package/src/work/events.ts +39 -4
- package/src/work/projection.ts +57 -2
|
@@ -0,0 +1,262 @@
|
|
|
1
|
+
import { readdir } from "node:fs/promises";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import { parseYamlFile } from "../config/yaml.js";
|
|
4
|
+
import { isOpenshainError } from "../errors.js";
|
|
5
|
+
import { readWorkspaceTextIfAny } from "../tool/files.js";
|
|
6
|
+
import { resolveWorkspacePath } from "../tool/paths.js";
|
|
7
|
+
import { RulesFileSchema, SourceFrontMatterSchema, } from "./schema.js";
|
|
8
|
+
/**
|
|
9
|
+
* Reading and checking what a person wrote under `knowledge/`. Every problem is collected, never
|
|
10
|
+
* thrown at the first one: a person fixing a set of files should see all of it in one pass.
|
|
11
|
+
*/
|
|
12
|
+
export const KNOWLEDGE_DIR_NAME = "knowledge";
|
|
13
|
+
/** How much a build reads in total, and how many files it reads, whatever is in the folder. */
|
|
14
|
+
export const MAX_KNOWLEDGE_FILES = 2000;
|
|
15
|
+
export const MAX_KNOWLEDGE_BYTES = 64 * 1024 * 1024;
|
|
16
|
+
const RULES_DIR = "rules";
|
|
17
|
+
const SOURCES_DIR = "sources";
|
|
18
|
+
/** True when the workspace has a `knowledge/` directory to read at all. */
|
|
19
|
+
export async function hasKnowledge(workspaceRoot) {
|
|
20
|
+
try {
|
|
21
|
+
await readdir(join(workspaceRoot, KNOWLEDGE_DIR_NAME));
|
|
22
|
+
return true;
|
|
23
|
+
}
|
|
24
|
+
catch {
|
|
25
|
+
return false;
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Reads `knowledge/rules/*.yaml` and `knowledge/sources/*.md` and checks them against each other.
|
|
30
|
+
* The rules that come back are the ones a build would index; when `problems` is not empty, nothing
|
|
31
|
+
* should be written.
|
|
32
|
+
*/
|
|
33
|
+
export async function checkKnowledge(workspaceRoot, options = {}) {
|
|
34
|
+
const dir = join(workspaceRoot, KNOWLEDGE_DIR_NAME);
|
|
35
|
+
const problems = [];
|
|
36
|
+
const budget = { files: MAX_KNOWLEDGE_FILES, bytes: MAX_KNOWLEDGE_BYTES };
|
|
37
|
+
const rules = [...(await readRules(dir, problems, budget)), ...(options.adding ?? [])];
|
|
38
|
+
const sources = await readSources(dir, problems, budget);
|
|
39
|
+
await checkPaths(workspaceRoot, sources, problems);
|
|
40
|
+
crossCheck(rules, sources, problems);
|
|
41
|
+
return { rules, sources, problems: problems.sort() };
|
|
42
|
+
}
|
|
43
|
+
/** Files of a directory in code point order; an unreadable directory is simply empty. */
|
|
44
|
+
async function filesOf(dir, extension) {
|
|
45
|
+
try {
|
|
46
|
+
return (await readdir(dir))
|
|
47
|
+
.filter((name) => name.endsWith(extension) && !name.startsWith("."))
|
|
48
|
+
.sort();
|
|
49
|
+
}
|
|
50
|
+
catch {
|
|
51
|
+
return [];
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
/** The text of a file, or a reason it was not read. Spends the budget as it goes. */
|
|
55
|
+
async function within(dir, relative, file, budget, problems) {
|
|
56
|
+
if (budget.files <= 0 || budget.bytes <= 0) {
|
|
57
|
+
problems.push(`${file}: not read; a build reads at most ${MAX_KNOWLEDGE_FILES} files`);
|
|
58
|
+
return undefined;
|
|
59
|
+
}
|
|
60
|
+
budget.files -= 1;
|
|
61
|
+
const text = await readWorkspaceTextIfAny(dir, relative);
|
|
62
|
+
if (text === undefined) {
|
|
63
|
+
problems.push(`${file}: cannot read the file`);
|
|
64
|
+
return undefined;
|
|
65
|
+
}
|
|
66
|
+
budget.bytes -= Buffer.byteLength(text, "utf8");
|
|
67
|
+
return text;
|
|
68
|
+
}
|
|
69
|
+
async function readRules(dir, problems, budget) {
|
|
70
|
+
const rules = [];
|
|
71
|
+
for (const name of await filesOf(join(dir, RULES_DIR), ".yaml")) {
|
|
72
|
+
const file = `${KNOWLEDGE_DIR_NAME}/${RULES_DIR}/${name}`;
|
|
73
|
+
const text = await within(dir, join(RULES_DIR, name), file, budget, problems);
|
|
74
|
+
if (text === undefined)
|
|
75
|
+
continue;
|
|
76
|
+
try {
|
|
77
|
+
const { data } = parseYamlFile(text, RulesFileSchema, file);
|
|
78
|
+
for (const rule of data.rules)
|
|
79
|
+
rules.push({ ...rule, file });
|
|
80
|
+
}
|
|
81
|
+
catch (err) {
|
|
82
|
+
problems.push(...messageOf(err, file));
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
return rules;
|
|
86
|
+
}
|
|
87
|
+
async function readSources(dir, problems, budget) {
|
|
88
|
+
const sources = [];
|
|
89
|
+
for (const name of await filesOf(join(dir, SOURCES_DIR), ".md")) {
|
|
90
|
+
const file = `${KNOWLEDGE_DIR_NAME}/${SOURCES_DIR}/${name}`;
|
|
91
|
+
const text = await within(dir, join(SOURCES_DIR, name), file, budget, problems);
|
|
92
|
+
if (text === undefined)
|
|
93
|
+
continue;
|
|
94
|
+
const split = frontMatter(text);
|
|
95
|
+
if (!split) {
|
|
96
|
+
problems.push(`${file}: the file must start with front matter between --- lines`);
|
|
97
|
+
continue;
|
|
98
|
+
}
|
|
99
|
+
try {
|
|
100
|
+
const { data } = parseYamlFile(split.head, SourceFrontMatterSchema, file);
|
|
101
|
+
sources.push({ ...data, body: split.body, file });
|
|
102
|
+
}
|
|
103
|
+
catch (err) {
|
|
104
|
+
problems.push(...messageOf(err, file));
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
return sources;
|
|
108
|
+
}
|
|
109
|
+
/** The YAML between the opening `---` and the next one, and everything after it. */
|
|
110
|
+
function frontMatter(text) {
|
|
111
|
+
const lines = text.split("\n");
|
|
112
|
+
if (lines[0]?.trim() !== "---")
|
|
113
|
+
return undefined;
|
|
114
|
+
const end = lines.indexOf("---", 1);
|
|
115
|
+
if (end < 0)
|
|
116
|
+
return undefined;
|
|
117
|
+
return {
|
|
118
|
+
head: lines.slice(1, end).join("\n"),
|
|
119
|
+
body: lines
|
|
120
|
+
.slice(end + 1)
|
|
121
|
+
.join("\n")
|
|
122
|
+
.trim(),
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
function messageOf(err, file) {
|
|
126
|
+
if (isOpenshainError(err))
|
|
127
|
+
return err.message.split("\n");
|
|
128
|
+
return [`${file}: ${err instanceof Error ? err.message : String(err)}`];
|
|
129
|
+
}
|
|
130
|
+
/**
|
|
131
|
+
* A source that names a file names it inside the company folder. The guard the tools run under
|
|
132
|
+
* answers this, so a source cannot pull `authority/`, a principal's record or anything outside
|
|
133
|
+
* the folder into the index.
|
|
134
|
+
*/
|
|
135
|
+
async function checkPaths(workspaceRoot, sources, problems) {
|
|
136
|
+
for (const source of sources) {
|
|
137
|
+
if (source.path === undefined)
|
|
138
|
+
continue;
|
|
139
|
+
try {
|
|
140
|
+
await resolveWorkspacePath(workspaceRoot, source.path);
|
|
141
|
+
}
|
|
142
|
+
catch (err) {
|
|
143
|
+
problems.push(`${source.file}: ${source.id} points at ${source.path}, which the runtime does not read (${err instanceof Error ? err.message : String(err)})`);
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
/** The checks that need more than one file: references, scope, effective days. */
|
|
148
|
+
function crossCheck(rules, sources, problems) {
|
|
149
|
+
const byId = new Map(sources.map((source) => [source.id, source]));
|
|
150
|
+
for (const source of duplicates(sources)) {
|
|
151
|
+
problems.push(`${source.file}: source ${source.id} is defined more than once`);
|
|
152
|
+
}
|
|
153
|
+
for (const rule of duplicates(rules)) {
|
|
154
|
+
problems.push(`${rule.file}: rule ${rule.id} is defined more than once`);
|
|
155
|
+
}
|
|
156
|
+
for (const item of [...sources, ...rules])
|
|
157
|
+
endsAfterItStarts(item, problems);
|
|
158
|
+
for (const rule of rules)
|
|
159
|
+
againstItsSource(rule, byId.get(rule.source.id), problems);
|
|
160
|
+
overlaps(rules, problems);
|
|
161
|
+
}
|
|
162
|
+
/** A day of the calendar comes before another; a rule or a source that ends first says nothing. */
|
|
163
|
+
function endsAfterItStarts(item, problems) {
|
|
164
|
+
if (item.effective_to !== null && item.effective_to < item.effective_from) {
|
|
165
|
+
problems.push(`${item.file}: ${item.id} ends before it starts`);
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
/**
|
|
169
|
+
* A rule stands on the source it cites, so it may not exist without it, outlive it, begin before
|
|
170
|
+
* it, or be read by people who may not read it.
|
|
171
|
+
*/
|
|
172
|
+
function againstItsSource(rule, source, problems) {
|
|
173
|
+
if (!source) {
|
|
174
|
+
problems.push(`${rule.file}: ${rule.id} cites ${rule.source.id}, which no source declares`);
|
|
175
|
+
return;
|
|
176
|
+
}
|
|
177
|
+
if (rule.effective_from < source.effective_from) {
|
|
178
|
+
problems.push(`${rule.file}: ${rule.id} starts before ${source.id}, the source it cites, is in effect`);
|
|
179
|
+
}
|
|
180
|
+
if (source.effective_to !== null && (rule.effective_to ?? FOREVER) > source.effective_to) {
|
|
181
|
+
problems.push(`${rule.file}: ${rule.id} outlives ${source.id}, the source it cites; close it on ${source.effective_to} or cite a newer source`);
|
|
182
|
+
}
|
|
183
|
+
if (!covers(source.scope, rule.scope)) {
|
|
184
|
+
problems.push(`${rule.file}: ${rule.id} may be read by more people than ${source.id}, the source it cites; its citation would name a source they cannot read`);
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
/** A day later than any a person would write, for a rule or a source with no end. */
|
|
188
|
+
const FOREVER = "9999-12-31";
|
|
189
|
+
/** The items whose id was already taken by an earlier item. */
|
|
190
|
+
function duplicates(items) {
|
|
191
|
+
const seen = new Set();
|
|
192
|
+
const again = [];
|
|
193
|
+
for (const item of items) {
|
|
194
|
+
if (seen.has(item.id))
|
|
195
|
+
again.push(item);
|
|
196
|
+
else
|
|
197
|
+
seen.add(item.id);
|
|
198
|
+
}
|
|
199
|
+
return again;
|
|
200
|
+
}
|
|
201
|
+
/** The people a scope names, and how it names them. Company-wide names everyone. */
|
|
202
|
+
function named(scope) {
|
|
203
|
+
if (scope === undefined || "visibility" in scope)
|
|
204
|
+
return { kind: "everyone", who: [] };
|
|
205
|
+
if ("principals" in scope)
|
|
206
|
+
return { kind: "principals", who: scope.principals };
|
|
207
|
+
return { kind: "roles", who: scope.roles };
|
|
208
|
+
}
|
|
209
|
+
/** Whether everyone who may read `inner` may also read `outer`. */
|
|
210
|
+
function covers(outer, inner) {
|
|
211
|
+
const wider = named(outer);
|
|
212
|
+
const narrower = named(inner);
|
|
213
|
+
if (wider.kind === "everyone")
|
|
214
|
+
return true;
|
|
215
|
+
if (narrower.kind === "everyone")
|
|
216
|
+
return false;
|
|
217
|
+
// Two lists of different kinds cannot be compared without knowing who holds which role.
|
|
218
|
+
if (wider.kind !== narrower.kind)
|
|
219
|
+
return false;
|
|
220
|
+
const allowed = new Set(wider.who);
|
|
221
|
+
return narrower.who.every((name) => allowed.has(name));
|
|
222
|
+
}
|
|
223
|
+
/**
|
|
224
|
+
* Two rules drawn from the very same passage, both in effect, are two answers to one question
|
|
225
|
+
* unless one says it replaces the other or they are for different people or professions.
|
|
226
|
+
*
|
|
227
|
+
* The passage, not the document: one policy document backs many rules — receipts over one
|
|
228
|
+
* amount, an approval over another — and that is how a company writes. Only rules that name the
|
|
229
|
+
* same section of the same source are compared, so ordinary writing is never refused.
|
|
230
|
+
*/
|
|
231
|
+
function overlaps(rules, problems) {
|
|
232
|
+
const replaced = new Set(rules.map((rule) => rule.supersedes).filter(Boolean));
|
|
233
|
+
for (const [i, rule] of rules.entries()) {
|
|
234
|
+
for (const other of rules.slice(i + 1)) {
|
|
235
|
+
if (rule.source.id !== other.source.id)
|
|
236
|
+
continue;
|
|
237
|
+
if (rule.source.section === undefined || rule.source.section !== other.source.section) {
|
|
238
|
+
continue;
|
|
239
|
+
}
|
|
240
|
+
if (replaced.has(rule.id) || replaced.has(other.id))
|
|
241
|
+
continue;
|
|
242
|
+
if (!inEffectTogether(rule, other))
|
|
243
|
+
continue;
|
|
244
|
+
if (disjoint(rule, other))
|
|
245
|
+
continue;
|
|
246
|
+
problems.push(`${other.file}: ${other.id} and ${rule.id} both cite ${rule.source.id} の ${rule.source.section} and are in effect at the same time; close one with effective_to, or say supersedes`);
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
function inEffectTogether(a, b) {
|
|
251
|
+
const end = (rule) => rule.effective_to ?? FOREVER;
|
|
252
|
+
return a.effective_from <= end(b) && b.effective_from <= end(a);
|
|
253
|
+
}
|
|
254
|
+
/** Rules that no one person and no one profession sees together do not contradict each other. */
|
|
255
|
+
function disjoint(a, b) {
|
|
256
|
+
const professions = (rule) => rule.applies_to?.profession;
|
|
257
|
+
const pa = professions(a);
|
|
258
|
+
const pb = professions(b);
|
|
259
|
+
if (pa && pb && !pa.some((name) => pb.includes(name)))
|
|
260
|
+
return true;
|
|
261
|
+
return !covers(a.scope, b.scope) && !covers(b.scope, a.scope);
|
|
262
|
+
}
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
/**
|
|
3
|
+
* Who may read it. A company with one principal may leave it out; with two or more it is written,
|
|
4
|
+
* because from then on leaving it out would mean deciding by accident.
|
|
5
|
+
*/
|
|
6
|
+
export declare const ScopeSchema: z.ZodUnion<readonly [z.ZodObject<{
|
|
7
|
+
visibility: z.ZodLiteral<"company">;
|
|
8
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
9
|
+
principals: z.ZodArray<z.ZodString>;
|
|
10
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
11
|
+
roles: z.ZodArray<z.ZodString>;
|
|
12
|
+
}, z.core.$strict>]>;
|
|
13
|
+
export type Scope = z.infer<typeof ScopeSchema>;
|
|
14
|
+
export declare const RuleSchema: z.ZodObject<{
|
|
15
|
+
id: z.ZodString;
|
|
16
|
+
statement: z.ZodString;
|
|
17
|
+
aliases: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
18
|
+
applies_to: z.ZodOptional<z.ZodObject<{
|
|
19
|
+
profession: z.ZodArray<z.ZodString>;
|
|
20
|
+
}, z.core.$strict>>;
|
|
21
|
+
scope: z.ZodOptional<z.ZodUnion<readonly [z.ZodObject<{
|
|
22
|
+
visibility: z.ZodLiteral<"company">;
|
|
23
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
24
|
+
principals: z.ZodArray<z.ZodString>;
|
|
25
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
26
|
+
roles: z.ZodArray<z.ZodString>;
|
|
27
|
+
}, z.core.$strict>]>>;
|
|
28
|
+
effective_from: z.ZodString;
|
|
29
|
+
effective_to: z.ZodNullable<z.ZodString>;
|
|
30
|
+
supersedes: z.ZodOptional<z.ZodString>;
|
|
31
|
+
expertise: z.ZodString;
|
|
32
|
+
source: z.ZodObject<{
|
|
33
|
+
id: z.ZodString;
|
|
34
|
+
section: z.ZodOptional<z.ZodString>;
|
|
35
|
+
}, z.core.$strict>;
|
|
36
|
+
}, z.core.$strict>;
|
|
37
|
+
export type Rule = z.infer<typeof RuleSchema>;
|
|
38
|
+
export declare const RulesFileSchema: z.ZodObject<{
|
|
39
|
+
version: z.ZodLiteral<1>;
|
|
40
|
+
rules: z.ZodArray<z.ZodObject<{
|
|
41
|
+
id: z.ZodString;
|
|
42
|
+
statement: z.ZodString;
|
|
43
|
+
aliases: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
44
|
+
applies_to: z.ZodOptional<z.ZodObject<{
|
|
45
|
+
profession: z.ZodArray<z.ZodString>;
|
|
46
|
+
}, z.core.$strict>>;
|
|
47
|
+
scope: z.ZodOptional<z.ZodUnion<readonly [z.ZodObject<{
|
|
48
|
+
visibility: z.ZodLiteral<"company">;
|
|
49
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
50
|
+
principals: z.ZodArray<z.ZodString>;
|
|
51
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
52
|
+
roles: z.ZodArray<z.ZodString>;
|
|
53
|
+
}, z.core.$strict>]>>;
|
|
54
|
+
effective_from: z.ZodString;
|
|
55
|
+
effective_to: z.ZodNullable<z.ZodString>;
|
|
56
|
+
supersedes: z.ZodOptional<z.ZodString>;
|
|
57
|
+
expertise: z.ZodString;
|
|
58
|
+
source: z.ZodObject<{
|
|
59
|
+
id: z.ZodString;
|
|
60
|
+
section: z.ZodOptional<z.ZodString>;
|
|
61
|
+
}, z.core.$strict>;
|
|
62
|
+
}, z.core.$strict>>;
|
|
63
|
+
}, z.core.$strict>;
|
|
64
|
+
/** The front matter of a source document. The body below it is the citation itself. */
|
|
65
|
+
export declare const SourceFrontMatterSchema: z.ZodObject<{
|
|
66
|
+
id: z.ZodString;
|
|
67
|
+
title: z.ZodString;
|
|
68
|
+
publisher: z.ZodString;
|
|
69
|
+
url: z.ZodOptional<z.ZodURL>;
|
|
70
|
+
path: z.ZodOptional<z.ZodString>;
|
|
71
|
+
retrieved_at: z.ZodString;
|
|
72
|
+
version: z.ZodOptional<z.ZodString>;
|
|
73
|
+
effective_from: z.ZodString;
|
|
74
|
+
effective_to: z.ZodNullable<z.ZodString>;
|
|
75
|
+
scope: z.ZodOptional<z.ZodUnion<readonly [z.ZodObject<{
|
|
76
|
+
visibility: z.ZodLiteral<"company">;
|
|
77
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
78
|
+
principals: z.ZodArray<z.ZodString>;
|
|
79
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
80
|
+
roles: z.ZodArray<z.ZodString>;
|
|
81
|
+
}, z.core.$strict>]>>;
|
|
82
|
+
expertise: z.ZodString;
|
|
83
|
+
}, z.core.$strict>;
|
|
84
|
+
export type SourceFrontMatter = z.infer<typeof SourceFrontMatterSchema>;
|
|
85
|
+
/** A source as it was read: its front matter, the body, and the file it came from. */
|
|
86
|
+
export interface Source extends SourceFrontMatter {
|
|
87
|
+
body: string;
|
|
88
|
+
file: string;
|
|
89
|
+
}
|
|
90
|
+
/** A rule as it was read, with the file it came from for the messages. */
|
|
91
|
+
export interface LoadedRule extends Rule {
|
|
92
|
+
file: string;
|
|
93
|
+
}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
/**
|
|
3
|
+
* What a person writes under `knowledge/`: the company's own rules, and the sources behind them.
|
|
4
|
+
* The shapes only; what makes a set of them consistent is in `check.ts`.
|
|
5
|
+
*/
|
|
6
|
+
/** An id a person writes: lowercase, and readable as a path of meaning (`expenses.receipt-required`). */
|
|
7
|
+
const knowledgeId = z
|
|
8
|
+
.string()
|
|
9
|
+
.min(1)
|
|
10
|
+
.max(200)
|
|
11
|
+
.regex(/^[a-z0-9][a-z0-9._-]*$/, "use lowercase letters, digits, . - or _, starting with a letter or a digit");
|
|
12
|
+
/**
|
|
13
|
+
* A date as the company writes it. The runtime compares these as strings, so the form is fixed,
|
|
14
|
+
* and it must be a day that exists: the index closes a superseded rule the day before the next
|
|
15
|
+
* one starts, and `2026-13-45` would end that arithmetic in an error rather than a refusal.
|
|
16
|
+
*/
|
|
17
|
+
const day = z
|
|
18
|
+
.string()
|
|
19
|
+
.regex(/^\d{4}-\d{2}-\d{2}$/, "write a date as YYYY-MM-DD")
|
|
20
|
+
.refine(isDay, "that day does not exist");
|
|
21
|
+
function isDay(text) {
|
|
22
|
+
const at = new Date(`${text}T00:00:00Z`);
|
|
23
|
+
return !Number.isNaN(at.getTime()) && at.toISOString().startsWith(text);
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Who may read it. A company with one principal may leave it out; with two or more it is written,
|
|
27
|
+
* because from then on leaving it out would mean deciding by accident.
|
|
28
|
+
*/
|
|
29
|
+
export const ScopeSchema = z.union([
|
|
30
|
+
z.strictObject({ visibility: z.literal("company") }),
|
|
31
|
+
z.strictObject({ principals: z.array(knowledgeId).min(1) }),
|
|
32
|
+
z.strictObject({ roles: z.array(knowledgeId).min(1) }),
|
|
33
|
+
]);
|
|
34
|
+
/**
|
|
35
|
+
* The professional domain a piece of knowledge belongs to. `none` is the only value core knows;
|
|
36
|
+
* the rest is a string a profession pack or the company defines. Core compares it and holds no
|
|
37
|
+
* list of qualifications (docs/design/core.md, spec/professional-boundary.md).
|
|
38
|
+
*/
|
|
39
|
+
const expertise = z
|
|
40
|
+
.string()
|
|
41
|
+
.min(1)
|
|
42
|
+
.max(40)
|
|
43
|
+
.regex(/^[a-z][a-z0-9-]*$/, "use lowercase letters, digits and -");
|
|
44
|
+
export const RuleSchema = z.strictObject({
|
|
45
|
+
id: knowledgeId,
|
|
46
|
+
statement: z.string().min(10).max(240),
|
|
47
|
+
aliases: z.array(z.string().min(1).max(80)).max(20).optional(),
|
|
48
|
+
applies_to: z.strictObject({ profession: z.array(knowledgeId).min(1) }).optional(),
|
|
49
|
+
scope: ScopeSchema.optional(),
|
|
50
|
+
effective_from: day,
|
|
51
|
+
// Written even when there is no end, so that an open-ended rule is a decision, not an omission.
|
|
52
|
+
effective_to: day.nullable(),
|
|
53
|
+
supersedes: knowledgeId.optional(),
|
|
54
|
+
expertise,
|
|
55
|
+
source: z.strictObject({ id: knowledgeId, section: z.string().min(1).max(200).optional() }),
|
|
56
|
+
});
|
|
57
|
+
export const RulesFileSchema = z.strictObject({
|
|
58
|
+
version: z.literal(1),
|
|
59
|
+
rules: z.array(RuleSchema).min(1),
|
|
60
|
+
});
|
|
61
|
+
/** The front matter of a source document. The body below it is the citation itself. */
|
|
62
|
+
export const SourceFrontMatterSchema = z
|
|
63
|
+
.strictObject({
|
|
64
|
+
id: knowledgeId,
|
|
65
|
+
title: z.string().min(1).max(200),
|
|
66
|
+
publisher: z.string().min(1).max(200),
|
|
67
|
+
url: z.url().optional(),
|
|
68
|
+
path: z.string().min(1).max(1000).optional(),
|
|
69
|
+
retrieved_at: day,
|
|
70
|
+
version: z.string().min(1).max(80).optional(),
|
|
71
|
+
effective_from: day,
|
|
72
|
+
effective_to: day.nullable(),
|
|
73
|
+
scope: ScopeSchema.optional(),
|
|
74
|
+
expertise,
|
|
75
|
+
})
|
|
76
|
+
.refine((source) => source.url !== undefined || source.path !== undefined, "say where it came from: url or path");
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import { type IndexUnit, type KnowledgeIndex } from "./build.ts";
|
|
2
|
+
/**
|
|
3
|
+
* Finding a unit by the characters it shares with a question. There is no embedding and no
|
|
4
|
+
* outside search engine: the index is groups of two and three characters, and the score is how
|
|
5
|
+
* much of the question a unit accounts for.
|
|
6
|
+
*/
|
|
7
|
+
export interface Hit {
|
|
8
|
+
unit: IndexUnit;
|
|
9
|
+
/** Between 0 and 1. The share of the question's grams the unit holds, corrected for length. */
|
|
10
|
+
score: number;
|
|
11
|
+
}
|
|
12
|
+
/** The shortest question the index can answer. One character matches almost everything. */
|
|
13
|
+
export declare const MIN_QUERY_LENGTH = 2;
|
|
14
|
+
export interface SearchOptions {
|
|
15
|
+
/** How many hits to return. */
|
|
16
|
+
limit?: number;
|
|
17
|
+
/** Only these units are searched. Everything else is invisible, count included. */
|
|
18
|
+
allowed?: (unit: IndexUnit) => boolean;
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* The best units for a question, most fitting first. Two hits with the same score keep the order
|
|
22
|
+
* of their keys, so the same index and question always answer the same way.
|
|
23
|
+
*/
|
|
24
|
+
export declare function search(index: KnowledgeIndex, query: string, options?: SearchOptions): Hit[];
|
|
25
|
+
/** Whether a unit is in effect on a day. */
|
|
26
|
+
export declare function inEffect(unit: IndexUnit, day: string): boolean;
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import { grams, normalize } from "./build.js";
|
|
2
|
+
/** The shortest question the index can answer. One character matches almost everything. */
|
|
3
|
+
export const MIN_QUERY_LENGTH = 2;
|
|
4
|
+
/**
|
|
5
|
+
* The best units for a question, most fitting first. Two hits with the same score keep the order
|
|
6
|
+
* of their keys, so the same index and question always answer the same way.
|
|
7
|
+
*/
|
|
8
|
+
export function search(index, query, options = {}) {
|
|
9
|
+
const text = normalize(query);
|
|
10
|
+
if (text.length < MIN_QUERY_LENGTH)
|
|
11
|
+
return [];
|
|
12
|
+
// A question of two characters has no group of three, and two-character words are ordinary in
|
|
13
|
+
// Japanese business writing, so the shorter grams answer it.
|
|
14
|
+
const n = text.length < 3 ? 2 : 3;
|
|
15
|
+
const wanted = grams(query, n);
|
|
16
|
+
if (wanted.size === 0)
|
|
17
|
+
return [];
|
|
18
|
+
const postings = n === 2 ? index.postings.pairs : index.postings.triples;
|
|
19
|
+
const allowed = options.allowed ?? (() => true);
|
|
20
|
+
// Narrow before scoring: what a person may not read costs nothing to rank, and the time a
|
|
21
|
+
// search takes then says nothing about how much of it there is.
|
|
22
|
+
const visible = new Set();
|
|
23
|
+
for (const [at, unit] of index.units.entries())
|
|
24
|
+
if (allowed(unit))
|
|
25
|
+
visible.add(at);
|
|
26
|
+
const matched = new Map();
|
|
27
|
+
for (const gram of wanted) {
|
|
28
|
+
for (const at of postings[gram] ?? []) {
|
|
29
|
+
if (visible.has(at))
|
|
30
|
+
matched.set(at, (matched.get(at) ?? 0) + 1);
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
const hits = [];
|
|
34
|
+
for (const [at, count] of matched) {
|
|
35
|
+
const unit = index.units[at];
|
|
36
|
+
if (!unit)
|
|
37
|
+
continue;
|
|
38
|
+
hits.push({ unit, score: (count / wanted.size) * lengthCorrection(index.sizes[at] ?? 0) });
|
|
39
|
+
}
|
|
40
|
+
hits.sort((a, b) => b.score - a.score || (a.unit.key < b.unit.key ? -1 : 1));
|
|
41
|
+
return hits.slice(0, options.limit ?? 5);
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* A long document holds more groups of characters and so matches more questions by size alone.
|
|
45
|
+
* The correction keeps a short rule from losing to a page of prose that happens to contain the
|
|
46
|
+
* same words; it lowers a long unit's score without ever ruling it out.
|
|
47
|
+
*/
|
|
48
|
+
function lengthCorrection(size) {
|
|
49
|
+
return 1 / (1 + Math.log10(1 + size / 40));
|
|
50
|
+
}
|
|
51
|
+
/** Whether a unit is in effect on a day. */
|
|
52
|
+
export function inEffect(unit, day) {
|
|
53
|
+
return unit.from <= day && (unit.to === null || day <= unit.to);
|
|
54
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Writes a file under `knowledge/` through a temporary file renamed into place, so a reader
|
|
3
|
+
* never sees half of one. The temporary name is easy to guess, so it is opened without following
|
|
4
|
+
* a link.
|
|
5
|
+
*/
|
|
6
|
+
export declare function writeKnowledgeFile(workspaceRoot: string, parts: string[], text: string): Promise<void>;
|
|
7
|
+
/** Reads a file under `knowledge/`, or nothing when it is missing, a link, or too large. */
|
|
8
|
+
export declare function readKnowledgeFile(workspaceRoot: string, parts: string[]): Promise<string | undefined>;
|
|
9
|
+
/** Where a file under `knowledge/` is, for a message a person reads. */
|
|
10
|
+
export declare function knowledgePath(parts: string[]): string;
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import { constants } from "node:fs";
|
|
2
|
+
import { mkdir, open, realpath, rename } from "node:fs/promises";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
import { OpenshainError } from "../errors.js";
|
|
5
|
+
import { KNOWLEDGE_DIR_NAME } from "./check.js";
|
|
6
|
+
/**
|
|
7
|
+
* Touching the files under `knowledge/`. The tools cannot reach them — it is a reserved path, so
|
|
8
|
+
* that what the index filters cannot be read around — which leaves the runtime to read and write
|
|
9
|
+
* them itself, without the guard the tools go through. These functions are that guard: the
|
|
10
|
+
* directory must be the one inside the company folder, and a link must never carry a write.
|
|
11
|
+
*/
|
|
12
|
+
/** How large a file under `knowledge/` may be for the runtime to read it whole. */
|
|
13
|
+
const MAX_BYTES = 64 * 1024 * 1024;
|
|
14
|
+
/**
|
|
15
|
+
* The directory the path names inside `knowledge/`, once it is known to be that directory. A
|
|
16
|
+
* link left in the folder would otherwise send a write anywhere the person can write.
|
|
17
|
+
*/
|
|
18
|
+
async function directory(workspaceRoot, parts) {
|
|
19
|
+
const root = await realpath(workspaceRoot);
|
|
20
|
+
const dir = join(root, KNOWLEDGE_DIR_NAME, ...parts);
|
|
21
|
+
await mkdir(dir, { recursive: true });
|
|
22
|
+
if ((await realpath(dir)) !== dir) {
|
|
23
|
+
throw new OpenshainError("invalid_path", `${[KNOWLEDGE_DIR_NAME, ...parts].join("/")} leads out of the company folder; nothing was written`);
|
|
24
|
+
}
|
|
25
|
+
return dir;
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Writes a file under `knowledge/` through a temporary file renamed into place, so a reader
|
|
29
|
+
* never sees half of one. The temporary name is easy to guess, so it is opened without following
|
|
30
|
+
* a link.
|
|
31
|
+
*/
|
|
32
|
+
export async function writeKnowledgeFile(workspaceRoot, parts, text) {
|
|
33
|
+
const name = parts.at(-1);
|
|
34
|
+
const dir = await directory(workspaceRoot, parts.slice(0, -1));
|
|
35
|
+
const path = join(dir, name);
|
|
36
|
+
const temporary = `${path}.writing`;
|
|
37
|
+
const flags = constants.O_WRONLY | constants.O_CREAT | constants.O_TRUNC | (constants.O_NOFOLLOW ?? 0);
|
|
38
|
+
const handle = await open(temporary, flags, 0o644);
|
|
39
|
+
try {
|
|
40
|
+
await handle.writeFile(text, "utf8");
|
|
41
|
+
}
|
|
42
|
+
finally {
|
|
43
|
+
await handle.close();
|
|
44
|
+
}
|
|
45
|
+
await rename(temporary, path);
|
|
46
|
+
}
|
|
47
|
+
/** Reads a file under `knowledge/`, or nothing when it is missing, a link, or too large. */
|
|
48
|
+
export async function readKnowledgeFile(workspaceRoot, parts) {
|
|
49
|
+
const path = join(workspaceRoot, KNOWLEDGE_DIR_NAME, ...parts);
|
|
50
|
+
try {
|
|
51
|
+
const handle = await open(path, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0));
|
|
52
|
+
try {
|
|
53
|
+
const { size } = await handle.stat();
|
|
54
|
+
if (size > MAX_BYTES)
|
|
55
|
+
return undefined;
|
|
56
|
+
return await handle.readFile("utf8");
|
|
57
|
+
}
|
|
58
|
+
finally {
|
|
59
|
+
await handle.close();
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
catch {
|
|
63
|
+
return undefined;
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
/** Where a file under `knowledge/` is, for a message a person reads. */
|
|
67
|
+
export function knowledgePath(parts) {
|
|
68
|
+
return [KNOWLEDGE_DIR_NAME, ...parts].join("/");
|
|
69
|
+
}
|
package/dist/runtime.d.ts
CHANGED
|
@@ -9,7 +9,9 @@ export interface RuntimeProviders {
|
|
|
9
9
|
/** Model providers by the id used in openshain.yaml. */
|
|
10
10
|
models: Record<string, (model: ModelConfig) => ModelProvider>;
|
|
11
11
|
/** Tool providers by the id used in openshain.yaml. Modules are loaded from the config directly. */
|
|
12
|
-
|
|
12
|
+
/** By the provider id used in openshain.yaml. The workspace is given, since what a provider
|
|
13
|
+
* offers can depend on what is in it. */
|
|
14
|
+
tools: Record<string, (workspaceRoot: string) => ToolProvider>;
|
|
13
15
|
}
|
|
14
16
|
export interface CreateRuntimeOptions {
|
|
15
17
|
workspaceRoot: string;
|
package/dist/runtime.js
CHANGED
|
@@ -62,7 +62,7 @@ export async function createToolRegistry(workspaceRoot, config, tools) {
|
|
|
62
62
|
if (!factory) {
|
|
63
63
|
throw new OpenshainError("config", `unknown tool provider "${entry.provider}"; known providers: ${Object.keys(tools).join(", ")}`);
|
|
64
64
|
}
|
|
65
|
-
await registry.register(factory(), registerOptions);
|
|
65
|
+
await registry.register(factory(workspaceRoot), registerOptions);
|
|
66
66
|
}
|
|
67
67
|
else {
|
|
68
68
|
await registry.register(await loadToolModule(workspaceRoot, entry.module), registerOptions);
|
|
@@ -177,6 +177,8 @@ async function callTool(input) {
|
|
|
177
177
|
result = await tool.provider.call(call, {
|
|
178
178
|
workId: work.id,
|
|
179
179
|
principalId: config.principal.id,
|
|
180
|
+
profession: config.profession.id,
|
|
181
|
+
businessDate: businessDate(config.company.timezone),
|
|
180
182
|
workspaceRoot,
|
|
181
183
|
});
|
|
182
184
|
}
|
package/dist/schemas.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { JsonSchema } from "./tool/types.ts";
|
|
2
|
-
export type SchemaName = "config.v1" | "events.v1" | "work.v1" | "authority-policy.v1" | "authority-delegations.v1";
|
|
2
|
+
export type SchemaName = "config.v1" | "events.v1" | "work.v1" | "authority-policy.v1" | "authority-delegations.v1" | "knowledge-rules.v1" | "knowledge-source.v1";
|
|
3
3
|
/**
|
|
4
4
|
* The JSON Schemas (draft 2020-12) of the files openshain reads and writes, derived from the zod
|
|
5
5
|
* schemas that validate them. `spec/schemas/` holds this output; `bun run schemas` regenerates it.
|
package/dist/schemas.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
2
|
import { DelegationsFileSchema, PolicyFileSchema } from "./authority/policy.js";
|
|
3
3
|
import { ConfigFileSchema } from "./config/schema.js";
|
|
4
|
+
import { RulesFileSchema, SourceFrontMatterSchema } from "./knowledge/schema.js";
|
|
4
5
|
import { EventFileSchema, payloadFileSchemas } from "./work/events.js";
|
|
5
6
|
import { WorkFileSchema } from "./work/work.js";
|
|
6
7
|
/**
|
|
@@ -16,6 +17,8 @@ export function jsonSchemas() {
|
|
|
16
17
|
"work.v1": describe(WorkFileSchema, "work.json", "The state of a work as projected from its event log. Never the source of truth."),
|
|
17
18
|
"authority-policy.v1": describe(PolicyFileSchema, "authority/policy.yaml", "The rules that judge tool calls: the first matching rule decides, else the default."),
|
|
18
19
|
"authority-delegations.v1": describe(DelegationsFileSchema, "authority/delegations.yaml", "Who the agent may act for, as which profession, and when."),
|
|
20
|
+
"knowledge-rules.v1": describe(RulesFileSchema, "knowledge/rules/*.yaml", "The company's own rules, each with the source behind it and the days it is in effect."),
|
|
21
|
+
"knowledge-source.v1": describe(SourceFrontMatterSchema, "knowledge/sources/*.md (front matter)", "Where a cited document came from, when it applies, and who may read it."),
|
|
19
22
|
};
|
|
20
23
|
}
|
|
21
24
|
/**
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Reading and writing a file of the company folder. Everything that reaches a file on behalf of
|
|
3
|
+
* a tool goes through here, so the path guard and the size limit are applied in one place rather
|
|
4
|
+
* than remembered at each call. A caller that opens a file itself is a caller that can forget.
|
|
5
|
+
*/
|
|
6
|
+
/** The most a tool reads from one file. Larger files are refused, not truncated. */
|
|
7
|
+
export declare const MAX_READ_BYTES: number;
|
|
8
|
+
/** The most a tool writes to one file. */
|
|
9
|
+
export declare const MAX_WRITE_BYTES: number;
|
|
10
|
+
/**
|
|
11
|
+
* Reads a text file through one descriptor: the size check and the read see the same file, so a
|
|
12
|
+
* swap between the two cannot slip a larger file past the limit.
|
|
13
|
+
*/
|
|
14
|
+
export declare function readWorkspaceText(root: string, path: string): Promise<string>;
|
|
15
|
+
/** The same read, but a file that is missing, too large or unreadable comes back as undefined. */
|
|
16
|
+
export declare function readWorkspaceTextIfAny(root: string, path: string): Promise<string | undefined>;
|
|
17
|
+
/** Writes through a descriptor opened with O_NOFOLLOW, so the final component may not be a symlink. */
|
|
18
|
+
export declare function writeWorkspaceText(root: string, path: string, content: string): Promise<{
|
|
19
|
+
path: string;
|
|
20
|
+
sha256: string;
|
|
21
|
+
}>;
|