factbag 0.0.0-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +5 -0
- package/LICENSE +202 -0
- package/README.md +44 -0
- package/dist/core/chunk-4IP7CXNV.js +86 -0
- package/dist/core/chunk-UVOCOH2I.js +41 -0
- package/dist/core/chunk-X3LSDJL4.js +59 -0
- package/dist/core/chunk-YF4PP42K.js +8 -0
- package/dist/core/dsl/enumeration.d.ts +3 -0
- package/dist/core/dsl/formula.d.ts +62 -0
- package/dist/core/dsl/index.d.ts +21 -0
- package/dist/core/dsl/index.js +132 -0
- package/dist/core/dsl/operators/aggregate.d.ts +30 -0
- package/dist/core/dsl/operators/arithmetic.d.ts +27 -0
- package/dist/core/dsl/operators/comparison.d.ts +13 -0
- package/dist/core/dsl/predicate.d.ts +33 -0
- package/dist/core/dsl/types.d.ts +59 -0
- package/dist/core/dsl/variable.d.ts +3 -0
- package/dist/core/exceptions/programmer-errors.d.ts +36 -0
- package/dist/core/explanation/explain.d.ts +20 -0
- package/dist/core/explanation/index.d.ts +8 -0
- package/dist/core/explanation/index.js +187 -0
- package/dist/core/explanation/types.d.ts +172 -0
- package/dist/core/knowledge-base/assertions/exclusive-definitions.d.ts +3 -0
- package/dist/core/knowledge-base/index.d.ts +13 -0
- package/dist/core/knowledge-base/index.js +196 -0
- package/dist/core/knowledge-base/knowledge-base.d.ts +81 -0
- package/dist/core/query/index.d.ts +10 -0
- package/dist/core/query/index.js +1947 -0
- package/dist/core/query/query.d.ts +47 -0
- package/dist/core/solver/candidate-search.d.ts +12 -0
- package/dist/core/solver/goal-value.d.ts +6 -0
- package/dist/core/solver/matching.d.ts +10 -0
- package/dist/core/solver/query-validation/constructive-recursion.d.ts +3 -0
- package/dist/core/solver/query-validation/constructive-variable-use.d.ts +3 -0
- package/dist/core/solver/query-validation/context.d.ts +10 -0
- package/dist/core/solver/query-validation/index.d.ts +1 -0
- package/dist/core/solver/query-validation/utils.d.ts +27 -0
- package/dist/core/solver/query-validation/validate-query.d.ts +6 -0
- package/dist/core/solver/solve-all.d.ts +6 -0
- package/dist/core/solver/solve.d.ts +6 -0
- package/dist/core/types/annotations.d.ts +6 -0
- package/dist/core/types/atom.d.ts +55 -0
- package/dist/core/types/common.d.ts +7 -0
- package/dist/core/types/evidence-metadata.d.ts +18 -0
- package/dist/core/types/goal.d.ts +103 -0
- package/dist/core/types/judgment.d.ts +137 -0
- package/dist/core/types/literal.d.ts +25 -0
- package/dist/core/types/predicate.d.ts +35 -0
- package/dist/core/types/rule.d.ts +16 -0
- package/dist/core/types/schema.d.ts +33 -0
- package/dist/core/types/variable.d.ts +13 -0
- package/dist/core/utils/assert.d.ts +4 -0
- package/dist/core/utils/normalize-to-list.d.ts +2 -0
- package/dist/corpus/chunkers.d.ts +7 -0
- package/dist/corpus/corpus.d.ts +41 -0
- package/dist/corpus/index.d.ts +8 -0
- package/dist/corpus/index.js +312 -0
- package/dist/corpus/records.d.ts +8 -0
- package/dist/corpus/text.d.ts +6 -0
- package/dist/corpus/types.d.ts +62 -0
- package/dist/guards/guarded-kb.d.ts +59 -0
- package/dist/guards/index.d.ts +7 -0
- package/dist/guards/index.js +121 -0
- package/dist/guards/types.d.ts +15 -0
- package/dist/jev/chunk-2UXHML67.js +206 -0
- package/dist/jev/chunk-M7RHUI2O.js +15 -0
- package/dist/jev/classifier.d.ts +28 -0
- package/dist/jev/classifier.js +171 -0
- package/dist/jev/client.d.ts +62 -0
- package/dist/jev/errors.d.ts +22 -0
- package/dist/jev/errors.js +6 -0
- package/dist/jev/jobs.d.ts +61 -0
- package/dist/jev/resolve.d.ts +82 -0
- package/dist/jev/resolve.js +99 -0
- package/dist/jev/tools.d.ts +9 -0
- package/dist/jev/types.d.ts +63 -0
- package/package.json +92 -0
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import type { Chunk, Chunker, CorpusDocument, DocumentInput, DocumentReference, Metadata, ResolvedChunk, TextRange } from "./types.js";
|
|
2
|
+
/**
|
|
3
|
+
* In-memory corpus of immutable documents and addressable source ranges.
|
|
4
|
+
* {@link Corpus.filter} returns views over the same storage; the filter is evaluated on read.
|
|
5
|
+
*/
|
|
6
|
+
export declare class Corpus<M extends Metadata = Metadata> {
|
|
7
|
+
#private;
|
|
8
|
+
/**
|
|
9
|
+
* Add a document to the shared storage; re-adding identical content is idempotent,
|
|
10
|
+
* different content throws. On a view, the document is visible only if it matches.
|
|
11
|
+
*/
|
|
12
|
+
add(input: DocumentInput<M>): CorpusDocument<M>;
|
|
13
|
+
/** Validate every input, including conflicts, before storing any; then store them in input order. */
|
|
14
|
+
add(inputs: readonly DocumentInput<M>[]): readonly CorpusDocument<M>[];
|
|
15
|
+
/** Read a document. Throws when absent or excluded by this view's filter. */
|
|
16
|
+
document(documentId: string): CorpusDocument<M>;
|
|
17
|
+
/** List every document matching this view's filter, in insertion order. */
|
|
18
|
+
documents(): readonly CorpusDocument<M>[];
|
|
19
|
+
/**
|
|
20
|
+
* Return a view over the same storage that also requires this predicate. Documents
|
|
21
|
+
* added later through any view appear when they match.
|
|
22
|
+
*/
|
|
23
|
+
filter(predicate: (document: CorpusDocument<M>) => boolean): Corpus<M>;
|
|
24
|
+
/**
|
|
25
|
+
* Split a document lazily, validating each range as it is consumed. If a later
|
|
26
|
+
* range is invalid, earlier yielded chunks remain valid and resolvable.
|
|
27
|
+
*/
|
|
28
|
+
chunk(target: DocumentReference, chunker: Chunker): Iterable<Chunk>;
|
|
29
|
+
private splitDocument;
|
|
30
|
+
/** Select an exact range of a document. */
|
|
31
|
+
slice(target: DocumentReference, selection: TextRange): Chunk;
|
|
32
|
+
/**
|
|
33
|
+
* Resolve a chunk ID to its exact text and document. The ID encodes the document and
|
|
34
|
+
* range, so any valid ID resolves while its document exists; resolution ignores this
|
|
35
|
+
* view's filter. Throws for malformed IDs, unknown documents and invalid ranges.
|
|
36
|
+
*/
|
|
37
|
+
resolve(chunkId: string): ResolvedChunk<M>;
|
|
38
|
+
private store;
|
|
39
|
+
private requireStored;
|
|
40
|
+
private requireVisible;
|
|
41
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Immutable documents and exact source chunks.
|
|
3
|
+
*
|
|
4
|
+
* @packageDocumentation
|
|
5
|
+
*/
|
|
6
|
+
export { Corpus } from "./corpus.js";
|
|
7
|
+
export * as chunkers from "./chunkers.js";
|
|
8
|
+
export type { Chunk, Chunker, ChunkReference, DocumentInput, CorpusDocument, DocumentReference, DocumentSource, Metadata, RecursiveChunkOptions, ResolvedChunk, TextRange, } from "./types.js";
|
|
@@ -0,0 +1,312 @@
|
|
|
1
|
+
var __defProp = Object.defineProperty;
|
|
2
|
+
var __export = (target, all) => {
|
|
3
|
+
for (var name in all)
|
|
4
|
+
__defProp(target, name, { get: all[name], enumerable: true });
|
|
5
|
+
};
|
|
6
|
+
|
|
7
|
+
// src/records.ts
|
|
8
|
+
function record(value) {
|
|
9
|
+
if (!isRecord(value)) {
|
|
10
|
+
throw new Error("Expected an object");
|
|
11
|
+
}
|
|
12
|
+
return value;
|
|
13
|
+
}
|
|
14
|
+
function isRecord(value) {
|
|
15
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
16
|
+
}
|
|
17
|
+
function string(value, name) {
|
|
18
|
+
if (typeof value !== "string") {
|
|
19
|
+
throw new Error(`${name} must be a string`);
|
|
20
|
+
}
|
|
21
|
+
return value;
|
|
22
|
+
}
|
|
23
|
+
function makeDocument(value) {
|
|
24
|
+
const input = record(value);
|
|
25
|
+
const documentId = string(input["documentId"], "documentId");
|
|
26
|
+
if (documentId.trim().length === 0) {
|
|
27
|
+
throw new Error("documentId must not be empty");
|
|
28
|
+
}
|
|
29
|
+
const rawMetadata = record(input["metadata"] ?? {});
|
|
30
|
+
if (Object.getPrototypeOf(rawMetadata) !== Object.prototype && Object.getPrototypeOf(rawMetadata) !== null) {
|
|
31
|
+
throw new Error("Metadata must be a plain object");
|
|
32
|
+
}
|
|
33
|
+
const metadata = value.metadata === void 0 ? {} : { ...value.metadata };
|
|
34
|
+
const rawSource = input["source"] === void 0 ? void 0 : record(input["source"]);
|
|
35
|
+
const source = rawSource === void 0 ? void 0 : Object.freeze({
|
|
36
|
+
uri: string(rawSource["uri"], "source.uri"),
|
|
37
|
+
...rawSource["mediaType"] === void 0 ? {} : { mediaType: string(rawSource["mediaType"], "source.mediaType") }
|
|
38
|
+
});
|
|
39
|
+
return Object.freeze({
|
|
40
|
+
documentId,
|
|
41
|
+
text: string(input["text"], "text"),
|
|
42
|
+
...input["title"] === void 0 ? {} : { title: string(input["title"], "title") },
|
|
43
|
+
...source === void 0 ? {} : { source },
|
|
44
|
+
metadata: Object.freeze(metadata)
|
|
45
|
+
});
|
|
46
|
+
}
|
|
47
|
+
function makeChunk(document, range) {
|
|
48
|
+
return Object.freeze({
|
|
49
|
+
id: `${encodeURIComponent(document.documentId)}#${range.start}-${range.end}`,
|
|
50
|
+
documentId: document.documentId,
|
|
51
|
+
range: Object.freeze({ start: range.start, end: range.end })
|
|
52
|
+
});
|
|
53
|
+
}
|
|
54
|
+
function parseChunkId(id) {
|
|
55
|
+
const match = /^([^#]+)#(0|[1-9]\d*)-(0|[1-9]\d*)$/u.exec(id);
|
|
56
|
+
const [, encoded, start, end] = match ?? [];
|
|
57
|
+
if (encoded === void 0 || start === void 0 || end === void 0) {
|
|
58
|
+
throw new Error(`Invalid chunk ID: ${id}`);
|
|
59
|
+
}
|
|
60
|
+
let documentId;
|
|
61
|
+
try {
|
|
62
|
+
documentId = decodeURIComponent(encoded);
|
|
63
|
+
} catch {
|
|
64
|
+
throw new Error(`Invalid chunk ID: ${id}`);
|
|
65
|
+
}
|
|
66
|
+
if (encodeURIComponent(documentId) !== encoded) {
|
|
67
|
+
throw new Error(`Invalid chunk ID: ${id}`);
|
|
68
|
+
}
|
|
69
|
+
return { documentId, range: { start: Number(start), end: Number(end) } };
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
// src/text.ts
|
|
73
|
+
function positiveInteger(value, name) {
|
|
74
|
+
if (!Number.isSafeInteger(value) || value < 1) {
|
|
75
|
+
throw new Error(`${name} must be a positive safe integer`);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
function nonnegativeInteger(value, name) {
|
|
79
|
+
if (!Number.isSafeInteger(value) || value < 0) {
|
|
80
|
+
throw new Error(`${name} must be a nonnegative safe integer`);
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
function graphemeBoundaries(text) {
|
|
84
|
+
const segmenter = new Intl.Segmenter("en", { granularity: "grapheme" });
|
|
85
|
+
return [...Array.from(segmenter.segment(text), (segment) => segment.index), text.length];
|
|
86
|
+
}
|
|
87
|
+
function validateRange(range, boundaries) {
|
|
88
|
+
if (!Number.isSafeInteger(range.start) || !Number.isSafeInteger(range.end) || range.start >= range.end || !boundaries.has(range.start) || !boundaries.has(range.end)) {
|
|
89
|
+
throw new Error("Range must be nonempty, within the text, and on grapheme boundaries");
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
function boundaryAt(boundaries, index) {
|
|
93
|
+
const boundary = boundaries[index];
|
|
94
|
+
if (boundary === void 0) {
|
|
95
|
+
throw new Error("Invalid text boundary index");
|
|
96
|
+
}
|
|
97
|
+
return boundary;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
// src/corpus.ts
|
|
101
|
+
var Corpus = class _Corpus {
|
|
102
|
+
#storage = { documents: /* @__PURE__ */ new Map() };
|
|
103
|
+
#filter = () => true;
|
|
104
|
+
add(input) {
|
|
105
|
+
if (!isBatch(input)) {
|
|
106
|
+
const document = makeDocument(input);
|
|
107
|
+
const existing = this.#storage.documents.get(document.documentId);
|
|
108
|
+
if (existing !== void 0) {
|
|
109
|
+
assertSameContent(existing, document);
|
|
110
|
+
return existing;
|
|
111
|
+
}
|
|
112
|
+
return this.store(document);
|
|
113
|
+
}
|
|
114
|
+
const pending = /* @__PURE__ */ new Map();
|
|
115
|
+
const documents = input.map((item) => {
|
|
116
|
+
const document = makeDocument(item);
|
|
117
|
+
const existing = pending.get(document.documentId) ?? this.#storage.documents.get(document.documentId);
|
|
118
|
+
if (existing !== void 0) {
|
|
119
|
+
assertSameContent(existing, document);
|
|
120
|
+
return existing;
|
|
121
|
+
}
|
|
122
|
+
pending.set(document.documentId, document);
|
|
123
|
+
return document;
|
|
124
|
+
});
|
|
125
|
+
for (const document of pending.values()) {
|
|
126
|
+
this.store(document);
|
|
127
|
+
}
|
|
128
|
+
return Object.freeze(documents);
|
|
129
|
+
}
|
|
130
|
+
/** Read a document. Throws when absent or excluded by this view's filter. */
|
|
131
|
+
document(documentId) {
|
|
132
|
+
return this.requireVisible({ documentId });
|
|
133
|
+
}
|
|
134
|
+
/** List every document matching this view's filter, in insertion order. */
|
|
135
|
+
documents() {
|
|
136
|
+
return Object.freeze([...this.#storage.documents.values()].filter(this.#filter));
|
|
137
|
+
}
|
|
138
|
+
/**
|
|
139
|
+
* Return a view over the same storage that also requires this predicate. Documents
|
|
140
|
+
* added later through any view appear when they match.
|
|
141
|
+
*/
|
|
142
|
+
filter(predicate) {
|
|
143
|
+
const parent = this.#filter;
|
|
144
|
+
const view = new _Corpus();
|
|
145
|
+
view.#storage = this.#storage;
|
|
146
|
+
view.#filter = (document) => parent(document) && predicate(document);
|
|
147
|
+
return view;
|
|
148
|
+
}
|
|
149
|
+
/**
|
|
150
|
+
* Split a document lazily, validating each range as it is consumed. If a later
|
|
151
|
+
* range is invalid, earlier yielded chunks remain valid and resolvable.
|
|
152
|
+
*/
|
|
153
|
+
chunk(target, chunker) {
|
|
154
|
+
return this.splitDocument(this.requireVisible(target), chunker);
|
|
155
|
+
}
|
|
156
|
+
*splitDocument(document, chunker) {
|
|
157
|
+
const boundaries = new Set(graphemeBoundaries(document.text));
|
|
158
|
+
let previousStart = -1;
|
|
159
|
+
for (const range of chunker.split(document.text)) {
|
|
160
|
+
validateRange(range, boundaries);
|
|
161
|
+
if (range.start <= previousStart) {
|
|
162
|
+
throw new Error("Chunker ranges must advance in source order");
|
|
163
|
+
}
|
|
164
|
+
previousStart = range.start;
|
|
165
|
+
yield materialize(document, makeChunk(document, range));
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
/** Select an exact range of a document. */
|
|
169
|
+
slice(target, selection) {
|
|
170
|
+
const document = this.requireVisible(target);
|
|
171
|
+
validateRange(selection, new Set(graphemeBoundaries(document.text)));
|
|
172
|
+
return materialize(document, makeChunk(document, selection));
|
|
173
|
+
}
|
|
174
|
+
/**
|
|
175
|
+
* Resolve a chunk ID to its exact text and document. The ID encodes the document and
|
|
176
|
+
* range, so any valid ID resolves while its document exists; resolution ignores this
|
|
177
|
+
* view's filter. Throws for malformed IDs, unknown documents and invalid ranges.
|
|
178
|
+
*/
|
|
179
|
+
resolve(chunkId) {
|
|
180
|
+
const { documentId, range } = parseChunkId(chunkId);
|
|
181
|
+
const document = this.requireStored({ documentId });
|
|
182
|
+
validateRange(range, new Set(graphemeBoundaries(document.text)));
|
|
183
|
+
return Object.freeze({ chunk: materialize(document, makeChunk(document, range)), document });
|
|
184
|
+
}
|
|
185
|
+
store(document) {
|
|
186
|
+
this.#storage.documents.set(document.documentId, document);
|
|
187
|
+
return document;
|
|
188
|
+
}
|
|
189
|
+
requireStored(reference) {
|
|
190
|
+
const document = this.#storage.documents.get(reference.documentId);
|
|
191
|
+
if (document === void 0) {
|
|
192
|
+
throw new Error(`Unknown document: ${reference.documentId}`);
|
|
193
|
+
}
|
|
194
|
+
return document;
|
|
195
|
+
}
|
|
196
|
+
requireVisible(reference) {
|
|
197
|
+
const document = this.requireStored(reference);
|
|
198
|
+
if (!this.#filter(document)) {
|
|
199
|
+
throw new Error(`Document "${document.documentId}" is excluded by this corpus view`);
|
|
200
|
+
}
|
|
201
|
+
return document;
|
|
202
|
+
}
|
|
203
|
+
};
|
|
204
|
+
function materialize(document, reference) {
|
|
205
|
+
return Object.freeze({
|
|
206
|
+
...reference,
|
|
207
|
+
text: document.text.slice(reference.range.start, reference.range.end)
|
|
208
|
+
});
|
|
209
|
+
}
|
|
210
|
+
function assertSameContent(existing, document) {
|
|
211
|
+
const keys = Object.keys(existing.metadata);
|
|
212
|
+
if (existing.text !== document.text || existing.title !== document.title || existing.source?.uri !== document.source?.uri || existing.source?.mediaType !== document.source?.mediaType || keys.length !== Object.keys(document.metadata).length || !keys.every(
|
|
213
|
+
(key) => Object.hasOwn(document.metadata, key) && Object.is(existing.metadata[key], document.metadata[key])
|
|
214
|
+
)) {
|
|
215
|
+
throw new Error(`Document "${document.documentId}" already exists with different content`);
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
function isBatch(input) {
|
|
219
|
+
return Array.isArray(input);
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
// src/chunkers.ts
|
|
223
|
+
var chunkers_exports = {};
|
|
224
|
+
__export(chunkers_exports, {
|
|
225
|
+
recursive: () => recursive
|
|
226
|
+
});
|
|
227
|
+
function recursive(options) {
|
|
228
|
+
const { size, overlap = 0, measure } = options;
|
|
229
|
+
const separators = [...options.separators ?? ["\n\n", "\n", " "]];
|
|
230
|
+
positiveInteger(size, "size");
|
|
231
|
+
nonnegativeInteger(overlap, "overlap");
|
|
232
|
+
if (overlap >= size) {
|
|
233
|
+
throw new RangeError("overlap must be smaller than size");
|
|
234
|
+
}
|
|
235
|
+
if (separators.some((separator) => separator.length === 0)) {
|
|
236
|
+
throw new Error("Separators must be nonempty; grapheme fallback is automatic");
|
|
237
|
+
}
|
|
238
|
+
return Object.freeze({
|
|
239
|
+
name: "recursive",
|
|
240
|
+
*split(text) {
|
|
241
|
+
const boundaries = graphemeBoundaries(text);
|
|
242
|
+
const indices = new Map(boundaries.map((offset, index) => [offset, index]));
|
|
243
|
+
const last = boundaries.length - 1;
|
|
244
|
+
function count(start2, end) {
|
|
245
|
+
if (measure === void 0) {
|
|
246
|
+
return end - start2;
|
|
247
|
+
}
|
|
248
|
+
const value = measure(
|
|
249
|
+
text.slice(boundaryAt(boundaries, start2), boundaryAt(boundaries, end))
|
|
250
|
+
);
|
|
251
|
+
if (!Number.isSafeInteger(value) || value < 0) {
|
|
252
|
+
throw new RangeError("measure must return a nonnegative safe integer");
|
|
253
|
+
}
|
|
254
|
+
return value;
|
|
255
|
+
}
|
|
256
|
+
function boundedEnd(start2) {
|
|
257
|
+
if (measure === void 0) {
|
|
258
|
+
return Math.min(start2 + size, last);
|
|
259
|
+
}
|
|
260
|
+
let end = start2;
|
|
261
|
+
while (end < last && count(start2, end + 1) <= size) {
|
|
262
|
+
end += 1;
|
|
263
|
+
}
|
|
264
|
+
if (end === start2) {
|
|
265
|
+
throw new RangeError("A single grapheme exceeds the chunk size");
|
|
266
|
+
}
|
|
267
|
+
return end;
|
|
268
|
+
}
|
|
269
|
+
function preferredEnd(start2, hardEnd) {
|
|
270
|
+
if (hardEnd === last) {
|
|
271
|
+
return hardEnd;
|
|
272
|
+
}
|
|
273
|
+
const startOffset = boundaryAt(boundaries, start2);
|
|
274
|
+
const endOffset = boundaryAt(boundaries, hardEnd);
|
|
275
|
+
for (const separator of separators) {
|
|
276
|
+
const offset = text.lastIndexOf(separator, endOffset - separator.length);
|
|
277
|
+
const candidate = indices.get(offset + separator.length);
|
|
278
|
+
if (offset >= startOffset && candidate !== void 0 && candidate > start2 && candidate <= hardEnd) {
|
|
279
|
+
return candidate;
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
return hardEnd;
|
|
283
|
+
}
|
|
284
|
+
function nextStart(start2, end) {
|
|
285
|
+
if (overlap === 0) {
|
|
286
|
+
return end;
|
|
287
|
+
}
|
|
288
|
+
if (measure === void 0) {
|
|
289
|
+
return Math.max(start2 + 1, end - overlap);
|
|
290
|
+
}
|
|
291
|
+
let next = end;
|
|
292
|
+
while (next > start2 + 1 && count(next - 1, end) <= overlap) {
|
|
293
|
+
next -= 1;
|
|
294
|
+
}
|
|
295
|
+
return next;
|
|
296
|
+
}
|
|
297
|
+
let start = 0;
|
|
298
|
+
while (start < last) {
|
|
299
|
+
const end = preferredEnd(start, boundedEnd(start));
|
|
300
|
+
yield { start: boundaryAt(boundaries, start), end: boundaryAt(boundaries, end) };
|
|
301
|
+
if (end === last) {
|
|
302
|
+
return;
|
|
303
|
+
}
|
|
304
|
+
start = nextStart(start, end);
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
});
|
|
308
|
+
}
|
|
309
|
+
export {
|
|
310
|
+
Corpus,
|
|
311
|
+
chunkers_exports as chunkers
|
|
312
|
+
};
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { ChunkReference, CorpusDocument, DocumentInput, Metadata, TextRange } from "./types.js";
|
|
2
|
+
export declare function makeDocument<M extends Metadata = Metadata>(value: DocumentInput<M>): CorpusDocument<M>;
|
|
3
|
+
export declare function makeChunk(document: CorpusDocument, range: TextRange): ChunkReference;
|
|
4
|
+
/** Parse a chunk ID back into its document and range. Only the canonical spelling is accepted. */
|
|
5
|
+
export declare function parseChunkId(id: string): Readonly<{
|
|
6
|
+
documentId: string;
|
|
7
|
+
range: TextRange;
|
|
8
|
+
}>;
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import type { TextRange } from "./types.js";
|
|
2
|
+
export declare function positiveInteger(value: number, name: string): void;
|
|
3
|
+
export declare function nonnegativeInteger(value: number, name: string): void;
|
|
4
|
+
export declare function graphemeBoundaries(text: string): readonly number[];
|
|
5
|
+
export declare function validateRange(range: TextRange, boundaries: ReadonlySet<number>): void;
|
|
6
|
+
export declare function boundaryAt(boundaries: readonly number[], index: number): number;
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
/** Application-defined metadata. Values may include native objects and functions. */
|
|
2
|
+
export type Metadata = Readonly<Record<string, unknown>>;
|
|
3
|
+
/** A half-open range, measured in JavaScript UTF-16 code units. */
|
|
4
|
+
export type TextRange = Readonly<{
|
|
5
|
+
start: number;
|
|
6
|
+
end: number;
|
|
7
|
+
}>;
|
|
8
|
+
/** Location of the original artifact; text offsets refer to the stored text, not file bytes. */
|
|
9
|
+
export type DocumentSource = Readonly<{
|
|
10
|
+
uri: string;
|
|
11
|
+
mediaType?: string;
|
|
12
|
+
}>;
|
|
13
|
+
/** Text supplied by the caller, after any file extraction or normalization. */
|
|
14
|
+
export type DocumentInput<M extends Metadata = Metadata> = Readonly<{
|
|
15
|
+
documentId: string;
|
|
16
|
+
text: string;
|
|
17
|
+
title?: string;
|
|
18
|
+
source?: DocumentSource;
|
|
19
|
+
metadata?: M;
|
|
20
|
+
}>;
|
|
21
|
+
/** Identifies one immutable document. */
|
|
22
|
+
export type DocumentReference = Readonly<{
|
|
23
|
+
documentId: string;
|
|
24
|
+
}>;
|
|
25
|
+
/** An immutable document. Re-adding its ID with different content is rejected. */
|
|
26
|
+
export type CorpusDocument<M extends Metadata = Metadata> = DocumentReference & Readonly<{
|
|
27
|
+
text: string;
|
|
28
|
+
title?: string;
|
|
29
|
+
source?: DocumentSource;
|
|
30
|
+
/** Shallow snapshot; nested values retain identity and remain caller-owned. */
|
|
31
|
+
metadata: Readonly<Partial<M>>;
|
|
32
|
+
}>;
|
|
33
|
+
/** A stable source range; its ID depends only on the document and absolute range. */
|
|
34
|
+
export type ChunkReference = DocumentReference & Readonly<{
|
|
35
|
+
id: string;
|
|
36
|
+
range: TextRange;
|
|
37
|
+
}>;
|
|
38
|
+
/** A source range with text materialized for convenient consumption by models or UIs. */
|
|
39
|
+
export type Chunk = ChunkReference & Readonly<{
|
|
40
|
+
text: string;
|
|
41
|
+
}>;
|
|
42
|
+
/** A resolved chunk together with its complete document. */
|
|
43
|
+
export type ResolvedChunk<M extends Metadata = Metadata> = Readonly<{
|
|
44
|
+
chunk: Chunk;
|
|
45
|
+
document: CorpusDocument<M>;
|
|
46
|
+
}>;
|
|
47
|
+
/** Synchronous segmentation into source-ordered ranges relative to the supplied text. */
|
|
48
|
+
export type Chunker = Readonly<{
|
|
49
|
+
name: string;
|
|
50
|
+
split: (text: string) => Iterable<TextRange>;
|
|
51
|
+
}>;
|
|
52
|
+
/** Bounded recursive splitting; size and overlap use grapheme counts unless measure is supplied. */
|
|
53
|
+
export type RecursiveChunkOptions = Readonly<{
|
|
54
|
+
/** Positive integer maximum size of each emitted substring. */
|
|
55
|
+
size: number;
|
|
56
|
+
/** Desired maximum overlap, smaller than size. Defaults to zero. */
|
|
57
|
+
overlap?: number;
|
|
58
|
+
/** Deterministic nonnegative integer size of a substring, e.g. an exact tokenizer's count. */
|
|
59
|
+
measure?: (text: string) => number;
|
|
60
|
+
/** Preferred separators, in order. Defaults to paragraphs, lines, then words. */
|
|
61
|
+
separators?: readonly string[];
|
|
62
|
+
}>;
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import type { Atom, GroundAtom, PredicateDeclaration, PredicateSchema } from "../core/dsl/index.js";
|
|
2
|
+
import type { EvidenceMetadata, KnowledgeBase, NegativeLiteral, PositiveLiteral } from "../core/knowledge-base/index.js";
|
|
3
|
+
import type { QueryAllJudgment, QueryJudgment } from "../core/query/index.js";
|
|
4
|
+
import type { AllowedPermission, Operation, PermissionFilter } from "./types.js";
|
|
5
|
+
/**
|
|
6
|
+
* A permission-enforcing view over one KB. Permission edits return new views;
|
|
7
|
+
* permitted writes change the shared underlying KB.
|
|
8
|
+
* Nothing is permitted by default, and blocks override permits regardless of order.
|
|
9
|
+
* `"*"` matches every operation or predicate and is evaluated when permissions are read.
|
|
10
|
+
*/
|
|
11
|
+
export declare class GuardedKB {
|
|
12
|
+
#private;
|
|
13
|
+
/** The underlying, unrestricted KB shared by this view and its branches. */
|
|
14
|
+
readonly kb: KnowledgeBase;
|
|
15
|
+
constructor(
|
|
16
|
+
/** The underlying, unrestricted KB shared by this view and its branches. */
|
|
17
|
+
kb: KnowledgeBase);
|
|
18
|
+
/** Return a new view permitting every operation for every predicate; shorthand for `permit("*", "*")`. */
|
|
19
|
+
permit(operation: "*"): GuardedKB;
|
|
20
|
+
/** Return a new view permitting the operations for every predicate, including ones the KB references later. */
|
|
21
|
+
permit(operation: Operation | "*", predicates: "*"): GuardedKB;
|
|
22
|
+
/** Return a new view additionally permitting these predicates for the operations. */
|
|
23
|
+
permit(operation: Operation | "*", ...predicates: readonly PredicateDeclaration[]): GuardedKB;
|
|
24
|
+
/** Return a new view blocking every operation for every predicate; shorthand for `block("*", "*")`. */
|
|
25
|
+
block(operation: "*"): GuardedKB;
|
|
26
|
+
/** Return a new view blocking the operations for every predicate, including ones the KB references later. */
|
|
27
|
+
block(operation: Operation | "*", predicates: "*"): GuardedKB;
|
|
28
|
+
/** Return a new view blocking these predicates, including any later permits. */
|
|
29
|
+
block(operation: Operation | "*", ...predicates: readonly PredicateDeclaration[]): GuardedKB;
|
|
30
|
+
/**
|
|
31
|
+
* Return a view with the same permissions over a fork of the underlying KB.
|
|
32
|
+
* Later writes to either KB do not affect the other; wildcards resolve against each KB separately.
|
|
33
|
+
*/
|
|
34
|
+
fork(): GuardedKB;
|
|
35
|
+
/** Assert evidence after checking permission; throws before writing when forbidden. */
|
|
36
|
+
asserts<Schema extends PredicateSchema>(atom: Atom<Schema>, metadata?: EvidenceMetadata): PositiveLiteral<Schema>;
|
|
37
|
+
/** Deny a fact after checking permission; throws before writing when forbidden. */
|
|
38
|
+
denies<Schema extends PredicateSchema>(atom: Atom<Schema>, metadata?: EvidenceMetadata): NegativeLiteral<Schema>;
|
|
39
|
+
/**
|
|
40
|
+
* Query a permitted ground atom. Permission applies to the requested predicate;
|
|
41
|
+
* evaluation uses the full KB, including supporting predicates and their proofs.
|
|
42
|
+
*/
|
|
43
|
+
query<Schema extends PredicateSchema>(atom: GroundAtom<Schema>): QueryJudgment<Schema>;
|
|
44
|
+
/** Query all matches for a permitted predicate using the full KB. */
|
|
45
|
+
queryAll<Schema extends PredicateSchema>(pattern: Atom<Schema>): QueryAllJudgment<Schema>;
|
|
46
|
+
/** Return all effective grants, in first-permitted order. */
|
|
47
|
+
resolve(): readonly AllowedPermission[];
|
|
48
|
+
/** Return effective grants for one operation. */
|
|
49
|
+
resolve(operation: Operation): readonly AllowedPermission[];
|
|
50
|
+
/** Return effective grants for any of these predicates, including query grants. */
|
|
51
|
+
resolve(predicate: PredicateDeclaration, ...predicates: readonly PredicateDeclaration[]): readonly AllowedPermission[];
|
|
52
|
+
/** Filter effective grants without changing their order or bypassing blocks. */
|
|
53
|
+
resolve(filter: PermissionFilter): readonly AllowedPermission[];
|
|
54
|
+
private requirePermission;
|
|
55
|
+
private isBlocked;
|
|
56
|
+
private withRule;
|
|
57
|
+
}
|
|
58
|
+
/** Wrap a KB with a guard initially permitting no operations. Does not copy the KB. */
|
|
59
|
+
export declare function guard(kb: KnowledgeBase): GuardedKB;
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
// src/guarded-kb.ts
|
|
2
|
+
import { query, queryAll } from "../core/query/index.js";
|
|
3
|
+
var operations = ["assert", "deny", "query"];
|
|
4
|
+
var GuardedKB = class _GuardedKB {
|
|
5
|
+
constructor(kb) {
|
|
6
|
+
this.kb = kb;
|
|
7
|
+
}
|
|
8
|
+
kb;
|
|
9
|
+
#rules = [];
|
|
10
|
+
permit(operation, ...predicates) {
|
|
11
|
+
return this.withRule("permit", operation, predicates);
|
|
12
|
+
}
|
|
13
|
+
block(operation, ...predicates) {
|
|
14
|
+
return this.withRule("block", operation, predicates);
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* Return a view with the same permissions over a fork of the underlying KB.
|
|
18
|
+
* Later writes to either KB do not affect the other; wildcards resolve against each KB separately.
|
|
19
|
+
*/
|
|
20
|
+
fork() {
|
|
21
|
+
const next = new _GuardedKB(this.kb.fork());
|
|
22
|
+
next.#rules = this.#rules;
|
|
23
|
+
return next;
|
|
24
|
+
}
|
|
25
|
+
/** Assert evidence after checking permission; throws before writing when forbidden. */
|
|
26
|
+
asserts(atom, metadata) {
|
|
27
|
+
this.requirePermission("assert", atom.predicate);
|
|
28
|
+
return this.kb.asserts(atom, metadata);
|
|
29
|
+
}
|
|
30
|
+
/** Deny a fact after checking permission; throws before writing when forbidden. */
|
|
31
|
+
denies(atom, metadata) {
|
|
32
|
+
this.requirePermission("deny", atom.predicate);
|
|
33
|
+
return this.kb.denies(atom, metadata);
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Query a permitted ground atom. Permission applies to the requested predicate;
|
|
37
|
+
* evaluation uses the full KB, including supporting predicates and their proofs.
|
|
38
|
+
*/
|
|
39
|
+
query(atom) {
|
|
40
|
+
this.requirePermission("query", atom.predicate);
|
|
41
|
+
return query(this.kb, atom);
|
|
42
|
+
}
|
|
43
|
+
/** Query all matches for a permitted predicate using the full KB. */
|
|
44
|
+
queryAll(pattern) {
|
|
45
|
+
this.requirePermission("query", pattern.predicate);
|
|
46
|
+
return queryAll(this.kb, pattern);
|
|
47
|
+
}
|
|
48
|
+
resolve(selection, ...predicates) {
|
|
49
|
+
let filter = {};
|
|
50
|
+
if (typeof selection === "string") {
|
|
51
|
+
filter = { operations: [selection] };
|
|
52
|
+
} else if (selection !== void 0) {
|
|
53
|
+
filter = "name" in selection ? { predicates: [selection, ...predicates] } : selection;
|
|
54
|
+
}
|
|
55
|
+
const known = this.#rules.some((rule) => rule.predicates === "*") ? this.kb.predicates() : [];
|
|
56
|
+
const subjectsOf = (rule) => rule.predicates === "*" ? known : rule.predicates;
|
|
57
|
+
const seen = /* @__PURE__ */ new Map();
|
|
58
|
+
const allowed = [];
|
|
59
|
+
for (const rule of this.#rules) {
|
|
60
|
+
if (rule.effect !== "permit") {
|
|
61
|
+
continue;
|
|
62
|
+
}
|
|
63
|
+
for (const operation of rule.operations) {
|
|
64
|
+
if (filter.operations && !filter.operations.includes(operation)) {
|
|
65
|
+
continue;
|
|
66
|
+
}
|
|
67
|
+
const subjects = seen.get(operation) ?? /* @__PURE__ */ new Set();
|
|
68
|
+
for (const predicate of subjectsOf(rule)) {
|
|
69
|
+
if (filter.predicates && !filter.predicates.includes(predicate)) {
|
|
70
|
+
continue;
|
|
71
|
+
}
|
|
72
|
+
if (subjects.has(predicate) || this.isBlocked(operation, predicate)) {
|
|
73
|
+
continue;
|
|
74
|
+
}
|
|
75
|
+
subjects.add(predicate);
|
|
76
|
+
allowed.push({ operation, predicate });
|
|
77
|
+
}
|
|
78
|
+
seen.set(operation, subjects);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
return allowed;
|
|
82
|
+
}
|
|
83
|
+
requirePermission(operation, predicate) {
|
|
84
|
+
const permitted = this.#rules.some(
|
|
85
|
+
(rule) => rule.effect === "permit" && matches(rule, operation, predicate)
|
|
86
|
+
);
|
|
87
|
+
if (!permitted || this.isBlocked(operation, predicate)) {
|
|
88
|
+
throw new Error(
|
|
89
|
+
`Operation "${operation}" is not permitted for predicate "${predicate.name}"`
|
|
90
|
+
);
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
isBlocked(operation, predicate) {
|
|
94
|
+
return this.#rules.some(
|
|
95
|
+
(rule) => rule.effect === "block" && matches(rule, operation, predicate)
|
|
96
|
+
);
|
|
97
|
+
}
|
|
98
|
+
withRule(effect, operation, predicates) {
|
|
99
|
+
const wildcard = predicates[0] === "*" || operation === "*" && predicates.length === 0;
|
|
100
|
+
const next = new _GuardedKB(this.kb);
|
|
101
|
+
next.#rules = [
|
|
102
|
+
...this.#rules,
|
|
103
|
+
{
|
|
104
|
+
effect,
|
|
105
|
+
operations: operation === "*" ? operations : [operation],
|
|
106
|
+
predicates: wildcard ? "*" : predicates.filter((predicate) => predicate !== "*")
|
|
107
|
+
}
|
|
108
|
+
];
|
|
109
|
+
return next;
|
|
110
|
+
}
|
|
111
|
+
};
|
|
112
|
+
function matches(rule, operation, predicate) {
|
|
113
|
+
return rule.operations.includes(operation) && (rule.predicates === "*" || rule.predicates.includes(predicate));
|
|
114
|
+
}
|
|
115
|
+
function guard(kb) {
|
|
116
|
+
return new GuardedKB(kb);
|
|
117
|
+
}
|
|
118
|
+
export {
|
|
119
|
+
GuardedKB,
|
|
120
|
+
guard
|
|
121
|
+
};
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import type { PredicateDeclaration } from "../core/dsl/index.js";
|
|
2
|
+
/** Guarded knowledge-base operations that can be permitted independently. */
|
|
3
|
+
export type Operation = "assert" | "deny" | "query";
|
|
4
|
+
/** An effective grant after blocks and duplicates have been removed. */
|
|
5
|
+
export type AllowedPermission = Readonly<{
|
|
6
|
+
operation: Operation;
|
|
7
|
+
predicate: PredicateDeclaration;
|
|
8
|
+
}>;
|
|
9
|
+
/** OR within each dimension, AND between dimensions. Empty arrays match nothing. */
|
|
10
|
+
export type PermissionFilter = Readonly<{
|
|
11
|
+
/** Omit to match every operation. */
|
|
12
|
+
operations?: readonly Operation[];
|
|
13
|
+
/** Omit to match every predicate. Predicates are matched by identity. */
|
|
14
|
+
predicates?: readonly PredicateDeclaration[];
|
|
15
|
+
}>;
|