@crewhaus/tool-state 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +268 -0
- package/dist/index.d.ts +43 -0
- package/dist/index.js +1858 -0
- package/dist/lib/bm25.d.ts +117 -0
- package/dist/lib/bm25.js +245 -0
- package/dist/lib/jsonl.d.ts +47 -0
- package/dist/lib/jsonl.js +102 -0
- package/dist/lib/names.d.ts +41 -0
- package/dist/lib/names.js +144 -0
- package/dist/lib/records.d.ts +66 -0
- package/dist/lib/records.js +106 -0
- package/dist/paths.d.ts +52 -0
- package/dist/paths.js +203 -0
- package/dist/store.d.ts +128 -0
- package/dist/store.js +502 -0
- package/dist/vector.d.ts +248 -0
- package/dist/vector.js +445 -0
- package/package.json +45 -0
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Lexical retrieval: an inverted index and Okapi BM25 ranking over it.
|
|
3
|
+
*
|
|
4
|
+
* LEXICAL ONLY. There are no embeddings here and no semantic matching of any
|
|
5
|
+
* kind — a document scores because it literally contains the query's terms.
|
|
6
|
+
* A search for "car" will not find "automobile". That is the honest limit of
|
|
7
|
+
* a deterministic retriever, and it is the reason this is worth having: the
|
|
8
|
+
* same corpus and the same query always produce the same ranking, with no
|
|
9
|
+
* model call and no vector store.
|
|
10
|
+
*
|
|
11
|
+
* The scoring function is the standard one:
|
|
12
|
+
*
|
|
13
|
+
* score(D,Q) = Σ_t idf(t) · ( f(t,D)·(k1+1) ) / ( f(t,D) + k1·(1−b+b·|D|/avgdl) )
|
|
14
|
+
* idf(t) = ln( 1 + (N − df(t) + 0.5) / (df(t) + 0.5) )
|
|
15
|
+
*
|
|
16
|
+
* with `k1` controlling how fast term frequency saturates and `b` how hard
|
|
17
|
+
* long documents are penalised. The defaults are the usual `k1 = 1.2`,
|
|
18
|
+
* `b = 0.75`; both are caller-tunable and both are reported in every result
|
|
19
|
+
* so a ranking can be reproduced later.
|
|
20
|
+
*
|
|
21
|
+
* The `+1` inside the idf logarithm is the non-negative variant. The classic
|
|
22
|
+
* Robertson/Sparck-Jones idf goes NEGATIVE once a term appears in more than
|
|
23
|
+
* half the corpus, which perversely penalises a document for containing a
|
|
24
|
+
* query term; this form floors it instead, so a term that appears everywhere
|
|
25
|
+
* keeps a small positive weight and a rare term still dominates it.
|
|
26
|
+
*
|
|
27
|
+
* Tokenisation is Unicode-aware case folding on non-word boundaries. There is
|
|
28
|
+
* no stemming and no stop-word list: "running" and "run" are different terms,
|
|
29
|
+
* and common words are handled by idf rather than by a hardcoded list that
|
|
30
|
+
* would be wrong in another language.
|
|
31
|
+
*/
|
|
32
|
+
/** Default term-frequency saturation. */
|
|
33
|
+
export declare const DEFAULT_K1 = 1.2;
|
|
34
|
+
/** Default length normalisation: 0 disables it, 1 applies it fully. */
|
|
35
|
+
export declare const DEFAULT_B = 0.75;
|
|
36
|
+
/** A document as the caller hands it over. */
|
|
37
|
+
export type IndexDoc = {
|
|
38
|
+
readonly id: string;
|
|
39
|
+
readonly text: string;
|
|
40
|
+
readonly title?: string;
|
|
41
|
+
readonly bytes?: number;
|
|
42
|
+
readonly mtimeMs?: number;
|
|
43
|
+
};
|
|
44
|
+
/** What the index remembers about a document — never its text. */
|
|
45
|
+
export type IndexEntry = {
|
|
46
|
+
readonly id: string;
|
|
47
|
+
/** Token count, used for length normalisation. */
|
|
48
|
+
readonly length: number;
|
|
49
|
+
readonly title?: string;
|
|
50
|
+
readonly bytes?: number;
|
|
51
|
+
readonly mtimeMs?: number;
|
|
52
|
+
};
|
|
53
|
+
/** A posting: the document's position in `docs`, and the term frequency. */
|
|
54
|
+
export type Posting = readonly [number, number];
|
|
55
|
+
/** A serialisable inverted index. Term keys are sorted, so JSON is stable. */
|
|
56
|
+
export type InvertedIndex = {
|
|
57
|
+
readonly version: 1;
|
|
58
|
+
readonly docs: ReadonlyArray<IndexEntry>;
|
|
59
|
+
readonly postings: Readonly<Record<string, ReadonlyArray<Posting>>>;
|
|
60
|
+
/** Total tokens across every document, for the average-length term. */
|
|
61
|
+
readonly totalTerms: number;
|
|
62
|
+
};
|
|
63
|
+
export type Bm25Params = {
|
|
64
|
+
readonly k1: number;
|
|
65
|
+
readonly b: number;
|
|
66
|
+
};
|
|
67
|
+
export type Hit = {
|
|
68
|
+
readonly id: string;
|
|
69
|
+
readonly score: number;
|
|
70
|
+
/** Which query terms this document actually contained, sorted. */
|
|
71
|
+
readonly matchedTerms: ReadonlyArray<string>;
|
|
72
|
+
readonly entry: IndexEntry;
|
|
73
|
+
};
|
|
74
|
+
export type SearchResult = {
|
|
75
|
+
/** The query's distinct terms, sorted. */
|
|
76
|
+
readonly terms: ReadonlyArray<string>;
|
|
77
|
+
/** Terms with no posting list at all, sorted. */
|
|
78
|
+
readonly unknownTerms: ReadonlyArray<string>;
|
|
79
|
+
/**
|
|
80
|
+
* How many documents matched at least one term and cleared `minScore`,
|
|
81
|
+
* before `limit` was applied.
|
|
82
|
+
*/
|
|
83
|
+
readonly matched: number;
|
|
84
|
+
readonly hits: ReadonlyArray<Hit>;
|
|
85
|
+
};
|
|
86
|
+
/** Case-folded word tokens: letters, digits and underscore, nothing else. */
|
|
87
|
+
export declare function tokenize(text: string): string[];
|
|
88
|
+
/** Distinct tokens of a query, sorted so the reported term list is stable. */
|
|
89
|
+
export declare function queryTerms(query: string): string[];
|
|
90
|
+
/**
|
|
91
|
+
* Build an inverted index over `docs`, in the order given. Callers sort the
|
|
92
|
+
* documents first so that two builds over the same corpus are byte-identical.
|
|
93
|
+
*/
|
|
94
|
+
export declare function buildIndex(docs: ReadonlyArray<IndexDoc>): InvertedIndex;
|
|
95
|
+
/**
|
|
96
|
+
* Rank the index's documents against `query`.
|
|
97
|
+
*
|
|
98
|
+
* Ties break on document id, ascending, by plain code-unit comparison — never
|
|
99
|
+
* `localeCompare`, whose ordering is machine-dependent.
|
|
100
|
+
*/
|
|
101
|
+
export declare function searchIndex(index: InvertedIndex, query: string, options?: {
|
|
102
|
+
readonly k1?: number;
|
|
103
|
+
readonly b?: number;
|
|
104
|
+
readonly limit?: number;
|
|
105
|
+
readonly minScore?: number;
|
|
106
|
+
}): SearchResult;
|
|
107
|
+
/** Structural check, so a hand-edited or half-written index file is reported. */
|
|
108
|
+
export declare function isInvertedIndex(value: unknown): value is InvertedIndex;
|
|
109
|
+
/**
|
|
110
|
+
* The first whole-word occurrence of any term, as a trimmed window of text.
|
|
111
|
+
*
|
|
112
|
+
* Deliberately simple: the earliest match wins rather than the densest one,
|
|
113
|
+
* so the snippet is cheap and stable. Returns `undefined` when no term occurs
|
|
114
|
+
* in the text (a document can still score on a term it shares with the query
|
|
115
|
+
* only through another field, e.g. its title).
|
|
116
|
+
*/
|
|
117
|
+
export declare function snippet(text: string, terms: ReadonlyArray<string>, maxChars: number): string | undefined;
|
package/dist/lib/bm25.js
ADDED
|
@@ -0,0 +1,245 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Lexical retrieval: an inverted index and Okapi BM25 ranking over it.
|
|
3
|
+
*
|
|
4
|
+
* LEXICAL ONLY. There are no embeddings here and no semantic matching of any
|
|
5
|
+
* kind — a document scores because it literally contains the query's terms.
|
|
6
|
+
* A search for "car" will not find "automobile". That is the honest limit of
|
|
7
|
+
* a deterministic retriever, and it is the reason this is worth having: the
|
|
8
|
+
* same corpus and the same query always produce the same ranking, with no
|
|
9
|
+
* model call and no vector store.
|
|
10
|
+
*
|
|
11
|
+
* The scoring function is the standard one:
|
|
12
|
+
*
|
|
13
|
+
* score(D,Q) = Σ_t idf(t) · ( f(t,D)·(k1+1) ) / ( f(t,D) + k1·(1−b+b·|D|/avgdl) )
|
|
14
|
+
* idf(t) = ln( 1 + (N − df(t) + 0.5) / (df(t) + 0.5) )
|
|
15
|
+
*
|
|
16
|
+
* with `k1` controlling how fast term frequency saturates and `b` how hard
|
|
17
|
+
* long documents are penalised. The defaults are the usual `k1 = 1.2`,
|
|
18
|
+
* `b = 0.75`; both are caller-tunable and both are reported in every result
|
|
19
|
+
* so a ranking can be reproduced later.
|
|
20
|
+
*
|
|
21
|
+
* The `+1` inside the idf logarithm is the non-negative variant. The classic
|
|
22
|
+
* Robertson/Sparck-Jones idf goes NEGATIVE once a term appears in more than
|
|
23
|
+
* half the corpus, which perversely penalises a document for containing a
|
|
24
|
+
* query term; this form floors it instead, so a term that appears everywhere
|
|
25
|
+
* keeps a small positive weight and a rare term still dominates it.
|
|
26
|
+
*
|
|
27
|
+
* Tokenisation is Unicode-aware case folding on non-word boundaries. There is
|
|
28
|
+
* no stemming and no stop-word list: "running" and "run" are different terms,
|
|
29
|
+
* and common words are handled by idf rather than by a hardcoded list that
|
|
30
|
+
* would be wrong in another language.
|
|
31
|
+
*/
|
|
32
|
+
/** Default term-frequency saturation. */
|
|
33
|
+
export const DEFAULT_K1 = 1.2;
|
|
34
|
+
/** Default length normalisation: 0 disables it, 1 applies it fully. */
|
|
35
|
+
export const DEFAULT_B = 0.75;
|
|
36
|
+
/** Longest token kept; anything longer is a checksum or a minified blob. */
|
|
37
|
+
const MAX_TOKEN_LENGTH = 64;
|
|
38
|
+
/** Case-folded word tokens: letters, digits and underscore, nothing else. */
|
|
39
|
+
export function tokenize(text) {
|
|
40
|
+
const out = [];
|
|
41
|
+
for (const raw of text.toLowerCase().split(/[^\p{L}\p{N}_]+/u)) {
|
|
42
|
+
if (raw.length > 0 && raw.length <= MAX_TOKEN_LENGTH)
|
|
43
|
+
out.push(raw);
|
|
44
|
+
}
|
|
45
|
+
return out;
|
|
46
|
+
}
|
|
47
|
+
/** Distinct tokens of a query, sorted so the reported term list is stable. */
|
|
48
|
+
export function queryTerms(query) {
|
|
49
|
+
return [...new Set(tokenize(query))].sort();
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* Build an inverted index over `docs`, in the order given. Callers sort the
|
|
53
|
+
* documents first so that two builds over the same corpus are byte-identical.
|
|
54
|
+
*/
|
|
55
|
+
export function buildIndex(docs) {
|
|
56
|
+
const entries = [];
|
|
57
|
+
const byTerm = new Map();
|
|
58
|
+
let totalTerms = 0;
|
|
59
|
+
docs.forEach((doc, docIndex) => {
|
|
60
|
+
const tokens = tokenize(doc.text);
|
|
61
|
+
const frequencies = new Map();
|
|
62
|
+
for (const token of tokens)
|
|
63
|
+
frequencies.set(token, (frequencies.get(token) ?? 0) + 1);
|
|
64
|
+
for (const [term, tf] of frequencies) {
|
|
65
|
+
const list = byTerm.get(term);
|
|
66
|
+
if (list === undefined)
|
|
67
|
+
byTerm.set(term, [[docIndex, tf]]);
|
|
68
|
+
else
|
|
69
|
+
list.push([docIndex, tf]);
|
|
70
|
+
}
|
|
71
|
+
totalTerms += tokens.length;
|
|
72
|
+
entries.push({
|
|
73
|
+
id: doc.id,
|
|
74
|
+
length: tokens.length,
|
|
75
|
+
...(doc.title !== undefined ? { title: doc.title } : {}),
|
|
76
|
+
...(doc.bytes !== undefined ? { bytes: doc.bytes } : {}),
|
|
77
|
+
...(doc.mtimeMs !== undefined ? { mtimeMs: doc.mtimeMs } : {}),
|
|
78
|
+
});
|
|
79
|
+
});
|
|
80
|
+
// A NULL-PROTOTYPE object, because a term is arbitrary text from a document:
|
|
81
|
+
// `postings["__proto__"] = […]` on a normal object literal sets the
|
|
82
|
+
// prototype instead of a key, which silently loses that term AND corrupts
|
|
83
|
+
// the index. With no prototype there is nothing to shadow.
|
|
84
|
+
const postings = Object.create(null);
|
|
85
|
+
for (const term of [...byTerm.keys()].sort()) {
|
|
86
|
+
postings[term] = byTerm.get(term) ?? [];
|
|
87
|
+
}
|
|
88
|
+
return { version: 1, docs: entries, postings, totalTerms };
|
|
89
|
+
}
|
|
90
|
+
/**
|
|
91
|
+
* The posting list for `term`, or `undefined` when the index has none.
|
|
92
|
+
*
|
|
93
|
+
* Read through `hasOwnProperty`, never as a plain property access: an index
|
|
94
|
+
* that has been through JSON has an ordinary prototype, so `postings["toString"]`
|
|
95
|
+
* would hand back a FUNCTION and `postings["__proto__"]` an object — and the
|
|
96
|
+
* loop below would then throw "not iterable" out of the tool for any query
|
|
97
|
+
* containing those perfectly ordinary English words. The shape is checked
|
|
98
|
+
* too, so a hand-edited index cannot smuggle a non-pair through the scorer.
|
|
99
|
+
*/
|
|
100
|
+
function postingsFor(index, term) {
|
|
101
|
+
if (!Object.prototype.hasOwnProperty.call(index.postings, term))
|
|
102
|
+
return undefined;
|
|
103
|
+
const postings = index.postings[term];
|
|
104
|
+
if (!Array.isArray(postings) || postings.length === 0)
|
|
105
|
+
return undefined;
|
|
106
|
+
const usable = postings.filter((posting) => Array.isArray(posting) &&
|
|
107
|
+
posting.length === 2 &&
|
|
108
|
+
typeof posting[0] === "number" &&
|
|
109
|
+
typeof posting[1] === "number");
|
|
110
|
+
return usable.length === 0 ? undefined : usable;
|
|
111
|
+
}
|
|
112
|
+
/** Scores are rounded here so the same ranking serialises to the same bytes. */
|
|
113
|
+
function round(value) {
|
|
114
|
+
return Math.round(value * 1e6) / 1e6;
|
|
115
|
+
}
|
|
116
|
+
/**
|
|
117
|
+
* Rank the index's documents against `query`.
|
|
118
|
+
*
|
|
119
|
+
* Ties break on document id, ascending, by plain code-unit comparison — never
|
|
120
|
+
* `localeCompare`, whose ordering is machine-dependent.
|
|
121
|
+
*/
|
|
122
|
+
export function searchIndex(index, query, options = {}) {
|
|
123
|
+
const k1 = options.k1 ?? DEFAULT_K1;
|
|
124
|
+
const b = options.b ?? DEFAULT_B;
|
|
125
|
+
const limit = options.limit ?? 10;
|
|
126
|
+
const minScore = options.minScore ?? 0;
|
|
127
|
+
const terms = queryTerms(query);
|
|
128
|
+
const docCount = index.docs.length;
|
|
129
|
+
const avgLength = docCount === 0 ? 0 : index.totalTerms / docCount;
|
|
130
|
+
const scores = new Map();
|
|
131
|
+
const matchedTerms = new Map();
|
|
132
|
+
const unknownTerms = [];
|
|
133
|
+
for (const term of terms) {
|
|
134
|
+
const postings = postingsFor(index, term);
|
|
135
|
+
if (postings === undefined) {
|
|
136
|
+
unknownTerms.push(term);
|
|
137
|
+
continue;
|
|
138
|
+
}
|
|
139
|
+
const df = postings.length;
|
|
140
|
+
const idf = Math.log(1 + (docCount - df + 0.5) / (df + 0.5));
|
|
141
|
+
for (const [docIndex, tf] of postings) {
|
|
142
|
+
const entry = index.docs[docIndex];
|
|
143
|
+
if (entry === undefined)
|
|
144
|
+
continue; // a posting pointing nowhere: skip it
|
|
145
|
+
const norm = avgLength === 0 ? 1 : 1 - b + (b * entry.length) / avgLength;
|
|
146
|
+
const contribution = (idf * (tf * (k1 + 1))) / (tf + k1 * norm);
|
|
147
|
+
scores.set(docIndex, (scores.get(docIndex) ?? 0) + contribution);
|
|
148
|
+
const seen = matchedTerms.get(docIndex);
|
|
149
|
+
if (seen === undefined)
|
|
150
|
+
matchedTerms.set(docIndex, [term]);
|
|
151
|
+
else
|
|
152
|
+
seen.push(term);
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
const hits = [];
|
|
156
|
+
for (const [docIndex, raw] of scores) {
|
|
157
|
+
const entry = index.docs[docIndex];
|
|
158
|
+
if (entry === undefined)
|
|
159
|
+
continue;
|
|
160
|
+
const score = round(raw);
|
|
161
|
+
if (score < minScore)
|
|
162
|
+
continue;
|
|
163
|
+
hits.push({
|
|
164
|
+
id: entry.id,
|
|
165
|
+
score,
|
|
166
|
+
matchedTerms: (matchedTerms.get(docIndex) ?? []).slice().sort(),
|
|
167
|
+
entry,
|
|
168
|
+
});
|
|
169
|
+
}
|
|
170
|
+
hits.sort((x, y) => (y.score !== x.score ? y.score - x.score : x.id < y.id ? -1 : 1));
|
|
171
|
+
return {
|
|
172
|
+
terms,
|
|
173
|
+
unknownTerms,
|
|
174
|
+
matched: hits.length,
|
|
175
|
+
hits: limit >= 0 ? hits.slice(0, limit) : hits,
|
|
176
|
+
};
|
|
177
|
+
}
|
|
178
|
+
/** Structural check, so a hand-edited or half-written index file is reported. */
|
|
179
|
+
export function isInvertedIndex(value) {
|
|
180
|
+
if (typeof value !== "object" || value === null)
|
|
181
|
+
return false;
|
|
182
|
+
const candidate = value;
|
|
183
|
+
if (candidate["version"] !== 1)
|
|
184
|
+
return false;
|
|
185
|
+
if (typeof candidate["totalTerms"] !== "number")
|
|
186
|
+
return false;
|
|
187
|
+
const docs = candidate["docs"];
|
|
188
|
+
if (!Array.isArray(docs))
|
|
189
|
+
return false;
|
|
190
|
+
for (const doc of docs) {
|
|
191
|
+
if (typeof doc !== "object" || doc === null)
|
|
192
|
+
return false;
|
|
193
|
+
const entry = doc;
|
|
194
|
+
if (typeof entry["id"] !== "string" || typeof entry["length"] !== "number")
|
|
195
|
+
return false;
|
|
196
|
+
}
|
|
197
|
+
const postings = candidate["postings"];
|
|
198
|
+
if (typeof postings !== "object" || postings === null || Array.isArray(postings))
|
|
199
|
+
return false;
|
|
200
|
+
for (const list of Object.values(postings)) {
|
|
201
|
+
if (!Array.isArray(list))
|
|
202
|
+
return false;
|
|
203
|
+
}
|
|
204
|
+
return true;
|
|
205
|
+
}
|
|
206
|
+
function isWordChar(ch) {
|
|
207
|
+
return /[\p{L}\p{N}_]/u.test(ch);
|
|
208
|
+
}
|
|
209
|
+
/**
|
|
210
|
+
* The first whole-word occurrence of any term, as a trimmed window of text.
|
|
211
|
+
*
|
|
212
|
+
* Deliberately simple: the earliest match wins rather than the densest one,
|
|
213
|
+
* so the snippet is cheap and stable. Returns `undefined` when no term occurs
|
|
214
|
+
* in the text (a document can still score on a term it shares with the query
|
|
215
|
+
* only through another field, e.g. its title).
|
|
216
|
+
*/
|
|
217
|
+
export function snippet(text, terms, maxChars) {
|
|
218
|
+
if (maxChars <= 0 || terms.length === 0)
|
|
219
|
+
return undefined;
|
|
220
|
+
const haystack = text.toLowerCase();
|
|
221
|
+
let best = -1;
|
|
222
|
+
for (const term of terms) {
|
|
223
|
+
let from = 0;
|
|
224
|
+
while (from <= haystack.length) {
|
|
225
|
+
const at = haystack.indexOf(term, from);
|
|
226
|
+
if (at === -1)
|
|
227
|
+
break;
|
|
228
|
+
const before = at === 0 ? "" : (haystack[at - 1] ?? "");
|
|
229
|
+
const after = haystack[at + term.length] ?? "";
|
|
230
|
+
if (!isWordChar(before) && !isWordChar(after)) {
|
|
231
|
+
if (best === -1 || at < best)
|
|
232
|
+
best = at;
|
|
233
|
+
break;
|
|
234
|
+
}
|
|
235
|
+
from = at + 1;
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
if (best === -1)
|
|
239
|
+
return undefined;
|
|
240
|
+
const lead = Math.floor(maxChars / 3);
|
|
241
|
+
const start = Math.max(0, best - lead);
|
|
242
|
+
const end = Math.min(text.length, start + maxChars);
|
|
243
|
+
const body = text.slice(start, end).replace(/\s+/g, " ").trim();
|
|
244
|
+
return `${start > 0 ? "…" : ""}${body}${end < text.length ? "…" : ""}`;
|
|
245
|
+
}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
export type JsonlRecord = Record<string, unknown>;
|
|
2
|
+
/** A line that could not be used, and why. */
|
|
3
|
+
export type CorruptLine = {
|
|
4
|
+
readonly line: number;
|
|
5
|
+
readonly reason: string;
|
|
6
|
+
};
|
|
7
|
+
export type JsonlParseResult = {
|
|
8
|
+
readonly records: ReadonlyArray<JsonlRecord>;
|
|
9
|
+
readonly corrupt: ReadonlyArray<CorruptLine>;
|
|
10
|
+
};
|
|
11
|
+
/**
|
|
12
|
+
* Parse JSONL. Blank lines are ignored. A line that is not JSON, or is JSON
|
|
13
|
+
* but not an object, lands in `corrupt` with its 1-based line number; a
|
|
14
|
+
* malformed final line with no trailing newline is additionally called out as
|
|
15
|
+
* truncated, which is the signature of an interrupted write.
|
|
16
|
+
*/
|
|
17
|
+
export declare function parseJsonl(text: string): JsonlParseResult;
|
|
18
|
+
/** Serialise one record as a line, newline included. Never contains a newline. */
|
|
19
|
+
export declare function toJsonlLine(record: JsonlRecord): string;
|
|
20
|
+
export type RecordFilter = {
|
|
21
|
+
readonly sinceSeq?: number;
|
|
22
|
+
readonly untilSeq?: number;
|
|
23
|
+
readonly kind?: string;
|
|
24
|
+
readonly author?: string;
|
|
25
|
+
/** Case-sensitive substring, tested against the record's JSON. */
|
|
26
|
+
readonly contains?: string;
|
|
27
|
+
readonly limit?: number;
|
|
28
|
+
readonly order?: "asc" | "desc";
|
|
29
|
+
};
|
|
30
|
+
export type FilterResult = {
|
|
31
|
+
/** Records that matched, before `limit`. */
|
|
32
|
+
readonly matched: number;
|
|
33
|
+
readonly selected: ReadonlyArray<JsonlRecord>;
|
|
34
|
+
/** True when `limit` cut records off the result. */
|
|
35
|
+
readonly truncated: boolean;
|
|
36
|
+
};
|
|
37
|
+
/**
|
|
38
|
+
* Filter and window a run of records.
|
|
39
|
+
*
|
|
40
|
+
* `sinceSeq` is exclusive and `untilSeq` inclusive, which is what a resuming
|
|
41
|
+
* reader wants: it remembers the last sequence it handled and asks for what
|
|
42
|
+
* came after. With `order: "desc"` the LAST `limit` matches are returned,
|
|
43
|
+
* newest first — the tail of a log, not its head.
|
|
44
|
+
*/
|
|
45
|
+
export declare function filterRecords(records: ReadonlyArray<JsonlRecord>, filter: RecordFilter): FilterResult;
|
|
46
|
+
/** The highest `seq` present, or 0 for an empty run. Used to resume counting. */
|
|
47
|
+
export declare function highestSeq(records: ReadonlyArray<JsonlRecord>): number;
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* JSONL: the append-only line format behind the journal and the blackboard.
|
|
3
|
+
*
|
|
4
|
+
* One JSON object per line, written with `O_APPEND` so that two processes
|
|
5
|
+
* appending at once cannot interleave a partial line into the middle of
|
|
6
|
+
* another. The parser is written for the failure that actually happens —
|
|
7
|
+
* a process killed mid-write leaves a truncated last line — so a bad line is
|
|
8
|
+
* REPORTED with its number and the rest of the file is still returned. A
|
|
9
|
+
* journal that refuses to be read because its tail is half-written is a
|
|
10
|
+
* journal that fails exactly when it is needed.
|
|
11
|
+
*/
|
|
12
|
+
import { isPlainObject } from "./records";
|
|
13
|
+
/**
|
|
14
|
+
* Parse JSONL. Blank lines are ignored. A line that is not JSON, or is JSON
|
|
15
|
+
* but not an object, lands in `corrupt` with its 1-based line number; a
|
|
16
|
+
* malformed final line with no trailing newline is additionally called out as
|
|
17
|
+
* truncated, which is the signature of an interrupted write.
|
|
18
|
+
*/
|
|
19
|
+
export function parseJsonl(text) {
|
|
20
|
+
const records = [];
|
|
21
|
+
const corrupt = [];
|
|
22
|
+
const endsWithNewline = text.length === 0 || text.endsWith("\n");
|
|
23
|
+
const lines = text.split("\n");
|
|
24
|
+
// `split` leaves a trailing "" for a newline-terminated file.
|
|
25
|
+
if (endsWithNewline)
|
|
26
|
+
lines.pop();
|
|
27
|
+
lines.forEach((line, index) => {
|
|
28
|
+
const trimmed = line.trim();
|
|
29
|
+
if (trimmed.length === 0)
|
|
30
|
+
return;
|
|
31
|
+
const isLast = index === lines.length - 1;
|
|
32
|
+
let parsed;
|
|
33
|
+
try {
|
|
34
|
+
parsed = JSON.parse(trimmed);
|
|
35
|
+
}
|
|
36
|
+
catch (err) {
|
|
37
|
+
const truncated = isLast && !endsWithNewline;
|
|
38
|
+
corrupt.push({
|
|
39
|
+
line: index + 1,
|
|
40
|
+
reason: truncated
|
|
41
|
+
? `truncated final line (${err.message}) — most likely a write interrupted part-way`
|
|
42
|
+
: `not valid JSON: ${err.message}`,
|
|
43
|
+
});
|
|
44
|
+
return;
|
|
45
|
+
}
|
|
46
|
+
if (!isPlainObject(parsed)) {
|
|
47
|
+
corrupt.push({ line: index + 1, reason: "line is valid JSON but not an object" });
|
|
48
|
+
return;
|
|
49
|
+
}
|
|
50
|
+
records.push(parsed);
|
|
51
|
+
});
|
|
52
|
+
return { records, corrupt };
|
|
53
|
+
}
|
|
54
|
+
/** Serialise one record as a line, newline included. Never contains a newline. */
|
|
55
|
+
export function toJsonlLine(record) {
|
|
56
|
+
return `${JSON.stringify(record)}\n`;
|
|
57
|
+
}
|
|
58
|
+
function seqOf(record) {
|
|
59
|
+
const seq = record["seq"];
|
|
60
|
+
return typeof seq === "number" ? seq : undefined;
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Filter and window a run of records.
|
|
64
|
+
*
|
|
65
|
+
* `sinceSeq` is exclusive and `untilSeq` inclusive, which is what a resuming
|
|
66
|
+
* reader wants: it remembers the last sequence it handled and asks for what
|
|
67
|
+
* came after. With `order: "desc"` the LAST `limit` matches are returned,
|
|
68
|
+
* newest first — the tail of a log, not its head.
|
|
69
|
+
*/
|
|
70
|
+
export function filterRecords(records, filter) {
|
|
71
|
+
const matches = [];
|
|
72
|
+
for (const record of records) {
|
|
73
|
+
const seq = seqOf(record);
|
|
74
|
+
if (filter.sinceSeq !== undefined && (seq === undefined || seq <= filter.sinceSeq))
|
|
75
|
+
continue;
|
|
76
|
+
if (filter.untilSeq !== undefined && (seq === undefined || seq > filter.untilSeq))
|
|
77
|
+
continue;
|
|
78
|
+
if (filter.kind !== undefined && record["kind"] !== filter.kind)
|
|
79
|
+
continue;
|
|
80
|
+
if (filter.author !== undefined && record["author"] !== filter.author)
|
|
81
|
+
continue;
|
|
82
|
+
if (filter.contains !== undefined && !JSON.stringify(record).includes(filter.contains))
|
|
83
|
+
continue;
|
|
84
|
+
matches.push(record);
|
|
85
|
+
}
|
|
86
|
+
const limit = filter.limit ?? matches.length;
|
|
87
|
+
const truncated = matches.length > limit;
|
|
88
|
+
if (filter.order === "desc") {
|
|
89
|
+
return { matched: matches.length, selected: matches.slice(-limit).reverse(), truncated };
|
|
90
|
+
}
|
|
91
|
+
return { matched: matches.length, selected: matches.slice(0, limit), truncated };
|
|
92
|
+
}
|
|
93
|
+
/** The highest `seq` present, or 0 for an empty run. Used to resume counting. */
|
|
94
|
+
export function highestSeq(records) {
|
|
95
|
+
let highest = 0;
|
|
96
|
+
for (const record of records) {
|
|
97
|
+
const seq = seqOf(record);
|
|
98
|
+
if (seq !== undefined && Number.isFinite(seq) && seq > highest)
|
|
99
|
+
highest = seq;
|
|
100
|
+
}
|
|
101
|
+
return highest;
|
|
102
|
+
}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Longest encoded filename produced. Beyond it the name is truncated and a
|
|
3
|
+
* digest appended, which keeps every filename inside the 255-byte limit every
|
|
4
|
+
* mainstream filesystem shares. A truncated name cannot be decoded, which is
|
|
5
|
+
* why each record also stores its own key in its contents.
|
|
6
|
+
*/
|
|
7
|
+
export declare const MAX_ENCODED_LENGTH = 200;
|
|
8
|
+
/**
|
|
9
|
+
* Percent-encode `raw` into a filename-safe segment.
|
|
10
|
+
*
|
|
11
|
+
* Deterministic: the same input always produces the same filename, including
|
|
12
|
+
* the truncated form, whose digest is taken over the whole original.
|
|
13
|
+
*/
|
|
14
|
+
export declare function encodeSegment(raw: string): string;
|
|
15
|
+
/**
|
|
16
|
+
* Reverse `encodeSegment`, or `undefined` when the segment is a truncated
|
|
17
|
+
* form (it carries a `~`) or is not a well-formed encoding. Callers that may
|
|
18
|
+
* meet a truncated name read the identifier out of the file instead.
|
|
19
|
+
*/
|
|
20
|
+
export declare function decodeSegment(encoded: string): string | undefined;
|
|
21
|
+
/** Longest a NAME may be. Short enough that its encoded form never truncates. */
|
|
22
|
+
export declare const MAX_NAME_LENGTH = 64;
|
|
23
|
+
/** Longest a KEY may be. Longer keys are refused rather than silently cut. */
|
|
24
|
+
export declare const MAX_KEY_LENGTH = 512;
|
|
25
|
+
/**
|
|
26
|
+
* Validate a NAME, returning a readable complaint or `undefined` when it is
|
|
27
|
+
* fine. `label` names the field so the message tells the caller which one.
|
|
28
|
+
*/
|
|
29
|
+
export declare function validateName(label: string, value: string): string | undefined;
|
|
30
|
+
/**
|
|
31
|
+
* Validate a KEY. Anything printable is allowed — keys are external ids — but
|
|
32
|
+
* control characters are refused because they would make the record's own
|
|
33
|
+
* JSON, and every log line quoting it, unreadable.
|
|
34
|
+
*/
|
|
35
|
+
export declare function validateKey(label: string, value: string, max?: number): string | undefined;
|
|
36
|
+
/**
|
|
37
|
+
* Plain lexicographic comparison by UTF-16 code unit — deliberately NOT
|
|
38
|
+
* `localeCompare`, whose ordering depends on the machine's locale and would
|
|
39
|
+
* make the same listing differ between two runs on two laptops.
|
|
40
|
+
*/
|
|
41
|
+
export declare function compareStrings(a: string, b: string): number;
|