@modelprofile.com/flexharness 18.4.0 → 18.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist_ts/00_commitinfo_data.js +1 -1
- package/dist_ts/classes.flexharness.core.js +5 -2
- package/dist_ts/classes.flexharness.prompts.d.ts +2 -0
- package/dist_ts/classes.flexharness.prompts.js +11 -2
- package/dist_ts/errors.d.ts +5 -0
- package/dist_ts/errors.js +10 -1
- package/dist_ts/index.d.ts +1 -0
- package/dist_ts/index.js +2 -1
- package/dist_ts/interfaces.d.ts +6 -0
- package/dist_ts/interfaces.js +1 -1
- package/dist_ts/plugins.d.ts +2 -2
- package/dist_ts/plugins.js +3 -3
- package/dist_ts/utils.json.d.ts +34 -1
- package/dist_ts/utils.json.js +47 -35
- package/dist_ts/utils.projectionindex.d.ts +58 -0
- package/dist_ts/utils.projectionindex.js +534 -0
- package/dist_ts/utils.projectionindextree.d.ts +63 -0
- package/dist_ts/utils.projectionindextree.js +314 -0
- package/dist_ts_stores_nosqldb/classes.nosqldbstores.js +11 -1
- package/dist_ts_stores_nosqldb/classes.recordstore.d.ts +12 -0
- package/dist_ts_stores_nosqldb/classes.recordstore.js +93 -9
- package/package.json +3 -3
- package/readme.md +5 -1
- package/ts/00_commitinfo_data.ts +1 -1
- package/ts/classes.flexharness.core.ts +4 -0
- package/ts/classes.flexharness.prompts.ts +17 -0
- package/ts/errors.ts +14 -0
- package/ts/index.ts +13 -0
- package/ts/interfaces.ts +7 -0
- package/ts/plugins.ts +2 -0
- package/ts/readme.md +5 -1
- package/ts/utils.json.ts +65 -43
- package/ts/utils.projectionindex.ts +693 -0
- package/ts/utils.projectionindextree.ts +328 -0
- package/ts_stores_nosqldb/classes.nosqldbstores.ts +12 -0
- package/ts_stores_nosqldb/classes.recordstore.ts +100 -7
- package/ts_stores_nosqldb/readme.md +2 -0
|
@@ -0,0 +1,328 @@
|
|
|
1
|
+
import * as plugins from './plugins.js';
|
|
2
|
+
import { FlexHarnessStoreFormatError } from './errors.js';
|
|
3
|
+
import { assertJsonSerializable } from './utils.json.js';
|
|
4
|
+
|
|
5
|
+
/** A content address is the SHA-256 of the canonical JSON node, including its domain tag. */
|
|
6
|
+
export interface IProjectionIndexNodeWrite {
|
|
7
|
+
readonly hash: string;
|
|
8
|
+
readonly node: IProjectionIndexTreeNode;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export type TProjectionIndexTree =
|
|
12
|
+
| 'messages'
|
|
13
|
+
| 'messageIds'
|
|
14
|
+
| 'runs'
|
|
15
|
+
| 'staged'
|
|
16
|
+
| 'segments'
|
|
17
|
+
| 'segmentRuns'
|
|
18
|
+
| 'events'
|
|
19
|
+
| 'captures';
|
|
20
|
+
|
|
21
|
+
export interface IProjectionIndexTreeNode {
|
|
22
|
+
readonly kind: 'projection-index-tree-v1';
|
|
23
|
+
readonly tree: TProjectionIndexTree;
|
|
24
|
+
readonly key: string;
|
|
25
|
+
readonly value: unknown;
|
|
26
|
+
readonly left?: string;
|
|
27
|
+
readonly right?: string;
|
|
28
|
+
readonly height: number;
|
|
29
|
+
readonly size: number;
|
|
30
|
+
/** Completed segment count, populated only by the ordered segment tree. */
|
|
31
|
+
readonly completed: number;
|
|
32
|
+
readonly capturing: number;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export interface IProjectionIndexTreeReader {
|
|
36
|
+
readNode(hash: string): Promise<unknown>;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function canonicalJson(value: unknown): string {
|
|
40
|
+
if (Array.isArray(value)) return `[${value.map(canonicalJson).join(',')}]`;
|
|
41
|
+
if (value !== null && typeof value === 'object') {
|
|
42
|
+
const record = value as Record<string, unknown>;
|
|
43
|
+
return `{${Object.keys(record).sort().map((key) =>
|
|
44
|
+
`${JSON.stringify(key)}:${canonicalJson(record[key])}`).join(',')}}`;
|
|
45
|
+
}
|
|
46
|
+
const primitive = JSON.stringify(value);
|
|
47
|
+
if (primitive === undefined) throw new FlexHarnessStoreFormatError('Projection index node is not JSON.');
|
|
48
|
+
return primitive;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export function projectionIndexHash(node: unknown): string {
|
|
52
|
+
assertJsonSerializable(node, '$indexNode');
|
|
53
|
+
return plugins.crypto.createHash('sha256').update(canonicalJson(node)).digest('hex');
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function assertHash(hash: string): void {
|
|
57
|
+
if (!/^[a-f0-9]{64}$/.test(hash)) {
|
|
58
|
+
throw new FlexHarnessStoreFormatError('Projection index node hash is invalid.');
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
63
|
+
return value !== null && typeof value === 'object' && !Array.isArray(value);
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
function assertTreeNode(value: unknown, tree: TProjectionIndexTree): asserts value is IProjectionIndexTreeNode {
|
|
67
|
+
if (!isRecord(value) || value.kind !== 'projection-index-tree-v1' || value.tree !== tree
|
|
68
|
+
|| typeof value.key !== 'string' || !Object.hasOwn(value, 'value')
|
|
69
|
+
|| !Number.isSafeInteger(value.height) || Number(value.height) < 1
|
|
70
|
+
|| !Number.isSafeInteger(value.size) || Number(value.size) < 1
|
|
71
|
+
|| !Number.isSafeInteger(value.completed) || Number(value.completed) < 0
|
|
72
|
+
|| !Number.isSafeInteger(value.capturing) || Number(value.capturing) < 0
|
|
73
|
+
|| (value.left !== undefined && (typeof value.left !== 'string' || !/^[a-f0-9]{64}$/.test(value.left)))
|
|
74
|
+
|| (value.right !== undefined && (typeof value.right !== 'string' || !/^[a-f0-9]{64}$/.test(value.right)))) {
|
|
75
|
+
throw new FlexHarnessStoreFormatError('Projection index tree node is malformed.');
|
|
76
|
+
}
|
|
77
|
+
const keys = Object.keys(value);
|
|
78
|
+
if (keys.some((key) => !['kind', 'tree', 'key', 'value', 'left', 'right', 'height', 'size', 'completed', 'capturing'].includes(key))) {
|
|
79
|
+
throw new FlexHarnessStoreFormatError('Projection index tree node has an unknown field.');
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/** One invocation's authenticated, immutable AVL nodes and staged replacements. */
|
|
84
|
+
export class ProjectionIndexTree {
|
|
85
|
+
private readonly cached = new Map<string, IProjectionIndexTreeNode>();
|
|
86
|
+
private readonly staged = new Map<string, IProjectionIndexTreeNode>();
|
|
87
|
+
|
|
88
|
+
public constructor(
|
|
89
|
+
private readonly tree: TProjectionIndexTree,
|
|
90
|
+
private readonly reader?: IProjectionIndexTreeReader,
|
|
91
|
+
) {}
|
|
92
|
+
|
|
93
|
+
private async read(hash: string | undefined): Promise<IProjectionIndexTreeNode | undefined> {
|
|
94
|
+
if (hash === undefined) return undefined;
|
|
95
|
+
assertHash(hash);
|
|
96
|
+
const cached = this.cached.get(hash);
|
|
97
|
+
if (cached) return cached;
|
|
98
|
+
if (!this.reader) throw new FlexHarnessStoreFormatError('Projection index node is missing.');
|
|
99
|
+
const value = await this.reader.readNode(hash);
|
|
100
|
+
assertTreeNode(value, this.tree);
|
|
101
|
+
if (projectionIndexHash(value) !== hash) {
|
|
102
|
+
throw new FlexHarnessStoreFormatError('Projection index node digest does not match its address.');
|
|
103
|
+
}
|
|
104
|
+
this.cached.set(hash, value);
|
|
105
|
+
return value;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
private async write(
|
|
109
|
+
key: string,
|
|
110
|
+
value: unknown,
|
|
111
|
+
left: string | undefined,
|
|
112
|
+
right: string | undefined,
|
|
113
|
+
): Promise<string> {
|
|
114
|
+
const [leftNode, rightNode] = await Promise.all([this.read(left), this.read(right)]);
|
|
115
|
+
const node: IProjectionIndexTreeNode = {
|
|
116
|
+
kind: 'projection-index-tree-v1',
|
|
117
|
+
tree: this.tree,
|
|
118
|
+
key,
|
|
119
|
+
value,
|
|
120
|
+
...(left === undefined ? {} : { left }),
|
|
121
|
+
...(right === undefined ? {} : { right }),
|
|
122
|
+
height: 1 + Math.max(leftNode?.height ?? 0, rightNode?.height ?? 0),
|
|
123
|
+
size: 1 + (leftNode?.size ?? 0) + (rightNode?.size ?? 0),
|
|
124
|
+
completed: (this.tree === 'segments' && isRecord(value) && value.status === 'completed' ? 1 : 0)
|
|
125
|
+
+ (leftNode?.completed ?? 0) + (rightNode?.completed ?? 0),
|
|
126
|
+
capturing: (this.tree === 'segments' && isRecord(value) && value.status === 'capturing' && value.captureId !== undefined ? 1 : 0)
|
|
127
|
+
+ (leftNode?.capturing ?? 0) + (rightNode?.capturing ?? 0),
|
|
128
|
+
};
|
|
129
|
+
const hash = projectionIndexHash(node);
|
|
130
|
+
this.cached.set(hash, node);
|
|
131
|
+
this.staged.set(hash, node);
|
|
132
|
+
return hash;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
private async rotateLeft(hash: string): Promise<string> {
|
|
136
|
+
const node = await this.read(hash);
|
|
137
|
+
const right = await this.read(node?.right);
|
|
138
|
+
if (!node || !right) throw new FlexHarnessStoreFormatError('Projection index rotation is invalid.');
|
|
139
|
+
const newLeft = await this.write(node.key, node.value, node.left, right.left);
|
|
140
|
+
return this.write(right.key, right.value, newLeft, right.right);
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
private async rotateRight(hash: string): Promise<string> {
|
|
144
|
+
const node = await this.read(hash);
|
|
145
|
+
const left = await this.read(node?.left);
|
|
146
|
+
if (!node || !left) throw new FlexHarnessStoreFormatError('Projection index rotation is invalid.');
|
|
147
|
+
const newRight = await this.write(node.key, node.value, left.right, node.right);
|
|
148
|
+
return this.write(left.key, left.value, left.left, newRight);
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
private async balance(hash: string): Promise<string> {
|
|
152
|
+
const node = await this.read(hash);
|
|
153
|
+
if (!node) throw new FlexHarnessStoreFormatError('Projection index node is missing.');
|
|
154
|
+
const [left, right] = await Promise.all([this.read(node.left), this.read(node.right)]);
|
|
155
|
+
const difference = (left?.height ?? 0) - (right?.height ?? 0);
|
|
156
|
+
if (difference > 1) {
|
|
157
|
+
if (!left) throw new FlexHarnessStoreFormatError('Projection index left branch is missing.');
|
|
158
|
+
const [inner, outer] = await Promise.all([this.read(left.right), this.read(left.left)]);
|
|
159
|
+
const leftHash = (inner?.height ?? 0) > (outer?.height ?? 0)
|
|
160
|
+
? await this.rotateLeft(node.left as string) : node.left as string;
|
|
161
|
+
return this.rotateRight(await this.write(node.key, node.value, leftHash, node.right));
|
|
162
|
+
}
|
|
163
|
+
if (difference < -1) {
|
|
164
|
+
if (!right) throw new FlexHarnessStoreFormatError('Projection index right branch is missing.');
|
|
165
|
+
const [inner, outer] = await Promise.all([this.read(right.left), this.read(right.right)]);
|
|
166
|
+
const rightHash = (inner?.height ?? 0) > (outer?.height ?? 0)
|
|
167
|
+
? await this.rotateRight(node.right as string) : node.right as string;
|
|
168
|
+
return this.rotateLeft(await this.write(node.key, node.value, node.left, rightHash));
|
|
169
|
+
}
|
|
170
|
+
return hash;
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
public async get(root: string | undefined, key: string): Promise<unknown> {
|
|
174
|
+
let cursor = root;
|
|
175
|
+
while (cursor !== undefined) {
|
|
176
|
+
const node = await this.read(cursor);
|
|
177
|
+
if (!node) break;
|
|
178
|
+
if (key === node.key) return node.value;
|
|
179
|
+
cursor = key < node.key ? node.left : node.right;
|
|
180
|
+
}
|
|
181
|
+
return undefined;
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
public async set(root: string | undefined, key: string, value: unknown): Promise<string> {
|
|
185
|
+
if (root === undefined) return this.write(key, value, undefined, undefined);
|
|
186
|
+
const node = await this.read(root);
|
|
187
|
+
if (!node) throw new FlexHarnessStoreFormatError('Projection index node is missing.');
|
|
188
|
+
if (key === node.key) return this.write(key, value, node.left, node.right);
|
|
189
|
+
const left = key < node.key ? await this.set(node.left, key, value) : node.left;
|
|
190
|
+
const right = key > node.key ? await this.set(node.right, key, value) : node.right;
|
|
191
|
+
return this.balance(await this.write(node.key, node.value, left, right));
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
/** Builds a canonical balanced tree in linear work from entries sorted by unique key. */
|
|
195
|
+
public async build(entries: readonly { key: string; value: unknown }[]): Promise<string | undefined> {
|
|
196
|
+
for (let index = 1; index < entries.length; index++) {
|
|
197
|
+
if (entries[index - 1].key >= entries[index].key) {
|
|
198
|
+
throw new FlexHarnessStoreFormatError('Projection index bootstrap keys must be unique and sorted.');
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
const visit = async (start: number, end: number): Promise<string | undefined> => {
|
|
202
|
+
if (start === end) return undefined;
|
|
203
|
+
const middle = Math.floor((start + end) / 2);
|
|
204
|
+
const left = await visit(start, middle);
|
|
205
|
+
const right = await visit(middle + 1, end);
|
|
206
|
+
return this.write(entries[middle].key, entries[middle].value, left, right);
|
|
207
|
+
};
|
|
208
|
+
return visit(0, entries.length);
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
private async removeMin(root: string): Promise<{ root: string | undefined; node: IProjectionIndexTreeNode }> {
|
|
212
|
+
const node = await this.read(root);
|
|
213
|
+
if (!node) throw new FlexHarnessStoreFormatError('Projection index node is missing.');
|
|
214
|
+
if (!node.left) return { root: node.right, node };
|
|
215
|
+
const result = await this.removeMin(node.left);
|
|
216
|
+
return {
|
|
217
|
+
root: await this.balance(await this.write(node.key, node.value, result.root, node.right)),
|
|
218
|
+
node: result.node,
|
|
219
|
+
};
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
public async delete(root: string | undefined, key: string): Promise<string | undefined> {
|
|
223
|
+
if (root === undefined) return undefined;
|
|
224
|
+
const node = await this.read(root);
|
|
225
|
+
if (!node) throw new FlexHarnessStoreFormatError('Projection index node is missing.');
|
|
226
|
+
if (key < node.key) {
|
|
227
|
+
if (!node.left) return root;
|
|
228
|
+
return this.balance(await this.write(node.key, node.value, await this.delete(node.left, key), node.right));
|
|
229
|
+
}
|
|
230
|
+
if (key > node.key) {
|
|
231
|
+
if (!node.right) return root;
|
|
232
|
+
return this.balance(await this.write(node.key, node.value, node.left, await this.delete(node.right, key)));
|
|
233
|
+
}
|
|
234
|
+
if (!node.left) return node.right;
|
|
235
|
+
if (!node.right) return node.left;
|
|
236
|
+
const next = await this.removeMin(node.right);
|
|
237
|
+
return this.balance(await this.write(next.node.key, next.node.value, node.left, next.root));
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
public async size(root: string | undefined): Promise<number> {
|
|
241
|
+
return (await this.read(root))?.size ?? 0;
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
public async completed(root: string | undefined): Promise<number> {
|
|
245
|
+
return (await this.read(root))?.completed ?? 0;
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
public async capturing(root: string | undefined): Promise<number> {
|
|
249
|
+
return (await this.read(root))?.capturing ?? 0;
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
public async select(root: string | undefined, index: number): Promise<{ key: string; value: unknown } | undefined> {
|
|
253
|
+
let cursor = root;
|
|
254
|
+
let offset = index;
|
|
255
|
+
while (cursor !== undefined) {
|
|
256
|
+
const node = await this.read(cursor);
|
|
257
|
+
if (!node) break;
|
|
258
|
+
const leftSize = (await this.read(node.left))?.size ?? 0;
|
|
259
|
+
if (offset === leftSize) return { key: node.key, value: node.value };
|
|
260
|
+
if (offset < leftSize) cursor = node.left;
|
|
261
|
+
else { offset -= leftSize + 1; cursor = node.right; }
|
|
262
|
+
}
|
|
263
|
+
return undefined;
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
public async rank(root: string | undefined, key: string): Promise<number | undefined> {
|
|
267
|
+
let cursor = root;
|
|
268
|
+
let before = 0;
|
|
269
|
+
while (cursor !== undefined) {
|
|
270
|
+
const node = await this.read(cursor);
|
|
271
|
+
if (!node) break;
|
|
272
|
+
const leftSize = (await this.read(node.left))?.size ?? 0;
|
|
273
|
+
if (key === node.key) return before + leftSize;
|
|
274
|
+
if (key < node.key) cursor = node.left;
|
|
275
|
+
else { before += leftSize + 1; cursor = node.right; }
|
|
276
|
+
}
|
|
277
|
+
return undefined;
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
public async selectCompleted(root: string | undefined, index: number): Promise<{ key: string; value: unknown } | undefined> {
|
|
281
|
+
let cursor = root;
|
|
282
|
+
let offset = index;
|
|
283
|
+
while (cursor !== undefined) {
|
|
284
|
+
const node = await this.read(cursor);
|
|
285
|
+
if (!node) break;
|
|
286
|
+
const leftCompleted = (await this.read(node.left))?.completed ?? 0;
|
|
287
|
+
if (offset < leftCompleted) { cursor = node.left; continue; }
|
|
288
|
+
offset -= leftCompleted;
|
|
289
|
+
const self = this.tree === 'segments' && isRecord(node.value) && node.value.status === 'completed';
|
|
290
|
+
if (self && offset === 0) return { key: node.key, value: node.value };
|
|
291
|
+
if (self) offset--;
|
|
292
|
+
cursor = node.right;
|
|
293
|
+
}
|
|
294
|
+
return undefined;
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
public async range(root: string | undefined, start: number, end: number): Promise<Array<{ key: string; value: unknown }>> {
|
|
298
|
+
const result: Array<{ key: string; value: unknown }> = [];
|
|
299
|
+
const visit = async (hash: string | undefined, base: number): Promise<void> => {
|
|
300
|
+
const node = await this.read(hash);
|
|
301
|
+
if (!node) return;
|
|
302
|
+
const leftSize = (await this.read(node.left))?.size ?? 0;
|
|
303
|
+
const position = base + leftSize;
|
|
304
|
+
if (start < position) await visit(node.left, base);
|
|
305
|
+
if (start <= position && position < end) result.push({ key: node.key, value: node.value });
|
|
306
|
+
if (position + 1 < end) await visit(node.right, position + 1);
|
|
307
|
+
};
|
|
308
|
+
await visit(root, 0);
|
|
309
|
+
return result;
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
/** Only nodes reachable from a returned root need to be persisted. */
|
|
313
|
+
public writes(root: string | undefined): IProjectionIndexNodeWrite[] {
|
|
314
|
+
const result: IProjectionIndexNodeWrite[] = [];
|
|
315
|
+
const visited = new Set<string>();
|
|
316
|
+
const visit = (hash: string | undefined): void => {
|
|
317
|
+
if (hash === undefined || visited.has(hash)) return;
|
|
318
|
+
visited.add(hash);
|
|
319
|
+
const node = this.staged.get(hash);
|
|
320
|
+
if (!node) return;
|
|
321
|
+
result.push({ hash, node });
|
|
322
|
+
visit(node.left);
|
|
323
|
+
visit(node.right);
|
|
324
|
+
};
|
|
325
|
+
visit(root);
|
|
326
|
+
return result;
|
|
327
|
+
}
|
|
328
|
+
}
|
|
@@ -193,6 +193,11 @@ class NoSqlProjectionStore implements IFlexProjectionStore {
|
|
|
193
193
|
return this.records.readLog({ domain: 'projection', storageKey, sessionId }, this.log(sessionId));
|
|
194
194
|
}
|
|
195
195
|
|
|
196
|
+
public loadHead(storageKey: string, sessionId: string) {
|
|
197
|
+
providerKey(storageKey, sessionId);
|
|
198
|
+
return this.records.readLogHead({ domain: 'projection', storageKey, sessionId });
|
|
199
|
+
}
|
|
200
|
+
|
|
196
201
|
public async save(
|
|
197
202
|
storageKey: string,
|
|
198
203
|
sessionId: string,
|
|
@@ -344,6 +349,13 @@ class NoSqlAgentEventStore implements plugins.IAgentEventStore {
|
|
|
344
349
|
);
|
|
345
350
|
}
|
|
346
351
|
|
|
352
|
+
public loadHead(sessionId: string) {
|
|
353
|
+
this.assertSession(sessionId);
|
|
354
|
+
return this.records.readLogHead({
|
|
355
|
+
domain: 'agentEvents', storageKey: this.storageKey, sessionId,
|
|
356
|
+
});
|
|
357
|
+
}
|
|
358
|
+
|
|
347
359
|
public async save(
|
|
348
360
|
sessionId: string,
|
|
349
361
|
change: plugins.IAgentEventChange,
|
|
@@ -128,6 +128,11 @@ export class FlexRecordDocument extends plugins.SmartDataDbDoc<FlexRecordDocumen
|
|
|
128
128
|
public blobId!: string | null;
|
|
129
129
|
public chunkCount!: number;
|
|
130
130
|
public payloadSha256!: string | null;
|
|
131
|
+
/** Hash of the log base and every committed change, updated with the head CAS. */
|
|
132
|
+
public contentRoot?: string | null;
|
|
133
|
+
/** The exact head revision and token whose content the root commits. */
|
|
134
|
+
public contentRootRevision?: number;
|
|
135
|
+
public contentRootWriteToken?: string;
|
|
131
136
|
/** Set before the first chunk of this record is written; the record's writes then sweep. */
|
|
132
137
|
public chunked!: boolean;
|
|
133
138
|
public sweepCount!: number;
|
|
@@ -222,6 +227,9 @@ export function bindFlexRecordModels(db: plugins.SmartdataDb): IFlexRecordModels
|
|
|
222
227
|
'blobId',
|
|
223
228
|
'chunkCount',
|
|
224
229
|
'payloadSha256',
|
|
230
|
+
'contentRoot',
|
|
231
|
+
'contentRootRevision',
|
|
232
|
+
'contentRootWriteToken',
|
|
225
233
|
'chunked',
|
|
226
234
|
'sweepCount',
|
|
227
235
|
'logId',
|
|
@@ -232,7 +240,7 @@ export function bindFlexRecordModels(db: plugins.SmartdataDb): IFlexRecordModels
|
|
|
232
240
|
'baseLength',
|
|
233
241
|
'retiredLogId',
|
|
234
242
|
],
|
|
235
|
-
numericFields: ['revision', 'chunkCount', 'sweepCount', 'changeCount', 'changeLength', 'baseLength'],
|
|
243
|
+
numericFields: ['revision', 'contentRootRevision', 'chunkCount', 'sweepCount', 'changeCount', 'changeLength', 'baseLength'],
|
|
236
244
|
identityFields: ['recordId'],
|
|
237
245
|
indexes: [
|
|
238
246
|
{ name: 'flexharness_record_id', key: { recordId: 1 }, options: { unique: true } },
|
|
@@ -314,6 +322,12 @@ interface IStoredLog<TSnapshot> {
|
|
|
314
322
|
value?: TSnapshot;
|
|
315
323
|
}
|
|
316
324
|
|
|
325
|
+
/** A content commitment read from one bounded log head, without its base or change pages. */
|
|
326
|
+
export interface IFlexRecordLogHead {
|
|
327
|
+
revision: number;
|
|
328
|
+
contentRoot: string;
|
|
329
|
+
}
|
|
330
|
+
|
|
317
331
|
type TDeletionSelector =
|
|
318
332
|
| { recordId: string; domain: TFlexRecordDomain }
|
|
319
333
|
| { ownerKey: string; domain: TFlexRecordDomain }
|
|
@@ -326,6 +340,31 @@ const deletionBatchSize = 4;
|
|
|
326
340
|
const digest = (value: string): string =>
|
|
327
341
|
plugins.crypto.createHash('sha256').update(value, 'utf8').digest('hex');
|
|
328
342
|
|
|
343
|
+
const logBaseRoot = (text: string): string => digest(JSON.stringify(['flex-record-log-v1', 'base', digest(text)]));
|
|
344
|
+
const logChangeRoot = (root: string, text: string): string =>
|
|
345
|
+
digest(JSON.stringify(['flex-record-log-v1', 'change', root, digest(text)]));
|
|
346
|
+
|
|
347
|
+
/** An older writer may advance a head without knowing these fields; then the root is unqualified. */
|
|
348
|
+
function committedLogRoot(head: FlexRecordDocument, resolved: IResolvedAddress): string | undefined {
|
|
349
|
+
const root = head.contentRoot;
|
|
350
|
+
const revision = head.contentRootRevision;
|
|
351
|
+
const token = head.contentRootWriteToken;
|
|
352
|
+
if (root === undefined || root === null) {
|
|
353
|
+
if (revision !== undefined || token !== undefined) {
|
|
354
|
+
throw new FlexHarnessStoreFormatError(`Incomplete log content commitment at ${resolved.path}.`);
|
|
355
|
+
}
|
|
356
|
+
return undefined;
|
|
357
|
+
}
|
|
358
|
+
if (typeof root !== 'string' || !/^[a-f0-9]{64}$/.test(root)
|
|
359
|
+
|| !Number.isSafeInteger(revision) || Number(revision) < 1
|
|
360
|
+
|| typeof token !== 'string' || token.length === 0
|
|
361
|
+
|| !Number.isSafeInteger(head.revision) || head.revision < 1
|
|
362
|
+
|| typeof head.writeToken !== 'string' || head.writeToken.length === 0) {
|
|
363
|
+
throw new FlexHarnessStoreFormatError(`Invalid log content commitment at ${resolved.path}.`);
|
|
364
|
+
}
|
|
365
|
+
return revision === head.revision && token === head.writeToken ? root : undefined;
|
|
366
|
+
}
|
|
367
|
+
|
|
329
368
|
function splitPayload(text: string): string[] {
|
|
330
369
|
const chunks: string[] = [];
|
|
331
370
|
let start = 0;
|
|
@@ -511,6 +550,33 @@ export class FlexRecordStore {
|
|
|
511
550
|
return this.readVisible(resolved, async () => (await this.loadLog(resolved, log)).value);
|
|
512
551
|
}
|
|
513
552
|
|
|
553
|
+
/** A missing or legacy commitment requires the caller to take the complete log path. */
|
|
554
|
+
public async readLogHead(address: IFlexRecordAddress): Promise<IFlexRecordLogHead | undefined> {
|
|
555
|
+
await this.prepare();
|
|
556
|
+
const resolved = this.resolve(address);
|
|
557
|
+
return this.readVisible(resolved, async () => {
|
|
558
|
+
const head = await this.findHead(resolved);
|
|
559
|
+
if (!head) return undefined;
|
|
560
|
+
const root = committedLogRoot(head, resolved);
|
|
561
|
+
if (head.contentRoot === undefined || head.contentRoot === null) return undefined;
|
|
562
|
+
if (!Number.isSafeInteger(head.revision) || head.revision < 1
|
|
563
|
+
|| typeof head.logId !== 'string' || !head.logId
|
|
564
|
+
|| typeof head.payloadSha256 !== 'string' || !/^[a-f0-9]{64}$/.test(head.payloadSha256)
|
|
565
|
+
|| typeof head.blobId !== 'string' || !head.blobId
|
|
566
|
+
|| head.payload !== null || head.chunked !== true
|
|
567
|
+
|| !Number.isSafeInteger(head.chunkCount) || head.chunkCount < 1
|
|
568
|
+
|| !Number.isSafeInteger(head.changeCount) || head.changeCount < 0
|
|
569
|
+
|| !Array.isArray(head.recentChanges)
|
|
570
|
+
|| head.recentChanges.some((change) => typeof change !== 'string')
|
|
571
|
+
|| head.changeCount < head.recentChanges.length
|
|
572
|
+
|| head.recentChanges.reduce((length, change) => length + change.length, 0)
|
|
573
|
+
> FLEX_RECORD_INLINE_CHANGE_CODE_UNITS) {
|
|
574
|
+
throw new FlexHarnessStoreFormatError(`Invalid content commitment at ${resolved.path}.`);
|
|
575
|
+
}
|
|
576
|
+
return root === undefined ? undefined : { revision: head.revision, contentRoot: root };
|
|
577
|
+
});
|
|
578
|
+
}
|
|
579
|
+
|
|
514
580
|
/**
|
|
515
581
|
* Saves a change to a log record while its head holds the expected revision. Only the head is
|
|
516
582
|
* read when the record takes the change; a compaction loads the record and checks the result in
|
|
@@ -532,8 +598,9 @@ export class FlexRecordStore {
|
|
|
532
598
|
const head = await this.findHead(resolved);
|
|
533
599
|
actualRevision = head?.revision ?? 0;
|
|
534
600
|
if (actualRevision !== change.expectedRevision) throw change.conflict(actualRevision);
|
|
535
|
-
|
|
536
|
-
|
|
601
|
+
const root = head ? committedLogRoot(head, resolved) : undefined;
|
|
602
|
+
if (head && root && this.takesChange(head, text)) {
|
|
603
|
+
if (await this.commitChange(resolved, head, revision, text, root)) return;
|
|
537
604
|
continue;
|
|
538
605
|
}
|
|
539
606
|
const stored = await this.loadLog(resolved, log);
|
|
@@ -881,10 +948,29 @@ export class FlexRecordStore {
|
|
|
881
948
|
for (let attempt = 0; attempt < chunkReadAttempts; attempt++) {
|
|
882
949
|
const head = await this.readHead(resolved);
|
|
883
950
|
const document = head.document;
|
|
884
|
-
if (!document
|
|
885
|
-
|
|
951
|
+
if (!document) return {};
|
|
952
|
+
const root = committedLogRoot(document, resolved);
|
|
953
|
+
if (head.text === undefined) {
|
|
954
|
+
if (document.contentRoot !== undefined && document.contentRoot !== null) {
|
|
955
|
+
throw new FlexHarnessStoreFormatError(`${resolved.path} has a log commitment without a base.`);
|
|
956
|
+
}
|
|
957
|
+
return { document };
|
|
958
|
+
}
|
|
959
|
+
if (!document.logId) {
|
|
960
|
+
if (document.contentRoot !== undefined && document.contentRoot !== null) {
|
|
961
|
+
throw new FlexHarnessStoreFormatError(`${resolved.path} has a log commitment without a log.`);
|
|
962
|
+
}
|
|
963
|
+
return { document, value: this.parse(resolved, head.text, log.validate) };
|
|
964
|
+
}
|
|
886
965
|
const changes = await this.readChanges(document);
|
|
887
966
|
if (changes) {
|
|
967
|
+
if (root !== undefined) {
|
|
968
|
+
let actualRoot = logBaseRoot(head.text);
|
|
969
|
+
for (const change of changes) actualRoot = logChangeRoot(actualRoot, change);
|
|
970
|
+
if (actualRoot !== root) {
|
|
971
|
+
throw new FlexHarnessStoreFormatError(`${resolved.path} has a mismatched log content commitment.`);
|
|
972
|
+
}
|
|
973
|
+
}
|
|
888
974
|
return { document, value: this.fold(resolved, document, head.text, changes, log) };
|
|
889
975
|
}
|
|
890
976
|
const latest = await this.findHead(resolved);
|
|
@@ -1007,6 +1093,8 @@ export class FlexRecordStore {
|
|
|
1007
1093
|
blobId,
|
|
1008
1094
|
chunkCount: chunked ? splitPayload(text).length : 0,
|
|
1009
1095
|
payloadSha256: chunked ? digest(text) : null,
|
|
1096
|
+
...(log ? { contentRoot: logBaseRoot(text) } : {}),
|
|
1097
|
+
...(log ? { contentRootRevision: write.revision, contentRootWriteToken: writeToken } : {}),
|
|
1010
1098
|
logId: log ? writeToken : null,
|
|
1011
1099
|
changeCount: 0,
|
|
1012
1100
|
recentChanges: [],
|
|
@@ -1015,6 +1103,7 @@ export class FlexRecordStore {
|
|
|
1015
1103
|
baseLength: log ? text.length : 0,
|
|
1016
1104
|
retiredLogId: current.logId ?? null,
|
|
1017
1105
|
},
|
|
1106
|
+
...(log ? {} : { $unset: { contentRoot: '', contentRootRevision: '', contentRootWriteToken: '' } }),
|
|
1018
1107
|
},
|
|
1019
1108
|
{ session: transaction },
|
|
1020
1109
|
));
|
|
@@ -1046,6 +1135,7 @@ export class FlexRecordStore {
|
|
|
1046
1135
|
head: FlexRecordDocument,
|
|
1047
1136
|
revision: number,
|
|
1048
1137
|
text: string,
|
|
1138
|
+
root: string,
|
|
1049
1139
|
): Promise<boolean> {
|
|
1050
1140
|
const recentLength = head.recentChanges.reduce((length, data) => length + data.length, 0);
|
|
1051
1141
|
const page = recentLength + text.length > FLEX_RECORD_INLINE_CHANGE_CODE_UNITS
|
|
@@ -1067,8 +1157,8 @@ export class FlexRecordStore {
|
|
|
1067
1157
|
const updated = await this.records.atomicUpdate(
|
|
1068
1158
|
{ recordId: resolved.recordId, writeToken: head.writeToken },
|
|
1069
1159
|
page
|
|
1070
|
-
? { $set: { revision, writeToken, recentChanges: [], lastPageId: page.pageId }, $inc: counts }
|
|
1071
|
-
: { $set: { revision, writeToken }, $inc: counts, $push: { recentChanges: text } },
|
|
1160
|
+
? { $set: { revision, writeToken, contentRoot: logChangeRoot(root, text), contentRootRevision: revision, contentRootWriteToken: writeToken, recentChanges: [], lastPageId: page.pageId }, $inc: counts }
|
|
1161
|
+
: { $set: { revision, writeToken, contentRoot: logChangeRoot(root, text), contentRootRevision: revision, contentRootWriteToken: writeToken }, $inc: counts, $push: { recentChanges: text } },
|
|
1072
1162
|
{ session: transaction },
|
|
1073
1163
|
);
|
|
1074
1164
|
if (updated.matchedCount === 1 && page) await this.changes.insert(page, { session: transaction });
|
|
@@ -1119,6 +1209,9 @@ export class FlexRecordStore {
|
|
|
1119
1209
|
head.blobId = chunk.blobId;
|
|
1120
1210
|
head.chunkCount = 1;
|
|
1121
1211
|
head.payloadSha256 = digest(text);
|
|
1212
|
+
head.contentRoot = logBaseRoot(text);
|
|
1213
|
+
head.contentRootRevision = revision;
|
|
1214
|
+
head.contentRootWriteToken = head.writeToken;
|
|
1122
1215
|
head.logId = head.writeToken;
|
|
1123
1216
|
head.baseLength = text.length;
|
|
1124
1217
|
} else {
|
|
@@ -56,6 +56,8 @@ All writes in one namespace now contend on its namespace document, not only crea
|
|
|
56
56
|
|
|
57
57
|
A projection and an Agent event history are each stored as a base plus the changes saved since. FlexHarness saves a projection as an `IFlexProjectionChange` and an Agent event history as an `IAgentEventChange`, which carry only the entries that changed, and the store appends the change to the record's head in the compare-and-swap that commits it, so a save reads and writes the head alone and costs the change rather than the whole session. The head holds the latest changes up to 16 Ki characters of JSON; the save that would exceed that first writes them, with its own change, as one page in `flexharness_changes`, and empties them in its swap. The base lives in chunks whatever its length, so the head each change swaps holds no payload; a session's first save creates the record together with a base of up to one chunk in one transaction. The changes are kept while together they stay no longer than the base, or than 64 Ki characters of JSON while the base is shorter, and no single change is longer than one chunk. The save that would exceed that compacts the record instead: it reads the record, applies the change, checks the resulting snapshot in full and writes it as the new base. The earlier base and its pages are removed like unreferenced chunks, by the record's next compaction, which finds those pages by their log instead of scanning the current one. A record therefore holds its base and changes no longer than it or that minimum, plus the earlier base and its pages until the next compaction. A session whose history shrank, after an undo branch for example, keeps its larger base until its changes outgrow it. A read applies every change to the base and checks the result in full, so it costs the base and every change since, and a record whose pages are incomplete fails with `FlexHarnessStoreFormatError`. A page a crashed writer left behind without swapping the head onto it is removed with the rest of its log, or by the record's deletion.
|
|
58
58
|
|
|
59
|
+
The projection store and each Agent event store also expose `loadHead`, an optional bounded read of `{ revision, contentRoot }`. A rooted log's head commits to its serialized base and ordered changes; each append advances that SHA-256 root and binds it to the new revision and write token in the head CAS. A full read verifies a current root. Missing and older unrooted logs return no head. An older writer that advances a rooted head but preserves fields it does not know leaves a well-formed stale binding: `loadHead` returns no root, and the next save validates and rewrites the whole log. Malformed commitment fields fail closed. This commitment does not certify a repaired session or replace recovery evidence.
|
|
60
|
+
|
|
59
61
|
A save of a scope, permission or job snapshot checks its expected revision against the record's head alone: it never reads the stored snapshot or its chunks, which it replaces whole. A projection or Agent event change checks it against the head in the same way and checks only its own form; the store reads the record only when it compacts. A record swap that loses to a concurrent writer is decided again against the newer record, up to eight times, with a bounded exponential backoff between attempts; a writer that still loses gets the domain's conflict error.
|
|
60
62
|
|
|
61
63
|
A projection or Agent event record that an earlier version wrote whole is read whole, and its next save rewrites it as a base. Versions before 12.0.0 cannot read a base: they fail closed on such a record with `FlexHarnessStoreFormatError` instead of reading the session without its changes, so rolling back to them is unsupported for every session 12.x has saved, as is running them beside 12.x on one database, and they leave the `flexharness_changes` documents in place.
|