@modelprofile.com/flexharness 9.10.0 → 10.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist_ts/00_commitinfo_data.js +1 -1
- package/dist_ts/classes.flexharness.d.ts +2 -0
- package/dist_ts/classes.flexharness.js +101 -7
- package/dist_ts/classes.stores.js +11 -119
- package/dist_ts/errors.d.ts +7 -2
- package/dist_ts/errors.js +1 -1
- package/dist_ts/interfaces.d.ts +1 -0
- package/dist_ts/utils.harnessoptions.js +2 -1
- package/dist_ts/utils.storerules.d.ts +28 -0
- package/dist_ts/utils.storerules.js +107 -0
- package/dist_ts_chat_harness/classes.transcript.d.ts +88 -0
- package/dist_ts_chat_harness/classes.transcript.js +268 -0
- package/dist_ts_chat_harness/functions.intents.d.ts +62 -0
- package/dist_ts_chat_harness/functions.intents.js +96 -0
- package/dist_ts_chat_harness/functions.mapping.d.ts +33 -0
- package/dist_ts_chat_harness/functions.mapping.js +220 -0
- package/dist_ts_chat_harness/index.d.ts +3 -0
- package/dist_ts_chat_harness/index.js +4 -0
- package/dist_ts_chat_harness/plugins.d.ts +1 -0
- package/dist_ts_chat_harness/plugins.js +2 -0
- package/dist_ts_kvm/classes.browserkvm.js +3 -3
- package/dist_ts_kvm/plugins.d.ts +1 -1
- package/dist_ts_kvm/plugins.js +2 -2
- package/dist_ts_stores_nosqldb/classes.nosqldbstores.d.ts +31 -0
- package/dist_ts_stores_nosqldb/classes.nosqldbstores.js +312 -0
- package/dist_ts_stores_nosqldb/classes.recordstore.d.ts +110 -0
- package/dist_ts_stores_nosqldb/classes.recordstore.js +437 -0
- package/dist_ts_stores_nosqldb/index.d.ts +2 -0
- package/dist_ts_stores_nosqldb/index.js +3 -0
- package/dist_ts_stores_nosqldb/plugins.d.ts +7 -0
- package/dist_ts_stores_nosqldb/plugins.js +10 -0
- package/package.json +27 -7
- package/readme.md +14 -6
- package/ts/00_commitinfo_data.ts +1 -1
- package/ts/classes.flexharness.ts +102 -7
- package/ts/classes.stores.ts +28 -172
- package/ts/errors.ts +8 -2
- package/ts/interfaces.ts +1 -0
- package/ts/readme.md +14 -6
- package/ts/utils.harnessoptions.ts +1 -0
- package/ts/utils.storerules.ts +177 -0
- package/ts_chat_harness/classes.transcript.ts +365 -0
- package/ts_chat_harness/functions.intents.ts +169 -0
- package/ts_chat_harness/functions.mapping.ts +267 -0
- package/ts_chat_harness/index.ts +33 -0
- package/ts_chat_harness/plugins.ts +18 -0
- package/ts_chat_harness/readme.md +114 -0
- package/ts_kvm/classes.browserkvm.ts +2 -2
- package/ts_kvm/plugins.ts +1 -1
- package/ts_stores_nosqldb/classes.nosqldbstores.ts +465 -0
- package/ts_stores_nosqldb/classes.recordstore.ts +560 -0
- package/ts_stores_nosqldb/index.ts +9 -0
- package/ts_stores_nosqldb/plugins.ts +38 -0
- package/ts_stores_nosqldb/readme.md +53 -0
|
@@ -0,0 +1,560 @@
|
|
|
1
|
+
import * as plugins from './plugins.js';
|
|
2
|
+
import {
|
|
3
|
+
FlexHarnessStoreCommitUncertainError,
|
|
4
|
+
FlexHarnessStoreFormatError,
|
|
5
|
+
FlexHarnessValidationError,
|
|
6
|
+
} from '../ts/errors.js';
|
|
7
|
+
|
|
8
|
+
export type TFlexRecordDomain =
|
|
9
|
+
| 'scope'
|
|
10
|
+
| 'projection'
|
|
11
|
+
| 'permission'
|
|
12
|
+
| 'projectManagement'
|
|
13
|
+
| 'agentEvents'
|
|
14
|
+
| 'agentEventArchive'
|
|
15
|
+
| 'jobs';
|
|
16
|
+
|
|
17
|
+
/** Names one record: a domain snapshot of a storage key, a session or an archive. */
|
|
18
|
+
export interface IFlexRecordAddress {
|
|
19
|
+
domain: TFlexRecordDomain;
|
|
20
|
+
storageKey: string;
|
|
21
|
+
sessionId?: string;
|
|
22
|
+
archiveId?: string;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/** The next stored value a mutation decided on. */
|
|
26
|
+
export interface IFlexRecordWrite {
|
|
27
|
+
revision: number;
|
|
28
|
+
value: unknown;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export interface IFlexRecordMutation<TValue> {
|
|
32
|
+
validate: (value: unknown) => TValue;
|
|
33
|
+
/** Decides on the stored value: the next value, nothing to write, or a thrown rejection. */
|
|
34
|
+
decide: (current: TValue | undefined) => IFlexRecordWrite | undefined;
|
|
35
|
+
/** The conflict to report when concurrent writers keep winning the compare-and-swap. */
|
|
36
|
+
contention: (actualRevision: number) => Error;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export const FLEX_RECORD_COLLECTION = 'flexharness_records';
|
|
40
|
+
export const FLEX_CHUNK_COLLECTION = 'flexharness_chunks';
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* UTF-16 code units a record holds inline, and the size of each chunk of a longer payload. A
|
|
44
|
+
* chunk stays below 3 MiB of UTF-8, far under the 16 MiB document limit NoSQLDB and MongoDB
|
|
45
|
+
* enforce, so a record and each of its chunks always fit one document.
|
|
46
|
+
*/
|
|
47
|
+
export const FLEX_RECORD_CHUNK_CODE_UNITS = 1024 * 1024;
|
|
48
|
+
|
|
49
|
+
const mutationAttempts = 8;
|
|
50
|
+
const mutationBackoffBaseMs = 5;
|
|
51
|
+
const mutationBackoffMaxMs = 200;
|
|
52
|
+
const chunkReadAttempts = 4;
|
|
53
|
+
const defaultOperationTimeoutMs = 30_000;
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* The head of one record. A payload up to one chunk is stored inline; a longer one lives in
|
|
57
|
+
* immutable chunks named by `blobId`. `writeToken` changes on every committed write and is the
|
|
58
|
+
* compare-and-swap fence. A head without payload or blob is a placeholder that reserves the
|
|
59
|
+
* record for a chunked first write; it reads as absent.
|
|
60
|
+
*/
|
|
61
|
+
export class FlexRecordDocument extends plugins.SmartDataDbDoc<FlexRecordDocument, FlexRecordDocument> {
|
|
62
|
+
public recordId!: string;
|
|
63
|
+
public domain!: TFlexRecordDomain;
|
|
64
|
+
public ownerKey!: string;
|
|
65
|
+
public namespaceKey!: string;
|
|
66
|
+
public revision!: number;
|
|
67
|
+
public writeToken!: string;
|
|
68
|
+
public payload!: string | null;
|
|
69
|
+
public blobId!: string | null;
|
|
70
|
+
public chunkCount!: number;
|
|
71
|
+
public payloadSha256!: string | null;
|
|
72
|
+
/** Set before the first chunk of this record is written; the record's writes then sweep. */
|
|
73
|
+
public chunked!: boolean;
|
|
74
|
+
public sweepCount!: number;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/** One immutable piece of a chunked payload, written against the head token it will replace. */
|
|
78
|
+
export class FlexChunkDocument extends plugins.SmartDataDbDoc<FlexChunkDocument, FlexChunkDocument> {
|
|
79
|
+
public chunkId!: string;
|
|
80
|
+
public recordId!: string;
|
|
81
|
+
public domain!: TFlexRecordDomain;
|
|
82
|
+
public ownerKey!: string;
|
|
83
|
+
public namespaceKey!: string;
|
|
84
|
+
public blobId!: string;
|
|
85
|
+
public baseToken!: string;
|
|
86
|
+
public index!: number;
|
|
87
|
+
public data!: string;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
type TRecordModel = typeof FlexRecordDocument;
|
|
91
|
+
type TChunkModel = typeof FlexChunkDocument;
|
|
92
|
+
|
|
93
|
+
function bindModels(db: plugins.SmartdataDb): { records: TRecordModel; chunks: TChunkModel } {
|
|
94
|
+
class BoundRecord extends FlexRecordDocument {}
|
|
95
|
+
class BoundChunk extends FlexChunkDocument {}
|
|
96
|
+
plugins.defineCollectionModel(BoundRecord, db, {
|
|
97
|
+
collectionName: FLEX_RECORD_COLLECTION,
|
|
98
|
+
persistedFields: [
|
|
99
|
+
'recordId',
|
|
100
|
+
'domain',
|
|
101
|
+
'ownerKey',
|
|
102
|
+
'namespaceKey',
|
|
103
|
+
'revision',
|
|
104
|
+
'writeToken',
|
|
105
|
+
'payload',
|
|
106
|
+
'blobId',
|
|
107
|
+
'chunkCount',
|
|
108
|
+
'payloadSha256',
|
|
109
|
+
'chunked',
|
|
110
|
+
'sweepCount',
|
|
111
|
+
],
|
|
112
|
+
numericFields: ['revision', 'chunkCount', 'sweepCount'],
|
|
113
|
+
identityFields: ['recordId'],
|
|
114
|
+
indexes: [
|
|
115
|
+
{ name: 'flexharness_record_id', key: { recordId: 1 }, options: { unique: true } },
|
|
116
|
+
{ name: 'flexharness_record_owner', key: { ownerKey: 1, domain: 1 } },
|
|
117
|
+
{ name: 'flexharness_record_namespace', key: { namespaceKey: 1, domain: 1 } },
|
|
118
|
+
],
|
|
119
|
+
});
|
|
120
|
+
plugins.defineCollectionModel(BoundChunk, db, {
|
|
121
|
+
collectionName: FLEX_CHUNK_COLLECTION,
|
|
122
|
+
persistedFields: [
|
|
123
|
+
'chunkId',
|
|
124
|
+
'recordId',
|
|
125
|
+
'domain',
|
|
126
|
+
'ownerKey',
|
|
127
|
+
'namespaceKey',
|
|
128
|
+
'blobId',
|
|
129
|
+
'baseToken',
|
|
130
|
+
'index',
|
|
131
|
+
'data',
|
|
132
|
+
],
|
|
133
|
+
numericFields: ['index'],
|
|
134
|
+
identityFields: ['chunkId'],
|
|
135
|
+
indexes: [
|
|
136
|
+
{ name: 'flexharness_chunk_id', key: { chunkId: 1 }, options: { unique: true } },
|
|
137
|
+
{ name: 'flexharness_chunk_blob', key: { blobId: 1 } },
|
|
138
|
+
{ name: 'flexharness_chunk_record', key: { recordId: 1 } },
|
|
139
|
+
{ name: 'flexharness_chunk_owner', key: { ownerKey: 1, domain: 1 } },
|
|
140
|
+
{ name: 'flexharness_chunk_namespace', key: { namespaceKey: 1, domain: 1 } },
|
|
141
|
+
],
|
|
142
|
+
});
|
|
143
|
+
return { records: BoundRecord, chunks: BoundChunk };
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
interface IResolvedAddress {
|
|
147
|
+
recordId: string;
|
|
148
|
+
domain: TFlexRecordDomain;
|
|
149
|
+
ownerKey: string;
|
|
150
|
+
namespaceKey: string;
|
|
151
|
+
path: string;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
interface IStoredHead {
|
|
155
|
+
document?: FlexRecordDocument;
|
|
156
|
+
text?: string;
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
const digest = (value: string): string =>
|
|
160
|
+
plugins.crypto.createHash('sha256').update(value, 'utf8').digest('hex');
|
|
161
|
+
|
|
162
|
+
function splitPayload(text: string): string[] {
|
|
163
|
+
const chunks: string[] = [];
|
|
164
|
+
let start = 0;
|
|
165
|
+
while (start < text.length) {
|
|
166
|
+
let end = Math.min(start + FLEX_RECORD_CHUNK_CODE_UNITS, text.length);
|
|
167
|
+
const last = text.charCodeAt(end - 1);
|
|
168
|
+
if (end < text.length && last >= 0xd800 && last <= 0xdbff) end--;
|
|
169
|
+
chunks.push(text.slice(start, end));
|
|
170
|
+
start = end;
|
|
171
|
+
}
|
|
172
|
+
return chunks;
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
/** A rejection the database states before anything is written; every other failure may have applied. */
|
|
176
|
+
function isDefiniteRejection(error: unknown): boolean {
|
|
177
|
+
return error instanceof plugins.SmartdataPersistenceError && [
|
|
178
|
+
'invalid_argument',
|
|
179
|
+
'invalid_configuration',
|
|
180
|
+
'invalid_document',
|
|
181
|
+
'unsupported_operation',
|
|
182
|
+
'unique_conflict',
|
|
183
|
+
].includes(error.code);
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
/**
|
|
187
|
+
* Waits before retrying a lost head swap: exponential from 5 ms, capped at 200 ms, with full
|
|
188
|
+
* jitter over its upper half, so writers that lost the same race spread out.
|
|
189
|
+
*/
|
|
190
|
+
function mutationBackoff(attempt: number): Promise<void> {
|
|
191
|
+
const ceiling = Math.min(mutationBackoffMaxMs, mutationBackoffBaseMs * 2 ** (attempt - 1));
|
|
192
|
+
const delayMs = ceiling / 2 + Math.random() * (ceiling / 2);
|
|
193
|
+
return new Promise((resolve) => setTimeout(resolve, delayMs));
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
function isUniqueConflict(error: unknown): boolean {
|
|
197
|
+
return error instanceof plugins.SmartdataPersistenceError && error.code === 'unique_conflict';
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
export interface IFlexRecordStoreOptions {
|
|
201
|
+
db: plugins.SmartdataDb;
|
|
202
|
+
operationTimeoutMs?: number;
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
/**
|
|
206
|
+
* Compare-and-swap records on `@lossless.org/client/nosqldb`. A write reads the head, decides,
|
|
207
|
+
* writes any chunks, and swaps the head only while its `writeToken` is unchanged. Chunks of a
|
|
208
|
+
* replaced or abandoned payload are swept in a transaction that also touches the head, so a
|
|
209
|
+
* concurrent deletion or writer can never lose chunks its head references.
|
|
210
|
+
*/
|
|
211
|
+
export class FlexRecordStore {
|
|
212
|
+
private readonly db: plugins.SmartdataDb;
|
|
213
|
+
private readonly timeoutMs: number;
|
|
214
|
+
private readonly records: TRecordModel;
|
|
215
|
+
private readonly chunks: TChunkModel;
|
|
216
|
+
private preparation?: Promise<void>;
|
|
217
|
+
|
|
218
|
+
constructor(options: IFlexRecordStoreOptions) {
|
|
219
|
+
const timeoutMs = options.operationTimeoutMs ?? defaultOperationTimeoutMs;
|
|
220
|
+
if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > 120_000) {
|
|
221
|
+
throw new FlexHarnessValidationError('operationTimeoutMs must be an integer from 1 to 120000.');
|
|
222
|
+
}
|
|
223
|
+
this.db = options.db;
|
|
224
|
+
this.timeoutMs = timeoutMs;
|
|
225
|
+
({ records: this.records, chunks: this.chunks } = bindModels(options.db));
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
/** Creates or verifies both collections and their indexes once; a failed attempt is retried. */
|
|
229
|
+
public prepare(): Promise<void> {
|
|
230
|
+
this.preparation ??= this.prepareCollections().catch((error: unknown) => {
|
|
231
|
+
this.preparation = undefined;
|
|
232
|
+
throw error;
|
|
233
|
+
});
|
|
234
|
+
return this.preparation;
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
private async prepareCollections(): Promise<void> {
|
|
238
|
+
const capabilities = this.db.capabilities;
|
|
239
|
+
if (!capabilities) {
|
|
240
|
+
throw new FlexHarnessValidationError(
|
|
241
|
+
'NoSqlFlexHarnessStores requires an initialized SmartdataDb.',
|
|
242
|
+
);
|
|
243
|
+
}
|
|
244
|
+
if (capabilities.transactions === 'unsupported') {
|
|
245
|
+
throw new FlexHarnessValidationError(
|
|
246
|
+
'NoSqlFlexHarnessStores requires a database that supports transactions.',
|
|
247
|
+
);
|
|
248
|
+
}
|
|
249
|
+
const options = { mode: 'ensureCompatible' as const, timeoutMs: this.timeoutMs };
|
|
250
|
+
await this.records.prepareCollection(options);
|
|
251
|
+
await this.chunks.prepareCollection(options);
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
private resolve(address: IFlexRecordAddress): IResolvedAddress {
|
|
255
|
+
const recordId = digest(JSON.stringify([
|
|
256
|
+
address.domain,
|
|
257
|
+
address.storageKey,
|
|
258
|
+
address.sessionId ?? null,
|
|
259
|
+
address.archiveId ?? null,
|
|
260
|
+
]));
|
|
261
|
+
return {
|
|
262
|
+
recordId,
|
|
263
|
+
domain: address.domain,
|
|
264
|
+
ownerKey: digest(JSON.stringify([address.storageKey, address.sessionId ?? null])),
|
|
265
|
+
namespaceKey: digest(address.storageKey),
|
|
266
|
+
path: `${FLEX_RECORD_COLLECTION}/${address.domain}/${recordId}`,
|
|
267
|
+
};
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
public async read<TValue>(
|
|
271
|
+
address: IFlexRecordAddress,
|
|
272
|
+
validate: (value: unknown) => TValue,
|
|
273
|
+
): Promise<TValue | undefined> {
|
|
274
|
+
await this.prepare();
|
|
275
|
+
const resolved = this.resolve(address);
|
|
276
|
+
return this.parse(resolved, await this.readHead(resolved), validate);
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
public async mutate<TValue>(
|
|
280
|
+
address: IFlexRecordAddress,
|
|
281
|
+
mutation: IFlexRecordMutation<TValue>,
|
|
282
|
+
): Promise<void> {
|
|
283
|
+
await this.prepare();
|
|
284
|
+
const resolved = this.resolve(address);
|
|
285
|
+
let actualRevision = 0;
|
|
286
|
+
for (let attempt = 0; attempt < mutationAttempts; attempt++) {
|
|
287
|
+
if (attempt > 0) await mutationBackoff(attempt);
|
|
288
|
+
const head = await this.readHead(resolved);
|
|
289
|
+
const current = this.parse(resolved, head, mutation.validate);
|
|
290
|
+
actualRevision = head.document?.revision ?? 0;
|
|
291
|
+
const write = mutation.decide(current);
|
|
292
|
+
if (!write) return;
|
|
293
|
+
if (await this.commit(resolved, head.document, write)) return;
|
|
294
|
+
}
|
|
295
|
+
throw mutation.contention(actualRevision);
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
/** Deletes every record and chunk one session owns in the given domains. */
|
|
299
|
+
public async deleteOwner(
|
|
300
|
+
storageKey: string,
|
|
301
|
+
sessionId: string,
|
|
302
|
+
domains: readonly TFlexRecordDomain[],
|
|
303
|
+
): Promise<void> {
|
|
304
|
+
await this.prepare();
|
|
305
|
+
const ownerKey = digest(JSON.stringify([storageKey, sessionId]));
|
|
306
|
+
await this.deleteInTransaction(
|
|
307
|
+
`${FLEX_RECORD_COLLECTION}/${domains.join('+')}/owner/${ownerKey}`,
|
|
308
|
+
{ ownerKey, domain: { $in: [...domains] } },
|
|
309
|
+
);
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
/** Deletes every record and chunk of one domain in a storage namespace. */
|
|
313
|
+
public async purgeNamespace(storageKey: string, domain: TFlexRecordDomain): Promise<void> {
|
|
314
|
+
await this.prepare();
|
|
315
|
+
const namespaceKey = digest(storageKey);
|
|
316
|
+
await this.deleteInTransaction(
|
|
317
|
+
`${FLEX_RECORD_COLLECTION}/${domain}/namespace/${namespaceKey}`,
|
|
318
|
+
{ namespaceKey, domain },
|
|
319
|
+
);
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
private async deleteInTransaction(
|
|
323
|
+
path: string,
|
|
324
|
+
selector:
|
|
325
|
+
| { ownerKey: string; domain: { $in: TFlexRecordDomain[] } }
|
|
326
|
+
| { namespaceKey: string; domain: TFlexRecordDomain },
|
|
327
|
+
): Promise<void> {
|
|
328
|
+
const session = this.db.createSession();
|
|
329
|
+
try {
|
|
330
|
+
await session.withTransaction(async (transaction) => {
|
|
331
|
+
await this.records.atomicDeleteMany(selector, { session: transaction });
|
|
332
|
+
await this.chunks.atomicDeleteMany(selector, { session: transaction });
|
|
333
|
+
}, { timeoutMS: this.timeoutMs });
|
|
334
|
+
} catch (error) {
|
|
335
|
+
if (plugins.SmartdataSession.getCommitOutcome(error) === 'not_committed') throw error;
|
|
336
|
+
throw new FlexHarnessStoreCommitUncertainError(path, 'delete', { cause: error });
|
|
337
|
+
} finally {
|
|
338
|
+
await session.close();
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
private parse<TValue>(
|
|
343
|
+
resolved: IResolvedAddress,
|
|
344
|
+
head: IStoredHead,
|
|
345
|
+
validate: (value: unknown) => TValue,
|
|
346
|
+
): TValue | undefined {
|
|
347
|
+
if (head.text === undefined) return undefined;
|
|
348
|
+
let parsed: unknown;
|
|
349
|
+
try {
|
|
350
|
+
parsed = JSON.parse(head.text);
|
|
351
|
+
} catch (error) {
|
|
352
|
+
throw new FlexHarnessStoreFormatError(`Malformed JSON at ${resolved.path}.`, { cause: error });
|
|
353
|
+
}
|
|
354
|
+
try {
|
|
355
|
+
return validate(parsed);
|
|
356
|
+
} catch (error) {
|
|
357
|
+
if (error instanceof FlexHarnessStoreFormatError) {
|
|
358
|
+
throw new FlexHarnessStoreFormatError(`Invalid record at ${resolved.path}: ${error.message}`, {
|
|
359
|
+
cause: error,
|
|
360
|
+
});
|
|
361
|
+
}
|
|
362
|
+
throw error;
|
|
363
|
+
}
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
private findHead(resolved: IResolvedAddress): Promise<FlexRecordDocument | null> {
|
|
367
|
+
return this.records.getInstance({ recordId: resolved.recordId }, { timeoutMS: this.timeoutMs });
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
private async readHead(resolved: IResolvedAddress): Promise<IStoredHead> {
|
|
371
|
+
for (let attempt = 0; attempt < chunkReadAttempts; attempt++) {
|
|
372
|
+
const document = await this.findHead(resolved);
|
|
373
|
+
if (!document) return {};
|
|
374
|
+
if (document.payload !== null) return { document, text: document.payload };
|
|
375
|
+
if (document.blobId === null) return { document };
|
|
376
|
+
const text = await this.readBlob(document);
|
|
377
|
+
if (text !== undefined) return { document, text };
|
|
378
|
+
const latest = await this.findHead(resolved);
|
|
379
|
+
if (latest?.writeToken === document.writeToken) {
|
|
380
|
+
throw new FlexHarnessStoreFormatError(
|
|
381
|
+
`${resolved.path} references ${document.chunkCount} chunks of blob "${document.blobId}", but they are incomplete.`,
|
|
382
|
+
);
|
|
383
|
+
}
|
|
384
|
+
}
|
|
385
|
+
throw new FlexHarnessStoreFormatError(
|
|
386
|
+
`${resolved.path} changed during ${chunkReadAttempts} consecutive reads.`,
|
|
387
|
+
);
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
private async readBlob(document: FlexRecordDocument): Promise<string | undefined> {
|
|
391
|
+
const stored = await this.chunks.getInstances(
|
|
392
|
+
{ blobId: document.blobId! },
|
|
393
|
+
{ timeoutMS: this.timeoutMs },
|
|
394
|
+
);
|
|
395
|
+
if (stored.length !== document.chunkCount) return undefined;
|
|
396
|
+
const text = stored
|
|
397
|
+
.sort((left, right) => left.index - right.index)
|
|
398
|
+
.map((chunk) => chunk.data)
|
|
399
|
+
.join('');
|
|
400
|
+
return digest(text) === document.payloadSha256 ? text : undefined;
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
/** Returns false when another writer swapped the head first; nothing of this write remains. */
|
|
404
|
+
private async commit(
|
|
405
|
+
resolved: IResolvedAddress,
|
|
406
|
+
stored: FlexRecordDocument | undefined,
|
|
407
|
+
write: IFlexRecordWrite,
|
|
408
|
+
): Promise<boolean> {
|
|
409
|
+
const text = JSON.stringify(write.value);
|
|
410
|
+
const chunked = text.length > FLEX_RECORD_CHUNK_CODE_UNITS;
|
|
411
|
+
let current = stored;
|
|
412
|
+
if (!current) {
|
|
413
|
+
if (!chunked) return this.insertHead(resolved, write.revision, text);
|
|
414
|
+
current = await this.insertPlaceholder(resolved);
|
|
415
|
+
if (!current) return false;
|
|
416
|
+
} else if (current.chunked) {
|
|
417
|
+
await this.sweep(current);
|
|
418
|
+
} else if (chunked && !await this.markChunked(current)) {
|
|
419
|
+
return false;
|
|
420
|
+
}
|
|
421
|
+
const blobId = chunked ? await this.writeBlob(resolved, current.writeToken, text) : null;
|
|
422
|
+
let swapped: boolean;
|
|
423
|
+
try {
|
|
424
|
+
const outcome = await this.records.atomicUpdate(
|
|
425
|
+
{ recordId: resolved.recordId, writeToken: current.writeToken },
|
|
426
|
+
{
|
|
427
|
+
$set: {
|
|
428
|
+
revision: write.revision,
|
|
429
|
+
writeToken: plugins.crypto.randomUUID(),
|
|
430
|
+
payload: chunked ? null : text,
|
|
431
|
+
blobId,
|
|
432
|
+
chunkCount: chunked ? splitPayload(text).length : 0,
|
|
433
|
+
payloadSha256: chunked ? digest(text) : null,
|
|
434
|
+
},
|
|
435
|
+
},
|
|
436
|
+
{ timeoutMS: this.timeoutMs },
|
|
437
|
+
);
|
|
438
|
+
swapped = outcome.matchedCount === 1;
|
|
439
|
+
} catch (error) {
|
|
440
|
+
if (isDefiniteRejection(error)) throw error;
|
|
441
|
+
throw new FlexHarnessStoreCommitUncertainError(resolved.path, 'write', { cause: error });
|
|
442
|
+
}
|
|
443
|
+
if (!swapped && blobId !== null) {
|
|
444
|
+
await this.chunks.atomicDeleteMany({ blobId }, { timeoutMS: this.timeoutMs });
|
|
445
|
+
}
|
|
446
|
+
return swapped;
|
|
447
|
+
}
|
|
448
|
+
|
|
449
|
+
private newHead(resolved: IResolvedAddress): FlexRecordDocument {
|
|
450
|
+
const head = new this.records();
|
|
451
|
+
head.recordId = resolved.recordId;
|
|
452
|
+
head.domain = resolved.domain;
|
|
453
|
+
head.ownerKey = resolved.ownerKey;
|
|
454
|
+
head.namespaceKey = resolved.namespaceKey;
|
|
455
|
+
head.writeToken = plugins.crypto.randomUUID();
|
|
456
|
+
head.blobId = null;
|
|
457
|
+
head.chunkCount = 0;
|
|
458
|
+
head.payloadSha256 = null;
|
|
459
|
+
head.sweepCount = 0;
|
|
460
|
+
return head;
|
|
461
|
+
}
|
|
462
|
+
|
|
463
|
+
private async insertHead(
|
|
464
|
+
resolved: IResolvedAddress,
|
|
465
|
+
revision: number,
|
|
466
|
+
text: string,
|
|
467
|
+
): Promise<boolean> {
|
|
468
|
+
const head = this.newHead(resolved);
|
|
469
|
+
head.revision = revision;
|
|
470
|
+
head.payload = text;
|
|
471
|
+
head.chunked = false;
|
|
472
|
+
try {
|
|
473
|
+
await this.records.insert(head, { timeoutMS: this.timeoutMs });
|
|
474
|
+
return true;
|
|
475
|
+
} catch (error) {
|
|
476
|
+
if (isUniqueConflict(error)) return false;
|
|
477
|
+
if (isDefiniteRejection(error)) throw error;
|
|
478
|
+
throw new FlexHarnessStoreCommitUncertainError(resolved.path, 'write', { cause: error });
|
|
479
|
+
}
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
/** Reserves the record for a chunked first write, so its chunks have a head token to replace. */
|
|
483
|
+
private async insertPlaceholder(
|
|
484
|
+
resolved: IResolvedAddress,
|
|
485
|
+
): Promise<FlexRecordDocument | undefined> {
|
|
486
|
+
const head = this.newHead(resolved);
|
|
487
|
+
head.revision = 0;
|
|
488
|
+
head.payload = null;
|
|
489
|
+
head.chunked = true;
|
|
490
|
+
try {
|
|
491
|
+
await this.records.insert(head, { timeoutMS: this.timeoutMs });
|
|
492
|
+
return head;
|
|
493
|
+
} catch (error) {
|
|
494
|
+
if (isUniqueConflict(error)) return undefined;
|
|
495
|
+
throw error;
|
|
496
|
+
}
|
|
497
|
+
}
|
|
498
|
+
|
|
499
|
+
private async markChunked(current: FlexRecordDocument): Promise<boolean> {
|
|
500
|
+
const outcome = await this.records.atomicUpdate(
|
|
501
|
+
{ recordId: current.recordId, writeToken: current.writeToken },
|
|
502
|
+
{ $set: { chunked: true } },
|
|
503
|
+
{ timeoutMS: this.timeoutMs },
|
|
504
|
+
);
|
|
505
|
+
return outcome.matchedCount === 1;
|
|
506
|
+
}
|
|
507
|
+
|
|
508
|
+
private async writeBlob(
|
|
509
|
+
resolved: IResolvedAddress,
|
|
510
|
+
baseToken: string,
|
|
511
|
+
text: string,
|
|
512
|
+
): Promise<string> {
|
|
513
|
+
const blobId = plugins.crypto.randomUUID();
|
|
514
|
+
const pieces = splitPayload(text);
|
|
515
|
+
for (let index = 0; index < pieces.length; index++) {
|
|
516
|
+
const chunk = new this.chunks();
|
|
517
|
+
chunk.chunkId = `${blobId}:${index}`;
|
|
518
|
+
chunk.recordId = resolved.recordId;
|
|
519
|
+
chunk.domain = resolved.domain;
|
|
520
|
+
chunk.ownerKey = resolved.ownerKey;
|
|
521
|
+
chunk.namespaceKey = resolved.namespaceKey;
|
|
522
|
+
chunk.blobId = blobId;
|
|
523
|
+
chunk.baseToken = baseToken;
|
|
524
|
+
chunk.index = index;
|
|
525
|
+
chunk.data = pieces[index];
|
|
526
|
+
await this.chunks.insert(chunk, { timeoutMS: this.timeoutMs });
|
|
527
|
+
}
|
|
528
|
+
return blobId;
|
|
529
|
+
}
|
|
530
|
+
|
|
531
|
+
/**
|
|
532
|
+
* Deletes the record's chunks that no head can reference any more: every blob but the current
|
|
533
|
+
* one that was written against another head token. A writer that read the current token keeps
|
|
534
|
+
* its chunks. Touching the head in the same transaction makes a concurrent deletion conflict,
|
|
535
|
+
* so chunks written after a deletion are never swept by a transaction that predates it.
|
|
536
|
+
*/
|
|
537
|
+
private async sweep(current: FlexRecordDocument): Promise<void> {
|
|
538
|
+
const session = this.db.createSession();
|
|
539
|
+
try {
|
|
540
|
+
await session.withTransaction(async (transaction) => {
|
|
541
|
+
const touched = await this.records.atomicUpdate(
|
|
542
|
+
{ recordId: current.recordId, writeToken: current.writeToken },
|
|
543
|
+
{ $inc: { sweepCount: 1 } },
|
|
544
|
+
{ session: transaction },
|
|
545
|
+
);
|
|
546
|
+
if (touched.matchedCount !== 1) return;
|
|
547
|
+
await this.chunks.atomicDeleteMany(
|
|
548
|
+
{
|
|
549
|
+
recordId: current.recordId,
|
|
550
|
+
baseToken: { $ne: current.writeToken },
|
|
551
|
+
...(current.blobId === null ? {} : { blobId: { $ne: current.blobId } }),
|
|
552
|
+
},
|
|
553
|
+
{ session: transaction },
|
|
554
|
+
);
|
|
555
|
+
}, { timeoutMS: this.timeoutMs });
|
|
556
|
+
} finally {
|
|
557
|
+
await session.close();
|
|
558
|
+
}
|
|
559
|
+
}
|
|
560
|
+
}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
// node native scope
|
|
2
|
+
import * as crypto from 'node:crypto';
|
|
3
|
+
|
|
4
|
+
export { crypto };
|
|
5
|
+
|
|
6
|
+
// @modelprofile.com scope
|
|
7
|
+
import {
|
|
8
|
+
AgentEventStoreConflictError,
|
|
9
|
+
ToolJobStoreConflictError,
|
|
10
|
+
} from '@modelprofile.com/flexharness-agent';
|
|
11
|
+
|
|
12
|
+
export { AgentEventStoreConflictError, ToolJobStoreConflictError };
|
|
13
|
+
export type {
|
|
14
|
+
IAgentEventArchiveV2,
|
|
15
|
+
IAgentEventSnapshotV2,
|
|
16
|
+
IAgentEventStoreV2,
|
|
17
|
+
IToolJobSnapshot,
|
|
18
|
+
IToolJobState,
|
|
19
|
+
IToolJobStore,
|
|
20
|
+
TAgentEvent,
|
|
21
|
+
} from '@modelprofile.com/flexharness-agent';
|
|
22
|
+
|
|
23
|
+
// @lossless.org scope
|
|
24
|
+
import {
|
|
25
|
+
SmartDataDbDoc,
|
|
26
|
+
SmartdataDb,
|
|
27
|
+
SmartdataPersistenceError,
|
|
28
|
+
SmartdataSession,
|
|
29
|
+
defineCollectionModel,
|
|
30
|
+
} from '@lossless.org/client/nosqldb';
|
|
31
|
+
|
|
32
|
+
export {
|
|
33
|
+
SmartDataDbDoc,
|
|
34
|
+
SmartdataDb,
|
|
35
|
+
SmartdataPersistenceError,
|
|
36
|
+
SmartdataSession,
|
|
37
|
+
defineCollectionModel,
|
|
38
|
+
};
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
FlexHarness session stores on `@lossless.org/client/nosqldb`, with compare-and-swap that holds across processes.
|
|
2
|
+
|
|
3
|
+
```sh
|
|
4
|
+
pnpm add @modelprofile.com/flexharness @lossless.org/client
|
|
5
|
+
```
|
|
6
|
+
|
|
7
|
+
```typescript
|
|
8
|
+
import { SmartdataDb } from '@lossless.org/client/nosqldb';
|
|
9
|
+
import { FlexHarness } from '@modelprofile.com/flexharness';
|
|
10
|
+
import { NoSqlFlexHarnessStores } from '@modelprofile.com/flexharness/stores/nosqldb';
|
|
11
|
+
|
|
12
|
+
const db = new SmartdataDb({ mongoDbUrl, mongoDbName: 'app' });
|
|
13
|
+
await db.init();
|
|
14
|
+
const stores = new NoSqlFlexHarnessStores({ db });
|
|
15
|
+
await stores.prepare();
|
|
16
|
+
|
|
17
|
+
const harness = new FlexHarness({ scopeResolver, modelResolver, stores });
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
`NoSqlFlexHarnessStores` implements every `IFlexHarnessStores` domain: scopes, projections with undo and redo history, remembered permissions, project management, Agent events with their archives, and background jobs. Several processes may open the same database; every save is a compare-and-swap on the stored record, so a stale writer gets the domain's conflict error instead of overwriting a newer snapshot.
|
|
21
|
+
|
|
22
|
+
The database is the application's. The stores never close it, and they need NoSQLDB or a MongoDB deployment with transactions. `prepare()` creates or verifies the `flexharness_records` and `flexharness_chunks` collections and their indexes; every store operation awaits the same preparation, so calling it at startup only moves that work and its failure forward. `operationTimeoutMs` (1 to 120000, default 30000) bounds every database call and transaction.
|
|
23
|
+
|
|
24
|
+
A snapshot of up to `FLEX_RECORD_CHUNK_CODE_UNITS` UTF-16 code units of JSON is stored inside its record. A longer one is split into chunks of that size, each below 3 MiB of UTF-8, so a snapshot may exceed the 16 MiB document limit of NoSQLDB and MongoDB. Chunks are written before the record swaps to them. The chunks a record no longer references are removed by a later write of that record, in a transaction that also touches the record, and deleting a session or purging a namespace removes its records and chunks in one transaction. A writer that crashes between writing its chunks and swapping the record leaves only unreferenced chunks, never a record that points at missing ones. They are written against the record's current write token, which a concurrent writer may still win with, so the write after the next one removes them, or the deletion of the session or namespace does.
|
|
25
|
+
|
|
26
|
+
A record swap that loses to a concurrent writer is decided again against the newer record, up to eight times, with a bounded exponential backoff between attempts; a writer that still loses gets the domain's conflict error.
|
|
27
|
+
|
|
28
|
+
A save that may have reached the database without a confirmation, such as a timed-out or interrupted record write or a deletion transaction with an unknown commit outcome, rejects with `FlexHarnessStoreCommitUncertainError` (operation `write` or `delete`). FlexHarness then fences the namespace, as it does for the JSON file stores.
|
|
29
|
+
|
|
30
|
+
## Issue Reporting and Security
|
|
31
|
+
|
|
32
|
+
Report issues and security concerns through [community.foss.global](https://community.foss.global).
|
|
33
|
+
|
|
34
|
+
## License and Legal Information
|
|
35
|
+
|
|
36
|
+
This repository contains open-source code licensed under the MIT License. A copy of the license can be found in the repository license file.
|
|
37
|
+
|
|
38
|
+
**Please note:** The MIT License does not grant permission to use the trade names, trademarks, service marks, or product names of the project, except as required for reasonable and customary use in describing the origin of the work and reproducing the content of the NOTICE file.
|
|
39
|
+
|
|
40
|
+
### Trademarks
|
|
41
|
+
|
|
42
|
+
This project is owned and maintained by Task Venture Capital GmbH. The names and logos associated with Task Venture Capital GmbH and any related products or services are trademarks of Task Venture Capital GmbH or third parties, and are not included within the scope of the MIT license granted herein.
|
|
43
|
+
|
|
44
|
+
Use of these trademarks must comply with Task Venture Capital GmbH's Trademark Guidelines or the guidelines of the respective third-party owners, and any usage must be approved in writing. Third-party trademarks used herein are the property of their respective owners and used only in a descriptive manner, e.g. for an implementation of an API or similar.
|
|
45
|
+
|
|
46
|
+
### Company Information
|
|
47
|
+
|
|
48
|
+
Task Venture Capital GmbH<br>
|
|
49
|
+
Registered at District Court Bremen HRB 35230 HB, Germany
|
|
50
|
+
|
|
51
|
+
For any legal inquiries or further information, please contact us via email at hello@task.vc.
|
|
52
|
+
|
|
53
|
+
By using this repository, you acknowledge that you have read this section, agree to comply with its terms, and understand that the licensing of the code does not imply endorsement by Task Venture Capital GmbH of any derivative works.
|