@fortemi/core 2026.8.0 → 2026.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/benchmarks/dataset-materialization/small-corpus.v1.json +12 -0
- package/dist/index.d.ts +1173 -130
- package/dist/index.js +2121 -405
- package/dist/index.js.map +1 -1
- package/package.json +10 -1
- package/schemas/dataset-execution-capabilities/fixtures/browser-local.json +14 -0
- package/schemas/dataset-execution-capabilities/fixtures/invalid-field-lineage-without-evidence.json +10 -0
- package/schemas/dataset-execution-capabilities/fixtures/invalid-incremental-without-checkpoint.json +10 -0
- package/schemas/dataset-execution-capabilities/fixtures/portable-shard.json +12 -0
- package/schemas/dataset-execution-capabilities/fixtures/remote-alpha.json +12 -0
- package/schemas/dataset-execution-capabilities/fixtures/static-cache.json +12 -0
- package/schemas/dataset-execution-capabilities/v1.schema.json +164 -0
- package/schemas/dataset-ingest/v1.schema.json +138 -0
- package/schemas/dataset-lineage/fixtures/golden-observed-field.json +20 -0
- package/schemas/dataset-lineage/v1.schema.json +126 -0
- package/schemas/dataset-materialization/fixtures/browser.json +11 -0
- package/schemas/dataset-materialization/fixtures/degraded.json +12 -0
- package/schemas/dataset-materialization/fixtures/deterministic.json +11 -0
- package/schemas/dataset-materialization/fixtures/external-adapter.json +11 -0
- package/schemas/dataset-materialization/fixtures/nondeterministic.json +11 -0
- package/schemas/dataset-materialization/fixtures/server.json +11 -0
- package/schemas/dataset-materialization/fixtures/supported.json +12 -0
- package/schemas/dataset-materialization/fixtures/unsupported.json +11 -0
- package/schemas/dataset-materialization/v1.schema.json +169 -0
package/dist/index.js
CHANGED
|
@@ -2,14 +2,1081 @@ import { v7, v5 } from 'uuid';
|
|
|
2
2
|
import { sha256 } from '@noble/hashes/sha256';
|
|
3
3
|
import { blake3 } from '@noble/hashes/blake3';
|
|
4
4
|
import { bytesToHex } from '@noble/hashes/utils';
|
|
5
|
+
import Ajv20202 from 'ajv/dist/2020.js';
|
|
5
6
|
import { z } from 'zod';
|
|
6
7
|
import { gzipSync, gunzipSync } from 'fflate';
|
|
7
|
-
import Ajv2020 from 'ajv/dist/2020.js';
|
|
8
8
|
|
|
9
9
|
// src/uuid.ts
|
|
10
10
|
function generateId() {
|
|
11
11
|
return v7();
|
|
12
12
|
}
|
|
13
|
+
function computeHash(data) {
|
|
14
|
+
const digest2 = sha256(data);
|
|
15
|
+
return `sha256:${bytesToHex(digest2)}`;
|
|
16
|
+
}
|
|
17
|
+
function computeBlobHash(data) {
|
|
18
|
+
return `blake3:${bytesToHex(blake3(data))}`;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
// src/dataset-ingest.ts
|
|
22
|
+
var DATASET_INGEST_CONTRACT = "fortemi.dataset-ingest/v1";
|
|
23
|
+
var DATASET_INGEST_SCHEMA_VERSION = "1.0.0";
|
|
24
|
+
var DatasetIngestError = class extends Error {
|
|
25
|
+
constructor(code, message) {
|
|
26
|
+
super(message);
|
|
27
|
+
this.code = code;
|
|
28
|
+
this.name = "DatasetIngestError";
|
|
29
|
+
}
|
|
30
|
+
};
|
|
31
|
+
function clone(value) {
|
|
32
|
+
return structuredClone(value);
|
|
33
|
+
}
|
|
34
|
+
var MemoryDatasetIngestStore = class {
|
|
35
|
+
scopes = /* @__PURE__ */ new Map();
|
|
36
|
+
queues = /* @__PURE__ */ new Map();
|
|
37
|
+
async transact(scopeKey, operation) {
|
|
38
|
+
const predecessor = this.queues.get(scopeKey) ?? Promise.resolve();
|
|
39
|
+
let release;
|
|
40
|
+
const current = new Promise((resolve) => {
|
|
41
|
+
release = resolve;
|
|
42
|
+
});
|
|
43
|
+
const queued = predecessor.then(() => current);
|
|
44
|
+
this.queues.set(scopeKey, queued);
|
|
45
|
+
await predecessor;
|
|
46
|
+
try {
|
|
47
|
+
const existing = this.scopes.get(scopeKey) ?? {
|
|
48
|
+
records: /* @__PURE__ */ new Map(),
|
|
49
|
+
receipts: /* @__PURE__ */ new Map()
|
|
50
|
+
};
|
|
51
|
+
const draft = {
|
|
52
|
+
records: new Map([...existing.records].map(([key, value]) => [key, clone(value)])),
|
|
53
|
+
receipts: new Map([...existing.receipts].map(([key, value]) => [key, clone(value)])),
|
|
54
|
+
...existing.checkpoint ? { checkpoint: clone(existing.checkpoint) } : {}
|
|
55
|
+
};
|
|
56
|
+
const transaction = {
|
|
57
|
+
getRecord: (id) => draft.records.get(id),
|
|
58
|
+
setRecord: (record) => draft.records.set(record.logicalId, clone(record)),
|
|
59
|
+
getReceipt: (key) => draft.receipts.get(key),
|
|
60
|
+
setReceipt: (receipt) => draft.receipts.set(receipt.idempotencyKey, clone(receipt)),
|
|
61
|
+
getCheckpoint: () => draft.checkpoint,
|
|
62
|
+
setCheckpoint: (checkpoint) => {
|
|
63
|
+
draft.checkpoint = clone(checkpoint);
|
|
64
|
+
}
|
|
65
|
+
};
|
|
66
|
+
const result = await operation(transaction);
|
|
67
|
+
this.scopes.set(scopeKey, draft);
|
|
68
|
+
return result;
|
|
69
|
+
} finally {
|
|
70
|
+
release();
|
|
71
|
+
if (this.queues.get(scopeKey) === queued) this.queues.delete(scopeKey);
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
async getReceipt(scopeKey, idempotencyKey) {
|
|
75
|
+
return clone(this.scopes.get(scopeKey)?.receipts.get(idempotencyKey));
|
|
76
|
+
}
|
|
77
|
+
async getCheckpoint(scopeKey) {
|
|
78
|
+
return clone(this.scopes.get(scopeKey)?.checkpoint);
|
|
79
|
+
}
|
|
80
|
+
async getRecords(scopeKey) {
|
|
81
|
+
return [...this.scopes.get(scopeKey)?.records.values() ?? []].map(clone).sort((a, b) => a.logicalId.localeCompare(b.logicalId));
|
|
82
|
+
}
|
|
83
|
+
};
|
|
84
|
+
function canonicalJson(value) {
|
|
85
|
+
if (value === null || typeof value !== "object") return JSON.stringify(value);
|
|
86
|
+
if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`;
|
|
87
|
+
return `{${Object.entries(value).sort(([a], [b]) => a.localeCompare(b)).map(([key, item]) => `${JSON.stringify(key)}:${canonicalJson(item)}`).join(",")}}`;
|
|
88
|
+
}
|
|
89
|
+
function digest(value) {
|
|
90
|
+
return computeHash(new TextEncoder().encode(canonicalJson(value)));
|
|
91
|
+
}
|
|
92
|
+
function datasetDestinationScopeKey(scope) {
|
|
93
|
+
return [scope.tenant, scope.dataset, scope.sourceBinding, scope.stream, scope.partition ?? ""].map(encodeURIComponent).join("/");
|
|
94
|
+
}
|
|
95
|
+
function deriveDatasetIngestIdempotencyKey(plan, batch) {
|
|
96
|
+
if (batch.idempotencyKey) return batch.idempotencyKey;
|
|
97
|
+
return digest({
|
|
98
|
+
contract: DATASET_INGEST_CONTRACT,
|
|
99
|
+
planId: plan.planId,
|
|
100
|
+
planDigest: plan.planDigest,
|
|
101
|
+
sourceRevision: plan.sourceRevision,
|
|
102
|
+
configurationDigest: plan.configurationDigest,
|
|
103
|
+
transformationDigest: plan.transformationDigest,
|
|
104
|
+
destination: plan.destination,
|
|
105
|
+
mode: plan.mode,
|
|
106
|
+
sequence: batch.sequence,
|
|
107
|
+
checkpointBefore: batch.checkpointBefore,
|
|
108
|
+
checkpointAfter: batch.checkpointAfter,
|
|
109
|
+
mutations: batch.mutations
|
|
110
|
+
});
|
|
111
|
+
}
|
|
112
|
+
function assertContract(plan, batch) {
|
|
113
|
+
if (plan.contract !== DATASET_INGEST_CONTRACT || batch.contract !== DATASET_INGEST_CONTRACT) {
|
|
114
|
+
throw new DatasetIngestError("INGEST_CONTRACT_UNSUPPORTED", "Unsupported dataset ingest contract");
|
|
115
|
+
}
|
|
116
|
+
if (!plan.schemaVersion.startsWith("1.") || !batch.schemaVersion.startsWith("1.")) {
|
|
117
|
+
throw new DatasetIngestError("INGEST_SCHEMA_UNSUPPORTED", "Unsupported dataset ingest schema version");
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
function assertCheckpointScope(checkpoint, scope) {
|
|
121
|
+
if (checkpoint.contract !== DATASET_INGEST_CONTRACT) throw new DatasetIngestError("INGEST_CONTRACT_UNSUPPORTED", "Unsupported checkpoint contract");
|
|
122
|
+
if (!checkpoint.schemaVersion.startsWith("1.")) throw new DatasetIngestError("CHECKPOINT_VERSION_UNSUPPORTED", `Unsupported checkpoint schema ${checkpoint.schemaVersion}`);
|
|
123
|
+
if (datasetDestinationScopeKey(checkpoint.scope) !== datasetDestinationScopeKey(scope)) {
|
|
124
|
+
throw new DatasetIngestError("CHECKPOINT_SCOPE_MISMATCH", "Checkpoint belongs to another destination scope");
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
var DatasetIngestExecutor = class {
|
|
128
|
+
constructor(store) {
|
|
129
|
+
this.store = store;
|
|
130
|
+
}
|
|
131
|
+
attempts = /* @__PURE__ */ new Map();
|
|
132
|
+
successes = /* @__PURE__ */ new Map();
|
|
133
|
+
preview(plan, batch) {
|
|
134
|
+
assertContract(plan, batch);
|
|
135
|
+
return {
|
|
136
|
+
idempotencyKey: deriveDatasetIngestIdempotencyKey(plan, batch),
|
|
137
|
+
upserts: batch.mutations.filter((item) => item.operation === "upsert").length,
|
|
138
|
+
tombstones: batch.mutations.filter((item) => item.operation === "tombstone").length
|
|
139
|
+
};
|
|
140
|
+
}
|
|
141
|
+
connectionCheck(plan, batch) {
|
|
142
|
+
assertContract(plan, batch);
|
|
143
|
+
assertCheckpointScope(batch.checkpointAfter, plan.destination);
|
|
144
|
+
if (batch.checkpointBefore) assertCheckpointScope(batch.checkpointBefore, plan.destination);
|
|
145
|
+
return {
|
|
146
|
+
idempotencyKey: deriveDatasetIngestIdempotencyKey(plan, batch),
|
|
147
|
+
scopeKey: datasetDestinationScopeKey(plan.destination),
|
|
148
|
+
compatible: true
|
|
149
|
+
};
|
|
150
|
+
}
|
|
151
|
+
async executeBatch(plan, batch, options = {}) {
|
|
152
|
+
assertContract(plan, batch);
|
|
153
|
+
const scopeKey = datasetDestinationScopeKey(plan.destination);
|
|
154
|
+
const idempotencyKey = deriveDatasetIngestIdempotencyKey(plan, batch);
|
|
155
|
+
const requestDigest = digest({ plan, batch });
|
|
156
|
+
const runId = generateId();
|
|
157
|
+
this.attempts.set(scopeKey, { runId, state: "running", verification: "pending", idempotencyKey });
|
|
158
|
+
const cancel = () => {
|
|
159
|
+
if (options.signal?.aborted) throw new DatasetIngestError("INGEST_CANCELLED", "Dataset ingest was cancelled before commit");
|
|
160
|
+
};
|
|
161
|
+
try {
|
|
162
|
+
cancel();
|
|
163
|
+
const receipt = await this.store.transact(scopeKey, async (transaction) => {
|
|
164
|
+
const prior = transaction.getReceipt(idempotencyKey);
|
|
165
|
+
if (prior) {
|
|
166
|
+
if (prior.requestDigest !== requestDigest) throw new DatasetIngestError("IDEMPOTENCY_CONFLICT", "Idempotency key was previously used with different canonical content");
|
|
167
|
+
return prior;
|
|
168
|
+
}
|
|
169
|
+
const currentCheckpoint = transaction.getCheckpoint();
|
|
170
|
+
if (batch.checkpointBefore) {
|
|
171
|
+
assertCheckpointScope(batch.checkpointBefore, plan.destination);
|
|
172
|
+
if (!currentCheckpoint || canonicalJson(currentCheckpoint) !== canonicalJson(batch.checkpointBefore)) {
|
|
173
|
+
throw new DatasetIngestError("CHECKPOINT_MISMATCH", "checkpointBefore does not match committed state");
|
|
174
|
+
}
|
|
175
|
+
} else if (currentCheckpoint) {
|
|
176
|
+
throw new DatasetIngestError("CHECKPOINT_MISMATCH", "A committed checkpoint exists but checkpointBefore was omitted");
|
|
177
|
+
}
|
|
178
|
+
assertCheckpointScope(batch.checkpointAfter, plan.destination);
|
|
179
|
+
if (batch.checkpointAfter.sequence !== batch.sequence) throw new DatasetIngestError("BATCH_OUT_OF_ORDER", "Checkpoint sequence must equal batch sequence");
|
|
180
|
+
if (currentCheckpoint && batch.checkpointAfter.sequence <= currentCheckpoint.sequence) throw new DatasetIngestError("CHECKPOINT_REGRESSION", "Checkpoint sequence must advance");
|
|
181
|
+
const expectedSequence = (currentCheckpoint?.sequence ?? 0) + 1;
|
|
182
|
+
if (batch.sequence !== expectedSequence) throw new DatasetIngestError("BATCH_OUT_OF_ORDER", `Expected batch sequence ${expectedSequence}, received ${batch.sequence}`);
|
|
183
|
+
const tombstones = batch.mutations.filter((item) => item.operation === "tombstone").length;
|
|
184
|
+
if (tombstones > 0) {
|
|
185
|
+
if (!plan.reconciliation.enabled) throw new DatasetIngestError("RECONCILIATION_NOT_ENABLED", "Tombstones require reconciliation policy");
|
|
186
|
+
if (!batch.enumeration?.complete) throw new DatasetIngestError("RECONCILIATION_INCOMPLETE", "Tombstones require complete source enumeration");
|
|
187
|
+
if (tombstones > plan.reconciliation.maxTombstones && !batch.enumeration.approvalId) {
|
|
188
|
+
throw new DatasetIngestError("RECONCILIATION_APPROVAL_REQUIRED", `Tombstone count ${tombstones} exceeds approved threshold`);
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
const rejections = [];
|
|
192
|
+
const accepted = [];
|
|
193
|
+
for (const mutation of batch.mutations) {
|
|
194
|
+
cancel();
|
|
195
|
+
const failure2 = options.validateRecord?.(mutation);
|
|
196
|
+
if (!failure2) {
|
|
197
|
+
accepted.push(mutation);
|
|
198
|
+
continue;
|
|
199
|
+
}
|
|
200
|
+
const rejection = {
|
|
201
|
+
logicalIdDigest: digest(mutation.logicalId),
|
|
202
|
+
...mutation.locator ? { locator: mutation.locator } : {},
|
|
203
|
+
code: failure2.code,
|
|
204
|
+
message: "Record rejected by validation policy"
|
|
205
|
+
};
|
|
206
|
+
if (plan.rejectionPolicy.mode === "fail-fast") throw new DatasetIngestError("RECORD_REJECTED", `${failure2.code}: record rejected`);
|
|
207
|
+
rejections.push(rejection);
|
|
208
|
+
if (rejections.length > plan.rejectionPolicy.maxRejectedRecords) throw new DatasetIngestError("REJECTION_LIMIT_EXCEEDED", "Rejected-record limit exceeded");
|
|
209
|
+
}
|
|
210
|
+
cancel();
|
|
211
|
+
for (const mutation of accepted) {
|
|
212
|
+
transaction.setRecord(mutation.operation === "upsert" ? { logicalId: mutation.logicalId, revision: mutation.revision, digest: mutation.digest, value: clone(mutation.value), tombstoned: false } : { logicalId: mutation.logicalId, revision: mutation.revision, digest: mutation.digest, tombstoned: true });
|
|
213
|
+
}
|
|
214
|
+
const effects = accepted.map(({ operation, logicalId, revision, digest: itemDigest }) => ({ operation, logicalId, revision, digest: itemDigest }));
|
|
215
|
+
const receipt2 = {
|
|
216
|
+
contract: DATASET_INGEST_CONTRACT,
|
|
217
|
+
schemaVersion: DATASET_INGEST_SCHEMA_VERSION,
|
|
218
|
+
runId,
|
|
219
|
+
idempotencyKey,
|
|
220
|
+
requestDigest,
|
|
221
|
+
planId: plan.planId,
|
|
222
|
+
planDigest: plan.planDigest,
|
|
223
|
+
sourceRevision: plan.sourceRevision,
|
|
224
|
+
destination: clone(plan.destination),
|
|
225
|
+
mode: plan.mode,
|
|
226
|
+
state: rejections.length ? "degraded" : "committed",
|
|
227
|
+
effects,
|
|
228
|
+
acceptedRecords: accepted.length,
|
|
229
|
+
rejectedRecords: rejections.length,
|
|
230
|
+
rejections,
|
|
231
|
+
outputDigest: digest(effects),
|
|
232
|
+
...batch.checkpointBefore ? { checkpointBefore: clone(batch.checkpointBefore) } : {},
|
|
233
|
+
checkpointAfter: clone(batch.checkpointAfter),
|
|
234
|
+
verification: "verified"
|
|
235
|
+
};
|
|
236
|
+
await options.hooks?.beforeCommit?.();
|
|
237
|
+
cancel();
|
|
238
|
+
transaction.setReceipt(receipt2);
|
|
239
|
+
transaction.setCheckpoint(batch.checkpointAfter);
|
|
240
|
+
return receipt2;
|
|
241
|
+
});
|
|
242
|
+
this.successes.set(scopeKey, clone(receipt));
|
|
243
|
+
this.attempts.set(scopeKey, { runId: receipt.runId, state: receipt.state, verification: "verified", idempotencyKey });
|
|
244
|
+
await options.hooks?.afterCommit?.(clone(receipt));
|
|
245
|
+
return receipt;
|
|
246
|
+
} catch (error) {
|
|
247
|
+
const code = error instanceof DatasetIngestError ? error.code : void 0;
|
|
248
|
+
this.attempts.set(scopeKey, { runId, state: code === "INGEST_CANCELLED" ? "cancelled" : "failed", verification: "failed", idempotencyKey, ...code ? { errorCode: code } : {} });
|
|
249
|
+
throw error;
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
async resolveAmbiguousCommit(plan, batch) {
|
|
253
|
+
return this.store.getReceipt(datasetDestinationScopeKey(plan.destination), deriveDatasetIngestIdempotencyKey(plan, batch));
|
|
254
|
+
}
|
|
255
|
+
status(scope, expectedSourceRevision) {
|
|
256
|
+
const key = datasetDestinationScopeKey(scope);
|
|
257
|
+
const lastSuccessful = this.successes.get(key);
|
|
258
|
+
return {
|
|
259
|
+
scope: clone(scope),
|
|
260
|
+
...this.attempts.get(key) ? { lastAttempt: clone(this.attempts.get(key)) } : {},
|
|
261
|
+
...lastSuccessful ? { lastSuccessful: clone(lastSuccessful) } : {},
|
|
262
|
+
freshness: lastSuccessful ? expectedSourceRevision && expectedSourceRevision !== lastSuccessful.sourceRevision ? "stale" : "current" : "never"
|
|
263
|
+
};
|
|
264
|
+
}
|
|
265
|
+
};
|
|
266
|
+
|
|
267
|
+
// src/dataset-lineage.ts
|
|
268
|
+
var DATASET_LINEAGE_CONTRACT = "fortemi.dataset-lineage/v1";
|
|
269
|
+
var DATASET_LINEAGE_SCHEMA_VERSION = "1.0.0";
|
|
270
|
+
var LINEAGE_ENTITY_KINDS = [
|
|
271
|
+
"dataset",
|
|
272
|
+
"dataset-revision",
|
|
273
|
+
"distribution",
|
|
274
|
+
"record",
|
|
275
|
+
"field",
|
|
276
|
+
"chunk",
|
|
277
|
+
"index",
|
|
278
|
+
"embedding-set",
|
|
279
|
+
"graph-artifact",
|
|
280
|
+
"community-artifact",
|
|
281
|
+
"processing-plan",
|
|
282
|
+
"run"
|
|
283
|
+
];
|
|
284
|
+
var LINEAGE_RELATIONSHIP_KINDS = [
|
|
285
|
+
"derived-from",
|
|
286
|
+
"field-derived-from",
|
|
287
|
+
"extracted-from",
|
|
288
|
+
"chunk-of",
|
|
289
|
+
"indexed-from",
|
|
290
|
+
"embedded-from",
|
|
291
|
+
"graph-derived-from",
|
|
292
|
+
"community-derived-from",
|
|
293
|
+
"revision-of",
|
|
294
|
+
"distributed-as",
|
|
295
|
+
"join-influence",
|
|
296
|
+
"filter-influence",
|
|
297
|
+
"aggregation-influence",
|
|
298
|
+
"ordering-influence",
|
|
299
|
+
"similarity-influence"
|
|
300
|
+
];
|
|
301
|
+
var LineageValidationError = class extends Error {
|
|
302
|
+
constructor(code, message) {
|
|
303
|
+
super(message);
|
|
304
|
+
this.code = code;
|
|
305
|
+
this.name = "LineageValidationError";
|
|
306
|
+
}
|
|
307
|
+
};
|
|
308
|
+
function canonicalJson2(value) {
|
|
309
|
+
if (value === void 0) return "null";
|
|
310
|
+
if (value === null || typeof value !== "object") return JSON.stringify(value);
|
|
311
|
+
if (Array.isArray(value)) return `[${value.map(canonicalJson2).join(",")}]`;
|
|
312
|
+
const entries = Object.entries(value).filter(([, item]) => item !== void 0).sort(([left], [right]) => left.localeCompare(right));
|
|
313
|
+
return `{${entries.map(([key, item]) => `${JSON.stringify(key)}:${canonicalJson2(item)}`).join(",")}}`;
|
|
314
|
+
}
|
|
315
|
+
function computeLineageDigest(value) {
|
|
316
|
+
return computeHash(new TextEncoder().encode(canonicalJson2(value)));
|
|
317
|
+
}
|
|
318
|
+
function clone2(value) {
|
|
319
|
+
return structuredClone(value);
|
|
320
|
+
}
|
|
321
|
+
function requireIdentity(id, label) {
|
|
322
|
+
if (!id || id.trim() !== id) throw new LineageValidationError("IDENTITY_REQUIRED", `${label} must be a non-empty canonical identity`);
|
|
323
|
+
}
|
|
324
|
+
function requireTimestamp(value, label) {
|
|
325
|
+
if (!/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{3})?Z$/.test(value) || !Number.isFinite(Date.parse(value))) {
|
|
326
|
+
throw new LineageValidationError("VALUE_INVALID", `${label} must be an RFC 3339 UTC timestamp`);
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
var ENTITY_KIND_SET = new Set(LINEAGE_ENTITY_KINDS);
|
|
330
|
+
var RELATIONSHIP_KIND_SET = new Set(LINEAGE_RELATIONSHIP_KINDS);
|
|
331
|
+
var RELATIONSHIP_ENDPOINTS = {
|
|
332
|
+
"field-derived-from": [["field"], ["field"]],
|
|
333
|
+
"chunk-of": [["chunk"], ["record", "distribution"]],
|
|
334
|
+
"indexed-from": [["index"], ["dataset", "dataset-revision", "distribution", "record", "field", "chunk"]],
|
|
335
|
+
"embedded-from": [["embedding-set"], ["dataset", "dataset-revision", "record", "field", "chunk"]],
|
|
336
|
+
"graph-derived-from": [["graph-artifact"], ["dataset", "dataset-revision", "index", "embedding-set"]],
|
|
337
|
+
"community-derived-from": [["community-artifact"], ["graph-artifact"]],
|
|
338
|
+
"revision-of": [["dataset-revision"], ["dataset"]],
|
|
339
|
+
"distributed-as": [["dataset", "dataset-revision"], ["distribution"]]
|
|
340
|
+
};
|
|
341
|
+
var DatasetLineageLedger = class _DatasetLineageLedger {
|
|
342
|
+
sequence = 0;
|
|
343
|
+
entities = /* @__PURE__ */ new Map();
|
|
344
|
+
agents = /* @__PURE__ */ new Map();
|
|
345
|
+
activities = /* @__PURE__ */ new Map();
|
|
346
|
+
evidence = /* @__PURE__ */ new Map();
|
|
347
|
+
assertions = /* @__PURE__ */ new Map();
|
|
348
|
+
corrections = /* @__PURE__ */ new Map();
|
|
349
|
+
maximumTraversalDepth;
|
|
350
|
+
maximumTraversalResults;
|
|
351
|
+
maximumPageSize;
|
|
352
|
+
constructor(options = {}) {
|
|
353
|
+
this.maximumTraversalDepth = options.maximumTraversalDepth ?? 16;
|
|
354
|
+
this.maximumTraversalResults = options.maximumTraversalResults ?? 1e4;
|
|
355
|
+
this.maximumPageSize = options.maximumPageSize ?? 500;
|
|
356
|
+
}
|
|
357
|
+
get snapshot() {
|
|
358
|
+
return this.sequence;
|
|
359
|
+
}
|
|
360
|
+
appendEntity(entity) {
|
|
361
|
+
requireIdentity(entity.id, "entity.id");
|
|
362
|
+
if (!ENTITY_KIND_SET.has(entity.kind)) throw new LineageValidationError("VALUE_INVALID", `Unsupported entity kind ${String(entity.kind)}`);
|
|
363
|
+
requireIdentity(entity.schemaId, "entity.schemaId");
|
|
364
|
+
requireIdentity(entity.schemaVersion, "entity.schemaVersion");
|
|
365
|
+
requireTimestamp(entity.createdAt, "entity.createdAt");
|
|
366
|
+
this.ensureUnique(this.entities, entity.id, "entity");
|
|
367
|
+
return this.append(this.entities, entity.id, entity);
|
|
368
|
+
}
|
|
369
|
+
appendAgent(agent) {
|
|
370
|
+
requireIdentity(agent.id, "agent.id");
|
|
371
|
+
requireIdentity(agent.name, "agent.name");
|
|
372
|
+
this.ensureUnique(this.agents, agent.id, "agent");
|
|
373
|
+
return this.append(this.agents, agent.id, agent);
|
|
374
|
+
}
|
|
375
|
+
appendActivity(activity) {
|
|
376
|
+
requireIdentity(activity.id, "activity.id");
|
|
377
|
+
requireTimestamp(activity.startedAt, "activity.startedAt");
|
|
378
|
+
if (activity.endedAt) requireTimestamp(activity.endedAt, "activity.endedAt");
|
|
379
|
+
for (const agentId of activity.agentIds) {
|
|
380
|
+
if (!this.agents.has(agentId)) throw new LineageValidationError("AGENT_DANGLING", `Activity references unknown agent ${agentId}`);
|
|
381
|
+
}
|
|
382
|
+
this.ensureUnique(this.activities, activity.id, "activity");
|
|
383
|
+
return this.append(this.activities, activity.id, activity);
|
|
384
|
+
}
|
|
385
|
+
appendEvidence(item) {
|
|
386
|
+
requireIdentity(item.id, "evidence.id");
|
|
387
|
+
requireIdentity(item.revision, "evidence.revision");
|
|
388
|
+
requireIdentity(item.locator, "evidence.locator");
|
|
389
|
+
requireTimestamp(item.capturedAt, "evidence.capturedAt");
|
|
390
|
+
if (item.payload !== void 0 && computeLineageDigest(item.payload) !== item.digest) {
|
|
391
|
+
throw new LineageValidationError("EVIDENCE_DIGEST_MISMATCH", `Evidence ${item.id} payload does not match ${item.digest}`);
|
|
392
|
+
}
|
|
393
|
+
const key = `${item.id}@${item.revision}`;
|
|
394
|
+
this.ensureUnique(this.evidence, key, "evidence revision");
|
|
395
|
+
return this.append(this.evidence, key, item);
|
|
396
|
+
}
|
|
397
|
+
appendAssertion(assertion) {
|
|
398
|
+
requireIdentity(assertion.id, "assertion.id");
|
|
399
|
+
requireIdentity(assertion.revision, "assertion.revision");
|
|
400
|
+
requireIdentity(assertion.method, "assertion.method");
|
|
401
|
+
requireIdentity(assertion.schemaId, "assertion.schemaId");
|
|
402
|
+
requireIdentity(assertion.schemaVersion, "assertion.schemaVersion");
|
|
403
|
+
requireTimestamp(assertion.assertedAt, "assertion.assertedAt");
|
|
404
|
+
if (!RELATIONSHIP_KIND_SET.has(assertion.relationship)) throw new LineageValidationError("VALUE_INVALID", `Unsupported relationship ${String(assertion.relationship)}`);
|
|
405
|
+
if (!Number.isFinite(assertion.confidence) || assertion.confidence < 0 || assertion.confidence > 1) {
|
|
406
|
+
throw new LineageValidationError("VALUE_INVALID", "assertion.confidence must be between 0 and 1");
|
|
407
|
+
}
|
|
408
|
+
const source = this.entities.get(assertion.sourceEntityId)?.value;
|
|
409
|
+
const target = this.entities.get(assertion.targetEntityId)?.value;
|
|
410
|
+
if (!source || !target) throw new LineageValidationError("IDENTITY_DANGLING", "Assertion source and target must both exist");
|
|
411
|
+
const endpointRule = RELATIONSHIP_ENDPOINTS[assertion.relationship];
|
|
412
|
+
if (endpointRule && (!endpointRule[0].includes(source.kind) || !endpointRule[1].includes(target.kind))) {
|
|
413
|
+
throw new LineageValidationError("TYPE_DIRECTION_INVALID", `${assertion.relationship} does not allow ${source.kind} -> ${target.kind}`);
|
|
414
|
+
}
|
|
415
|
+
if (!this.agents.has(assertion.issuerAgentId)) throw new LineageValidationError("AGENT_DANGLING", `Unknown issuer agent ${assertion.issuerAgentId}`);
|
|
416
|
+
if (assertion.producingActivityId && !this.activities.has(assertion.producingActivityId)) {
|
|
417
|
+
throw new LineageValidationError("IDENTITY_DANGLING", `Unknown producing activity ${assertion.producingActivityId}`);
|
|
418
|
+
}
|
|
419
|
+
if (assertion.assertionKind === "observed" && !assertion.producingActivityId) {
|
|
420
|
+
throw new LineageValidationError("ACTIVITY_REQUIRED", "Observed assertions require a producing activity");
|
|
421
|
+
}
|
|
422
|
+
if (assertion.assertionKind === "observed" && assertion.evidence.length === 0) {
|
|
423
|
+
throw new LineageValidationError("EVIDENCE_DANGLING", "Observed assertions require evidence");
|
|
424
|
+
}
|
|
425
|
+
for (const reference of assertion.evidence) {
|
|
426
|
+
const item = this.evidence.get(`${reference.evidenceId}@${reference.revision}`)?.value;
|
|
427
|
+
if (!item) throw new LineageValidationError("EVIDENCE_DANGLING", `Unknown evidence ${reference.evidenceId}@${reference.revision}`);
|
|
428
|
+
if (item.digest !== reference.digest) throw new LineageValidationError("EVIDENCE_DIGEST_MISMATCH", `Evidence reference digest differs for ${reference.evidenceId}`);
|
|
429
|
+
if (reference.locator && reference.locator !== item.locator) throw new LineageValidationError("EVIDENCE_DIGEST_MISMATCH", `Evidence locator differs for ${reference.evidenceId}`);
|
|
430
|
+
}
|
|
431
|
+
const key = `${assertion.id}@${assertion.revision}`;
|
|
432
|
+
if (this.assertions.has(key)) throw new LineageValidationError("IDENTITY_DUPLICATE", `Duplicate assertion revision ${key}`);
|
|
433
|
+
const prior = [...this.assertions.values()].some((entry) => entry.value.id === assertion.id);
|
|
434
|
+
if (prior && !this.hasReplacementPermission(assertion.id, assertion.revision)) {
|
|
435
|
+
throw new LineageValidationError("CORRECTION_INVALID", `Assertion ${assertion.id} can only receive revision ${assertion.revision} after an explicit correction`);
|
|
436
|
+
}
|
|
437
|
+
return this.append(this.assertions, key, assertion);
|
|
438
|
+
}
|
|
439
|
+
appendCorrection(correction) {
|
|
440
|
+
requireIdentity(correction.id, "correction.id");
|
|
441
|
+
requireIdentity(correction.reason, "correction.reason");
|
|
442
|
+
requireTimestamp(correction.recordedAt, "correction.recordedAt");
|
|
443
|
+
if (!this.assertions.has(`${correction.assertionId}@${correction.assertionRevision}`)) throw new LineageValidationError("IDENTITY_DANGLING", `Unknown assertion ${correction.assertionId}@${correction.assertionRevision}`);
|
|
444
|
+
if (!this.agents.has(correction.issuerAgentId)) throw new LineageValidationError("AGENT_DANGLING", `Unknown correction issuer ${correction.issuerAgentId}`);
|
|
445
|
+
const activity = this.activities.get(correction.activityId)?.value;
|
|
446
|
+
if (!activity || activity.kind !== "correction") throw new LineageValidationError("CORRECTION_INVALID", "Correction requires a correction activity");
|
|
447
|
+
if (correction.action === "retract" && (correction.replacementAssertionId || correction.replacementRevision)) throw new LineageValidationError("CORRECTION_INVALID", "Retraction cannot name a replacement");
|
|
448
|
+
if (correction.action !== "retract" && (!correction.replacementAssertionId || !correction.replacementRevision)) throw new LineageValidationError("CORRECTION_INVALID", `${correction.action} requires a replacement assertion identity and revision`);
|
|
449
|
+
if (correction.replacementAssertionId === correction.assertionId && correction.action === "supersede") {
|
|
450
|
+
throw new LineageValidationError("CORRECTION_INVALID", "Supersession must identify a different assertion");
|
|
451
|
+
}
|
|
452
|
+
this.ensureUnique(this.corrections, correction.id, "correction");
|
|
453
|
+
return this.append(this.corrections, correction.id, correction);
|
|
454
|
+
}
|
|
455
|
+
traverse(request, policy) {
|
|
456
|
+
this.validateTraversalRequest(request);
|
|
457
|
+
const snapshot = request.snapshot ?? this.sequence;
|
|
458
|
+
if (snapshot < 0 || snapshot > this.sequence) throw new LineageValidationError("SNAPSHOT_UNAVAILABLE", `Snapshot ${snapshot} is unavailable`);
|
|
459
|
+
const entityMap = this.visibleAt(this.entities, snapshot);
|
|
460
|
+
const assertions = [...this.visibleAt(this.assertions, snapshot).values()].filter((assertion) => !request.relationshipKinds || request.relationshipKinds.includes(assertion.relationship)).filter((assertion) => !request.assertionKinds || request.assertionKinds.includes(assertion.assertionKind)).filter((assertion) => policy.canReadAssertion(clone2(assertion))).filter((assertion) => {
|
|
461
|
+
const source = entityMap.get(assertion.sourceEntityId);
|
|
462
|
+
const target = entityMap.get(assertion.targetEntityId);
|
|
463
|
+
return Boolean(source && target && policy.canReadEntity(clone2(source)) && policy.canReadEntity(clone2(target)));
|
|
464
|
+
}).sort(compareAssertions);
|
|
465
|
+
for (const id of request.startEntityIds) {
|
|
466
|
+
const entity = entityMap.get(id);
|
|
467
|
+
if (!entity || !policy.canReadEntity(clone2(entity))) throw new LineageValidationError("IDENTITY_DANGLING", `Start entity ${id} is unavailable`);
|
|
468
|
+
}
|
|
469
|
+
const queue = [...new Set(request.startEntityIds)].sort().map((id) => ({ id, depth: 0, path: [] }));
|
|
470
|
+
const visitedDepth = /* @__PURE__ */ new Map();
|
|
471
|
+
const nodeMap = /* @__PURE__ */ new Map();
|
|
472
|
+
const edgeMap = /* @__PURE__ */ new Map();
|
|
473
|
+
while (queue.length > 0) {
|
|
474
|
+
const current = queue.shift();
|
|
475
|
+
const priorDepth = visitedDepth.get(current.id);
|
|
476
|
+
if (priorDepth !== void 0 && priorDepth <= current.depth) continue;
|
|
477
|
+
visitedDepth.set(current.id, current.depth);
|
|
478
|
+
const entity = entityMap.get(current.id);
|
|
479
|
+
if (!request.entityKinds || request.entityKinds.includes(entity.kind) || current.depth === 0) {
|
|
480
|
+
nodeMap.set(current.id, { entity: clone2(entity), depth: current.depth, pathAssertionIds: [...current.path] });
|
|
481
|
+
}
|
|
482
|
+
if (current.depth >= request.maximumDepth) continue;
|
|
483
|
+
for (const assertion of assertions) {
|
|
484
|
+
const downstream = assertion.sourceEntityId === current.id;
|
|
485
|
+
const upstream = assertion.targetEntityId === current.id;
|
|
486
|
+
if (request.direction === "downstream" && !downstream || request.direction === "upstream" && !upstream || request.direction === "both" && !downstream && !upstream) continue;
|
|
487
|
+
const nextId = downstream ? assertion.targetEntityId : assertion.sourceEntityId;
|
|
488
|
+
edgeMap.set(`${assertion.id}@${assertion.revision}`, {
|
|
489
|
+
assertion: clone2(assertion),
|
|
490
|
+
status: this.statusAt(assertion.id, assertion.revision, snapshot),
|
|
491
|
+
...request.includeEvidence ? { evidence: assertion.evidence.map((reference) => this.evidence.get(`${reference.evidenceId}@${reference.revision}`)).filter((entry) => Boolean(entry && entry.sequence <= snapshot)).map((entry) => entry.value).filter((item) => policy.canReadEvidence(clone2(item))).map(clone2) } : {}
|
|
492
|
+
});
|
|
493
|
+
queue.push({ id: nextId, depth: current.depth + 1, path: [...current.path, assertion.id] });
|
|
494
|
+
}
|
|
495
|
+
if (nodeMap.size + edgeMap.size > request.maximumResults) break;
|
|
496
|
+
}
|
|
497
|
+
const allNodes = [...nodeMap.values()].sort((left, right) => left.depth - right.depth || left.entity.id.localeCompare(right.entity.id));
|
|
498
|
+
const allEdges = [...edgeMap.values()].sort((left, right) => compareAssertions(left.assertion, right.assertion));
|
|
499
|
+
const rows = [
|
|
500
|
+
...allNodes.map((value) => ({ kind: "node", key: `n:${String(value.depth).padStart(8, "0")}:${value.entity.id}`, value })),
|
|
501
|
+
...allEdges.map((value) => ({ kind: "edge", key: `e:${value.assertion.sourceEntityId}:${value.assertion.targetEntityId}:${value.assertion.relationship}:${value.assertion.id}`, value }))
|
|
502
|
+
].sort((left, right) => left.key.localeCompare(right.key));
|
|
503
|
+
const requestFingerprint = this.traversalFingerprint(request, snapshot);
|
|
504
|
+
const offset = request.cursor ? this.decodeCursor(request.cursor, requestFingerprint, snapshot) : 0;
|
|
505
|
+
const bounded = rows.slice(0, request.maximumResults);
|
|
506
|
+
const page = bounded.slice(offset, offset + request.pageSize);
|
|
507
|
+
const nextOffset = offset + page.length;
|
|
508
|
+
const truncated = rows.length > request.maximumResults;
|
|
509
|
+
return {
|
|
510
|
+
contract: DATASET_LINEAGE_CONTRACT,
|
|
511
|
+
schemaVersion: DATASET_LINEAGE_SCHEMA_VERSION,
|
|
512
|
+
snapshot,
|
|
513
|
+
nodes: page.filter((row) => row.kind === "node").map((row) => clone2(row.value)),
|
|
514
|
+
edges: page.filter((row) => row.kind === "edge").map((row) => clone2(row.value)),
|
|
515
|
+
...nextOffset < bounded.length ? { nextCursor: `${snapshot}.${nextOffset}.${requestFingerprint}` } : {},
|
|
516
|
+
truncated
|
|
517
|
+
};
|
|
518
|
+
}
|
|
519
|
+
exportArchive(snapshot = this.sequence) {
|
|
520
|
+
if (snapshot < 0 || snapshot > this.sequence) throw new LineageValidationError("SNAPSHOT_UNAVAILABLE", `Snapshot ${snapshot} is unavailable`);
|
|
521
|
+
const content = {
|
|
522
|
+
contract: DATASET_LINEAGE_CONTRACT,
|
|
523
|
+
schemaVersion: DATASET_LINEAGE_SCHEMA_VERSION,
|
|
524
|
+
snapshot,
|
|
525
|
+
entities: this.sorted(this.visibleAt(this.entities, snapshot)),
|
|
526
|
+
agents: this.sorted(this.visibleAt(this.agents, snapshot)),
|
|
527
|
+
activities: this.sorted(this.visibleAt(this.activities, snapshot)),
|
|
528
|
+
evidence: this.sorted(this.visibleAt(this.evidence, snapshot), (item) => `${item.id}@${item.revision}`),
|
|
529
|
+
assertions: this.sorted(this.visibleAt(this.assertions, snapshot)),
|
|
530
|
+
corrections: this.sorted(this.visibleAt(this.corrections, snapshot))
|
|
531
|
+
};
|
|
532
|
+
return clone2({ ...content, digest: computeLineageDigest(content) });
|
|
533
|
+
}
|
|
534
|
+
static importArchive(archive, options = {}) {
|
|
535
|
+
if (archive.contract !== DATASET_LINEAGE_CONTRACT || !archive.schemaVersion.startsWith("1.")) throw new LineageValidationError("VALUE_INVALID", "Unsupported lineage archive contract");
|
|
536
|
+
const { digest: digest2, ...content } = archive;
|
|
537
|
+
if (computeLineageDigest(content) !== digest2) throw new LineageValidationError("EVIDENCE_DIGEST_MISMATCH", "Lineage archive digest does not match canonical content");
|
|
538
|
+
const ledger = new _DatasetLineageLedger(options);
|
|
539
|
+
for (const item of archive.agents) ledger.appendAgent(item);
|
|
540
|
+
for (const item of archive.entities) ledger.appendEntity(item);
|
|
541
|
+
for (const item of archive.activities) ledger.appendActivity(item);
|
|
542
|
+
for (const item of archive.evidence) ledger.appendEvidence(item);
|
|
543
|
+
const replacementKeys = new Set(archive.corrections.flatMap((item) => item.replacementAssertionId && item.replacementRevision ? [`${item.replacementAssertionId}@${item.replacementRevision}`] : []));
|
|
544
|
+
for (const item of archive.assertions.filter((item2) => !replacementKeys.has(`${item2.id}@${item2.revision}`))) ledger.appendAssertion(item);
|
|
545
|
+
const pending = [...archive.corrections];
|
|
546
|
+
while (pending.length > 0) {
|
|
547
|
+
const index = pending.findIndex((item) => ledger.assertions.has(`${item.assertionId}@${item.assertionRevision}`));
|
|
548
|
+
if (index < 0) throw new LineageValidationError("CORRECTION_INVALID", "Correction history contains an unreachable assertion revision");
|
|
549
|
+
const correction = pending.splice(index, 1)[0];
|
|
550
|
+
ledger.appendCorrection(correction);
|
|
551
|
+
if (correction.replacementAssertionId && correction.replacementRevision) {
|
|
552
|
+
const key = `${correction.replacementAssertionId}@${correction.replacementRevision}`;
|
|
553
|
+
if (!ledger.assertions.has(key)) {
|
|
554
|
+
const replacement = archive.assertions.find((item) => `${item.id}@${item.revision}` === key);
|
|
555
|
+
if (!replacement) throw new LineageValidationError("IDENTITY_DANGLING", `Missing replacement assertion ${key}`);
|
|
556
|
+
ledger.appendAssertion(replacement);
|
|
557
|
+
}
|
|
558
|
+
}
|
|
559
|
+
}
|
|
560
|
+
return ledger;
|
|
561
|
+
}
|
|
562
|
+
project(capabilities, snapshot = this.sequence) {
|
|
563
|
+
const source = this.exportArchive(snapshot);
|
|
564
|
+
const losses = [];
|
|
565
|
+
const entities = source.entities.filter((item, index) => {
|
|
566
|
+
const keep = capabilities.entityKinds.includes(item.kind);
|
|
567
|
+
if (!keep) losses.push({ path: `/entities/${index}`, reason: "unsupported-entity-kind", canonicalDigest: computeLineageDigest(item) });
|
|
568
|
+
return keep;
|
|
569
|
+
});
|
|
570
|
+
const entityIds = new Set(entities.map((item) => item.id));
|
|
571
|
+
const assertions = source.assertions.filter((item, index) => {
|
|
572
|
+
const reason = !capabilities.relationshipKinds.includes(item.relationship) ? "unsupported-relationship-kind" : !capabilities.assertionKinds.includes(item.assertionKind) ? "unsupported-assertion-kind" : !entityIds.has(item.sourceEntityId) || !entityIds.has(item.targetEntityId) ? "unsupported-entity-kind" : void 0;
|
|
573
|
+
if (reason) losses.push({ path: `/assertions/${index}`, reason, canonicalDigest: computeLineageDigest(item) });
|
|
574
|
+
return !reason;
|
|
575
|
+
});
|
|
576
|
+
const evidenceIds = new Set(assertions.flatMap((item) => item.evidence.map((reference) => `${reference.evidenceId}@${reference.revision}`)));
|
|
577
|
+
const evidence = capabilities.preservesEvidence ? source.evidence.filter((item) => evidenceIds.has(`${item.id}@${item.revision}`)) : [];
|
|
578
|
+
if (!capabilities.preservesEvidence && evidenceIds.size > 0) losses.push({ path: "/evidence", reason: "evidence-omitted", canonicalDigest: computeLineageDigest(source.evidence) });
|
|
579
|
+
const assertionIds = new Set(assertions.map((item) => item.id));
|
|
580
|
+
const corrections = capabilities.preservesCorrections ? source.corrections.filter((item) => assertionIds.has(item.assertionId)) : [];
|
|
581
|
+
if (!capabilities.preservesCorrections && source.corrections.length > 0) losses.push({ path: "/corrections", reason: "corrections-omitted", canonicalDigest: computeLineageDigest(source.corrections) });
|
|
582
|
+
losses.sort((left, right) => left.path.localeCompare(right.path) || left.reason.localeCompare(right.reason));
|
|
583
|
+
const projectionContent = { entities, assertions, evidence, corrections };
|
|
584
|
+
const projectionDigest = computeLineageDigest(projectionContent);
|
|
585
|
+
return clone2({
|
|
586
|
+
contract: DATASET_LINEAGE_CONTRACT,
|
|
587
|
+
schemaVersion: DATASET_LINEAGE_SCHEMA_VERSION,
|
|
588
|
+
canonical: false,
|
|
589
|
+
regenerable: true,
|
|
590
|
+
sourceDigest: source.digest,
|
|
591
|
+
...projectionContent,
|
|
592
|
+
digest: projectionDigest,
|
|
593
|
+
lossReceipt: {
|
|
594
|
+
contract: DATASET_LINEAGE_CONTRACT,
|
|
595
|
+
schemaVersion: DATASET_LINEAGE_SCHEMA_VERSION,
|
|
596
|
+
sourceDigest: source.digest,
|
|
597
|
+
projectionDigest,
|
|
598
|
+
lossless: losses.length === 0,
|
|
599
|
+
losses
|
|
600
|
+
}
|
|
601
|
+
});
|
|
602
|
+
}
|
|
603
|
+
append(map, key, value) {
|
|
604
|
+
this.sequence += 1;
|
|
605
|
+
map.set(key, { sequence: this.sequence, value: clone2(value) });
|
|
606
|
+
return this.sequence;
|
|
607
|
+
}
|
|
608
|
+
ensureUnique(map, key, label) {
|
|
609
|
+
if (map.has(key)) throw new LineageValidationError("IDENTITY_DUPLICATE", `Duplicate ${label} identity ${key}`);
|
|
610
|
+
}
|
|
611
|
+
visibleAt(map, snapshot) {
|
|
612
|
+
return new Map([...map].filter(([, entry]) => entry.sequence <= snapshot).map(([key, entry]) => [key, clone2(entry.value)]));
|
|
613
|
+
}
|
|
614
|
+
sorted(map, key = (item) => item.id) {
|
|
615
|
+
return [...map.values()].sort((left, right) => key(left).localeCompare(key(right))).map(clone2);
|
|
616
|
+
}
|
|
617
|
+
hasReplacementPermission(assertionId, revision) {
|
|
618
|
+
return [...this.corrections.values()].some((entry) => entry.value.action === "correct" && entry.value.replacementAssertionId === assertionId && entry.value.replacementRevision === revision);
|
|
619
|
+
}
|
|
620
|
+
statusAt(assertionId, revision, snapshot) {
|
|
621
|
+
const latest = [...this.corrections.values()].filter((entry) => entry.sequence <= snapshot && entry.value.assertionId === assertionId && entry.value.assertionRevision === revision).sort((left, right) => right.sequence - left.sequence)[0]?.value;
|
|
622
|
+
if (!latest) return "active";
|
|
623
|
+
return latest.action === "correct" ? "corrected" : latest.action === "retract" ? "retracted" : "superseded";
|
|
624
|
+
}
|
|
625
|
+
validateTraversalRequest(request) {
|
|
626
|
+
if (request.startEntityIds.length === 0) throw new LineageValidationError("IDENTITY_REQUIRED", "At least one start entity is required");
|
|
627
|
+
if (!Number.isSafeInteger(request.maximumDepth) || request.maximumDepth < 0 || request.maximumDepth > this.maximumTraversalDepth) throw new LineageValidationError("TRAVERSAL_LIMIT_EXCEEDED", `maximumDepth exceeds ${this.maximumTraversalDepth}`);
|
|
628
|
+
if (!Number.isSafeInteger(request.maximumResults) || request.maximumResults < 1 || request.maximumResults > this.maximumTraversalResults) throw new LineageValidationError("TRAVERSAL_LIMIT_EXCEEDED", `maximumResults exceeds ${this.maximumTraversalResults}`);
|
|
629
|
+
if (!Number.isSafeInteger(request.pageSize) || request.pageSize < 1 || request.pageSize > this.maximumPageSize || request.pageSize > request.maximumResults) throw new LineageValidationError("TRAVERSAL_LIMIT_EXCEEDED", `pageSize exceeds ${this.maximumPageSize}`);
|
|
630
|
+
}
|
|
631
|
+
traversalFingerprint(request, snapshot) {
|
|
632
|
+
const parameters = { ...request };
|
|
633
|
+
delete parameters.cursor;
|
|
634
|
+
delete parameters.snapshot;
|
|
635
|
+
return computeLineageDigest({ ...parameters, startEntityIds: [...new Set(parameters.startEntityIds)].sort(), snapshot });
|
|
636
|
+
}
|
|
637
|
+
decodeCursor(cursor, fingerprint, snapshot) {
|
|
638
|
+
const match = /^(\d+)\.(\d+)\.(sha256:[0-9a-f]{64})$/.exec(cursor);
|
|
639
|
+
if (!match || Number(match[1]) !== snapshot || match[3] !== fingerprint) throw new LineageValidationError("CURSOR_INVALID", "Cursor does not match this query and snapshot");
|
|
640
|
+
const offset = Number(match[2]);
|
|
641
|
+
if (!Number.isSafeInteger(offset) || offset < 0) throw new LineageValidationError("CURSOR_INVALID", "Cursor offset is invalid");
|
|
642
|
+
return offset;
|
|
643
|
+
}
|
|
644
|
+
};
|
|
645
|
+
function compareAssertions(left, right) {
|
|
646
|
+
return left.sourceEntityId.localeCompare(right.sourceEntityId) || left.targetEntityId.localeCompare(right.targetEntityId) || left.relationship.localeCompare(right.relationship) || left.id.localeCompare(right.id);
|
|
647
|
+
}
|
|
648
|
+
|
|
649
|
+
// src/dataset-execution-capabilities.ts
|
|
650
|
+
var DATASET_EXECUTION_CONTRACT = "fortemi.dataset-execution-capabilities/v1";
|
|
651
|
+
var DATASET_EXECUTION_SCHEMA_VERSION = "1.0.0";
|
|
652
|
+
var DATASET_EXECUTION_CAPABILITY_IDS = [
|
|
653
|
+
"ingest.full",
|
|
654
|
+
"ingest.snapshot",
|
|
655
|
+
"ingest.incremental",
|
|
656
|
+
"ingest.stream",
|
|
657
|
+
"schema.inspect",
|
|
658
|
+
"identity.stable-revision",
|
|
659
|
+
"identity.record",
|
|
660
|
+
"mutation.upsert",
|
|
661
|
+
"mutation.tombstone",
|
|
662
|
+
"mutation.reconcile",
|
|
663
|
+
"checkpoint.read",
|
|
664
|
+
"checkpoint.write",
|
|
665
|
+
"execution.cancel",
|
|
666
|
+
"rejection.record",
|
|
667
|
+
"index.lexical",
|
|
668
|
+
"index.chunk",
|
|
669
|
+
"index.vector",
|
|
670
|
+
"index.hybrid",
|
|
671
|
+
"index.rerank",
|
|
672
|
+
"index.graph",
|
|
673
|
+
"index.community",
|
|
674
|
+
"lineage.dataset",
|
|
675
|
+
"lineage.record",
|
|
676
|
+
"lineage.field",
|
|
677
|
+
"lineage.relationship-evidence",
|
|
678
|
+
"transaction.atomic-batch",
|
|
679
|
+
"privacy.pre-materialization-filter",
|
|
680
|
+
"pagination.cursor",
|
|
681
|
+
"ordering.deterministic"
|
|
682
|
+
];
|
|
683
|
+
var ID_SET = new Set(DATASET_EXECUTION_CAPABILITY_IDS);
|
|
684
|
+
var LIMIT_KEYS = ["maxInputBytes", "maxRecordBytes", "maxBatchRecords", "maxConcurrency", "maxPageSize", "maxTraversalDepth"];
|
|
685
|
+
function major(version) {
|
|
686
|
+
const match = /^(\d+)\.(\d+)\.(\d+)(?:[-+].*)?$/.exec(version);
|
|
687
|
+
return match ? Number(match[1]) : null;
|
|
688
|
+
}
|
|
689
|
+
function compareVersions(left, right) {
|
|
690
|
+
const parse = (value) => {
|
|
691
|
+
const match = /^(\d+)\.(\d+)\.(\d+)(?:[-+].*)?$/.exec(value);
|
|
692
|
+
return match ? [Number(match[1]), Number(match[2]), Number(match[3])] : null;
|
|
693
|
+
};
|
|
694
|
+
const a = parse(left);
|
|
695
|
+
const b = parse(right);
|
|
696
|
+
if (!a || !b) return null;
|
|
697
|
+
for (let index = 0; index < 3; index++) {
|
|
698
|
+
if (a[index] !== b[index]) return a[index] > b[index] ? 1 : -1;
|
|
699
|
+
}
|
|
700
|
+
return 0;
|
|
701
|
+
}
|
|
702
|
+
function isSupported(capability) {
|
|
703
|
+
return capability !== void 0 && capability.status !== "unsupported";
|
|
704
|
+
}
|
|
705
|
+
function validateDatasetExecutionDescriptor(descriptor) {
|
|
706
|
+
const diagnostics = [];
|
|
707
|
+
if (descriptor.contract !== DATASET_EXECUTION_CONTRACT) {
|
|
708
|
+
diagnostics.push({ code: "CONTRACT_MAJOR_UNSUPPORTED", path: "/contract", message: `Unsupported contract ${String(descriptor.contract)}` });
|
|
709
|
+
}
|
|
710
|
+
if (major(descriptor.schemaVersion) !== 1) {
|
|
711
|
+
diagnostics.push({ code: "SCHEMA_VERSION_UNSUPPORTED", path: "/schemaVersion", message: `Unsupported descriptor schema version ${descriptor.schemaVersion}` });
|
|
712
|
+
}
|
|
713
|
+
const capabilities = /* @__PURE__ */ new Map();
|
|
714
|
+
descriptor.capabilities.forEach((capability, index) => {
|
|
715
|
+
if (!ID_SET.has(capability.id) || compareVersions(capability.version, capability.version) === null) {
|
|
716
|
+
diagnostics.push({ code: "DESCRIPTOR_INVALID", path: `/capabilities/${index}`, message: `Invalid capability declaration ${capability.id}` });
|
|
717
|
+
return;
|
|
718
|
+
}
|
|
719
|
+
if (capabilities.has(capability.id)) {
|
|
720
|
+
diagnostics.push({ code: "CAPABILITY_DUPLICATE", capability: capability.id, path: `/capabilities/${index}/id`, message: `Capability ${capability.id} is declared more than once` });
|
|
721
|
+
}
|
|
722
|
+
capabilities.set(capability.id, capability);
|
|
723
|
+
for (const key of LIMIT_KEYS) {
|
|
724
|
+
const value = capability.limits?.[key];
|
|
725
|
+
if (value !== void 0 && (!Number.isSafeInteger(value) || value < 0)) {
|
|
726
|
+
diagnostics.push({ code: "DESCRIPTOR_INVALID", capability: capability.id, path: `/capabilities/${index}/limits/${key}`, message: `${key} must be a non-negative safe integer` });
|
|
727
|
+
}
|
|
728
|
+
}
|
|
729
|
+
if (capability.status !== "unsupported" && capability.evidence.length === 0) {
|
|
730
|
+
diagnostics.push({ code: "DESCRIPTOR_INVALID", capability: capability.id, path: `/capabilities/${index}/evidence`, message: `Supported capability ${capability.id} requires evidence` });
|
|
731
|
+
}
|
|
732
|
+
for (const evidenceId of capability.evidence) {
|
|
733
|
+
if (!descriptor.evidence.some((evidence) => evidence.id === evidenceId)) {
|
|
734
|
+
diagnostics.push({ code: "DESCRIPTOR_INVALID", capability: capability.id, path: `/capabilities/${index}/evidence`, message: `Unknown evidence ${evidenceId}` });
|
|
735
|
+
}
|
|
736
|
+
}
|
|
737
|
+
});
|
|
738
|
+
const requireTogether = (source, requirements) => {
|
|
739
|
+
if (!isSupported(capabilities.get(source))) return;
|
|
740
|
+
for (const requirement of requirements) {
|
|
741
|
+
if (!isSupported(capabilities.get(requirement))) {
|
|
742
|
+
diagnostics.push({ code: "CAPABILITY_INCONSISTENT", capability: source, message: `${source} requires ${requirement}` });
|
|
743
|
+
}
|
|
744
|
+
}
|
|
745
|
+
};
|
|
746
|
+
requireTogether("ingest.incremental", ["identity.stable-revision", "checkpoint.read", "checkpoint.write"]);
|
|
747
|
+
requireTogether("lineage.field", ["lineage.relationship-evidence"]);
|
|
748
|
+
requireTogether("mutation.reconcile", ["mutation.upsert", "mutation.tombstone"]);
|
|
749
|
+
requireTogether("index.hybrid", ["index.lexical", "index.vector"]);
|
|
750
|
+
if (descriptor.guarantees.transaction === "atomic-batch" && !isSupported(capabilities.get("transaction.atomic-batch"))) {
|
|
751
|
+
diagnostics.push({ code: "CAPABILITY_INCONSISTENT", capability: "transaction.atomic-batch", message: "Atomic-batch guarantee requires transaction.atomic-batch capability" });
|
|
752
|
+
}
|
|
753
|
+
if (descriptor.runtime.plane === "static-cache" && descriptor.runtime.dataClass !== "static-cache") {
|
|
754
|
+
diagnostics.push({ code: "CAPABILITY_INCONSISTENT", message: "Static-cache execution plane must declare static-cache data class" });
|
|
755
|
+
}
|
|
756
|
+
if (descriptor.runtime.plane === "live-remote-persistence" && descriptor.runtime.maturity === "stable" && !descriptor.evidence.some((item) => item.kind === "live-qualification")) {
|
|
757
|
+
diagnostics.push({ code: "CAPABILITY_INCONSISTENT", message: "Stable live remote persistence requires live qualification evidence" });
|
|
758
|
+
}
|
|
759
|
+
return diagnostics;
|
|
760
|
+
}
|
|
761
|
+
function assessRequirement(requirement, capabilities) {
|
|
762
|
+
const capability = capabilities.get(requirement.id);
|
|
763
|
+
if (!isSupported(capability)) {
|
|
764
|
+
return { ok: false, reason: "unsupported", diagnostics: [{ code: "REQUIRED_CAPABILITY_MISSING", capability: requirement.id, message: `Capability ${requirement.id} is unsupported` }] };
|
|
765
|
+
}
|
|
766
|
+
if (requirement.minimumVersion) {
|
|
767
|
+
const comparison = compareVersions(capability.version, requirement.minimumVersion);
|
|
768
|
+
if (comparison === null || comparison < 0) {
|
|
769
|
+
return { ok: false, reason: "version-insufficient", diagnostics: [{ code: "CAPABILITY_VERSION_INSUFFICIENT", capability: requirement.id, message: `${capability.version} does not satisfy ${requirement.minimumVersion}` }] };
|
|
770
|
+
}
|
|
771
|
+
}
|
|
772
|
+
for (const key of LIMIT_KEYS) {
|
|
773
|
+
const required = requirement.minimumLimits?.[key];
|
|
774
|
+
if (required !== void 0 && (capability.limits?.[key] ?? -1) < required) {
|
|
775
|
+
return { ok: false, reason: "limit-insufficient", diagnostics: [{ code: "CAPABILITY_LIMIT_INSUFFICIENT", capability: requirement.id, path: `/minimumLimits/${key}`, message: `${key} does not satisfy ${required}` }] };
|
|
776
|
+
}
|
|
777
|
+
}
|
|
778
|
+
return { ok: true, diagnostics: [] };
|
|
779
|
+
}
|
|
780
|
+
function negotiateDatasetExecutionCapabilities(descriptor, request) {
|
|
781
|
+
const diagnostics = validateDatasetExecutionDescriptor(descriptor);
|
|
782
|
+
if (request.contract !== DATASET_EXECUTION_CONTRACT) {
|
|
783
|
+
diagnostics.push({ code: "CONTRACT_MAJOR_UNSUPPORTED", path: "/contract", message: `Unsupported request contract ${String(request.contract)}` });
|
|
784
|
+
}
|
|
785
|
+
const capabilities = new Map(descriptor.capabilities.map((capability) => [capability.id, capability]));
|
|
786
|
+
const selected = [];
|
|
787
|
+
const degradations = [];
|
|
788
|
+
if (diagnostics.length === 0) {
|
|
789
|
+
for (const requirement of request.required) {
|
|
790
|
+
const assessment = assessRequirement(requirement, capabilities);
|
|
791
|
+
if (assessment.ok) selected.push(requirement.id);
|
|
792
|
+
else diagnostics.push(...assessment.diagnostics);
|
|
793
|
+
}
|
|
794
|
+
for (const requirement of request.optional ?? []) {
|
|
795
|
+
const assessment = assessRequirement(requirement, capabilities);
|
|
796
|
+
if (assessment.ok) {
|
|
797
|
+
selected.push(requirement.id);
|
|
798
|
+
continue;
|
|
799
|
+
}
|
|
800
|
+
const fallback = requirement.fallback?.find((id) => isSupported(capabilities.get(id)));
|
|
801
|
+
if (fallback) selected.push(fallback);
|
|
802
|
+
degradations.push({
|
|
803
|
+
requested: requirement.id,
|
|
804
|
+
...fallback ? { selected: fallback } : {},
|
|
805
|
+
reason: assessment.reason,
|
|
806
|
+
changedGuarantees: fallback ? [`${requirement.id} replaced by ${fallback}`] : [`${requirement.id} omitted`]
|
|
807
|
+
});
|
|
808
|
+
}
|
|
809
|
+
}
|
|
810
|
+
return {
|
|
811
|
+
contract: DATASET_EXECUTION_CONTRACT,
|
|
812
|
+
accepted: diagnostics.length === 0,
|
|
813
|
+
runtime: { ...descriptor.runtime },
|
|
814
|
+
selected: [...new Set(selected)],
|
|
815
|
+
degradations,
|
|
816
|
+
diagnostics
|
|
817
|
+
};
|
|
818
|
+
}
|
|
819
|
+
|
|
820
|
+
// src/dataset-execution-descriptors.ts
|
|
821
|
+
var FORTEMI_BROWSER_LOCAL_DATASET_EXECUTION_DESCRIPTOR = {
|
|
822
|
+
contract: DATASET_EXECUTION_CONTRACT,
|
|
823
|
+
schemaVersion: DATASET_EXECUTION_SCHEMA_VERSION,
|
|
824
|
+
runtime: { id: "fortemi-browser", version: "2026.8.0", plane: "browser-local-archive", dataClass: "canonical", maturity: "stable" },
|
|
825
|
+
guarantees: { transaction: "atomic-batch", isolation: "serializable", durability: "wal", availability: "local-process", ordering: "backend-cursor" },
|
|
826
|
+
capabilities: [
|
|
827
|
+
{ id: "ingest.full", version: "1.0.0", status: "supported", limits: { maxBatchRecords: 1e3, maxConcurrency: 1 }, evidence: ["browser-conformance"] },
|
|
828
|
+
{ id: "identity.record", version: "1.0.0", status: "supported", evidence: ["browser-conformance"] },
|
|
829
|
+
{ id: "mutation.upsert", version: "1.0.0", status: "supported", evidence: ["browser-conformance"] },
|
|
830
|
+
{ id: "transaction.atomic-batch", version: "1.0.0", status: "supported", evidence: ["browser-conformance"] },
|
|
831
|
+
{ id: "ordering.deterministic", version: "1.0.0", status: "supported", evidence: ["browser-conformance"] }
|
|
832
|
+
],
|
|
833
|
+
evidence: [{ id: "browser-conformance", kind: "conformance-report", uri: "fortemi://conformance/browser-local/v1" }]
|
|
834
|
+
};
|
|
835
|
+
var FORTEMI_STATIC_CACHE_DATASET_EXECUTION_DESCRIPTOR = {
|
|
836
|
+
contract: DATASET_EXECUTION_CONTRACT,
|
|
837
|
+
schemaVersion: DATASET_EXECUTION_SCHEMA_VERSION,
|
|
838
|
+
runtime: { id: "fortemi-static", version: "2026.8.0", plane: "static-cache", dataClass: "static-cache", maturity: "stable" },
|
|
839
|
+
guarantees: { transaction: "none", isolation: "snapshot", durability: "filesystem", availability: "local-process", ordering: "stable-identity" },
|
|
840
|
+
capabilities: [
|
|
841
|
+
{ id: "index.lexical", version: "1.0.0", status: "supported", limits: { maxPageSize: 1e3 }, evidence: ["static-conformance"] },
|
|
842
|
+
{ id: "pagination.cursor", version: "1.0.0", status: "supported", evidence: ["static-conformance"] },
|
|
843
|
+
{ id: "ordering.deterministic", version: "1.0.0", status: "supported", evidence: ["static-conformance"] }
|
|
844
|
+
],
|
|
845
|
+
evidence: [{ id: "static-conformance", kind: "conformance-report", uri: "fortemi://conformance/static-cache/v1" }]
|
|
846
|
+
};
|
|
847
|
+
var FORTEMI_PORTABLE_SHARD_DATASET_EXECUTION_DESCRIPTOR = {
|
|
848
|
+
contract: DATASET_EXECUTION_CONTRACT,
|
|
849
|
+
schemaVersion: DATASET_EXECUTION_SCHEMA_VERSION,
|
|
850
|
+
runtime: { id: "fortemi-shard", version: "2026.8.0", plane: "portable-shard", dataClass: "portable-projection", maturity: "stable" },
|
|
851
|
+
guarantees: { transaction: "none", isolation: "snapshot", durability: "filesystem", availability: "local-process", ordering: "stable-identity" },
|
|
852
|
+
capabilities: [
|
|
853
|
+
{ id: "schema.inspect", version: "1.0.0", status: "supported", evidence: ["shard-conformance"] },
|
|
854
|
+
{ id: "lineage.dataset", version: "1.0.0", status: "experimental", evidence: ["shard-conformance"] },
|
|
855
|
+
{ id: "ordering.deterministic", version: "1.0.0", status: "supported", evidence: ["shard-conformance"] }
|
|
856
|
+
],
|
|
857
|
+
evidence: [{ id: "shard-conformance", kind: "conformance-report", uri: "fortemi://conformance/knowledge-shard/v1" }]
|
|
858
|
+
};
|
|
859
|
+
var DATASET_MATERIALIZATION_CONTRACT = "fortemi.dataset-materialization-profile/v1";
|
|
860
|
+
var DATASET_MATERIALIZATION_SCHEMA_VERSION = "1.0.0";
|
|
861
|
+
var DATASET_MATERIALIZATION_KINDS = [
|
|
862
|
+
"chunking",
|
|
863
|
+
"lexical",
|
|
864
|
+
"vector",
|
|
865
|
+
"hybrid",
|
|
866
|
+
"rerank",
|
|
867
|
+
"entity-relationship-extraction",
|
|
868
|
+
"graph-retrieval",
|
|
869
|
+
"community"
|
|
870
|
+
];
|
|
871
|
+
var DatasetMaterializationError = class extends Error {
|
|
872
|
+
constructor(code, message) {
|
|
873
|
+
super(message);
|
|
874
|
+
this.code = code;
|
|
875
|
+
this.name = "DatasetMaterializationError";
|
|
876
|
+
}
|
|
877
|
+
};
|
|
878
|
+
var encoder = new TextEncoder();
|
|
879
|
+
function canonicalJson3(value) {
|
|
880
|
+
if (value === null || typeof value !== "object") return JSON.stringify(value);
|
|
881
|
+
if (Array.isArray(value)) return `[${value.map(canonicalJson3).join(",")}]`;
|
|
882
|
+
return `{${Object.entries(value).filter(([, item]) => item !== void 0).sort(([left], [right]) => left.localeCompare(right)).map(([key, item]) => `${JSON.stringify(key)}:${canonicalJson3(item)}`).join(",")}}`;
|
|
883
|
+
}
|
|
884
|
+
function digestDatasetMaterializationValue(value) {
|
|
885
|
+
return computeHash(encoder.encode(canonicalJson3(value)));
|
|
886
|
+
}
|
|
887
|
+
function profileDiagnostics(profile) {
|
|
888
|
+
const diagnostics = [];
|
|
889
|
+
const requiredBoundaries = ["chunking", "model-invocation", "index-persistence"];
|
|
890
|
+
if (profile.contract !== DATASET_MATERIALIZATION_CONTRACT || !/^1\./.test(profile.schemaVersion)) {
|
|
891
|
+
diagnostics.push({ code: "PROFILE_INVALID", path: "/contract", message: "Unsupported profile contract or schema major" });
|
|
892
|
+
}
|
|
893
|
+
if (profile.output.dataClass !== "regenerable-index" || profile.output.canonicalMutation !== false) {
|
|
894
|
+
diagnostics.push({ code: "PROFILE_INVALID", path: "/output", message: "Profiles may only produce derived, regenerable artifacts" });
|
|
895
|
+
}
|
|
896
|
+
for (const boundary of requiredBoundaries) {
|
|
897
|
+
if (!profile.privacy.filtersBefore.includes(boundary)) {
|
|
898
|
+
diagnostics.push({ code: "PROFILE_INVALID", path: "/privacy/filtersBefore", message: `Privacy filtering must precede ${boundary}` });
|
|
899
|
+
}
|
|
900
|
+
}
|
|
901
|
+
if (profile.determinism.class === "seeded" && profile.determinism.seedRequired !== true) {
|
|
902
|
+
diagnostics.push({ code: "PROFILE_INVALID", path: "/determinism/seedRequired", message: "Seeded profiles must require a seed" });
|
|
903
|
+
}
|
|
904
|
+
if (profile.resourceLimits.maxInputBytes < 0 || profile.resourceLimits.maxRecords < 0 || profile.resourceLimits.maxConcurrency < 1) {
|
|
905
|
+
diagnostics.push({ code: "PROFILE_INVALID", path: "/resourceLimits", message: "Resource limits must be non-negative and concurrency must be positive" });
|
|
906
|
+
}
|
|
907
|
+
return diagnostics;
|
|
908
|
+
}
|
|
909
|
+
function negotiateDatasetMaterializationProfile(request) {
|
|
910
|
+
const candidates = [request.profile, ...request.fallbackProfiles ?? []];
|
|
911
|
+
const primaryDiagnostics = profileDiagnostics(request.profile);
|
|
912
|
+
if (!request.profile.operations.includes(request.operation)) {
|
|
913
|
+
primaryDiagnostics.push({ code: "OPERATION_UNSUPPORTED", path: "/operations", message: `${request.operation} is not supported by ${request.profile.id}` });
|
|
914
|
+
}
|
|
915
|
+
if (request.profile.status === "unsupported") {
|
|
916
|
+
primaryDiagnostics.push({ code: "PROFILE_UNSUPPORTED", path: "/status", message: `${request.profile.id} is unsupported` });
|
|
917
|
+
}
|
|
918
|
+
for (let index = 0; index < candidates.length; index += 1) {
|
|
919
|
+
const candidate = candidates[index];
|
|
920
|
+
const diagnostics = index === 0 ? primaryDiagnostics : profileDiagnostics(candidate);
|
|
921
|
+
if (!candidate.operations.includes(request.operation) || candidate.status === "unsupported" || diagnostics.length > 0) continue;
|
|
922
|
+
const capability = negotiateDatasetExecutionCapabilities(request.runtime, {
|
|
923
|
+
contract: DATASET_EXECUTION_CONTRACT,
|
|
924
|
+
required: candidate.requiredRuntimeCapabilities,
|
|
925
|
+
optional: candidate.optionalRuntimeCapabilities
|
|
926
|
+
});
|
|
927
|
+
if (capability.accepted) {
|
|
928
|
+
const fallback = index > 0;
|
|
929
|
+
return {
|
|
930
|
+
accepted: true,
|
|
931
|
+
requestedProfile: request.profile.id,
|
|
932
|
+
selectedProfile: candidate.id,
|
|
933
|
+
runtime: { ...request.runtime.runtime },
|
|
934
|
+
degraded: fallback || capability.degradations.length > 0,
|
|
935
|
+
degradations: capability.degradations,
|
|
936
|
+
diagnostics: fallback ? primaryDiagnostics : []
|
|
937
|
+
};
|
|
938
|
+
}
|
|
939
|
+
if (index === 0) primaryDiagnostics.push(...capability.diagnostics);
|
|
940
|
+
}
|
|
941
|
+
return {
|
|
942
|
+
accepted: false,
|
|
943
|
+
requestedProfile: request.profile.id,
|
|
944
|
+
runtime: { ...request.runtime.runtime },
|
|
945
|
+
degraded: false,
|
|
946
|
+
degradations: [],
|
|
947
|
+
diagnostics: [...primaryDiagnostics, { code: "NO_FALLBACK_PROFILE", message: `No compatible ${request.operation} profile is available` }]
|
|
948
|
+
};
|
|
949
|
+
}
|
|
950
|
+
function clone3(value) {
|
|
951
|
+
return structuredClone(value);
|
|
952
|
+
}
|
|
953
|
+
function assertResources(snapshot, profile) {
|
|
954
|
+
const inputBytes = encoder.encode(canonicalJson3(snapshot.records)).byteLength;
|
|
955
|
+
if (snapshot.records.length > profile.resourceLimits.maxRecords || inputBytes > profile.resourceLimits.maxInputBytes) {
|
|
956
|
+
throw new DatasetMaterializationError("RESOURCE_LIMIT_EXCEEDED", `Input exceeds profile ${profile.id} resource limits`);
|
|
957
|
+
}
|
|
958
|
+
return inputBytes;
|
|
959
|
+
}
|
|
960
|
+
function stableArtifacts(artifacts) {
|
|
961
|
+
return [...artifacts].sort((left, right) => {
|
|
962
|
+
const score = (right.score ?? Number.NEGATIVE_INFINITY) - (left.score ?? Number.NEGATIVE_INFINITY);
|
|
963
|
+
return score || left.logicalId.localeCompare(right.logicalId) || left.digest.localeCompare(right.digest);
|
|
964
|
+
});
|
|
965
|
+
}
|
|
966
|
+
async function executeDatasetMaterialization(request, runtime, adapter, authorize, options = {}) {
|
|
967
|
+
const negotiation = negotiateDatasetMaterializationProfile({ operation: "build", profile: request.profile, runtime, fallbackProfiles: options.fallbackProfiles });
|
|
968
|
+
if (!negotiation.accepted || !negotiation.selectedProfile) {
|
|
969
|
+
throw new DatasetMaterializationError("NEGOTIATION_FAILED", negotiation.diagnostics.map((item) => item.message).join("; "));
|
|
970
|
+
}
|
|
971
|
+
const selected = [request.profile, ...options.fallbackProfiles ?? []].find((item) => item.id === negotiation.selectedProfile);
|
|
972
|
+
try {
|
|
973
|
+
const validateConfiguration = new Ajv20202({ strict: true, allErrors: true }).compile(selected.configurationSchema);
|
|
974
|
+
if (!validateConfiguration(request.configuration)) {
|
|
975
|
+
throw new DatasetMaterializationError("CONFIGURATION_INVALID", `Configuration does not satisfy ${selected.id}: ${JSON.stringify(validateConfiguration.errors)}`);
|
|
976
|
+
}
|
|
977
|
+
} catch (error) {
|
|
978
|
+
if (error instanceof DatasetMaterializationError) throw error;
|
|
979
|
+
throw new DatasetMaterializationError("PROFILE_INVALID", `Invalid configuration schema for ${selected.id}: ${error instanceof Error ? error.message : String(error)}`);
|
|
980
|
+
}
|
|
981
|
+
const inputBytes = assertResources(request.snapshot, selected);
|
|
982
|
+
const allowed = [];
|
|
983
|
+
const denied = [];
|
|
984
|
+
for (const record of request.snapshot.records) {
|
|
985
|
+
if (await authorize(Object.freeze(clone3(record)))) allowed.push(clone3(record));
|
|
986
|
+
else denied.push(record.digest);
|
|
987
|
+
}
|
|
988
|
+
const privacy = {
|
|
989
|
+
policyId: "caller-authorization",
|
|
990
|
+
policyVersion: "1.0.0",
|
|
991
|
+
policyDigest: digestDatasetMaterializationValue({ policy: "caller-authorization", version: "1.0.0" }),
|
|
992
|
+
allowedRecordDigests: allowed.map((item) => item.digest).sort(),
|
|
993
|
+
deniedRecordDigests: denied.sort(),
|
|
994
|
+
evaluatedAt: (options.now ?? (() => (/* @__PURE__ */ new Date()).toISOString()))()
|
|
995
|
+
};
|
|
996
|
+
const affected = request.affected ?? {
|
|
997
|
+
sourceRevisions: [request.snapshot.revision],
|
|
998
|
+
recordDigests: allowed.map((item) => item.digest).sort(),
|
|
999
|
+
chunkDigests: []
|
|
1000
|
+
};
|
|
1001
|
+
const detachedSnapshot = clone3({ ...request.snapshot, records: allowed });
|
|
1002
|
+
const result = await adapter.materialize(Object.freeze({
|
|
1003
|
+
snapshot: detachedSnapshot,
|
|
1004
|
+
profile: clone3(selected),
|
|
1005
|
+
configuration: clone3(request.configuration),
|
|
1006
|
+
mode: request.mode,
|
|
1007
|
+
affected: clone3(affected)
|
|
1008
|
+
}));
|
|
1009
|
+
const artifacts = selected.determinism.class === "deterministic" ? stableArtifacts(result.artifacts) : clone3(result.artifacts);
|
|
1010
|
+
const digests = artifacts.map((item) => item.digest).sort();
|
|
1011
|
+
const counts = artifacts.reduce((all, item) => {
|
|
1012
|
+
all[item.kind] = (all[item.kind] ?? 0) + 1;
|
|
1013
|
+
return all;
|
|
1014
|
+
}, {});
|
|
1015
|
+
const profileDigest = digestDatasetMaterializationValue(selected);
|
|
1016
|
+
const receiptBase = {
|
|
1017
|
+
contract: DATASET_MATERIALIZATION_CONTRACT,
|
|
1018
|
+
schemaVersion: DATASET_MATERIALIZATION_SCHEMA_VERSION,
|
|
1019
|
+
runId: request.runId,
|
|
1020
|
+
processingRunId: request.processingRunId,
|
|
1021
|
+
source: { datasetId: request.snapshot.datasetId, revision: request.snapshot.revision, digests: [...request.snapshot.sourceDigests].sort() },
|
|
1022
|
+
schema: { id: request.snapshot.schemaId, version: request.snapshot.schemaVersion, digest: request.snapshot.schemaDigest },
|
|
1023
|
+
profile: { id: selected.id, version: selected.version, digest: profileDigest, configurationDigest: digestDatasetMaterializationValue(request.configuration), implementation: clone3(selected.implementation) },
|
|
1024
|
+
runtime: clone3(runtime.runtime),
|
|
1025
|
+
negotiation: { requestedProfile: negotiation.requestedProfile, selectedProfile: negotiation.selectedProfile, degraded: negotiation.degraded, degradations: clone3(negotiation.degradations) },
|
|
1026
|
+
mode: request.mode,
|
|
1027
|
+
affected: clone3(affected),
|
|
1028
|
+
output: { counts, digests, aggregateDigest: digestDatasetMaterializationValue(digests) },
|
|
1029
|
+
privacy,
|
|
1030
|
+
resources: { ...clone3(result.resources), inputBytes },
|
|
1031
|
+
createdAt: (options.now ?? (() => (/* @__PURE__ */ new Date()).toISOString()))()
|
|
1032
|
+
};
|
|
1033
|
+
const receipt = {
|
|
1034
|
+
...receiptBase,
|
|
1035
|
+
receiptId: digestDatasetMaterializationValue(receiptBase)
|
|
1036
|
+
};
|
|
1037
|
+
return { artifacts, receipt };
|
|
1038
|
+
}
|
|
1039
|
+
async function executeDatasetRetrieval(request, runtime, adapter, receiptId, fallbackProfiles = []) {
|
|
1040
|
+
const negotiation = negotiateDatasetMaterializationProfile({ operation: "query", profile: request.profile, runtime, fallbackProfiles });
|
|
1041
|
+
if (!negotiation.accepted || !negotiation.selectedProfile) throw new DatasetMaterializationError("NEGOTIATION_FAILED", negotiation.diagnostics.map((item) => item.message).join("; "));
|
|
1042
|
+
if (!adapter.retrieve) throw new DatasetMaterializationError("ADAPTER_MISMATCH", `Backend ${adapter.backend.id} cannot query`);
|
|
1043
|
+
const selected = [request.profile, ...fallbackProfiles].find((item) => item.id === negotiation.selectedProfile);
|
|
1044
|
+
const results = await adapter.retrieve(Object.freeze({ request: clone3(request), profile: clone3(selected) }));
|
|
1045
|
+
return {
|
|
1046
|
+
contract: DATASET_MATERIALIZATION_CONTRACT,
|
|
1047
|
+
schemaVersion: DATASET_MATERIALIZATION_SCHEMA_VERSION,
|
|
1048
|
+
queryId: request.queryId,
|
|
1049
|
+
requestedProfile: request.profile.id,
|
|
1050
|
+
actualProfile: selected.id,
|
|
1051
|
+
actualBackend: { ...adapter.backend, plane: runtime.runtime.plane },
|
|
1052
|
+
degraded: negotiation.degraded,
|
|
1053
|
+
...negotiation.degraded ? { fallbackReason: negotiation.diagnostics.map((item) => item.message).join("; ") || "optional capability degradation" } : {},
|
|
1054
|
+
scoreSemantics: { implementationScoped: true, comparableAcrossImplementations: false },
|
|
1055
|
+
results: selected.determinism.class === "deterministic" ? stableArtifacts(results).slice(0, request.limit) : clone3(results).slice(0, request.limit),
|
|
1056
|
+
receiptId
|
|
1057
|
+
};
|
|
1058
|
+
}
|
|
1059
|
+
function compareDatasetIncrementalParity(full, incremental) {
|
|
1060
|
+
const select = (items, kind) => items.filter((item) => !kind || item.kind === kind).map((item) => `${item.logicalId}:${item.digest}`).sort();
|
|
1061
|
+
const mismatches = [];
|
|
1062
|
+
if (canonicalJson3(select(full)) !== canonicalJson3(select(incremental))) mismatches.push("identities");
|
|
1063
|
+
if (canonicalJson3(select(full, "chunk")) !== canonicalJson3(select(incremental, "chunk"))) mismatches.push("chunks");
|
|
1064
|
+
if (canonicalJson3(select(full, "relationship")) !== canonicalJson3(select(incremental, "relationship"))) mismatches.push("relationships");
|
|
1065
|
+
if (canonicalJson3(select(full, "community")) !== canonicalJson3(select(incremental, "community"))) mismatches.push("communities");
|
|
1066
|
+
if (canonicalJson3(full.map((item) => item.digest).sort()) !== canonicalJson3(incremental.map((item) => item.digest).sort())) mismatches.push("digests");
|
|
1067
|
+
const order = (items) => items.map((item) => `${item.logicalId}:${item.digest}`);
|
|
1068
|
+
if (canonicalJson3(order(full)) !== canonicalJson3(order(incremental))) mismatches.push("ordering");
|
|
1069
|
+
return { equivalent: mismatches.length === 0, mismatches };
|
|
1070
|
+
}
|
|
1071
|
+
function validateDatasetBenchmarkEvidence(evidence) {
|
|
1072
|
+
const errors = [];
|
|
1073
|
+
if (evidence.contract !== DATASET_MATERIALIZATION_CONTRACT || !/^1\./.test(evidence.schemaVersion)) errors.push("unsupported benchmark contract");
|
|
1074
|
+
if (!evidence.correctness.passed) errors.push("correctness suite did not pass");
|
|
1075
|
+
if (evidence.freshness.sourceRevision !== evidence.corpus.revision) errors.push("benchmark evidence is stale");
|
|
1076
|
+
if (!evidence.claims.corpusScoped || evidence.claims.universalScaleLimit) errors.push("benchmark claims must remain corpus-scoped");
|
|
1077
|
+
if (evidence.measurements.length === 0) errors.push("benchmark has no measurements");
|
|
1078
|
+
return errors;
|
|
1079
|
+
}
|
|
13
1080
|
|
|
14
1081
|
// src/event-bus.ts
|
|
15
1082
|
var TypedEventBus = class {
|
|
@@ -1135,13 +2202,6 @@ async function restoreDbSnapshot(source, options = {}) {
|
|
|
1135
2202
|
const createOptions = { loadDataDir: data };
|
|
1136
2203
|
return createPGliteInstance(options.persistence ?? "memory", options.archiveName ?? "default", createOptions);
|
|
1137
2204
|
}
|
|
1138
|
-
function computeHash(data) {
|
|
1139
|
-
const digest = sha256(data);
|
|
1140
|
-
return `sha256:${bytesToHex(digest)}`;
|
|
1141
|
-
}
|
|
1142
|
-
function computeBlobHash(data) {
|
|
1143
|
-
return `blake3:${bytesToHex(blake3(data))}`;
|
|
1144
|
-
}
|
|
1145
2205
|
|
|
1146
2206
|
// src/repositories/notes-repository.ts
|
|
1147
2207
|
var NotesRepository = class {
|
|
@@ -4421,7 +5481,7 @@ function sourceHash(source) {
|
|
|
4421
5481
|
function contentDigest(content) {
|
|
4422
5482
|
return computeHash(new TextEncoder().encode(content));
|
|
4423
5483
|
}
|
|
4424
|
-
async function insertNote(tx, input, noteId,
|
|
5484
|
+
async function insertNote(tx, input, noteId, digest2) {
|
|
4425
5485
|
const originalId = generateId();
|
|
4426
5486
|
if (input.source.archive_id) {
|
|
4427
5487
|
await tx.query(
|
|
@@ -4446,7 +5506,7 @@ async function insertNote(tx, input, noteId, digest) {
|
|
|
4446
5506
|
await tx.query(
|
|
4447
5507
|
`INSERT INTO note_original (id, note_id, content, content_hash)
|
|
4448
5508
|
VALUES ($1, $2, $3, $4)`,
|
|
4449
|
-
[originalId, noteId, input.content,
|
|
5509
|
+
[originalId, noteId, input.content, digest2]
|
|
4450
5510
|
);
|
|
4451
5511
|
await tx.query(
|
|
4452
5512
|
`INSERT INTO note_revised_current (note_id, content, ai_metadata)
|
|
@@ -4549,7 +5609,7 @@ var SourceUpsertRepository = class {
|
|
|
4549
5609
|
const preview = [];
|
|
4550
5610
|
for (const [index, item] of items.entries()) {
|
|
4551
5611
|
const externalIdHash = sourceHash(item.source);
|
|
4552
|
-
const
|
|
5612
|
+
const digest2 = contentDigest(item.content);
|
|
4553
5613
|
const existing = await this.db.query(
|
|
4554
5614
|
`SELECT note_id, content_digest
|
|
4555
5615
|
FROM source_identity
|
|
@@ -4561,18 +5621,18 @@ var SourceUpsertRepository = class {
|
|
|
4561
5621
|
[item.source.tenant_id ?? "default", item.source.archive_id ?? null, item.source.namespace, item.source.external_id]
|
|
4562
5622
|
);
|
|
4563
5623
|
if (existing.rows.length === 0) {
|
|
4564
|
-
preview.push({ index, outcome: "inserted", external_id_hash: externalIdHash, content_digest:
|
|
4565
|
-
} else if (existing.rows[0].content_digest ===
|
|
4566
|
-
preview.push({ index, outcome: "unchanged", note_id: existing.rows[0].note_id, external_id_hash: externalIdHash, content_digest:
|
|
5624
|
+
preview.push({ index, outcome: "inserted", external_id_hash: externalIdHash, content_digest: digest2 });
|
|
5625
|
+
} else if (existing.rows[0].content_digest === digest2) {
|
|
5626
|
+
preview.push({ index, outcome: "unchanged", note_id: existing.rows[0].note_id, external_id_hash: externalIdHash, content_digest: digest2 });
|
|
4567
5627
|
} else if ((item.policy ?? "version") === "conflict") {
|
|
4568
|
-
preview.push({ index, outcome: "conflict", note_id: existing.rows[0].note_id, external_id_hash: externalIdHash, content_digest:
|
|
5628
|
+
preview.push({ index, outcome: "conflict", note_id: existing.rows[0].note_id, external_id_hash: externalIdHash, content_digest: digest2 });
|
|
4569
5629
|
} else {
|
|
4570
5630
|
preview.push({
|
|
4571
5631
|
index,
|
|
4572
5632
|
outcome: item.policy === "replace" ? "replaced" : "versioned",
|
|
4573
5633
|
note_id: existing.rows[0].note_id,
|
|
4574
5634
|
external_id_hash: externalIdHash,
|
|
4575
|
-
content_digest:
|
|
5635
|
+
content_digest: digest2
|
|
4576
5636
|
});
|
|
4577
5637
|
}
|
|
4578
5638
|
}
|
|
@@ -4581,7 +5641,7 @@ var SourceUpsertRepository = class {
|
|
|
4581
5641
|
await this.db.transaction(async (tx) => {
|
|
4582
5642
|
for (const [index, item] of items.entries()) {
|
|
4583
5643
|
const externalIdHash = sourceHash(item.source);
|
|
4584
|
-
const
|
|
5644
|
+
const digest2 = contentDigest(item.content);
|
|
4585
5645
|
const existing = await tx.query(
|
|
4586
5646
|
`SELECT note_id, content_digest
|
|
4587
5647
|
FROM source_identity
|
|
@@ -4594,7 +5654,7 @@ var SourceUpsertRepository = class {
|
|
|
4594
5654
|
);
|
|
4595
5655
|
if (existing.rows.length === 0) {
|
|
4596
5656
|
const noteId = item.source.caller_stable_id ?? generateId();
|
|
4597
|
-
await insertNote(tx, item, noteId,
|
|
5657
|
+
await insertNote(tx, item, noteId, digest2);
|
|
4598
5658
|
await tx.query(
|
|
4599
5659
|
`INSERT INTO source_identity
|
|
4600
5660
|
(id, tenant_id, archive_id, namespace, external_id, external_id_hash,
|
|
@@ -4608,23 +5668,23 @@ var SourceUpsertRepository = class {
|
|
|
4608
5668
|
item.source.external_id,
|
|
4609
5669
|
externalIdHash,
|
|
4610
5670
|
item.source.source_schema_version,
|
|
4611
|
-
|
|
5671
|
+
digest2,
|
|
4612
5672
|
item.source.import_run_id,
|
|
4613
5673
|
item.source.caller_stable_id ?? null,
|
|
4614
5674
|
noteId
|
|
4615
5675
|
]
|
|
4616
5676
|
);
|
|
4617
|
-
outcomes[index] = { index, outcome: "inserted", note_id: noteId, external_id_hash: externalIdHash, content_digest:
|
|
5677
|
+
outcomes[index] = { index, outcome: "inserted", note_id: noteId, external_id_hash: externalIdHash, content_digest: digest2 };
|
|
4618
5678
|
continue;
|
|
4619
5679
|
}
|
|
4620
5680
|
const row = existing.rows[0];
|
|
4621
|
-
if (row.content_digest ===
|
|
4622
|
-
outcomes[index] = { index, outcome: "unchanged", note_id: row.note_id, external_id_hash: externalIdHash, content_digest:
|
|
5681
|
+
if (row.content_digest === digest2) {
|
|
5682
|
+
outcomes[index] = { index, outcome: "unchanged", note_id: row.note_id, external_id_hash: externalIdHash, content_digest: digest2 };
|
|
4623
5683
|
continue;
|
|
4624
5684
|
}
|
|
4625
5685
|
const policy = item.policy ?? "version";
|
|
4626
5686
|
if (policy === "conflict") {
|
|
4627
|
-
outcomes[index] = { index, outcome: "conflict", note_id: row.note_id, external_id_hash: externalIdHash, content_digest:
|
|
5687
|
+
outcomes[index] = { index, outcome: "conflict", note_id: row.note_id, external_id_hash: externalIdHash, content_digest: digest2 };
|
|
4628
5688
|
continue;
|
|
4629
5689
|
}
|
|
4630
5690
|
const outcome = policy === "replace" ? "replaced" : "versioned";
|
|
@@ -4639,7 +5699,7 @@ var SourceUpsertRepository = class {
|
|
|
4639
5699
|
AND external_id = $8`,
|
|
4640
5700
|
[
|
|
4641
5701
|
item.source.source_schema_version,
|
|
4642
|
-
|
|
5702
|
+
digest2,
|
|
4643
5703
|
item.source.import_run_id,
|
|
4644
5704
|
row.note_id,
|
|
4645
5705
|
item.source.tenant_id ?? "default",
|
|
@@ -4648,7 +5708,7 @@ var SourceUpsertRepository = class {
|
|
|
4648
5708
|
item.source.external_id
|
|
4649
5709
|
]
|
|
4650
5710
|
);
|
|
4651
|
-
outcomes[index] = { index, outcome, note_id: row.note_id, external_id_hash: externalIdHash, content_digest:
|
|
5711
|
+
outcomes[index] = { index, outcome, note_id: row.note_id, external_id_hash: externalIdHash, content_digest: digest2 };
|
|
4652
5712
|
}
|
|
4653
5713
|
if (hasMaterialChange(outcomes)) {
|
|
4654
5714
|
await tx.query(
|
|
@@ -5591,7 +6651,7 @@ function titleGenerationHandler(job, db) {
|
|
|
5591
6651
|
|
|
5592
6652
|
Note content:
|
|
5593
6653
|
${content.slice(0, 1e3)}`;
|
|
5594
|
-
const llmTitle = (await llmFn2(prompt, { maxTokens: 60, temperature: 0.3 })).trim();
|
|
6654
|
+
const llmTitle = (await llmFn2(prompt, { maxTokens: 60, temperature: 0.3, task: "chat.general" })).trim();
|
|
5595
6655
|
if (llmTitle) {
|
|
5596
6656
|
const title2 = llmTitle.length > 200 ? llmTitle.slice(0, 197) + "..." : llmTitle;
|
|
5597
6657
|
await db.query(
|
|
@@ -5632,7 +6692,7 @@ Respond with ONLY the enhanced note content, no explanation.
|
|
|
5632
6692
|
|
|
5633
6693
|
Original note:
|
|
5634
6694
|
${content}`;
|
|
5635
|
-
const revised = (await llmFn2(prompt, { maxTokens: 2e3, temperature: 0.4 })).trim();
|
|
6695
|
+
const revised = (await llmFn2(prompt, { maxTokens: 2e3, temperature: 0.4, task: "chat.revision" })).trim();
|
|
5636
6696
|
if (!revised || revised === content) return { skipped: true, reason: "no changes from LLM" };
|
|
5637
6697
|
const revResult = await db.query(
|
|
5638
6698
|
`SELECT COALESCE(MAX(revision_number), 0) as max_rev FROM note_revision WHERE note_id = $1`,
|
|
@@ -5673,7 +6733,7 @@ Text:
|
|
|
5673
6733
|
${content.slice(0, 1500)}
|
|
5674
6734
|
|
|
5675
6735
|
Tags:`;
|
|
5676
|
-
const response = (await llmFn2(prompt, { maxTokens: 60, temperature: 0.1 })).trim();
|
|
6736
|
+
const response = (await llmFn2(prompt, { maxTokens: 60, temperature: 0.1, task: "chat.tagging" })).trim();
|
|
5677
6737
|
const tags = response.split(/[,\n]/).map((t) => t.trim().toLowerCase().replace(/^[-*\d.]+\s*/, "").replace(/['"]/g, "")).filter((t) => {
|
|
5678
6738
|
if (t.length < 2 || t.length > 40) return false;
|
|
5679
6739
|
if (/^\d+$/.test(t)) return false;
|
|
@@ -6956,10 +8016,24 @@ function chunkText(text, maxChars = 800, overlap = 100) {
|
|
|
6956
8016
|
}
|
|
6957
8017
|
|
|
6958
8018
|
// src/capabilities/embedding-handler.ts
|
|
8019
|
+
var DEFAULT_LARGE_DOCUMENT_CHARS = 12e3;
|
|
8020
|
+
var DEFAULT_LARGE_DOCUMENT_CHUNKS = 12;
|
|
6959
8021
|
var embedFn = null;
|
|
8022
|
+
var embeddingTaskSelectionOptions = {};
|
|
6960
8023
|
function setEmbedFunction(fn) {
|
|
6961
8024
|
embedFn = fn;
|
|
6962
8025
|
}
|
|
8026
|
+
function setEmbeddingTaskSelectionOptions(options = {}) {
|
|
8027
|
+
embeddingTaskSelectionOptions = { ...options };
|
|
8028
|
+
}
|
|
8029
|
+
function getEmbeddingTaskSelectionOptions() {
|
|
8030
|
+
return { ...embeddingTaskSelectionOptions };
|
|
8031
|
+
}
|
|
8032
|
+
function selectEmbeddingTask(content, chunks, options = {}) {
|
|
8033
|
+
const largeDocumentChars = options.largeDocumentChars ?? DEFAULT_LARGE_DOCUMENT_CHARS;
|
|
8034
|
+
const largeDocumentChunks = options.largeDocumentChunks ?? DEFAULT_LARGE_DOCUMENT_CHUNKS;
|
|
8035
|
+
return content.length >= largeDocumentChars || chunks.length >= largeDocumentChunks ? "embedding.large-document" : "embedding.document";
|
|
8036
|
+
}
|
|
6963
8037
|
function getEmbedFunction() {
|
|
6964
8038
|
return embedFn;
|
|
6965
8039
|
}
|
|
@@ -6985,7 +8059,8 @@ async function embeddingGenerationHandler(job, db) {
|
|
|
6985
8059
|
if (!noteText) return { skipped: true, reason: "note missing, deleted, or has no content" };
|
|
6986
8060
|
const content = noteText.combined;
|
|
6987
8061
|
const chunks = chunkText(content);
|
|
6988
|
-
const
|
|
8062
|
+
const task = selectEmbeddingTask(content, chunks, embeddingTaskSelectionOptions);
|
|
8063
|
+
const embeddings = await fn(chunks, { task });
|
|
6989
8064
|
const vector = averageEmbeddings(embeddings);
|
|
6990
8065
|
const embeddingSets = new EmbeddingSetsRepository(db);
|
|
6991
8066
|
const set = await embeddingSets.ensureDefault();
|
|
@@ -6994,7 +8069,7 @@ async function embeddingGenerationHandler(job, db) {
|
|
|
6994
8069
|
embedding_set_id: set.id,
|
|
6995
8070
|
vector
|
|
6996
8071
|
});
|
|
6997
|
-
return { chunks: chunks.length, embeddings: embeddings.length, setId: set.id };
|
|
8072
|
+
return { chunks: chunks.length, embeddings: embeddings.length, setId: set.id, task };
|
|
6998
8073
|
}
|
|
6999
8074
|
|
|
7000
8075
|
// src/capabilities/auto-tag.ts
|
|
@@ -7153,6 +8228,226 @@ function unregisterLlmCapability() {
|
|
|
7153
8228
|
setLlmFunction(null);
|
|
7154
8229
|
}
|
|
7155
8230
|
|
|
8231
|
+
// src/capabilities/fallback-router.ts
|
|
8232
|
+
var DEFAULT_COOLDOWNS = {
|
|
8233
|
+
rateLimit: 3e4,
|
|
8234
|
+
serverError: 6e4,
|
|
8235
|
+
connectionFailure: 3e5,
|
|
8236
|
+
contentPolicy: 0
|
|
8237
|
+
};
|
|
8238
|
+
function classifyError(error) {
|
|
8239
|
+
const msg = error instanceof Error ? error.message : String(error);
|
|
8240
|
+
const lower = msg.toLowerCase();
|
|
8241
|
+
if (lower.includes("429") || lower.includes("rate limit")) return "rate_limit";
|
|
8242
|
+
if (lower.includes("500") || lower.includes("502") || lower.includes("503") || lower.includes("504")) return "server_error";
|
|
8243
|
+
if (lower.includes("content") && (lower.includes("policy") || lower.includes("filter"))) return "content_policy";
|
|
8244
|
+
if (lower.includes("context") && (lower.includes("window") || lower.includes("length") || lower.includes("too long"))) return "context_window";
|
|
8245
|
+
if (lower.includes("fetch") || lower.includes("network") || lower.includes("connection") || lower.includes("econnrefused") || lower.includes("timeout")) return "connection_failure";
|
|
8246
|
+
return "unknown";
|
|
8247
|
+
}
|
|
8248
|
+
var FallbackRouter = class {
|
|
8249
|
+
id = "fallback-router";
|
|
8250
|
+
name = "Fallback Router";
|
|
8251
|
+
tier = "remote";
|
|
8252
|
+
providers;
|
|
8253
|
+
cooldowns;
|
|
8254
|
+
cooldownMap = /* @__PURE__ */ new Map();
|
|
8255
|
+
events;
|
|
8256
|
+
get capabilities() {
|
|
8257
|
+
const available = this.getAvailableProviders();
|
|
8258
|
+
return {
|
|
8259
|
+
embeddings: available.some((p) => p.capabilities.embeddings),
|
|
8260
|
+
chat: available.some((p) => p.capabilities.chat),
|
|
8261
|
+
streaming: available.some((p) => p.capabilities.streaming),
|
|
8262
|
+
vision: available.some((p) => p.capabilities.vision),
|
|
8263
|
+
toolCalling: available.some((p) => p.capabilities.toolCalling),
|
|
8264
|
+
structuredOutput: available.some((p) => p.capabilities.structuredOutput),
|
|
8265
|
+
maxContextTokens: Math.max(
|
|
8266
|
+
...available.map((p) => p.capabilities.maxContextTokens ?? 0),
|
|
8267
|
+
0
|
|
8268
|
+
)
|
|
8269
|
+
};
|
|
8270
|
+
}
|
|
8271
|
+
constructor(config) {
|
|
8272
|
+
this.providers = [...config.providers];
|
|
8273
|
+
this.cooldowns = { ...DEFAULT_COOLDOWNS, ...config.cooldowns };
|
|
8274
|
+
this.events = config.events;
|
|
8275
|
+
}
|
|
8276
|
+
// -------------------------------------------------------------------------
|
|
8277
|
+
// Provider management
|
|
8278
|
+
// -------------------------------------------------------------------------
|
|
8279
|
+
/** Get providers not currently in cooldown */
|
|
8280
|
+
getAvailableProviders() {
|
|
8281
|
+
const now2 = Date.now();
|
|
8282
|
+
return this.providers.filter((p) => {
|
|
8283
|
+
const cd = this.cooldownMap.get(p.id);
|
|
8284
|
+
if (!cd) return true;
|
|
8285
|
+
if (now2 >= cd.expiresAt) {
|
|
8286
|
+
this.cooldownMap.delete(p.id);
|
|
8287
|
+
return true;
|
|
8288
|
+
}
|
|
8289
|
+
return false;
|
|
8290
|
+
});
|
|
8291
|
+
}
|
|
8292
|
+
/** Get providers in cooldown with their expiry info */
|
|
8293
|
+
getCoolingDown() {
|
|
8294
|
+
const now2 = Date.now();
|
|
8295
|
+
const result = [];
|
|
8296
|
+
for (const [id, entry] of this.cooldownMap) {
|
|
8297
|
+
if (now2 < entry.expiresAt) {
|
|
8298
|
+
result.push({ providerId: id, category: entry.category, expiresAt: entry.expiresAt });
|
|
8299
|
+
}
|
|
8300
|
+
}
|
|
8301
|
+
return result;
|
|
8302
|
+
}
|
|
8303
|
+
/** Manually clear cooldown for a provider */
|
|
8304
|
+
clearCooldown(providerId) {
|
|
8305
|
+
this.cooldownMap.delete(providerId);
|
|
8306
|
+
}
|
|
8307
|
+
/** Clear all cooldowns */
|
|
8308
|
+
clearAllCooldowns() {
|
|
8309
|
+
this.cooldownMap.clear();
|
|
8310
|
+
}
|
|
8311
|
+
/** Add a provider to the chain (appended at lowest priority) */
|
|
8312
|
+
addProvider(provider) {
|
|
8313
|
+
this.providers.push(provider);
|
|
8314
|
+
}
|
|
8315
|
+
/** Remove a provider from the chain */
|
|
8316
|
+
removeProvider(id) {
|
|
8317
|
+
this.providers = this.providers.filter((p) => p.id !== id);
|
|
8318
|
+
this.cooldownMap.delete(id);
|
|
8319
|
+
}
|
|
8320
|
+
/** Reorder providers (new priority order) */
|
|
8321
|
+
setOrder(ids) {
|
|
8322
|
+
const byId = new Map(this.providers.map((p) => [p.id, p]));
|
|
8323
|
+
const reordered = [];
|
|
8324
|
+
for (const id of ids) {
|
|
8325
|
+
const p = byId.get(id);
|
|
8326
|
+
if (p) reordered.push(p);
|
|
8327
|
+
}
|
|
8328
|
+
for (const p of this.providers) {
|
|
8329
|
+
if (!ids.includes(p.id)) reordered.push(p);
|
|
8330
|
+
}
|
|
8331
|
+
this.providers = reordered;
|
|
8332
|
+
}
|
|
8333
|
+
// -------------------------------------------------------------------------
|
|
8334
|
+
// InferenceProvider interface — with fallback
|
|
8335
|
+
// -------------------------------------------------------------------------
|
|
8336
|
+
async embed(request) {
|
|
8337
|
+
return this.withFallback(
|
|
8338
|
+
(p) => p.capabilities.embeddings && !!p.embed,
|
|
8339
|
+
(p) => p.embed(request)
|
|
8340
|
+
);
|
|
8341
|
+
}
|
|
8342
|
+
async complete(request) {
|
|
8343
|
+
return this.withFallback(
|
|
8344
|
+
(p) => p.capabilities.chat && !!p.complete,
|
|
8345
|
+
(p) => p.complete(request)
|
|
8346
|
+
);
|
|
8347
|
+
}
|
|
8348
|
+
async *stream(request) {
|
|
8349
|
+
const candidates = this.getAvailableProviders().filter((p) => p.capabilities.streaming && p.stream);
|
|
8350
|
+
if (candidates.length === 0) {
|
|
8351
|
+
throw new Error("No available providers with streaming capability");
|
|
8352
|
+
}
|
|
8353
|
+
yield* candidates[0].stream(request);
|
|
8354
|
+
}
|
|
8355
|
+
async listModels() {
|
|
8356
|
+
const available = this.getAvailableProviders();
|
|
8357
|
+
const results = await Promise.allSettled(
|
|
8358
|
+
available.map((p) => p.listModels())
|
|
8359
|
+
);
|
|
8360
|
+
const models = [];
|
|
8361
|
+
for (const r of results) {
|
|
8362
|
+
if (r.status === "fulfilled") models.push(...r.value);
|
|
8363
|
+
}
|
|
8364
|
+
return models;
|
|
8365
|
+
}
|
|
8366
|
+
async probe() {
|
|
8367
|
+
const available = this.getAvailableProviders();
|
|
8368
|
+
if (available.length === 0) {
|
|
8369
|
+
return { status: "down", latencyMs: 0, message: "All providers in cooldown" };
|
|
8370
|
+
}
|
|
8371
|
+
const start = Date.now();
|
|
8372
|
+
const results = await Promise.allSettled(
|
|
8373
|
+
available.map((p) => p.probe())
|
|
8374
|
+
);
|
|
8375
|
+
const okCount = results.filter(
|
|
8376
|
+
(r) => r.status === "fulfilled" && r.value.status === "ok"
|
|
8377
|
+
).length;
|
|
8378
|
+
return {
|
|
8379
|
+
status: okCount === available.length ? "ok" : okCount > 0 ? "degraded" : "down",
|
|
8380
|
+
latencyMs: Date.now() - start,
|
|
8381
|
+
message: `${okCount}/${available.length} providers healthy`
|
|
8382
|
+
};
|
|
8383
|
+
}
|
|
8384
|
+
dispose() {
|
|
8385
|
+
for (const p of this.providers) {
|
|
8386
|
+
p.dispose();
|
|
8387
|
+
}
|
|
8388
|
+
this.providers = [];
|
|
8389
|
+
this.cooldownMap.clear();
|
|
8390
|
+
}
|
|
8391
|
+
// -------------------------------------------------------------------------
|
|
8392
|
+
// Core fallback logic
|
|
8393
|
+
// -------------------------------------------------------------------------
|
|
8394
|
+
async withFallback(filter, execute) {
|
|
8395
|
+
const candidates = this.getAvailableProviders().filter(filter);
|
|
8396
|
+
if (candidates.length === 0) {
|
|
8397
|
+
throw new Error("No available providers for this request");
|
|
8398
|
+
}
|
|
8399
|
+
let lastError;
|
|
8400
|
+
for (const provider of candidates) {
|
|
8401
|
+
try {
|
|
8402
|
+
return await execute(provider);
|
|
8403
|
+
} catch (err) {
|
|
8404
|
+
lastError = err instanceof Error ? err : new Error(String(err));
|
|
8405
|
+
const category = classifyError(err);
|
|
8406
|
+
this.applyCooldown(provider.id, category);
|
|
8407
|
+
const nextCandidate = candidates[candidates.indexOf(provider) + 1];
|
|
8408
|
+
if (nextCandidate) {
|
|
8409
|
+
this.events?.emit("provider.fallback", {
|
|
8410
|
+
fromProvider: provider.id,
|
|
8411
|
+
toProvider: nextCandidate.id,
|
|
8412
|
+
errorCategory: category,
|
|
8413
|
+
error: lastError.message
|
|
8414
|
+
});
|
|
8415
|
+
}
|
|
8416
|
+
}
|
|
8417
|
+
}
|
|
8418
|
+
throw lastError ?? new Error("All providers failed");
|
|
8419
|
+
}
|
|
8420
|
+
applyCooldown(providerId, category) {
|
|
8421
|
+
let cooldownMs;
|
|
8422
|
+
switch (category) {
|
|
8423
|
+
case "rate_limit":
|
|
8424
|
+
cooldownMs = this.cooldowns.rateLimit;
|
|
8425
|
+
break;
|
|
8426
|
+
case "server_error":
|
|
8427
|
+
cooldownMs = this.cooldowns.serverError;
|
|
8428
|
+
break;
|
|
8429
|
+
case "connection_failure":
|
|
8430
|
+
cooldownMs = this.cooldowns.connectionFailure;
|
|
8431
|
+
break;
|
|
8432
|
+
case "content_policy":
|
|
8433
|
+
cooldownMs = this.cooldowns.contentPolicy;
|
|
8434
|
+
break;
|
|
8435
|
+
default:
|
|
8436
|
+
cooldownMs = this.cooldowns.serverError;
|
|
8437
|
+
}
|
|
8438
|
+
if (cooldownMs > 0) {
|
|
8439
|
+
const expiresAt = Date.now() + cooldownMs;
|
|
8440
|
+
this.cooldownMap.set(providerId, { expiresAt, category });
|
|
8441
|
+
this.events?.emit("provider.cooldown", {
|
|
8442
|
+
providerId,
|
|
8443
|
+
errorCategory: category,
|
|
8444
|
+
cooldownMs,
|
|
8445
|
+
expiresAt
|
|
8446
|
+
});
|
|
8447
|
+
}
|
|
8448
|
+
}
|
|
8449
|
+
};
|
|
8450
|
+
|
|
7156
8451
|
// src/capabilities/provider-registry.ts
|
|
7157
8452
|
var ProviderRegistry = class {
|
|
7158
8453
|
constructor(events) {
|
|
@@ -7160,6 +8455,7 @@ var ProviderRegistry = class {
|
|
|
7160
8455
|
}
|
|
7161
8456
|
providers = /* @__PURE__ */ new Map();
|
|
7162
8457
|
activeId = null;
|
|
8458
|
+
routes = /* @__PURE__ */ new Map();
|
|
7163
8459
|
/** Register a provider. First provider with embedding capability becomes active. */
|
|
7164
8460
|
add(provider) {
|
|
7165
8461
|
if (this.providers.has(provider.id)) {
|
|
@@ -7197,6 +8493,36 @@ var ProviderRegistry = class {
|
|
|
7197
8493
|
this.syncLegacyFunctions();
|
|
7198
8494
|
this.events?.emit("provider.active", { id, name: provider.name });
|
|
7199
8495
|
}
|
|
8496
|
+
setRoute(task, policy) {
|
|
8497
|
+
const cloned = cloneProviderRoutePolicy(policy);
|
|
8498
|
+
this.routes.set(task, cloned);
|
|
8499
|
+
this.events?.emit("provider.route.configured", {
|
|
8500
|
+
task,
|
|
8501
|
+
providerIds: cloned.providerIds ?? [],
|
|
8502
|
+
model: cloned.model,
|
|
8503
|
+
fallback: cloned.fallback,
|
|
8504
|
+
hasRequirements: Boolean(cloned.requirements)
|
|
8505
|
+
});
|
|
8506
|
+
}
|
|
8507
|
+
getRoute(task) {
|
|
8508
|
+
const route = this.routes.get(task);
|
|
8509
|
+
return route ? cloneProviderRoutePolicy(route) : void 0;
|
|
8510
|
+
}
|
|
8511
|
+
clearRoute(task) {
|
|
8512
|
+
this.routes.delete(task);
|
|
8513
|
+
this.events?.emit("provider.route.cleared", { task });
|
|
8514
|
+
}
|
|
8515
|
+
clearRoutes() {
|
|
8516
|
+
const tasks = Array.from(this.routes.keys());
|
|
8517
|
+
this.routes.clear();
|
|
8518
|
+
if (tasks.length === 0) {
|
|
8519
|
+
this.events?.emit("provider.route.cleared", {});
|
|
8520
|
+
return;
|
|
8521
|
+
}
|
|
8522
|
+
for (const task of tasks) {
|
|
8523
|
+
this.events?.emit("provider.route.cleared", { task });
|
|
8524
|
+
}
|
|
8525
|
+
}
|
|
7200
8526
|
/** Get the currently active provider */
|
|
7201
8527
|
getActive() {
|
|
7202
8528
|
if (!this.activeId) return null;
|
|
@@ -7228,27 +8554,51 @@ var ProviderRegistry = class {
|
|
|
7228
8554
|
}
|
|
7229
8555
|
/** Convenience: embed using active provider */
|
|
7230
8556
|
async embed(request) {
|
|
7231
|
-
const
|
|
7232
|
-
|
|
7233
|
-
throw new Error("No
|
|
7234
|
-
|
|
7235
|
-
|
|
8557
|
+
const route = request.task ? this.routes.get(request.task) : void 0;
|
|
8558
|
+
return this.withRouteFallback(request.task, "embeddings", request.model, async (selection) => {
|
|
8559
|
+
if (!selection.provider.embed) throw new Error("No routed provider with embedding capability");
|
|
8560
|
+
return selection.provider.embed(withResolvedModel(request, selection.model));
|
|
8561
|
+
}, route?.fallback !== false);
|
|
7236
8562
|
}
|
|
7237
8563
|
/** Convenience: complete using active provider */
|
|
7238
8564
|
async complete(request) {
|
|
7239
|
-
const
|
|
7240
|
-
|
|
7241
|
-
throw new Error("No
|
|
7242
|
-
|
|
7243
|
-
|
|
8565
|
+
const route = request.task ? this.routes.get(request.task) : void 0;
|
|
8566
|
+
return this.withRouteFallback(request.task, "chat", request.model, async (selection) => {
|
|
8567
|
+
if (!selection.provider.complete) throw new Error("No routed provider with chat capability");
|
|
8568
|
+
return selection.provider.complete(withResolvedModel(request, selection.model));
|
|
8569
|
+
}, route?.fallback !== false);
|
|
7244
8570
|
}
|
|
7245
8571
|
/** Convenience: stream using active provider */
|
|
7246
8572
|
stream(request) {
|
|
7247
|
-
const provider = this.
|
|
7248
|
-
if (!provider
|
|
7249
|
-
|
|
7250
|
-
|
|
7251
|
-
|
|
8573
|
+
const { provider, model } = this.resolveProvider(request.task, "streaming", request.model, true);
|
|
8574
|
+
if (!provider.stream) throw new Error("No routed provider with streaming capability");
|
|
8575
|
+
return provider.stream(withResolvedModel(request, model));
|
|
8576
|
+
}
|
|
8577
|
+
previewRoute(task, capability, requestModel) {
|
|
8578
|
+
const { provider, model, routeMatched } = this.resolveProvider(task, capability, requestModel, false);
|
|
8579
|
+
return {
|
|
8580
|
+
provider,
|
|
8581
|
+
providerId: provider.id,
|
|
8582
|
+
providerName: provider.name,
|
|
8583
|
+
tier: provider.tier,
|
|
8584
|
+
capability,
|
|
8585
|
+
task,
|
|
8586
|
+
model,
|
|
8587
|
+
routeMatched
|
|
8588
|
+
};
|
|
8589
|
+
}
|
|
8590
|
+
async probeRoute(task, capability, requestModel) {
|
|
8591
|
+
const selection = this.previewRoute(task, capability, requestModel);
|
|
8592
|
+
const probe = await selection.provider.probe();
|
|
8593
|
+
return { ...selection, probe };
|
|
8594
|
+
}
|
|
8595
|
+
validateRoute(task, capability = inferInferenceTaskCapability(task), policy = this.routes.get(task)) {
|
|
8596
|
+
return validateProviderRoute(task, capability, policy, this.list());
|
|
8597
|
+
}
|
|
8598
|
+
validateRoutes() {
|
|
8599
|
+
return Array.from(this.routes.entries()).map(
|
|
8600
|
+
([task, policy]) => this.validateRoute(task, inferInferenceTaskCapability(task), policy)
|
|
8601
|
+
);
|
|
7252
8602
|
}
|
|
7253
8603
|
/** Dispose all providers */
|
|
7254
8604
|
dispose() {
|
|
@@ -7270,29 +8620,357 @@ var ProviderRegistry = class {
|
|
|
7270
8620
|
syncLegacyFunctions() {
|
|
7271
8621
|
const active = this.getActive();
|
|
7272
8622
|
if (active?.embed && active.capabilities.embeddings) {
|
|
7273
|
-
const embedBridge = (texts) =>
|
|
8623
|
+
const embedBridge = (texts, options) => {
|
|
8624
|
+
const request = {
|
|
8625
|
+
texts,
|
|
8626
|
+
task: options?.task ?? "embedding.document"
|
|
8627
|
+
};
|
|
8628
|
+
if (options?.model) request.model = options.model;
|
|
8629
|
+
return this.embed(request).then((r) => r.vectors);
|
|
8630
|
+
};
|
|
7274
8631
|
setEmbedFunction(embedBridge);
|
|
7275
8632
|
} else {
|
|
7276
8633
|
setEmbedFunction(null);
|
|
7277
8634
|
}
|
|
7278
8635
|
if (active?.complete && active.capabilities.chat) {
|
|
7279
|
-
const llmBridge = (prompt, options) =>
|
|
7280
|
-
|
|
7281
|
-
|
|
7282
|
-
|
|
7283
|
-
|
|
8636
|
+
const llmBridge = (prompt, options) => {
|
|
8637
|
+
const request = {
|
|
8638
|
+
prompt,
|
|
8639
|
+
task: options?.task ?? "chat.general"
|
|
8640
|
+
};
|
|
8641
|
+
if (options?.model) request.model = options.model;
|
|
8642
|
+
if (options?.maxTokens !== void 0) request.maxTokens = options.maxTokens;
|
|
8643
|
+
if (options?.temperature !== void 0) request.temperature = options.temperature;
|
|
8644
|
+
return this.complete(request).then((r) => r.text);
|
|
8645
|
+
};
|
|
7284
8646
|
setLlmFunction(llmBridge);
|
|
7285
8647
|
} else {
|
|
7286
8648
|
setLlmFunction(null);
|
|
7287
8649
|
}
|
|
7288
8650
|
}
|
|
8651
|
+
resolveProvider(task, capability, requestModel, emitSelection = false) {
|
|
8652
|
+
const candidates = this.resolveRouteSelections(task, capability, requestModel);
|
|
8653
|
+
const selection = candidates[0];
|
|
8654
|
+
if (!selection) {
|
|
8655
|
+
const label = task ? ` for task '${task}'` : "";
|
|
8656
|
+
throw new Error(`No provider satisfies ${String(capability)} route${label}`);
|
|
8657
|
+
}
|
|
8658
|
+
if (emitSelection) {
|
|
8659
|
+
this.emitRouteSelected(selection);
|
|
8660
|
+
}
|
|
8661
|
+
return selection;
|
|
8662
|
+
}
|
|
8663
|
+
resolveRouteSelections(task, capability, requestModel) {
|
|
8664
|
+
const route = task ? this.routes.get(task) : void 0;
|
|
8665
|
+
const model = requestModel ?? route?.model;
|
|
8666
|
+
const routeMatched = Boolean(route);
|
|
8667
|
+
return this.resolveCandidates(route, capability).filter((provider) => Boolean(provider.capabilities[capability])).filter((provider) => capability !== "embeddings" || Boolean(provider.embed)).filter((provider) => capability !== "chat" || Boolean(provider.complete)).filter((provider) => capability !== "streaming" || Boolean(provider.stream)).map((provider) => ({
|
|
8668
|
+
provider,
|
|
8669
|
+
providerId: provider.id,
|
|
8670
|
+
providerName: provider.name,
|
|
8671
|
+
tier: provider.tier,
|
|
8672
|
+
capability,
|
|
8673
|
+
task,
|
|
8674
|
+
model,
|
|
8675
|
+
routeMatched
|
|
8676
|
+
}));
|
|
8677
|
+
}
|
|
8678
|
+
emitRouteSelected(selection) {
|
|
8679
|
+
this.events?.emit("provider.route.selected", {
|
|
8680
|
+
providerId: selection.providerId,
|
|
8681
|
+
providerName: selection.providerName,
|
|
8682
|
+
tier: selection.tier,
|
|
8683
|
+
capability: String(selection.capability),
|
|
8684
|
+
task: selection.task,
|
|
8685
|
+
model: selection.model,
|
|
8686
|
+
routeMatched: selection.routeMatched
|
|
8687
|
+
});
|
|
8688
|
+
}
|
|
8689
|
+
async withRouteFallback(task, capability, requestModel, execute, allowFallback) {
|
|
8690
|
+
const selections = this.resolveRouteSelections(task, capability, requestModel);
|
|
8691
|
+
const candidates = allowFallback ? selections : selections.slice(0, 1);
|
|
8692
|
+
if (!candidates.length) {
|
|
8693
|
+
const label = task ? ` for task '${task}'` : "";
|
|
8694
|
+
const route = task ? this.routes.get(task) : void 0;
|
|
8695
|
+
this.emitRouteFailed(
|
|
8696
|
+
void 0,
|
|
8697
|
+
capability,
|
|
8698
|
+
task,
|
|
8699
|
+
requestModel ?? route?.model,
|
|
8700
|
+
Boolean(route),
|
|
8701
|
+
0,
|
|
8702
|
+
0,
|
|
8703
|
+
0,
|
|
8704
|
+
"no_provider",
|
|
8705
|
+
`No provider satisfies ${String(capability)} route${label}`
|
|
8706
|
+
);
|
|
8707
|
+
throw new Error(`No provider satisfies ${String(capability)} route${label}`);
|
|
8708
|
+
}
|
|
8709
|
+
let lastError;
|
|
8710
|
+
let lastSelection;
|
|
8711
|
+
let lastErrorCategory = "unknown";
|
|
8712
|
+
const start = Date.now();
|
|
8713
|
+
for (let index = 0; index < candidates.length; index++) {
|
|
8714
|
+
const selection = candidates[index];
|
|
8715
|
+
try {
|
|
8716
|
+
this.emitRouteSelected(selection);
|
|
8717
|
+
const result = await execute(selection);
|
|
8718
|
+
this.emitRouteCompleted(selection, index + 1, index, Date.now() - start);
|
|
8719
|
+
return result;
|
|
8720
|
+
} catch (err) {
|
|
8721
|
+
lastSelection = selection;
|
|
8722
|
+
lastError = err instanceof Error ? err : new Error(String(err));
|
|
8723
|
+
lastErrorCategory = classifyError(err);
|
|
8724
|
+
const next = candidates[index + 1];
|
|
8725
|
+
if (next) {
|
|
8726
|
+
this.events?.emit("provider.fallback", {
|
|
8727
|
+
fromProvider: selection.providerId,
|
|
8728
|
+
toProvider: next.providerId,
|
|
8729
|
+
errorCategory: lastErrorCategory,
|
|
8730
|
+
error: lastError.message
|
|
8731
|
+
});
|
|
8732
|
+
}
|
|
8733
|
+
}
|
|
8734
|
+
}
|
|
8735
|
+
this.emitRouteFailed(
|
|
8736
|
+
lastSelection,
|
|
8737
|
+
capability,
|
|
8738
|
+
task,
|
|
8739
|
+
requestModel,
|
|
8740
|
+
candidates[0]?.routeMatched ?? Boolean(task && this.routes.get(task)),
|
|
8741
|
+
candidates.length,
|
|
8742
|
+
Math.max(candidates.length - 1, 0),
|
|
8743
|
+
Date.now() - start,
|
|
8744
|
+
lastErrorCategory,
|
|
8745
|
+
lastError?.message ?? "All routed providers failed"
|
|
8746
|
+
);
|
|
8747
|
+
throw lastError ?? new Error("All routed providers failed");
|
|
8748
|
+
}
|
|
8749
|
+
emitRouteCompleted(selection, attempt, fallbackCount, latencyMs) {
|
|
8750
|
+
this.events?.emit("provider.route.completed", {
|
|
8751
|
+
providerId: selection.providerId,
|
|
8752
|
+
providerName: selection.providerName,
|
|
8753
|
+
tier: selection.tier,
|
|
8754
|
+
capability: String(selection.capability),
|
|
8755
|
+
task: selection.task,
|
|
8756
|
+
model: selection.model,
|
|
8757
|
+
routeMatched: selection.routeMatched,
|
|
8758
|
+
attempt,
|
|
8759
|
+
fallbackCount,
|
|
8760
|
+
latencyMs
|
|
8761
|
+
});
|
|
8762
|
+
}
|
|
8763
|
+
emitRouteFailed(selection, capability, task, model, routeMatched, attempt, fallbackCount, latencyMs, errorCategory, error) {
|
|
8764
|
+
this.events?.emit("provider.route.failed", {
|
|
8765
|
+
providerId: selection?.providerId,
|
|
8766
|
+
providerName: selection?.providerName,
|
|
8767
|
+
tier: selection?.tier,
|
|
8768
|
+
capability: String(capability),
|
|
8769
|
+
task,
|
|
8770
|
+
model: selection?.model ?? model,
|
|
8771
|
+
routeMatched,
|
|
8772
|
+
attempt,
|
|
8773
|
+
fallbackCount,
|
|
8774
|
+
latencyMs,
|
|
8775
|
+
errorCategory,
|
|
8776
|
+
error
|
|
8777
|
+
});
|
|
8778
|
+
}
|
|
8779
|
+
resolveCandidates(route, capability) {
|
|
8780
|
+
const all = this.list();
|
|
8781
|
+
if (!route) {
|
|
8782
|
+
const active = this.getActive();
|
|
8783
|
+
return active ? [active, ...all.filter((provider) => provider.id !== active.id)] : all;
|
|
8784
|
+
}
|
|
8785
|
+
const byId = new Map(all.map((provider) => [provider.id, provider]));
|
|
8786
|
+
const selected = [];
|
|
8787
|
+
for (const id of route.providerIds ?? []) {
|
|
8788
|
+
const provider = byId.get(id);
|
|
8789
|
+
if (provider) selected.push(provider);
|
|
8790
|
+
}
|
|
8791
|
+
const hasExplicitProviderIds = Boolean(route.providerIds?.length);
|
|
8792
|
+
const remaining = hasExplicitProviderIds || route.fallback === false ? selected : all;
|
|
8793
|
+
return remaining.filter((provider) => {
|
|
8794
|
+
if (route.tiers?.length && !route.tiers.includes(provider.tier)) return false;
|
|
8795
|
+
if (!providerSatisfiesRouteRequirements(provider, route.requirements, capability)) return false;
|
|
8796
|
+
return true;
|
|
8797
|
+
});
|
|
8798
|
+
}
|
|
7289
8799
|
};
|
|
8800
|
+
function withResolvedModel(request, model) {
|
|
8801
|
+
return model === void 0 ? request : { ...request, model };
|
|
8802
|
+
}
|
|
8803
|
+
function inferInferenceTaskCapability(task) {
|
|
8804
|
+
if (task.startsWith("embedding.")) return "embeddings";
|
|
8805
|
+
if (task.startsWith("vision.")) return "vision";
|
|
8806
|
+
return "chat";
|
|
8807
|
+
}
|
|
8808
|
+
function validateProviderRoute(task, capability, policy, providers) {
|
|
8809
|
+
const byId = new Map(providers.map((provider) => [provider.id, provider]));
|
|
8810
|
+
const providerIds = policy?.providerIds ?? [];
|
|
8811
|
+
const issues = [];
|
|
8812
|
+
const eligibleProviderIds = [];
|
|
8813
|
+
if (policy && providerIds.length === 0 && policy.fallback === false) {
|
|
8814
|
+
issues.push({
|
|
8815
|
+
severity: "error",
|
|
8816
|
+
code: "empty-explicit-chain",
|
|
8817
|
+
message: `Route '${task}' disables fallback but does not name a provider.`
|
|
8818
|
+
});
|
|
8819
|
+
}
|
|
8820
|
+
const candidates = providerIds.length ? providerIds.map((id) => byId.get(id)).filter((provider) => Boolean(provider)) : providers;
|
|
8821
|
+
for (const providerId of providerIds) {
|
|
8822
|
+
if (!byId.has(providerId)) {
|
|
8823
|
+
issues.push({
|
|
8824
|
+
severity: "error",
|
|
8825
|
+
code: "missing-provider",
|
|
8826
|
+
providerId,
|
|
8827
|
+
message: `Route '${task}' references missing provider '${providerId}'.`
|
|
8828
|
+
});
|
|
8829
|
+
}
|
|
8830
|
+
}
|
|
8831
|
+
for (const provider of candidates) {
|
|
8832
|
+
if (policy?.tiers?.length && !policy.tiers.includes(provider.tier)) continue;
|
|
8833
|
+
if (!provider.capabilities[capability]) {
|
|
8834
|
+
issues.push({
|
|
8835
|
+
severity: "error",
|
|
8836
|
+
code: "unsupported-capability",
|
|
8837
|
+
providerId: provider.id,
|
|
8838
|
+
message: `Provider '${provider.id}' does not support ${String(capability)} for route '${task}'.`
|
|
8839
|
+
});
|
|
8840
|
+
continue;
|
|
8841
|
+
}
|
|
8842
|
+
if (capability === "embeddings" && !provider.embed) {
|
|
8843
|
+
issues.push({
|
|
8844
|
+
severity: "error",
|
|
8845
|
+
code: "missing-handler",
|
|
8846
|
+
providerId: provider.id,
|
|
8847
|
+
message: `Provider '${provider.id}' has no embedding handler for route '${task}'.`
|
|
8848
|
+
});
|
|
8849
|
+
continue;
|
|
8850
|
+
}
|
|
8851
|
+
if (capability === "chat" && !provider.complete) {
|
|
8852
|
+
issues.push({
|
|
8853
|
+
severity: "error",
|
|
8854
|
+
code: "missing-handler",
|
|
8855
|
+
providerId: provider.id,
|
|
8856
|
+
message: `Provider '${provider.id}' has no chat handler for route '${task}'.`
|
|
8857
|
+
});
|
|
8858
|
+
continue;
|
|
8859
|
+
}
|
|
8860
|
+
if (capability === "streaming" && !provider.stream) {
|
|
8861
|
+
issues.push({
|
|
8862
|
+
severity: "error",
|
|
8863
|
+
code: "missing-handler",
|
|
8864
|
+
providerId: provider.id,
|
|
8865
|
+
message: `Provider '${provider.id}' has no streaming handler for route '${task}'.`
|
|
8866
|
+
});
|
|
8867
|
+
continue;
|
|
8868
|
+
}
|
|
8869
|
+
const requirementIssue = getProviderRouteRequirementIssue(provider, policy?.requirements, capability);
|
|
8870
|
+
if (requirementIssue) {
|
|
8871
|
+
issues.push({
|
|
8872
|
+
severity: "error",
|
|
8873
|
+
code: "profile-requirement",
|
|
8874
|
+
providerId: provider.id,
|
|
8875
|
+
message: requirementIssue
|
|
8876
|
+
});
|
|
8877
|
+
continue;
|
|
8878
|
+
}
|
|
8879
|
+
eligibleProviderIds.push(provider.id);
|
|
8880
|
+
}
|
|
8881
|
+
if (!eligibleProviderIds.length) {
|
|
8882
|
+
issues.push({
|
|
8883
|
+
severity: "error",
|
|
8884
|
+
code: "no-eligible-provider",
|
|
8885
|
+
message: `Route '${task}' has no eligible ${String(capability)} provider.`
|
|
8886
|
+
});
|
|
8887
|
+
}
|
|
8888
|
+
return {
|
|
8889
|
+
task,
|
|
8890
|
+
capability,
|
|
8891
|
+
policy: policy ? cloneProviderRoutePolicy(policy) : void 0,
|
|
8892
|
+
providerIds,
|
|
8893
|
+
eligibleProviderIds,
|
|
8894
|
+
issues,
|
|
8895
|
+
ok: !issues.some((issue) => issue.severity === "error")
|
|
8896
|
+
};
|
|
8897
|
+
}
|
|
8898
|
+
function cloneProviderRoutePolicy(policy) {
|
|
8899
|
+
return {
|
|
8900
|
+
...policy,
|
|
8901
|
+
providerIds: policy.providerIds ? [...policy.providerIds] : void 0,
|
|
8902
|
+
tiers: policy.tiers ? [...policy.tiers] : void 0,
|
|
8903
|
+
requirements: policy.requirements ? {
|
|
8904
|
+
...policy.requirements,
|
|
8905
|
+
privacyTiers: policy.requirements.privacyTiers ? [...policy.requirements.privacyTiers] : void 0
|
|
8906
|
+
} : void 0
|
|
8907
|
+
};
|
|
8908
|
+
}
|
|
8909
|
+
var COST_ORDER = ["free", "low", "medium", "high"];
|
|
8910
|
+
function providerSatisfiesRouteRequirements(provider, requirements, capability) {
|
|
8911
|
+
return getProviderRouteRequirementIssue(provider, requirements, capability) === void 0;
|
|
8912
|
+
}
|
|
8913
|
+
function getProviderRouteRequirementIssue(provider, requirements, capability) {
|
|
8914
|
+
if (!requirements) return void 0;
|
|
8915
|
+
const profile = provider.profile;
|
|
8916
|
+
if (requirements.privacyTiers?.length) {
|
|
8917
|
+
const privacy = profile?.privacyTier ?? privacyTierFromProviderTier(provider.tier);
|
|
8918
|
+
if (!requirements.privacyTiers.includes(privacy)) {
|
|
8919
|
+
return `Provider '${provider.id}' does not match required privacy tier`;
|
|
8920
|
+
}
|
|
8921
|
+
}
|
|
8922
|
+
if (requirements.maxCostTier) {
|
|
8923
|
+
const cost = profile?.costTier;
|
|
8924
|
+
if (!cost || COST_ORDER.indexOf(cost) > COST_ORDER.indexOf(requirements.maxCostTier)) {
|
|
8925
|
+
return `Provider '${provider.id}' does not match required cost tier`;
|
|
8926
|
+
}
|
|
8927
|
+
}
|
|
8928
|
+
if (requirements.minContextTokens) {
|
|
8929
|
+
const context = provider.capabilities.maxContextTokens;
|
|
8930
|
+
if (!context || context < requirements.minContextTokens) {
|
|
8931
|
+
return `Provider '${provider.id}' does not advertise enough context`;
|
|
8932
|
+
}
|
|
8933
|
+
}
|
|
8934
|
+
if (capability === "embeddings" && requirements.minEmbeddingDimensions) {
|
|
8935
|
+
const dimensions = profile?.embeddingDimensions ?? [];
|
|
8936
|
+
if (!dimensions.some((dimension) => dimension >= requirements.minEmbeddingDimensions)) {
|
|
8937
|
+
return `Provider '${provider.id}' does not advertise enough embedding dimensions`;
|
|
8938
|
+
}
|
|
8939
|
+
}
|
|
8940
|
+
if (requirements.dataClass) {
|
|
8941
|
+
const allowed = profile?.dataClasses ?? defaultDataClasses(provider.tier);
|
|
8942
|
+
if (!allowed.includes(requirements.dataClass)) {
|
|
8943
|
+
return `Provider '${provider.id}' does not allow required data class`;
|
|
8944
|
+
}
|
|
8945
|
+
}
|
|
8946
|
+
if (requirements.maxInputChars) {
|
|
8947
|
+
const maxInputChars = profile?.maxInputChars;
|
|
8948
|
+
if (!maxInputChars || maxInputChars < requirements.maxInputChars) {
|
|
8949
|
+
return `Provider '${provider.id}' does not advertise enough input capacity`;
|
|
8950
|
+
}
|
|
8951
|
+
}
|
|
8952
|
+
return void 0;
|
|
8953
|
+
}
|
|
8954
|
+
function privacyTierFromProviderTier(tier) {
|
|
8955
|
+
if (tier === "remote") return "external";
|
|
8956
|
+
if (tier === "chrome-ai") return "host-managed";
|
|
8957
|
+
return "local";
|
|
8958
|
+
}
|
|
8959
|
+
function defaultDataClasses(tier) {
|
|
8960
|
+
if (tier === "remote") return ["public"];
|
|
8961
|
+
return ["public", "private", "sensitive"];
|
|
8962
|
+
}
|
|
7290
8963
|
function createLegacyProvider(options) {
|
|
7291
8964
|
const { embedFn: embedFn2, llmFn: llmFn2, id = "legacy", name = "Legacy Provider" } = options;
|
|
7292
8965
|
return {
|
|
7293
8966
|
id,
|
|
7294
8967
|
name,
|
|
7295
8968
|
tier: "in-browser",
|
|
8969
|
+
profile: options.profile ?? {
|
|
8970
|
+
privacyTier: "local",
|
|
8971
|
+
costTier: "free",
|
|
8972
|
+
dataClasses: ["public", "private", "sensitive"]
|
|
8973
|
+
},
|
|
7296
8974
|
capabilities: {
|
|
7297
8975
|
embeddings: !!embedFn2,
|
|
7298
8976
|
chat: !!llmFn2,
|
|
@@ -7302,14 +8980,11 @@ function createLegacyProvider(options) {
|
|
|
7302
8980
|
structuredOutput: false
|
|
7303
8981
|
},
|
|
7304
8982
|
embed: embedFn2 ? async (request) => ({
|
|
7305
|
-
vectors: await embedFn2(request.texts),
|
|
8983
|
+
vectors: await embedFn2(request.texts, embedOptionsFromRequest(request)),
|
|
7306
8984
|
model: "legacy"
|
|
7307
8985
|
}) : void 0,
|
|
7308
8986
|
complete: llmFn2 ? async (request) => ({
|
|
7309
|
-
text: await llmFn2(request.prompt,
|
|
7310
|
-
maxTokens: request.maxTokens,
|
|
7311
|
-
temperature: request.temperature
|
|
7312
|
-
}),
|
|
8987
|
+
text: await llmFn2(request.prompt, llmOptionsFromRequest(request)),
|
|
7313
8988
|
model: "legacy"
|
|
7314
8989
|
}) : void 0,
|
|
7315
8990
|
async listModels() {
|
|
@@ -7325,6 +9000,114 @@ function createLegacyProvider(options) {
|
|
|
7325
9000
|
}
|
|
7326
9001
|
};
|
|
7327
9002
|
}
|
|
9003
|
+
function embedOptionsFromRequest(request) {
|
|
9004
|
+
const options = {};
|
|
9005
|
+
if (request.task) options.task = request.task;
|
|
9006
|
+
if (request.model) options.model = request.model;
|
|
9007
|
+
return Object.keys(options).length ? options : void 0;
|
|
9008
|
+
}
|
|
9009
|
+
function llmOptionsFromRequest(request) {
|
|
9010
|
+
const options = {};
|
|
9011
|
+
if (request.maxTokens !== void 0) options.maxTokens = request.maxTokens;
|
|
9012
|
+
if (request.temperature !== void 0) options.temperature = request.temperature;
|
|
9013
|
+
if (request.task) options.task = request.task;
|
|
9014
|
+
if (request.model) options.model = request.model;
|
|
9015
|
+
return Object.keys(options).length ? options : void 0;
|
|
9016
|
+
}
|
|
9017
|
+
|
|
9018
|
+
// src/capabilities/local-discovery.ts
|
|
9019
|
+
var LOCAL_ENDPOINTS = [
|
|
9020
|
+
{ id: "ollama", name: "Ollama", baseURL: "http://localhost:11434/v1", defaultPort: 11434 },
|
|
9021
|
+
{ id: "lm-studio", name: "LM Studio", baseURL: "http://localhost:1234/v1", defaultPort: 1234 },
|
|
9022
|
+
{ id: "llama-cpp", name: "llama.cpp", baseURL: "http://localhost:8080/v1", defaultPort: 8080 },
|
|
9023
|
+
{ id: "vllm", name: "vLLM", baseURL: "http://localhost:8000/v1", defaultPort: 8e3 },
|
|
9024
|
+
{ id: "jan", name: "Jan", baseURL: "http://localhost:1337/v1", defaultPort: 1337 },
|
|
9025
|
+
{ id: "localai", name: "LocalAI", baseURL: "http://localhost:8080/v1", defaultPort: 8080 }
|
|
9026
|
+
];
|
|
9027
|
+
function classifyModel(modelId) {
|
|
9028
|
+
const lower = modelId.toLowerCase();
|
|
9029
|
+
if (lower.includes("embed") || lower.includes("e5-") || lower.includes("bge-") || lower.includes("nomic-") || lower.includes("mxbai-") || lower.includes("all-minilm") || lower.includes("gte-")) {
|
|
9030
|
+
return "embedding";
|
|
9031
|
+
}
|
|
9032
|
+
if (lower.includes("vision") || lower.includes("llava") || lower.includes("moondream") || lower.includes("minicpm-v") || lower.includes("bakllava")) {
|
|
9033
|
+
return "vision";
|
|
9034
|
+
}
|
|
9035
|
+
return "chat";
|
|
9036
|
+
}
|
|
9037
|
+
function inferLocalEmbeddingDimensions(models) {
|
|
9038
|
+
const embeddingModel = models.find((model) => model.capabilities.embeddings)?.id.toLowerCase();
|
|
9039
|
+
if (!embeddingModel) return void 0;
|
|
9040
|
+
if (embeddingModel.includes("nomic-embed")) return [768];
|
|
9041
|
+
if (embeddingModel.includes("bge-large")) return [1024];
|
|
9042
|
+
if (embeddingModel.includes("bge-base")) return [768];
|
|
9043
|
+
if (embeddingModel.includes("bge-small")) return [384];
|
|
9044
|
+
if (embeddingModel.includes("all-minilm")) return [384];
|
|
9045
|
+
if (embeddingModel.includes("mxbai-embed-large")) return [1024];
|
|
9046
|
+
return void 0;
|
|
9047
|
+
}
|
|
9048
|
+
function createLocalProviderProfile(models = []) {
|
|
9049
|
+
const embeddingDimensions = inferLocalEmbeddingDimensions(models);
|
|
9050
|
+
return {
|
|
9051
|
+
privacyTier: "local",
|
|
9052
|
+
costTier: "free",
|
|
9053
|
+
embeddingDimensions,
|
|
9054
|
+
dataClasses: ["public", "private", "sensitive"]
|
|
9055
|
+
};
|
|
9056
|
+
}
|
|
9057
|
+
async function probeEndpoint(endpoint, timeoutMs) {
|
|
9058
|
+
try {
|
|
9059
|
+
const response = await globalThis.fetch(`${endpoint.baseURL}/models`, {
|
|
9060
|
+
signal: AbortSignal.timeout(timeoutMs)
|
|
9061
|
+
});
|
|
9062
|
+
if (!response.ok) return null;
|
|
9063
|
+
const data = await response.json();
|
|
9064
|
+
let modelIds;
|
|
9065
|
+
if (data.data && Array.isArray(data.data)) {
|
|
9066
|
+
modelIds = data.data.map((m) => m.id);
|
|
9067
|
+
} else if (data.models && Array.isArray(data.models)) {
|
|
9068
|
+
modelIds = data.models.map((m) => m.name ?? m.model ?? "");
|
|
9069
|
+
} else {
|
|
9070
|
+
modelIds = [];
|
|
9071
|
+
}
|
|
9072
|
+
const models = modelIds.filter(Boolean).map((id) => {
|
|
9073
|
+
const category = classifyModel(id);
|
|
9074
|
+
return {
|
|
9075
|
+
id,
|
|
9076
|
+
name: id,
|
|
9077
|
+
capabilities: {
|
|
9078
|
+
embeddings: category === "embedding",
|
|
9079
|
+
chat: category === "chat" || category === "vision",
|
|
9080
|
+
vision: category === "vision"
|
|
9081
|
+
}
|
|
9082
|
+
};
|
|
9083
|
+
});
|
|
9084
|
+
return {
|
|
9085
|
+
id: endpoint.id,
|
|
9086
|
+
name: endpoint.name,
|
|
9087
|
+
baseURL: endpoint.baseURL,
|
|
9088
|
+
models
|
|
9089
|
+
};
|
|
9090
|
+
} catch {
|
|
9091
|
+
return null;
|
|
9092
|
+
}
|
|
9093
|
+
}
|
|
9094
|
+
async function discoverLocalProviders(options = {}) {
|
|
9095
|
+
const { extraEndpoints = [], timeoutMs = 2e3, skipPorts = [] } = options;
|
|
9096
|
+
const allEndpoints = [...LOCAL_ENDPOINTS, ...extraEndpoints];
|
|
9097
|
+
const seen = /* @__PURE__ */ new Set();
|
|
9098
|
+
const uniqueEndpoints = allEndpoints.filter((ep) => {
|
|
9099
|
+
if (seen.has(ep.baseURL)) return false;
|
|
9100
|
+
if (skipPorts.includes(ep.defaultPort)) return false;
|
|
9101
|
+
seen.add(ep.baseURL);
|
|
9102
|
+
return true;
|
|
9103
|
+
});
|
|
9104
|
+
const results = await Promise.allSettled(
|
|
9105
|
+
uniqueEndpoints.map((ep) => probeEndpoint(ep, timeoutMs))
|
|
9106
|
+
);
|
|
9107
|
+
return results.filter(
|
|
9108
|
+
(r) => r.status === "fulfilled" && r.value !== null
|
|
9109
|
+
).map((r) => r.value);
|
|
9110
|
+
}
|
|
7328
9111
|
|
|
7329
9112
|
// src/capabilities/openai-provider.ts
|
|
7330
9113
|
var OpenAICompatibleProvider = class {
|
|
@@ -7332,6 +9115,7 @@ var OpenAICompatibleProvider = class {
|
|
|
7332
9115
|
name;
|
|
7333
9116
|
tier;
|
|
7334
9117
|
capabilities;
|
|
9118
|
+
profile;
|
|
7335
9119
|
baseURL;
|
|
7336
9120
|
apiKey;
|
|
7337
9121
|
defaultModel;
|
|
@@ -7357,6 +9141,7 @@ var OpenAICompatibleProvider = class {
|
|
|
7357
9141
|
toolCalling: false,
|
|
7358
9142
|
structuredOutput: false
|
|
7359
9143
|
};
|
|
9144
|
+
this.profile = config.profile ?? (this.tier === "local-server" ? { privacyTier: "local", costTier: "free" } : { privacyTier: "external" });
|
|
7360
9145
|
}
|
|
7361
9146
|
// -------------------------------------------------------------------------
|
|
7362
9147
|
// InferenceProvider interface
|
|
@@ -7451,15 +9236,19 @@ var OpenAICompatibleProvider = class {
|
|
|
7451
9236
|
try {
|
|
7452
9237
|
const response = await this.fetch("/models", void 0, "GET");
|
|
7453
9238
|
const data = response;
|
|
7454
|
-
return data.data.map((m) =>
|
|
7455
|
-
|
|
7456
|
-
|
|
7457
|
-
|
|
7458
|
-
|
|
7459
|
-
|
|
7460
|
-
|
|
7461
|
-
|
|
7462
|
-
|
|
9239
|
+
return data.data.map((m) => {
|
|
9240
|
+
const category = classifyModel(m.id);
|
|
9241
|
+
return {
|
|
9242
|
+
id: m.id,
|
|
9243
|
+
name: m.id,
|
|
9244
|
+
capabilities: {
|
|
9245
|
+
chat: category === "chat" || category === "vision",
|
|
9246
|
+
embeddings: category === "embedding",
|
|
9247
|
+
vision: category === "vision"
|
|
9248
|
+
},
|
|
9249
|
+
owned_by: m.owned_by
|
|
9250
|
+
};
|
|
9251
|
+
});
|
|
7463
9252
|
} catch {
|
|
7464
9253
|
return [];
|
|
7465
9254
|
}
|
|
@@ -7507,10 +9296,6 @@ var OpenAICompatibleProvider = class {
|
|
|
7507
9296
|
return void 0;
|
|
7508
9297
|
}
|
|
7509
9298
|
}
|
|
7510
|
-
isEmbeddingModel(id) {
|
|
7511
|
-
const lower = id.toLowerCase();
|
|
7512
|
-
return lower.includes("embed") || lower.includes("e5-") || lower.includes("bge-") || lower.includes("nomic-") || lower.includes("mxbai-") || lower.includes("all-minilm");
|
|
7513
|
-
}
|
|
7514
9299
|
isLocalURL() {
|
|
7515
9300
|
try {
|
|
7516
9301
|
const url = new URL(this.baseURL);
|
|
@@ -7551,299 +9336,77 @@ var OpenAICompatibleProvider = class {
|
|
|
7551
9336
|
}
|
|
7552
9337
|
};
|
|
7553
9338
|
|
|
7554
|
-
// src/capabilities/
|
|
7555
|
-
|
|
7556
|
-
{
|
|
7557
|
-
|
|
7558
|
-
|
|
7559
|
-
|
|
7560
|
-
|
|
7561
|
-
|
|
7562
|
-
|
|
7563
|
-
|
|
7564
|
-
const lower = modelId.toLowerCase();
|
|
7565
|
-
if (lower.includes("embed") || lower.includes("e5-") || lower.includes("bge-") || lower.includes("nomic-") || lower.includes("mxbai-") || lower.includes("all-minilm") || lower.includes("gte-")) {
|
|
7566
|
-
return "embedding";
|
|
7567
|
-
}
|
|
7568
|
-
if (lower.includes("vision") || lower.includes("llava") || lower.includes("moondream") || lower.includes("minicpm-v") || lower.includes("bakllava")) {
|
|
7569
|
-
return "vision";
|
|
7570
|
-
}
|
|
7571
|
-
return "chat";
|
|
7572
|
-
}
|
|
7573
|
-
async function probeEndpoint(endpoint, timeoutMs) {
|
|
7574
|
-
try {
|
|
7575
|
-
const response = await globalThis.fetch(`${endpoint.baseURL}/models`, {
|
|
7576
|
-
signal: AbortSignal.timeout(timeoutMs)
|
|
7577
|
-
});
|
|
7578
|
-
if (!response.ok) return null;
|
|
7579
|
-
const data = await response.json();
|
|
7580
|
-
let modelIds;
|
|
7581
|
-
if (data.data && Array.isArray(data.data)) {
|
|
7582
|
-
modelIds = data.data.map((m) => m.id);
|
|
7583
|
-
} else if (data.models && Array.isArray(data.models)) {
|
|
7584
|
-
modelIds = data.models.map((m) => m.name ?? m.model ?? "");
|
|
7585
|
-
} else {
|
|
7586
|
-
modelIds = [];
|
|
7587
|
-
}
|
|
7588
|
-
const models = modelIds.filter(Boolean).map((id) => {
|
|
7589
|
-
const category = classifyModel(id);
|
|
7590
|
-
return {
|
|
7591
|
-
id,
|
|
7592
|
-
name: id,
|
|
7593
|
-
capabilities: {
|
|
7594
|
-
embeddings: category === "embedding",
|
|
7595
|
-
chat: category === "chat" || category === "vision",
|
|
7596
|
-
vision: category === "vision"
|
|
7597
|
-
}
|
|
7598
|
-
};
|
|
7599
|
-
});
|
|
7600
|
-
return {
|
|
7601
|
-
id: endpoint.id,
|
|
7602
|
-
name: endpoint.name,
|
|
7603
|
-
baseURL: endpoint.baseURL,
|
|
7604
|
-
models
|
|
7605
|
-
};
|
|
7606
|
-
} catch {
|
|
7607
|
-
return null;
|
|
7608
|
-
}
|
|
7609
|
-
}
|
|
7610
|
-
async function discoverLocalProviders(options = {}) {
|
|
7611
|
-
const { extraEndpoints = [], timeoutMs = 2e3, skipPorts = [] } = options;
|
|
7612
|
-
const allEndpoints = [...LOCAL_ENDPOINTS, ...extraEndpoints];
|
|
7613
|
-
const seen = /* @__PURE__ */ new Set();
|
|
7614
|
-
const uniqueEndpoints = allEndpoints.filter((ep) => {
|
|
7615
|
-
if (seen.has(ep.baseURL)) return false;
|
|
7616
|
-
if (skipPorts.includes(ep.defaultPort)) return false;
|
|
7617
|
-
seen.add(ep.baseURL);
|
|
7618
|
-
return true;
|
|
7619
|
-
});
|
|
7620
|
-
const results = await Promise.allSettled(
|
|
7621
|
-
uniqueEndpoints.map((ep) => probeEndpoint(ep, timeoutMs))
|
|
7622
|
-
);
|
|
7623
|
-
return results.filter(
|
|
7624
|
-
(r) => r.status === "fulfilled" && r.value !== null
|
|
7625
|
-
).map((r) => r.value);
|
|
7626
|
-
}
|
|
7627
|
-
|
|
7628
|
-
// src/capabilities/fallback-router.ts
|
|
7629
|
-
var DEFAULT_COOLDOWNS = {
|
|
7630
|
-
rateLimit: 3e4,
|
|
7631
|
-
serverError: 6e4,
|
|
7632
|
-
connectionFailure: 3e5,
|
|
7633
|
-
contentPolicy: 0
|
|
7634
|
-
};
|
|
7635
|
-
function classifyError(error) {
|
|
7636
|
-
const msg = error instanceof Error ? error.message : String(error);
|
|
7637
|
-
const lower = msg.toLowerCase();
|
|
7638
|
-
if (lower.includes("429") || lower.includes("rate limit")) return "rate_limit";
|
|
7639
|
-
if (lower.includes("500") || lower.includes("502") || lower.includes("503") || lower.includes("504")) return "server_error";
|
|
7640
|
-
if (lower.includes("content") && (lower.includes("policy") || lower.includes("filter"))) return "content_policy";
|
|
7641
|
-
if (lower.includes("context") && (lower.includes("window") || lower.includes("length") || lower.includes("too long"))) return "context_window";
|
|
7642
|
-
if (lower.includes("fetch") || lower.includes("network") || lower.includes("connection") || lower.includes("econnrefused") || lower.includes("timeout")) return "connection_failure";
|
|
7643
|
-
return "unknown";
|
|
9339
|
+
// src/capabilities/bridge-provider.ts
|
|
9340
|
+
function capabilitiesFromBridge(info) {
|
|
9341
|
+
return {
|
|
9342
|
+
embeddings: Boolean(info.capabilities.embeddings),
|
|
9343
|
+
chat: Boolean(info.capabilities.chat),
|
|
9344
|
+
streaming: Boolean(info.capabilities.streaming),
|
|
9345
|
+
vision: false,
|
|
9346
|
+
toolCalling: false,
|
|
9347
|
+
structuredOutput: false
|
|
9348
|
+
};
|
|
7644
9349
|
}
|
|
7645
|
-
var
|
|
7646
|
-
|
|
7647
|
-
|
|
7648
|
-
|
|
7649
|
-
|
|
7650
|
-
|
|
7651
|
-
|
|
7652
|
-
|
|
7653
|
-
|
|
7654
|
-
const available = this.getAvailableProviders();
|
|
7655
|
-
return {
|
|
7656
|
-
embeddings: available.some((p) => p.capabilities.embeddings),
|
|
7657
|
-
chat: available.some((p) => p.capabilities.chat),
|
|
7658
|
-
streaming: available.some((p) => p.capabilities.streaming),
|
|
7659
|
-
vision: available.some((p) => p.capabilities.vision),
|
|
7660
|
-
toolCalling: available.some((p) => p.capabilities.toolCalling),
|
|
7661
|
-
structuredOutput: available.some((p) => p.capabilities.structuredOutput),
|
|
7662
|
-
maxContextTokens: Math.max(
|
|
7663
|
-
...available.map((p) => p.capabilities.maxContextTokens ?? 0),
|
|
7664
|
-
0
|
|
7665
|
-
)
|
|
9350
|
+
var BridgeInferenceProvider = class {
|
|
9351
|
+
constructor(bridge, info) {
|
|
9352
|
+
this.bridge = bridge;
|
|
9353
|
+
this.id = info.id;
|
|
9354
|
+
this.name = info.name;
|
|
9355
|
+
this.tier = info.tier;
|
|
9356
|
+
this.capabilities = capabilitiesFromBridge(info);
|
|
9357
|
+
this.profile = info.profile ?? {
|
|
9358
|
+
privacyTier: "host-managed"
|
|
7666
9359
|
};
|
|
7667
9360
|
}
|
|
7668
|
-
|
|
7669
|
-
|
|
7670
|
-
|
|
7671
|
-
|
|
7672
|
-
|
|
7673
|
-
|
|
7674
|
-
|
|
7675
|
-
|
|
7676
|
-
/** Get providers not currently in cooldown */
|
|
7677
|
-
getAvailableProviders() {
|
|
7678
|
-
const now2 = Date.now();
|
|
7679
|
-
return this.providers.filter((p) => {
|
|
7680
|
-
const cd = this.cooldownMap.get(p.id);
|
|
7681
|
-
if (!cd) return true;
|
|
7682
|
-
if (now2 >= cd.expiresAt) {
|
|
7683
|
-
this.cooldownMap.delete(p.id);
|
|
7684
|
-
return true;
|
|
7685
|
-
}
|
|
7686
|
-
return false;
|
|
7687
|
-
});
|
|
7688
|
-
}
|
|
7689
|
-
/** Get providers in cooldown with their expiry info */
|
|
7690
|
-
getCoolingDown() {
|
|
7691
|
-
const now2 = Date.now();
|
|
7692
|
-
const result = [];
|
|
7693
|
-
for (const [id, entry] of this.cooldownMap) {
|
|
7694
|
-
if (now2 < entry.expiresAt) {
|
|
7695
|
-
result.push({ providerId: id, category: entry.category, expiresAt: entry.expiresAt });
|
|
7696
|
-
}
|
|
7697
|
-
}
|
|
7698
|
-
return result;
|
|
7699
|
-
}
|
|
7700
|
-
/** Manually clear cooldown for a provider */
|
|
7701
|
-
clearCooldown(providerId) {
|
|
7702
|
-
this.cooldownMap.delete(providerId);
|
|
7703
|
-
}
|
|
7704
|
-
/** Clear all cooldowns */
|
|
7705
|
-
clearAllCooldowns() {
|
|
7706
|
-
this.cooldownMap.clear();
|
|
7707
|
-
}
|
|
7708
|
-
/** Add a provider to the chain (appended at lowest priority) */
|
|
7709
|
-
addProvider(provider) {
|
|
7710
|
-
this.providers.push(provider);
|
|
7711
|
-
}
|
|
7712
|
-
/** Remove a provider from the chain */
|
|
7713
|
-
removeProvider(id) {
|
|
7714
|
-
this.providers = this.providers.filter((p) => p.id !== id);
|
|
7715
|
-
this.cooldownMap.delete(id);
|
|
7716
|
-
}
|
|
7717
|
-
/** Reorder providers (new priority order) */
|
|
7718
|
-
setOrder(ids) {
|
|
7719
|
-
const byId = new Map(this.providers.map((p) => [p.id, p]));
|
|
7720
|
-
const reordered = [];
|
|
7721
|
-
for (const id of ids) {
|
|
7722
|
-
const p = byId.get(id);
|
|
7723
|
-
if (p) reordered.push(p);
|
|
9361
|
+
id;
|
|
9362
|
+
name;
|
|
9363
|
+
tier;
|
|
9364
|
+
capabilities;
|
|
9365
|
+
profile;
|
|
9366
|
+
async embed(request) {
|
|
9367
|
+
if (!this.capabilities.embeddings) {
|
|
9368
|
+
throw new Error(`Bridge provider '${this.id}' does not support embeddings`);
|
|
7724
9369
|
}
|
|
7725
|
-
|
|
7726
|
-
|
|
9370
|
+
if (!this.bridge.inference) {
|
|
9371
|
+
throw new Error("Fortemi bridge inference router is unavailable");
|
|
7727
9372
|
}
|
|
7728
|
-
this.
|
|
7729
|
-
}
|
|
7730
|
-
// -------------------------------------------------------------------------
|
|
7731
|
-
// InferenceProvider interface — with fallback
|
|
7732
|
-
// -------------------------------------------------------------------------
|
|
7733
|
-
async embed(request) {
|
|
7734
|
-
return this.withFallback(
|
|
7735
|
-
(p) => p.capabilities.embeddings && !!p.embed,
|
|
7736
|
-
(p) => p.embed(request)
|
|
7737
|
-
);
|
|
9373
|
+
return this.bridge.inference.embed(this.id, request);
|
|
7738
9374
|
}
|
|
7739
9375
|
async complete(request) {
|
|
7740
|
-
|
|
7741
|
-
(
|
|
7742
|
-
|
|
7743
|
-
)
|
|
9376
|
+
if (!this.capabilities.chat) {
|
|
9377
|
+
throw new Error(`Bridge provider '${this.id}' does not support chat`);
|
|
9378
|
+
}
|
|
9379
|
+
if (!this.bridge.inference) {
|
|
9380
|
+
throw new Error("Fortemi bridge inference router is unavailable");
|
|
9381
|
+
}
|
|
9382
|
+
return this.bridge.inference.complete(this.id, request);
|
|
7744
9383
|
}
|
|
7745
|
-
|
|
7746
|
-
|
|
7747
|
-
|
|
7748
|
-
throw new Error("No available providers with streaming capability");
|
|
9384
|
+
stream(request) {
|
|
9385
|
+
if (!this.capabilities.streaming) {
|
|
9386
|
+
throw new Error(`Bridge provider '${this.id}' does not support streaming`);
|
|
7749
9387
|
}
|
|
7750
|
-
|
|
9388
|
+
if (!this.bridge.inference?.stream) {
|
|
9389
|
+
throw new Error("Fortemi bridge streaming router is unavailable");
|
|
9390
|
+
}
|
|
9391
|
+
return this.bridge.inference.stream(this.id, request);
|
|
7751
9392
|
}
|
|
7752
9393
|
async listModels() {
|
|
7753
|
-
|
|
7754
|
-
const results = await Promise.allSettled(
|
|
7755
|
-
available.map((p) => p.listModels())
|
|
7756
|
-
);
|
|
7757
|
-
const models = [];
|
|
7758
|
-
for (const r of results) {
|
|
7759
|
-
if (r.status === "fulfilled") models.push(...r.value);
|
|
7760
|
-
}
|
|
7761
|
-
return models;
|
|
9394
|
+
return [];
|
|
7762
9395
|
}
|
|
7763
9396
|
async probe() {
|
|
7764
|
-
|
|
7765
|
-
|
|
7766
|
-
return { status: "down", latencyMs: 0, message: "All providers in cooldown" };
|
|
9397
|
+
if (!this.bridge.inference) {
|
|
9398
|
+
return { status: "down", latencyMs: 0, message: "Fortemi bridge inference router is unavailable" };
|
|
7767
9399
|
}
|
|
7768
|
-
|
|
7769
|
-
const results = await Promise.allSettled(
|
|
7770
|
-
available.map((p) => p.probe())
|
|
7771
|
-
);
|
|
7772
|
-
const okCount = results.filter(
|
|
7773
|
-
(r) => r.status === "fulfilled" && r.value.status === "ok"
|
|
7774
|
-
).length;
|
|
7775
|
-
return {
|
|
7776
|
-
status: okCount === available.length ? "ok" : okCount > 0 ? "degraded" : "down",
|
|
7777
|
-
latencyMs: Date.now() - start,
|
|
7778
|
-
message: `${okCount}/${available.length} providers healthy`
|
|
7779
|
-
};
|
|
9400
|
+
return this.bridge.inference.probeProvider(this.id);
|
|
7780
9401
|
}
|
|
7781
9402
|
dispose() {
|
|
7782
|
-
for (const p of this.providers) {
|
|
7783
|
-
p.dispose();
|
|
7784
|
-
}
|
|
7785
|
-
this.providers = [];
|
|
7786
|
-
this.cooldownMap.clear();
|
|
7787
|
-
}
|
|
7788
|
-
// -------------------------------------------------------------------------
|
|
7789
|
-
// Core fallback logic
|
|
7790
|
-
// -------------------------------------------------------------------------
|
|
7791
|
-
async withFallback(filter, execute) {
|
|
7792
|
-
const candidates = this.getAvailableProviders().filter(filter);
|
|
7793
|
-
if (candidates.length === 0) {
|
|
7794
|
-
throw new Error("No available providers for this request");
|
|
7795
|
-
}
|
|
7796
|
-
let lastError;
|
|
7797
|
-
for (const provider of candidates) {
|
|
7798
|
-
try {
|
|
7799
|
-
return await execute(provider);
|
|
7800
|
-
} catch (err) {
|
|
7801
|
-
lastError = err instanceof Error ? err : new Error(String(err));
|
|
7802
|
-
const category = classifyError(err);
|
|
7803
|
-
this.applyCooldown(provider.id, category);
|
|
7804
|
-
const nextCandidate = candidates[candidates.indexOf(provider) + 1];
|
|
7805
|
-
if (nextCandidate) {
|
|
7806
|
-
this.events?.emit("provider.fallback", {
|
|
7807
|
-
fromProvider: provider.id,
|
|
7808
|
-
toProvider: nextCandidate.id,
|
|
7809
|
-
errorCategory: category,
|
|
7810
|
-
error: lastError.message
|
|
7811
|
-
});
|
|
7812
|
-
}
|
|
7813
|
-
}
|
|
7814
|
-
}
|
|
7815
|
-
throw lastError ?? new Error("All providers failed");
|
|
7816
|
-
}
|
|
7817
|
-
applyCooldown(providerId, category) {
|
|
7818
|
-
let cooldownMs;
|
|
7819
|
-
switch (category) {
|
|
7820
|
-
case "rate_limit":
|
|
7821
|
-
cooldownMs = this.cooldowns.rateLimit;
|
|
7822
|
-
break;
|
|
7823
|
-
case "server_error":
|
|
7824
|
-
cooldownMs = this.cooldowns.serverError;
|
|
7825
|
-
break;
|
|
7826
|
-
case "connection_failure":
|
|
7827
|
-
cooldownMs = this.cooldowns.connectionFailure;
|
|
7828
|
-
break;
|
|
7829
|
-
case "content_policy":
|
|
7830
|
-
cooldownMs = this.cooldowns.contentPolicy;
|
|
7831
|
-
break;
|
|
7832
|
-
default:
|
|
7833
|
-
cooldownMs = this.cooldowns.serverError;
|
|
7834
|
-
}
|
|
7835
|
-
if (cooldownMs > 0) {
|
|
7836
|
-
const expiresAt = Date.now() + cooldownMs;
|
|
7837
|
-
this.cooldownMap.set(providerId, { expiresAt, category });
|
|
7838
|
-
this.events?.emit("provider.cooldown", {
|
|
7839
|
-
providerId,
|
|
7840
|
-
errorCategory: category,
|
|
7841
|
-
cooldownMs,
|
|
7842
|
-
expiresAt
|
|
7843
|
-
});
|
|
7844
|
-
}
|
|
7845
9403
|
}
|
|
7846
9404
|
};
|
|
9405
|
+
async function createBridgeInferenceProviders(bridge) {
|
|
9406
|
+
if (!bridge.inference) return [];
|
|
9407
|
+
const providers = await bridge.inference.listProviders();
|
|
9408
|
+
return providers.map((info) => new BridgeInferenceProvider(bridge, info));
|
|
9409
|
+
}
|
|
7847
9410
|
|
|
7848
9411
|
// src/fortemi-bridge.ts
|
|
7849
9412
|
function getFortemiBridge(host = globalThis) {
|
|
@@ -7872,6 +9435,159 @@ async function hasFortemiSecureSecrets(host = globalThis) {
|
|
|
7872
9435
|
}
|
|
7873
9436
|
}
|
|
7874
9437
|
|
|
9438
|
+
// src/capabilities/inference-runtime.ts
|
|
9439
|
+
function defineInferenceRuntime(config) {
|
|
9440
|
+
return config;
|
|
9441
|
+
}
|
|
9442
|
+
function defineInferenceProvider(provider) {
|
|
9443
|
+
return { kind: "provider", provider };
|
|
9444
|
+
}
|
|
9445
|
+
function defineOpenAICompatibleProvider(config) {
|
|
9446
|
+
return { kind: "openai-compatible", config };
|
|
9447
|
+
}
|
|
9448
|
+
function defineLegacyInferenceProvider(config) {
|
|
9449
|
+
return { kind: "legacy", ...config };
|
|
9450
|
+
}
|
|
9451
|
+
function mergeInferenceRuntimeConfigs(...configs) {
|
|
9452
|
+
const merged = {};
|
|
9453
|
+
for (const config of configs) {
|
|
9454
|
+
if (!config) continue;
|
|
9455
|
+
if (config.providers?.length) {
|
|
9456
|
+
merged.providers = mergeConfiguredProviders(merged.providers, config.providers);
|
|
9457
|
+
}
|
|
9458
|
+
if (config.routes) {
|
|
9459
|
+
merged.routes = { ...merged.routes ?? {}, ...config.routes };
|
|
9460
|
+
}
|
|
9461
|
+
if (config.activeProviderId !== void 0) {
|
|
9462
|
+
merged.activeProviderId = config.activeProviderId;
|
|
9463
|
+
}
|
|
9464
|
+
if (config.bridgeHost !== void 0) {
|
|
9465
|
+
merged.bridgeHost = config.bridgeHost;
|
|
9466
|
+
}
|
|
9467
|
+
if (config.includeBridgeProviders !== void 0) {
|
|
9468
|
+
merged.includeBridgeProviders = config.includeBridgeProviders;
|
|
9469
|
+
}
|
|
9470
|
+
if (config.discoverLocal !== void 0) {
|
|
9471
|
+
merged.discoverLocal = config.discoverLocal;
|
|
9472
|
+
}
|
|
9473
|
+
if (config.embeddingTaskSelection) {
|
|
9474
|
+
merged.embeddingTaskSelection = {
|
|
9475
|
+
...merged.embeddingTaskSelection ?? {},
|
|
9476
|
+
...config.embeddingTaskSelection
|
|
9477
|
+
};
|
|
9478
|
+
}
|
|
9479
|
+
}
|
|
9480
|
+
return merged;
|
|
9481
|
+
}
|
|
9482
|
+
function getConfiguredInferenceProviderId(config) {
|
|
9483
|
+
switch (config.kind) {
|
|
9484
|
+
case "provider":
|
|
9485
|
+
return config.provider.id;
|
|
9486
|
+
case "openai-compatible":
|
|
9487
|
+
return config.config.id;
|
|
9488
|
+
case "legacy":
|
|
9489
|
+
return config.id ?? "legacy";
|
|
9490
|
+
}
|
|
9491
|
+
}
|
|
9492
|
+
function mergeConfiguredProviders(previous, next) {
|
|
9493
|
+
const byId = /* @__PURE__ */ new Map();
|
|
9494
|
+
for (const provider of previous ?? []) {
|
|
9495
|
+
byId.set(getConfiguredInferenceProviderId(provider), provider);
|
|
9496
|
+
}
|
|
9497
|
+
for (const provider of next) {
|
|
9498
|
+
const id = getConfiguredInferenceProviderId(provider);
|
|
9499
|
+
byId.delete(id);
|
|
9500
|
+
byId.set(id, provider);
|
|
9501
|
+
}
|
|
9502
|
+
return Array.from(byId.values());
|
|
9503
|
+
}
|
|
9504
|
+
async function configureInferenceRuntime(options = {}) {
|
|
9505
|
+
const registry = options.registry ?? new ProviderRegistry(options.events);
|
|
9506
|
+
setEmbeddingTaskSelectionOptions(options.embeddingTaskSelection);
|
|
9507
|
+
for (const providerConfig of options.providers ?? []) {
|
|
9508
|
+
registry.add(createConfiguredProvider(providerConfig));
|
|
9509
|
+
}
|
|
9510
|
+
if (options.includeBridgeProviders !== false) {
|
|
9511
|
+
const bridge = getFortemiBridge(options.bridgeHost);
|
|
9512
|
+
if (bridge) {
|
|
9513
|
+
for (const provider of await createBridgeInferenceProviders(bridge)) {
|
|
9514
|
+
if (!registry.get(provider.id)) registry.add(provider);
|
|
9515
|
+
}
|
|
9516
|
+
}
|
|
9517
|
+
}
|
|
9518
|
+
if (options.discoverLocal) {
|
|
9519
|
+
const discoveryOptions = options.discoverLocal === true ? {} : options.discoverLocal;
|
|
9520
|
+
const discovered = await discoverLocalProviders(discoveryOptions);
|
|
9521
|
+
for (const provider of discovered) {
|
|
9522
|
+
const id = `local:${provider.id}`;
|
|
9523
|
+
if (registry.get(id)) continue;
|
|
9524
|
+
registry.add(new OpenAICompatibleProvider({
|
|
9525
|
+
id,
|
|
9526
|
+
name: provider.name,
|
|
9527
|
+
baseURL: provider.baseURL,
|
|
9528
|
+
tier: "local-server",
|
|
9529
|
+
defaultModel: provider.models.find((model) => model.capabilities.chat)?.id,
|
|
9530
|
+
defaultEmbeddingModel: provider.models.find((model) => model.capabilities.embeddings)?.id,
|
|
9531
|
+
profile: createLocalProviderProfile(provider.models)
|
|
9532
|
+
}));
|
|
9533
|
+
}
|
|
9534
|
+
}
|
|
9535
|
+
for (const [task, route] of Object.entries(options.routes ?? {})) {
|
|
9536
|
+
if (route) registry.setRoute(task, route);
|
|
9537
|
+
}
|
|
9538
|
+
if (options.activeProviderId) {
|
|
9539
|
+
registry.setActive(options.activeProviderId);
|
|
9540
|
+
}
|
|
9541
|
+
if (options.capabilityManager) {
|
|
9542
|
+
wireCapabilities(options.capabilityManager, registry);
|
|
9543
|
+
}
|
|
9544
|
+
const routeValidation = registry.validateRoutes();
|
|
9545
|
+
return {
|
|
9546
|
+
registry,
|
|
9547
|
+
providers: registry.list(),
|
|
9548
|
+
routeValidation,
|
|
9549
|
+
routeIssues: routeValidation.flatMap((route) => route.issues)
|
|
9550
|
+
};
|
|
9551
|
+
}
|
|
9552
|
+
function createConfiguredProvider(config) {
|
|
9553
|
+
switch (config.kind) {
|
|
9554
|
+
case "provider":
|
|
9555
|
+
return config.provider;
|
|
9556
|
+
case "openai-compatible":
|
|
9557
|
+
return new OpenAICompatibleProvider(config.config);
|
|
9558
|
+
case "legacy":
|
|
9559
|
+
return createLegacyProvider(config);
|
|
9560
|
+
}
|
|
9561
|
+
}
|
|
9562
|
+
function wireCapabilities(manager, registry) {
|
|
9563
|
+
if (registry.hasEmbeddings()) {
|
|
9564
|
+
manager.registerLoader("semantic", async () => {
|
|
9565
|
+
setEmbedFunction((texts, options) => {
|
|
9566
|
+
const request = {
|
|
9567
|
+
texts,
|
|
9568
|
+
task: options?.task ?? "embedding.document"
|
|
9569
|
+
};
|
|
9570
|
+
if (options?.model) Object.assign(request, { model: options.model });
|
|
9571
|
+
return registry.embed(request).then((result) => result.vectors);
|
|
9572
|
+
});
|
|
9573
|
+
});
|
|
9574
|
+
}
|
|
9575
|
+
if (registry.hasChat()) {
|
|
9576
|
+
manager.registerLoader("llm", async () => {
|
|
9577
|
+
setLlmFunction((prompt, options) => {
|
|
9578
|
+
const request = {
|
|
9579
|
+
prompt,
|
|
9580
|
+
task: options?.task ?? "chat.general"
|
|
9581
|
+
};
|
|
9582
|
+
if (options?.model) Object.assign(request, { model: options.model });
|
|
9583
|
+
if (options?.maxTokens !== void 0) Object.assign(request, { maxTokens: options.maxTokens });
|
|
9584
|
+
if (options?.temperature !== void 0) Object.assign(request, { temperature: options.temperature });
|
|
9585
|
+
return registry.complete(request).then((result) => result.text);
|
|
9586
|
+
});
|
|
9587
|
+
});
|
|
9588
|
+
}
|
|
9589
|
+
}
|
|
9590
|
+
|
|
7875
9591
|
// src/security/plugin-content.ts
|
|
7876
9592
|
var DEFAULT_DIRECTIVES = {
|
|
7877
9593
|
"default-src": ["'self'"],
|
|
@@ -7913,12 +9629,12 @@ function parseSriToken(integrity) {
|
|
|
7913
9629
|
const separator = token.indexOf("-");
|
|
7914
9630
|
if (separator <= 0) throw new Error("Invalid SRI token");
|
|
7915
9631
|
const algorithm = token.slice(0, separator).toLowerCase();
|
|
7916
|
-
const
|
|
9632
|
+
const digest2 = token.slice(separator + 1);
|
|
7917
9633
|
if (!SUPPORTED_SRI_ALGORITHMS.has(algorithm)) {
|
|
7918
9634
|
throw new Error("Unsupported SRI algorithm: " + algorithm);
|
|
7919
9635
|
}
|
|
7920
|
-
if (!
|
|
7921
|
-
return { algorithm, digest };
|
|
9636
|
+
if (!digest2) throw new Error("Invalid SRI digest");
|
|
9637
|
+
return { algorithm, digest: digest2 };
|
|
7922
9638
|
}
|
|
7923
9639
|
function toBase64(bytes) {
|
|
7924
9640
|
const data = new Uint8Array(bytes);
|
|
@@ -7935,8 +9651,8 @@ async function computeSri(data, algorithm = "sha384") {
|
|
|
7935
9651
|
if (!SUPPORTED_SRI_ALGORITHMS.has(normalized)) {
|
|
7936
9652
|
throw new Error("Unsupported SRI algorithm: " + algorithm);
|
|
7937
9653
|
}
|
|
7938
|
-
const
|
|
7939
|
-
return normalized + "-" + toBase64(
|
|
9654
|
+
const digest2 = await crypto.subtle.digest(normalized.toUpperCase().replace("SHA", "SHA-"), toArrayBuffer(data));
|
|
9655
|
+
return normalized + "-" + toBase64(digest2);
|
|
7940
9656
|
}
|
|
7941
9657
|
async function verifySri(data, integrity) {
|
|
7942
9658
|
const expected = parseSriToken(integrity);
|
|
@@ -8968,12 +10684,12 @@ async function verifyShardSignature(input) {
|
|
|
8968
10684
|
false,
|
|
8969
10685
|
["verify"]
|
|
8970
10686
|
);
|
|
8971
|
-
const
|
|
10687
|
+
const digest2 = await sha256Hex(canonicalPayloadBytes(payload));
|
|
8972
10688
|
signatureValid = await globalThis.crypto.subtle.verify(
|
|
8973
10689
|
"Ed25519",
|
|
8974
10690
|
key,
|
|
8975
10691
|
toBufferSource(base64urlToBytes(envelope.signature)),
|
|
8976
|
-
toBufferSource(new TextEncoder().encode(
|
|
10692
|
+
toBufferSource(new TextEncoder().encode(digest2))
|
|
8977
10693
|
);
|
|
8978
10694
|
} catch (err) {
|
|
8979
10695
|
return { ok: false, reason: "malformed", detail: err instanceof Error ? err.message : String(err) };
|
|
@@ -9018,11 +10734,11 @@ async function signShard(input) {
|
|
|
9018
10734
|
manifest_digest: await sha256Hex(manifest),
|
|
9019
10735
|
blob_digests: sidecarBlobDigests(input.files)
|
|
9020
10736
|
};
|
|
9021
|
-
const
|
|
10737
|
+
const digest2 = await sha256Hex(canonicalPayloadBytes(payload));
|
|
9022
10738
|
const signature = await globalThis.crypto.subtle.sign(
|
|
9023
10739
|
"Ed25519",
|
|
9024
10740
|
input.privateKey,
|
|
9025
|
-
toBufferSource(new TextEncoder().encode(
|
|
10741
|
+
toBufferSource(new TextEncoder().encode(digest2))
|
|
9026
10742
|
);
|
|
9027
10743
|
const envelope = {
|
|
9028
10744
|
...payload,
|
|
@@ -27567,7 +29283,7 @@ function addCanonicalFormats(ajv) {
|
|
|
27567
29283
|
}
|
|
27568
29284
|
function getLegacyAjv() {
|
|
27569
29285
|
if (!legacyAjvInstance) {
|
|
27570
|
-
legacyAjvInstance = new
|
|
29286
|
+
legacyAjvInstance = new Ajv20202({
|
|
27571
29287
|
allErrors: true,
|
|
27572
29288
|
strict: true,
|
|
27573
29289
|
validateFormats: false
|
|
@@ -27578,7 +29294,7 @@ function getLegacyAjv() {
|
|
|
27578
29294
|
}
|
|
27579
29295
|
function getCoreAjv() {
|
|
27580
29296
|
if (!coreAjvInstance) {
|
|
27581
|
-
coreAjvInstance = new
|
|
29297
|
+
coreAjvInstance = new Ajv20202({
|
|
27582
29298
|
allErrors: true,
|
|
27583
29299
|
strict: true,
|
|
27584
29300
|
validateFormats: true
|
|
@@ -27598,7 +29314,7 @@ function getCoreAjv() {
|
|
|
27598
29314
|
}
|
|
27599
29315
|
function getRecordAjv() {
|
|
27600
29316
|
if (!recordAjvInstance) {
|
|
27601
|
-
recordAjvInstance = new
|
|
29317
|
+
recordAjvInstance = new Ajv20202({
|
|
27602
29318
|
allErrors: true,
|
|
27603
29319
|
strict: true,
|
|
27604
29320
|
validateFormats: true
|
|
@@ -27618,7 +29334,7 @@ function getRecordAjv() {
|
|
|
27618
29334
|
}
|
|
27619
29335
|
function getFullAjv() {
|
|
27620
29336
|
if (!fullAjvInstance) {
|
|
27621
|
-
fullAjvInstance = new
|
|
29337
|
+
fullAjvInstance = new Ajv20202({
|
|
27622
29338
|
allErrors: true,
|
|
27623
29339
|
strict: true,
|
|
27624
29340
|
validateFormats: true
|
|
@@ -28278,7 +29994,7 @@ async function promoteBlobs(blobStore, blobs) {
|
|
|
28278
29994
|
|
|
28279
29995
|
// src/shard/full-v1-store.ts
|
|
28280
29996
|
var decoder3 = new TextDecoder();
|
|
28281
|
-
var
|
|
29997
|
+
var encoder2 = new TextEncoder();
|
|
28282
29998
|
function emptyCounts() {
|
|
28283
29999
|
return {
|
|
28284
30000
|
notes: 0,
|
|
@@ -28329,15 +30045,15 @@ function componentRecords(component, bytes) {
|
|
|
28329
30045
|
const parsed = spec.encoding === "json-array" ? parseJsonArrayBytes(bytes) : parseJsonlBytes(bytes);
|
|
28330
30046
|
return parsed;
|
|
28331
30047
|
}
|
|
28332
|
-
function
|
|
28333
|
-
if (Array.isArray(value)) return `[${value.map(
|
|
30048
|
+
function canonicalJson4(value) {
|
|
30049
|
+
if (Array.isArray(value)) return `[${value.map(canonicalJson4).join(",")}]`;
|
|
28334
30050
|
if (value && typeof value === "object") {
|
|
28335
|
-
return `{${Object.entries(value).sort(([left], [right]) => left.localeCompare(right)).map(([key, item]) => `${JSON.stringify(key)}:${
|
|
30051
|
+
return `{${Object.entries(value).sort(([left], [right]) => left.localeCompare(right)).map(([key, item]) => `${JSON.stringify(key)}:${canonicalJson4(item)}`).join(",")}}`;
|
|
28336
30052
|
}
|
|
28337
30053
|
return JSON.stringify(value);
|
|
28338
30054
|
}
|
|
28339
30055
|
function encodeComponentRecords(component, records) {
|
|
28340
|
-
return
|
|
30056
|
+
return encoder2.encode(FULL_V1_COMPONENT_FILES[component].encoding === "json-array" ? JSON.stringify(records) : records.map((record) => JSON.stringify(record)).join("\n"));
|
|
28341
30057
|
}
|
|
28342
30058
|
function attachmentBlobReferences(files) {
|
|
28343
30059
|
const refs = /* @__PURE__ */ new Map();
|
|
@@ -28588,7 +30304,7 @@ async function exportFullV1Snapshot(db, blobStore) {
|
|
|
28588
30304
|
for (const row of persisted.rows) currentRecords.get(row.component)?.push(row.record_json);
|
|
28589
30305
|
const changed = [...currentRecords].some(([component, records]) => {
|
|
28590
30306
|
const original = componentRecords(component, files.get(FULL_V1_COMPONENT_FILES[component].file));
|
|
28591
|
-
return
|
|
30307
|
+
return canonicalJson4(records) !== canonicalJson4(original);
|
|
28592
30308
|
});
|
|
28593
30309
|
let blobRefs;
|
|
28594
30310
|
if (changed) {
|
|
@@ -28607,7 +30323,7 @@ async function exportFullV1Snapshot(db, blobStore) {
|
|
|
28607
30323
|
name: "fortemi-react-full-v1-store",
|
|
28608
30324
|
version: manifest.producer?.version ?? "unknown"
|
|
28609
30325
|
};
|
|
28610
|
-
files.set("manifest.json",
|
|
30326
|
+
files.set("manifest.json", encoder2.encode(JSON.stringify(manifest, null, 2)));
|
|
28611
30327
|
files.delete("signature.json");
|
|
28612
30328
|
blobRefs = attachmentBlobReferences(files);
|
|
28613
30329
|
} else {
|
|
@@ -28631,7 +30347,7 @@ async function exportFullV1Snapshot(db, blobStore) {
|
|
|
28631
30347
|
}
|
|
28632
30348
|
return { success: true, archive: packTarGz(files), errors: [], capability_report: capability };
|
|
28633
30349
|
}
|
|
28634
|
-
var
|
|
30350
|
+
var encoder3 = new TextEncoder();
|
|
28635
30351
|
var decoder4 = new TextDecoder();
|
|
28636
30352
|
function iso2(value) {
|
|
28637
30353
|
if (value === null) return null;
|
|
@@ -28660,7 +30376,7 @@ function readRecords(files, component) {
|
|
|
28660
30376
|
function writeRecords(files, component, records) {
|
|
28661
30377
|
const spec = FULL_V1_COMPONENT_FILES[component];
|
|
28662
30378
|
const text = spec.encoding === "json-array" ? JSON.stringify(records) : records.map((record) => JSON.stringify(record)).join("\n");
|
|
28663
|
-
files.set(spec.file,
|
|
30379
|
+
files.set(spec.file, encoder3.encode(text));
|
|
28664
30380
|
}
|
|
28665
30381
|
async function liveRepresentationLosses(db) {
|
|
28666
30382
|
const tables = [
|
|
@@ -29127,7 +30843,7 @@ async function exportLiveFullV1(db, coreArchive, legacyArchive, options) {
|
|
|
29127
30843
|
capability_report: capability
|
|
29128
30844
|
};
|
|
29129
30845
|
}
|
|
29130
|
-
files.set("manifest.json",
|
|
30846
|
+
files.set("manifest.json", encoder3.encode(JSON.stringify(manifest, null, 2)));
|
|
29131
30847
|
for (const note of records.notes) {
|
|
29132
30848
|
const attachments = Array.isArray(note.attachments) ? note.attachments : [];
|
|
29133
30849
|
for (const projection of attachments) {
|
|
@@ -29172,7 +30888,7 @@ async function exportLiveFullV1(db, coreArchive, legacyArchive, options) {
|
|
|
29172
30888
|
}
|
|
29173
30889
|
|
|
29174
30890
|
// src/shard/shard-export.ts
|
|
29175
|
-
var
|
|
30891
|
+
var encoder4 = new TextEncoder();
|
|
29176
30892
|
var CORE_V1_FILES = /* @__PURE__ */ new Set([
|
|
29177
30893
|
"notes.jsonl",
|
|
29178
30894
|
"collections.json",
|
|
@@ -29591,11 +31307,11 @@ async function exportShardBytes(db, options, mode) {
|
|
|
29591
31307
|
const slice = shardNotes.slice(offset, offset + clusterSize);
|
|
29592
31308
|
const href = `notes/${String(offset).padStart(6, "0")}.jsonl`;
|
|
29593
31309
|
clusters.push({ href, offset });
|
|
29594
|
-
files.set(href,
|
|
31310
|
+
files.set(href, encoder4.encode(slice.map((n) => JSON.stringify(n)).join("\n")));
|
|
29595
31311
|
}
|
|
29596
31312
|
layout = { clusters: { notes: clusters } };
|
|
29597
31313
|
} else {
|
|
29598
|
-
files.set("notes.jsonl",
|
|
31314
|
+
files.set("notes.jsonl", encoder4.encode(shardNotes.map((n) => JSON.stringify(n)).join("\n")));
|
|
29599
31315
|
}
|
|
29600
31316
|
components.push("notes");
|
|
29601
31317
|
counts.notes = notes.length;
|
|
@@ -29623,7 +31339,7 @@ async function exportShardBytes(db, options, mode) {
|
|
|
29623
31339
|
mode?.nativeSchema2Presence
|
|
29624
31340
|
);
|
|
29625
31341
|
}
|
|
29626
|
-
files.set("collections.json",
|
|
31342
|
+
files.set("collections.json", encoder4.encode(JSON.stringify(shardCollections)));
|
|
29627
31343
|
components.push("collections");
|
|
29628
31344
|
counts.collections = shardCollections.length;
|
|
29629
31345
|
const allTagRows = await db.query(
|
|
@@ -29639,7 +31355,7 @@ async function exportShardBytes(db, options, mode) {
|
|
|
29639
31355
|
const shardTags = tagsToShard(
|
|
29640
31356
|
relevantTags.map((r) => ({ name: r.tag, created_at: r.created_at }))
|
|
29641
31357
|
);
|
|
29642
|
-
files.set("tags.json",
|
|
31358
|
+
files.set("tags.json", encoder4.encode(JSON.stringify(shardTags)));
|
|
29643
31359
|
components.push("tags");
|
|
29644
31360
|
counts.tags = shardTags.length;
|
|
29645
31361
|
const templateRows = await db.query(`SELECT * FROM template ORDER BY created_at, id`);
|
|
@@ -29655,7 +31371,7 @@ async function exportShardBytes(db, options, mode) {
|
|
|
29655
31371
|
mode?.nativeSchema2Presence
|
|
29656
31372
|
);
|
|
29657
31373
|
}
|
|
29658
|
-
files.set("templates.json",
|
|
31374
|
+
files.set("templates.json", encoder4.encode(JSON.stringify(shardTemplates)));
|
|
29659
31375
|
components.push("templates");
|
|
29660
31376
|
counts.templates = shardTemplates.length;
|
|
29661
31377
|
}
|
|
@@ -29680,7 +31396,7 @@ async function exportShardBytes(db, options, mode) {
|
|
|
29680
31396
|
);
|
|
29681
31397
|
}
|
|
29682
31398
|
const linksJsonl = shardLinks.map((l) => JSON.stringify(l)).join("\n");
|
|
29683
|
-
files.set("links.jsonl",
|
|
31399
|
+
files.set("links.jsonl", encoder4.encode(linksJsonl));
|
|
29684
31400
|
components.push("links");
|
|
29685
31401
|
counts.links = shardLinks.length;
|
|
29686
31402
|
const allNoteSkosRows = await db.query(`SELECT * FROM note_skos_tag ORDER BY created_at`);
|
|
@@ -29697,25 +31413,25 @@ async function exportShardBytes(db, options, mode) {
|
|
|
29697
31413
|
(row) => exportedConceptIds.has(row.source_concept_id) && exportedConceptIds.has(row.target_concept_id)
|
|
29698
31414
|
) : allRelationRows.rows;
|
|
29699
31415
|
const shardSkosSchemes = filteredSchemeRows.map(skosSchemeToShard);
|
|
29700
|
-
files.set("skos_schemes.json",
|
|
31416
|
+
files.set("skos_schemes.json", encoder4.encode(JSON.stringify(shardSkosSchemes)));
|
|
29701
31417
|
components.push("skos_schemes");
|
|
29702
31418
|
counts.skos_schemes = shardSkosSchemes.length;
|
|
29703
31419
|
const shardSkosConcepts = filteredConceptRows.map(skosConceptToShard);
|
|
29704
|
-
files.set("skos_concepts.json",
|
|
31420
|
+
files.set("skos_concepts.json", encoder4.encode(JSON.stringify(shardSkosConcepts)));
|
|
29705
31421
|
components.push("skos_concepts");
|
|
29706
31422
|
counts.skos_concepts = shardSkosConcepts.length;
|
|
29707
31423
|
const skosRelationsJsonl = filteredRelationRows.map((row) => JSON.stringify(skosRelationToShard(row))).join("\n");
|
|
29708
|
-
files.set("skos_relations.jsonl",
|
|
31424
|
+
files.set("skos_relations.jsonl", encoder4.encode(skosRelationsJsonl));
|
|
29709
31425
|
components.push("skos_relations");
|
|
29710
31426
|
counts.skos_relations = filteredRelationRows.length;
|
|
29711
31427
|
const noteSkosJsonl = filteredNoteSkosRows.map((row) => JSON.stringify(noteSkosTagToShard(row))).join("\n");
|
|
29712
|
-
files.set("note_skos_tags.jsonl",
|
|
31428
|
+
files.set("note_skos_tags.jsonl", encoder4.encode(noteSkosJsonl));
|
|
29713
31429
|
components.push("note_skos_tags");
|
|
29714
31430
|
counts.note_skos_tags = filteredNoteSkosRows.length;
|
|
29715
31431
|
const provenanceRows = await db.query(`SELECT * FROM provenance_edge ORDER BY started_at`);
|
|
29716
31432
|
const filteredProvenanceRows = isFiltered ? provenanceRows.rows.filter((row) => row.entity_type !== "note" || exportedNoteIds.has(row.entity_id)) : provenanceRows.rows;
|
|
29717
31433
|
const provenanceJsonl = filteredProvenanceRows.map((row) => JSON.stringify(provenanceEdgeToShard(row))).join("\n");
|
|
29718
|
-
files.set("provenance_edges.jsonl",
|
|
31434
|
+
files.set("provenance_edges.jsonl", encoder4.encode(provenanceJsonl));
|
|
29719
31435
|
components.push("provenance_edges");
|
|
29720
31436
|
counts.provenance_edges = filteredProvenanceRows.length;
|
|
29721
31437
|
if (options?.includeEmbeddings) {
|
|
@@ -29755,7 +31471,7 @@ async function exportShardBytes(db, options, mode) {
|
|
|
29755
31471
|
freshness_json: { status: "unknown" }
|
|
29756
31472
|
} : row
|
|
29757
31473
|
));
|
|
29758
|
-
files.set("embedding_sets.json",
|
|
31474
|
+
files.set("embedding_sets.json", encoder4.encode(JSON.stringify(shardEmbSets)));
|
|
29759
31475
|
components.push("embedding_sets");
|
|
29760
31476
|
counts.embedding_sets = shardEmbSets.length;
|
|
29761
31477
|
const embeddingConfigRows = await db.query(
|
|
@@ -29765,7 +31481,7 @@ async function exportShardBytes(db, options, mode) {
|
|
|
29765
31481
|
);
|
|
29766
31482
|
if (embeddingConfigRows.rows.length > 0) {
|
|
29767
31483
|
const shardEmbeddingConfigs = embeddingConfigRows.rows.map((row) => embeddingConfigToShard(row));
|
|
29768
|
-
files.set("embedding_configs.json",
|
|
31484
|
+
files.set("embedding_configs.json", encoder4.encode(JSON.stringify(shardEmbeddingConfigs)));
|
|
29769
31485
|
components.push("embedding_configs");
|
|
29770
31486
|
counts.embedding_configs = shardEmbeddingConfigs.length;
|
|
29771
31487
|
}
|
|
@@ -29778,7 +31494,7 @@ async function exportShardBytes(db, options, mode) {
|
|
|
29778
31494
|
(member) => exportedSetIds.has(member.embedding_set_id) && exportedNoteIds.has(member.note_id) && (includeMaterializedSelectors || !virtualSetIds.has(member.embedding_set_id))
|
|
29779
31495
|
);
|
|
29780
31496
|
const membersJsonl = scopedEmbMemberRows.map((m) => JSON.stringify(embeddingSetMemberToShard(m))).join("\n");
|
|
29781
|
-
files.set("embedding_set_members.jsonl",
|
|
31497
|
+
files.set("embedding_set_members.jsonl", encoder4.encode(membersJsonl));
|
|
29782
31498
|
components.push("embedding_set_members");
|
|
29783
31499
|
counts.embedding_set_members = scopedEmbMemberRows.length;
|
|
29784
31500
|
const embRows = await db.query(
|
|
@@ -29804,7 +31520,7 @@ async function exportShardBytes(db, options, mode) {
|
|
|
29804
31520
|
(embedding) => exportedSetIds.has(embedding.embedding_set_id) && exportedNoteIds.has(embedding.note_id) && (memberEmbeddingIds.size === 0 || memberEmbeddingIds.has(embedding.id))
|
|
29805
31521
|
);
|
|
29806
31522
|
const embJsonl = scopedEmbRows.map((e) => JSON.stringify(embeddingToShard(e))).join("\n");
|
|
29807
|
-
files.set("embeddings.jsonl",
|
|
31523
|
+
files.set("embeddings.jsonl", encoder4.encode(embJsonl));
|
|
29808
31524
|
components.push("embeddings");
|
|
29809
31525
|
counts.embeddings = scopedEmbRows.length;
|
|
29810
31526
|
}
|
|
@@ -29830,7 +31546,7 @@ async function exportShardBytes(db, options, mode) {
|
|
|
29830
31546
|
freshness: jsonObject4(row.freshness_json) ?? { status: "unknown" },
|
|
29831
31547
|
created_at: iso3(row.created_at)
|
|
29832
31548
|
}));
|
|
29833
|
-
files.set("graph_sources.json",
|
|
31549
|
+
files.set("graph_sources.json", encoder4.encode(JSON.stringify(shardGraphSources)));
|
|
29834
31550
|
components.push("graph_sources");
|
|
29835
31551
|
counts.graph_sources = shardGraphSources.length;
|
|
29836
31552
|
}
|
|
@@ -29846,7 +31562,7 @@ async function exportShardBytes(db, options, mode) {
|
|
|
29846
31562
|
rank: row.rank,
|
|
29847
31563
|
metadata: jsonObject4(row.metadata_json)
|
|
29848
31564
|
})).join("\n");
|
|
29849
|
-
files.set("graph_edges.jsonl",
|
|
31565
|
+
files.set("graph_edges.jsonl", encoder4.encode(graphEdgesJsonl));
|
|
29850
31566
|
components.push("graph_edges");
|
|
29851
31567
|
counts.graph_edges = scopedGraphEdgeRows.length;
|
|
29852
31568
|
}
|
|
@@ -29882,7 +31598,7 @@ async function exportShardBytes(db, options, mode) {
|
|
|
29882
31598
|
})),
|
|
29883
31599
|
created_at: iso3(row.created_at)
|
|
29884
31600
|
}));
|
|
29885
|
-
files.set("communities.json",
|
|
31601
|
+
files.set("communities.json", encoder4.encode(JSON.stringify(shardCommunitySets)));
|
|
29886
31602
|
components.push("communities");
|
|
29887
31603
|
counts.community_sets = shardCommunitySets.length;
|
|
29888
31604
|
counts.communities = scopedCommunityRows.length;
|
|
@@ -29898,7 +31614,7 @@ async function exportShardBytes(db, options, mode) {
|
|
|
29898
31614
|
source_type: row.source_type,
|
|
29899
31615
|
metadata: jsonObject4(row.metadata_json)
|
|
29900
31616
|
})).join("\n");
|
|
29901
|
-
files.set("community_assignments.jsonl",
|
|
31617
|
+
files.set("community_assignments.jsonl", encoder4.encode(assignmentsJsonl));
|
|
29902
31618
|
components.push("community_assignments");
|
|
29903
31619
|
counts.community_assignments = scopedAssignmentRows.length;
|
|
29904
31620
|
}
|
|
@@ -29921,7 +31637,7 @@ async function exportShardBytes(db, options, mode) {
|
|
|
29921
31637
|
const coreTags = [...tagsByName.values()].sort(
|
|
29922
31638
|
(left, right) => left.name.localeCompare(right.name)
|
|
29923
31639
|
);
|
|
29924
|
-
files.set("tags.json",
|
|
31640
|
+
files.set("tags.json", encoder4.encode(JSON.stringify(coreTags)));
|
|
29925
31641
|
components.splice(0, components.length, ...CORE_V1_COMPONENTS);
|
|
29926
31642
|
for (const key of Object.keys(counts)) {
|
|
29927
31643
|
if (!CORE_V1_COMPONENTS.includes(key)) delete counts[key];
|
|
@@ -29961,7 +31677,7 @@ async function exportShardBytes(db, options, mode) {
|
|
|
29961
31677
|
...!coreV1 ? { migrated_from: null } : {},
|
|
29962
31678
|
...!coreV1 && layout ? { layout } : {}
|
|
29963
31679
|
};
|
|
29964
|
-
files.set("manifest.json",
|
|
31680
|
+
files.set("manifest.json", encoder4.encode(JSON.stringify(manifest, null, 2)));
|
|
29965
31681
|
if (options?.includeBlobs && options.blobStore) {
|
|
29966
31682
|
const packed = /* @__PURE__ */ new Set();
|
|
29967
31683
|
for (const row of attachmentRows.rows) {
|
|
@@ -34467,7 +36183,7 @@ function getAiwgFortemiIndexExportSchema() {
|
|
|
34467
36183
|
}
|
|
34468
36184
|
function getAjv() {
|
|
34469
36185
|
if (!ajvInstance) {
|
|
34470
|
-
ajvInstance = new
|
|
36186
|
+
ajvInstance = new Ajv20202({
|
|
34471
36187
|
allErrors: true,
|
|
34472
36188
|
strict: false,
|
|
34473
36189
|
validateFormats: false
|
|
@@ -34530,7 +36246,7 @@ function validateAiwgFortemiProjectedRecordSchema(value) {
|
|
|
34530
36246
|
const valid = validate(value);
|
|
34531
36247
|
return { valid, errors: formatErrors2(validate.errors) };
|
|
34532
36248
|
}
|
|
34533
|
-
var
|
|
36249
|
+
var encoder5 = new TextEncoder();
|
|
34534
36250
|
var UUID_NAMESPACE = "7ab5d1f8-29d2-5e35-9e2f-3a45de171a9e";
|
|
34535
36251
|
function uuid(kind, id) {
|
|
34536
36252
|
return v5(`${kind}:${id}`, UUID_NAMESPACE);
|
|
@@ -34546,7 +36262,7 @@ function addLoss(losses, code, message, details = {}) {
|
|
|
34546
36262
|
losses.push({ code, message, ...details });
|
|
34547
36263
|
}
|
|
34548
36264
|
function encode(values, encoding) {
|
|
34549
|
-
return
|
|
36265
|
+
return encoder5.encode(encoding === "json-array" ? JSON.stringify(values) : values.map((value) => JSON.stringify(value)).join("\n"));
|
|
34550
36266
|
}
|
|
34551
36267
|
function noteTitle(record, losses) {
|
|
34552
36268
|
if (own(record, "title")) return record.title ?? null;
|
|
@@ -34714,7 +36430,7 @@ async function convertAiwgIndexToFullV1(index, options = {}) {
|
|
|
34714
36430
|
tags: recordTags,
|
|
34715
36431
|
attachments: []
|
|
34716
36432
|
});
|
|
34717
|
-
const hash = await sha256Hex(
|
|
36433
|
+
const hash = await sha256Hex(encoder5.encode(content));
|
|
34718
36434
|
rows.get("note_originals").push({
|
|
34719
36435
|
id: uuid("note-original", record.id),
|
|
34720
36436
|
note_id: noteId,
|
|
@@ -34806,7 +36522,7 @@ async function convertAiwgIndexToFullV1(index, options = {}) {
|
|
|
34806
36522
|
metric: null,
|
|
34807
36523
|
algorithm: null,
|
|
34808
36524
|
parameters: null,
|
|
34809
|
-
input_hash: `sha256:${await sha256Hex(
|
|
36525
|
+
input_hash: `sha256:${await sha256Hex(encoder5.encode(JSON.stringify(relationshipInput)))}`,
|
|
34810
36526
|
freshness: { status: "fresh", checked_at: exportedAt },
|
|
34811
36527
|
created_at: exportedAt
|
|
34812
36528
|
});
|
|
@@ -35264,7 +36980,7 @@ async function convertAiwgIndexToFullV1(index, options = {}) {
|
|
|
35264
36980
|
checksums,
|
|
35265
36981
|
min_reader_version: "2.0.0"
|
|
35266
36982
|
};
|
|
35267
|
-
const manifestBytes =
|
|
36983
|
+
const manifestBytes = encoder5.encode(JSON.stringify(manifest, null, 2));
|
|
35268
36984
|
files.set("manifest.json", manifestBytes);
|
|
35269
36985
|
const validation = await validateFullV1ShardArchive(files);
|
|
35270
36986
|
if (!validation.valid) {
|
|
@@ -36499,7 +38215,7 @@ function createRecordBackend(store, options = {}) {
|
|
|
36499
38215
|
}
|
|
36500
38216
|
|
|
36501
38217
|
// src/records/record-shard.ts
|
|
36502
|
-
var
|
|
38218
|
+
var encoder6 = new TextEncoder();
|
|
36503
38219
|
var decoder8 = new TextDecoder();
|
|
36504
38220
|
function emptyCounts3() {
|
|
36505
38221
|
return {
|
|
@@ -36757,11 +38473,11 @@ async function buildRecordShardArchive(store, options, profile) {
|
|
|
36757
38473
|
const slice = shardNotes.slice(offset, offset + clusterSize);
|
|
36758
38474
|
const href = `notes/${String(offset).padStart(6, "0")}.jsonl`;
|
|
36759
38475
|
clusters.push({ href, offset });
|
|
36760
|
-
files.set(href,
|
|
38476
|
+
files.set(href, encoder6.encode(slice.map((n) => JSON.stringify(n)).join("\n")));
|
|
36761
38477
|
}
|
|
36762
38478
|
layout = { clusters: { notes: clusters } };
|
|
36763
38479
|
} else {
|
|
36764
|
-
files.set("notes.jsonl",
|
|
38480
|
+
files.set("notes.jsonl", encoder6.encode(shardNotes.map((n) => JSON.stringify(n)).join("\n")));
|
|
36765
38481
|
}
|
|
36766
38482
|
components.push("notes");
|
|
36767
38483
|
counts.notes = shardNotes.length;
|
|
@@ -36789,7 +38505,7 @@ async function buildRecordShardArchive(store, options, profile) {
|
|
|
36789
38505
|
note_count: mapped.note_count ?? 0
|
|
36790
38506
|
};
|
|
36791
38507
|
});
|
|
36792
|
-
files.set("collections.json",
|
|
38508
|
+
files.set("collections.json", encoder6.encode(JSON.stringify(shardCollections)));
|
|
36793
38509
|
components.push("collections");
|
|
36794
38510
|
counts.collections = shardCollections.length;
|
|
36795
38511
|
const distinctTags = [...new Set(
|
|
@@ -36806,7 +38522,7 @@ async function buildRecordShardArchive(store, options, profile) {
|
|
|
36806
38522
|
created_at: tagCreatedAt.get(name) ?? (/* @__PURE__ */ new Date(0)).toISOString()
|
|
36807
38523
|
}))
|
|
36808
38524
|
);
|
|
36809
|
-
files.set("tags.json",
|
|
38525
|
+
files.set("tags.json", encoder6.encode(JSON.stringify(shardTags)));
|
|
36810
38526
|
components.push("tags");
|
|
36811
38527
|
counts.tags = shardTags.length;
|
|
36812
38528
|
const links = await store.list("link");
|
|
@@ -36828,7 +38544,7 @@ async function buildRecordShardArchive(store, options, profile) {
|
|
|
36828
38544
|
}
|
|
36829
38545
|
return shard;
|
|
36830
38546
|
});
|
|
36831
|
-
files.set("links.jsonl",
|
|
38547
|
+
files.set("links.jsonl", encoder6.encode(shardLinks.map((l) => JSON.stringify(l)).join("\n")));
|
|
36832
38548
|
components.push("links");
|
|
36833
38549
|
counts.links = shardLinks.length;
|
|
36834
38550
|
const checksums = {};
|
|
@@ -36958,7 +38674,7 @@ async function buildRecordShardArchive(store, options, profile) {
|
|
|
36958
38674
|
);
|
|
36959
38675
|
}
|
|
36960
38676
|
}
|
|
36961
|
-
files.set("manifest.json",
|
|
38677
|
+
files.set("manifest.json", encoder6.encode(JSON.stringify(manifest, null, 2)));
|
|
36962
38678
|
if (options?.includeBlobs && options.blobStore) {
|
|
36963
38679
|
const packed = /* @__PURE__ */ new Set();
|
|
36964
38680
|
for (const checksum of exportedBlobChecksums) {
|
|
@@ -37344,7 +39060,7 @@ async function importShardToRecords(store, data, options) {
|
|
|
37344
39060
|
id: existingOriginal?.id ?? generateId(),
|
|
37345
39061
|
note_id: note.id,
|
|
37346
39062
|
content: note.original_content,
|
|
37347
|
-
content_hash: computeHash(
|
|
39063
|
+
content_hash: computeHash(encoder6.encode(note.original_content)),
|
|
37348
39064
|
created_at: createdAt2
|
|
37349
39065
|
};
|
|
37350
39066
|
mutations.push({ op: "put", collection: "note_original", record: originalRecord });
|
|
@@ -37870,8 +39586,8 @@ async function purgeRecordStoreGraph(store, selector, operationKey) {
|
|
|
37870
39586
|
}
|
|
37871
39587
|
|
|
37872
39588
|
// src/index.ts
|
|
37873
|
-
var VERSION = "2026.
|
|
39589
|
+
var VERSION = "2026.9.0";
|
|
37874
39590
|
|
|
37875
|
-
export { AIWG_SCAN_REQUIRED_FIELDS, AllowlistTrustStore, ArchiveManager, AttachmentsRepository, CORE_V1_COMPONENTS, CURRENT_MIGRATION_HEAD, CURRENT_SHARD_VERSION, CanonicalAttachmentsRepository, CanonicalNotesRepository, CapabilityManager, CaptureKnowledgeInputSchema, CollectionsRepository, CommunitiesRepository, DB_SNAPSHOT_SCHEMA_VERSION, DbSnapshotVersionError, EMBED_REQUEST_KIND, EMBED_RESPONSE_KIND, EmbeddingSetsRepository, FORTEMI_COMPATIBILITY_PATH, FORTEMI_COMPATIBILITY_STATES, FORTEMI_REQUIRED_COMPATIBILITY_CAPABILITIES, FORTEMI_SERVER_COMPATIBILITY_REVISION, FallbackRouter, FortemiToolManifest, GetNoteInputSchema, GraphRepository, IdbRecordStore, JOB_CAPABILITIES, JOB_PRIORITIES, JobQueueWorker, LOCAL_ENDPOINTS, LifecyclePurgeRepository, LinksRepository, ListNotesInputSchema, ManageArchiveInputSchema, ManageAttachmentsInputSchema, ManageCapabilitiesInputSchema, ManageCollectionsInputSchema, ManageLinksInputSchema, ManageNoteInputSchema, ManageTagsInputSchema, MemoryBlobStore, MemoryRecordStore, MigrationRunner, NotesRepository, OpenAICompatibleProvider, PGliteStorageBackend, PGliteStorageBackendFactory, PGliteWorkerClient, PGliteWorkerStorageBackend, PGliteWorkerStorageBackendFactory, ProvenanceRepository, ProviderRegistry, RECORD_COLLECTIONS, RECORD_SCHEMA_VERSION, RECORD_STORE_CAPABILITIES, REGISTERED_METADATA_PATHS, SHARD_FORMAT, SIGNATURE_ENTRY, SIGNING_ENVELOPE_VERSION, SUPPORTED_PGLITE_VERSION, SearchInputSchema, SearchRepository, SkosRepository, SourceUpsertRepository, TagsRepository, TransactionProxy, TypedEventBus, VERSION, aiRevisionHandler, aiwgFortemiIndexFromKnowledgeShard, aiwgFortemiIndexToCommunityGraph, aiwgFortemiIndexToKnowledgeShard, aiwgFortemiIndexToKnowledgeShardWithReport, allMigrations, appendPluginScript, assertAiwgFortemiChunkManifest, assertAiwgFortemiChunkPart, assertAiwgFortemiIndexExport, assertAiwgStaticEmbeddingSet, assertShardComponentRecord, buildAiwgChunkedIndex, buildAiwgStaticEmbeddingSet, buildMetadataPredicateConditions, buildNoteConditions, buildPluginCsp, captureKnowledge, chunkText, classifyError, classifyModel, clearPrefetchedShard, collectionFromShard, collectionToShard, computeBlobHash, computeHash, computeSri, conceptTaggingHandler, cosineSimilarity, createAiwgFetchChunkLoader, createAiwgFetchDetailLoader, createAiwgIndexController, createAiwgReviewDecisionExport, createBlobStore, createCosineSemanticProvider, createCspReportHandler, createFortemi, createLazyBlobStore, createLegacyProvider, createPGliteBackend, createPGliteInstance, createRecordBackend, createRecordStore, createRemoteBackend, createRoutes, createShardBackend, createShardCapabilityReport, createWorkerEmbedFunction, defaultStorageBackendFactory, detectCommunities, detectGpuCapabilities, detectInferenceCapabilities, discoverLocalProviders, dropAttachmentProjection, dropNoteProjection, dumpDbSnapshot, embeddingConfigToShard, embeddingFromShard, embeddingGenerationHandler, embeddingSetFromShard, embeddingSetMemberToShard, embeddingSetToShard, embeddingToShard, enqueueFullWorkflow, enqueueJob, enqueueNoteCreationJobs, estimateModelFit, estimateVramMB, estimateVramTier, exportShard, exportShardFromRecords, exportShardFromRecordsWithReport, exportShardWithReport, fetchAndValidateFortemiCompatibility, fetchPluginScript, filterAiwgRecordsByPrivacy, findAiwgStaticDuplicatePairs, formatFortemiCompatibilitySummary, fortemiCompatibilityUrl, fortemiManifest, fromPrefetched, generateId, getAiwgFortemiFacets, getAiwgFortemiIndexExportSchema, getEmbedFunction, getFortemiBridge, getFortemiSecretStore, getJobQueueStatus, getKnowledgeShardContractReceipt, getKnowledgeShardProfileRegistry, getKnowledgeShardSchema, getLlmFunction, getNote, getPrefetchedSha256, handleEmbedRequests, hasFortemiSecureSecrets, importShard, importShardToRecords, isPluginScriptAllowed, isShardPrefetched, isShardSigningSupported, linkFromShard, linkToShard, linkingHandler, listNotes, manageArchive, manageAttachments, manageCapabilities, manageCollections, manageLinks, manageNote, manageTags, matchRoute, migrateLegacyBlobStore, noteFromShard, noteSkosTagToShard, noteToShard, openShard, packTarGz, parseCspReport, prefetchShard, previewRecordStorePurge, profileSupportError, projectAttachments, projectNotes, projectRecords, provenanceEdgeToShard, purgeRecordStoreGraph, queryAiwgFortemiIndex, queryAiwgHybridIndex, queryAiwgSemanticIndex, registerLlmCapability, registerSemanticCapability, registerSemanticCapabilityWorker, registerServiceWorker, restoreDbSnapshot, searchTool, selectBackend, selectLlmModel, setEmbedFunction, setLlmFunction, sha256Hex, sidecarBlobDigests, signShard, skosConceptToShard, skosRelationToShard, skosSchemeToShard, suggestTags, tagsToShard, templateToShard, titleGenerationHandler, unpackTarGz, unregisterLlmCapability, unregisterSemanticCapability, upsertRecordStoreSources, urlLinkToShard, validateAiwgFortemiChunkManifest, validateAiwgFortemiChunkPart, validateAiwgFortemiIndexExport, validateAiwgFortemiIndexExportSchema, validateAiwgFortemiProjectedRecordSchema, validateAiwgStaticEmbeddingSet, validateChecksums, validateCoreV1ShardArchive, validateFortemiCompatibilityResponse, validateFullV1ShardArchive, validateRecordV1ShardArchive, validateShardArchive, validateShardComponentRecord, validateShardManifest, verifyDbSnapshotMeta, verifyShardSignature, verifySri };
|
|
39591
|
+
export { AIWG_SCAN_REQUIRED_FIELDS, AllowlistTrustStore, ArchiveManager, AttachmentsRepository, BridgeInferenceProvider, CORE_V1_COMPONENTS, CURRENT_MIGRATION_HEAD, CURRENT_SHARD_VERSION, CanonicalAttachmentsRepository, CanonicalNotesRepository, CapabilityManager, CaptureKnowledgeInputSchema, CollectionsRepository, CommunitiesRepository, DATASET_EXECUTION_CAPABILITY_IDS, DATASET_EXECUTION_CONTRACT, DATASET_EXECUTION_SCHEMA_VERSION, DATASET_INGEST_CONTRACT, DATASET_INGEST_SCHEMA_VERSION, DATASET_LINEAGE_CONTRACT, DATASET_LINEAGE_SCHEMA_VERSION, DATASET_MATERIALIZATION_CONTRACT, DATASET_MATERIALIZATION_KINDS, DATASET_MATERIALIZATION_SCHEMA_VERSION, DB_SNAPSHOT_SCHEMA_VERSION, DEFAULT_LARGE_DOCUMENT_CHARS, DEFAULT_LARGE_DOCUMENT_CHUNKS, DatasetIngestError, DatasetIngestExecutor, DatasetLineageLedger, DatasetMaterializationError, DbSnapshotVersionError, EMBED_REQUEST_KIND, EMBED_RESPONSE_KIND, EmbeddingSetsRepository, FORTEMI_BROWSER_LOCAL_DATASET_EXECUTION_DESCRIPTOR, FORTEMI_COMPATIBILITY_PATH, FORTEMI_COMPATIBILITY_STATES, FORTEMI_PORTABLE_SHARD_DATASET_EXECUTION_DESCRIPTOR, FORTEMI_REQUIRED_COMPATIBILITY_CAPABILITIES, FORTEMI_SERVER_COMPATIBILITY_REVISION, FORTEMI_STATIC_CACHE_DATASET_EXECUTION_DESCRIPTOR, FallbackRouter, FortemiToolManifest, GetNoteInputSchema, GraphRepository, IdbRecordStore, JOB_CAPABILITIES, JOB_PRIORITIES, JobQueueWorker, LINEAGE_ENTITY_KINDS, LINEAGE_RELATIONSHIP_KINDS, LOCAL_ENDPOINTS, LifecyclePurgeRepository, LineageValidationError, LinksRepository, ListNotesInputSchema, ManageArchiveInputSchema, ManageAttachmentsInputSchema, ManageCapabilitiesInputSchema, ManageCollectionsInputSchema, ManageLinksInputSchema, ManageNoteInputSchema, ManageTagsInputSchema, MemoryBlobStore, MemoryDatasetIngestStore, MemoryRecordStore, MigrationRunner, NotesRepository, OpenAICompatibleProvider, PGliteStorageBackend, PGliteStorageBackendFactory, PGliteWorkerClient, PGliteWorkerStorageBackend, PGliteWorkerStorageBackendFactory, ProvenanceRepository, ProviderRegistry, RECORD_COLLECTIONS, RECORD_SCHEMA_VERSION, RECORD_STORE_CAPABILITIES, REGISTERED_METADATA_PATHS, SHARD_FORMAT, SIGNATURE_ENTRY, SIGNING_ENVELOPE_VERSION, SUPPORTED_PGLITE_VERSION, SearchInputSchema, SearchRepository, SkosRepository, SourceUpsertRepository, TagsRepository, TransactionProxy, TypedEventBus, VERSION, aiRevisionHandler, aiwgFortemiIndexFromKnowledgeShard, aiwgFortemiIndexToCommunityGraph, aiwgFortemiIndexToKnowledgeShard, aiwgFortemiIndexToKnowledgeShardWithReport, allMigrations, appendPluginScript, assertAiwgFortemiChunkManifest, assertAiwgFortemiChunkPart, assertAiwgFortemiIndexExport, assertAiwgStaticEmbeddingSet, assertShardComponentRecord, buildAiwgChunkedIndex, buildAiwgStaticEmbeddingSet, buildMetadataPredicateConditions, buildNoteConditions, buildPluginCsp, captureKnowledge, chunkText, classifyError, classifyModel, clearPrefetchedShard, collectionFromShard, collectionToShard, compareDatasetIncrementalParity, computeBlobHash, computeHash, computeLineageDigest, computeSri, conceptTaggingHandler, configureInferenceRuntime, cosineSimilarity, createAiwgFetchChunkLoader, createAiwgFetchDetailLoader, createAiwgIndexController, createAiwgReviewDecisionExport, createBlobStore, createBridgeInferenceProviders, createCosineSemanticProvider, createCspReportHandler, createFortemi, createLazyBlobStore, createLegacyProvider, createLocalProviderProfile, createPGliteBackend, createPGliteInstance, createRecordBackend, createRecordStore, createRemoteBackend, createRoutes, createShardBackend, createShardCapabilityReport, createWorkerEmbedFunction, datasetDestinationScopeKey, defaultStorageBackendFactory, defineInferenceProvider, defineInferenceRuntime, defineLegacyInferenceProvider, defineOpenAICompatibleProvider, deriveDatasetIngestIdempotencyKey, detectCommunities, detectGpuCapabilities, detectInferenceCapabilities, digestDatasetMaterializationValue, discoverLocalProviders, dropAttachmentProjection, dropNoteProjection, dumpDbSnapshot, embeddingConfigToShard, embeddingFromShard, embeddingGenerationHandler, embeddingSetFromShard, embeddingSetMemberToShard, embeddingSetToShard, embeddingToShard, enqueueFullWorkflow, enqueueJob, enqueueNoteCreationJobs, estimateModelFit, estimateVramMB, estimateVramTier, executeDatasetMaterialization, executeDatasetRetrieval, exportShard, exportShardFromRecords, exportShardFromRecordsWithReport, exportShardWithReport, fetchAndValidateFortemiCompatibility, fetchPluginScript, filterAiwgRecordsByPrivacy, findAiwgStaticDuplicatePairs, formatFortemiCompatibilitySummary, fortemiCompatibilityUrl, fortemiManifest, fromPrefetched, generateId, getAiwgFortemiFacets, getAiwgFortemiIndexExportSchema, getConfiguredInferenceProviderId, getEmbedFunction, getEmbeddingTaskSelectionOptions, getFortemiBridge, getFortemiSecretStore, getJobQueueStatus, getKnowledgeShardContractReceipt, getKnowledgeShardProfileRegistry, getKnowledgeShardSchema, getLlmFunction, getNote, getPrefetchedSha256, getProviderRouteRequirementIssue, handleEmbedRequests, hasFortemiSecureSecrets, importShard, importShardToRecords, inferInferenceTaskCapability, inferLocalEmbeddingDimensions, isPluginScriptAllowed, isShardPrefetched, isShardSigningSupported, linkFromShard, linkToShard, linkingHandler, listNotes, manageArchive, manageAttachments, manageCapabilities, manageCollections, manageLinks, manageNote, manageTags, matchRoute, mergeInferenceRuntimeConfigs, migrateLegacyBlobStore, negotiateDatasetExecutionCapabilities, negotiateDatasetMaterializationProfile, noteFromShard, noteSkosTagToShard, noteToShard, openShard, packTarGz, parseCspReport, prefetchShard, previewRecordStorePurge, profileSupportError, projectAttachments, projectNotes, projectRecords, provenanceEdgeToShard, providerSatisfiesRouteRequirements, purgeRecordStoreGraph, queryAiwgFortemiIndex, queryAiwgHybridIndex, queryAiwgSemanticIndex, registerLlmCapability, registerSemanticCapability, registerSemanticCapabilityWorker, registerServiceWorker, restoreDbSnapshot, searchTool, selectBackend, selectEmbeddingTask, selectLlmModel, setEmbedFunction, setEmbeddingTaskSelectionOptions, setLlmFunction, sha256Hex, sidecarBlobDigests, signShard, skosConceptToShard, skosRelationToShard, skosSchemeToShard, suggestTags, tagsToShard, templateToShard, titleGenerationHandler, unpackTarGz, unregisterLlmCapability, unregisterSemanticCapability, upsertRecordStoreSources, urlLinkToShard, validateAiwgFortemiChunkManifest, validateAiwgFortemiChunkPart, validateAiwgFortemiIndexExport, validateAiwgFortemiIndexExportSchema, validateAiwgFortemiProjectedRecordSchema, validateAiwgStaticEmbeddingSet, validateChecksums, validateCoreV1ShardArchive, validateDatasetBenchmarkEvidence, validateDatasetExecutionDescriptor, validateFortemiCompatibilityResponse, validateFullV1ShardArchive, validateProviderRoute, validateRecordV1ShardArchive, validateShardArchive, validateShardComponentRecord, validateShardManifest, verifyDbSnapshotMeta, verifyShardSignature, verifySri };
|
|
37876
39592
|
//# sourceMappingURL=index.js.map
|
|
37877
39593
|
//# sourceMappingURL=index.js.map
|