@fortemi/core 2026.8.0 → 2026.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -1
- package/benchmarks/dataset-materialization/small-corpus.v1.json +12 -0
- package/dist/index.d.ts +1255 -134
- package/dist/index.js +2734 -763
- package/dist/index.js.map +1 -1
- package/package.json +13 -1
- package/schemas/dataset-execution-capabilities/fixtures/browser-local.json +14 -0
- package/schemas/dataset-execution-capabilities/fixtures/invalid-field-lineage-without-evidence.json +10 -0
- package/schemas/dataset-execution-capabilities/fixtures/invalid-incremental-without-checkpoint.json +10 -0
- package/schemas/dataset-execution-capabilities/fixtures/portable-shard.json +12 -0
- package/schemas/dataset-execution-capabilities/fixtures/remote-alpha.json +12 -0
- package/schemas/dataset-execution-capabilities/fixtures/static-cache.json +12 -0
- package/schemas/dataset-execution-capabilities/v1.schema.json +164 -0
- package/schemas/dataset-ingest/v1.schema.json +138 -0
- package/schemas/dataset-lineage/fixtures/golden-observed-field.json +20 -0
- package/schemas/dataset-lineage/v1.schema.json +126 -0
- package/schemas/dataset-materialization/fixtures/browser.json +11 -0
- package/schemas/dataset-materialization/fixtures/degraded.json +12 -0
- package/schemas/dataset-materialization/fixtures/deterministic.json +11 -0
- package/schemas/dataset-materialization/fixtures/external-adapter.json +11 -0
- package/schemas/dataset-materialization/fixtures/nondeterministic.json +11 -0
- package/schemas/dataset-materialization/fixtures/server.json +11 -0
- package/schemas/dataset-materialization/fixtures/supported.json +12 -0
- package/schemas/dataset-materialization/fixtures/unsupported.json +11 -0
- package/schemas/dataset-materialization/v1.schema.json +169 -0
- package/schemas/source-note-upsert/contract.receipt.json +78 -0
- package/schemas/source-note-upsert/v1.conformance.json +100 -0
package/dist/index.js
CHANGED
|
@@ -2,14 +2,1081 @@ import { v7, v5 } from 'uuid';
|
|
|
2
2
|
import { sha256 } from '@noble/hashes/sha256';
|
|
3
3
|
import { blake3 } from '@noble/hashes/blake3';
|
|
4
4
|
import { bytesToHex } from '@noble/hashes/utils';
|
|
5
|
+
import Ajv20202 from 'ajv/dist/2020.js';
|
|
5
6
|
import { z } from 'zod';
|
|
6
7
|
import { gzipSync, gunzipSync } from 'fflate';
|
|
7
|
-
import Ajv2020 from 'ajv/dist/2020.js';
|
|
8
8
|
|
|
9
9
|
// src/uuid.ts
|
|
10
10
|
function generateId() {
|
|
11
11
|
return v7();
|
|
12
12
|
}
|
|
13
|
+
function computeHash(data) {
|
|
14
|
+
const digest2 = sha256(data);
|
|
15
|
+
return `sha256:${bytesToHex(digest2)}`;
|
|
16
|
+
}
|
|
17
|
+
function computeBlobHash(data) {
|
|
18
|
+
return `blake3:${bytesToHex(blake3(data))}`;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
// src/dataset-ingest.ts
|
|
22
|
+
var DATASET_INGEST_CONTRACT = "fortemi.dataset-ingest/v1";
|
|
23
|
+
var DATASET_INGEST_SCHEMA_VERSION = "1.0.0";
|
|
24
|
+
var DatasetIngestError = class extends Error {
|
|
25
|
+
constructor(code, message) {
|
|
26
|
+
super(message);
|
|
27
|
+
this.code = code;
|
|
28
|
+
this.name = "DatasetIngestError";
|
|
29
|
+
}
|
|
30
|
+
};
|
|
31
|
+
function clone(value) {
|
|
32
|
+
return structuredClone(value);
|
|
33
|
+
}
|
|
34
|
+
var MemoryDatasetIngestStore = class {
|
|
35
|
+
scopes = /* @__PURE__ */ new Map();
|
|
36
|
+
queues = /* @__PURE__ */ new Map();
|
|
37
|
+
async transact(scopeKey, operation) {
|
|
38
|
+
const predecessor = this.queues.get(scopeKey) ?? Promise.resolve();
|
|
39
|
+
let release;
|
|
40
|
+
const current = new Promise((resolve) => {
|
|
41
|
+
release = resolve;
|
|
42
|
+
});
|
|
43
|
+
const queued = predecessor.then(() => current);
|
|
44
|
+
this.queues.set(scopeKey, queued);
|
|
45
|
+
await predecessor;
|
|
46
|
+
try {
|
|
47
|
+
const existing = this.scopes.get(scopeKey) ?? {
|
|
48
|
+
records: /* @__PURE__ */ new Map(),
|
|
49
|
+
receipts: /* @__PURE__ */ new Map()
|
|
50
|
+
};
|
|
51
|
+
const draft = {
|
|
52
|
+
records: new Map([...existing.records].map(([key, value]) => [key, clone(value)])),
|
|
53
|
+
receipts: new Map([...existing.receipts].map(([key, value]) => [key, clone(value)])),
|
|
54
|
+
...existing.checkpoint ? { checkpoint: clone(existing.checkpoint) } : {}
|
|
55
|
+
};
|
|
56
|
+
const transaction = {
|
|
57
|
+
getRecord: (id) => draft.records.get(id),
|
|
58
|
+
setRecord: (record) => draft.records.set(record.logicalId, clone(record)),
|
|
59
|
+
getReceipt: (key) => draft.receipts.get(key),
|
|
60
|
+
setReceipt: (receipt) => draft.receipts.set(receipt.idempotencyKey, clone(receipt)),
|
|
61
|
+
getCheckpoint: () => draft.checkpoint,
|
|
62
|
+
setCheckpoint: (checkpoint) => {
|
|
63
|
+
draft.checkpoint = clone(checkpoint);
|
|
64
|
+
}
|
|
65
|
+
};
|
|
66
|
+
const result = await operation(transaction);
|
|
67
|
+
this.scopes.set(scopeKey, draft);
|
|
68
|
+
return result;
|
|
69
|
+
} finally {
|
|
70
|
+
release();
|
|
71
|
+
if (this.queues.get(scopeKey) === queued) this.queues.delete(scopeKey);
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
async getReceipt(scopeKey, idempotencyKey) {
|
|
75
|
+
return clone(this.scopes.get(scopeKey)?.receipts.get(idempotencyKey));
|
|
76
|
+
}
|
|
77
|
+
async getCheckpoint(scopeKey) {
|
|
78
|
+
return clone(this.scopes.get(scopeKey)?.checkpoint);
|
|
79
|
+
}
|
|
80
|
+
async getRecords(scopeKey) {
|
|
81
|
+
return [...this.scopes.get(scopeKey)?.records.values() ?? []].map(clone).sort((a, b) => a.logicalId.localeCompare(b.logicalId));
|
|
82
|
+
}
|
|
83
|
+
};
|
|
84
|
+
function canonicalJson(value) {
|
|
85
|
+
if (value === null || typeof value !== "object") return JSON.stringify(value);
|
|
86
|
+
if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`;
|
|
87
|
+
return `{${Object.entries(value).sort(([a], [b]) => a.localeCompare(b)).map(([key, item]) => `${JSON.stringify(key)}:${canonicalJson(item)}`).join(",")}}`;
|
|
88
|
+
}
|
|
89
|
+
function digest(value) {
|
|
90
|
+
return computeHash(new TextEncoder().encode(canonicalJson(value)));
|
|
91
|
+
}
|
|
92
|
+
function datasetDestinationScopeKey(scope) {
|
|
93
|
+
return [scope.tenant, scope.dataset, scope.sourceBinding, scope.stream, scope.partition ?? ""].map(encodeURIComponent).join("/");
|
|
94
|
+
}
|
|
95
|
+
function deriveDatasetIngestIdempotencyKey(plan, batch) {
|
|
96
|
+
if (batch.idempotencyKey) return batch.idempotencyKey;
|
|
97
|
+
return digest({
|
|
98
|
+
contract: DATASET_INGEST_CONTRACT,
|
|
99
|
+
planId: plan.planId,
|
|
100
|
+
planDigest: plan.planDigest,
|
|
101
|
+
sourceRevision: plan.sourceRevision,
|
|
102
|
+
configurationDigest: plan.configurationDigest,
|
|
103
|
+
transformationDigest: plan.transformationDigest,
|
|
104
|
+
destination: plan.destination,
|
|
105
|
+
mode: plan.mode,
|
|
106
|
+
sequence: batch.sequence,
|
|
107
|
+
checkpointBefore: batch.checkpointBefore,
|
|
108
|
+
checkpointAfter: batch.checkpointAfter,
|
|
109
|
+
mutations: batch.mutations
|
|
110
|
+
});
|
|
111
|
+
}
|
|
112
|
+
function assertContract(plan, batch) {
|
|
113
|
+
if (plan.contract !== DATASET_INGEST_CONTRACT || batch.contract !== DATASET_INGEST_CONTRACT) {
|
|
114
|
+
throw new DatasetIngestError("INGEST_CONTRACT_UNSUPPORTED", "Unsupported dataset ingest contract");
|
|
115
|
+
}
|
|
116
|
+
if (!plan.schemaVersion.startsWith("1.") || !batch.schemaVersion.startsWith("1.")) {
|
|
117
|
+
throw new DatasetIngestError("INGEST_SCHEMA_UNSUPPORTED", "Unsupported dataset ingest schema version");
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
function assertCheckpointScope(checkpoint, scope) {
|
|
121
|
+
if (checkpoint.contract !== DATASET_INGEST_CONTRACT) throw new DatasetIngestError("INGEST_CONTRACT_UNSUPPORTED", "Unsupported checkpoint contract");
|
|
122
|
+
if (!checkpoint.schemaVersion.startsWith("1.")) throw new DatasetIngestError("CHECKPOINT_VERSION_UNSUPPORTED", `Unsupported checkpoint schema ${checkpoint.schemaVersion}`);
|
|
123
|
+
if (datasetDestinationScopeKey(checkpoint.scope) !== datasetDestinationScopeKey(scope)) {
|
|
124
|
+
throw new DatasetIngestError("CHECKPOINT_SCOPE_MISMATCH", "Checkpoint belongs to another destination scope");
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
var DatasetIngestExecutor = class {
|
|
128
|
+
constructor(store) {
|
|
129
|
+
this.store = store;
|
|
130
|
+
}
|
|
131
|
+
attempts = /* @__PURE__ */ new Map();
|
|
132
|
+
successes = /* @__PURE__ */ new Map();
|
|
133
|
+
preview(plan, batch) {
|
|
134
|
+
assertContract(plan, batch);
|
|
135
|
+
return {
|
|
136
|
+
idempotencyKey: deriveDatasetIngestIdempotencyKey(plan, batch),
|
|
137
|
+
upserts: batch.mutations.filter((item) => item.operation === "upsert").length,
|
|
138
|
+
tombstones: batch.mutations.filter((item) => item.operation === "tombstone").length
|
|
139
|
+
};
|
|
140
|
+
}
|
|
141
|
+
connectionCheck(plan, batch) {
|
|
142
|
+
assertContract(plan, batch);
|
|
143
|
+
assertCheckpointScope(batch.checkpointAfter, plan.destination);
|
|
144
|
+
if (batch.checkpointBefore) assertCheckpointScope(batch.checkpointBefore, plan.destination);
|
|
145
|
+
return {
|
|
146
|
+
idempotencyKey: deriveDatasetIngestIdempotencyKey(plan, batch),
|
|
147
|
+
scopeKey: datasetDestinationScopeKey(plan.destination),
|
|
148
|
+
compatible: true
|
|
149
|
+
};
|
|
150
|
+
}
|
|
151
|
+
async executeBatch(plan, batch, options = {}) {
|
|
152
|
+
assertContract(plan, batch);
|
|
153
|
+
const scopeKey = datasetDestinationScopeKey(plan.destination);
|
|
154
|
+
const idempotencyKey = deriveDatasetIngestIdempotencyKey(plan, batch);
|
|
155
|
+
const requestDigest = digest({ plan, batch });
|
|
156
|
+
const runId = generateId();
|
|
157
|
+
this.attempts.set(scopeKey, { runId, state: "running", verification: "pending", idempotencyKey });
|
|
158
|
+
const cancel = () => {
|
|
159
|
+
if (options.signal?.aborted) throw new DatasetIngestError("INGEST_CANCELLED", "Dataset ingest was cancelled before commit");
|
|
160
|
+
};
|
|
161
|
+
try {
|
|
162
|
+
cancel();
|
|
163
|
+
const receipt = await this.store.transact(scopeKey, async (transaction) => {
|
|
164
|
+
const prior = transaction.getReceipt(idempotencyKey);
|
|
165
|
+
if (prior) {
|
|
166
|
+
if (prior.requestDigest !== requestDigest) throw new DatasetIngestError("IDEMPOTENCY_CONFLICT", "Idempotency key was previously used with different canonical content");
|
|
167
|
+
return prior;
|
|
168
|
+
}
|
|
169
|
+
const currentCheckpoint = transaction.getCheckpoint();
|
|
170
|
+
if (batch.checkpointBefore) {
|
|
171
|
+
assertCheckpointScope(batch.checkpointBefore, plan.destination);
|
|
172
|
+
if (!currentCheckpoint || canonicalJson(currentCheckpoint) !== canonicalJson(batch.checkpointBefore)) {
|
|
173
|
+
throw new DatasetIngestError("CHECKPOINT_MISMATCH", "checkpointBefore does not match committed state");
|
|
174
|
+
}
|
|
175
|
+
} else if (currentCheckpoint) {
|
|
176
|
+
throw new DatasetIngestError("CHECKPOINT_MISMATCH", "A committed checkpoint exists but checkpointBefore was omitted");
|
|
177
|
+
}
|
|
178
|
+
assertCheckpointScope(batch.checkpointAfter, plan.destination);
|
|
179
|
+
if (batch.checkpointAfter.sequence !== batch.sequence) throw new DatasetIngestError("BATCH_OUT_OF_ORDER", "Checkpoint sequence must equal batch sequence");
|
|
180
|
+
if (currentCheckpoint && batch.checkpointAfter.sequence <= currentCheckpoint.sequence) throw new DatasetIngestError("CHECKPOINT_REGRESSION", "Checkpoint sequence must advance");
|
|
181
|
+
const expectedSequence = (currentCheckpoint?.sequence ?? 0) + 1;
|
|
182
|
+
if (batch.sequence !== expectedSequence) throw new DatasetIngestError("BATCH_OUT_OF_ORDER", `Expected batch sequence ${expectedSequence}, received ${batch.sequence}`);
|
|
183
|
+
const tombstones = batch.mutations.filter((item) => item.operation === "tombstone").length;
|
|
184
|
+
if (tombstones > 0) {
|
|
185
|
+
if (!plan.reconciliation.enabled) throw new DatasetIngestError("RECONCILIATION_NOT_ENABLED", "Tombstones require reconciliation policy");
|
|
186
|
+
if (!batch.enumeration?.complete) throw new DatasetIngestError("RECONCILIATION_INCOMPLETE", "Tombstones require complete source enumeration");
|
|
187
|
+
if (tombstones > plan.reconciliation.maxTombstones && !batch.enumeration.approvalId) {
|
|
188
|
+
throw new DatasetIngestError("RECONCILIATION_APPROVAL_REQUIRED", `Tombstone count ${tombstones} exceeds approved threshold`);
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
const rejections = [];
|
|
192
|
+
const accepted = [];
|
|
193
|
+
for (const mutation of batch.mutations) {
|
|
194
|
+
cancel();
|
|
195
|
+
const failure2 = options.validateRecord?.(mutation);
|
|
196
|
+
if (!failure2) {
|
|
197
|
+
accepted.push(mutation);
|
|
198
|
+
continue;
|
|
199
|
+
}
|
|
200
|
+
const rejection = {
|
|
201
|
+
logicalIdDigest: digest(mutation.logicalId),
|
|
202
|
+
...mutation.locator ? { locator: mutation.locator } : {},
|
|
203
|
+
code: failure2.code,
|
|
204
|
+
message: "Record rejected by validation policy"
|
|
205
|
+
};
|
|
206
|
+
if (plan.rejectionPolicy.mode === "fail-fast") throw new DatasetIngestError("RECORD_REJECTED", `${failure2.code}: record rejected`);
|
|
207
|
+
rejections.push(rejection);
|
|
208
|
+
if (rejections.length > plan.rejectionPolicy.maxRejectedRecords) throw new DatasetIngestError("REJECTION_LIMIT_EXCEEDED", "Rejected-record limit exceeded");
|
|
209
|
+
}
|
|
210
|
+
cancel();
|
|
211
|
+
for (const mutation of accepted) {
|
|
212
|
+
transaction.setRecord(mutation.operation === "upsert" ? { logicalId: mutation.logicalId, revision: mutation.revision, digest: mutation.digest, value: clone(mutation.value), tombstoned: false } : { logicalId: mutation.logicalId, revision: mutation.revision, digest: mutation.digest, tombstoned: true });
|
|
213
|
+
}
|
|
214
|
+
const effects = accepted.map(({ operation, logicalId, revision, digest: itemDigest }) => ({ operation, logicalId, revision, digest: itemDigest }));
|
|
215
|
+
const receipt2 = {
|
|
216
|
+
contract: DATASET_INGEST_CONTRACT,
|
|
217
|
+
schemaVersion: DATASET_INGEST_SCHEMA_VERSION,
|
|
218
|
+
runId,
|
|
219
|
+
idempotencyKey,
|
|
220
|
+
requestDigest,
|
|
221
|
+
planId: plan.planId,
|
|
222
|
+
planDigest: plan.planDigest,
|
|
223
|
+
sourceRevision: plan.sourceRevision,
|
|
224
|
+
destination: clone(plan.destination),
|
|
225
|
+
mode: plan.mode,
|
|
226
|
+
state: rejections.length ? "degraded" : "committed",
|
|
227
|
+
effects,
|
|
228
|
+
acceptedRecords: accepted.length,
|
|
229
|
+
rejectedRecords: rejections.length,
|
|
230
|
+
rejections,
|
|
231
|
+
outputDigest: digest(effects),
|
|
232
|
+
...batch.checkpointBefore ? { checkpointBefore: clone(batch.checkpointBefore) } : {},
|
|
233
|
+
checkpointAfter: clone(batch.checkpointAfter),
|
|
234
|
+
verification: "verified"
|
|
235
|
+
};
|
|
236
|
+
await options.hooks?.beforeCommit?.();
|
|
237
|
+
cancel();
|
|
238
|
+
transaction.setReceipt(receipt2);
|
|
239
|
+
transaction.setCheckpoint(batch.checkpointAfter);
|
|
240
|
+
return receipt2;
|
|
241
|
+
});
|
|
242
|
+
this.successes.set(scopeKey, clone(receipt));
|
|
243
|
+
this.attempts.set(scopeKey, { runId: receipt.runId, state: receipt.state, verification: "verified", idempotencyKey });
|
|
244
|
+
await options.hooks?.afterCommit?.(clone(receipt));
|
|
245
|
+
return receipt;
|
|
246
|
+
} catch (error) {
|
|
247
|
+
const code = error instanceof DatasetIngestError ? error.code : void 0;
|
|
248
|
+
this.attempts.set(scopeKey, { runId, state: code === "INGEST_CANCELLED" ? "cancelled" : "failed", verification: "failed", idempotencyKey, ...code ? { errorCode: code } : {} });
|
|
249
|
+
throw error;
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
async resolveAmbiguousCommit(plan, batch) {
|
|
253
|
+
return this.store.getReceipt(datasetDestinationScopeKey(plan.destination), deriveDatasetIngestIdempotencyKey(plan, batch));
|
|
254
|
+
}
|
|
255
|
+
status(scope, expectedSourceRevision) {
|
|
256
|
+
const key = datasetDestinationScopeKey(scope);
|
|
257
|
+
const lastSuccessful = this.successes.get(key);
|
|
258
|
+
return {
|
|
259
|
+
scope: clone(scope),
|
|
260
|
+
...this.attempts.get(key) ? { lastAttempt: clone(this.attempts.get(key)) } : {},
|
|
261
|
+
...lastSuccessful ? { lastSuccessful: clone(lastSuccessful) } : {},
|
|
262
|
+
freshness: lastSuccessful ? expectedSourceRevision && expectedSourceRevision !== lastSuccessful.sourceRevision ? "stale" : "current" : "never"
|
|
263
|
+
};
|
|
264
|
+
}
|
|
265
|
+
};
|
|
266
|
+
|
|
267
|
+
// src/dataset-lineage.ts
|
|
268
|
+
var DATASET_LINEAGE_CONTRACT = "fortemi.dataset-lineage/v1";
|
|
269
|
+
var DATASET_LINEAGE_SCHEMA_VERSION = "1.0.0";
|
|
270
|
+
var LINEAGE_ENTITY_KINDS = [
|
|
271
|
+
"dataset",
|
|
272
|
+
"dataset-revision",
|
|
273
|
+
"distribution",
|
|
274
|
+
"record",
|
|
275
|
+
"field",
|
|
276
|
+
"chunk",
|
|
277
|
+
"index",
|
|
278
|
+
"embedding-set",
|
|
279
|
+
"graph-artifact",
|
|
280
|
+
"community-artifact",
|
|
281
|
+
"processing-plan",
|
|
282
|
+
"run"
|
|
283
|
+
];
|
|
284
|
+
var LINEAGE_RELATIONSHIP_KINDS = [
|
|
285
|
+
"derived-from",
|
|
286
|
+
"field-derived-from",
|
|
287
|
+
"extracted-from",
|
|
288
|
+
"chunk-of",
|
|
289
|
+
"indexed-from",
|
|
290
|
+
"embedded-from",
|
|
291
|
+
"graph-derived-from",
|
|
292
|
+
"community-derived-from",
|
|
293
|
+
"revision-of",
|
|
294
|
+
"distributed-as",
|
|
295
|
+
"join-influence",
|
|
296
|
+
"filter-influence",
|
|
297
|
+
"aggregation-influence",
|
|
298
|
+
"ordering-influence",
|
|
299
|
+
"similarity-influence"
|
|
300
|
+
];
|
|
301
|
+
var LineageValidationError = class extends Error {
|
|
302
|
+
constructor(code, message) {
|
|
303
|
+
super(message);
|
|
304
|
+
this.code = code;
|
|
305
|
+
this.name = "LineageValidationError";
|
|
306
|
+
}
|
|
307
|
+
};
|
|
308
|
+
function canonicalJson2(value) {
|
|
309
|
+
if (value === void 0) return "null";
|
|
310
|
+
if (value === null || typeof value !== "object") return JSON.stringify(value);
|
|
311
|
+
if (Array.isArray(value)) return `[${value.map(canonicalJson2).join(",")}]`;
|
|
312
|
+
const entries = Object.entries(value).filter(([, item]) => item !== void 0).sort(([left], [right]) => left.localeCompare(right));
|
|
313
|
+
return `{${entries.map(([key, item]) => `${JSON.stringify(key)}:${canonicalJson2(item)}`).join(",")}}`;
|
|
314
|
+
}
|
|
315
|
+
function computeLineageDigest(value) {
|
|
316
|
+
return computeHash(new TextEncoder().encode(canonicalJson2(value)));
|
|
317
|
+
}
|
|
318
|
+
function clone2(value) {
|
|
319
|
+
return structuredClone(value);
|
|
320
|
+
}
|
|
321
|
+
function requireIdentity(id, label) {
|
|
322
|
+
if (!id || id.trim() !== id) throw new LineageValidationError("IDENTITY_REQUIRED", `${label} must be a non-empty canonical identity`);
|
|
323
|
+
}
|
|
324
|
+
function requireTimestamp(value, label) {
|
|
325
|
+
if (!/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{3})?Z$/.test(value) || !Number.isFinite(Date.parse(value))) {
|
|
326
|
+
throw new LineageValidationError("VALUE_INVALID", `${label} must be an RFC 3339 UTC timestamp`);
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
var ENTITY_KIND_SET = new Set(LINEAGE_ENTITY_KINDS);
|
|
330
|
+
var RELATIONSHIP_KIND_SET = new Set(LINEAGE_RELATIONSHIP_KINDS);
|
|
331
|
+
var RELATIONSHIP_ENDPOINTS = {
|
|
332
|
+
"field-derived-from": [["field"], ["field"]],
|
|
333
|
+
"chunk-of": [["chunk"], ["record", "distribution"]],
|
|
334
|
+
"indexed-from": [["index"], ["dataset", "dataset-revision", "distribution", "record", "field", "chunk"]],
|
|
335
|
+
"embedded-from": [["embedding-set"], ["dataset", "dataset-revision", "record", "field", "chunk"]],
|
|
336
|
+
"graph-derived-from": [["graph-artifact"], ["dataset", "dataset-revision", "index", "embedding-set"]],
|
|
337
|
+
"community-derived-from": [["community-artifact"], ["graph-artifact"]],
|
|
338
|
+
"revision-of": [["dataset-revision"], ["dataset"]],
|
|
339
|
+
"distributed-as": [["dataset", "dataset-revision"], ["distribution"]]
|
|
340
|
+
};
|
|
341
|
+
var DatasetLineageLedger = class _DatasetLineageLedger {
|
|
342
|
+
sequence = 0;
|
|
343
|
+
entities = /* @__PURE__ */ new Map();
|
|
344
|
+
agents = /* @__PURE__ */ new Map();
|
|
345
|
+
activities = /* @__PURE__ */ new Map();
|
|
346
|
+
evidence = /* @__PURE__ */ new Map();
|
|
347
|
+
assertions = /* @__PURE__ */ new Map();
|
|
348
|
+
corrections = /* @__PURE__ */ new Map();
|
|
349
|
+
maximumTraversalDepth;
|
|
350
|
+
maximumTraversalResults;
|
|
351
|
+
maximumPageSize;
|
|
352
|
+
constructor(options = {}) {
|
|
353
|
+
this.maximumTraversalDepth = options.maximumTraversalDepth ?? 16;
|
|
354
|
+
this.maximumTraversalResults = options.maximumTraversalResults ?? 1e4;
|
|
355
|
+
this.maximumPageSize = options.maximumPageSize ?? 500;
|
|
356
|
+
}
|
|
357
|
+
get snapshot() {
|
|
358
|
+
return this.sequence;
|
|
359
|
+
}
|
|
360
|
+
appendEntity(entity) {
|
|
361
|
+
requireIdentity(entity.id, "entity.id");
|
|
362
|
+
if (!ENTITY_KIND_SET.has(entity.kind)) throw new LineageValidationError("VALUE_INVALID", `Unsupported entity kind ${String(entity.kind)}`);
|
|
363
|
+
requireIdentity(entity.schemaId, "entity.schemaId");
|
|
364
|
+
requireIdentity(entity.schemaVersion, "entity.schemaVersion");
|
|
365
|
+
requireTimestamp(entity.createdAt, "entity.createdAt");
|
|
366
|
+
this.ensureUnique(this.entities, entity.id, "entity");
|
|
367
|
+
return this.append(this.entities, entity.id, entity);
|
|
368
|
+
}
|
|
369
|
+
appendAgent(agent) {
|
|
370
|
+
requireIdentity(agent.id, "agent.id");
|
|
371
|
+
requireIdentity(agent.name, "agent.name");
|
|
372
|
+
this.ensureUnique(this.agents, agent.id, "agent");
|
|
373
|
+
return this.append(this.agents, agent.id, agent);
|
|
374
|
+
}
|
|
375
|
+
appendActivity(activity) {
|
|
376
|
+
requireIdentity(activity.id, "activity.id");
|
|
377
|
+
requireTimestamp(activity.startedAt, "activity.startedAt");
|
|
378
|
+
if (activity.endedAt) requireTimestamp(activity.endedAt, "activity.endedAt");
|
|
379
|
+
for (const agentId of activity.agentIds) {
|
|
380
|
+
if (!this.agents.has(agentId)) throw new LineageValidationError("AGENT_DANGLING", `Activity references unknown agent ${agentId}`);
|
|
381
|
+
}
|
|
382
|
+
this.ensureUnique(this.activities, activity.id, "activity");
|
|
383
|
+
return this.append(this.activities, activity.id, activity);
|
|
384
|
+
}
|
|
385
|
+
appendEvidence(item) {
|
|
386
|
+
requireIdentity(item.id, "evidence.id");
|
|
387
|
+
requireIdentity(item.revision, "evidence.revision");
|
|
388
|
+
requireIdentity(item.locator, "evidence.locator");
|
|
389
|
+
requireTimestamp(item.capturedAt, "evidence.capturedAt");
|
|
390
|
+
if (item.payload !== void 0 && computeLineageDigest(item.payload) !== item.digest) {
|
|
391
|
+
throw new LineageValidationError("EVIDENCE_DIGEST_MISMATCH", `Evidence ${item.id} payload does not match ${item.digest}`);
|
|
392
|
+
}
|
|
393
|
+
const key = `${item.id}@${item.revision}`;
|
|
394
|
+
this.ensureUnique(this.evidence, key, "evidence revision");
|
|
395
|
+
return this.append(this.evidence, key, item);
|
|
396
|
+
}
|
|
397
|
+
appendAssertion(assertion) {
|
|
398
|
+
requireIdentity(assertion.id, "assertion.id");
|
|
399
|
+
requireIdentity(assertion.revision, "assertion.revision");
|
|
400
|
+
requireIdentity(assertion.method, "assertion.method");
|
|
401
|
+
requireIdentity(assertion.schemaId, "assertion.schemaId");
|
|
402
|
+
requireIdentity(assertion.schemaVersion, "assertion.schemaVersion");
|
|
403
|
+
requireTimestamp(assertion.assertedAt, "assertion.assertedAt");
|
|
404
|
+
if (!RELATIONSHIP_KIND_SET.has(assertion.relationship)) throw new LineageValidationError("VALUE_INVALID", `Unsupported relationship ${String(assertion.relationship)}`);
|
|
405
|
+
if (!Number.isFinite(assertion.confidence) || assertion.confidence < 0 || assertion.confidence > 1) {
|
|
406
|
+
throw new LineageValidationError("VALUE_INVALID", "assertion.confidence must be between 0 and 1");
|
|
407
|
+
}
|
|
408
|
+
const source = this.entities.get(assertion.sourceEntityId)?.value;
|
|
409
|
+
const target = this.entities.get(assertion.targetEntityId)?.value;
|
|
410
|
+
if (!source || !target) throw new LineageValidationError("IDENTITY_DANGLING", "Assertion source and target must both exist");
|
|
411
|
+
const endpointRule = RELATIONSHIP_ENDPOINTS[assertion.relationship];
|
|
412
|
+
if (endpointRule && (!endpointRule[0].includes(source.kind) || !endpointRule[1].includes(target.kind))) {
|
|
413
|
+
throw new LineageValidationError("TYPE_DIRECTION_INVALID", `${assertion.relationship} does not allow ${source.kind} -> ${target.kind}`);
|
|
414
|
+
}
|
|
415
|
+
if (!this.agents.has(assertion.issuerAgentId)) throw new LineageValidationError("AGENT_DANGLING", `Unknown issuer agent ${assertion.issuerAgentId}`);
|
|
416
|
+
if (assertion.producingActivityId && !this.activities.has(assertion.producingActivityId)) {
|
|
417
|
+
throw new LineageValidationError("IDENTITY_DANGLING", `Unknown producing activity ${assertion.producingActivityId}`);
|
|
418
|
+
}
|
|
419
|
+
if (assertion.assertionKind === "observed" && !assertion.producingActivityId) {
|
|
420
|
+
throw new LineageValidationError("ACTIVITY_REQUIRED", "Observed assertions require a producing activity");
|
|
421
|
+
}
|
|
422
|
+
if (assertion.assertionKind === "observed" && assertion.evidence.length === 0) {
|
|
423
|
+
throw new LineageValidationError("EVIDENCE_DANGLING", "Observed assertions require evidence");
|
|
424
|
+
}
|
|
425
|
+
for (const reference of assertion.evidence) {
|
|
426
|
+
const item = this.evidence.get(`${reference.evidenceId}@${reference.revision}`)?.value;
|
|
427
|
+
if (!item) throw new LineageValidationError("EVIDENCE_DANGLING", `Unknown evidence ${reference.evidenceId}@${reference.revision}`);
|
|
428
|
+
if (item.digest !== reference.digest) throw new LineageValidationError("EVIDENCE_DIGEST_MISMATCH", `Evidence reference digest differs for ${reference.evidenceId}`);
|
|
429
|
+
if (reference.locator && reference.locator !== item.locator) throw new LineageValidationError("EVIDENCE_DIGEST_MISMATCH", `Evidence locator differs for ${reference.evidenceId}`);
|
|
430
|
+
}
|
|
431
|
+
const key = `${assertion.id}@${assertion.revision}`;
|
|
432
|
+
if (this.assertions.has(key)) throw new LineageValidationError("IDENTITY_DUPLICATE", `Duplicate assertion revision ${key}`);
|
|
433
|
+
const prior = [...this.assertions.values()].some((entry) => entry.value.id === assertion.id);
|
|
434
|
+
if (prior && !this.hasReplacementPermission(assertion.id, assertion.revision)) {
|
|
435
|
+
throw new LineageValidationError("CORRECTION_INVALID", `Assertion ${assertion.id} can only receive revision ${assertion.revision} after an explicit correction`);
|
|
436
|
+
}
|
|
437
|
+
return this.append(this.assertions, key, assertion);
|
|
438
|
+
}
|
|
439
|
+
appendCorrection(correction) {
|
|
440
|
+
requireIdentity(correction.id, "correction.id");
|
|
441
|
+
requireIdentity(correction.reason, "correction.reason");
|
|
442
|
+
requireTimestamp(correction.recordedAt, "correction.recordedAt");
|
|
443
|
+
if (!this.assertions.has(`${correction.assertionId}@${correction.assertionRevision}`)) throw new LineageValidationError("IDENTITY_DANGLING", `Unknown assertion ${correction.assertionId}@${correction.assertionRevision}`);
|
|
444
|
+
if (!this.agents.has(correction.issuerAgentId)) throw new LineageValidationError("AGENT_DANGLING", `Unknown correction issuer ${correction.issuerAgentId}`);
|
|
445
|
+
const activity = this.activities.get(correction.activityId)?.value;
|
|
446
|
+
if (!activity || activity.kind !== "correction") throw new LineageValidationError("CORRECTION_INVALID", "Correction requires a correction activity");
|
|
447
|
+
if (correction.action === "retract" && (correction.replacementAssertionId || correction.replacementRevision)) throw new LineageValidationError("CORRECTION_INVALID", "Retraction cannot name a replacement");
|
|
448
|
+
if (correction.action !== "retract" && (!correction.replacementAssertionId || !correction.replacementRevision)) throw new LineageValidationError("CORRECTION_INVALID", `${correction.action} requires a replacement assertion identity and revision`);
|
|
449
|
+
if (correction.replacementAssertionId === correction.assertionId && correction.action === "supersede") {
|
|
450
|
+
throw new LineageValidationError("CORRECTION_INVALID", "Supersession must identify a different assertion");
|
|
451
|
+
}
|
|
452
|
+
this.ensureUnique(this.corrections, correction.id, "correction");
|
|
453
|
+
return this.append(this.corrections, correction.id, correction);
|
|
454
|
+
}
|
|
455
|
+
traverse(request, policy) {
|
|
456
|
+
this.validateTraversalRequest(request);
|
|
457
|
+
const snapshot = request.snapshot ?? this.sequence;
|
|
458
|
+
if (snapshot < 0 || snapshot > this.sequence) throw new LineageValidationError("SNAPSHOT_UNAVAILABLE", `Snapshot ${snapshot} is unavailable`);
|
|
459
|
+
const entityMap = this.visibleAt(this.entities, snapshot);
|
|
460
|
+
const assertions = [...this.visibleAt(this.assertions, snapshot).values()].filter((assertion) => !request.relationshipKinds || request.relationshipKinds.includes(assertion.relationship)).filter((assertion) => !request.assertionKinds || request.assertionKinds.includes(assertion.assertionKind)).filter((assertion) => policy.canReadAssertion(clone2(assertion))).filter((assertion) => {
|
|
461
|
+
const source = entityMap.get(assertion.sourceEntityId);
|
|
462
|
+
const target = entityMap.get(assertion.targetEntityId);
|
|
463
|
+
return Boolean(source && target && policy.canReadEntity(clone2(source)) && policy.canReadEntity(clone2(target)));
|
|
464
|
+
}).sort(compareAssertions);
|
|
465
|
+
for (const id of request.startEntityIds) {
|
|
466
|
+
const entity = entityMap.get(id);
|
|
467
|
+
if (!entity || !policy.canReadEntity(clone2(entity))) throw new LineageValidationError("IDENTITY_DANGLING", `Start entity ${id} is unavailable`);
|
|
468
|
+
}
|
|
469
|
+
const queue = [...new Set(request.startEntityIds)].sort().map((id) => ({ id, depth: 0, path: [] }));
|
|
470
|
+
const visitedDepth = /* @__PURE__ */ new Map();
|
|
471
|
+
const nodeMap = /* @__PURE__ */ new Map();
|
|
472
|
+
const edgeMap = /* @__PURE__ */ new Map();
|
|
473
|
+
while (queue.length > 0) {
|
|
474
|
+
const current = queue.shift();
|
|
475
|
+
const priorDepth = visitedDepth.get(current.id);
|
|
476
|
+
if (priorDepth !== void 0 && priorDepth <= current.depth) continue;
|
|
477
|
+
visitedDepth.set(current.id, current.depth);
|
|
478
|
+
const entity = entityMap.get(current.id);
|
|
479
|
+
if (!request.entityKinds || request.entityKinds.includes(entity.kind) || current.depth === 0) {
|
|
480
|
+
nodeMap.set(current.id, { entity: clone2(entity), depth: current.depth, pathAssertionIds: [...current.path] });
|
|
481
|
+
}
|
|
482
|
+
if (current.depth >= request.maximumDepth) continue;
|
|
483
|
+
for (const assertion of assertions) {
|
|
484
|
+
const downstream = assertion.sourceEntityId === current.id;
|
|
485
|
+
const upstream = assertion.targetEntityId === current.id;
|
|
486
|
+
if (request.direction === "downstream" && !downstream || request.direction === "upstream" && !upstream || request.direction === "both" && !downstream && !upstream) continue;
|
|
487
|
+
const nextId = downstream ? assertion.targetEntityId : assertion.sourceEntityId;
|
|
488
|
+
edgeMap.set(`${assertion.id}@${assertion.revision}`, {
|
|
489
|
+
assertion: clone2(assertion),
|
|
490
|
+
status: this.statusAt(assertion.id, assertion.revision, snapshot),
|
|
491
|
+
...request.includeEvidence ? { evidence: assertion.evidence.map((reference) => this.evidence.get(`${reference.evidenceId}@${reference.revision}`)).filter((entry) => Boolean(entry && entry.sequence <= snapshot)).map((entry) => entry.value).filter((item) => policy.canReadEvidence(clone2(item))).map(clone2) } : {}
|
|
492
|
+
});
|
|
493
|
+
queue.push({ id: nextId, depth: current.depth + 1, path: [...current.path, assertion.id] });
|
|
494
|
+
}
|
|
495
|
+
if (nodeMap.size + edgeMap.size > request.maximumResults) break;
|
|
496
|
+
}
|
|
497
|
+
const allNodes = [...nodeMap.values()].sort((left, right) => left.depth - right.depth || left.entity.id.localeCompare(right.entity.id));
|
|
498
|
+
const allEdges = [...edgeMap.values()].sort((left, right) => compareAssertions(left.assertion, right.assertion));
|
|
499
|
+
const rows = [
|
|
500
|
+
...allNodes.map((value) => ({ kind: "node", key: `n:${String(value.depth).padStart(8, "0")}:${value.entity.id}`, value })),
|
|
501
|
+
...allEdges.map((value) => ({ kind: "edge", key: `e:${value.assertion.sourceEntityId}:${value.assertion.targetEntityId}:${value.assertion.relationship}:${value.assertion.id}`, value }))
|
|
502
|
+
].sort((left, right) => left.key.localeCompare(right.key));
|
|
503
|
+
const requestFingerprint = this.traversalFingerprint(request, snapshot);
|
|
504
|
+
const offset = request.cursor ? this.decodeCursor(request.cursor, requestFingerprint, snapshot) : 0;
|
|
505
|
+
const bounded = rows.slice(0, request.maximumResults);
|
|
506
|
+
const page = bounded.slice(offset, offset + request.pageSize);
|
|
507
|
+
const nextOffset = offset + page.length;
|
|
508
|
+
const truncated = rows.length > request.maximumResults;
|
|
509
|
+
return {
|
|
510
|
+
contract: DATASET_LINEAGE_CONTRACT,
|
|
511
|
+
schemaVersion: DATASET_LINEAGE_SCHEMA_VERSION,
|
|
512
|
+
snapshot,
|
|
513
|
+
nodes: page.filter((row) => row.kind === "node").map((row) => clone2(row.value)),
|
|
514
|
+
edges: page.filter((row) => row.kind === "edge").map((row) => clone2(row.value)),
|
|
515
|
+
...nextOffset < bounded.length ? { nextCursor: `${snapshot}.${nextOffset}.${requestFingerprint}` } : {},
|
|
516
|
+
truncated
|
|
517
|
+
};
|
|
518
|
+
}
|
|
519
|
+
exportArchive(snapshot = this.sequence) {
|
|
520
|
+
if (snapshot < 0 || snapshot > this.sequence) throw new LineageValidationError("SNAPSHOT_UNAVAILABLE", `Snapshot ${snapshot} is unavailable`);
|
|
521
|
+
const content = {
|
|
522
|
+
contract: DATASET_LINEAGE_CONTRACT,
|
|
523
|
+
schemaVersion: DATASET_LINEAGE_SCHEMA_VERSION,
|
|
524
|
+
snapshot,
|
|
525
|
+
entities: this.sorted(this.visibleAt(this.entities, snapshot)),
|
|
526
|
+
agents: this.sorted(this.visibleAt(this.agents, snapshot)),
|
|
527
|
+
activities: this.sorted(this.visibleAt(this.activities, snapshot)),
|
|
528
|
+
evidence: this.sorted(this.visibleAt(this.evidence, snapshot), (item) => `${item.id}@${item.revision}`),
|
|
529
|
+
assertions: this.sorted(this.visibleAt(this.assertions, snapshot)),
|
|
530
|
+
corrections: this.sorted(this.visibleAt(this.corrections, snapshot))
|
|
531
|
+
};
|
|
532
|
+
return clone2({ ...content, digest: computeLineageDigest(content) });
|
|
533
|
+
}
|
|
534
|
+
static importArchive(archive, options = {}) {
|
|
535
|
+
if (archive.contract !== DATASET_LINEAGE_CONTRACT || !archive.schemaVersion.startsWith("1.")) throw new LineageValidationError("VALUE_INVALID", "Unsupported lineage archive contract");
|
|
536
|
+
const { digest: digest2, ...content } = archive;
|
|
537
|
+
if (computeLineageDigest(content) !== digest2) throw new LineageValidationError("EVIDENCE_DIGEST_MISMATCH", "Lineage archive digest does not match canonical content");
|
|
538
|
+
const ledger = new _DatasetLineageLedger(options);
|
|
539
|
+
for (const item of archive.agents) ledger.appendAgent(item);
|
|
540
|
+
for (const item of archive.entities) ledger.appendEntity(item);
|
|
541
|
+
for (const item of archive.activities) ledger.appendActivity(item);
|
|
542
|
+
for (const item of archive.evidence) ledger.appendEvidence(item);
|
|
543
|
+
const replacementKeys = new Set(archive.corrections.flatMap((item) => item.replacementAssertionId && item.replacementRevision ? [`${item.replacementAssertionId}@${item.replacementRevision}`] : []));
|
|
544
|
+
for (const item of archive.assertions.filter((item2) => !replacementKeys.has(`${item2.id}@${item2.revision}`))) ledger.appendAssertion(item);
|
|
545
|
+
const pending = [...archive.corrections];
|
|
546
|
+
while (pending.length > 0) {
|
|
547
|
+
const index = pending.findIndex((item) => ledger.assertions.has(`${item.assertionId}@${item.assertionRevision}`));
|
|
548
|
+
if (index < 0) throw new LineageValidationError("CORRECTION_INVALID", "Correction history contains an unreachable assertion revision");
|
|
549
|
+
const correction = pending.splice(index, 1)[0];
|
|
550
|
+
ledger.appendCorrection(correction);
|
|
551
|
+
if (correction.replacementAssertionId && correction.replacementRevision) {
|
|
552
|
+
const key = `${correction.replacementAssertionId}@${correction.replacementRevision}`;
|
|
553
|
+
if (!ledger.assertions.has(key)) {
|
|
554
|
+
const replacement = archive.assertions.find((item) => `${item.id}@${item.revision}` === key);
|
|
555
|
+
if (!replacement) throw new LineageValidationError("IDENTITY_DANGLING", `Missing replacement assertion ${key}`);
|
|
556
|
+
ledger.appendAssertion(replacement);
|
|
557
|
+
}
|
|
558
|
+
}
|
|
559
|
+
}
|
|
560
|
+
return ledger;
|
|
561
|
+
}
|
|
562
|
+
project(capabilities, snapshot = this.sequence) {
|
|
563
|
+
const source = this.exportArchive(snapshot);
|
|
564
|
+
const losses = [];
|
|
565
|
+
const entities = source.entities.filter((item, index) => {
|
|
566
|
+
const keep = capabilities.entityKinds.includes(item.kind);
|
|
567
|
+
if (!keep) losses.push({ path: `/entities/${index}`, reason: "unsupported-entity-kind", canonicalDigest: computeLineageDigest(item) });
|
|
568
|
+
return keep;
|
|
569
|
+
});
|
|
570
|
+
const entityIds = new Set(entities.map((item) => item.id));
|
|
571
|
+
const assertions = source.assertions.filter((item, index) => {
|
|
572
|
+
const reason = !capabilities.relationshipKinds.includes(item.relationship) ? "unsupported-relationship-kind" : !capabilities.assertionKinds.includes(item.assertionKind) ? "unsupported-assertion-kind" : !entityIds.has(item.sourceEntityId) || !entityIds.has(item.targetEntityId) ? "unsupported-entity-kind" : void 0;
|
|
573
|
+
if (reason) losses.push({ path: `/assertions/${index}`, reason, canonicalDigest: computeLineageDigest(item) });
|
|
574
|
+
return !reason;
|
|
575
|
+
});
|
|
576
|
+
const evidenceIds = new Set(assertions.flatMap((item) => item.evidence.map((reference) => `${reference.evidenceId}@${reference.revision}`)));
|
|
577
|
+
const evidence = capabilities.preservesEvidence ? source.evidence.filter((item) => evidenceIds.has(`${item.id}@${item.revision}`)) : [];
|
|
578
|
+
if (!capabilities.preservesEvidence && evidenceIds.size > 0) losses.push({ path: "/evidence", reason: "evidence-omitted", canonicalDigest: computeLineageDigest(source.evidence) });
|
|
579
|
+
const assertionIds = new Set(assertions.map((item) => item.id));
|
|
580
|
+
const corrections = capabilities.preservesCorrections ? source.corrections.filter((item) => assertionIds.has(item.assertionId)) : [];
|
|
581
|
+
if (!capabilities.preservesCorrections && source.corrections.length > 0) losses.push({ path: "/corrections", reason: "corrections-omitted", canonicalDigest: computeLineageDigest(source.corrections) });
|
|
582
|
+
losses.sort((left, right) => left.path.localeCompare(right.path) || left.reason.localeCompare(right.reason));
|
|
583
|
+
const projectionContent = { entities, assertions, evidence, corrections };
|
|
584
|
+
const projectionDigest = computeLineageDigest(projectionContent);
|
|
585
|
+
return clone2({
|
|
586
|
+
contract: DATASET_LINEAGE_CONTRACT,
|
|
587
|
+
schemaVersion: DATASET_LINEAGE_SCHEMA_VERSION,
|
|
588
|
+
canonical: false,
|
|
589
|
+
regenerable: true,
|
|
590
|
+
sourceDigest: source.digest,
|
|
591
|
+
...projectionContent,
|
|
592
|
+
digest: projectionDigest,
|
|
593
|
+
lossReceipt: {
|
|
594
|
+
contract: DATASET_LINEAGE_CONTRACT,
|
|
595
|
+
schemaVersion: DATASET_LINEAGE_SCHEMA_VERSION,
|
|
596
|
+
sourceDigest: source.digest,
|
|
597
|
+
projectionDigest,
|
|
598
|
+
lossless: losses.length === 0,
|
|
599
|
+
losses
|
|
600
|
+
}
|
|
601
|
+
});
|
|
602
|
+
}
|
|
603
|
+
append(map, key, value) {
|
|
604
|
+
this.sequence += 1;
|
|
605
|
+
map.set(key, { sequence: this.sequence, value: clone2(value) });
|
|
606
|
+
return this.sequence;
|
|
607
|
+
}
|
|
608
|
+
ensureUnique(map, key, label) {
|
|
609
|
+
if (map.has(key)) throw new LineageValidationError("IDENTITY_DUPLICATE", `Duplicate ${label} identity ${key}`);
|
|
610
|
+
}
|
|
611
|
+
visibleAt(map, snapshot) {
|
|
612
|
+
return new Map([...map].filter(([, entry]) => entry.sequence <= snapshot).map(([key, entry]) => [key, clone2(entry.value)]));
|
|
613
|
+
}
|
|
614
|
+
sorted(map, key = (item) => item.id) {
|
|
615
|
+
return [...map.values()].sort((left, right) => key(left).localeCompare(key(right))).map(clone2);
|
|
616
|
+
}
|
|
617
|
+
hasReplacementPermission(assertionId, revision) {
|
|
618
|
+
return [...this.corrections.values()].some((entry) => entry.value.action === "correct" && entry.value.replacementAssertionId === assertionId && entry.value.replacementRevision === revision);
|
|
619
|
+
}
|
|
620
|
+
statusAt(assertionId, revision, snapshot) {
|
|
621
|
+
const latest = [...this.corrections.values()].filter((entry) => entry.sequence <= snapshot && entry.value.assertionId === assertionId && entry.value.assertionRevision === revision).sort((left, right) => right.sequence - left.sequence)[0]?.value;
|
|
622
|
+
if (!latest) return "active";
|
|
623
|
+
return latest.action === "correct" ? "corrected" : latest.action === "retract" ? "retracted" : "superseded";
|
|
624
|
+
}
|
|
625
|
+
validateTraversalRequest(request) {
|
|
626
|
+
if (request.startEntityIds.length === 0) throw new LineageValidationError("IDENTITY_REQUIRED", "At least one start entity is required");
|
|
627
|
+
if (!Number.isSafeInteger(request.maximumDepth) || request.maximumDepth < 0 || request.maximumDepth > this.maximumTraversalDepth) throw new LineageValidationError("TRAVERSAL_LIMIT_EXCEEDED", `maximumDepth exceeds ${this.maximumTraversalDepth}`);
|
|
628
|
+
if (!Number.isSafeInteger(request.maximumResults) || request.maximumResults < 1 || request.maximumResults > this.maximumTraversalResults) throw new LineageValidationError("TRAVERSAL_LIMIT_EXCEEDED", `maximumResults exceeds ${this.maximumTraversalResults}`);
|
|
629
|
+
if (!Number.isSafeInteger(request.pageSize) || request.pageSize < 1 || request.pageSize > this.maximumPageSize || request.pageSize > request.maximumResults) throw new LineageValidationError("TRAVERSAL_LIMIT_EXCEEDED", `pageSize exceeds ${this.maximumPageSize}`);
|
|
630
|
+
}
|
|
631
|
+
traversalFingerprint(request, snapshot) {
|
|
632
|
+
const parameters = { ...request };
|
|
633
|
+
delete parameters.cursor;
|
|
634
|
+
delete parameters.snapshot;
|
|
635
|
+
return computeLineageDigest({ ...parameters, startEntityIds: [...new Set(parameters.startEntityIds)].sort(), snapshot });
|
|
636
|
+
}
|
|
637
|
+
decodeCursor(cursor, fingerprint, snapshot) {
|
|
638
|
+
const match = /^(\d+)\.(\d+)\.(sha256:[0-9a-f]{64})$/.exec(cursor);
|
|
639
|
+
if (!match || Number(match[1]) !== snapshot || match[3] !== fingerprint) throw new LineageValidationError("CURSOR_INVALID", "Cursor does not match this query and snapshot");
|
|
640
|
+
const offset = Number(match[2]);
|
|
641
|
+
if (!Number.isSafeInteger(offset) || offset < 0) throw new LineageValidationError("CURSOR_INVALID", "Cursor offset is invalid");
|
|
642
|
+
return offset;
|
|
643
|
+
}
|
|
644
|
+
};
|
|
645
|
+
function compareAssertions(left, right) {
|
|
646
|
+
return left.sourceEntityId.localeCompare(right.sourceEntityId) || left.targetEntityId.localeCompare(right.targetEntityId) || left.relationship.localeCompare(right.relationship) || left.id.localeCompare(right.id);
|
|
647
|
+
}
|
|
648
|
+
|
|
649
|
+
// src/dataset-execution-capabilities.ts
|
|
650
|
+
var DATASET_EXECUTION_CONTRACT = "fortemi.dataset-execution-capabilities/v1";
|
|
651
|
+
var DATASET_EXECUTION_SCHEMA_VERSION = "1.0.0";
|
|
652
|
+
var DATASET_EXECUTION_CAPABILITY_IDS = [
|
|
653
|
+
"ingest.full",
|
|
654
|
+
"ingest.snapshot",
|
|
655
|
+
"ingest.incremental",
|
|
656
|
+
"ingest.stream",
|
|
657
|
+
"schema.inspect",
|
|
658
|
+
"identity.stable-revision",
|
|
659
|
+
"identity.record",
|
|
660
|
+
"mutation.upsert",
|
|
661
|
+
"mutation.tombstone",
|
|
662
|
+
"mutation.reconcile",
|
|
663
|
+
"checkpoint.read",
|
|
664
|
+
"checkpoint.write",
|
|
665
|
+
"execution.cancel",
|
|
666
|
+
"rejection.record",
|
|
667
|
+
"index.lexical",
|
|
668
|
+
"index.chunk",
|
|
669
|
+
"index.vector",
|
|
670
|
+
"index.hybrid",
|
|
671
|
+
"index.rerank",
|
|
672
|
+
"index.graph",
|
|
673
|
+
"index.community",
|
|
674
|
+
"lineage.dataset",
|
|
675
|
+
"lineage.record",
|
|
676
|
+
"lineage.field",
|
|
677
|
+
"lineage.relationship-evidence",
|
|
678
|
+
"transaction.atomic-batch",
|
|
679
|
+
"privacy.pre-materialization-filter",
|
|
680
|
+
"pagination.cursor",
|
|
681
|
+
"ordering.deterministic"
|
|
682
|
+
];
|
|
683
|
+
var ID_SET = new Set(DATASET_EXECUTION_CAPABILITY_IDS);
|
|
684
|
+
var LIMIT_KEYS = ["maxInputBytes", "maxRecordBytes", "maxBatchRecords", "maxConcurrency", "maxPageSize", "maxTraversalDepth"];
|
|
685
|
+
function major(version) {
|
|
686
|
+
const match = /^(\d+)\.(\d+)\.(\d+)(?:[-+].*)?$/.exec(version);
|
|
687
|
+
return match ? Number(match[1]) : null;
|
|
688
|
+
}
|
|
689
|
+
function compareVersions(left, right) {
|
|
690
|
+
const parse = (value) => {
|
|
691
|
+
const match = /^(\d+)\.(\d+)\.(\d+)(?:[-+].*)?$/.exec(value);
|
|
692
|
+
return match ? [Number(match[1]), Number(match[2]), Number(match[3])] : null;
|
|
693
|
+
};
|
|
694
|
+
const a = parse(left);
|
|
695
|
+
const b = parse(right);
|
|
696
|
+
if (!a || !b) return null;
|
|
697
|
+
for (let index = 0; index < 3; index++) {
|
|
698
|
+
if (a[index] !== b[index]) return a[index] > b[index] ? 1 : -1;
|
|
699
|
+
}
|
|
700
|
+
return 0;
|
|
701
|
+
}
|
|
702
|
+
function isSupported(capability) {
|
|
703
|
+
return capability !== void 0 && capability.status !== "unsupported";
|
|
704
|
+
}
|
|
705
|
+
function validateDatasetExecutionDescriptor(descriptor) {
|
|
706
|
+
const diagnostics = [];
|
|
707
|
+
if (descriptor.contract !== DATASET_EXECUTION_CONTRACT) {
|
|
708
|
+
diagnostics.push({ code: "CONTRACT_MAJOR_UNSUPPORTED", path: "/contract", message: `Unsupported contract ${String(descriptor.contract)}` });
|
|
709
|
+
}
|
|
710
|
+
if (major(descriptor.schemaVersion) !== 1) {
|
|
711
|
+
diagnostics.push({ code: "SCHEMA_VERSION_UNSUPPORTED", path: "/schemaVersion", message: `Unsupported descriptor schema version ${descriptor.schemaVersion}` });
|
|
712
|
+
}
|
|
713
|
+
const capabilities = /* @__PURE__ */ new Map();
|
|
714
|
+
descriptor.capabilities.forEach((capability, index) => {
|
|
715
|
+
if (!ID_SET.has(capability.id) || compareVersions(capability.version, capability.version) === null) {
|
|
716
|
+
diagnostics.push({ code: "DESCRIPTOR_INVALID", path: `/capabilities/${index}`, message: `Invalid capability declaration ${capability.id}` });
|
|
717
|
+
return;
|
|
718
|
+
}
|
|
719
|
+
if (capabilities.has(capability.id)) {
|
|
720
|
+
diagnostics.push({ code: "CAPABILITY_DUPLICATE", capability: capability.id, path: `/capabilities/${index}/id`, message: `Capability ${capability.id} is declared more than once` });
|
|
721
|
+
}
|
|
722
|
+
capabilities.set(capability.id, capability);
|
|
723
|
+
for (const key of LIMIT_KEYS) {
|
|
724
|
+
const value = capability.limits?.[key];
|
|
725
|
+
if (value !== void 0 && (!Number.isSafeInteger(value) || value < 0)) {
|
|
726
|
+
diagnostics.push({ code: "DESCRIPTOR_INVALID", capability: capability.id, path: `/capabilities/${index}/limits/${key}`, message: `${key} must be a non-negative safe integer` });
|
|
727
|
+
}
|
|
728
|
+
}
|
|
729
|
+
if (capability.status !== "unsupported" && capability.evidence.length === 0) {
|
|
730
|
+
diagnostics.push({ code: "DESCRIPTOR_INVALID", capability: capability.id, path: `/capabilities/${index}/evidence`, message: `Supported capability ${capability.id} requires evidence` });
|
|
731
|
+
}
|
|
732
|
+
for (const evidenceId of capability.evidence) {
|
|
733
|
+
if (!descriptor.evidence.some((evidence) => evidence.id === evidenceId)) {
|
|
734
|
+
diagnostics.push({ code: "DESCRIPTOR_INVALID", capability: capability.id, path: `/capabilities/${index}/evidence`, message: `Unknown evidence ${evidenceId}` });
|
|
735
|
+
}
|
|
736
|
+
}
|
|
737
|
+
});
|
|
738
|
+
const requireTogether = (source, requirements) => {
|
|
739
|
+
if (!isSupported(capabilities.get(source))) return;
|
|
740
|
+
for (const requirement of requirements) {
|
|
741
|
+
if (!isSupported(capabilities.get(requirement))) {
|
|
742
|
+
diagnostics.push({ code: "CAPABILITY_INCONSISTENT", capability: source, message: `${source} requires ${requirement}` });
|
|
743
|
+
}
|
|
744
|
+
}
|
|
745
|
+
};
|
|
746
|
+
requireTogether("ingest.incremental", ["identity.stable-revision", "checkpoint.read", "checkpoint.write"]);
|
|
747
|
+
requireTogether("lineage.field", ["lineage.relationship-evidence"]);
|
|
748
|
+
requireTogether("mutation.reconcile", ["mutation.upsert", "mutation.tombstone"]);
|
|
749
|
+
requireTogether("index.hybrid", ["index.lexical", "index.vector"]);
|
|
750
|
+
if (descriptor.guarantees.transaction === "atomic-batch" && !isSupported(capabilities.get("transaction.atomic-batch"))) {
|
|
751
|
+
diagnostics.push({ code: "CAPABILITY_INCONSISTENT", capability: "transaction.atomic-batch", message: "Atomic-batch guarantee requires transaction.atomic-batch capability" });
|
|
752
|
+
}
|
|
753
|
+
if (descriptor.runtime.plane === "static-cache" && descriptor.runtime.dataClass !== "static-cache") {
|
|
754
|
+
diagnostics.push({ code: "CAPABILITY_INCONSISTENT", message: "Static-cache execution plane must declare static-cache data class" });
|
|
755
|
+
}
|
|
756
|
+
if (descriptor.runtime.plane === "live-remote-persistence" && descriptor.runtime.maturity === "stable" && !descriptor.evidence.some((item) => item.kind === "live-qualification")) {
|
|
757
|
+
diagnostics.push({ code: "CAPABILITY_INCONSISTENT", message: "Stable live remote persistence requires live qualification evidence" });
|
|
758
|
+
}
|
|
759
|
+
return diagnostics;
|
|
760
|
+
}
|
|
761
|
+
function assessRequirement(requirement, capabilities) {
|
|
762
|
+
const capability = capabilities.get(requirement.id);
|
|
763
|
+
if (!isSupported(capability)) {
|
|
764
|
+
return { ok: false, reason: "unsupported", diagnostics: [{ code: "REQUIRED_CAPABILITY_MISSING", capability: requirement.id, message: `Capability ${requirement.id} is unsupported` }] };
|
|
765
|
+
}
|
|
766
|
+
if (requirement.minimumVersion) {
|
|
767
|
+
const comparison = compareVersions(capability.version, requirement.minimumVersion);
|
|
768
|
+
if (comparison === null || comparison < 0) {
|
|
769
|
+
return { ok: false, reason: "version-insufficient", diagnostics: [{ code: "CAPABILITY_VERSION_INSUFFICIENT", capability: requirement.id, message: `${capability.version} does not satisfy ${requirement.minimumVersion}` }] };
|
|
770
|
+
}
|
|
771
|
+
}
|
|
772
|
+
for (const key of LIMIT_KEYS) {
|
|
773
|
+
const required = requirement.minimumLimits?.[key];
|
|
774
|
+
if (required !== void 0 && (capability.limits?.[key] ?? -1) < required) {
|
|
775
|
+
return { ok: false, reason: "limit-insufficient", diagnostics: [{ code: "CAPABILITY_LIMIT_INSUFFICIENT", capability: requirement.id, path: `/minimumLimits/${key}`, message: `${key} does not satisfy ${required}` }] };
|
|
776
|
+
}
|
|
777
|
+
}
|
|
778
|
+
return { ok: true, diagnostics: [] };
|
|
779
|
+
}
|
|
780
|
+
function negotiateDatasetExecutionCapabilities(descriptor, request) {
|
|
781
|
+
const diagnostics = validateDatasetExecutionDescriptor(descriptor);
|
|
782
|
+
if (request.contract !== DATASET_EXECUTION_CONTRACT) {
|
|
783
|
+
diagnostics.push({ code: "CONTRACT_MAJOR_UNSUPPORTED", path: "/contract", message: `Unsupported request contract ${String(request.contract)}` });
|
|
784
|
+
}
|
|
785
|
+
const capabilities = new Map(descriptor.capabilities.map((capability) => [capability.id, capability]));
|
|
786
|
+
const selected = [];
|
|
787
|
+
const degradations = [];
|
|
788
|
+
if (diagnostics.length === 0) {
|
|
789
|
+
for (const requirement of request.required) {
|
|
790
|
+
const assessment = assessRequirement(requirement, capabilities);
|
|
791
|
+
if (assessment.ok) selected.push(requirement.id);
|
|
792
|
+
else diagnostics.push(...assessment.diagnostics);
|
|
793
|
+
}
|
|
794
|
+
for (const requirement of request.optional ?? []) {
|
|
795
|
+
const assessment = assessRequirement(requirement, capabilities);
|
|
796
|
+
if (assessment.ok) {
|
|
797
|
+
selected.push(requirement.id);
|
|
798
|
+
continue;
|
|
799
|
+
}
|
|
800
|
+
const fallback = requirement.fallback?.find((id) => isSupported(capabilities.get(id)));
|
|
801
|
+
if (fallback) selected.push(fallback);
|
|
802
|
+
degradations.push({
|
|
803
|
+
requested: requirement.id,
|
|
804
|
+
...fallback ? { selected: fallback } : {},
|
|
805
|
+
reason: assessment.reason,
|
|
806
|
+
changedGuarantees: fallback ? [`${requirement.id} replaced by ${fallback}`] : [`${requirement.id} omitted`]
|
|
807
|
+
});
|
|
808
|
+
}
|
|
809
|
+
}
|
|
810
|
+
return {
|
|
811
|
+
contract: DATASET_EXECUTION_CONTRACT,
|
|
812
|
+
accepted: diagnostics.length === 0,
|
|
813
|
+
runtime: { ...descriptor.runtime },
|
|
814
|
+
selected: [...new Set(selected)],
|
|
815
|
+
degradations,
|
|
816
|
+
diagnostics
|
|
817
|
+
};
|
|
818
|
+
}
|
|
819
|
+
|
|
820
|
+
// src/dataset-execution-descriptors.ts
|
|
821
|
+
var FORTEMI_BROWSER_LOCAL_DATASET_EXECUTION_DESCRIPTOR = {
|
|
822
|
+
contract: DATASET_EXECUTION_CONTRACT,
|
|
823
|
+
schemaVersion: DATASET_EXECUTION_SCHEMA_VERSION,
|
|
824
|
+
runtime: { id: "fortemi-browser", version: "2026.8.0", plane: "browser-local-archive", dataClass: "canonical", maturity: "stable" },
|
|
825
|
+
guarantees: { transaction: "atomic-batch", isolation: "serializable", durability: "wal", availability: "local-process", ordering: "backend-cursor" },
|
|
826
|
+
capabilities: [
|
|
827
|
+
{ id: "ingest.full", version: "1.0.0", status: "supported", limits: { maxBatchRecords: 1e3, maxConcurrency: 1 }, evidence: ["browser-conformance"] },
|
|
828
|
+
{ id: "identity.record", version: "1.0.0", status: "supported", evidence: ["browser-conformance"] },
|
|
829
|
+
{ id: "mutation.upsert", version: "1.0.0", status: "supported", evidence: ["browser-conformance"] },
|
|
830
|
+
{ id: "transaction.atomic-batch", version: "1.0.0", status: "supported", evidence: ["browser-conformance"] },
|
|
831
|
+
{ id: "ordering.deterministic", version: "1.0.0", status: "supported", evidence: ["browser-conformance"] }
|
|
832
|
+
],
|
|
833
|
+
evidence: [{ id: "browser-conformance", kind: "conformance-report", uri: "fortemi://conformance/browser-local/v1" }]
|
|
834
|
+
};
|
|
835
|
+
var FORTEMI_STATIC_CACHE_DATASET_EXECUTION_DESCRIPTOR = {
|
|
836
|
+
contract: DATASET_EXECUTION_CONTRACT,
|
|
837
|
+
schemaVersion: DATASET_EXECUTION_SCHEMA_VERSION,
|
|
838
|
+
runtime: { id: "fortemi-static", version: "2026.8.0", plane: "static-cache", dataClass: "static-cache", maturity: "stable" },
|
|
839
|
+
guarantees: { transaction: "none", isolation: "snapshot", durability: "filesystem", availability: "local-process", ordering: "stable-identity" },
|
|
840
|
+
capabilities: [
|
|
841
|
+
{ id: "index.lexical", version: "1.0.0", status: "supported", limits: { maxPageSize: 1e3 }, evidence: ["static-conformance"] },
|
|
842
|
+
{ id: "pagination.cursor", version: "1.0.0", status: "supported", evidence: ["static-conformance"] },
|
|
843
|
+
{ id: "ordering.deterministic", version: "1.0.0", status: "supported", evidence: ["static-conformance"] }
|
|
844
|
+
],
|
|
845
|
+
evidence: [{ id: "static-conformance", kind: "conformance-report", uri: "fortemi://conformance/static-cache/v1" }]
|
|
846
|
+
};
|
|
847
|
+
var FORTEMI_PORTABLE_SHARD_DATASET_EXECUTION_DESCRIPTOR = {
|
|
848
|
+
contract: DATASET_EXECUTION_CONTRACT,
|
|
849
|
+
schemaVersion: DATASET_EXECUTION_SCHEMA_VERSION,
|
|
850
|
+
runtime: { id: "fortemi-shard", version: "2026.8.0", plane: "portable-shard", dataClass: "portable-projection", maturity: "stable" },
|
|
851
|
+
guarantees: { transaction: "none", isolation: "snapshot", durability: "filesystem", availability: "local-process", ordering: "stable-identity" },
|
|
852
|
+
capabilities: [
|
|
853
|
+
{ id: "schema.inspect", version: "1.0.0", status: "supported", evidence: ["shard-conformance"] },
|
|
854
|
+
{ id: "lineage.dataset", version: "1.0.0", status: "experimental", evidence: ["shard-conformance"] },
|
|
855
|
+
{ id: "ordering.deterministic", version: "1.0.0", status: "supported", evidence: ["shard-conformance"] }
|
|
856
|
+
],
|
|
857
|
+
evidence: [{ id: "shard-conformance", kind: "conformance-report", uri: "fortemi://conformance/knowledge-shard/v1" }]
|
|
858
|
+
};
|
|
859
|
+
var DATASET_MATERIALIZATION_CONTRACT = "fortemi.dataset-materialization-profile/v1";
|
|
860
|
+
var DATASET_MATERIALIZATION_SCHEMA_VERSION = "1.0.0";
|
|
861
|
+
var DATASET_MATERIALIZATION_KINDS = [
|
|
862
|
+
"chunking",
|
|
863
|
+
"lexical",
|
|
864
|
+
"vector",
|
|
865
|
+
"hybrid",
|
|
866
|
+
"rerank",
|
|
867
|
+
"entity-relationship-extraction",
|
|
868
|
+
"graph-retrieval",
|
|
869
|
+
"community"
|
|
870
|
+
];
|
|
871
|
+
var DatasetMaterializationError = class extends Error {
|
|
872
|
+
constructor(code, message) {
|
|
873
|
+
super(message);
|
|
874
|
+
this.code = code;
|
|
875
|
+
this.name = "DatasetMaterializationError";
|
|
876
|
+
}
|
|
877
|
+
};
|
|
878
|
+
var encoder = new TextEncoder();
|
|
879
|
+
function canonicalJson3(value) {
|
|
880
|
+
if (value === null || typeof value !== "object") return JSON.stringify(value);
|
|
881
|
+
if (Array.isArray(value)) return `[${value.map(canonicalJson3).join(",")}]`;
|
|
882
|
+
return `{${Object.entries(value).filter(([, item]) => item !== void 0).sort(([left], [right]) => left.localeCompare(right)).map(([key, item]) => `${JSON.stringify(key)}:${canonicalJson3(item)}`).join(",")}}`;
|
|
883
|
+
}
|
|
884
|
+
function digestDatasetMaterializationValue(value) {
|
|
885
|
+
return computeHash(encoder.encode(canonicalJson3(value)));
|
|
886
|
+
}
|
|
887
|
+
function profileDiagnostics(profile) {
|
|
888
|
+
const diagnostics = [];
|
|
889
|
+
const requiredBoundaries = ["chunking", "model-invocation", "index-persistence"];
|
|
890
|
+
if (profile.contract !== DATASET_MATERIALIZATION_CONTRACT || !/^1\./.test(profile.schemaVersion)) {
|
|
891
|
+
diagnostics.push({ code: "PROFILE_INVALID", path: "/contract", message: "Unsupported profile contract or schema major" });
|
|
892
|
+
}
|
|
893
|
+
if (profile.output.dataClass !== "regenerable-index" || profile.output.canonicalMutation !== false) {
|
|
894
|
+
diagnostics.push({ code: "PROFILE_INVALID", path: "/output", message: "Profiles may only produce derived, regenerable artifacts" });
|
|
895
|
+
}
|
|
896
|
+
for (const boundary of requiredBoundaries) {
|
|
897
|
+
if (!profile.privacy.filtersBefore.includes(boundary)) {
|
|
898
|
+
diagnostics.push({ code: "PROFILE_INVALID", path: "/privacy/filtersBefore", message: `Privacy filtering must precede ${boundary}` });
|
|
899
|
+
}
|
|
900
|
+
}
|
|
901
|
+
if (profile.determinism.class === "seeded" && profile.determinism.seedRequired !== true) {
|
|
902
|
+
diagnostics.push({ code: "PROFILE_INVALID", path: "/determinism/seedRequired", message: "Seeded profiles must require a seed" });
|
|
903
|
+
}
|
|
904
|
+
if (profile.resourceLimits.maxInputBytes < 0 || profile.resourceLimits.maxRecords < 0 || profile.resourceLimits.maxConcurrency < 1) {
|
|
905
|
+
diagnostics.push({ code: "PROFILE_INVALID", path: "/resourceLimits", message: "Resource limits must be non-negative and concurrency must be positive" });
|
|
906
|
+
}
|
|
907
|
+
return diagnostics;
|
|
908
|
+
}
|
|
909
|
+
function negotiateDatasetMaterializationProfile(request) {
|
|
910
|
+
const candidates = [request.profile, ...request.fallbackProfiles ?? []];
|
|
911
|
+
const primaryDiagnostics = profileDiagnostics(request.profile);
|
|
912
|
+
if (!request.profile.operations.includes(request.operation)) {
|
|
913
|
+
primaryDiagnostics.push({ code: "OPERATION_UNSUPPORTED", path: "/operations", message: `${request.operation} is not supported by ${request.profile.id}` });
|
|
914
|
+
}
|
|
915
|
+
if (request.profile.status === "unsupported") {
|
|
916
|
+
primaryDiagnostics.push({ code: "PROFILE_UNSUPPORTED", path: "/status", message: `${request.profile.id} is unsupported` });
|
|
917
|
+
}
|
|
918
|
+
for (let index = 0; index < candidates.length; index += 1) {
|
|
919
|
+
const candidate = candidates[index];
|
|
920
|
+
const diagnostics = index === 0 ? primaryDiagnostics : profileDiagnostics(candidate);
|
|
921
|
+
if (!candidate.operations.includes(request.operation) || candidate.status === "unsupported" || diagnostics.length > 0) continue;
|
|
922
|
+
const capability = negotiateDatasetExecutionCapabilities(request.runtime, {
|
|
923
|
+
contract: DATASET_EXECUTION_CONTRACT,
|
|
924
|
+
required: candidate.requiredRuntimeCapabilities,
|
|
925
|
+
optional: candidate.optionalRuntimeCapabilities
|
|
926
|
+
});
|
|
927
|
+
if (capability.accepted) {
|
|
928
|
+
const fallback = index > 0;
|
|
929
|
+
return {
|
|
930
|
+
accepted: true,
|
|
931
|
+
requestedProfile: request.profile.id,
|
|
932
|
+
selectedProfile: candidate.id,
|
|
933
|
+
runtime: { ...request.runtime.runtime },
|
|
934
|
+
degraded: fallback || capability.degradations.length > 0,
|
|
935
|
+
degradations: capability.degradations,
|
|
936
|
+
diagnostics: fallback ? primaryDiagnostics : []
|
|
937
|
+
};
|
|
938
|
+
}
|
|
939
|
+
if (index === 0) primaryDiagnostics.push(...capability.diagnostics);
|
|
940
|
+
}
|
|
941
|
+
return {
|
|
942
|
+
accepted: false,
|
|
943
|
+
requestedProfile: request.profile.id,
|
|
944
|
+
runtime: { ...request.runtime.runtime },
|
|
945
|
+
degraded: false,
|
|
946
|
+
degradations: [],
|
|
947
|
+
diagnostics: [...primaryDiagnostics, { code: "NO_FALLBACK_PROFILE", message: `No compatible ${request.operation} profile is available` }]
|
|
948
|
+
};
|
|
949
|
+
}
|
|
950
|
+
function clone3(value) {
|
|
951
|
+
return structuredClone(value);
|
|
952
|
+
}
|
|
953
|
+
function assertResources(snapshot, profile) {
|
|
954
|
+
const inputBytes = encoder.encode(canonicalJson3(snapshot.records)).byteLength;
|
|
955
|
+
if (snapshot.records.length > profile.resourceLimits.maxRecords || inputBytes > profile.resourceLimits.maxInputBytes) {
|
|
956
|
+
throw new DatasetMaterializationError("RESOURCE_LIMIT_EXCEEDED", `Input exceeds profile ${profile.id} resource limits`);
|
|
957
|
+
}
|
|
958
|
+
return inputBytes;
|
|
959
|
+
}
|
|
960
|
+
function stableArtifacts(artifacts) {
|
|
961
|
+
return [...artifacts].sort((left, right) => {
|
|
962
|
+
const score = (right.score ?? Number.NEGATIVE_INFINITY) - (left.score ?? Number.NEGATIVE_INFINITY);
|
|
963
|
+
return score || left.logicalId.localeCompare(right.logicalId) || left.digest.localeCompare(right.digest);
|
|
964
|
+
});
|
|
965
|
+
}
|
|
966
|
+
async function executeDatasetMaterialization(request, runtime, adapter, authorize, options = {}) {
|
|
967
|
+
const negotiation = negotiateDatasetMaterializationProfile({ operation: "build", profile: request.profile, runtime, fallbackProfiles: options.fallbackProfiles });
|
|
968
|
+
if (!negotiation.accepted || !negotiation.selectedProfile) {
|
|
969
|
+
throw new DatasetMaterializationError("NEGOTIATION_FAILED", negotiation.diagnostics.map((item) => item.message).join("; "));
|
|
970
|
+
}
|
|
971
|
+
const selected = [request.profile, ...options.fallbackProfiles ?? []].find((item) => item.id === negotiation.selectedProfile);
|
|
972
|
+
try {
|
|
973
|
+
const validateConfiguration = new Ajv20202({ strict: true, allErrors: true }).compile(selected.configurationSchema);
|
|
974
|
+
if (!validateConfiguration(request.configuration)) {
|
|
975
|
+
throw new DatasetMaterializationError("CONFIGURATION_INVALID", `Configuration does not satisfy ${selected.id}: ${JSON.stringify(validateConfiguration.errors)}`);
|
|
976
|
+
}
|
|
977
|
+
} catch (error) {
|
|
978
|
+
if (error instanceof DatasetMaterializationError) throw error;
|
|
979
|
+
throw new DatasetMaterializationError("PROFILE_INVALID", `Invalid configuration schema for ${selected.id}: ${error instanceof Error ? error.message : String(error)}`);
|
|
980
|
+
}
|
|
981
|
+
const inputBytes = assertResources(request.snapshot, selected);
|
|
982
|
+
const allowed = [];
|
|
983
|
+
const denied = [];
|
|
984
|
+
for (const record of request.snapshot.records) {
|
|
985
|
+
if (await authorize(Object.freeze(clone3(record)))) allowed.push(clone3(record));
|
|
986
|
+
else denied.push(record.digest);
|
|
987
|
+
}
|
|
988
|
+
const privacy = {
|
|
989
|
+
policyId: "caller-authorization",
|
|
990
|
+
policyVersion: "1.0.0",
|
|
991
|
+
policyDigest: digestDatasetMaterializationValue({ policy: "caller-authorization", version: "1.0.0" }),
|
|
992
|
+
allowedRecordDigests: allowed.map((item) => item.digest).sort(),
|
|
993
|
+
deniedRecordDigests: denied.sort(),
|
|
994
|
+
evaluatedAt: (options.now ?? (() => (/* @__PURE__ */ new Date()).toISOString()))()
|
|
995
|
+
};
|
|
996
|
+
const affected = request.affected ?? {
|
|
997
|
+
sourceRevisions: [request.snapshot.revision],
|
|
998
|
+
recordDigests: allowed.map((item) => item.digest).sort(),
|
|
999
|
+
chunkDigests: []
|
|
1000
|
+
};
|
|
1001
|
+
const detachedSnapshot = clone3({ ...request.snapshot, records: allowed });
|
|
1002
|
+
const result = await adapter.materialize(Object.freeze({
|
|
1003
|
+
snapshot: detachedSnapshot,
|
|
1004
|
+
profile: clone3(selected),
|
|
1005
|
+
configuration: clone3(request.configuration),
|
|
1006
|
+
mode: request.mode,
|
|
1007
|
+
affected: clone3(affected)
|
|
1008
|
+
}));
|
|
1009
|
+
const artifacts = selected.determinism.class === "deterministic" ? stableArtifacts(result.artifacts) : clone3(result.artifacts);
|
|
1010
|
+
const digests = artifacts.map((item) => item.digest).sort();
|
|
1011
|
+
const counts = artifacts.reduce((all, item) => {
|
|
1012
|
+
all[item.kind] = (all[item.kind] ?? 0) + 1;
|
|
1013
|
+
return all;
|
|
1014
|
+
}, {});
|
|
1015
|
+
const profileDigest = digestDatasetMaterializationValue(selected);
|
|
1016
|
+
const receiptBase = {
|
|
1017
|
+
contract: DATASET_MATERIALIZATION_CONTRACT,
|
|
1018
|
+
schemaVersion: DATASET_MATERIALIZATION_SCHEMA_VERSION,
|
|
1019
|
+
runId: request.runId,
|
|
1020
|
+
processingRunId: request.processingRunId,
|
|
1021
|
+
source: { datasetId: request.snapshot.datasetId, revision: request.snapshot.revision, digests: [...request.snapshot.sourceDigests].sort() },
|
|
1022
|
+
schema: { id: request.snapshot.schemaId, version: request.snapshot.schemaVersion, digest: request.snapshot.schemaDigest },
|
|
1023
|
+
profile: { id: selected.id, version: selected.version, digest: profileDigest, configurationDigest: digestDatasetMaterializationValue(request.configuration), implementation: clone3(selected.implementation) },
|
|
1024
|
+
runtime: clone3(runtime.runtime),
|
|
1025
|
+
negotiation: { requestedProfile: negotiation.requestedProfile, selectedProfile: negotiation.selectedProfile, degraded: negotiation.degraded, degradations: clone3(negotiation.degradations) },
|
|
1026
|
+
mode: request.mode,
|
|
1027
|
+
affected: clone3(affected),
|
|
1028
|
+
output: { counts, digests, aggregateDigest: digestDatasetMaterializationValue(digests) },
|
|
1029
|
+
privacy,
|
|
1030
|
+
resources: { ...clone3(result.resources), inputBytes },
|
|
1031
|
+
createdAt: (options.now ?? (() => (/* @__PURE__ */ new Date()).toISOString()))()
|
|
1032
|
+
};
|
|
1033
|
+
const receipt = {
|
|
1034
|
+
...receiptBase,
|
|
1035
|
+
receiptId: digestDatasetMaterializationValue(receiptBase)
|
|
1036
|
+
};
|
|
1037
|
+
return { artifacts, receipt };
|
|
1038
|
+
}
|
|
1039
|
+
async function executeDatasetRetrieval(request, runtime, adapter, receiptId, fallbackProfiles = []) {
|
|
1040
|
+
const negotiation = negotiateDatasetMaterializationProfile({ operation: "query", profile: request.profile, runtime, fallbackProfiles });
|
|
1041
|
+
if (!negotiation.accepted || !negotiation.selectedProfile) throw new DatasetMaterializationError("NEGOTIATION_FAILED", negotiation.diagnostics.map((item) => item.message).join("; "));
|
|
1042
|
+
if (!adapter.retrieve) throw new DatasetMaterializationError("ADAPTER_MISMATCH", `Backend ${adapter.backend.id} cannot query`);
|
|
1043
|
+
const selected = [request.profile, ...fallbackProfiles].find((item) => item.id === negotiation.selectedProfile);
|
|
1044
|
+
const results = await adapter.retrieve(Object.freeze({ request: clone3(request), profile: clone3(selected) }));
|
|
1045
|
+
return {
|
|
1046
|
+
contract: DATASET_MATERIALIZATION_CONTRACT,
|
|
1047
|
+
schemaVersion: DATASET_MATERIALIZATION_SCHEMA_VERSION,
|
|
1048
|
+
queryId: request.queryId,
|
|
1049
|
+
requestedProfile: request.profile.id,
|
|
1050
|
+
actualProfile: selected.id,
|
|
1051
|
+
actualBackend: { ...adapter.backend, plane: runtime.runtime.plane },
|
|
1052
|
+
degraded: negotiation.degraded,
|
|
1053
|
+
...negotiation.degraded ? { fallbackReason: negotiation.diagnostics.map((item) => item.message).join("; ") || "optional capability degradation" } : {},
|
|
1054
|
+
scoreSemantics: { implementationScoped: true, comparableAcrossImplementations: false },
|
|
1055
|
+
results: selected.determinism.class === "deterministic" ? stableArtifacts(results).slice(0, request.limit) : clone3(results).slice(0, request.limit),
|
|
1056
|
+
receiptId
|
|
1057
|
+
};
|
|
1058
|
+
}
|
|
1059
|
+
function compareDatasetIncrementalParity(full, incremental) {
|
|
1060
|
+
const select = (items, kind) => items.filter((item) => !kind || item.kind === kind).map((item) => `${item.logicalId}:${item.digest}`).sort();
|
|
1061
|
+
const mismatches = [];
|
|
1062
|
+
if (canonicalJson3(select(full)) !== canonicalJson3(select(incremental))) mismatches.push("identities");
|
|
1063
|
+
if (canonicalJson3(select(full, "chunk")) !== canonicalJson3(select(incremental, "chunk"))) mismatches.push("chunks");
|
|
1064
|
+
if (canonicalJson3(select(full, "relationship")) !== canonicalJson3(select(incremental, "relationship"))) mismatches.push("relationships");
|
|
1065
|
+
if (canonicalJson3(select(full, "community")) !== canonicalJson3(select(incremental, "community"))) mismatches.push("communities");
|
|
1066
|
+
if (canonicalJson3(full.map((item) => item.digest).sort()) !== canonicalJson3(incremental.map((item) => item.digest).sort())) mismatches.push("digests");
|
|
1067
|
+
const order = (items) => items.map((item) => `${item.logicalId}:${item.digest}`);
|
|
1068
|
+
if (canonicalJson3(order(full)) !== canonicalJson3(order(incremental))) mismatches.push("ordering");
|
|
1069
|
+
return { equivalent: mismatches.length === 0, mismatches };
|
|
1070
|
+
}
|
|
1071
|
+
function validateDatasetBenchmarkEvidence(evidence) {
|
|
1072
|
+
const errors = [];
|
|
1073
|
+
if (evidence.contract !== DATASET_MATERIALIZATION_CONTRACT || !/^1\./.test(evidence.schemaVersion)) errors.push("unsupported benchmark contract");
|
|
1074
|
+
if (!evidence.correctness.passed) errors.push("correctness suite did not pass");
|
|
1075
|
+
if (evidence.freshness.sourceRevision !== evidence.corpus.revision) errors.push("benchmark evidence is stale");
|
|
1076
|
+
if (!evidence.claims.corpusScoped || evidence.claims.universalScaleLimit) errors.push("benchmark claims must remain corpus-scoped");
|
|
1077
|
+
if (evidence.measurements.length === 0) errors.push("benchmark has no measurements");
|
|
1078
|
+
return errors;
|
|
1079
|
+
}
|
|
13
1080
|
|
|
14
1081
|
// src/event-bus.ts
|
|
15
1082
|
var TypedEventBus = class {
|
|
@@ -1005,6 +2072,54 @@ var migration0023 = {
|
|
|
1005
2072
|
`
|
|
1006
2073
|
};
|
|
1007
2074
|
|
|
2075
|
+
// src/migrations/0024_source_upsert_contract.ts
|
|
2076
|
+
var migration0024 = {
|
|
2077
|
+
version: 24,
|
|
2078
|
+
name: "0024_source_upsert_contract",
|
|
2079
|
+
sql: `
|
|
2080
|
+
ALTER TABLE source_import_run ADD COLUMN IF NOT EXISTS external_run_id TEXT;
|
|
2081
|
+
ALTER TABLE source_import_run ADD COLUMN IF NOT EXISTS source_id TEXT;
|
|
2082
|
+
ALTER TABLE source_import_run ADD COLUMN IF NOT EXISTS source_schema_version TEXT;
|
|
2083
|
+
ALTER TABLE source_import_run ADD COLUMN IF NOT EXISTS workspace_id TEXT;
|
|
2084
|
+
UPDATE source_import_run SET external_run_id = id WHERE external_run_id IS NULL;
|
|
2085
|
+
UPDATE source_import_run SET source_schema_version = 'legacy' WHERE source_schema_version IS NULL;
|
|
2086
|
+
ALTER TABLE source_import_run ALTER COLUMN external_run_id SET NOT NULL;
|
|
2087
|
+
ALTER TABLE source_import_run ALTER COLUMN source_schema_version SET NOT NULL;
|
|
2088
|
+
ALTER TABLE source_import_run ADD CONSTRAINT source_import_run_contract_lengths
|
|
2089
|
+
CHECK (
|
|
2090
|
+
length(external_run_id) BETWEEN 1 AND 200
|
|
2091
|
+
AND (source_id IS NULL OR length(source_id) BETWEEN 1 AND 500)
|
|
2092
|
+
AND length(source_schema_version) BETWEEN 1 AND 100
|
|
2093
|
+
AND (workspace_id IS NULL OR length(workspace_id) BETWEEN 1 AND 500)
|
|
2094
|
+
);
|
|
2095
|
+
CREATE UNIQUE INDEX IF NOT EXISTS idx_source_import_run_scope
|
|
2096
|
+
ON source_import_run(tenant_id, COALESCE(archive_id, ''), namespace, external_run_id);
|
|
2097
|
+
|
|
2098
|
+
ALTER TABLE source_identity ADD COLUMN IF NOT EXISTS source_id TEXT;
|
|
2099
|
+
ALTER TABLE source_identity ADD CONSTRAINT source_identity_source_id_length
|
|
2100
|
+
CHECK (source_id IS NULL OR length(source_id) BETWEEN 1 AND 500);
|
|
2101
|
+
|
|
2102
|
+
CREATE TABLE IF NOT EXISTS source_import_batch (
|
|
2103
|
+
id TEXT PRIMARY KEY,
|
|
2104
|
+
tenant_id TEXT NOT NULL DEFAULT 'default',
|
|
2105
|
+
archive_id TEXT,
|
|
2106
|
+
namespace TEXT NOT NULL,
|
|
2107
|
+
batch_id TEXT NOT NULL,
|
|
2108
|
+
request_digest TEXT NOT NULL,
|
|
2109
|
+
import_run_id TEXT NOT NULL,
|
|
2110
|
+
outcome TEXT NOT NULL,
|
|
2111
|
+
checkpoint JSONB NOT NULL DEFAULT '{}',
|
|
2112
|
+
receipt JSONB NOT NULL DEFAULT '{}',
|
|
2113
|
+
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
|
2114
|
+
);
|
|
2115
|
+
|
|
2116
|
+
CREATE UNIQUE INDEX IF NOT EXISTS idx_source_import_batch_scope
|
|
2117
|
+
ON source_import_batch(tenant_id, COALESCE(archive_id, ''), namespace, batch_id);
|
|
2118
|
+
CREATE INDEX IF NOT EXISTS idx_source_import_batch_run
|
|
2119
|
+
ON source_import_batch(import_run_id);
|
|
2120
|
+
`
|
|
2121
|
+
};
|
|
2122
|
+
|
|
1008
2123
|
// src/migrations/index.ts
|
|
1009
2124
|
var allMigrations = [
|
|
1010
2125
|
migration0001,
|
|
@@ -1029,7 +2144,8 @@ var allMigrations = [
|
|
|
1029
2144
|
migration0020,
|
|
1030
2145
|
migration0021,
|
|
1031
2146
|
migration0022,
|
|
1032
|
-
migration0023
|
|
2147
|
+
migration0023,
|
|
2148
|
+
migration0024
|
|
1033
2149
|
];
|
|
1034
2150
|
|
|
1035
2151
|
// src/data-archive.ts
|
|
@@ -1135,13 +2251,6 @@ async function restoreDbSnapshot(source, options = {}) {
|
|
|
1135
2251
|
const createOptions = { loadDataDir: data };
|
|
1136
2252
|
return createPGliteInstance(options.persistence ?? "memory", options.archiveName ?? "default", createOptions);
|
|
1137
2253
|
}
|
|
1138
|
-
function computeHash(data) {
|
|
1139
|
-
const digest = sha256(data);
|
|
1140
|
-
return `sha256:${bytesToHex(digest)}`;
|
|
1141
|
-
}
|
|
1142
|
-
function computeBlobHash(data) {
|
|
1143
|
-
return `blake3:${bytesToHex(blake3(data))}`;
|
|
1144
|
-
}
|
|
1145
2254
|
|
|
1146
2255
|
// src/repositories/notes-repository.ts
|
|
1147
2256
|
var NotesRepository = class {
|
|
@@ -4401,52 +5510,79 @@ function createLazyBlobStore(archiveName, options) {
|
|
|
4401
5510
|
}
|
|
4402
5511
|
|
|
4403
5512
|
// src/repositories/source-upsert-repository.ts
|
|
4404
|
-
var
|
|
5513
|
+
var SOURCE_UPSERT_CONTRACT_VERSION = "1.0.0";
|
|
5514
|
+
var SOURCE_UPSERT_MAX_ITEMS = 500;
|
|
4405
5515
|
function assertSource(input) {
|
|
4406
|
-
if (!input.
|
|
4407
|
-
if (!input.
|
|
4408
|
-
if (!input.
|
|
4409
|
-
|
|
4410
|
-
|
|
4411
|
-
if (!input.import_run_id || input.import_run_id.length > 128) throw new Error("Source import_run_id is required and must be <= 128 characters");
|
|
5516
|
+
if (!input.tenant_id || input.tenant_id.length > 200) throw new Error("invalid_batch_metadata");
|
|
5517
|
+
if (!input.namespace || input.namespace.length > 200) throw new Error("invalid_batch_metadata");
|
|
5518
|
+
if (!input.external_id || input.external_id.length > 1e3) throw new Error("invalid_item");
|
|
5519
|
+
if (!input.source_schema_version || input.source_schema_version.length > 100) throw new Error("invalid_batch_metadata");
|
|
5520
|
+
if (!input.import_run_id || input.import_run_id.length > 200) throw new Error("invalid_batch_metadata");
|
|
4412
5521
|
}
|
|
4413
|
-
function
|
|
5522
|
+
function sourceIdentityHash(source) {
|
|
4414
5523
|
return computeHash(new TextEncoder().encode([
|
|
4415
5524
|
source.tenant_id ?? "default",
|
|
4416
|
-
source.archive_id ?? "",
|
|
5525
|
+
source.archive_id ?? "public",
|
|
4417
5526
|
source.namespace,
|
|
4418
|
-
source.external_id
|
|
5527
|
+
source.external_id,
|
|
5528
|
+
""
|
|
4419
5529
|
].join("\0")));
|
|
4420
5530
|
}
|
|
4421
|
-
function
|
|
5531
|
+
function sourceContentDigest(content) {
|
|
4422
5532
|
return computeHash(new TextEncoder().encode(content));
|
|
4423
5533
|
}
|
|
4424
|
-
|
|
5534
|
+
function sourceRequestDigest(items, options) {
|
|
5535
|
+
return computeHash(new TextEncoder().encode(JSON.stringify({
|
|
5536
|
+
items: items.map((item) => ({
|
|
5537
|
+
source: item.source,
|
|
5538
|
+
title: item.title ?? null,
|
|
5539
|
+
content: item.content,
|
|
5540
|
+
content_digest: item.content_digest ?? null,
|
|
5541
|
+
format: item.format ?? "markdown",
|
|
5542
|
+
visibility: item.visibility ?? "private",
|
|
5543
|
+
metadata: item.metadata ?? null,
|
|
5544
|
+
policy: item.policy ?? options.policy ?? "version"
|
|
5545
|
+
})),
|
|
5546
|
+
checkpoint: options.checkpoint ?? null,
|
|
5547
|
+
dry_run: options.dryRun === true
|
|
5548
|
+
})));
|
|
5549
|
+
}
|
|
5550
|
+
function deriveSourceBatchId(requestDigest) {
|
|
5551
|
+
return `derived-${requestDigest.slice("sha256:".length, "sha256:".length + 32)}`;
|
|
5552
|
+
}
|
|
5553
|
+
function sourceRunRecordId(source) {
|
|
5554
|
+
return computeHash(new TextEncoder().encode([
|
|
5555
|
+
source.tenant_id ?? "default",
|
|
5556
|
+
source.archive_id ?? "public",
|
|
5557
|
+
source.namespace,
|
|
5558
|
+
source.import_run_id,
|
|
5559
|
+
""
|
|
5560
|
+
].join("\0")));
|
|
5561
|
+
}
|
|
5562
|
+
function parseReceipt(value) {
|
|
5563
|
+
const parsed = typeof value === "string" ? JSON.parse(value) : value;
|
|
5564
|
+
if (!parsed || typeof parsed !== "object") return null;
|
|
5565
|
+
return parsed;
|
|
5566
|
+
}
|
|
5567
|
+
function duplicateReceipt(receipt) {
|
|
5568
|
+
const items = receipt.items.map((item) => ({ ...item, outcome: "unchanged", reason: void 0, reason_code: void 0 }));
|
|
5569
|
+
return finish(receipt.import_run_id, receipt.batch_id, false, "duplicate", items, receipt.checkpoint);
|
|
5570
|
+
}
|
|
5571
|
+
async function insertNote(tx, input, noteId, digest2) {
|
|
4425
5572
|
const originalId = generateId();
|
|
4426
5573
|
if (input.source.archive_id) {
|
|
4427
|
-
await tx.query(
|
|
4428
|
-
`INSERT INTO archive (id, name)
|
|
4429
|
-
VALUES ($1, $2)
|
|
4430
|
-
ON CONFLICT (id) DO NOTHING`,
|
|
4431
|
-
[input.source.archive_id, input.source.archive_id]
|
|
4432
|
-
);
|
|
5574
|
+
await tx.query("INSERT INTO archive (id, name) VALUES ($1, $2) ON CONFLICT (id) DO NOTHING", [input.source.archive_id, input.source.archive_id]);
|
|
4433
5575
|
}
|
|
4434
5576
|
await tx.query(
|
|
4435
5577
|
`INSERT INTO note (id, archive_id, title, format, source, visibility)
|
|
4436
5578
|
VALUES ($1, $2, $3, $4, $5, $6)`,
|
|
4437
|
-
[
|
|
4438
|
-
noteId,
|
|
4439
|
-
input.source.archive_id ?? null,
|
|
4440
|
-
input.title ?? null,
|
|
4441
|
-
input.format ?? "markdown",
|
|
4442
|
-
`source:${input.source.namespace}`,
|
|
4443
|
-
input.visibility ?? "private"
|
|
4444
|
-
]
|
|
5579
|
+
[noteId, input.source.archive_id ?? null, input.title ?? null, input.format ?? "markdown", `source:${input.source.namespace}`, input.visibility ?? "private"]
|
|
4445
5580
|
);
|
|
5581
|
+
await tx.query("INSERT INTO note_original (id, note_id, content, content_hash) VALUES ($1, $2, $3, $4)", [originalId, noteId, input.content, digest2]);
|
|
4446
5582
|
await tx.query(
|
|
4447
|
-
`INSERT INTO
|
|
4448
|
-
VALUES ($1, $2, $3, $4)`,
|
|
4449
|
-
[
|
|
5583
|
+
`INSERT INTO note_revision (id, note_id, revision_number, type, content, ai_metadata)
|
|
5584
|
+
VALUES ($1, $2, 1, 'source-import', $3, $4::jsonb)`,
|
|
5585
|
+
[generateId(), noteId, input.content, JSON.stringify(input.metadata ?? null)]
|
|
4450
5586
|
);
|
|
4451
5587
|
await tx.query(
|
|
4452
5588
|
`INSERT INTO note_revised_current (note_id, content, ai_metadata)
|
|
@@ -4454,55 +5590,26 @@ async function insertNote(tx, input, noteId, digest) {
|
|
|
4454
5590
|
[noteId, input.content, JSON.stringify(input.metadata ?? null)]
|
|
4455
5591
|
);
|
|
4456
5592
|
}
|
|
4457
|
-
async function updateNote(tx, input, noteId, outcome) {
|
|
5593
|
+
async function updateNote(tx, input, noteId, outcome, digest2) {
|
|
4458
5594
|
if (input.source.archive_id) {
|
|
4459
|
-
await tx.query(
|
|
4460
|
-
`INSERT INTO archive (id, name)
|
|
4461
|
-
VALUES ($1, $2)
|
|
4462
|
-
ON CONFLICT (id) DO NOTHING`,
|
|
4463
|
-
[input.source.archive_id, input.source.archive_id]
|
|
4464
|
-
);
|
|
5595
|
+
await tx.query("INSERT INTO archive (id, name) VALUES ($1, $2) ON CONFLICT (id) DO NOTHING", [input.source.archive_id, input.source.archive_id]);
|
|
4465
5596
|
}
|
|
4466
5597
|
if (outcome === "versioned") {
|
|
4467
|
-
const count2 = await tx.query(
|
|
4468
|
-
|
|
4469
|
-
|
|
4470
|
-
|
|
4471
|
-
|
|
4472
|
-
`SELECT content, ai_metadata FROM note_revised_current WHERE note_id = $1`,
|
|
4473
|
-
[noteId]
|
|
5598
|
+
const count2 = await tx.query("SELECT COUNT(*) AS count FROM note_revision WHERE note_id = $1", [noteId]);
|
|
5599
|
+
await tx.query(
|
|
5600
|
+
`INSERT INTO note_revision (id, note_id, revision_number, type, content, ai_metadata)
|
|
5601
|
+
VALUES ($1, $2, $3, 'source-import', $4, $5::jsonb)`,
|
|
5602
|
+
[generateId(), noteId, Number.parseInt(count2.rows[0]?.count ?? "0", 10) + 1, input.content, JSON.stringify(input.metadata ?? null)]
|
|
4474
5603
|
);
|
|
4475
|
-
|
|
4476
|
-
|
|
4477
|
-
await tx.query(
|
|
4478
|
-
`INSERT INTO note_revision (id, note_id, revision_number, type, content, ai_metadata)
|
|
4479
|
-
VALUES ($1, $2, $3, 'source-import', $4, $5::jsonb)`,
|
|
4480
|
-
[
|
|
4481
|
-
generateId(),
|
|
4482
|
-
noteId,
|
|
4483
|
-
nextRevision,
|
|
4484
|
-
current.rows[0].content,
|
|
4485
|
-
JSON.stringify(current.rows[0].ai_metadata ?? null)
|
|
4486
|
-
]
|
|
4487
|
-
);
|
|
4488
|
-
}
|
|
5604
|
+
} else {
|
|
5605
|
+
await tx.query("UPDATE note_original SET content = $1, content_hash = $2 WHERE note_id = $3", [input.content, digest2, noteId]);
|
|
4489
5606
|
}
|
|
4490
5607
|
await tx.query(
|
|
4491
|
-
`UPDATE note
|
|
4492
|
-
|
|
4493
|
-
WHERE id = $5`,
|
|
4494
|
-
[
|
|
4495
|
-
input.title ?? null,
|
|
4496
|
-
input.format ?? "markdown",
|
|
4497
|
-
input.visibility ?? "private",
|
|
4498
|
-
input.source.archive_id ?? null,
|
|
4499
|
-
noteId
|
|
4500
|
-
]
|
|
5608
|
+
`UPDATE note SET title = $1, format = $2, visibility = $3, archive_id = $4, updated_at = now(), deleted_at = NULL WHERE id = $5`,
|
|
5609
|
+
[input.title ?? null, input.format ?? "markdown", input.visibility ?? "private", input.source.archive_id ?? null, noteId]
|
|
4501
5610
|
);
|
|
4502
5611
|
await tx.query(
|
|
4503
|
-
`UPDATE note_revised_current
|
|
4504
|
-
SET content = $1, ai_metadata = $2::jsonb, is_user_edited = false, updated_at = now()
|
|
4505
|
-
WHERE note_id = $3`,
|
|
5612
|
+
`UPDATE note_revised_current SET content = $1, ai_metadata = $2::jsonb, is_user_edited = false, updated_at = now() WHERE note_id = $3`,
|
|
4506
5613
|
[input.content, JSON.stringify(input.metadata ?? null), noteId]
|
|
4507
5614
|
);
|
|
4508
5615
|
}
|
|
@@ -4511,184 +5618,260 @@ var SourceUpsertRepository = class {
|
|
|
4511
5618
|
this.db = db;
|
|
4512
5619
|
this.events = events;
|
|
4513
5620
|
}
|
|
4514
|
-
async
|
|
4515
|
-
const
|
|
4516
|
-
|
|
4517
|
-
|
|
4518
|
-
|
|
4519
|
-
|
|
4520
|
-
|
|
4521
|
-
|
|
4522
|
-
|
|
4523
|
-
|
|
4524
|
-
|
|
4525
|
-
|
|
4526
|
-
|
|
4527
|
-
|
|
4528
|
-
|
|
4529
|
-
|
|
4530
|
-
|
|
4531
|
-
|
|
4532
|
-
|
|
4533
|
-
|
|
4534
|
-
|
|
4535
|
-
|
|
4536
|
-
|
|
4537
|
-
|
|
4538
|
-
|
|
4539
|
-
|
|
4540
|
-
|
|
4541
|
-
|
|
4542
|
-
|
|
4543
|
-
|
|
5621
|
+
async upsertRequest(request, scope = {}) {
|
|
5622
|
+
const tenant = scope.tenant_id ?? "default";
|
|
5623
|
+
const memory = scope.archive_id ?? null;
|
|
5624
|
+
const items = request.items.map((item) => ({
|
|
5625
|
+
source: {
|
|
5626
|
+
tenant_id: tenant,
|
|
5627
|
+
archive_id: memory,
|
|
5628
|
+
namespace: request.source_namespace,
|
|
5629
|
+
external_id: item.external_id,
|
|
5630
|
+
source_schema_version: request.source_schema_version,
|
|
5631
|
+
import_run_id: request.import_run_id,
|
|
5632
|
+
source_id: request.source_id,
|
|
5633
|
+
workspace_id: request.workspace_id,
|
|
5634
|
+
caller_stable_id: item.caller_stable_id
|
|
5635
|
+
},
|
|
5636
|
+
title: item.title,
|
|
5637
|
+
content: item.content,
|
|
5638
|
+
content_digest: item.content_digest,
|
|
5639
|
+
format: item.format,
|
|
5640
|
+
metadata: item.metadata,
|
|
5641
|
+
policy: item.policy ?? request.policy
|
|
5642
|
+
}));
|
|
5643
|
+
const invalidCommon = request.source_id !== void 0 && (request.source_id.length === 0 || request.source_id.length > 500) || request.workspace_id !== void 0 && (request.workspace_id.length === 0 || request.workspace_id.length > 500);
|
|
5644
|
+
if (invalidCommon) {
|
|
5645
|
+
const batchId = request.batch_id ?? deriveSourceBatchId(sourceRequestDigest(items, {}));
|
|
5646
|
+
const rejected = items.map((item, index) => ({
|
|
5647
|
+
index,
|
|
5648
|
+
outcome: "rejected",
|
|
5649
|
+
external_id_hash: sourceIdentityHash(item.source),
|
|
5650
|
+
content_digest: sourceContentDigest(item.content),
|
|
5651
|
+
reason_code: "invalid_batch_metadata"
|
|
5652
|
+
}));
|
|
5653
|
+
return contractResponse(finish(request.import_run_id, batchId, request.dry_run === true, "rejected", rejected, request.checkpoint));
|
|
4544
5654
|
}
|
|
4545
|
-
|
|
4546
|
-
|
|
5655
|
+
return contractResponse(await this.upsertBatch(items, {
|
|
5656
|
+
dryRun: request.dry_run,
|
|
5657
|
+
batchId: request.batch_id,
|
|
5658
|
+
checkpoint: request.checkpoint,
|
|
5659
|
+
policy: request.policy
|
|
5660
|
+
}));
|
|
5661
|
+
}
|
|
5662
|
+
async upsertBatch(items, options = {}) {
|
|
5663
|
+
const maxItems = options.maxItems ?? SOURCE_UPSERT_MAX_ITEMS;
|
|
5664
|
+
const importRunId = items[0]?.source.import_run_id ?? "";
|
|
5665
|
+
const requestDigest = sourceRequestDigest(items, options);
|
|
5666
|
+
const batchId = options.batchId ?? deriveSourceBatchId(requestDigest);
|
|
5667
|
+
const batchReason = batchId.length === 0 || batchId.length > 200 ? "invalid_batch_metadata" : JSON.stringify(options.checkpoint ?? {}).length > 65536 ? "checkpoint_too_large" : void 0;
|
|
5668
|
+
const validation = validateItems(items, maxItems, batchReason);
|
|
5669
|
+
if (validation) {
|
|
5670
|
+
return finish(importRunId, batchId, options.dryRun === true, "rejected", validation, options.checkpoint);
|
|
4547
5671
|
}
|
|
4548
5672
|
if (options.dryRun) {
|
|
4549
|
-
const preview =
|
|
4550
|
-
|
|
4551
|
-
|
|
4552
|
-
|
|
4553
|
-
|
|
4554
|
-
|
|
4555
|
-
|
|
4556
|
-
|
|
4557
|
-
|
|
4558
|
-
|
|
4559
|
-
|
|
4560
|
-
|
|
4561
|
-
[item.source.tenant_id ?? "default", item.source.archive_id ?? null, item.source.namespace, item.source.external_id]
|
|
4562
|
-
);
|
|
4563
|
-
if (existing.rows.length === 0) {
|
|
4564
|
-
preview.push({ index, outcome: "inserted", external_id_hash: externalIdHash, content_digest: digest });
|
|
4565
|
-
} else if (existing.rows[0].content_digest === digest) {
|
|
4566
|
-
preview.push({ index, outcome: "unchanged", note_id: existing.rows[0].note_id, external_id_hash: externalIdHash, content_digest: digest });
|
|
4567
|
-
} else if ((item.policy ?? "version") === "conflict") {
|
|
4568
|
-
preview.push({ index, outcome: "conflict", note_id: existing.rows[0].note_id, external_id_hash: externalIdHash, content_digest: digest });
|
|
4569
|
-
} else {
|
|
4570
|
-
preview.push({
|
|
5673
|
+
const preview = await previewItems(this.db, items, options.policy);
|
|
5674
|
+
return finish(importRunId, batchId, true, "preview", preview, options.checkpoint);
|
|
5675
|
+
}
|
|
5676
|
+
const response = await this.db.transaction(async (tx) => {
|
|
5677
|
+
const prior = await tx.query(
|
|
5678
|
+
`SELECT request_digest, receipt FROM source_import_batch
|
|
5679
|
+
WHERE tenant_id = $1 AND archive_id IS NOT DISTINCT FROM $2 AND namespace = $3 AND batch_id = $4 LIMIT 1`,
|
|
5680
|
+
[items[0].source.tenant_id ?? "default", items[0].source.archive_id ?? null, items[0].source.namespace, batchId]
|
|
5681
|
+
);
|
|
5682
|
+
if (prior.rows[0]) {
|
|
5683
|
+
if (prior.rows[0].request_digest !== requestDigest) {
|
|
5684
|
+
const rejected = items.map((item, index) => ({
|
|
4571
5685
|
index,
|
|
4572
|
-
outcome:
|
|
4573
|
-
|
|
4574
|
-
|
|
4575
|
-
|
|
4576
|
-
});
|
|
5686
|
+
outcome: "rejected",
|
|
5687
|
+
external_id_hash: sourceIdentityHash(item.source),
|
|
5688
|
+
content_digest: sourceContentDigest(item.content),
|
|
5689
|
+
reason_code: "batch_id_reused_with_different_request"
|
|
5690
|
+
}));
|
|
5691
|
+
return finish(importRunId, batchId, false, "rejected", rejected, options.checkpoint);
|
|
5692
|
+
}
|
|
5693
|
+
const receipt = parseReceipt(prior.rows[0].receipt);
|
|
5694
|
+
if (!receipt) throw new Error("Stored source upsert receipt is invalid");
|
|
5695
|
+
return duplicateReceipt(receipt);
|
|
5696
|
+
}
|
|
5697
|
+
for (const item of items) {
|
|
5698
|
+
if (!item.source.caller_stable_id) continue;
|
|
5699
|
+
const collision = await tx.query("SELECT id FROM note WHERE id = $1 LIMIT 1", [item.source.caller_stable_id]);
|
|
5700
|
+
if (collision.rows[0]) {
|
|
5701
|
+
const mapped = await findExisting(tx, item);
|
|
5702
|
+
if (mapped?.note_id === item.source.caller_stable_id) continue;
|
|
5703
|
+
const rejected = items.map((candidate, index) => ({
|
|
5704
|
+
index,
|
|
5705
|
+
outcome: "rejected",
|
|
5706
|
+
external_id_hash: sourceIdentityHash(candidate.source),
|
|
5707
|
+
content_digest: sourceContentDigest(candidate.content),
|
|
5708
|
+
reason_code: "caller_stable_id_conflict"
|
|
5709
|
+
}));
|
|
5710
|
+
return finish(importRunId, batchId, false, "rejected", rejected, options.checkpoint);
|
|
4577
5711
|
}
|
|
4578
5712
|
}
|
|
4579
|
-
|
|
4580
|
-
}
|
|
4581
|
-
await this.db.transaction(async (tx) => {
|
|
5713
|
+
const outcomes = [];
|
|
4582
5714
|
for (const [index, item] of items.entries()) {
|
|
4583
|
-
const externalIdHash =
|
|
4584
|
-
const
|
|
4585
|
-
const existing = await tx
|
|
4586
|
-
|
|
4587
|
-
FROM source_identity
|
|
4588
|
-
WHERE tenant_id = $1
|
|
4589
|
-
AND archive_id IS NOT DISTINCT FROM $2
|
|
4590
|
-
AND namespace = $3
|
|
4591
|
-
AND external_id = $4
|
|
4592
|
-
LIMIT 1`,
|
|
4593
|
-
[item.source.tenant_id ?? "default", item.source.archive_id ?? null, item.source.namespace, item.source.external_id]
|
|
4594
|
-
);
|
|
4595
|
-
if (existing.rows.length === 0) {
|
|
5715
|
+
const externalIdHash = sourceIdentityHash(item.source);
|
|
5716
|
+
const digest2 = sourceContentDigest(item.content);
|
|
5717
|
+
const existing = await findExisting(tx, item);
|
|
5718
|
+
if (!existing) {
|
|
4596
5719
|
const noteId = item.source.caller_stable_id ?? generateId();
|
|
4597
|
-
await insertNote(tx, item, noteId,
|
|
5720
|
+
await insertNote(tx, item, noteId, digest2);
|
|
4598
5721
|
await tx.query(
|
|
4599
5722
|
`INSERT INTO source_identity
|
|
4600
|
-
(id, tenant_id, archive_id, namespace, external_id, external_id_hash,
|
|
4601
|
-
|
|
4602
|
-
|
|
4603
|
-
[
|
|
4604
|
-
generateId(),
|
|
4605
|
-
item.source.tenant_id ?? "default",
|
|
4606
|
-
item.source.archive_id ?? null,
|
|
4607
|
-
item.source.namespace,
|
|
4608
|
-
item.source.external_id,
|
|
4609
|
-
externalIdHash,
|
|
4610
|
-
item.source.source_schema_version,
|
|
4611
|
-
digest,
|
|
4612
|
-
item.source.import_run_id,
|
|
4613
|
-
item.source.caller_stable_id ?? null,
|
|
4614
|
-
noteId
|
|
4615
|
-
]
|
|
5723
|
+
(id, tenant_id, archive_id, namespace, external_id, external_id_hash, source_id, source_schema_version, content_digest, import_run_id, caller_stable_id, note_id)
|
|
5724
|
+
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)`,
|
|
5725
|
+
[generateId(), item.source.tenant_id ?? "default", item.source.archive_id ?? null, item.source.namespace, item.source.external_id, externalIdHash, item.source.source_id ?? null, item.source.source_schema_version, digest2, item.source.import_run_id, item.source.caller_stable_id ?? null, noteId]
|
|
4616
5726
|
);
|
|
4617
|
-
outcomes
|
|
5727
|
+
outcomes.push({ index, outcome: "inserted", note_id: noteId, external_id_hash: externalIdHash, content_digest: digest2 });
|
|
4618
5728
|
continue;
|
|
4619
5729
|
}
|
|
4620
|
-
|
|
4621
|
-
|
|
4622
|
-
outcomes[index] = { index, outcome: "unchanged", note_id: row.note_id, external_id_hash: externalIdHash, content_digest: digest };
|
|
5730
|
+
if (existing.content_digest === digest2) {
|
|
5731
|
+
outcomes.push({ index, outcome: "unchanged", note_id: existing.note_id, external_id_hash: externalIdHash, content_digest: digest2 });
|
|
4623
5732
|
continue;
|
|
4624
5733
|
}
|
|
4625
|
-
const policy = item.policy ?? "version";
|
|
5734
|
+
const policy = item.policy ?? options.policy ?? "version";
|
|
4626
5735
|
if (policy === "conflict") {
|
|
4627
|
-
outcomes
|
|
5736
|
+
outcomes.push({ index, outcome: "conflict", note_id: existing.note_id, external_id_hash: externalIdHash, content_digest: digest2 });
|
|
4628
5737
|
continue;
|
|
4629
5738
|
}
|
|
4630
5739
|
const outcome = policy === "replace" ? "replaced" : "versioned";
|
|
4631
|
-
await updateNote(tx, item,
|
|
5740
|
+
await updateNote(tx, item, existing.note_id, outcome, digest2);
|
|
4632
5741
|
await tx.query(
|
|
4633
|
-
`UPDATE source_identity
|
|
4634
|
-
|
|
4635
|
-
|
|
4636
|
-
AND tenant_id = $5
|
|
4637
|
-
AND archive_id IS NOT DISTINCT FROM $6
|
|
4638
|
-
AND namespace = $7
|
|
4639
|
-
AND external_id = $8`,
|
|
4640
|
-
[
|
|
4641
|
-
item.source.source_schema_version,
|
|
4642
|
-
digest,
|
|
4643
|
-
item.source.import_run_id,
|
|
4644
|
-
row.note_id,
|
|
4645
|
-
item.source.tenant_id ?? "default",
|
|
4646
|
-
item.source.archive_id ?? null,
|
|
4647
|
-
item.source.namespace,
|
|
4648
|
-
item.source.external_id
|
|
4649
|
-
]
|
|
4650
|
-
);
|
|
4651
|
-
outcomes[index] = { index, outcome, note_id: row.note_id, external_id_hash: externalIdHash, content_digest: digest };
|
|
4652
|
-
}
|
|
4653
|
-
if (hasMaterialChange(outcomes)) {
|
|
4654
|
-
await tx.query(
|
|
4655
|
-
`INSERT INTO source_import_run (id, tenant_id, archive_id, namespace, completed_at, checkpoint, receipt)
|
|
4656
|
-
VALUES ($1, $2, $3, $4, now(), $5::jsonb, $6::jsonb)
|
|
4657
|
-
ON CONFLICT (id) DO UPDATE
|
|
4658
|
-
SET completed_at = EXCLUDED.completed_at,
|
|
4659
|
-
checkpoint = EXCLUDED.checkpoint,
|
|
4660
|
-
receipt = EXCLUDED.receipt`,
|
|
4661
|
-
[
|
|
4662
|
-
items[0].source.import_run_id,
|
|
4663
|
-
items[0].source.tenant_id ?? "default",
|
|
4664
|
-
items[0].source.archive_id ?? null,
|
|
4665
|
-
items[0].source.namespace,
|
|
4666
|
-
JSON.stringify({ item_count: items.length }),
|
|
4667
|
-
JSON.stringify({ counts: countOutcomes(outcomes) })
|
|
4668
|
-
]
|
|
5742
|
+
`UPDATE source_identity SET source_id = $1, source_schema_version = $2, content_digest = $3, import_run_id = $4, updated_at = now()
|
|
5743
|
+
WHERE note_id = $5 AND tenant_id = $6 AND archive_id IS NOT DISTINCT FROM $7 AND namespace = $8 AND external_id = $9`,
|
|
5744
|
+
[item.source.source_id ?? null, item.source.source_schema_version, digest2, item.source.import_run_id, existing.note_id, item.source.tenant_id ?? "default", item.source.archive_id ?? null, item.source.namespace, item.source.external_id]
|
|
4669
5745
|
);
|
|
5746
|
+
outcomes.push({ index, outcome, note_id: existing.note_id, external_id_hash: externalIdHash, content_digest: digest2 });
|
|
4670
5747
|
}
|
|
5748
|
+
const committed = finish(importRunId, batchId, false, "committed", outcomes, options.checkpoint);
|
|
5749
|
+
await tx.query(
|
|
5750
|
+
`INSERT INTO source_import_run (id, tenant_id, archive_id, namespace, external_run_id, source_id, source_schema_version, workspace_id, completed_at, checkpoint, receipt)
|
|
5751
|
+
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, now(), $9::jsonb, $10::jsonb)
|
|
5752
|
+
ON CONFLICT (id) DO UPDATE SET source_id = EXCLUDED.source_id, source_schema_version = EXCLUDED.source_schema_version, workspace_id = EXCLUDED.workspace_id, completed_at = EXCLUDED.completed_at, checkpoint = EXCLUDED.checkpoint, receipt = EXCLUDED.receipt`,
|
|
5753
|
+
[sourceRunRecordId(items[0].source), items[0].source.tenant_id ?? "default", items[0].source.archive_id ?? null, items[0].source.namespace, importRunId, items[0].source.source_id ?? null, items[0].source.source_schema_version, items[0].source.workspace_id ?? null, JSON.stringify(options.checkpoint ?? {}), JSON.stringify(redactedReceipt(committed))]
|
|
5754
|
+
);
|
|
5755
|
+
await tx.query(
|
|
5756
|
+
`INSERT INTO source_import_batch (id, tenant_id, archive_id, namespace, batch_id, request_digest, import_run_id, outcome, checkpoint, receipt)
|
|
5757
|
+
VALUES ($1, $2, $3, $4, $5, $6, $7, 'committed', $8::jsonb, $9::jsonb)`,
|
|
5758
|
+
[generateId(), items[0].source.tenant_id ?? "default", items[0].source.archive_id ?? null, items[0].source.namespace, batchId, requestDigest, importRunId, JSON.stringify(options.checkpoint ?? {}), JSON.stringify(redactedReceipt(committed))]
|
|
5759
|
+
);
|
|
5760
|
+
return committed;
|
|
4671
5761
|
});
|
|
4672
|
-
if (hasMaterialChange(
|
|
4673
|
-
this.events?.emit("source.upserted", { importRunId
|
|
5762
|
+
if (response.outcome === "committed" && hasMaterialChange(response.items)) {
|
|
5763
|
+
this.events?.emit("source.upserted", { importRunId, counts: response.counts });
|
|
4674
5764
|
}
|
|
4675
|
-
return
|
|
4676
|
-
}
|
|
4677
|
-
finish(importRunId, dryRun, outcomes) {
|
|
4678
|
-
return { import_run_id: importRunId, dry_run: dryRun, outcomes, counts: countOutcomes(outcomes) };
|
|
5765
|
+
return response;
|
|
4679
5766
|
}
|
|
4680
5767
|
};
|
|
5768
|
+
async function findExisting(tx, item) {
|
|
5769
|
+
const result = await tx.query(
|
|
5770
|
+
`SELECT note_id, content_digest FROM source_identity
|
|
5771
|
+
WHERE tenant_id = $1 AND archive_id IS NOT DISTINCT FROM $2 AND namespace = $3 AND external_id = $4 LIMIT 1`,
|
|
5772
|
+
[item.source.tenant_id ?? "default", item.source.archive_id ?? null, item.source.namespace, item.source.external_id]
|
|
5773
|
+
);
|
|
5774
|
+
return result.rows[0] ?? null;
|
|
5775
|
+
}
|
|
5776
|
+
async function previewItems(db, items, batchPolicy) {
|
|
5777
|
+
const preview = [];
|
|
5778
|
+
for (const [index, item] of items.entries()) {
|
|
5779
|
+
const external_id_hash = sourceIdentityHash(item.source);
|
|
5780
|
+
const content_digest = sourceContentDigest(item.content);
|
|
5781
|
+
const existing = await findExisting(db, item);
|
|
5782
|
+
if (!existing) preview.push({ index, outcome: "inserted", external_id_hash, content_digest });
|
|
5783
|
+
else if (existing.content_digest === content_digest) preview.push({ index, outcome: "unchanged", note_id: existing.note_id, external_id_hash, content_digest });
|
|
5784
|
+
else if ((item.policy ?? batchPolicy ?? "version") === "conflict") preview.push({ index, outcome: "conflict", note_id: existing.note_id, external_id_hash, content_digest });
|
|
5785
|
+
else preview.push({ index, outcome: (item.policy ?? batchPolicy) === "replace" ? "replaced" : "versioned", note_id: existing.note_id, external_id_hash, content_digest });
|
|
5786
|
+
}
|
|
5787
|
+
return preview;
|
|
5788
|
+
}
|
|
5789
|
+
function validateItems(items, maxItems, initialReason) {
|
|
5790
|
+
const seen = /* @__PURE__ */ new Set();
|
|
5791
|
+
const stableIds = /* @__PURE__ */ new Set();
|
|
5792
|
+
const rejected = [];
|
|
5793
|
+
let batchReason = initialReason ?? (items.length === 0 || items.length > maxItems ? "batch_size_out_of_bounds" : null);
|
|
5794
|
+
for (const [index, item] of items.entries()) {
|
|
5795
|
+
const digest2 = sourceContentDigest(item.content ?? "");
|
|
5796
|
+
let reason = batchReason;
|
|
5797
|
+
try {
|
|
5798
|
+
assertSource(item.source);
|
|
5799
|
+
if (!item.content || item.content.length > 4194304) reason ??= "invalid_item";
|
|
5800
|
+
if ((item.format ?? "markdown").length > 100 || (item.title?.length ?? 0) > 2e3 || JSON.stringify(item.metadata ?? {}).length > 262144) reason ??= "invalid_item";
|
|
5801
|
+
if (item.content_digest && item.content_digest !== digest2) reason ??= "content_digest_mismatch";
|
|
5802
|
+
const identity = `${item.source.tenant_id ?? "default"}\0${item.source.archive_id ?? ""}\0${item.source.namespace}\0${item.source.external_id}`;
|
|
5803
|
+
if (seen.has(identity)) reason ??= "duplicate_external_id_in_batch";
|
|
5804
|
+
seen.add(identity);
|
|
5805
|
+
if (item.source.caller_stable_id && stableIds.has(item.source.caller_stable_id)) reason ??= "caller_stable_id_conflict";
|
|
5806
|
+
if (item.source.caller_stable_id) stableIds.add(item.source.caller_stable_id);
|
|
5807
|
+
} catch (error) {
|
|
5808
|
+
const code = error instanceof Error ? error.message : "invalid_item";
|
|
5809
|
+
reason ??= code === "invalid_batch_metadata" ? code : "invalid_item";
|
|
5810
|
+
}
|
|
5811
|
+
if (reason) {
|
|
5812
|
+
batchReason = reason;
|
|
5813
|
+
rejected.push({ index, outcome: "rejected", external_id_hash: sourceIdentityHash(item.source), content_digest: digest2, reason_code: reason });
|
|
5814
|
+
}
|
|
5815
|
+
}
|
|
5816
|
+
if (!batchReason) return null;
|
|
5817
|
+
if (rejected.length === items.length) return rejected;
|
|
5818
|
+
return items.map((item, index) => rejected.find((result) => result.index === index) ?? {
|
|
5819
|
+
index,
|
|
5820
|
+
outcome: "rejected",
|
|
5821
|
+
external_id_hash: sourceIdentityHash(item.source),
|
|
5822
|
+
content_digest: sourceContentDigest(item.content),
|
|
5823
|
+
reason_code: batchReason
|
|
5824
|
+
});
|
|
5825
|
+
}
|
|
5826
|
+
function finish(importRunId, batchId, dryRun, outcome, items, checkpoint) {
|
|
5827
|
+
return {
|
|
5828
|
+
contract_version: SOURCE_UPSERT_CONTRACT_VERSION,
|
|
5829
|
+
import_run_id: importRunId,
|
|
5830
|
+
batch_id: batchId,
|
|
5831
|
+
dry_run: dryRun,
|
|
5832
|
+
outcome,
|
|
5833
|
+
...checkpoint ? { checkpoint } : {},
|
|
5834
|
+
items,
|
|
5835
|
+
outcomes: items,
|
|
5836
|
+
counts: countOutcomes(items)
|
|
5837
|
+
};
|
|
5838
|
+
}
|
|
5839
|
+
function redactedReceipt(receipt) {
|
|
5840
|
+
return { ...contractResponse(receipt), items: receipt.items.map(sanitizeItem) };
|
|
5841
|
+
}
|
|
5842
|
+
function contractResponse(receipt) {
|
|
5843
|
+
return {
|
|
5844
|
+
contract_version: receipt.contract_version,
|
|
5845
|
+
import_run_id: receipt.import_run_id,
|
|
5846
|
+
batch_id: receipt.batch_id,
|
|
5847
|
+
dry_run: receipt.dry_run,
|
|
5848
|
+
outcome: receipt.outcome,
|
|
5849
|
+
...receipt.checkpoint ? { checkpoint: receipt.checkpoint } : {},
|
|
5850
|
+
counts: receipt.counts,
|
|
5851
|
+
items: receipt.items
|
|
5852
|
+
};
|
|
5853
|
+
}
|
|
5854
|
+
function sanitizeItem(item) {
|
|
5855
|
+
return {
|
|
5856
|
+
index: item.index,
|
|
5857
|
+
outcome: item.outcome,
|
|
5858
|
+
...item.note_id ? { note_id: item.note_id } : {},
|
|
5859
|
+
external_id_hash: item.external_id_hash,
|
|
5860
|
+
content_digest: item.content_digest,
|
|
5861
|
+
...item.reason_code ? { reason_code: item.reason_code } : {}
|
|
5862
|
+
};
|
|
5863
|
+
}
|
|
4681
5864
|
function hasMaterialChange(outcomes) {
|
|
4682
|
-
return outcomes.some((
|
|
5865
|
+
return outcomes.some((item) => item.outcome === "inserted" || item.outcome === "versioned" || item.outcome === "replaced");
|
|
4683
5866
|
}
|
|
4684
5867
|
function countOutcomes(outcomes) {
|
|
4685
5868
|
return {
|
|
4686
|
-
inserted: outcomes.filter((
|
|
4687
|
-
unchanged: outcomes.filter((
|
|
4688
|
-
versioned: outcomes.filter((
|
|
4689
|
-
replaced: outcomes.filter((
|
|
4690
|
-
conflict: outcomes.filter((
|
|
4691
|
-
rejected: outcomes.filter((
|
|
5869
|
+
inserted: outcomes.filter((item) => item.outcome === "inserted").length,
|
|
5870
|
+
unchanged: outcomes.filter((item) => item.outcome === "unchanged").length,
|
|
5871
|
+
versioned: outcomes.filter((item) => item.outcome === "versioned").length,
|
|
5872
|
+
replaced: outcomes.filter((item) => item.outcome === "replaced").length,
|
|
5873
|
+
conflict: outcomes.filter((item) => item.outcome === "conflict").length,
|
|
5874
|
+
rejected: outcomes.filter((item) => item.outcome === "rejected").length
|
|
4692
5875
|
};
|
|
4693
5876
|
}
|
|
4694
5877
|
|
|
@@ -5591,7 +6774,7 @@ function titleGenerationHandler(job, db) {
|
|
|
5591
6774
|
|
|
5592
6775
|
Note content:
|
|
5593
6776
|
${content.slice(0, 1e3)}`;
|
|
5594
|
-
const llmTitle = (await llmFn2(prompt, { maxTokens: 60, temperature: 0.3 })).trim();
|
|
6777
|
+
const llmTitle = (await llmFn2(prompt, { maxTokens: 60, temperature: 0.3, task: "chat.general" })).trim();
|
|
5595
6778
|
if (llmTitle) {
|
|
5596
6779
|
const title2 = llmTitle.length > 200 ? llmTitle.slice(0, 197) + "..." : llmTitle;
|
|
5597
6780
|
await db.query(
|
|
@@ -5632,7 +6815,7 @@ Respond with ONLY the enhanced note content, no explanation.
|
|
|
5632
6815
|
|
|
5633
6816
|
Original note:
|
|
5634
6817
|
${content}`;
|
|
5635
|
-
const revised = (await llmFn2(prompt, { maxTokens: 2e3, temperature: 0.4 })).trim();
|
|
6818
|
+
const revised = (await llmFn2(prompt, { maxTokens: 2e3, temperature: 0.4, task: "chat.revision" })).trim();
|
|
5636
6819
|
if (!revised || revised === content) return { skipped: true, reason: "no changes from LLM" };
|
|
5637
6820
|
const revResult = await db.query(
|
|
5638
6821
|
`SELECT COALESCE(MAX(revision_number), 0) as max_rev FROM note_revision WHERE note_id = $1`,
|
|
@@ -5673,7 +6856,7 @@ Text:
|
|
|
5673
6856
|
${content.slice(0, 1500)}
|
|
5674
6857
|
|
|
5675
6858
|
Tags:`;
|
|
5676
|
-
const response = (await llmFn2(prompt, { maxTokens: 60, temperature: 0.1 })).trim();
|
|
6859
|
+
const response = (await llmFn2(prompt, { maxTokens: 60, temperature: 0.1, task: "chat.tagging" })).trim();
|
|
5677
6860
|
const tags = response.split(/[,\n]/).map((t) => t.trim().toLowerCase().replace(/^[-*\d.]+\s*/, "").replace(/['"]/g, "")).filter((t) => {
|
|
5678
6861
|
if (t.length < 2 || t.length > 40) return false;
|
|
5679
6862
|
if (/^\d+$/.test(t)) return false;
|
|
@@ -6956,10 +8139,24 @@ function chunkText(text, maxChars = 800, overlap = 100) {
|
|
|
6956
8139
|
}
|
|
6957
8140
|
|
|
6958
8141
|
// src/capabilities/embedding-handler.ts
|
|
8142
|
+
var DEFAULT_LARGE_DOCUMENT_CHARS = 12e3;
|
|
8143
|
+
var DEFAULT_LARGE_DOCUMENT_CHUNKS = 12;
|
|
6959
8144
|
var embedFn = null;
|
|
8145
|
+
var embeddingTaskSelectionOptions = {};
|
|
6960
8146
|
function setEmbedFunction(fn) {
|
|
6961
8147
|
embedFn = fn;
|
|
6962
8148
|
}
|
|
8149
|
+
function setEmbeddingTaskSelectionOptions(options = {}) {
|
|
8150
|
+
embeddingTaskSelectionOptions = { ...options };
|
|
8151
|
+
}
|
|
8152
|
+
function getEmbeddingTaskSelectionOptions() {
|
|
8153
|
+
return { ...embeddingTaskSelectionOptions };
|
|
8154
|
+
}
|
|
8155
|
+
function selectEmbeddingTask(content, chunks, options = {}) {
|
|
8156
|
+
const largeDocumentChars = options.largeDocumentChars ?? DEFAULT_LARGE_DOCUMENT_CHARS;
|
|
8157
|
+
const largeDocumentChunks = options.largeDocumentChunks ?? DEFAULT_LARGE_DOCUMENT_CHUNKS;
|
|
8158
|
+
return content.length >= largeDocumentChars || chunks.length >= largeDocumentChunks ? "embedding.large-document" : "embedding.document";
|
|
8159
|
+
}
|
|
6963
8160
|
function getEmbedFunction() {
|
|
6964
8161
|
return embedFn;
|
|
6965
8162
|
}
|
|
@@ -6985,7 +8182,8 @@ async function embeddingGenerationHandler(job, db) {
|
|
|
6985
8182
|
if (!noteText) return { skipped: true, reason: "note missing, deleted, or has no content" };
|
|
6986
8183
|
const content = noteText.combined;
|
|
6987
8184
|
const chunks = chunkText(content);
|
|
6988
|
-
const
|
|
8185
|
+
const task = selectEmbeddingTask(content, chunks, embeddingTaskSelectionOptions);
|
|
8186
|
+
const embeddings = await fn(chunks, { task });
|
|
6989
8187
|
const vector = averageEmbeddings(embeddings);
|
|
6990
8188
|
const embeddingSets = new EmbeddingSetsRepository(db);
|
|
6991
8189
|
const set = await embeddingSets.ensureDefault();
|
|
@@ -6994,7 +8192,7 @@ async function embeddingGenerationHandler(job, db) {
|
|
|
6994
8192
|
embedding_set_id: set.id,
|
|
6995
8193
|
vector
|
|
6996
8194
|
});
|
|
6997
|
-
return { chunks: chunks.length, embeddings: embeddings.length, setId: set.id };
|
|
8195
|
+
return { chunks: chunks.length, embeddings: embeddings.length, setId: set.id, task };
|
|
6998
8196
|
}
|
|
6999
8197
|
|
|
7000
8198
|
// src/capabilities/auto-tag.ts
|
|
@@ -7153,6 +8351,226 @@ function unregisterLlmCapability() {
|
|
|
7153
8351
|
setLlmFunction(null);
|
|
7154
8352
|
}
|
|
7155
8353
|
|
|
8354
|
+
// src/capabilities/fallback-router.ts
|
|
8355
|
+
var DEFAULT_COOLDOWNS = {
|
|
8356
|
+
rateLimit: 3e4,
|
|
8357
|
+
serverError: 6e4,
|
|
8358
|
+
connectionFailure: 3e5,
|
|
8359
|
+
contentPolicy: 0
|
|
8360
|
+
};
|
|
8361
|
+
function classifyError(error) {
|
|
8362
|
+
const msg = error instanceof Error ? error.message : String(error);
|
|
8363
|
+
const lower = msg.toLowerCase();
|
|
8364
|
+
if (lower.includes("429") || lower.includes("rate limit")) return "rate_limit";
|
|
8365
|
+
if (lower.includes("500") || lower.includes("502") || lower.includes("503") || lower.includes("504")) return "server_error";
|
|
8366
|
+
if (lower.includes("content") && (lower.includes("policy") || lower.includes("filter"))) return "content_policy";
|
|
8367
|
+
if (lower.includes("context") && (lower.includes("window") || lower.includes("length") || lower.includes("too long"))) return "context_window";
|
|
8368
|
+
if (lower.includes("fetch") || lower.includes("network") || lower.includes("connection") || lower.includes("econnrefused") || lower.includes("timeout")) return "connection_failure";
|
|
8369
|
+
return "unknown";
|
|
8370
|
+
}
|
|
8371
|
+
var FallbackRouter = class {
|
|
8372
|
+
id = "fallback-router";
|
|
8373
|
+
name = "Fallback Router";
|
|
8374
|
+
tier = "remote";
|
|
8375
|
+
providers;
|
|
8376
|
+
cooldowns;
|
|
8377
|
+
cooldownMap = /* @__PURE__ */ new Map();
|
|
8378
|
+
events;
|
|
8379
|
+
get capabilities() {
|
|
8380
|
+
const available = this.getAvailableProviders();
|
|
8381
|
+
return {
|
|
8382
|
+
embeddings: available.some((p) => p.capabilities.embeddings),
|
|
8383
|
+
chat: available.some((p) => p.capabilities.chat),
|
|
8384
|
+
streaming: available.some((p) => p.capabilities.streaming),
|
|
8385
|
+
vision: available.some((p) => p.capabilities.vision),
|
|
8386
|
+
toolCalling: available.some((p) => p.capabilities.toolCalling),
|
|
8387
|
+
structuredOutput: available.some((p) => p.capabilities.structuredOutput),
|
|
8388
|
+
maxContextTokens: Math.max(
|
|
8389
|
+
...available.map((p) => p.capabilities.maxContextTokens ?? 0),
|
|
8390
|
+
0
|
|
8391
|
+
)
|
|
8392
|
+
};
|
|
8393
|
+
}
|
|
8394
|
+
constructor(config) {
|
|
8395
|
+
this.providers = [...config.providers];
|
|
8396
|
+
this.cooldowns = { ...DEFAULT_COOLDOWNS, ...config.cooldowns };
|
|
8397
|
+
this.events = config.events;
|
|
8398
|
+
}
|
|
8399
|
+
// -------------------------------------------------------------------------
|
|
8400
|
+
// Provider management
|
|
8401
|
+
// -------------------------------------------------------------------------
|
|
8402
|
+
/** Get providers not currently in cooldown */
|
|
8403
|
+
getAvailableProviders() {
|
|
8404
|
+
const now2 = Date.now();
|
|
8405
|
+
return this.providers.filter((p) => {
|
|
8406
|
+
const cd = this.cooldownMap.get(p.id);
|
|
8407
|
+
if (!cd) return true;
|
|
8408
|
+
if (now2 >= cd.expiresAt) {
|
|
8409
|
+
this.cooldownMap.delete(p.id);
|
|
8410
|
+
return true;
|
|
8411
|
+
}
|
|
8412
|
+
return false;
|
|
8413
|
+
});
|
|
8414
|
+
}
|
|
8415
|
+
/** Get providers in cooldown with their expiry info */
|
|
8416
|
+
getCoolingDown() {
|
|
8417
|
+
const now2 = Date.now();
|
|
8418
|
+
const result = [];
|
|
8419
|
+
for (const [id, entry] of this.cooldownMap) {
|
|
8420
|
+
if (now2 < entry.expiresAt) {
|
|
8421
|
+
result.push({ providerId: id, category: entry.category, expiresAt: entry.expiresAt });
|
|
8422
|
+
}
|
|
8423
|
+
}
|
|
8424
|
+
return result;
|
|
8425
|
+
}
|
|
8426
|
+
/** Manually clear cooldown for a provider */
|
|
8427
|
+
clearCooldown(providerId) {
|
|
8428
|
+
this.cooldownMap.delete(providerId);
|
|
8429
|
+
}
|
|
8430
|
+
/** Clear all cooldowns */
|
|
8431
|
+
clearAllCooldowns() {
|
|
8432
|
+
this.cooldownMap.clear();
|
|
8433
|
+
}
|
|
8434
|
+
/** Add a provider to the chain (appended at lowest priority) */
|
|
8435
|
+
addProvider(provider) {
|
|
8436
|
+
this.providers.push(provider);
|
|
8437
|
+
}
|
|
8438
|
+
/** Remove a provider from the chain */
|
|
8439
|
+
removeProvider(id) {
|
|
8440
|
+
this.providers = this.providers.filter((p) => p.id !== id);
|
|
8441
|
+
this.cooldownMap.delete(id);
|
|
8442
|
+
}
|
|
8443
|
+
/** Reorder providers (new priority order) */
|
|
8444
|
+
setOrder(ids) {
|
|
8445
|
+
const byId = new Map(this.providers.map((p) => [p.id, p]));
|
|
8446
|
+
const reordered = [];
|
|
8447
|
+
for (const id of ids) {
|
|
8448
|
+
const p = byId.get(id);
|
|
8449
|
+
if (p) reordered.push(p);
|
|
8450
|
+
}
|
|
8451
|
+
for (const p of this.providers) {
|
|
8452
|
+
if (!ids.includes(p.id)) reordered.push(p);
|
|
8453
|
+
}
|
|
8454
|
+
this.providers = reordered;
|
|
8455
|
+
}
|
|
8456
|
+
// -------------------------------------------------------------------------
|
|
8457
|
+
// InferenceProvider interface — with fallback
|
|
8458
|
+
// -------------------------------------------------------------------------
|
|
8459
|
+
async embed(request) {
|
|
8460
|
+
return this.withFallback(
|
|
8461
|
+
(p) => p.capabilities.embeddings && !!p.embed,
|
|
8462
|
+
(p) => p.embed(request)
|
|
8463
|
+
);
|
|
8464
|
+
}
|
|
8465
|
+
async complete(request) {
|
|
8466
|
+
return this.withFallback(
|
|
8467
|
+
(p) => p.capabilities.chat && !!p.complete,
|
|
8468
|
+
(p) => p.complete(request)
|
|
8469
|
+
);
|
|
8470
|
+
}
|
|
8471
|
+
async *stream(request) {
|
|
8472
|
+
const candidates = this.getAvailableProviders().filter((p) => p.capabilities.streaming && p.stream);
|
|
8473
|
+
if (candidates.length === 0) {
|
|
8474
|
+
throw new Error("No available providers with streaming capability");
|
|
8475
|
+
}
|
|
8476
|
+
yield* candidates[0].stream(request);
|
|
8477
|
+
}
|
|
8478
|
+
async listModels() {
|
|
8479
|
+
const available = this.getAvailableProviders();
|
|
8480
|
+
const results = await Promise.allSettled(
|
|
8481
|
+
available.map((p) => p.listModels())
|
|
8482
|
+
);
|
|
8483
|
+
const models = [];
|
|
8484
|
+
for (const r of results) {
|
|
8485
|
+
if (r.status === "fulfilled") models.push(...r.value);
|
|
8486
|
+
}
|
|
8487
|
+
return models;
|
|
8488
|
+
}
|
|
8489
|
+
async probe() {
|
|
8490
|
+
const available = this.getAvailableProviders();
|
|
8491
|
+
if (available.length === 0) {
|
|
8492
|
+
return { status: "down", latencyMs: 0, message: "All providers in cooldown" };
|
|
8493
|
+
}
|
|
8494
|
+
const start = Date.now();
|
|
8495
|
+
const results = await Promise.allSettled(
|
|
8496
|
+
available.map((p) => p.probe())
|
|
8497
|
+
);
|
|
8498
|
+
const okCount = results.filter(
|
|
8499
|
+
(r) => r.status === "fulfilled" && r.value.status === "ok"
|
|
8500
|
+
).length;
|
|
8501
|
+
return {
|
|
8502
|
+
status: okCount === available.length ? "ok" : okCount > 0 ? "degraded" : "down",
|
|
8503
|
+
latencyMs: Date.now() - start,
|
|
8504
|
+
message: `${okCount}/${available.length} providers healthy`
|
|
8505
|
+
};
|
|
8506
|
+
}
|
|
8507
|
+
dispose() {
|
|
8508
|
+
for (const p of this.providers) {
|
|
8509
|
+
p.dispose();
|
|
8510
|
+
}
|
|
8511
|
+
this.providers = [];
|
|
8512
|
+
this.cooldownMap.clear();
|
|
8513
|
+
}
|
|
8514
|
+
// -------------------------------------------------------------------------
|
|
8515
|
+
// Core fallback logic
|
|
8516
|
+
// -------------------------------------------------------------------------
|
|
8517
|
+
async withFallback(filter, execute) {
|
|
8518
|
+
const candidates = this.getAvailableProviders().filter(filter);
|
|
8519
|
+
if (candidates.length === 0) {
|
|
8520
|
+
throw new Error("No available providers for this request");
|
|
8521
|
+
}
|
|
8522
|
+
let lastError;
|
|
8523
|
+
for (const provider of candidates) {
|
|
8524
|
+
try {
|
|
8525
|
+
return await execute(provider);
|
|
8526
|
+
} catch (err) {
|
|
8527
|
+
lastError = err instanceof Error ? err : new Error(String(err));
|
|
8528
|
+
const category = classifyError(err);
|
|
8529
|
+
this.applyCooldown(provider.id, category);
|
|
8530
|
+
const nextCandidate = candidates[candidates.indexOf(provider) + 1];
|
|
8531
|
+
if (nextCandidate) {
|
|
8532
|
+
this.events?.emit("provider.fallback", {
|
|
8533
|
+
fromProvider: provider.id,
|
|
8534
|
+
toProvider: nextCandidate.id,
|
|
8535
|
+
errorCategory: category,
|
|
8536
|
+
error: lastError.message
|
|
8537
|
+
});
|
|
8538
|
+
}
|
|
8539
|
+
}
|
|
8540
|
+
}
|
|
8541
|
+
throw lastError ?? new Error("All providers failed");
|
|
8542
|
+
}
|
|
8543
|
+
applyCooldown(providerId, category) {
|
|
8544
|
+
let cooldownMs;
|
|
8545
|
+
switch (category) {
|
|
8546
|
+
case "rate_limit":
|
|
8547
|
+
cooldownMs = this.cooldowns.rateLimit;
|
|
8548
|
+
break;
|
|
8549
|
+
case "server_error":
|
|
8550
|
+
cooldownMs = this.cooldowns.serverError;
|
|
8551
|
+
break;
|
|
8552
|
+
case "connection_failure":
|
|
8553
|
+
cooldownMs = this.cooldowns.connectionFailure;
|
|
8554
|
+
break;
|
|
8555
|
+
case "content_policy":
|
|
8556
|
+
cooldownMs = this.cooldowns.contentPolicy;
|
|
8557
|
+
break;
|
|
8558
|
+
default:
|
|
8559
|
+
cooldownMs = this.cooldowns.serverError;
|
|
8560
|
+
}
|
|
8561
|
+
if (cooldownMs > 0) {
|
|
8562
|
+
const expiresAt = Date.now() + cooldownMs;
|
|
8563
|
+
this.cooldownMap.set(providerId, { expiresAt, category });
|
|
8564
|
+
this.events?.emit("provider.cooldown", {
|
|
8565
|
+
providerId,
|
|
8566
|
+
errorCategory: category,
|
|
8567
|
+
cooldownMs,
|
|
8568
|
+
expiresAt
|
|
8569
|
+
});
|
|
8570
|
+
}
|
|
8571
|
+
}
|
|
8572
|
+
};
|
|
8573
|
+
|
|
7156
8574
|
// src/capabilities/provider-registry.ts
|
|
7157
8575
|
var ProviderRegistry = class {
|
|
7158
8576
|
constructor(events) {
|
|
@@ -7160,6 +8578,7 @@ var ProviderRegistry = class {
|
|
|
7160
8578
|
}
|
|
7161
8579
|
providers = /* @__PURE__ */ new Map();
|
|
7162
8580
|
activeId = null;
|
|
8581
|
+
routes = /* @__PURE__ */ new Map();
|
|
7163
8582
|
/** Register a provider. First provider with embedding capability becomes active. */
|
|
7164
8583
|
add(provider) {
|
|
7165
8584
|
if (this.providers.has(provider.id)) {
|
|
@@ -7197,6 +8616,36 @@ var ProviderRegistry = class {
|
|
|
7197
8616
|
this.syncLegacyFunctions();
|
|
7198
8617
|
this.events?.emit("provider.active", { id, name: provider.name });
|
|
7199
8618
|
}
|
|
8619
|
+
setRoute(task, policy) {
|
|
8620
|
+
const cloned = cloneProviderRoutePolicy(policy);
|
|
8621
|
+
this.routes.set(task, cloned);
|
|
8622
|
+
this.events?.emit("provider.route.configured", {
|
|
8623
|
+
task,
|
|
8624
|
+
providerIds: cloned.providerIds ?? [],
|
|
8625
|
+
model: cloned.model,
|
|
8626
|
+
fallback: cloned.fallback,
|
|
8627
|
+
hasRequirements: Boolean(cloned.requirements)
|
|
8628
|
+
});
|
|
8629
|
+
}
|
|
8630
|
+
getRoute(task) {
|
|
8631
|
+
const route = this.routes.get(task);
|
|
8632
|
+
return route ? cloneProviderRoutePolicy(route) : void 0;
|
|
8633
|
+
}
|
|
8634
|
+
clearRoute(task) {
|
|
8635
|
+
this.routes.delete(task);
|
|
8636
|
+
this.events?.emit("provider.route.cleared", { task });
|
|
8637
|
+
}
|
|
8638
|
+
clearRoutes() {
|
|
8639
|
+
const tasks = Array.from(this.routes.keys());
|
|
8640
|
+
this.routes.clear();
|
|
8641
|
+
if (tasks.length === 0) {
|
|
8642
|
+
this.events?.emit("provider.route.cleared", {});
|
|
8643
|
+
return;
|
|
8644
|
+
}
|
|
8645
|
+
for (const task of tasks) {
|
|
8646
|
+
this.events?.emit("provider.route.cleared", { task });
|
|
8647
|
+
}
|
|
8648
|
+
}
|
|
7200
8649
|
/** Get the currently active provider */
|
|
7201
8650
|
getActive() {
|
|
7202
8651
|
if (!this.activeId) return null;
|
|
@@ -7228,27 +8677,51 @@ var ProviderRegistry = class {
|
|
|
7228
8677
|
}
|
|
7229
8678
|
/** Convenience: embed using active provider */
|
|
7230
8679
|
async embed(request) {
|
|
7231
|
-
const
|
|
7232
|
-
|
|
7233
|
-
throw new Error("No
|
|
7234
|
-
|
|
7235
|
-
|
|
8680
|
+
const route = request.task ? this.routes.get(request.task) : void 0;
|
|
8681
|
+
return this.withRouteFallback(request.task, "embeddings", request.model, async (selection) => {
|
|
8682
|
+
if (!selection.provider.embed) throw new Error("No routed provider with embedding capability");
|
|
8683
|
+
return selection.provider.embed(withResolvedModel(request, selection.model));
|
|
8684
|
+
}, route?.fallback !== false);
|
|
7236
8685
|
}
|
|
7237
8686
|
/** Convenience: complete using active provider */
|
|
7238
8687
|
async complete(request) {
|
|
7239
|
-
const
|
|
7240
|
-
|
|
7241
|
-
throw new Error("No
|
|
7242
|
-
|
|
7243
|
-
|
|
8688
|
+
const route = request.task ? this.routes.get(request.task) : void 0;
|
|
8689
|
+
return this.withRouteFallback(request.task, "chat", request.model, async (selection) => {
|
|
8690
|
+
if (!selection.provider.complete) throw new Error("No routed provider with chat capability");
|
|
8691
|
+
return selection.provider.complete(withResolvedModel(request, selection.model));
|
|
8692
|
+
}, route?.fallback !== false);
|
|
7244
8693
|
}
|
|
7245
8694
|
/** Convenience: stream using active provider */
|
|
7246
8695
|
stream(request) {
|
|
7247
|
-
const provider = this.
|
|
7248
|
-
if (!provider
|
|
7249
|
-
|
|
7250
|
-
|
|
7251
|
-
|
|
8696
|
+
const { provider, model } = this.resolveProvider(request.task, "streaming", request.model, true);
|
|
8697
|
+
if (!provider.stream) throw new Error("No routed provider with streaming capability");
|
|
8698
|
+
return provider.stream(withResolvedModel(request, model));
|
|
8699
|
+
}
|
|
8700
|
+
previewRoute(task, capability, requestModel) {
|
|
8701
|
+
const { provider, model, routeMatched } = this.resolveProvider(task, capability, requestModel, false);
|
|
8702
|
+
return {
|
|
8703
|
+
provider,
|
|
8704
|
+
providerId: provider.id,
|
|
8705
|
+
providerName: provider.name,
|
|
8706
|
+
tier: provider.tier,
|
|
8707
|
+
capability,
|
|
8708
|
+
task,
|
|
8709
|
+
model,
|
|
8710
|
+
routeMatched
|
|
8711
|
+
};
|
|
8712
|
+
}
|
|
8713
|
+
async probeRoute(task, capability, requestModel) {
|
|
8714
|
+
const selection = this.previewRoute(task, capability, requestModel);
|
|
8715
|
+
const probe = await selection.provider.probe();
|
|
8716
|
+
return { ...selection, probe };
|
|
8717
|
+
}
|
|
8718
|
+
validateRoute(task, capability = inferInferenceTaskCapability(task), policy = this.routes.get(task)) {
|
|
8719
|
+
return validateProviderRoute(task, capability, policy, this.list());
|
|
8720
|
+
}
|
|
8721
|
+
validateRoutes() {
|
|
8722
|
+
return Array.from(this.routes.entries()).map(
|
|
8723
|
+
([task, policy]) => this.validateRoute(task, inferInferenceTaskCapability(task), policy)
|
|
8724
|
+
);
|
|
7252
8725
|
}
|
|
7253
8726
|
/** Dispose all providers */
|
|
7254
8727
|
dispose() {
|
|
@@ -7270,29 +8743,357 @@ var ProviderRegistry = class {
|
|
|
7270
8743
|
syncLegacyFunctions() {
|
|
7271
8744
|
const active = this.getActive();
|
|
7272
8745
|
if (active?.embed && active.capabilities.embeddings) {
|
|
7273
|
-
const embedBridge = (texts) =>
|
|
8746
|
+
const embedBridge = (texts, options) => {
|
|
8747
|
+
const request = {
|
|
8748
|
+
texts,
|
|
8749
|
+
task: options?.task ?? "embedding.document"
|
|
8750
|
+
};
|
|
8751
|
+
if (options?.model) request.model = options.model;
|
|
8752
|
+
return this.embed(request).then((r) => r.vectors);
|
|
8753
|
+
};
|
|
7274
8754
|
setEmbedFunction(embedBridge);
|
|
7275
8755
|
} else {
|
|
7276
8756
|
setEmbedFunction(null);
|
|
7277
8757
|
}
|
|
7278
8758
|
if (active?.complete && active.capabilities.chat) {
|
|
7279
|
-
const llmBridge = (prompt, options) =>
|
|
7280
|
-
|
|
7281
|
-
|
|
7282
|
-
|
|
7283
|
-
|
|
8759
|
+
const llmBridge = (prompt, options) => {
|
|
8760
|
+
const request = {
|
|
8761
|
+
prompt,
|
|
8762
|
+
task: options?.task ?? "chat.general"
|
|
8763
|
+
};
|
|
8764
|
+
if (options?.model) request.model = options.model;
|
|
8765
|
+
if (options?.maxTokens !== void 0) request.maxTokens = options.maxTokens;
|
|
8766
|
+
if (options?.temperature !== void 0) request.temperature = options.temperature;
|
|
8767
|
+
return this.complete(request).then((r) => r.text);
|
|
8768
|
+
};
|
|
7284
8769
|
setLlmFunction(llmBridge);
|
|
7285
8770
|
} else {
|
|
7286
8771
|
setLlmFunction(null);
|
|
7287
8772
|
}
|
|
7288
8773
|
}
|
|
8774
|
+
resolveProvider(task, capability, requestModel, emitSelection = false) {
|
|
8775
|
+
const candidates = this.resolveRouteSelections(task, capability, requestModel);
|
|
8776
|
+
const selection = candidates[0];
|
|
8777
|
+
if (!selection) {
|
|
8778
|
+
const label = task ? ` for task '${task}'` : "";
|
|
8779
|
+
throw new Error(`No provider satisfies ${String(capability)} route${label}`);
|
|
8780
|
+
}
|
|
8781
|
+
if (emitSelection) {
|
|
8782
|
+
this.emitRouteSelected(selection);
|
|
8783
|
+
}
|
|
8784
|
+
return selection;
|
|
8785
|
+
}
|
|
8786
|
+
resolveRouteSelections(task, capability, requestModel) {
|
|
8787
|
+
const route = task ? this.routes.get(task) : void 0;
|
|
8788
|
+
const model = requestModel ?? route?.model;
|
|
8789
|
+
const routeMatched = Boolean(route);
|
|
8790
|
+
return this.resolveCandidates(route, capability).filter((provider) => Boolean(provider.capabilities[capability])).filter((provider) => capability !== "embeddings" || Boolean(provider.embed)).filter((provider) => capability !== "chat" || Boolean(provider.complete)).filter((provider) => capability !== "streaming" || Boolean(provider.stream)).map((provider) => ({
|
|
8791
|
+
provider,
|
|
8792
|
+
providerId: provider.id,
|
|
8793
|
+
providerName: provider.name,
|
|
8794
|
+
tier: provider.tier,
|
|
8795
|
+
capability,
|
|
8796
|
+
task,
|
|
8797
|
+
model,
|
|
8798
|
+
routeMatched
|
|
8799
|
+
}));
|
|
8800
|
+
}
|
|
8801
|
+
emitRouteSelected(selection) {
|
|
8802
|
+
this.events?.emit("provider.route.selected", {
|
|
8803
|
+
providerId: selection.providerId,
|
|
8804
|
+
providerName: selection.providerName,
|
|
8805
|
+
tier: selection.tier,
|
|
8806
|
+
capability: String(selection.capability),
|
|
8807
|
+
task: selection.task,
|
|
8808
|
+
model: selection.model,
|
|
8809
|
+
routeMatched: selection.routeMatched
|
|
8810
|
+
});
|
|
8811
|
+
}
|
|
8812
|
+
async withRouteFallback(task, capability, requestModel, execute, allowFallback) {
|
|
8813
|
+
const selections = this.resolveRouteSelections(task, capability, requestModel);
|
|
8814
|
+
const candidates = allowFallback ? selections : selections.slice(0, 1);
|
|
8815
|
+
if (!candidates.length) {
|
|
8816
|
+
const label = task ? ` for task '${task}'` : "";
|
|
8817
|
+
const route = task ? this.routes.get(task) : void 0;
|
|
8818
|
+
this.emitRouteFailed(
|
|
8819
|
+
void 0,
|
|
8820
|
+
capability,
|
|
8821
|
+
task,
|
|
8822
|
+
requestModel ?? route?.model,
|
|
8823
|
+
Boolean(route),
|
|
8824
|
+
0,
|
|
8825
|
+
0,
|
|
8826
|
+
0,
|
|
8827
|
+
"no_provider",
|
|
8828
|
+
`No provider satisfies ${String(capability)} route${label}`
|
|
8829
|
+
);
|
|
8830
|
+
throw new Error(`No provider satisfies ${String(capability)} route${label}`);
|
|
8831
|
+
}
|
|
8832
|
+
let lastError;
|
|
8833
|
+
let lastSelection;
|
|
8834
|
+
let lastErrorCategory = "unknown";
|
|
8835
|
+
const start = Date.now();
|
|
8836
|
+
for (let index = 0; index < candidates.length; index++) {
|
|
8837
|
+
const selection = candidates[index];
|
|
8838
|
+
try {
|
|
8839
|
+
this.emitRouteSelected(selection);
|
|
8840
|
+
const result = await execute(selection);
|
|
8841
|
+
this.emitRouteCompleted(selection, index + 1, index, Date.now() - start);
|
|
8842
|
+
return result;
|
|
8843
|
+
} catch (err) {
|
|
8844
|
+
lastSelection = selection;
|
|
8845
|
+
lastError = err instanceof Error ? err : new Error(String(err));
|
|
8846
|
+
lastErrorCategory = classifyError(err);
|
|
8847
|
+
const next = candidates[index + 1];
|
|
8848
|
+
if (next) {
|
|
8849
|
+
this.events?.emit("provider.fallback", {
|
|
8850
|
+
fromProvider: selection.providerId,
|
|
8851
|
+
toProvider: next.providerId,
|
|
8852
|
+
errorCategory: lastErrorCategory,
|
|
8853
|
+
error: lastError.message
|
|
8854
|
+
});
|
|
8855
|
+
}
|
|
8856
|
+
}
|
|
8857
|
+
}
|
|
8858
|
+
this.emitRouteFailed(
|
|
8859
|
+
lastSelection,
|
|
8860
|
+
capability,
|
|
8861
|
+
task,
|
|
8862
|
+
requestModel,
|
|
8863
|
+
candidates[0]?.routeMatched ?? Boolean(task && this.routes.get(task)),
|
|
8864
|
+
candidates.length,
|
|
8865
|
+
Math.max(candidates.length - 1, 0),
|
|
8866
|
+
Date.now() - start,
|
|
8867
|
+
lastErrorCategory,
|
|
8868
|
+
lastError?.message ?? "All routed providers failed"
|
|
8869
|
+
);
|
|
8870
|
+
throw lastError ?? new Error("All routed providers failed");
|
|
8871
|
+
}
|
|
8872
|
+
emitRouteCompleted(selection, attempt, fallbackCount, latencyMs) {
|
|
8873
|
+
this.events?.emit("provider.route.completed", {
|
|
8874
|
+
providerId: selection.providerId,
|
|
8875
|
+
providerName: selection.providerName,
|
|
8876
|
+
tier: selection.tier,
|
|
8877
|
+
capability: String(selection.capability),
|
|
8878
|
+
task: selection.task,
|
|
8879
|
+
model: selection.model,
|
|
8880
|
+
routeMatched: selection.routeMatched,
|
|
8881
|
+
attempt,
|
|
8882
|
+
fallbackCount,
|
|
8883
|
+
latencyMs
|
|
8884
|
+
});
|
|
8885
|
+
}
|
|
8886
|
+
emitRouteFailed(selection, capability, task, model, routeMatched, attempt, fallbackCount, latencyMs, errorCategory, error) {
|
|
8887
|
+
this.events?.emit("provider.route.failed", {
|
|
8888
|
+
providerId: selection?.providerId,
|
|
8889
|
+
providerName: selection?.providerName,
|
|
8890
|
+
tier: selection?.tier,
|
|
8891
|
+
capability: String(capability),
|
|
8892
|
+
task,
|
|
8893
|
+
model: selection?.model ?? model,
|
|
8894
|
+
routeMatched,
|
|
8895
|
+
attempt,
|
|
8896
|
+
fallbackCount,
|
|
8897
|
+
latencyMs,
|
|
8898
|
+
errorCategory,
|
|
8899
|
+
error
|
|
8900
|
+
});
|
|
8901
|
+
}
|
|
8902
|
+
resolveCandidates(route, capability) {
|
|
8903
|
+
const all = this.list();
|
|
8904
|
+
if (!route) {
|
|
8905
|
+
const active = this.getActive();
|
|
8906
|
+
return active ? [active, ...all.filter((provider) => provider.id !== active.id)] : all;
|
|
8907
|
+
}
|
|
8908
|
+
const byId = new Map(all.map((provider) => [provider.id, provider]));
|
|
8909
|
+
const selected = [];
|
|
8910
|
+
for (const id of route.providerIds ?? []) {
|
|
8911
|
+
const provider = byId.get(id);
|
|
8912
|
+
if (provider) selected.push(provider);
|
|
8913
|
+
}
|
|
8914
|
+
const hasExplicitProviderIds = Boolean(route.providerIds?.length);
|
|
8915
|
+
const remaining = hasExplicitProviderIds || route.fallback === false ? selected : all;
|
|
8916
|
+
return remaining.filter((provider) => {
|
|
8917
|
+
if (route.tiers?.length && !route.tiers.includes(provider.tier)) return false;
|
|
8918
|
+
if (!providerSatisfiesRouteRequirements(provider, route.requirements, capability)) return false;
|
|
8919
|
+
return true;
|
|
8920
|
+
});
|
|
8921
|
+
}
|
|
7289
8922
|
};
|
|
8923
|
+
function withResolvedModel(request, model) {
|
|
8924
|
+
return model === void 0 ? request : { ...request, model };
|
|
8925
|
+
}
|
|
8926
|
+
function inferInferenceTaskCapability(task) {
|
|
8927
|
+
if (task.startsWith("embedding.")) return "embeddings";
|
|
8928
|
+
if (task.startsWith("vision.")) return "vision";
|
|
8929
|
+
return "chat";
|
|
8930
|
+
}
|
|
8931
|
+
function validateProviderRoute(task, capability, policy, providers) {
|
|
8932
|
+
const byId = new Map(providers.map((provider) => [provider.id, provider]));
|
|
8933
|
+
const providerIds = policy?.providerIds ?? [];
|
|
8934
|
+
const issues = [];
|
|
8935
|
+
const eligibleProviderIds = [];
|
|
8936
|
+
if (policy && providerIds.length === 0 && policy.fallback === false) {
|
|
8937
|
+
issues.push({
|
|
8938
|
+
severity: "error",
|
|
8939
|
+
code: "empty-explicit-chain",
|
|
8940
|
+
message: `Route '${task}' disables fallback but does not name a provider.`
|
|
8941
|
+
});
|
|
8942
|
+
}
|
|
8943
|
+
const candidates = providerIds.length ? providerIds.map((id) => byId.get(id)).filter((provider) => Boolean(provider)) : providers;
|
|
8944
|
+
for (const providerId of providerIds) {
|
|
8945
|
+
if (!byId.has(providerId)) {
|
|
8946
|
+
issues.push({
|
|
8947
|
+
severity: "error",
|
|
8948
|
+
code: "missing-provider",
|
|
8949
|
+
providerId,
|
|
8950
|
+
message: `Route '${task}' references missing provider '${providerId}'.`
|
|
8951
|
+
});
|
|
8952
|
+
}
|
|
8953
|
+
}
|
|
8954
|
+
for (const provider of candidates) {
|
|
8955
|
+
if (policy?.tiers?.length && !policy.tiers.includes(provider.tier)) continue;
|
|
8956
|
+
if (!provider.capabilities[capability]) {
|
|
8957
|
+
issues.push({
|
|
8958
|
+
severity: "error",
|
|
8959
|
+
code: "unsupported-capability",
|
|
8960
|
+
providerId: provider.id,
|
|
8961
|
+
message: `Provider '${provider.id}' does not support ${String(capability)} for route '${task}'.`
|
|
8962
|
+
});
|
|
8963
|
+
continue;
|
|
8964
|
+
}
|
|
8965
|
+
if (capability === "embeddings" && !provider.embed) {
|
|
8966
|
+
issues.push({
|
|
8967
|
+
severity: "error",
|
|
8968
|
+
code: "missing-handler",
|
|
8969
|
+
providerId: provider.id,
|
|
8970
|
+
message: `Provider '${provider.id}' has no embedding handler for route '${task}'.`
|
|
8971
|
+
});
|
|
8972
|
+
continue;
|
|
8973
|
+
}
|
|
8974
|
+
if (capability === "chat" && !provider.complete) {
|
|
8975
|
+
issues.push({
|
|
8976
|
+
severity: "error",
|
|
8977
|
+
code: "missing-handler",
|
|
8978
|
+
providerId: provider.id,
|
|
8979
|
+
message: `Provider '${provider.id}' has no chat handler for route '${task}'.`
|
|
8980
|
+
});
|
|
8981
|
+
continue;
|
|
8982
|
+
}
|
|
8983
|
+
if (capability === "streaming" && !provider.stream) {
|
|
8984
|
+
issues.push({
|
|
8985
|
+
severity: "error",
|
|
8986
|
+
code: "missing-handler",
|
|
8987
|
+
providerId: provider.id,
|
|
8988
|
+
message: `Provider '${provider.id}' has no streaming handler for route '${task}'.`
|
|
8989
|
+
});
|
|
8990
|
+
continue;
|
|
8991
|
+
}
|
|
8992
|
+
const requirementIssue = getProviderRouteRequirementIssue(provider, policy?.requirements, capability);
|
|
8993
|
+
if (requirementIssue) {
|
|
8994
|
+
issues.push({
|
|
8995
|
+
severity: "error",
|
|
8996
|
+
code: "profile-requirement",
|
|
8997
|
+
providerId: provider.id,
|
|
8998
|
+
message: requirementIssue
|
|
8999
|
+
});
|
|
9000
|
+
continue;
|
|
9001
|
+
}
|
|
9002
|
+
eligibleProviderIds.push(provider.id);
|
|
9003
|
+
}
|
|
9004
|
+
if (!eligibleProviderIds.length) {
|
|
9005
|
+
issues.push({
|
|
9006
|
+
severity: "error",
|
|
9007
|
+
code: "no-eligible-provider",
|
|
9008
|
+
message: `Route '${task}' has no eligible ${String(capability)} provider.`
|
|
9009
|
+
});
|
|
9010
|
+
}
|
|
9011
|
+
return {
|
|
9012
|
+
task,
|
|
9013
|
+
capability,
|
|
9014
|
+
policy: policy ? cloneProviderRoutePolicy(policy) : void 0,
|
|
9015
|
+
providerIds,
|
|
9016
|
+
eligibleProviderIds,
|
|
9017
|
+
issues,
|
|
9018
|
+
ok: !issues.some((issue) => issue.severity === "error")
|
|
9019
|
+
};
|
|
9020
|
+
}
|
|
9021
|
+
function cloneProviderRoutePolicy(policy) {
|
|
9022
|
+
return {
|
|
9023
|
+
...policy,
|
|
9024
|
+
providerIds: policy.providerIds ? [...policy.providerIds] : void 0,
|
|
9025
|
+
tiers: policy.tiers ? [...policy.tiers] : void 0,
|
|
9026
|
+
requirements: policy.requirements ? {
|
|
9027
|
+
...policy.requirements,
|
|
9028
|
+
privacyTiers: policy.requirements.privacyTiers ? [...policy.requirements.privacyTiers] : void 0
|
|
9029
|
+
} : void 0
|
|
9030
|
+
};
|
|
9031
|
+
}
|
|
9032
|
+
var COST_ORDER = ["free", "low", "medium", "high"];
|
|
9033
|
+
function providerSatisfiesRouteRequirements(provider, requirements, capability) {
|
|
9034
|
+
return getProviderRouteRequirementIssue(provider, requirements, capability) === void 0;
|
|
9035
|
+
}
|
|
9036
|
+
function getProviderRouteRequirementIssue(provider, requirements, capability) {
|
|
9037
|
+
if (!requirements) return void 0;
|
|
9038
|
+
const profile = provider.profile;
|
|
9039
|
+
if (requirements.privacyTiers?.length) {
|
|
9040
|
+
const privacy = profile?.privacyTier ?? privacyTierFromProviderTier(provider.tier);
|
|
9041
|
+
if (!requirements.privacyTiers.includes(privacy)) {
|
|
9042
|
+
return `Provider '${provider.id}' does not match required privacy tier`;
|
|
9043
|
+
}
|
|
9044
|
+
}
|
|
9045
|
+
if (requirements.maxCostTier) {
|
|
9046
|
+
const cost = profile?.costTier;
|
|
9047
|
+
if (!cost || COST_ORDER.indexOf(cost) > COST_ORDER.indexOf(requirements.maxCostTier)) {
|
|
9048
|
+
return `Provider '${provider.id}' does not match required cost tier`;
|
|
9049
|
+
}
|
|
9050
|
+
}
|
|
9051
|
+
if (requirements.minContextTokens) {
|
|
9052
|
+
const context = provider.capabilities.maxContextTokens;
|
|
9053
|
+
if (!context || context < requirements.minContextTokens) {
|
|
9054
|
+
return `Provider '${provider.id}' does not advertise enough context`;
|
|
9055
|
+
}
|
|
9056
|
+
}
|
|
9057
|
+
if (capability === "embeddings" && requirements.minEmbeddingDimensions) {
|
|
9058
|
+
const dimensions = profile?.embeddingDimensions ?? [];
|
|
9059
|
+
if (!dimensions.some((dimension) => dimension >= requirements.minEmbeddingDimensions)) {
|
|
9060
|
+
return `Provider '${provider.id}' does not advertise enough embedding dimensions`;
|
|
9061
|
+
}
|
|
9062
|
+
}
|
|
9063
|
+
if (requirements.dataClass) {
|
|
9064
|
+
const allowed = profile?.dataClasses ?? defaultDataClasses(provider.tier);
|
|
9065
|
+
if (!allowed.includes(requirements.dataClass)) {
|
|
9066
|
+
return `Provider '${provider.id}' does not allow required data class`;
|
|
9067
|
+
}
|
|
9068
|
+
}
|
|
9069
|
+
if (requirements.maxInputChars) {
|
|
9070
|
+
const maxInputChars = profile?.maxInputChars;
|
|
9071
|
+
if (!maxInputChars || maxInputChars < requirements.maxInputChars) {
|
|
9072
|
+
return `Provider '${provider.id}' does not advertise enough input capacity`;
|
|
9073
|
+
}
|
|
9074
|
+
}
|
|
9075
|
+
return void 0;
|
|
9076
|
+
}
|
|
9077
|
+
function privacyTierFromProviderTier(tier) {
|
|
9078
|
+
if (tier === "remote") return "external";
|
|
9079
|
+
if (tier === "chrome-ai") return "host-managed";
|
|
9080
|
+
return "local";
|
|
9081
|
+
}
|
|
9082
|
+
function defaultDataClasses(tier) {
|
|
9083
|
+
if (tier === "remote") return ["public"];
|
|
9084
|
+
return ["public", "private", "sensitive"];
|
|
9085
|
+
}
|
|
7290
9086
|
function createLegacyProvider(options) {
|
|
7291
9087
|
const { embedFn: embedFn2, llmFn: llmFn2, id = "legacy", name = "Legacy Provider" } = options;
|
|
7292
9088
|
return {
|
|
7293
9089
|
id,
|
|
7294
9090
|
name,
|
|
7295
9091
|
tier: "in-browser",
|
|
9092
|
+
profile: options.profile ?? {
|
|
9093
|
+
privacyTier: "local",
|
|
9094
|
+
costTier: "free",
|
|
9095
|
+
dataClasses: ["public", "private", "sensitive"]
|
|
9096
|
+
},
|
|
7296
9097
|
capabilities: {
|
|
7297
9098
|
embeddings: !!embedFn2,
|
|
7298
9099
|
chat: !!llmFn2,
|
|
@@ -7302,14 +9103,11 @@ function createLegacyProvider(options) {
|
|
|
7302
9103
|
structuredOutput: false
|
|
7303
9104
|
},
|
|
7304
9105
|
embed: embedFn2 ? async (request) => ({
|
|
7305
|
-
vectors: await embedFn2(request.texts),
|
|
9106
|
+
vectors: await embedFn2(request.texts, embedOptionsFromRequest(request)),
|
|
7306
9107
|
model: "legacy"
|
|
7307
9108
|
}) : void 0,
|
|
7308
9109
|
complete: llmFn2 ? async (request) => ({
|
|
7309
|
-
text: await llmFn2(request.prompt,
|
|
7310
|
-
maxTokens: request.maxTokens,
|
|
7311
|
-
temperature: request.temperature
|
|
7312
|
-
}),
|
|
9110
|
+
text: await llmFn2(request.prompt, llmOptionsFromRequest(request)),
|
|
7313
9111
|
model: "legacy"
|
|
7314
9112
|
}) : void 0,
|
|
7315
9113
|
async listModels() {
|
|
@@ -7325,6 +9123,114 @@ function createLegacyProvider(options) {
|
|
|
7325
9123
|
}
|
|
7326
9124
|
};
|
|
7327
9125
|
}
|
|
9126
|
+
function embedOptionsFromRequest(request) {
|
|
9127
|
+
const options = {};
|
|
9128
|
+
if (request.task) options.task = request.task;
|
|
9129
|
+
if (request.model) options.model = request.model;
|
|
9130
|
+
return Object.keys(options).length ? options : void 0;
|
|
9131
|
+
}
|
|
9132
|
+
function llmOptionsFromRequest(request) {
|
|
9133
|
+
const options = {};
|
|
9134
|
+
if (request.maxTokens !== void 0) options.maxTokens = request.maxTokens;
|
|
9135
|
+
if (request.temperature !== void 0) options.temperature = request.temperature;
|
|
9136
|
+
if (request.task) options.task = request.task;
|
|
9137
|
+
if (request.model) options.model = request.model;
|
|
9138
|
+
return Object.keys(options).length ? options : void 0;
|
|
9139
|
+
}
|
|
9140
|
+
|
|
9141
|
+
// src/capabilities/local-discovery.ts
|
|
9142
|
+
var LOCAL_ENDPOINTS = [
|
|
9143
|
+
{ id: "ollama", name: "Ollama", baseURL: "http://localhost:11434/v1", defaultPort: 11434 },
|
|
9144
|
+
{ id: "lm-studio", name: "LM Studio", baseURL: "http://localhost:1234/v1", defaultPort: 1234 },
|
|
9145
|
+
{ id: "llama-cpp", name: "llama.cpp", baseURL: "http://localhost:8080/v1", defaultPort: 8080 },
|
|
9146
|
+
{ id: "vllm", name: "vLLM", baseURL: "http://localhost:8000/v1", defaultPort: 8e3 },
|
|
9147
|
+
{ id: "jan", name: "Jan", baseURL: "http://localhost:1337/v1", defaultPort: 1337 },
|
|
9148
|
+
{ id: "localai", name: "LocalAI", baseURL: "http://localhost:8080/v1", defaultPort: 8080 }
|
|
9149
|
+
];
|
|
9150
|
+
function classifyModel(modelId) {
|
|
9151
|
+
const lower = modelId.toLowerCase();
|
|
9152
|
+
if (lower.includes("embed") || lower.includes("e5-") || lower.includes("bge-") || lower.includes("nomic-") || lower.includes("mxbai-") || lower.includes("all-minilm") || lower.includes("gte-")) {
|
|
9153
|
+
return "embedding";
|
|
9154
|
+
}
|
|
9155
|
+
if (lower.includes("vision") || lower.includes("llava") || lower.includes("moondream") || lower.includes("minicpm-v") || lower.includes("bakllava")) {
|
|
9156
|
+
return "vision";
|
|
9157
|
+
}
|
|
9158
|
+
return "chat";
|
|
9159
|
+
}
|
|
9160
|
+
function inferLocalEmbeddingDimensions(models) {
|
|
9161
|
+
const embeddingModel = models.find((model) => model.capabilities.embeddings)?.id.toLowerCase();
|
|
9162
|
+
if (!embeddingModel) return void 0;
|
|
9163
|
+
if (embeddingModel.includes("nomic-embed")) return [768];
|
|
9164
|
+
if (embeddingModel.includes("bge-large")) return [1024];
|
|
9165
|
+
if (embeddingModel.includes("bge-base")) return [768];
|
|
9166
|
+
if (embeddingModel.includes("bge-small")) return [384];
|
|
9167
|
+
if (embeddingModel.includes("all-minilm")) return [384];
|
|
9168
|
+
if (embeddingModel.includes("mxbai-embed-large")) return [1024];
|
|
9169
|
+
return void 0;
|
|
9170
|
+
}
|
|
9171
|
+
function createLocalProviderProfile(models = []) {
|
|
9172
|
+
const embeddingDimensions = inferLocalEmbeddingDimensions(models);
|
|
9173
|
+
return {
|
|
9174
|
+
privacyTier: "local",
|
|
9175
|
+
costTier: "free",
|
|
9176
|
+
embeddingDimensions,
|
|
9177
|
+
dataClasses: ["public", "private", "sensitive"]
|
|
9178
|
+
};
|
|
9179
|
+
}
|
|
9180
|
+
async function probeEndpoint(endpoint, timeoutMs) {
|
|
9181
|
+
try {
|
|
9182
|
+
const response = await globalThis.fetch(`${endpoint.baseURL}/models`, {
|
|
9183
|
+
signal: AbortSignal.timeout(timeoutMs)
|
|
9184
|
+
});
|
|
9185
|
+
if (!response.ok) return null;
|
|
9186
|
+
const data = await response.json();
|
|
9187
|
+
let modelIds;
|
|
9188
|
+
if (data.data && Array.isArray(data.data)) {
|
|
9189
|
+
modelIds = data.data.map((m) => m.id);
|
|
9190
|
+
} else if (data.models && Array.isArray(data.models)) {
|
|
9191
|
+
modelIds = data.models.map((m) => m.name ?? m.model ?? "");
|
|
9192
|
+
} else {
|
|
9193
|
+
modelIds = [];
|
|
9194
|
+
}
|
|
9195
|
+
const models = modelIds.filter(Boolean).map((id) => {
|
|
9196
|
+
const category = classifyModel(id);
|
|
9197
|
+
return {
|
|
9198
|
+
id,
|
|
9199
|
+
name: id,
|
|
9200
|
+
capabilities: {
|
|
9201
|
+
embeddings: category === "embedding",
|
|
9202
|
+
chat: category === "chat" || category === "vision",
|
|
9203
|
+
vision: category === "vision"
|
|
9204
|
+
}
|
|
9205
|
+
};
|
|
9206
|
+
});
|
|
9207
|
+
return {
|
|
9208
|
+
id: endpoint.id,
|
|
9209
|
+
name: endpoint.name,
|
|
9210
|
+
baseURL: endpoint.baseURL,
|
|
9211
|
+
models
|
|
9212
|
+
};
|
|
9213
|
+
} catch {
|
|
9214
|
+
return null;
|
|
9215
|
+
}
|
|
9216
|
+
}
|
|
9217
|
+
async function discoverLocalProviders(options = {}) {
|
|
9218
|
+
const { extraEndpoints = [], timeoutMs = 2e3, skipPorts = [] } = options;
|
|
9219
|
+
const allEndpoints = [...LOCAL_ENDPOINTS, ...extraEndpoints];
|
|
9220
|
+
const seen = /* @__PURE__ */ new Set();
|
|
9221
|
+
const uniqueEndpoints = allEndpoints.filter((ep) => {
|
|
9222
|
+
if (seen.has(ep.baseURL)) return false;
|
|
9223
|
+
if (skipPorts.includes(ep.defaultPort)) return false;
|
|
9224
|
+
seen.add(ep.baseURL);
|
|
9225
|
+
return true;
|
|
9226
|
+
});
|
|
9227
|
+
const results = await Promise.allSettled(
|
|
9228
|
+
uniqueEndpoints.map((ep) => probeEndpoint(ep, timeoutMs))
|
|
9229
|
+
);
|
|
9230
|
+
return results.filter(
|
|
9231
|
+
(r) => r.status === "fulfilled" && r.value !== null
|
|
9232
|
+
).map((r) => r.value);
|
|
9233
|
+
}
|
|
7328
9234
|
|
|
7329
9235
|
// src/capabilities/openai-provider.ts
|
|
7330
9236
|
var OpenAICompatibleProvider = class {
|
|
@@ -7332,6 +9238,7 @@ var OpenAICompatibleProvider = class {
|
|
|
7332
9238
|
name;
|
|
7333
9239
|
tier;
|
|
7334
9240
|
capabilities;
|
|
9241
|
+
profile;
|
|
7335
9242
|
baseURL;
|
|
7336
9243
|
apiKey;
|
|
7337
9244
|
defaultModel;
|
|
@@ -7357,6 +9264,7 @@ var OpenAICompatibleProvider = class {
|
|
|
7357
9264
|
toolCalling: false,
|
|
7358
9265
|
structuredOutput: false
|
|
7359
9266
|
};
|
|
9267
|
+
this.profile = config.profile ?? (this.tier === "local-server" ? { privacyTier: "local", costTier: "free" } : { privacyTier: "external" });
|
|
7360
9268
|
}
|
|
7361
9269
|
// -------------------------------------------------------------------------
|
|
7362
9270
|
// InferenceProvider interface
|
|
@@ -7451,15 +9359,19 @@ var OpenAICompatibleProvider = class {
|
|
|
7451
9359
|
try {
|
|
7452
9360
|
const response = await this.fetch("/models", void 0, "GET");
|
|
7453
9361
|
const data = response;
|
|
7454
|
-
return data.data.map((m) =>
|
|
7455
|
-
|
|
7456
|
-
|
|
7457
|
-
|
|
7458
|
-
|
|
7459
|
-
|
|
7460
|
-
|
|
7461
|
-
|
|
7462
|
-
|
|
9362
|
+
return data.data.map((m) => {
|
|
9363
|
+
const category = classifyModel(m.id);
|
|
9364
|
+
return {
|
|
9365
|
+
id: m.id,
|
|
9366
|
+
name: m.id,
|
|
9367
|
+
capabilities: {
|
|
9368
|
+
chat: category === "chat" || category === "vision",
|
|
9369
|
+
embeddings: category === "embedding",
|
|
9370
|
+
vision: category === "vision"
|
|
9371
|
+
},
|
|
9372
|
+
owned_by: m.owned_by
|
|
9373
|
+
};
|
|
9374
|
+
});
|
|
7463
9375
|
} catch {
|
|
7464
9376
|
return [];
|
|
7465
9377
|
}
|
|
@@ -7507,10 +9419,6 @@ var OpenAICompatibleProvider = class {
|
|
|
7507
9419
|
return void 0;
|
|
7508
9420
|
}
|
|
7509
9421
|
}
|
|
7510
|
-
isEmbeddingModel(id) {
|
|
7511
|
-
const lower = id.toLowerCase();
|
|
7512
|
-
return lower.includes("embed") || lower.includes("e5-") || lower.includes("bge-") || lower.includes("nomic-") || lower.includes("mxbai-") || lower.includes("all-minilm");
|
|
7513
|
-
}
|
|
7514
9422
|
isLocalURL() {
|
|
7515
9423
|
try {
|
|
7516
9424
|
const url = new URL(this.baseURL);
|
|
@@ -7551,299 +9459,77 @@ var OpenAICompatibleProvider = class {
|
|
|
7551
9459
|
}
|
|
7552
9460
|
};
|
|
7553
9461
|
|
|
7554
|
-
// src/capabilities/
|
|
7555
|
-
|
|
7556
|
-
{
|
|
7557
|
-
|
|
7558
|
-
|
|
7559
|
-
|
|
7560
|
-
|
|
7561
|
-
|
|
7562
|
-
|
|
7563
|
-
|
|
7564
|
-
const lower = modelId.toLowerCase();
|
|
7565
|
-
if (lower.includes("embed") || lower.includes("e5-") || lower.includes("bge-") || lower.includes("nomic-") || lower.includes("mxbai-") || lower.includes("all-minilm") || lower.includes("gte-")) {
|
|
7566
|
-
return "embedding";
|
|
7567
|
-
}
|
|
7568
|
-
if (lower.includes("vision") || lower.includes("llava") || lower.includes("moondream") || lower.includes("minicpm-v") || lower.includes("bakllava")) {
|
|
7569
|
-
return "vision";
|
|
7570
|
-
}
|
|
7571
|
-
return "chat";
|
|
7572
|
-
}
|
|
7573
|
-
async function probeEndpoint(endpoint, timeoutMs) {
|
|
7574
|
-
try {
|
|
7575
|
-
const response = await globalThis.fetch(`${endpoint.baseURL}/models`, {
|
|
7576
|
-
signal: AbortSignal.timeout(timeoutMs)
|
|
7577
|
-
});
|
|
7578
|
-
if (!response.ok) return null;
|
|
7579
|
-
const data = await response.json();
|
|
7580
|
-
let modelIds;
|
|
7581
|
-
if (data.data && Array.isArray(data.data)) {
|
|
7582
|
-
modelIds = data.data.map((m) => m.id);
|
|
7583
|
-
} else if (data.models && Array.isArray(data.models)) {
|
|
7584
|
-
modelIds = data.models.map((m) => m.name ?? m.model ?? "");
|
|
7585
|
-
} else {
|
|
7586
|
-
modelIds = [];
|
|
7587
|
-
}
|
|
7588
|
-
const models = modelIds.filter(Boolean).map((id) => {
|
|
7589
|
-
const category = classifyModel(id);
|
|
7590
|
-
return {
|
|
7591
|
-
id,
|
|
7592
|
-
name: id,
|
|
7593
|
-
capabilities: {
|
|
7594
|
-
embeddings: category === "embedding",
|
|
7595
|
-
chat: category === "chat" || category === "vision",
|
|
7596
|
-
vision: category === "vision"
|
|
7597
|
-
}
|
|
7598
|
-
};
|
|
7599
|
-
});
|
|
7600
|
-
return {
|
|
7601
|
-
id: endpoint.id,
|
|
7602
|
-
name: endpoint.name,
|
|
7603
|
-
baseURL: endpoint.baseURL,
|
|
7604
|
-
models
|
|
7605
|
-
};
|
|
7606
|
-
} catch {
|
|
7607
|
-
return null;
|
|
7608
|
-
}
|
|
7609
|
-
}
|
|
7610
|
-
async function discoverLocalProviders(options = {}) {
|
|
7611
|
-
const { extraEndpoints = [], timeoutMs = 2e3, skipPorts = [] } = options;
|
|
7612
|
-
const allEndpoints = [...LOCAL_ENDPOINTS, ...extraEndpoints];
|
|
7613
|
-
const seen = /* @__PURE__ */ new Set();
|
|
7614
|
-
const uniqueEndpoints = allEndpoints.filter((ep) => {
|
|
7615
|
-
if (seen.has(ep.baseURL)) return false;
|
|
7616
|
-
if (skipPorts.includes(ep.defaultPort)) return false;
|
|
7617
|
-
seen.add(ep.baseURL);
|
|
7618
|
-
return true;
|
|
7619
|
-
});
|
|
7620
|
-
const results = await Promise.allSettled(
|
|
7621
|
-
uniqueEndpoints.map((ep) => probeEndpoint(ep, timeoutMs))
|
|
7622
|
-
);
|
|
7623
|
-
return results.filter(
|
|
7624
|
-
(r) => r.status === "fulfilled" && r.value !== null
|
|
7625
|
-
).map((r) => r.value);
|
|
7626
|
-
}
|
|
7627
|
-
|
|
7628
|
-
// src/capabilities/fallback-router.ts
|
|
7629
|
-
var DEFAULT_COOLDOWNS = {
|
|
7630
|
-
rateLimit: 3e4,
|
|
7631
|
-
serverError: 6e4,
|
|
7632
|
-
connectionFailure: 3e5,
|
|
7633
|
-
contentPolicy: 0
|
|
7634
|
-
};
|
|
7635
|
-
function classifyError(error) {
|
|
7636
|
-
const msg = error instanceof Error ? error.message : String(error);
|
|
7637
|
-
const lower = msg.toLowerCase();
|
|
7638
|
-
if (lower.includes("429") || lower.includes("rate limit")) return "rate_limit";
|
|
7639
|
-
if (lower.includes("500") || lower.includes("502") || lower.includes("503") || lower.includes("504")) return "server_error";
|
|
7640
|
-
if (lower.includes("content") && (lower.includes("policy") || lower.includes("filter"))) return "content_policy";
|
|
7641
|
-
if (lower.includes("context") && (lower.includes("window") || lower.includes("length") || lower.includes("too long"))) return "context_window";
|
|
7642
|
-
if (lower.includes("fetch") || lower.includes("network") || lower.includes("connection") || lower.includes("econnrefused") || lower.includes("timeout")) return "connection_failure";
|
|
7643
|
-
return "unknown";
|
|
9462
|
+
// src/capabilities/bridge-provider.ts
|
|
9463
|
+
function capabilitiesFromBridge(info) {
|
|
9464
|
+
return {
|
|
9465
|
+
embeddings: Boolean(info.capabilities.embeddings),
|
|
9466
|
+
chat: Boolean(info.capabilities.chat),
|
|
9467
|
+
streaming: Boolean(info.capabilities.streaming),
|
|
9468
|
+
vision: false,
|
|
9469
|
+
toolCalling: false,
|
|
9470
|
+
structuredOutput: false
|
|
9471
|
+
};
|
|
7644
9472
|
}
|
|
7645
|
-
var
|
|
7646
|
-
|
|
7647
|
-
|
|
7648
|
-
|
|
7649
|
-
|
|
7650
|
-
|
|
7651
|
-
|
|
7652
|
-
|
|
7653
|
-
|
|
7654
|
-
const available = this.getAvailableProviders();
|
|
7655
|
-
return {
|
|
7656
|
-
embeddings: available.some((p) => p.capabilities.embeddings),
|
|
7657
|
-
chat: available.some((p) => p.capabilities.chat),
|
|
7658
|
-
streaming: available.some((p) => p.capabilities.streaming),
|
|
7659
|
-
vision: available.some((p) => p.capabilities.vision),
|
|
7660
|
-
toolCalling: available.some((p) => p.capabilities.toolCalling),
|
|
7661
|
-
structuredOutput: available.some((p) => p.capabilities.structuredOutput),
|
|
7662
|
-
maxContextTokens: Math.max(
|
|
7663
|
-
...available.map((p) => p.capabilities.maxContextTokens ?? 0),
|
|
7664
|
-
0
|
|
7665
|
-
)
|
|
9473
|
+
var BridgeInferenceProvider = class {
|
|
9474
|
+
constructor(bridge, info) {
|
|
9475
|
+
this.bridge = bridge;
|
|
9476
|
+
this.id = info.id;
|
|
9477
|
+
this.name = info.name;
|
|
9478
|
+
this.tier = info.tier;
|
|
9479
|
+
this.capabilities = capabilitiesFromBridge(info);
|
|
9480
|
+
this.profile = info.profile ?? {
|
|
9481
|
+
privacyTier: "host-managed"
|
|
7666
9482
|
};
|
|
7667
9483
|
}
|
|
7668
|
-
|
|
7669
|
-
|
|
7670
|
-
|
|
7671
|
-
|
|
7672
|
-
|
|
7673
|
-
|
|
7674
|
-
|
|
7675
|
-
|
|
7676
|
-
/** Get providers not currently in cooldown */
|
|
7677
|
-
getAvailableProviders() {
|
|
7678
|
-
const now2 = Date.now();
|
|
7679
|
-
return this.providers.filter((p) => {
|
|
7680
|
-
const cd = this.cooldownMap.get(p.id);
|
|
7681
|
-
if (!cd) return true;
|
|
7682
|
-
if (now2 >= cd.expiresAt) {
|
|
7683
|
-
this.cooldownMap.delete(p.id);
|
|
7684
|
-
return true;
|
|
7685
|
-
}
|
|
7686
|
-
return false;
|
|
7687
|
-
});
|
|
7688
|
-
}
|
|
7689
|
-
/** Get providers in cooldown with their expiry info */
|
|
7690
|
-
getCoolingDown() {
|
|
7691
|
-
const now2 = Date.now();
|
|
7692
|
-
const result = [];
|
|
7693
|
-
for (const [id, entry] of this.cooldownMap) {
|
|
7694
|
-
if (now2 < entry.expiresAt) {
|
|
7695
|
-
result.push({ providerId: id, category: entry.category, expiresAt: entry.expiresAt });
|
|
7696
|
-
}
|
|
7697
|
-
}
|
|
7698
|
-
return result;
|
|
7699
|
-
}
|
|
7700
|
-
/** Manually clear cooldown for a provider */
|
|
7701
|
-
clearCooldown(providerId) {
|
|
7702
|
-
this.cooldownMap.delete(providerId);
|
|
7703
|
-
}
|
|
7704
|
-
/** Clear all cooldowns */
|
|
7705
|
-
clearAllCooldowns() {
|
|
7706
|
-
this.cooldownMap.clear();
|
|
7707
|
-
}
|
|
7708
|
-
/** Add a provider to the chain (appended at lowest priority) */
|
|
7709
|
-
addProvider(provider) {
|
|
7710
|
-
this.providers.push(provider);
|
|
7711
|
-
}
|
|
7712
|
-
/** Remove a provider from the chain */
|
|
7713
|
-
removeProvider(id) {
|
|
7714
|
-
this.providers = this.providers.filter((p) => p.id !== id);
|
|
7715
|
-
this.cooldownMap.delete(id);
|
|
7716
|
-
}
|
|
7717
|
-
/** Reorder providers (new priority order) */
|
|
7718
|
-
setOrder(ids) {
|
|
7719
|
-
const byId = new Map(this.providers.map((p) => [p.id, p]));
|
|
7720
|
-
const reordered = [];
|
|
7721
|
-
for (const id of ids) {
|
|
7722
|
-
const p = byId.get(id);
|
|
7723
|
-
if (p) reordered.push(p);
|
|
9484
|
+
id;
|
|
9485
|
+
name;
|
|
9486
|
+
tier;
|
|
9487
|
+
capabilities;
|
|
9488
|
+
profile;
|
|
9489
|
+
async embed(request) {
|
|
9490
|
+
if (!this.capabilities.embeddings) {
|
|
9491
|
+
throw new Error(`Bridge provider '${this.id}' does not support embeddings`);
|
|
7724
9492
|
}
|
|
7725
|
-
|
|
7726
|
-
|
|
9493
|
+
if (!this.bridge.inference) {
|
|
9494
|
+
throw new Error("Fortemi bridge inference router is unavailable");
|
|
7727
9495
|
}
|
|
7728
|
-
this.
|
|
7729
|
-
}
|
|
7730
|
-
// -------------------------------------------------------------------------
|
|
7731
|
-
// InferenceProvider interface — with fallback
|
|
7732
|
-
// -------------------------------------------------------------------------
|
|
7733
|
-
async embed(request) {
|
|
7734
|
-
return this.withFallback(
|
|
7735
|
-
(p) => p.capabilities.embeddings && !!p.embed,
|
|
7736
|
-
(p) => p.embed(request)
|
|
7737
|
-
);
|
|
9496
|
+
return this.bridge.inference.embed(this.id, request);
|
|
7738
9497
|
}
|
|
7739
9498
|
async complete(request) {
|
|
7740
|
-
|
|
7741
|
-
(
|
|
7742
|
-
|
|
7743
|
-
)
|
|
9499
|
+
if (!this.capabilities.chat) {
|
|
9500
|
+
throw new Error(`Bridge provider '${this.id}' does not support chat`);
|
|
9501
|
+
}
|
|
9502
|
+
if (!this.bridge.inference) {
|
|
9503
|
+
throw new Error("Fortemi bridge inference router is unavailable");
|
|
9504
|
+
}
|
|
9505
|
+
return this.bridge.inference.complete(this.id, request);
|
|
7744
9506
|
}
|
|
7745
|
-
|
|
7746
|
-
|
|
7747
|
-
|
|
7748
|
-
throw new Error("No available providers with streaming capability");
|
|
9507
|
+
stream(request) {
|
|
9508
|
+
if (!this.capabilities.streaming) {
|
|
9509
|
+
throw new Error(`Bridge provider '${this.id}' does not support streaming`);
|
|
7749
9510
|
}
|
|
7750
|
-
|
|
9511
|
+
if (!this.bridge.inference?.stream) {
|
|
9512
|
+
throw new Error("Fortemi bridge streaming router is unavailable");
|
|
9513
|
+
}
|
|
9514
|
+
return this.bridge.inference.stream(this.id, request);
|
|
7751
9515
|
}
|
|
7752
9516
|
async listModels() {
|
|
7753
|
-
|
|
7754
|
-
const results = await Promise.allSettled(
|
|
7755
|
-
available.map((p) => p.listModels())
|
|
7756
|
-
);
|
|
7757
|
-
const models = [];
|
|
7758
|
-
for (const r of results) {
|
|
7759
|
-
if (r.status === "fulfilled") models.push(...r.value);
|
|
7760
|
-
}
|
|
7761
|
-
return models;
|
|
9517
|
+
return [];
|
|
7762
9518
|
}
|
|
7763
9519
|
async probe() {
|
|
7764
|
-
|
|
7765
|
-
|
|
7766
|
-
return { status: "down", latencyMs: 0, message: "All providers in cooldown" };
|
|
9520
|
+
if (!this.bridge.inference) {
|
|
9521
|
+
return { status: "down", latencyMs: 0, message: "Fortemi bridge inference router is unavailable" };
|
|
7767
9522
|
}
|
|
7768
|
-
|
|
7769
|
-
const results = await Promise.allSettled(
|
|
7770
|
-
available.map((p) => p.probe())
|
|
7771
|
-
);
|
|
7772
|
-
const okCount = results.filter(
|
|
7773
|
-
(r) => r.status === "fulfilled" && r.value.status === "ok"
|
|
7774
|
-
).length;
|
|
7775
|
-
return {
|
|
7776
|
-
status: okCount === available.length ? "ok" : okCount > 0 ? "degraded" : "down",
|
|
7777
|
-
latencyMs: Date.now() - start,
|
|
7778
|
-
message: `${okCount}/${available.length} providers healthy`
|
|
7779
|
-
};
|
|
9523
|
+
return this.bridge.inference.probeProvider(this.id);
|
|
7780
9524
|
}
|
|
7781
9525
|
dispose() {
|
|
7782
|
-
for (const p of this.providers) {
|
|
7783
|
-
p.dispose();
|
|
7784
|
-
}
|
|
7785
|
-
this.providers = [];
|
|
7786
|
-
this.cooldownMap.clear();
|
|
7787
|
-
}
|
|
7788
|
-
// -------------------------------------------------------------------------
|
|
7789
|
-
// Core fallback logic
|
|
7790
|
-
// -------------------------------------------------------------------------
|
|
7791
|
-
async withFallback(filter, execute) {
|
|
7792
|
-
const candidates = this.getAvailableProviders().filter(filter);
|
|
7793
|
-
if (candidates.length === 0) {
|
|
7794
|
-
throw new Error("No available providers for this request");
|
|
7795
|
-
}
|
|
7796
|
-
let lastError;
|
|
7797
|
-
for (const provider of candidates) {
|
|
7798
|
-
try {
|
|
7799
|
-
return await execute(provider);
|
|
7800
|
-
} catch (err) {
|
|
7801
|
-
lastError = err instanceof Error ? err : new Error(String(err));
|
|
7802
|
-
const category = classifyError(err);
|
|
7803
|
-
this.applyCooldown(provider.id, category);
|
|
7804
|
-
const nextCandidate = candidates[candidates.indexOf(provider) + 1];
|
|
7805
|
-
if (nextCandidate) {
|
|
7806
|
-
this.events?.emit("provider.fallback", {
|
|
7807
|
-
fromProvider: provider.id,
|
|
7808
|
-
toProvider: nextCandidate.id,
|
|
7809
|
-
errorCategory: category,
|
|
7810
|
-
error: lastError.message
|
|
7811
|
-
});
|
|
7812
|
-
}
|
|
7813
|
-
}
|
|
7814
|
-
}
|
|
7815
|
-
throw lastError ?? new Error("All providers failed");
|
|
7816
|
-
}
|
|
7817
|
-
applyCooldown(providerId, category) {
|
|
7818
|
-
let cooldownMs;
|
|
7819
|
-
switch (category) {
|
|
7820
|
-
case "rate_limit":
|
|
7821
|
-
cooldownMs = this.cooldowns.rateLimit;
|
|
7822
|
-
break;
|
|
7823
|
-
case "server_error":
|
|
7824
|
-
cooldownMs = this.cooldowns.serverError;
|
|
7825
|
-
break;
|
|
7826
|
-
case "connection_failure":
|
|
7827
|
-
cooldownMs = this.cooldowns.connectionFailure;
|
|
7828
|
-
break;
|
|
7829
|
-
case "content_policy":
|
|
7830
|
-
cooldownMs = this.cooldowns.contentPolicy;
|
|
7831
|
-
break;
|
|
7832
|
-
default:
|
|
7833
|
-
cooldownMs = this.cooldowns.serverError;
|
|
7834
|
-
}
|
|
7835
|
-
if (cooldownMs > 0) {
|
|
7836
|
-
const expiresAt = Date.now() + cooldownMs;
|
|
7837
|
-
this.cooldownMap.set(providerId, { expiresAt, category });
|
|
7838
|
-
this.events?.emit("provider.cooldown", {
|
|
7839
|
-
providerId,
|
|
7840
|
-
errorCategory: category,
|
|
7841
|
-
cooldownMs,
|
|
7842
|
-
expiresAt
|
|
7843
|
-
});
|
|
7844
|
-
}
|
|
7845
9526
|
}
|
|
7846
9527
|
};
|
|
9528
|
+
async function createBridgeInferenceProviders(bridge) {
|
|
9529
|
+
if (!bridge.inference) return [];
|
|
9530
|
+
const providers = await bridge.inference.listProviders();
|
|
9531
|
+
return providers.map((info) => new BridgeInferenceProvider(bridge, info));
|
|
9532
|
+
}
|
|
7847
9533
|
|
|
7848
9534
|
// src/fortemi-bridge.ts
|
|
7849
9535
|
function getFortemiBridge(host = globalThis) {
|
|
@@ -7872,6 +9558,159 @@ async function hasFortemiSecureSecrets(host = globalThis) {
|
|
|
7872
9558
|
}
|
|
7873
9559
|
}
|
|
7874
9560
|
|
|
9561
|
+
// src/capabilities/inference-runtime.ts
|
|
9562
|
+
function defineInferenceRuntime(config) {
|
|
9563
|
+
return config;
|
|
9564
|
+
}
|
|
9565
|
+
function defineInferenceProvider(provider) {
|
|
9566
|
+
return { kind: "provider", provider };
|
|
9567
|
+
}
|
|
9568
|
+
function defineOpenAICompatibleProvider(config) {
|
|
9569
|
+
return { kind: "openai-compatible", config };
|
|
9570
|
+
}
|
|
9571
|
+
function defineLegacyInferenceProvider(config) {
|
|
9572
|
+
return { kind: "legacy", ...config };
|
|
9573
|
+
}
|
|
9574
|
+
function mergeInferenceRuntimeConfigs(...configs) {
|
|
9575
|
+
const merged = {};
|
|
9576
|
+
for (const config of configs) {
|
|
9577
|
+
if (!config) continue;
|
|
9578
|
+
if (config.providers?.length) {
|
|
9579
|
+
merged.providers = mergeConfiguredProviders(merged.providers, config.providers);
|
|
9580
|
+
}
|
|
9581
|
+
if (config.routes) {
|
|
9582
|
+
merged.routes = { ...merged.routes ?? {}, ...config.routes };
|
|
9583
|
+
}
|
|
9584
|
+
if (config.activeProviderId !== void 0) {
|
|
9585
|
+
merged.activeProviderId = config.activeProviderId;
|
|
9586
|
+
}
|
|
9587
|
+
if (config.bridgeHost !== void 0) {
|
|
9588
|
+
merged.bridgeHost = config.bridgeHost;
|
|
9589
|
+
}
|
|
9590
|
+
if (config.includeBridgeProviders !== void 0) {
|
|
9591
|
+
merged.includeBridgeProviders = config.includeBridgeProviders;
|
|
9592
|
+
}
|
|
9593
|
+
if (config.discoverLocal !== void 0) {
|
|
9594
|
+
merged.discoverLocal = config.discoverLocal;
|
|
9595
|
+
}
|
|
9596
|
+
if (config.embeddingTaskSelection) {
|
|
9597
|
+
merged.embeddingTaskSelection = {
|
|
9598
|
+
...merged.embeddingTaskSelection ?? {},
|
|
9599
|
+
...config.embeddingTaskSelection
|
|
9600
|
+
};
|
|
9601
|
+
}
|
|
9602
|
+
}
|
|
9603
|
+
return merged;
|
|
9604
|
+
}
|
|
9605
|
+
function getConfiguredInferenceProviderId(config) {
|
|
9606
|
+
switch (config.kind) {
|
|
9607
|
+
case "provider":
|
|
9608
|
+
return config.provider.id;
|
|
9609
|
+
case "openai-compatible":
|
|
9610
|
+
return config.config.id;
|
|
9611
|
+
case "legacy":
|
|
9612
|
+
return config.id ?? "legacy";
|
|
9613
|
+
}
|
|
9614
|
+
}
|
|
9615
|
+
function mergeConfiguredProviders(previous, next) {
|
|
9616
|
+
const byId = /* @__PURE__ */ new Map();
|
|
9617
|
+
for (const provider of previous ?? []) {
|
|
9618
|
+
byId.set(getConfiguredInferenceProviderId(provider), provider);
|
|
9619
|
+
}
|
|
9620
|
+
for (const provider of next) {
|
|
9621
|
+
const id = getConfiguredInferenceProviderId(provider);
|
|
9622
|
+
byId.delete(id);
|
|
9623
|
+
byId.set(id, provider);
|
|
9624
|
+
}
|
|
9625
|
+
return Array.from(byId.values());
|
|
9626
|
+
}
|
|
9627
|
+
async function configureInferenceRuntime(options = {}) {
|
|
9628
|
+
const registry = options.registry ?? new ProviderRegistry(options.events);
|
|
9629
|
+
setEmbeddingTaskSelectionOptions(options.embeddingTaskSelection);
|
|
9630
|
+
for (const providerConfig of options.providers ?? []) {
|
|
9631
|
+
registry.add(createConfiguredProvider(providerConfig));
|
|
9632
|
+
}
|
|
9633
|
+
if (options.includeBridgeProviders !== false) {
|
|
9634
|
+
const bridge = getFortemiBridge(options.bridgeHost);
|
|
9635
|
+
if (bridge) {
|
|
9636
|
+
for (const provider of await createBridgeInferenceProviders(bridge)) {
|
|
9637
|
+
if (!registry.get(provider.id)) registry.add(provider);
|
|
9638
|
+
}
|
|
9639
|
+
}
|
|
9640
|
+
}
|
|
9641
|
+
if (options.discoverLocal) {
|
|
9642
|
+
const discoveryOptions = options.discoverLocal === true ? {} : options.discoverLocal;
|
|
9643
|
+
const discovered = await discoverLocalProviders(discoveryOptions);
|
|
9644
|
+
for (const provider of discovered) {
|
|
9645
|
+
const id = `local:${provider.id}`;
|
|
9646
|
+
if (registry.get(id)) continue;
|
|
9647
|
+
registry.add(new OpenAICompatibleProvider({
|
|
9648
|
+
id,
|
|
9649
|
+
name: provider.name,
|
|
9650
|
+
baseURL: provider.baseURL,
|
|
9651
|
+
tier: "local-server",
|
|
9652
|
+
defaultModel: provider.models.find((model) => model.capabilities.chat)?.id,
|
|
9653
|
+
defaultEmbeddingModel: provider.models.find((model) => model.capabilities.embeddings)?.id,
|
|
9654
|
+
profile: createLocalProviderProfile(provider.models)
|
|
9655
|
+
}));
|
|
9656
|
+
}
|
|
9657
|
+
}
|
|
9658
|
+
for (const [task, route] of Object.entries(options.routes ?? {})) {
|
|
9659
|
+
if (route) registry.setRoute(task, route);
|
|
9660
|
+
}
|
|
9661
|
+
if (options.activeProviderId) {
|
|
9662
|
+
registry.setActive(options.activeProviderId);
|
|
9663
|
+
}
|
|
9664
|
+
if (options.capabilityManager) {
|
|
9665
|
+
wireCapabilities(options.capabilityManager, registry);
|
|
9666
|
+
}
|
|
9667
|
+
const routeValidation = registry.validateRoutes();
|
|
9668
|
+
return {
|
|
9669
|
+
registry,
|
|
9670
|
+
providers: registry.list(),
|
|
9671
|
+
routeValidation,
|
|
9672
|
+
routeIssues: routeValidation.flatMap((route) => route.issues)
|
|
9673
|
+
};
|
|
9674
|
+
}
|
|
9675
|
+
function createConfiguredProvider(config) {
|
|
9676
|
+
switch (config.kind) {
|
|
9677
|
+
case "provider":
|
|
9678
|
+
return config.provider;
|
|
9679
|
+
case "openai-compatible":
|
|
9680
|
+
return new OpenAICompatibleProvider(config.config);
|
|
9681
|
+
case "legacy":
|
|
9682
|
+
return createLegacyProvider(config);
|
|
9683
|
+
}
|
|
9684
|
+
}
|
|
9685
|
+
function wireCapabilities(manager, registry) {
|
|
9686
|
+
if (registry.hasEmbeddings()) {
|
|
9687
|
+
manager.registerLoader("semantic", async () => {
|
|
9688
|
+
setEmbedFunction((texts, options) => {
|
|
9689
|
+
const request = {
|
|
9690
|
+
texts,
|
|
9691
|
+
task: options?.task ?? "embedding.document"
|
|
9692
|
+
};
|
|
9693
|
+
if (options?.model) Object.assign(request, { model: options.model });
|
|
9694
|
+
return registry.embed(request).then((result) => result.vectors);
|
|
9695
|
+
});
|
|
9696
|
+
});
|
|
9697
|
+
}
|
|
9698
|
+
if (registry.hasChat()) {
|
|
9699
|
+
manager.registerLoader("llm", async () => {
|
|
9700
|
+
setLlmFunction((prompt, options) => {
|
|
9701
|
+
const request = {
|
|
9702
|
+
prompt,
|
|
9703
|
+
task: options?.task ?? "chat.general"
|
|
9704
|
+
};
|
|
9705
|
+
if (options?.model) Object.assign(request, { model: options.model });
|
|
9706
|
+
if (options?.maxTokens !== void 0) Object.assign(request, { maxTokens: options.maxTokens });
|
|
9707
|
+
if (options?.temperature !== void 0) Object.assign(request, { temperature: options.temperature });
|
|
9708
|
+
return registry.complete(request).then((result) => result.text);
|
|
9709
|
+
});
|
|
9710
|
+
});
|
|
9711
|
+
}
|
|
9712
|
+
}
|
|
9713
|
+
|
|
7875
9714
|
// src/security/plugin-content.ts
|
|
7876
9715
|
var DEFAULT_DIRECTIVES = {
|
|
7877
9716
|
"default-src": ["'self'"],
|
|
@@ -7913,12 +9752,12 @@ function parseSriToken(integrity) {
|
|
|
7913
9752
|
const separator = token.indexOf("-");
|
|
7914
9753
|
if (separator <= 0) throw new Error("Invalid SRI token");
|
|
7915
9754
|
const algorithm = token.slice(0, separator).toLowerCase();
|
|
7916
|
-
const
|
|
9755
|
+
const digest2 = token.slice(separator + 1);
|
|
7917
9756
|
if (!SUPPORTED_SRI_ALGORITHMS.has(algorithm)) {
|
|
7918
9757
|
throw new Error("Unsupported SRI algorithm: " + algorithm);
|
|
7919
9758
|
}
|
|
7920
|
-
if (!
|
|
7921
|
-
return { algorithm, digest };
|
|
9759
|
+
if (!digest2) throw new Error("Invalid SRI digest");
|
|
9760
|
+
return { algorithm, digest: digest2 };
|
|
7922
9761
|
}
|
|
7923
9762
|
function toBase64(bytes) {
|
|
7924
9763
|
const data = new Uint8Array(bytes);
|
|
@@ -7935,8 +9774,8 @@ async function computeSri(data, algorithm = "sha384") {
|
|
|
7935
9774
|
if (!SUPPORTED_SRI_ALGORITHMS.has(normalized)) {
|
|
7936
9775
|
throw new Error("Unsupported SRI algorithm: " + algorithm);
|
|
7937
9776
|
}
|
|
7938
|
-
const
|
|
7939
|
-
return normalized + "-" + toBase64(
|
|
9777
|
+
const digest2 = await crypto.subtle.digest(normalized.toUpperCase().replace("SHA", "SHA-"), toArrayBuffer(data));
|
|
9778
|
+
return normalized + "-" + toBase64(digest2);
|
|
7940
9779
|
}
|
|
7941
9780
|
async function verifySri(data, integrity) {
|
|
7942
9781
|
const expected = parseSriToken(integrity);
|
|
@@ -8968,12 +10807,12 @@ async function verifyShardSignature(input) {
|
|
|
8968
10807
|
false,
|
|
8969
10808
|
["verify"]
|
|
8970
10809
|
);
|
|
8971
|
-
const
|
|
10810
|
+
const digest2 = await sha256Hex(canonicalPayloadBytes(payload));
|
|
8972
10811
|
signatureValid = await globalThis.crypto.subtle.verify(
|
|
8973
10812
|
"Ed25519",
|
|
8974
10813
|
key,
|
|
8975
10814
|
toBufferSource(base64urlToBytes(envelope.signature)),
|
|
8976
|
-
toBufferSource(new TextEncoder().encode(
|
|
10815
|
+
toBufferSource(new TextEncoder().encode(digest2))
|
|
8977
10816
|
);
|
|
8978
10817
|
} catch (err) {
|
|
8979
10818
|
return { ok: false, reason: "malformed", detail: err instanceof Error ? err.message : String(err) };
|
|
@@ -9018,11 +10857,11 @@ async function signShard(input) {
|
|
|
9018
10857
|
manifest_digest: await sha256Hex(manifest),
|
|
9019
10858
|
blob_digests: sidecarBlobDigests(input.files)
|
|
9020
10859
|
};
|
|
9021
|
-
const
|
|
10860
|
+
const digest2 = await sha256Hex(canonicalPayloadBytes(payload));
|
|
9022
10861
|
const signature = await globalThis.crypto.subtle.sign(
|
|
9023
10862
|
"Ed25519",
|
|
9024
10863
|
input.privateKey,
|
|
9025
|
-
toBufferSource(new TextEncoder().encode(
|
|
10864
|
+
toBufferSource(new TextEncoder().encode(digest2))
|
|
9026
10865
|
);
|
|
9027
10866
|
const envelope = {
|
|
9028
10867
|
...payload,
|
|
@@ -27567,7 +29406,7 @@ function addCanonicalFormats(ajv) {
|
|
|
27567
29406
|
}
|
|
27568
29407
|
function getLegacyAjv() {
|
|
27569
29408
|
if (!legacyAjvInstance) {
|
|
27570
|
-
legacyAjvInstance = new
|
|
29409
|
+
legacyAjvInstance = new Ajv20202({
|
|
27571
29410
|
allErrors: true,
|
|
27572
29411
|
strict: true,
|
|
27573
29412
|
validateFormats: false
|
|
@@ -27578,7 +29417,7 @@ function getLegacyAjv() {
|
|
|
27578
29417
|
}
|
|
27579
29418
|
function getCoreAjv() {
|
|
27580
29419
|
if (!coreAjvInstance) {
|
|
27581
|
-
coreAjvInstance = new
|
|
29420
|
+
coreAjvInstance = new Ajv20202({
|
|
27582
29421
|
allErrors: true,
|
|
27583
29422
|
strict: true,
|
|
27584
29423
|
validateFormats: true
|
|
@@ -27598,7 +29437,7 @@ function getCoreAjv() {
|
|
|
27598
29437
|
}
|
|
27599
29438
|
function getRecordAjv() {
|
|
27600
29439
|
if (!recordAjvInstance) {
|
|
27601
|
-
recordAjvInstance = new
|
|
29440
|
+
recordAjvInstance = new Ajv20202({
|
|
27602
29441
|
allErrors: true,
|
|
27603
29442
|
strict: true,
|
|
27604
29443
|
validateFormats: true
|
|
@@ -27618,7 +29457,7 @@ function getRecordAjv() {
|
|
|
27618
29457
|
}
|
|
27619
29458
|
function getFullAjv() {
|
|
27620
29459
|
if (!fullAjvInstance) {
|
|
27621
|
-
fullAjvInstance = new
|
|
29460
|
+
fullAjvInstance = new Ajv20202({
|
|
27622
29461
|
allErrors: true,
|
|
27623
29462
|
strict: true,
|
|
27624
29463
|
validateFormats: true
|
|
@@ -27691,7 +29530,7 @@ function profileOf(value) {
|
|
|
27691
29530
|
return typeof profile === "string" ? profile : void 0;
|
|
27692
29531
|
}
|
|
27693
29532
|
function validateShardManifest(value) {
|
|
27694
|
-
let
|
|
29533
|
+
let validate2;
|
|
27695
29534
|
const profile = profileOf(value);
|
|
27696
29535
|
if (profile === "core-v1") {
|
|
27697
29536
|
const version = value && typeof value === "object" && !Array.isArray(value) ? coreSchemaVersion(String(value.version ?? "")) : void 0;
|
|
@@ -27701,7 +29540,7 @@ function validateShardManifest(value) {
|
|
|
27701
29540
|
errors: ["(root) uses an unsupported canonical core-v1 schema version"]
|
|
27702
29541
|
};
|
|
27703
29542
|
}
|
|
27704
|
-
|
|
29543
|
+
validate2 = coreValidatorFor("manifest", version);
|
|
27705
29544
|
} else if (profile === "record-v1") {
|
|
27706
29545
|
const version = value && typeof value === "object" && !Array.isArray(value) ? recordSchemaVersion(String(value.version ?? "")) : void 0;
|
|
27707
29546
|
if (!version) {
|
|
@@ -27710,7 +29549,7 @@ function validateShardManifest(value) {
|
|
|
27710
29549
|
errors: ["(root) uses an unsupported canonical record-v1 schema version"]
|
|
27711
29550
|
};
|
|
27712
29551
|
}
|
|
27713
|
-
|
|
29552
|
+
validate2 = recordValidatorFor("manifest", version);
|
|
27714
29553
|
} else if (profile === "full-v1") {
|
|
27715
29554
|
const version = value && typeof value === "object" && !Array.isArray(value) ? fullSchemaVersion(String(value.version ?? "")) : void 0;
|
|
27716
29555
|
if (!version) {
|
|
@@ -27719,12 +29558,12 @@ function validateShardManifest(value) {
|
|
|
27719
29558
|
errors: ["(root) uses an unsupported canonical full-v1 schema version"]
|
|
27720
29559
|
};
|
|
27721
29560
|
}
|
|
27722
|
-
|
|
29561
|
+
validate2 = fullValidatorFor("manifest", version);
|
|
27723
29562
|
} else {
|
|
27724
|
-
|
|
29563
|
+
validate2 = legacyValidatorFor("manifest");
|
|
27725
29564
|
}
|
|
27726
|
-
const valid =
|
|
27727
|
-
return { valid, errors: formatErrors(
|
|
29565
|
+
const valid = validate2(value);
|
|
29566
|
+
return { valid, errors: formatErrors(validate2.errors) };
|
|
27728
29567
|
}
|
|
27729
29568
|
function parseJsonArray(bytes, path) {
|
|
27730
29569
|
if (!bytes) return { records: [], errors: [] };
|
|
@@ -28167,9 +30006,9 @@ async function validateFullV1ShardArchive(input) {
|
|
|
28167
30006
|
return { valid: errors.length === 0, errors };
|
|
28168
30007
|
}
|
|
28169
30008
|
function validateShardComponentRecord(component, value, profile, version = CURRENT_SHARD_VERSION) {
|
|
28170
|
-
let
|
|
30009
|
+
let validate2;
|
|
28171
30010
|
if (profile === "core-v1" && component in CORE_V1_COMPONENT_FILES) {
|
|
28172
|
-
|
|
30011
|
+
validate2 = coreValidatorFor(component, version);
|
|
28173
30012
|
} else if (profile === "record-v1" && component in RECORD_V1_COMPONENT_FILES) {
|
|
28174
30013
|
const canonicalVersion = recordSchemaVersion(version);
|
|
28175
30014
|
if (!canonicalVersion) {
|
|
@@ -28178,7 +30017,7 @@ function validateShardComponentRecord(component, value, profile, version = CURRE
|
|
|
28178
30017
|
errors: [`(root) uses an unsupported canonical record-v1 schema version ${version}`]
|
|
28179
30018
|
};
|
|
28180
30019
|
}
|
|
28181
|
-
|
|
30020
|
+
validate2 = recordValidatorFor(component, canonicalVersion);
|
|
28182
30021
|
} else if (profile === "full-v1" && component in FULL_V1_COMPONENT_FILES) {
|
|
28183
30022
|
const canonicalVersion = fullSchemaVersion(version);
|
|
28184
30023
|
if (!canonicalVersion) {
|
|
@@ -28187,7 +30026,7 @@ function validateShardComponentRecord(component, value, profile, version = CURRE
|
|
|
28187
30026
|
errors: [`(root) uses an unsupported canonical full-v1 schema version ${version}`]
|
|
28188
30027
|
};
|
|
28189
30028
|
}
|
|
28190
|
-
|
|
30029
|
+
validate2 = fullValidatorFor(component, canonicalVersion);
|
|
28191
30030
|
} else {
|
|
28192
30031
|
const legacyDef = LEGACY_COMPONENT_SCHEMA_DEFS[component];
|
|
28193
30032
|
if (!legacyDef) {
|
|
@@ -28196,10 +30035,10 @@ function validateShardComponentRecord(component, value, profile, version = CURRE
|
|
|
28196
30035
|
errors: [`(root) component '${component}' requires an explicit full-v1 profile`]
|
|
28197
30036
|
};
|
|
28198
30037
|
}
|
|
28199
|
-
|
|
30038
|
+
validate2 = legacyValidatorFor(legacyDef);
|
|
28200
30039
|
}
|
|
28201
|
-
const valid =
|
|
28202
|
-
return { valid, errors: formatErrors(
|
|
30040
|
+
const valid = validate2(value);
|
|
30041
|
+
return { valid, errors: formatErrors(validate2.errors) };
|
|
28203
30042
|
}
|
|
28204
30043
|
function assertShardComponentRecord(component, value, profile, version = CURRENT_SHARD_VERSION) {
|
|
28205
30044
|
const result = validateShardComponentRecord(component, value, profile, version);
|
|
@@ -28278,7 +30117,7 @@ async function promoteBlobs(blobStore, blobs) {
|
|
|
28278
30117
|
|
|
28279
30118
|
// src/shard/full-v1-store.ts
|
|
28280
30119
|
var decoder3 = new TextDecoder();
|
|
28281
|
-
var
|
|
30120
|
+
var encoder2 = new TextEncoder();
|
|
28282
30121
|
function emptyCounts() {
|
|
28283
30122
|
return {
|
|
28284
30123
|
notes: 0,
|
|
@@ -28329,15 +30168,15 @@ function componentRecords(component, bytes) {
|
|
|
28329
30168
|
const parsed = spec.encoding === "json-array" ? parseJsonArrayBytes(bytes) : parseJsonlBytes(bytes);
|
|
28330
30169
|
return parsed;
|
|
28331
30170
|
}
|
|
28332
|
-
function
|
|
28333
|
-
if (Array.isArray(value)) return `[${value.map(
|
|
30171
|
+
function canonicalJson4(value) {
|
|
30172
|
+
if (Array.isArray(value)) return `[${value.map(canonicalJson4).join(",")}]`;
|
|
28334
30173
|
if (value && typeof value === "object") {
|
|
28335
|
-
return `{${Object.entries(value).sort(([left], [right]) => left.localeCompare(right)).map(([key, item]) => `${JSON.stringify(key)}:${
|
|
30174
|
+
return `{${Object.entries(value).sort(([left], [right]) => left.localeCompare(right)).map(([key, item]) => `${JSON.stringify(key)}:${canonicalJson4(item)}`).join(",")}}`;
|
|
28336
30175
|
}
|
|
28337
30176
|
return JSON.stringify(value);
|
|
28338
30177
|
}
|
|
28339
30178
|
function encodeComponentRecords(component, records) {
|
|
28340
|
-
return
|
|
30179
|
+
return encoder2.encode(FULL_V1_COMPONENT_FILES[component].encoding === "json-array" ? JSON.stringify(records) : records.map((record) => JSON.stringify(record)).join("\n"));
|
|
28341
30180
|
}
|
|
28342
30181
|
function attachmentBlobReferences(files) {
|
|
28343
30182
|
const refs = /* @__PURE__ */ new Map();
|
|
@@ -28588,7 +30427,7 @@ async function exportFullV1Snapshot(db, blobStore) {
|
|
|
28588
30427
|
for (const row of persisted.rows) currentRecords.get(row.component)?.push(row.record_json);
|
|
28589
30428
|
const changed = [...currentRecords].some(([component, records]) => {
|
|
28590
30429
|
const original = componentRecords(component, files.get(FULL_V1_COMPONENT_FILES[component].file));
|
|
28591
|
-
return
|
|
30430
|
+
return canonicalJson4(records) !== canonicalJson4(original);
|
|
28592
30431
|
});
|
|
28593
30432
|
let blobRefs;
|
|
28594
30433
|
if (changed) {
|
|
@@ -28607,7 +30446,7 @@ async function exportFullV1Snapshot(db, blobStore) {
|
|
|
28607
30446
|
name: "fortemi-react-full-v1-store",
|
|
28608
30447
|
version: manifest.producer?.version ?? "unknown"
|
|
28609
30448
|
};
|
|
28610
|
-
files.set("manifest.json",
|
|
30449
|
+
files.set("manifest.json", encoder2.encode(JSON.stringify(manifest, null, 2)));
|
|
28611
30450
|
files.delete("signature.json");
|
|
28612
30451
|
blobRefs = attachmentBlobReferences(files);
|
|
28613
30452
|
} else {
|
|
@@ -28631,7 +30470,7 @@ async function exportFullV1Snapshot(db, blobStore) {
|
|
|
28631
30470
|
}
|
|
28632
30471
|
return { success: true, archive: packTarGz(files), errors: [], capability_report: capability };
|
|
28633
30472
|
}
|
|
28634
|
-
var
|
|
30473
|
+
var encoder3 = new TextEncoder();
|
|
28635
30474
|
var decoder4 = new TextDecoder();
|
|
28636
30475
|
function iso2(value) {
|
|
28637
30476
|
if (value === null) return null;
|
|
@@ -28660,7 +30499,7 @@ function readRecords(files, component) {
|
|
|
28660
30499
|
function writeRecords(files, component, records) {
|
|
28661
30500
|
const spec = FULL_V1_COMPONENT_FILES[component];
|
|
28662
30501
|
const text = spec.encoding === "json-array" ? JSON.stringify(records) : records.map((record) => JSON.stringify(record)).join("\n");
|
|
28663
|
-
files.set(spec.file,
|
|
30502
|
+
files.set(spec.file, encoder3.encode(text));
|
|
28664
30503
|
}
|
|
28665
30504
|
async function liveRepresentationLosses(db) {
|
|
28666
30505
|
const tables = [
|
|
@@ -29127,7 +30966,7 @@ async function exportLiveFullV1(db, coreArchive, legacyArchive, options) {
|
|
|
29127
30966
|
capability_report: capability
|
|
29128
30967
|
};
|
|
29129
30968
|
}
|
|
29130
|
-
files.set("manifest.json",
|
|
30969
|
+
files.set("manifest.json", encoder3.encode(JSON.stringify(manifest, null, 2)));
|
|
29131
30970
|
for (const note of records.notes) {
|
|
29132
30971
|
const attachments = Array.isArray(note.attachments) ? note.attachments : [];
|
|
29133
30972
|
for (const projection of attachments) {
|
|
@@ -29172,7 +31011,7 @@ async function exportLiveFullV1(db, coreArchive, legacyArchive, options) {
|
|
|
29172
31011
|
}
|
|
29173
31012
|
|
|
29174
31013
|
// src/shard/shard-export.ts
|
|
29175
|
-
var
|
|
31014
|
+
var encoder4 = new TextEncoder();
|
|
29176
31015
|
var CORE_V1_FILES = /* @__PURE__ */ new Set([
|
|
29177
31016
|
"notes.jsonl",
|
|
29178
31017
|
"collections.json",
|
|
@@ -29591,11 +31430,11 @@ async function exportShardBytes(db, options, mode) {
|
|
|
29591
31430
|
const slice = shardNotes.slice(offset, offset + clusterSize);
|
|
29592
31431
|
const href = `notes/${String(offset).padStart(6, "0")}.jsonl`;
|
|
29593
31432
|
clusters.push({ href, offset });
|
|
29594
|
-
files.set(href,
|
|
31433
|
+
files.set(href, encoder4.encode(slice.map((n) => JSON.stringify(n)).join("\n")));
|
|
29595
31434
|
}
|
|
29596
31435
|
layout = { clusters: { notes: clusters } };
|
|
29597
31436
|
} else {
|
|
29598
|
-
files.set("notes.jsonl",
|
|
31437
|
+
files.set("notes.jsonl", encoder4.encode(shardNotes.map((n) => JSON.stringify(n)).join("\n")));
|
|
29599
31438
|
}
|
|
29600
31439
|
components.push("notes");
|
|
29601
31440
|
counts.notes = notes.length;
|
|
@@ -29623,7 +31462,7 @@ async function exportShardBytes(db, options, mode) {
|
|
|
29623
31462
|
mode?.nativeSchema2Presence
|
|
29624
31463
|
);
|
|
29625
31464
|
}
|
|
29626
|
-
files.set("collections.json",
|
|
31465
|
+
files.set("collections.json", encoder4.encode(JSON.stringify(shardCollections)));
|
|
29627
31466
|
components.push("collections");
|
|
29628
31467
|
counts.collections = shardCollections.length;
|
|
29629
31468
|
const allTagRows = await db.query(
|
|
@@ -29639,7 +31478,7 @@ async function exportShardBytes(db, options, mode) {
|
|
|
29639
31478
|
const shardTags = tagsToShard(
|
|
29640
31479
|
relevantTags.map((r) => ({ name: r.tag, created_at: r.created_at }))
|
|
29641
31480
|
);
|
|
29642
|
-
files.set("tags.json",
|
|
31481
|
+
files.set("tags.json", encoder4.encode(JSON.stringify(shardTags)));
|
|
29643
31482
|
components.push("tags");
|
|
29644
31483
|
counts.tags = shardTags.length;
|
|
29645
31484
|
const templateRows = await db.query(`SELECT * FROM template ORDER BY created_at, id`);
|
|
@@ -29655,7 +31494,7 @@ async function exportShardBytes(db, options, mode) {
|
|
|
29655
31494
|
mode?.nativeSchema2Presence
|
|
29656
31495
|
);
|
|
29657
31496
|
}
|
|
29658
|
-
files.set("templates.json",
|
|
31497
|
+
files.set("templates.json", encoder4.encode(JSON.stringify(shardTemplates)));
|
|
29659
31498
|
components.push("templates");
|
|
29660
31499
|
counts.templates = shardTemplates.length;
|
|
29661
31500
|
}
|
|
@@ -29680,7 +31519,7 @@ async function exportShardBytes(db, options, mode) {
|
|
|
29680
31519
|
);
|
|
29681
31520
|
}
|
|
29682
31521
|
const linksJsonl = shardLinks.map((l) => JSON.stringify(l)).join("\n");
|
|
29683
|
-
files.set("links.jsonl",
|
|
31522
|
+
files.set("links.jsonl", encoder4.encode(linksJsonl));
|
|
29684
31523
|
components.push("links");
|
|
29685
31524
|
counts.links = shardLinks.length;
|
|
29686
31525
|
const allNoteSkosRows = await db.query(`SELECT * FROM note_skos_tag ORDER BY created_at`);
|
|
@@ -29697,25 +31536,25 @@ async function exportShardBytes(db, options, mode) {
|
|
|
29697
31536
|
(row) => exportedConceptIds.has(row.source_concept_id) && exportedConceptIds.has(row.target_concept_id)
|
|
29698
31537
|
) : allRelationRows.rows;
|
|
29699
31538
|
const shardSkosSchemes = filteredSchemeRows.map(skosSchemeToShard);
|
|
29700
|
-
files.set("skos_schemes.json",
|
|
31539
|
+
files.set("skos_schemes.json", encoder4.encode(JSON.stringify(shardSkosSchemes)));
|
|
29701
31540
|
components.push("skos_schemes");
|
|
29702
31541
|
counts.skos_schemes = shardSkosSchemes.length;
|
|
29703
31542
|
const shardSkosConcepts = filteredConceptRows.map(skosConceptToShard);
|
|
29704
|
-
files.set("skos_concepts.json",
|
|
31543
|
+
files.set("skos_concepts.json", encoder4.encode(JSON.stringify(shardSkosConcepts)));
|
|
29705
31544
|
components.push("skos_concepts");
|
|
29706
31545
|
counts.skos_concepts = shardSkosConcepts.length;
|
|
29707
31546
|
const skosRelationsJsonl = filteredRelationRows.map((row) => JSON.stringify(skosRelationToShard(row))).join("\n");
|
|
29708
|
-
files.set("skos_relations.jsonl",
|
|
31547
|
+
files.set("skos_relations.jsonl", encoder4.encode(skosRelationsJsonl));
|
|
29709
31548
|
components.push("skos_relations");
|
|
29710
31549
|
counts.skos_relations = filteredRelationRows.length;
|
|
29711
31550
|
const noteSkosJsonl = filteredNoteSkosRows.map((row) => JSON.stringify(noteSkosTagToShard(row))).join("\n");
|
|
29712
|
-
files.set("note_skos_tags.jsonl",
|
|
31551
|
+
files.set("note_skos_tags.jsonl", encoder4.encode(noteSkosJsonl));
|
|
29713
31552
|
components.push("note_skos_tags");
|
|
29714
31553
|
counts.note_skos_tags = filteredNoteSkosRows.length;
|
|
29715
31554
|
const provenanceRows = await db.query(`SELECT * FROM provenance_edge ORDER BY started_at`);
|
|
29716
31555
|
const filteredProvenanceRows = isFiltered ? provenanceRows.rows.filter((row) => row.entity_type !== "note" || exportedNoteIds.has(row.entity_id)) : provenanceRows.rows;
|
|
29717
31556
|
const provenanceJsonl = filteredProvenanceRows.map((row) => JSON.stringify(provenanceEdgeToShard(row))).join("\n");
|
|
29718
|
-
files.set("provenance_edges.jsonl",
|
|
31557
|
+
files.set("provenance_edges.jsonl", encoder4.encode(provenanceJsonl));
|
|
29719
31558
|
components.push("provenance_edges");
|
|
29720
31559
|
counts.provenance_edges = filteredProvenanceRows.length;
|
|
29721
31560
|
if (options?.includeEmbeddings) {
|
|
@@ -29755,7 +31594,7 @@ async function exportShardBytes(db, options, mode) {
|
|
|
29755
31594
|
freshness_json: { status: "unknown" }
|
|
29756
31595
|
} : row
|
|
29757
31596
|
));
|
|
29758
|
-
files.set("embedding_sets.json",
|
|
31597
|
+
files.set("embedding_sets.json", encoder4.encode(JSON.stringify(shardEmbSets)));
|
|
29759
31598
|
components.push("embedding_sets");
|
|
29760
31599
|
counts.embedding_sets = shardEmbSets.length;
|
|
29761
31600
|
const embeddingConfigRows = await db.query(
|
|
@@ -29765,7 +31604,7 @@ async function exportShardBytes(db, options, mode) {
|
|
|
29765
31604
|
);
|
|
29766
31605
|
if (embeddingConfigRows.rows.length > 0) {
|
|
29767
31606
|
const shardEmbeddingConfigs = embeddingConfigRows.rows.map((row) => embeddingConfigToShard(row));
|
|
29768
|
-
files.set("embedding_configs.json",
|
|
31607
|
+
files.set("embedding_configs.json", encoder4.encode(JSON.stringify(shardEmbeddingConfigs)));
|
|
29769
31608
|
components.push("embedding_configs");
|
|
29770
31609
|
counts.embedding_configs = shardEmbeddingConfigs.length;
|
|
29771
31610
|
}
|
|
@@ -29778,7 +31617,7 @@ async function exportShardBytes(db, options, mode) {
|
|
|
29778
31617
|
(member) => exportedSetIds.has(member.embedding_set_id) && exportedNoteIds.has(member.note_id) && (includeMaterializedSelectors || !virtualSetIds.has(member.embedding_set_id))
|
|
29779
31618
|
);
|
|
29780
31619
|
const membersJsonl = scopedEmbMemberRows.map((m) => JSON.stringify(embeddingSetMemberToShard(m))).join("\n");
|
|
29781
|
-
files.set("embedding_set_members.jsonl",
|
|
31620
|
+
files.set("embedding_set_members.jsonl", encoder4.encode(membersJsonl));
|
|
29782
31621
|
components.push("embedding_set_members");
|
|
29783
31622
|
counts.embedding_set_members = scopedEmbMemberRows.length;
|
|
29784
31623
|
const embRows = await db.query(
|
|
@@ -29804,7 +31643,7 @@ async function exportShardBytes(db, options, mode) {
|
|
|
29804
31643
|
(embedding) => exportedSetIds.has(embedding.embedding_set_id) && exportedNoteIds.has(embedding.note_id) && (memberEmbeddingIds.size === 0 || memberEmbeddingIds.has(embedding.id))
|
|
29805
31644
|
);
|
|
29806
31645
|
const embJsonl = scopedEmbRows.map((e) => JSON.stringify(embeddingToShard(e))).join("\n");
|
|
29807
|
-
files.set("embeddings.jsonl",
|
|
31646
|
+
files.set("embeddings.jsonl", encoder4.encode(embJsonl));
|
|
29808
31647
|
components.push("embeddings");
|
|
29809
31648
|
counts.embeddings = scopedEmbRows.length;
|
|
29810
31649
|
}
|
|
@@ -29830,7 +31669,7 @@ async function exportShardBytes(db, options, mode) {
|
|
|
29830
31669
|
freshness: jsonObject4(row.freshness_json) ?? { status: "unknown" },
|
|
29831
31670
|
created_at: iso3(row.created_at)
|
|
29832
31671
|
}));
|
|
29833
|
-
files.set("graph_sources.json",
|
|
31672
|
+
files.set("graph_sources.json", encoder4.encode(JSON.stringify(shardGraphSources)));
|
|
29834
31673
|
components.push("graph_sources");
|
|
29835
31674
|
counts.graph_sources = shardGraphSources.length;
|
|
29836
31675
|
}
|
|
@@ -29846,7 +31685,7 @@ async function exportShardBytes(db, options, mode) {
|
|
|
29846
31685
|
rank: row.rank,
|
|
29847
31686
|
metadata: jsonObject4(row.metadata_json)
|
|
29848
31687
|
})).join("\n");
|
|
29849
|
-
files.set("graph_edges.jsonl",
|
|
31688
|
+
files.set("graph_edges.jsonl", encoder4.encode(graphEdgesJsonl));
|
|
29850
31689
|
components.push("graph_edges");
|
|
29851
31690
|
counts.graph_edges = scopedGraphEdgeRows.length;
|
|
29852
31691
|
}
|
|
@@ -29882,7 +31721,7 @@ async function exportShardBytes(db, options, mode) {
|
|
|
29882
31721
|
})),
|
|
29883
31722
|
created_at: iso3(row.created_at)
|
|
29884
31723
|
}));
|
|
29885
|
-
files.set("communities.json",
|
|
31724
|
+
files.set("communities.json", encoder4.encode(JSON.stringify(shardCommunitySets)));
|
|
29886
31725
|
components.push("communities");
|
|
29887
31726
|
counts.community_sets = shardCommunitySets.length;
|
|
29888
31727
|
counts.communities = scopedCommunityRows.length;
|
|
@@ -29898,7 +31737,7 @@ async function exportShardBytes(db, options, mode) {
|
|
|
29898
31737
|
source_type: row.source_type,
|
|
29899
31738
|
metadata: jsonObject4(row.metadata_json)
|
|
29900
31739
|
})).join("\n");
|
|
29901
|
-
files.set("community_assignments.jsonl",
|
|
31740
|
+
files.set("community_assignments.jsonl", encoder4.encode(assignmentsJsonl));
|
|
29902
31741
|
components.push("community_assignments");
|
|
29903
31742
|
counts.community_assignments = scopedAssignmentRows.length;
|
|
29904
31743
|
}
|
|
@@ -29921,7 +31760,7 @@ async function exportShardBytes(db, options, mode) {
|
|
|
29921
31760
|
const coreTags = [...tagsByName.values()].sort(
|
|
29922
31761
|
(left, right) => left.name.localeCompare(right.name)
|
|
29923
31762
|
);
|
|
29924
|
-
files.set("tags.json",
|
|
31763
|
+
files.set("tags.json", encoder4.encode(JSON.stringify(coreTags)));
|
|
29925
31764
|
components.splice(0, components.length, ...CORE_V1_COMPONENTS);
|
|
29926
31765
|
for (const key of Object.keys(counts)) {
|
|
29927
31766
|
if (!CORE_V1_COMPONENTS.includes(key)) delete counts[key];
|
|
@@ -29961,7 +31800,7 @@ async function exportShardBytes(db, options, mode) {
|
|
|
29961
31800
|
...!coreV1 ? { migrated_from: null } : {},
|
|
29962
31801
|
...!coreV1 && layout ? { layout } : {}
|
|
29963
31802
|
};
|
|
29964
|
-
files.set("manifest.json",
|
|
31803
|
+
files.set("manifest.json", encoder4.encode(JSON.stringify(manifest, null, 2)));
|
|
29965
31804
|
if (options?.includeBlobs && options.blobStore) {
|
|
29966
31805
|
const packed = /* @__PURE__ */ new Set();
|
|
29967
31806
|
for (const row of attachmentRows.rows) {
|
|
@@ -34467,7 +36306,7 @@ function getAiwgFortemiIndexExportSchema() {
|
|
|
34467
36306
|
}
|
|
34468
36307
|
function getAjv() {
|
|
34469
36308
|
if (!ajvInstance) {
|
|
34470
|
-
ajvInstance = new
|
|
36309
|
+
ajvInstance = new Ajv20202({
|
|
34471
36310
|
allErrors: true,
|
|
34472
36311
|
strict: false,
|
|
34473
36312
|
validateFormats: false
|
|
@@ -34512,7 +36351,7 @@ function getProjectedRecordValidator() {
|
|
|
34512
36351
|
return projectedRecordValidator;
|
|
34513
36352
|
}
|
|
34514
36353
|
function validateAiwgFortemiIndexExportSchema(value) {
|
|
34515
|
-
const
|
|
36354
|
+
const validate2 = getExportValidator();
|
|
34516
36355
|
const schemaValue = value && typeof value === "object" && Array.isArray(value.items) ? {
|
|
34517
36356
|
...value,
|
|
34518
36357
|
items: value.items.map((item) => {
|
|
@@ -34522,15 +36361,15 @@ function validateAiwgFortemiIndexExportSchema(value) {
|
|
|
34522
36361
|
return schemaRecord;
|
|
34523
36362
|
})
|
|
34524
36363
|
} : value;
|
|
34525
|
-
const valid =
|
|
34526
|
-
return { valid, errors: formatErrors2(
|
|
36364
|
+
const valid = validate2(schemaValue);
|
|
36365
|
+
return { valid, errors: formatErrors2(validate2.errors) };
|
|
34527
36366
|
}
|
|
34528
36367
|
function validateAiwgFortemiProjectedRecordSchema(value) {
|
|
34529
|
-
const
|
|
34530
|
-
const valid =
|
|
34531
|
-
return { valid, errors: formatErrors2(
|
|
36368
|
+
const validate2 = getProjectedRecordValidator();
|
|
36369
|
+
const valid = validate2(value);
|
|
36370
|
+
return { valid, errors: formatErrors2(validate2.errors) };
|
|
34532
36371
|
}
|
|
34533
|
-
var
|
|
36372
|
+
var encoder5 = new TextEncoder();
|
|
34534
36373
|
var UUID_NAMESPACE = "7ab5d1f8-29d2-5e35-9e2f-3a45de171a9e";
|
|
34535
36374
|
function uuid(kind, id) {
|
|
34536
36375
|
return v5(`${kind}:${id}`, UUID_NAMESPACE);
|
|
@@ -34546,7 +36385,7 @@ function addLoss(losses, code, message, details = {}) {
|
|
|
34546
36385
|
losses.push({ code, message, ...details });
|
|
34547
36386
|
}
|
|
34548
36387
|
function encode(values, encoding) {
|
|
34549
|
-
return
|
|
36388
|
+
return encoder5.encode(encoding === "json-array" ? JSON.stringify(values) : values.map((value) => JSON.stringify(value)).join("\n"));
|
|
34550
36389
|
}
|
|
34551
36390
|
function noteTitle(record, losses) {
|
|
34552
36391
|
if (own(record, "title")) return record.title ?? null;
|
|
@@ -34714,7 +36553,7 @@ async function convertAiwgIndexToFullV1(index, options = {}) {
|
|
|
34714
36553
|
tags: recordTags,
|
|
34715
36554
|
attachments: []
|
|
34716
36555
|
});
|
|
34717
|
-
const hash = await sha256Hex(
|
|
36556
|
+
const hash = await sha256Hex(encoder5.encode(content));
|
|
34718
36557
|
rows.get("note_originals").push({
|
|
34719
36558
|
id: uuid("note-original", record.id),
|
|
34720
36559
|
note_id: noteId,
|
|
@@ -34806,7 +36645,7 @@ async function convertAiwgIndexToFullV1(index, options = {}) {
|
|
|
34806
36645
|
metric: null,
|
|
34807
36646
|
algorithm: null,
|
|
34808
36647
|
parameters: null,
|
|
34809
|
-
input_hash: `sha256:${await sha256Hex(
|
|
36648
|
+
input_hash: `sha256:${await sha256Hex(encoder5.encode(JSON.stringify(relationshipInput)))}`,
|
|
34810
36649
|
freshness: { status: "fresh", checked_at: exportedAt },
|
|
34811
36650
|
created_at: exportedAt
|
|
34812
36651
|
});
|
|
@@ -35264,7 +37103,7 @@ async function convertAiwgIndexToFullV1(index, options = {}) {
|
|
|
35264
37103
|
checksums,
|
|
35265
37104
|
min_reader_version: "2.0.0"
|
|
35266
37105
|
};
|
|
35267
|
-
const manifestBytes =
|
|
37106
|
+
const manifestBytes = encoder5.encode(JSON.stringify(manifest, null, 2));
|
|
35268
37107
|
files.set("manifest.json", manifestBytes);
|
|
35269
37108
|
const validation = await validateFullV1ShardArchive(files);
|
|
35270
37109
|
if (!validation.valid) {
|
|
@@ -35519,6 +37358,7 @@ var RECORD_COLLECTIONS = [
|
|
|
35519
37358
|
"note",
|
|
35520
37359
|
"note_original",
|
|
35521
37360
|
"note_revised_current",
|
|
37361
|
+
"note_revision",
|
|
35522
37362
|
"note_tag",
|
|
35523
37363
|
"link",
|
|
35524
37364
|
"collection",
|
|
@@ -35528,6 +37368,7 @@ var RECORD_COLLECTIONS = [
|
|
|
35528
37368
|
"shard_manifest",
|
|
35529
37369
|
"source_identity",
|
|
35530
37370
|
"source_import_run",
|
|
37371
|
+
"source_import_batch",
|
|
35531
37372
|
"deletion_receipt"
|
|
35532
37373
|
];
|
|
35533
37374
|
var RECORD_STORE_CAPABILITIES = {
|
|
@@ -35650,10 +37491,10 @@ var MemoryRecordStore = class {
|
|
|
35650
37491
|
};
|
|
35651
37492
|
|
|
35652
37493
|
// src/records/idb-record-store.ts
|
|
35653
|
-
var DB_VERSION =
|
|
37494
|
+
var DB_VERSION = 3;
|
|
35654
37495
|
var JOURNAL_STORE = "journal";
|
|
35655
37496
|
var META_STORE = "meta";
|
|
35656
|
-
var RECORD_SCHEMA_VERSION =
|
|
37497
|
+
var RECORD_SCHEMA_VERSION = 3;
|
|
35657
37498
|
function requestToPromise(request) {
|
|
35658
37499
|
return new Promise((resolve, reject) => {
|
|
35659
37500
|
request.onsuccess = () => resolve(request.result);
|
|
@@ -36210,10 +38051,11 @@ function collectionsParentFirst(collections) {
|
|
|
36210
38051
|
return ordered;
|
|
36211
38052
|
}
|
|
36212
38053
|
async function projectNotes(db, store) {
|
|
36213
|
-
const [noteRows, originals, revised, tags, links, collections, memberships] = await Promise.all([
|
|
38054
|
+
const [noteRows, originals, revised, revisions, tags, links, collections, memberships] = await Promise.all([
|
|
36214
38055
|
store.list("note"),
|
|
36215
38056
|
store.list("note_original"),
|
|
36216
38057
|
store.list("note_revised_current"),
|
|
38058
|
+
store.list("note_revision"),
|
|
36217
38059
|
store.list("note_tag"),
|
|
36218
38060
|
store.list("link"),
|
|
36219
38061
|
store.list("collection"),
|
|
@@ -36310,6 +38152,28 @@ async function projectNotes(db, store) {
|
|
|
36310
38152
|
]
|
|
36311
38153
|
);
|
|
36312
38154
|
}
|
|
38155
|
+
for (const r of revisions) {
|
|
38156
|
+
await db.query(
|
|
38157
|
+
`INSERT INTO note_revision (id, note_id, revision_number, type, content, ai_metadata, model, created_at)
|
|
38158
|
+
VALUES ($1, $2, $3, $4, $5, $6::jsonb, $7, $8)
|
|
38159
|
+
ON CONFLICT (id) DO UPDATE SET
|
|
38160
|
+
revision_number = EXCLUDED.revision_number,
|
|
38161
|
+
type = EXCLUDED.type,
|
|
38162
|
+
content = EXCLUDED.content,
|
|
38163
|
+
ai_metadata = EXCLUDED.ai_metadata,
|
|
38164
|
+
model = EXCLUDED.model`,
|
|
38165
|
+
[
|
|
38166
|
+
r.id,
|
|
38167
|
+
r.note_id,
|
|
38168
|
+
r.revision_number,
|
|
38169
|
+
r.type,
|
|
38170
|
+
r.content,
|
|
38171
|
+
r.ai_metadata == null ? null : JSON.stringify(r.ai_metadata),
|
|
38172
|
+
r.model,
|
|
38173
|
+
r.created_at
|
|
38174
|
+
]
|
|
38175
|
+
);
|
|
38176
|
+
}
|
|
36313
38177
|
for (const t of tags) {
|
|
36314
38178
|
await db.query(
|
|
36315
38179
|
`INSERT INTO note_tag (id, note_id, tag, created_at)
|
|
@@ -36346,6 +38210,7 @@ async function projectNotes(db, store) {
|
|
|
36346
38210
|
);
|
|
36347
38211
|
return {
|
|
36348
38212
|
notes: noteRows.length,
|
|
38213
|
+
revisions: revisions.length,
|
|
36349
38214
|
tags: tags.length,
|
|
36350
38215
|
links: links.length,
|
|
36351
38216
|
collections: collections.length,
|
|
@@ -36499,7 +38364,7 @@ function createRecordBackend(store, options = {}) {
|
|
|
36499
38364
|
}
|
|
36500
38365
|
|
|
36501
38366
|
// src/records/record-shard.ts
|
|
36502
|
-
var
|
|
38367
|
+
var encoder6 = new TextEncoder();
|
|
36503
38368
|
var decoder8 = new TextDecoder();
|
|
36504
38369
|
function emptyCounts3() {
|
|
36505
38370
|
return {
|
|
@@ -36757,11 +38622,11 @@ async function buildRecordShardArchive(store, options, profile) {
|
|
|
36757
38622
|
const slice = shardNotes.slice(offset, offset + clusterSize);
|
|
36758
38623
|
const href = `notes/${String(offset).padStart(6, "0")}.jsonl`;
|
|
36759
38624
|
clusters.push({ href, offset });
|
|
36760
|
-
files.set(href,
|
|
38625
|
+
files.set(href, encoder6.encode(slice.map((n) => JSON.stringify(n)).join("\n")));
|
|
36761
38626
|
}
|
|
36762
38627
|
layout = { clusters: { notes: clusters } };
|
|
36763
38628
|
} else {
|
|
36764
|
-
files.set("notes.jsonl",
|
|
38629
|
+
files.set("notes.jsonl", encoder6.encode(shardNotes.map((n) => JSON.stringify(n)).join("\n")));
|
|
36765
38630
|
}
|
|
36766
38631
|
components.push("notes");
|
|
36767
38632
|
counts.notes = shardNotes.length;
|
|
@@ -36789,7 +38654,7 @@ async function buildRecordShardArchive(store, options, profile) {
|
|
|
36789
38654
|
note_count: mapped.note_count ?? 0
|
|
36790
38655
|
};
|
|
36791
38656
|
});
|
|
36792
|
-
files.set("collections.json",
|
|
38657
|
+
files.set("collections.json", encoder6.encode(JSON.stringify(shardCollections)));
|
|
36793
38658
|
components.push("collections");
|
|
36794
38659
|
counts.collections = shardCollections.length;
|
|
36795
38660
|
const distinctTags = [...new Set(
|
|
@@ -36806,7 +38671,7 @@ async function buildRecordShardArchive(store, options, profile) {
|
|
|
36806
38671
|
created_at: tagCreatedAt.get(name) ?? (/* @__PURE__ */ new Date(0)).toISOString()
|
|
36807
38672
|
}))
|
|
36808
38673
|
);
|
|
36809
|
-
files.set("tags.json",
|
|
38674
|
+
files.set("tags.json", encoder6.encode(JSON.stringify(shardTags)));
|
|
36810
38675
|
components.push("tags");
|
|
36811
38676
|
counts.tags = shardTags.length;
|
|
36812
38677
|
const links = await store.list("link");
|
|
@@ -36828,7 +38693,7 @@ async function buildRecordShardArchive(store, options, profile) {
|
|
|
36828
38693
|
}
|
|
36829
38694
|
return shard;
|
|
36830
38695
|
});
|
|
36831
|
-
files.set("links.jsonl",
|
|
38696
|
+
files.set("links.jsonl", encoder6.encode(shardLinks.map((l) => JSON.stringify(l)).join("\n")));
|
|
36832
38697
|
components.push("links");
|
|
36833
38698
|
counts.links = shardLinks.length;
|
|
36834
38699
|
const checksums = {};
|
|
@@ -36958,7 +38823,7 @@ async function buildRecordShardArchive(store, options, profile) {
|
|
|
36958
38823
|
);
|
|
36959
38824
|
}
|
|
36960
38825
|
}
|
|
36961
|
-
files.set("manifest.json",
|
|
38826
|
+
files.set("manifest.json", encoder6.encode(JSON.stringify(manifest, null, 2)));
|
|
36962
38827
|
if (options?.includeBlobs && options.blobStore) {
|
|
36963
38828
|
const packed = /* @__PURE__ */ new Set();
|
|
36964
38829
|
for (const checksum of exportedBlobChecksums) {
|
|
@@ -37344,7 +39209,7 @@ async function importShardToRecords(store, data, options) {
|
|
|
37344
39209
|
id: existingOriginal?.id ?? generateId(),
|
|
37345
39210
|
note_id: note.id,
|
|
37346
39211
|
content: note.original_content,
|
|
37347
|
-
content_hash: computeHash(
|
|
39212
|
+
content_hash: computeHash(encoder6.encode(note.original_content)),
|
|
37348
39213
|
created_at: createdAt2
|
|
37349
39214
|
};
|
|
37350
39215
|
mutations.push({ op: "put", collection: "note_original", record: originalRecord });
|
|
@@ -37580,177 +39445,283 @@ async function importShardToRecords(store, data, options) {
|
|
|
37580
39445
|
function now() {
|
|
37581
39446
|
return (/* @__PURE__ */ new Date()).toISOString();
|
|
37582
39447
|
}
|
|
37583
|
-
function contentDigest2(content) {
|
|
37584
|
-
return computeHash(new TextEncoder().encode(content));
|
|
37585
|
-
}
|
|
37586
|
-
function sourceHash2(source) {
|
|
37587
|
-
return computeHash(new TextEncoder().encode([
|
|
37588
|
-
source.tenant_id ?? "default",
|
|
37589
|
-
source.archive_id ?? "",
|
|
37590
|
-
source.namespace,
|
|
37591
|
-
source.external_id
|
|
37592
|
-
].join("\0")));
|
|
37593
|
-
}
|
|
37594
39448
|
function countOutcomes2(outcomes) {
|
|
37595
39449
|
return {
|
|
37596
|
-
inserted: outcomes.filter((
|
|
37597
|
-
unchanged: outcomes.filter((
|
|
37598
|
-
versioned: outcomes.filter((
|
|
37599
|
-
replaced: outcomes.filter((
|
|
37600
|
-
conflict: outcomes.filter((
|
|
37601
|
-
rejected: outcomes.filter((
|
|
39450
|
+
inserted: outcomes.filter((item) => item.outcome === "inserted").length,
|
|
39451
|
+
unchanged: outcomes.filter((item) => item.outcome === "unchanged").length,
|
|
39452
|
+
versioned: outcomes.filter((item) => item.outcome === "versioned").length,
|
|
39453
|
+
replaced: outcomes.filter((item) => item.outcome === "replaced").length,
|
|
39454
|
+
conflict: outcomes.filter((item) => item.outcome === "conflict").length,
|
|
39455
|
+
rejected: outcomes.filter((item) => item.outcome === "rejected").length
|
|
39456
|
+
};
|
|
39457
|
+
}
|
|
39458
|
+
function finish2(importRunId, batchId, dryRun, outcome, items, checkpoint) {
|
|
39459
|
+
return {
|
|
39460
|
+
contract_version: SOURCE_UPSERT_CONTRACT_VERSION,
|
|
39461
|
+
import_run_id: importRunId,
|
|
39462
|
+
batch_id: batchId,
|
|
39463
|
+
dry_run: dryRun,
|
|
39464
|
+
outcome,
|
|
39465
|
+
...checkpoint ? { checkpoint } : {},
|
|
39466
|
+
items,
|
|
39467
|
+
outcomes: items,
|
|
39468
|
+
counts: countOutcomes2(items)
|
|
37602
39469
|
};
|
|
37603
39470
|
}
|
|
37604
39471
|
async function findSource(store, source) {
|
|
37605
39472
|
const identities = await store.list("source_identity");
|
|
37606
39473
|
return identities.find((identity) => identity.tenant_id === (source.tenant_id ?? "default") && identity.archive_id === (source.archive_id ?? null) && identity.namespace === source.namespace && identity.external_id === source.external_id) ?? null;
|
|
37607
39474
|
}
|
|
39475
|
+
async function upsertRecordStoreRequest(store, request, scope = {}) {
|
|
39476
|
+
const items = request.items.map((item) => ({
|
|
39477
|
+
source: {
|
|
39478
|
+
tenant_id: scope.tenant_id ?? "default",
|
|
39479
|
+
archive_id: scope.archive_id ?? null,
|
|
39480
|
+
namespace: request.source_namespace,
|
|
39481
|
+
external_id: item.external_id,
|
|
39482
|
+
source_schema_version: request.source_schema_version,
|
|
39483
|
+
import_run_id: request.import_run_id,
|
|
39484
|
+
source_id: request.source_id,
|
|
39485
|
+
workspace_id: request.workspace_id,
|
|
39486
|
+
caller_stable_id: item.caller_stable_id
|
|
39487
|
+
},
|
|
39488
|
+
title: item.title,
|
|
39489
|
+
content: item.content,
|
|
39490
|
+
content_digest: item.content_digest,
|
|
39491
|
+
format: item.format,
|
|
39492
|
+
metadata: item.metadata,
|
|
39493
|
+
policy: item.policy ?? request.policy
|
|
39494
|
+
}));
|
|
39495
|
+
const result = await upsertRecordStoreSources(store, items, {
|
|
39496
|
+
dryRun: request.dry_run,
|
|
39497
|
+
batchId: request.batch_id,
|
|
39498
|
+
checkpoint: request.checkpoint,
|
|
39499
|
+
policy: request.policy
|
|
39500
|
+
});
|
|
39501
|
+
return {
|
|
39502
|
+
contract_version: result.contract_version,
|
|
39503
|
+
import_run_id: result.import_run_id,
|
|
39504
|
+
batch_id: result.batch_id,
|
|
39505
|
+
dry_run: result.dry_run,
|
|
39506
|
+
outcome: result.outcome,
|
|
39507
|
+
...result.checkpoint ? { checkpoint: result.checkpoint } : {},
|
|
39508
|
+
counts: result.counts,
|
|
39509
|
+
items: result.items
|
|
39510
|
+
};
|
|
39511
|
+
}
|
|
37608
39512
|
async function upsertRecordStoreSources(store, items, options = {}) {
|
|
37609
|
-
const maxItems = options.maxItems ?? 500;
|
|
37610
|
-
if (items.length > maxItems) throw new Error(`Source upsert batch exceeds the ${maxItems} item bound`);
|
|
37611
39513
|
if (!store.applyBatch) throw new Error("RecordStore source upsert requires atomic applyBatch() support");
|
|
37612
|
-
|
|
37613
|
-
|
|
37614
|
-
|
|
37615
|
-
|
|
37616
|
-
|
|
37617
|
-
|
|
37618
|
-
|
|
39514
|
+
const importRunId = items[0]?.source.import_run_id ?? "";
|
|
39515
|
+
const requestDigest = sourceRequestDigest(items, options);
|
|
39516
|
+
const batchId = options.batchId ?? deriveSourceBatchId(requestDigest);
|
|
39517
|
+
const batchReason = batchId.length === 0 || batchId.length > 200 ? "invalid_batch_metadata" : JSON.stringify(options.checkpoint ?? {}).length > 65536 ? "checkpoint_too_large" : void 0;
|
|
39518
|
+
const validation = validate(items, options.maxItems ?? 500, batchReason);
|
|
39519
|
+
if (validation) return finish2(importRunId, batchId, options.dryRun === true, "rejected", validation, options.checkpoint);
|
|
39520
|
+
const batches = await store.list("source_import_batch");
|
|
39521
|
+
const prior = batches.find((batch2) => batch2.tenant_id === (items[0].source.tenant_id ?? "default") && batch2.archive_id === (items[0].source.archive_id ?? null) && batch2.namespace === items[0].source.namespace && batch2.batch_id === batchId);
|
|
39522
|
+
if (prior) {
|
|
39523
|
+
if (prior.request_digest !== requestDigest) {
|
|
39524
|
+
const rejected = items.map((item, index) => ({
|
|
39525
|
+
index,
|
|
39526
|
+
outcome: "rejected",
|
|
39527
|
+
external_id_hash: sourceIdentityHash(item.source),
|
|
39528
|
+
content_digest: sourceContentDigest(item.content),
|
|
39529
|
+
reason_code: "batch_id_reused_with_different_request"
|
|
39530
|
+
}));
|
|
39531
|
+
return finish2(importRunId, batchId, false, "rejected", rejected, options.checkpoint);
|
|
39532
|
+
}
|
|
39533
|
+
const receipt = prior.receipt;
|
|
39534
|
+
const unchanged = receipt.items.map((item) => ({ ...item, outcome: "unchanged", reason_code: void 0, reason: void 0 }));
|
|
39535
|
+
return finish2(importRunId, batchId, false, "duplicate", unchanged, receipt.checkpoint);
|
|
37619
39536
|
}
|
|
37620
39537
|
const outcomes = [];
|
|
37621
39538
|
const mutations = [];
|
|
37622
39539
|
const stamp = now();
|
|
37623
39540
|
for (const [index, item] of items.entries()) {
|
|
37624
|
-
const external_id_hash =
|
|
37625
|
-
const content_digest =
|
|
39541
|
+
const external_id_hash = sourceIdentityHash(item.source);
|
|
39542
|
+
const content_digest = sourceContentDigest(item.content);
|
|
37626
39543
|
const existing = await findSource(store, item.source);
|
|
37627
39544
|
if (!existing) {
|
|
37628
39545
|
const noteId = item.source.caller_stable_id ?? generateId();
|
|
37629
|
-
|
|
37630
|
-
|
|
37631
|
-
|
|
37632
|
-
|
|
37633
|
-
|
|
37634
|
-
|
|
37635
|
-
|
|
37636
|
-
|
|
37637
|
-
|
|
37638
|
-
revision_mode: "standard",
|
|
37639
|
-
is_starred: false,
|
|
37640
|
-
is_pinned: false,
|
|
37641
|
-
is_archived: false,
|
|
37642
|
-
created_at: stamp,
|
|
37643
|
-
updated_at: stamp,
|
|
37644
|
-
deleted_at: null
|
|
37645
|
-
};
|
|
37646
|
-
const original = {
|
|
37647
|
-
id: generateId(),
|
|
37648
|
-
note_id: noteId,
|
|
37649
|
-
content: item.content,
|
|
37650
|
-
content_hash: content_digest,
|
|
37651
|
-
created_at: stamp
|
|
37652
|
-
};
|
|
37653
|
-
const current2 = {
|
|
37654
|
-
id: noteId,
|
|
37655
|
-
content: item.content,
|
|
37656
|
-
ai_metadata: item.metadata ?? null,
|
|
37657
|
-
generation_count: 0,
|
|
37658
|
-
model: null,
|
|
37659
|
-
is_user_edited: false,
|
|
37660
|
-
updated_at: stamp
|
|
37661
|
-
};
|
|
37662
|
-
const identity = {
|
|
37663
|
-
id: generateId(),
|
|
37664
|
-
tenant_id: item.source.tenant_id ?? "default",
|
|
37665
|
-
archive_id: item.source.archive_id ?? null,
|
|
37666
|
-
namespace: item.source.namespace,
|
|
37667
|
-
external_id: item.source.external_id,
|
|
37668
|
-
external_id_hash,
|
|
37669
|
-
source_schema_version: item.source.source_schema_version,
|
|
37670
|
-
content_digest,
|
|
37671
|
-
import_run_id: item.source.import_run_id,
|
|
37672
|
-
caller_stable_id: item.source.caller_stable_id ?? null,
|
|
37673
|
-
note_id: noteId,
|
|
37674
|
-
created_at: stamp,
|
|
37675
|
-
updated_at: stamp
|
|
37676
|
-
};
|
|
37677
|
-
mutations.push(
|
|
37678
|
-
{ op: "put", collection: "note", record: note2 },
|
|
37679
|
-
{ op: "put", collection: "note_original", record: original },
|
|
37680
|
-
{ op: "put", collection: "note_revised_current", record: current2 },
|
|
37681
|
-
{ op: "put", collection: "source_identity", record: identity }
|
|
37682
|
-
);
|
|
39546
|
+
if (await store.get("note", noteId)) {
|
|
39547
|
+
const rejected = items.map((candidate, candidateIndex) => ({
|
|
39548
|
+
index: candidateIndex,
|
|
39549
|
+
outcome: "rejected",
|
|
39550
|
+
external_id_hash: sourceIdentityHash(candidate.source),
|
|
39551
|
+
content_digest: sourceContentDigest(candidate.content),
|
|
39552
|
+
reason_code: "caller_stable_id_conflict"
|
|
39553
|
+
}));
|
|
39554
|
+
return finish2(importRunId, batchId, false, "rejected", rejected, options.checkpoint);
|
|
37683
39555
|
}
|
|
39556
|
+
outcomes.push({ index, outcome: "inserted", note_id: noteId, external_id_hash, content_digest });
|
|
39557
|
+
if (!options.dryRun) addInsertMutations(mutations, item, noteId, content_digest, external_id_hash, stamp);
|
|
37684
39558
|
continue;
|
|
37685
39559
|
}
|
|
37686
39560
|
if (existing.content_digest === content_digest) {
|
|
37687
39561
|
outcomes.push({ index, outcome: "unchanged", note_id: existing.note_id, external_id_hash, content_digest });
|
|
37688
39562
|
continue;
|
|
37689
39563
|
}
|
|
37690
|
-
const policy = item.policy ?? "version";
|
|
39564
|
+
const policy = item.policy ?? options.policy ?? "version";
|
|
37691
39565
|
if (policy === "conflict") {
|
|
37692
39566
|
outcomes.push({ index, outcome: "conflict", note_id: existing.note_id, external_id_hash, content_digest });
|
|
37693
39567
|
continue;
|
|
37694
39568
|
}
|
|
37695
39569
|
const note = await store.get("note", existing.note_id);
|
|
37696
39570
|
const current = await store.get("note_revised_current", existing.note_id);
|
|
37697
|
-
|
|
37698
|
-
|
|
37699
|
-
|
|
39571
|
+
const originals = await store.list("note_original");
|
|
39572
|
+
const original = originals.find((candidate) => candidate.note_id === existing.note_id);
|
|
39573
|
+
if (!note || !current || !original) {
|
|
39574
|
+
const rejected = items.map((candidate, candidateIndex) => ({
|
|
39575
|
+
index: candidateIndex,
|
|
39576
|
+
outcome: "rejected",
|
|
39577
|
+
note_id: candidateIndex === index ? existing.note_id : void 0,
|
|
39578
|
+
external_id_hash: sourceIdentityHash(candidate.source),
|
|
39579
|
+
content_digest: sourceContentDigest(candidate.content),
|
|
39580
|
+
reason_code: "invalid_item"
|
|
39581
|
+
}));
|
|
39582
|
+
return finish2(importRunId, batchId, false, "rejected", rejected, options.checkpoint);
|
|
37700
39583
|
}
|
|
37701
39584
|
const outcome = policy === "replace" ? "replaced" : "versioned";
|
|
37702
39585
|
outcomes.push({ index, outcome, note_id: existing.note_id, external_id_hash, content_digest });
|
|
37703
39586
|
if (!options.dryRun) {
|
|
37704
|
-
|
|
37705
|
-
|
|
37706
|
-
|
|
37707
|
-
|
|
37708
|
-
|
|
37709
|
-
|
|
37710
|
-
|
|
37711
|
-
|
|
37712
|
-
|
|
37713
|
-
|
|
37714
|
-
|
|
37715
|
-
|
|
37716
|
-
|
|
37717
|
-
|
|
37718
|
-
|
|
37719
|
-
|
|
37720
|
-
|
|
37721
|
-
|
|
37722
|
-
|
|
37723
|
-
|
|
37724
|
-
|
|
37725
|
-
|
|
37726
|
-
|
|
37727
|
-
|
|
37728
|
-
|
|
37729
|
-
|
|
37730
|
-
|
|
37731
|
-
|
|
37732
|
-
|
|
37733
|
-
|
|
37734
|
-
|
|
37735
|
-
|
|
37736
|
-
|
|
37737
|
-
|
|
37738
|
-
|
|
37739
|
-
|
|
37740
|
-
|
|
39587
|
+
const revisionNumber = outcome === "versioned" ? (await store.list("note_revision")).filter((row) => row.note_id === existing.note_id).length + 1 : void 0;
|
|
39588
|
+
addUpdateMutations(mutations, item, existing, note, current, original, outcome, content_digest, stamp, revisionNumber);
|
|
39589
|
+
}
|
|
39590
|
+
}
|
|
39591
|
+
if (options.dryRun) return finish2(importRunId, batchId, true, "preview", outcomes, options.checkpoint);
|
|
39592
|
+
const committed = finish2(importRunId, batchId, false, "committed", outcomes, options.checkpoint);
|
|
39593
|
+
const contractReceipt = {
|
|
39594
|
+
contract_version: committed.contract_version,
|
|
39595
|
+
import_run_id: committed.import_run_id,
|
|
39596
|
+
batch_id: committed.batch_id,
|
|
39597
|
+
dry_run: committed.dry_run,
|
|
39598
|
+
outcome: committed.outcome,
|
|
39599
|
+
...committed.checkpoint ? { checkpoint: committed.checkpoint } : {},
|
|
39600
|
+
counts: committed.counts,
|
|
39601
|
+
items: committed.items
|
|
39602
|
+
};
|
|
39603
|
+
const run = {
|
|
39604
|
+
id: sourceRunRecordId(items[0].source),
|
|
39605
|
+
external_run_id: importRunId,
|
|
39606
|
+
source_id: items[0].source.source_id ?? null,
|
|
39607
|
+
source_schema_version: items[0].source.source_schema_version,
|
|
39608
|
+
workspace_id: items[0].source.workspace_id ?? null,
|
|
39609
|
+
tenant_id: items[0].source.tenant_id ?? "default",
|
|
39610
|
+
archive_id: items[0].source.archive_id ?? null,
|
|
39611
|
+
namespace: items[0].source.namespace,
|
|
39612
|
+
started_at: stamp,
|
|
39613
|
+
completed_at: stamp,
|
|
39614
|
+
checkpoint: options.checkpoint ?? {},
|
|
39615
|
+
receipt: contractReceipt
|
|
39616
|
+
};
|
|
39617
|
+
const batch = {
|
|
39618
|
+
id: generateId(),
|
|
39619
|
+
tenant_id: items[0].source.tenant_id ?? "default",
|
|
39620
|
+
archive_id: items[0].source.archive_id ?? null,
|
|
39621
|
+
namespace: items[0].source.namespace,
|
|
39622
|
+
batch_id: batchId,
|
|
39623
|
+
request_digest: requestDigest,
|
|
39624
|
+
import_run_id: importRunId,
|
|
39625
|
+
outcome: "committed",
|
|
39626
|
+
checkpoint: options.checkpoint ?? {},
|
|
39627
|
+
receipt: contractReceipt,
|
|
39628
|
+
created_at: stamp
|
|
39629
|
+
};
|
|
39630
|
+
mutations.push(
|
|
39631
|
+
{ op: "put", collection: "source_import_run", record: run },
|
|
39632
|
+
{ op: "put", collection: "source_import_batch", record: batch }
|
|
39633
|
+
);
|
|
39634
|
+
await store.applyBatch(mutations);
|
|
39635
|
+
return committed;
|
|
39636
|
+
}
|
|
39637
|
+
function addInsertMutations(mutations, item, noteId, contentDigest, externalIdHash, stamp) {
|
|
39638
|
+
const note = {
|
|
39639
|
+
id: noteId,
|
|
39640
|
+
archive_id: item.source.archive_id ?? null,
|
|
39641
|
+
title: item.title ?? null,
|
|
39642
|
+
format: item.format ?? "markdown",
|
|
39643
|
+
source: `source:${item.source.namespace}`,
|
|
39644
|
+
visibility: item.visibility ?? "private",
|
|
39645
|
+
revision_mode: "standard",
|
|
39646
|
+
is_starred: false,
|
|
39647
|
+
is_pinned: false,
|
|
39648
|
+
is_archived: false,
|
|
39649
|
+
created_at: stamp,
|
|
39650
|
+
updated_at: stamp,
|
|
39651
|
+
deleted_at: null
|
|
39652
|
+
};
|
|
39653
|
+
const original = { id: generateId(), note_id: noteId, content: item.content, content_hash: contentDigest, created_at: stamp };
|
|
39654
|
+
const current = { id: noteId, content: item.content, ai_metadata: item.metadata ?? null, generation_count: 0, model: null, is_user_edited: false, updated_at: stamp };
|
|
39655
|
+
const revision = { id: generateId(), note_id: noteId, revision_number: 1, type: "source-import", content: item.content, ai_metadata: item.metadata ?? null, model: null, created_at: stamp };
|
|
39656
|
+
const identity = {
|
|
39657
|
+
id: generateId(),
|
|
39658
|
+
tenant_id: item.source.tenant_id ?? "default",
|
|
39659
|
+
archive_id: item.source.archive_id ?? null,
|
|
39660
|
+
namespace: item.source.namespace,
|
|
39661
|
+
external_id: item.source.external_id,
|
|
39662
|
+
external_id_hash: externalIdHash,
|
|
39663
|
+
source_id: item.source.source_id ?? null,
|
|
39664
|
+
source_schema_version: item.source.source_schema_version,
|
|
39665
|
+
content_digest: contentDigest,
|
|
39666
|
+
import_run_id: item.source.import_run_id,
|
|
39667
|
+
caller_stable_id: item.source.caller_stable_id ?? null,
|
|
39668
|
+
note_id: noteId,
|
|
39669
|
+
created_at: stamp,
|
|
39670
|
+
updated_at: stamp
|
|
39671
|
+
};
|
|
39672
|
+
mutations.push(
|
|
39673
|
+
{ op: "put", collection: "note", record: note },
|
|
39674
|
+
{ op: "put", collection: "note_original", record: original },
|
|
39675
|
+
{ op: "put", collection: "note_revised_current", record: current },
|
|
39676
|
+
{ op: "put", collection: "note_revision", record: revision },
|
|
39677
|
+
{ op: "put", collection: "source_identity", record: identity }
|
|
39678
|
+
);
|
|
39679
|
+
}
|
|
39680
|
+
function addUpdateMutations(mutations, item, identity, note, current, original, outcome, contentDigest, stamp, revisionNumber) {
|
|
39681
|
+
mutations.push(
|
|
39682
|
+
{ op: "put", collection: "note", record: { ...note, title: item.title ?? null, archive_id: item.source.archive_id ?? null, format: item.format ?? "markdown", visibility: item.visibility ?? "private", deleted_at: null, updated_at: stamp } },
|
|
39683
|
+
{ op: "put", collection: "note_revised_current", record: { ...current, content: item.content, ai_metadata: item.metadata ?? null, is_user_edited: false, updated_at: stamp } },
|
|
39684
|
+
{ op: "put", collection: "source_identity", record: { ...identity, source_id: item.source.source_id ?? null, source_schema_version: item.source.source_schema_version, content_digest: contentDigest, import_run_id: item.source.import_run_id, updated_at: stamp } }
|
|
39685
|
+
);
|
|
39686
|
+
if (outcome === "replaced") {
|
|
39687
|
+
mutations.push({ op: "put", collection: "note_original", record: { ...original, content: item.content, content_hash: contentDigest } });
|
|
39688
|
+
} else {
|
|
39689
|
+
const revision = {
|
|
39690
|
+
id: generateId(),
|
|
39691
|
+
note_id: identity.note_id,
|
|
39692
|
+
revision_number: revisionNumber ?? 1,
|
|
39693
|
+
type: "source-import",
|
|
39694
|
+
content: item.content,
|
|
39695
|
+
ai_metadata: item.metadata ?? null,
|
|
39696
|
+
model: null,
|
|
39697
|
+
created_at: stamp
|
|
37741
39698
|
};
|
|
37742
|
-
mutations.push({ op: "put", collection: "
|
|
37743
|
-
await store.applyBatch(mutations);
|
|
39699
|
+
mutations.push({ op: "put", collection: "note_revision", record: revision });
|
|
37744
39700
|
}
|
|
37745
|
-
return {
|
|
37746
|
-
import_run_id: items[0].source.import_run_id,
|
|
37747
|
-
dry_run: options.dryRun === true,
|
|
37748
|
-
outcomes,
|
|
37749
|
-
counts: countOutcomes2(outcomes)
|
|
37750
|
-
};
|
|
37751
39701
|
}
|
|
37752
|
-
function
|
|
37753
|
-
|
|
39702
|
+
function validate(items, maxItems, initialReason) {
|
|
39703
|
+
let reason = initialReason ?? (items.length === 0 || items.length > maxItems ? "batch_size_out_of_bounds" : null);
|
|
39704
|
+
const seen = /* @__PURE__ */ new Set();
|
|
39705
|
+
const stableIds = /* @__PURE__ */ new Set();
|
|
39706
|
+
for (const item of items) {
|
|
39707
|
+
if (!item.source.tenant_id || !item.source.namespace || !item.source.external_id || !item.source.source_schema_version || !item.source.import_run_id || !item.content) reason ??= "invalid_item";
|
|
39708
|
+
if ((item.source.source_id?.length ?? 0) > 500 || item.source.source_id === "" || (item.source.workspace_id?.length ?? 0) > 500 || item.source.workspace_id === "") reason ??= "invalid_batch_metadata";
|
|
39709
|
+
if ((item.format ?? "markdown").length > 100 || (item.title?.length ?? 0) > 2e3 || JSON.stringify(item.metadata ?? {}).length > 262144) reason ??= "invalid_item";
|
|
39710
|
+
if (item.content_digest && item.content_digest !== sourceContentDigest(item.content)) reason ??= "content_digest_mismatch";
|
|
39711
|
+
const key = `${item.source.tenant_id}\0${item.source.archive_id ?? ""}\0${item.source.namespace}\0${item.source.external_id}`;
|
|
39712
|
+
if (seen.has(key)) reason ??= "duplicate_external_id_in_batch";
|
|
39713
|
+
seen.add(key);
|
|
39714
|
+
if (item.source.caller_stable_id && stableIds.has(item.source.caller_stable_id)) reason ??= "caller_stable_id_conflict";
|
|
39715
|
+
if (item.source.caller_stable_id) stableIds.add(item.source.caller_stable_id);
|
|
39716
|
+
}
|
|
39717
|
+
if (!reason) return null;
|
|
39718
|
+
return items.map((item, index) => ({
|
|
39719
|
+
index,
|
|
39720
|
+
outcome: "rejected",
|
|
39721
|
+
external_id_hash: sourceIdentityHash(item.source),
|
|
39722
|
+
content_digest: sourceContentDigest(item.content ?? ""),
|
|
39723
|
+
reason_code: reason
|
|
39724
|
+
}));
|
|
37754
39725
|
}
|
|
37755
39726
|
|
|
37756
39727
|
// src/records/lifecycle-purge.ts
|
|
@@ -37870,8 +39841,8 @@ async function purgeRecordStoreGraph(store, selector, operationKey) {
|
|
|
37870
39841
|
}
|
|
37871
39842
|
|
|
37872
39843
|
// src/index.ts
|
|
37873
|
-
var VERSION = "2026.
|
|
39844
|
+
var VERSION = "2026.9.1";
|
|
37874
39845
|
|
|
37875
|
-
export { AIWG_SCAN_REQUIRED_FIELDS, AllowlistTrustStore, ArchiveManager, AttachmentsRepository, CORE_V1_COMPONENTS, CURRENT_MIGRATION_HEAD, CURRENT_SHARD_VERSION, CanonicalAttachmentsRepository, CanonicalNotesRepository, CapabilityManager, CaptureKnowledgeInputSchema, CollectionsRepository, CommunitiesRepository, DB_SNAPSHOT_SCHEMA_VERSION, DbSnapshotVersionError, EMBED_REQUEST_KIND, EMBED_RESPONSE_KIND, EmbeddingSetsRepository, FORTEMI_COMPATIBILITY_PATH, FORTEMI_COMPATIBILITY_STATES, FORTEMI_REQUIRED_COMPATIBILITY_CAPABILITIES, FORTEMI_SERVER_COMPATIBILITY_REVISION, FallbackRouter, FortemiToolManifest, GetNoteInputSchema, GraphRepository, IdbRecordStore, JOB_CAPABILITIES, JOB_PRIORITIES, JobQueueWorker, LOCAL_ENDPOINTS, LifecyclePurgeRepository, LinksRepository, ListNotesInputSchema, ManageArchiveInputSchema, ManageAttachmentsInputSchema, ManageCapabilitiesInputSchema, ManageCollectionsInputSchema, ManageLinksInputSchema, ManageNoteInputSchema, ManageTagsInputSchema, MemoryBlobStore, MemoryRecordStore, MigrationRunner, NotesRepository, OpenAICompatibleProvider, PGliteStorageBackend, PGliteStorageBackendFactory, PGliteWorkerClient, PGliteWorkerStorageBackend, PGliteWorkerStorageBackendFactory, ProvenanceRepository, ProviderRegistry, RECORD_COLLECTIONS, RECORD_SCHEMA_VERSION, RECORD_STORE_CAPABILITIES, REGISTERED_METADATA_PATHS, SHARD_FORMAT, SIGNATURE_ENTRY, SIGNING_ENVELOPE_VERSION, SUPPORTED_PGLITE_VERSION, SearchInputSchema, SearchRepository, SkosRepository, SourceUpsertRepository, TagsRepository, TransactionProxy, TypedEventBus, VERSION, aiRevisionHandler, aiwgFortemiIndexFromKnowledgeShard, aiwgFortemiIndexToCommunityGraph, aiwgFortemiIndexToKnowledgeShard, aiwgFortemiIndexToKnowledgeShardWithReport, allMigrations, appendPluginScript, assertAiwgFortemiChunkManifest, assertAiwgFortemiChunkPart, assertAiwgFortemiIndexExport, assertAiwgStaticEmbeddingSet, assertShardComponentRecord, buildAiwgChunkedIndex, buildAiwgStaticEmbeddingSet, buildMetadataPredicateConditions, buildNoteConditions, buildPluginCsp, captureKnowledge, chunkText, classifyError, classifyModel, clearPrefetchedShard, collectionFromShard, collectionToShard, computeBlobHash, computeHash, computeSri, conceptTaggingHandler, cosineSimilarity, createAiwgFetchChunkLoader, createAiwgFetchDetailLoader, createAiwgIndexController, createAiwgReviewDecisionExport, createBlobStore, createCosineSemanticProvider, createCspReportHandler, createFortemi, createLazyBlobStore, createLegacyProvider, createPGliteBackend, createPGliteInstance, createRecordBackend, createRecordStore, createRemoteBackend, createRoutes, createShardBackend, createShardCapabilityReport, createWorkerEmbedFunction, defaultStorageBackendFactory, detectCommunities, detectGpuCapabilities, detectInferenceCapabilities, discoverLocalProviders, dropAttachmentProjection, dropNoteProjection, dumpDbSnapshot, embeddingConfigToShard, embeddingFromShard, embeddingGenerationHandler, embeddingSetFromShard, embeddingSetMemberToShard, embeddingSetToShard, embeddingToShard, enqueueFullWorkflow, enqueueJob, enqueueNoteCreationJobs, estimateModelFit, estimateVramMB, estimateVramTier, exportShard, exportShardFromRecords, exportShardFromRecordsWithReport, exportShardWithReport, fetchAndValidateFortemiCompatibility, fetchPluginScript, filterAiwgRecordsByPrivacy, findAiwgStaticDuplicatePairs, formatFortemiCompatibilitySummary, fortemiCompatibilityUrl, fortemiManifest, fromPrefetched, generateId, getAiwgFortemiFacets, getAiwgFortemiIndexExportSchema, getEmbedFunction, getFortemiBridge, getFortemiSecretStore, getJobQueueStatus, getKnowledgeShardContractReceipt, getKnowledgeShardProfileRegistry, getKnowledgeShardSchema, getLlmFunction, getNote, getPrefetchedSha256, handleEmbedRequests, hasFortemiSecureSecrets, importShard, importShardToRecords, isPluginScriptAllowed, isShardPrefetched, isShardSigningSupported, linkFromShard, linkToShard, linkingHandler, listNotes, manageArchive, manageAttachments, manageCapabilities, manageCollections, manageLinks, manageNote, manageTags, matchRoute, migrateLegacyBlobStore, noteFromShard, noteSkosTagToShard, noteToShard, openShard, packTarGz, parseCspReport, prefetchShard, previewRecordStorePurge, profileSupportError, projectAttachments, projectNotes, projectRecords, provenanceEdgeToShard, purgeRecordStoreGraph, queryAiwgFortemiIndex, queryAiwgHybridIndex, queryAiwgSemanticIndex, registerLlmCapability, registerSemanticCapability, registerSemanticCapabilityWorker, registerServiceWorker, restoreDbSnapshot, searchTool, selectBackend, selectLlmModel, setEmbedFunction, setLlmFunction, sha256Hex, sidecarBlobDigests, signShard, skosConceptToShard, skosRelationToShard, skosSchemeToShard, suggestTags, tagsToShard, templateToShard, titleGenerationHandler, unpackTarGz, unregisterLlmCapability, unregisterSemanticCapability, upsertRecordStoreSources, urlLinkToShard, validateAiwgFortemiChunkManifest, validateAiwgFortemiChunkPart, validateAiwgFortemiIndexExport, validateAiwgFortemiIndexExportSchema, validateAiwgFortemiProjectedRecordSchema, validateAiwgStaticEmbeddingSet, validateChecksums, validateCoreV1ShardArchive, validateFortemiCompatibilityResponse, validateFullV1ShardArchive, validateRecordV1ShardArchive, validateShardArchive, validateShardComponentRecord, validateShardManifest, verifyDbSnapshotMeta, verifyShardSignature, verifySri };
|
|
39846
|
+
export { AIWG_SCAN_REQUIRED_FIELDS, AllowlistTrustStore, ArchiveManager, AttachmentsRepository, BridgeInferenceProvider, CORE_V1_COMPONENTS, CURRENT_MIGRATION_HEAD, CURRENT_SHARD_VERSION, CanonicalAttachmentsRepository, CanonicalNotesRepository, CapabilityManager, CaptureKnowledgeInputSchema, CollectionsRepository, CommunitiesRepository, DATASET_EXECUTION_CAPABILITY_IDS, DATASET_EXECUTION_CONTRACT, DATASET_EXECUTION_SCHEMA_VERSION, DATASET_INGEST_CONTRACT, DATASET_INGEST_SCHEMA_VERSION, DATASET_LINEAGE_CONTRACT, DATASET_LINEAGE_SCHEMA_VERSION, DATASET_MATERIALIZATION_CONTRACT, DATASET_MATERIALIZATION_KINDS, DATASET_MATERIALIZATION_SCHEMA_VERSION, DB_SNAPSHOT_SCHEMA_VERSION, DEFAULT_LARGE_DOCUMENT_CHARS, DEFAULT_LARGE_DOCUMENT_CHUNKS, DatasetIngestError, DatasetIngestExecutor, DatasetLineageLedger, DatasetMaterializationError, DbSnapshotVersionError, EMBED_REQUEST_KIND, EMBED_RESPONSE_KIND, EmbeddingSetsRepository, FORTEMI_BROWSER_LOCAL_DATASET_EXECUTION_DESCRIPTOR, FORTEMI_COMPATIBILITY_PATH, FORTEMI_COMPATIBILITY_STATES, FORTEMI_PORTABLE_SHARD_DATASET_EXECUTION_DESCRIPTOR, FORTEMI_REQUIRED_COMPATIBILITY_CAPABILITIES, FORTEMI_SERVER_COMPATIBILITY_REVISION, FORTEMI_STATIC_CACHE_DATASET_EXECUTION_DESCRIPTOR, FallbackRouter, FortemiToolManifest, GetNoteInputSchema, GraphRepository, IdbRecordStore, JOB_CAPABILITIES, JOB_PRIORITIES, JobQueueWorker, LINEAGE_ENTITY_KINDS, LINEAGE_RELATIONSHIP_KINDS, LOCAL_ENDPOINTS, LifecyclePurgeRepository, LineageValidationError, LinksRepository, ListNotesInputSchema, ManageArchiveInputSchema, ManageAttachmentsInputSchema, ManageCapabilitiesInputSchema, ManageCollectionsInputSchema, ManageLinksInputSchema, ManageNoteInputSchema, ManageTagsInputSchema, MemoryBlobStore, MemoryDatasetIngestStore, MemoryRecordStore, MigrationRunner, NotesRepository, OpenAICompatibleProvider, PGliteStorageBackend, PGliteStorageBackendFactory, PGliteWorkerClient, PGliteWorkerStorageBackend, PGliteWorkerStorageBackendFactory, ProvenanceRepository, ProviderRegistry, RECORD_COLLECTIONS, RECORD_SCHEMA_VERSION, RECORD_STORE_CAPABILITIES, REGISTERED_METADATA_PATHS, SHARD_FORMAT, SIGNATURE_ENTRY, SIGNING_ENVELOPE_VERSION, SOURCE_UPSERT_CONTRACT_VERSION, SOURCE_UPSERT_MAX_ITEMS, SUPPORTED_PGLITE_VERSION, SearchInputSchema, SearchRepository, SkosRepository, SourceUpsertRepository, TagsRepository, TransactionProxy, TypedEventBus, VERSION, aiRevisionHandler, aiwgFortemiIndexFromKnowledgeShard, aiwgFortemiIndexToCommunityGraph, aiwgFortemiIndexToKnowledgeShard, aiwgFortemiIndexToKnowledgeShardWithReport, allMigrations, appendPluginScript, assertAiwgFortemiChunkManifest, assertAiwgFortemiChunkPart, assertAiwgFortemiIndexExport, assertAiwgStaticEmbeddingSet, assertShardComponentRecord, buildAiwgChunkedIndex, buildAiwgStaticEmbeddingSet, buildMetadataPredicateConditions, buildNoteConditions, buildPluginCsp, captureKnowledge, chunkText, classifyError, classifyModel, clearPrefetchedShard, collectionFromShard, collectionToShard, compareDatasetIncrementalParity, computeBlobHash, computeHash, computeLineageDigest, computeSri, conceptTaggingHandler, configureInferenceRuntime, cosineSimilarity, createAiwgFetchChunkLoader, createAiwgFetchDetailLoader, createAiwgIndexController, createAiwgReviewDecisionExport, createBlobStore, createBridgeInferenceProviders, createCosineSemanticProvider, createCspReportHandler, createFortemi, createLazyBlobStore, createLegacyProvider, createLocalProviderProfile, createPGliteBackend, createPGliteInstance, createRecordBackend, createRecordStore, createRemoteBackend, createRoutes, createShardBackend, createShardCapabilityReport, createWorkerEmbedFunction, datasetDestinationScopeKey, defaultStorageBackendFactory, defineInferenceProvider, defineInferenceRuntime, defineLegacyInferenceProvider, defineOpenAICompatibleProvider, deriveDatasetIngestIdempotencyKey, detectCommunities, detectGpuCapabilities, detectInferenceCapabilities, digestDatasetMaterializationValue, discoverLocalProviders, dropAttachmentProjection, dropNoteProjection, dumpDbSnapshot, embeddingConfigToShard, embeddingFromShard, embeddingGenerationHandler, embeddingSetFromShard, embeddingSetMemberToShard, embeddingSetToShard, embeddingToShard, enqueueFullWorkflow, enqueueJob, enqueueNoteCreationJobs, estimateModelFit, estimateVramMB, estimateVramTier, executeDatasetMaterialization, executeDatasetRetrieval, exportShard, exportShardFromRecords, exportShardFromRecordsWithReport, exportShardWithReport, fetchAndValidateFortemiCompatibility, fetchPluginScript, filterAiwgRecordsByPrivacy, findAiwgStaticDuplicatePairs, formatFortemiCompatibilitySummary, fortemiCompatibilityUrl, fortemiManifest, fromPrefetched, generateId, getAiwgFortemiFacets, getAiwgFortemiIndexExportSchema, getConfiguredInferenceProviderId, getEmbedFunction, getEmbeddingTaskSelectionOptions, getFortemiBridge, getFortemiSecretStore, getJobQueueStatus, getKnowledgeShardContractReceipt, getKnowledgeShardProfileRegistry, getKnowledgeShardSchema, getLlmFunction, getNote, getPrefetchedSha256, getProviderRouteRequirementIssue, handleEmbedRequests, hasFortemiSecureSecrets, importShard, importShardToRecords, inferInferenceTaskCapability, inferLocalEmbeddingDimensions, isPluginScriptAllowed, isShardPrefetched, isShardSigningSupported, linkFromShard, linkToShard, linkingHandler, listNotes, manageArchive, manageAttachments, manageCapabilities, manageCollections, manageLinks, manageNote, manageTags, matchRoute, mergeInferenceRuntimeConfigs, migrateLegacyBlobStore, negotiateDatasetExecutionCapabilities, negotiateDatasetMaterializationProfile, noteFromShard, noteSkosTagToShard, noteToShard, openShard, packTarGz, parseCspReport, prefetchShard, previewRecordStorePurge, profileSupportError, projectAttachments, projectNotes, projectRecords, provenanceEdgeToShard, providerSatisfiesRouteRequirements, purgeRecordStoreGraph, queryAiwgFortemiIndex, queryAiwgHybridIndex, queryAiwgSemanticIndex, registerLlmCapability, registerSemanticCapability, registerSemanticCapabilityWorker, registerServiceWorker, restoreDbSnapshot, searchTool, selectBackend, selectEmbeddingTask, selectLlmModel, setEmbedFunction, setEmbeddingTaskSelectionOptions, setLlmFunction, sha256Hex, sidecarBlobDigests, signShard, skosConceptToShard, skosRelationToShard, skosSchemeToShard, suggestTags, tagsToShard, templateToShard, titleGenerationHandler, unpackTarGz, unregisterLlmCapability, unregisterSemanticCapability, upsertRecordStoreRequest, upsertRecordStoreSources, urlLinkToShard, validateAiwgFortemiChunkManifest, validateAiwgFortemiChunkPart, validateAiwgFortemiIndexExport, validateAiwgFortemiIndexExportSchema, validateAiwgFortemiProjectedRecordSchema, validateAiwgStaticEmbeddingSet, validateChecksums, validateCoreV1ShardArchive, validateDatasetBenchmarkEvidence, validateDatasetExecutionDescriptor, validateFortemiCompatibilityResponse, validateFullV1ShardArchive, validateProviderRoute, validateRecordV1ShardArchive, validateShardArchive, validateShardComponentRecord, validateShardManifest, verifyDbSnapshotMeta, verifyShardSignature, verifySri };
|
|
37876
39847
|
//# sourceMappingURL=index.js.map
|
|
37877
39848
|
//# sourceMappingURL=index.js.map
|