@fortemi/core 2026.7.15 → 2026.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -1
- package/benchmarks/dataset-materialization/small-corpus.v1.json +12 -0
- package/dist/index.d.ts +1379 -130
- package/dist/index.js +3180 -425
- package/dist/index.js.map +1 -1
- package/package.json +10 -1
- package/schemas/dataset-execution-capabilities/fixtures/browser-local.json +14 -0
- package/schemas/dataset-execution-capabilities/fixtures/invalid-field-lineage-without-evidence.json +10 -0
- package/schemas/dataset-execution-capabilities/fixtures/invalid-incremental-without-checkpoint.json +10 -0
- package/schemas/dataset-execution-capabilities/fixtures/portable-shard.json +12 -0
- package/schemas/dataset-execution-capabilities/fixtures/remote-alpha.json +12 -0
- package/schemas/dataset-execution-capabilities/fixtures/static-cache.json +12 -0
- package/schemas/dataset-execution-capabilities/v1.schema.json +164 -0
- package/schemas/dataset-ingest/v1.schema.json +138 -0
- package/schemas/dataset-lineage/fixtures/golden-observed-field.json +20 -0
- package/schemas/dataset-lineage/v1.schema.json +126 -0
- package/schemas/dataset-materialization/fixtures/browser.json +11 -0
- package/schemas/dataset-materialization/fixtures/degraded.json +12 -0
- package/schemas/dataset-materialization/fixtures/deterministic.json +11 -0
- package/schemas/dataset-materialization/fixtures/external-adapter.json +11 -0
- package/schemas/dataset-materialization/fixtures/nondeterministic.json +11 -0
- package/schemas/dataset-materialization/fixtures/server.json +11 -0
- package/schemas/dataset-materialization/fixtures/supported.json +12 -0
- package/schemas/dataset-materialization/fixtures/unsupported.json +11 -0
- package/schemas/dataset-materialization/v1.schema.json +169 -0
package/dist/index.js
CHANGED
|
@@ -2,14 +2,1081 @@ import { v7, v5 } from 'uuid';
|
|
|
2
2
|
import { sha256 } from '@noble/hashes/sha256';
|
|
3
3
|
import { blake3 } from '@noble/hashes/blake3';
|
|
4
4
|
import { bytesToHex } from '@noble/hashes/utils';
|
|
5
|
+
import Ajv20202 from 'ajv/dist/2020.js';
|
|
5
6
|
import { z } from 'zod';
|
|
6
7
|
import { gzipSync, gunzipSync } from 'fflate';
|
|
7
|
-
import Ajv2020 from 'ajv/dist/2020.js';
|
|
8
8
|
|
|
9
9
|
// src/uuid.ts
|
|
10
10
|
function generateId() {
|
|
11
11
|
return v7();
|
|
12
12
|
}
|
|
13
|
+
function computeHash(data) {
|
|
14
|
+
const digest2 = sha256(data);
|
|
15
|
+
return `sha256:${bytesToHex(digest2)}`;
|
|
16
|
+
}
|
|
17
|
+
function computeBlobHash(data) {
|
|
18
|
+
return `blake3:${bytesToHex(blake3(data))}`;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
// src/dataset-ingest.ts
|
|
22
|
+
var DATASET_INGEST_CONTRACT = "fortemi.dataset-ingest/v1";
|
|
23
|
+
var DATASET_INGEST_SCHEMA_VERSION = "1.0.0";
|
|
24
|
+
var DatasetIngestError = class extends Error {
|
|
25
|
+
constructor(code, message) {
|
|
26
|
+
super(message);
|
|
27
|
+
this.code = code;
|
|
28
|
+
this.name = "DatasetIngestError";
|
|
29
|
+
}
|
|
30
|
+
};
|
|
31
|
+
function clone(value) {
|
|
32
|
+
return structuredClone(value);
|
|
33
|
+
}
|
|
34
|
+
var MemoryDatasetIngestStore = class {
|
|
35
|
+
scopes = /* @__PURE__ */ new Map();
|
|
36
|
+
queues = /* @__PURE__ */ new Map();
|
|
37
|
+
async transact(scopeKey, operation) {
|
|
38
|
+
const predecessor = this.queues.get(scopeKey) ?? Promise.resolve();
|
|
39
|
+
let release;
|
|
40
|
+
const current = new Promise((resolve) => {
|
|
41
|
+
release = resolve;
|
|
42
|
+
});
|
|
43
|
+
const queued = predecessor.then(() => current);
|
|
44
|
+
this.queues.set(scopeKey, queued);
|
|
45
|
+
await predecessor;
|
|
46
|
+
try {
|
|
47
|
+
const existing = this.scopes.get(scopeKey) ?? {
|
|
48
|
+
records: /* @__PURE__ */ new Map(),
|
|
49
|
+
receipts: /* @__PURE__ */ new Map()
|
|
50
|
+
};
|
|
51
|
+
const draft = {
|
|
52
|
+
records: new Map([...existing.records].map(([key, value]) => [key, clone(value)])),
|
|
53
|
+
receipts: new Map([...existing.receipts].map(([key, value]) => [key, clone(value)])),
|
|
54
|
+
...existing.checkpoint ? { checkpoint: clone(existing.checkpoint) } : {}
|
|
55
|
+
};
|
|
56
|
+
const transaction = {
|
|
57
|
+
getRecord: (id) => draft.records.get(id),
|
|
58
|
+
setRecord: (record) => draft.records.set(record.logicalId, clone(record)),
|
|
59
|
+
getReceipt: (key) => draft.receipts.get(key),
|
|
60
|
+
setReceipt: (receipt) => draft.receipts.set(receipt.idempotencyKey, clone(receipt)),
|
|
61
|
+
getCheckpoint: () => draft.checkpoint,
|
|
62
|
+
setCheckpoint: (checkpoint) => {
|
|
63
|
+
draft.checkpoint = clone(checkpoint);
|
|
64
|
+
}
|
|
65
|
+
};
|
|
66
|
+
const result = await operation(transaction);
|
|
67
|
+
this.scopes.set(scopeKey, draft);
|
|
68
|
+
return result;
|
|
69
|
+
} finally {
|
|
70
|
+
release();
|
|
71
|
+
if (this.queues.get(scopeKey) === queued) this.queues.delete(scopeKey);
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
async getReceipt(scopeKey, idempotencyKey) {
|
|
75
|
+
return clone(this.scopes.get(scopeKey)?.receipts.get(idempotencyKey));
|
|
76
|
+
}
|
|
77
|
+
async getCheckpoint(scopeKey) {
|
|
78
|
+
return clone(this.scopes.get(scopeKey)?.checkpoint);
|
|
79
|
+
}
|
|
80
|
+
async getRecords(scopeKey) {
|
|
81
|
+
return [...this.scopes.get(scopeKey)?.records.values() ?? []].map(clone).sort((a, b) => a.logicalId.localeCompare(b.logicalId));
|
|
82
|
+
}
|
|
83
|
+
};
|
|
84
|
+
function canonicalJson(value) {
|
|
85
|
+
if (value === null || typeof value !== "object") return JSON.stringify(value);
|
|
86
|
+
if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`;
|
|
87
|
+
return `{${Object.entries(value).sort(([a], [b]) => a.localeCompare(b)).map(([key, item]) => `${JSON.stringify(key)}:${canonicalJson(item)}`).join(",")}}`;
|
|
88
|
+
}
|
|
89
|
+
function digest(value) {
|
|
90
|
+
return computeHash(new TextEncoder().encode(canonicalJson(value)));
|
|
91
|
+
}
|
|
92
|
+
function datasetDestinationScopeKey(scope) {
|
|
93
|
+
return [scope.tenant, scope.dataset, scope.sourceBinding, scope.stream, scope.partition ?? ""].map(encodeURIComponent).join("/");
|
|
94
|
+
}
|
|
95
|
+
function deriveDatasetIngestIdempotencyKey(plan, batch) {
|
|
96
|
+
if (batch.idempotencyKey) return batch.idempotencyKey;
|
|
97
|
+
return digest({
|
|
98
|
+
contract: DATASET_INGEST_CONTRACT,
|
|
99
|
+
planId: plan.planId,
|
|
100
|
+
planDigest: plan.planDigest,
|
|
101
|
+
sourceRevision: plan.sourceRevision,
|
|
102
|
+
configurationDigest: plan.configurationDigest,
|
|
103
|
+
transformationDigest: plan.transformationDigest,
|
|
104
|
+
destination: plan.destination,
|
|
105
|
+
mode: plan.mode,
|
|
106
|
+
sequence: batch.sequence,
|
|
107
|
+
checkpointBefore: batch.checkpointBefore,
|
|
108
|
+
checkpointAfter: batch.checkpointAfter,
|
|
109
|
+
mutations: batch.mutations
|
|
110
|
+
});
|
|
111
|
+
}
|
|
112
|
+
function assertContract(plan, batch) {
|
|
113
|
+
if (plan.contract !== DATASET_INGEST_CONTRACT || batch.contract !== DATASET_INGEST_CONTRACT) {
|
|
114
|
+
throw new DatasetIngestError("INGEST_CONTRACT_UNSUPPORTED", "Unsupported dataset ingest contract");
|
|
115
|
+
}
|
|
116
|
+
if (!plan.schemaVersion.startsWith("1.") || !batch.schemaVersion.startsWith("1.")) {
|
|
117
|
+
throw new DatasetIngestError("INGEST_SCHEMA_UNSUPPORTED", "Unsupported dataset ingest schema version");
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
function assertCheckpointScope(checkpoint, scope) {
|
|
121
|
+
if (checkpoint.contract !== DATASET_INGEST_CONTRACT) throw new DatasetIngestError("INGEST_CONTRACT_UNSUPPORTED", "Unsupported checkpoint contract");
|
|
122
|
+
if (!checkpoint.schemaVersion.startsWith("1.")) throw new DatasetIngestError("CHECKPOINT_VERSION_UNSUPPORTED", `Unsupported checkpoint schema ${checkpoint.schemaVersion}`);
|
|
123
|
+
if (datasetDestinationScopeKey(checkpoint.scope) !== datasetDestinationScopeKey(scope)) {
|
|
124
|
+
throw new DatasetIngestError("CHECKPOINT_SCOPE_MISMATCH", "Checkpoint belongs to another destination scope");
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
var DatasetIngestExecutor = class {
|
|
128
|
+
constructor(store) {
|
|
129
|
+
this.store = store;
|
|
130
|
+
}
|
|
131
|
+
attempts = /* @__PURE__ */ new Map();
|
|
132
|
+
successes = /* @__PURE__ */ new Map();
|
|
133
|
+
preview(plan, batch) {
|
|
134
|
+
assertContract(plan, batch);
|
|
135
|
+
return {
|
|
136
|
+
idempotencyKey: deriveDatasetIngestIdempotencyKey(plan, batch),
|
|
137
|
+
upserts: batch.mutations.filter((item) => item.operation === "upsert").length,
|
|
138
|
+
tombstones: batch.mutations.filter((item) => item.operation === "tombstone").length
|
|
139
|
+
};
|
|
140
|
+
}
|
|
141
|
+
connectionCheck(plan, batch) {
|
|
142
|
+
assertContract(plan, batch);
|
|
143
|
+
assertCheckpointScope(batch.checkpointAfter, plan.destination);
|
|
144
|
+
if (batch.checkpointBefore) assertCheckpointScope(batch.checkpointBefore, plan.destination);
|
|
145
|
+
return {
|
|
146
|
+
idempotencyKey: deriveDatasetIngestIdempotencyKey(plan, batch),
|
|
147
|
+
scopeKey: datasetDestinationScopeKey(plan.destination),
|
|
148
|
+
compatible: true
|
|
149
|
+
};
|
|
150
|
+
}
|
|
151
|
+
async executeBatch(plan, batch, options = {}) {
|
|
152
|
+
assertContract(plan, batch);
|
|
153
|
+
const scopeKey = datasetDestinationScopeKey(plan.destination);
|
|
154
|
+
const idempotencyKey = deriveDatasetIngestIdempotencyKey(plan, batch);
|
|
155
|
+
const requestDigest = digest({ plan, batch });
|
|
156
|
+
const runId = generateId();
|
|
157
|
+
this.attempts.set(scopeKey, { runId, state: "running", verification: "pending", idempotencyKey });
|
|
158
|
+
const cancel = () => {
|
|
159
|
+
if (options.signal?.aborted) throw new DatasetIngestError("INGEST_CANCELLED", "Dataset ingest was cancelled before commit");
|
|
160
|
+
};
|
|
161
|
+
try {
|
|
162
|
+
cancel();
|
|
163
|
+
const receipt = await this.store.transact(scopeKey, async (transaction) => {
|
|
164
|
+
const prior = transaction.getReceipt(idempotencyKey);
|
|
165
|
+
if (prior) {
|
|
166
|
+
if (prior.requestDigest !== requestDigest) throw new DatasetIngestError("IDEMPOTENCY_CONFLICT", "Idempotency key was previously used with different canonical content");
|
|
167
|
+
return prior;
|
|
168
|
+
}
|
|
169
|
+
const currentCheckpoint = transaction.getCheckpoint();
|
|
170
|
+
if (batch.checkpointBefore) {
|
|
171
|
+
assertCheckpointScope(batch.checkpointBefore, plan.destination);
|
|
172
|
+
if (!currentCheckpoint || canonicalJson(currentCheckpoint) !== canonicalJson(batch.checkpointBefore)) {
|
|
173
|
+
throw new DatasetIngestError("CHECKPOINT_MISMATCH", "checkpointBefore does not match committed state");
|
|
174
|
+
}
|
|
175
|
+
} else if (currentCheckpoint) {
|
|
176
|
+
throw new DatasetIngestError("CHECKPOINT_MISMATCH", "A committed checkpoint exists but checkpointBefore was omitted");
|
|
177
|
+
}
|
|
178
|
+
assertCheckpointScope(batch.checkpointAfter, plan.destination);
|
|
179
|
+
if (batch.checkpointAfter.sequence !== batch.sequence) throw new DatasetIngestError("BATCH_OUT_OF_ORDER", "Checkpoint sequence must equal batch sequence");
|
|
180
|
+
if (currentCheckpoint && batch.checkpointAfter.sequence <= currentCheckpoint.sequence) throw new DatasetIngestError("CHECKPOINT_REGRESSION", "Checkpoint sequence must advance");
|
|
181
|
+
const expectedSequence = (currentCheckpoint?.sequence ?? 0) + 1;
|
|
182
|
+
if (batch.sequence !== expectedSequence) throw new DatasetIngestError("BATCH_OUT_OF_ORDER", `Expected batch sequence ${expectedSequence}, received ${batch.sequence}`);
|
|
183
|
+
const tombstones = batch.mutations.filter((item) => item.operation === "tombstone").length;
|
|
184
|
+
if (tombstones > 0) {
|
|
185
|
+
if (!plan.reconciliation.enabled) throw new DatasetIngestError("RECONCILIATION_NOT_ENABLED", "Tombstones require reconciliation policy");
|
|
186
|
+
if (!batch.enumeration?.complete) throw new DatasetIngestError("RECONCILIATION_INCOMPLETE", "Tombstones require complete source enumeration");
|
|
187
|
+
if (tombstones > plan.reconciliation.maxTombstones && !batch.enumeration.approvalId) {
|
|
188
|
+
throw new DatasetIngestError("RECONCILIATION_APPROVAL_REQUIRED", `Tombstone count ${tombstones} exceeds approved threshold`);
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
const rejections = [];
|
|
192
|
+
const accepted = [];
|
|
193
|
+
for (const mutation of batch.mutations) {
|
|
194
|
+
cancel();
|
|
195
|
+
const failure2 = options.validateRecord?.(mutation);
|
|
196
|
+
if (!failure2) {
|
|
197
|
+
accepted.push(mutation);
|
|
198
|
+
continue;
|
|
199
|
+
}
|
|
200
|
+
const rejection = {
|
|
201
|
+
logicalIdDigest: digest(mutation.logicalId),
|
|
202
|
+
...mutation.locator ? { locator: mutation.locator } : {},
|
|
203
|
+
code: failure2.code,
|
|
204
|
+
message: "Record rejected by validation policy"
|
|
205
|
+
};
|
|
206
|
+
if (plan.rejectionPolicy.mode === "fail-fast") throw new DatasetIngestError("RECORD_REJECTED", `${failure2.code}: record rejected`);
|
|
207
|
+
rejections.push(rejection);
|
|
208
|
+
if (rejections.length > plan.rejectionPolicy.maxRejectedRecords) throw new DatasetIngestError("REJECTION_LIMIT_EXCEEDED", "Rejected-record limit exceeded");
|
|
209
|
+
}
|
|
210
|
+
cancel();
|
|
211
|
+
for (const mutation of accepted) {
|
|
212
|
+
transaction.setRecord(mutation.operation === "upsert" ? { logicalId: mutation.logicalId, revision: mutation.revision, digest: mutation.digest, value: clone(mutation.value), tombstoned: false } : { logicalId: mutation.logicalId, revision: mutation.revision, digest: mutation.digest, tombstoned: true });
|
|
213
|
+
}
|
|
214
|
+
const effects = accepted.map(({ operation, logicalId, revision, digest: itemDigest }) => ({ operation, logicalId, revision, digest: itemDigest }));
|
|
215
|
+
const receipt2 = {
|
|
216
|
+
contract: DATASET_INGEST_CONTRACT,
|
|
217
|
+
schemaVersion: DATASET_INGEST_SCHEMA_VERSION,
|
|
218
|
+
runId,
|
|
219
|
+
idempotencyKey,
|
|
220
|
+
requestDigest,
|
|
221
|
+
planId: plan.planId,
|
|
222
|
+
planDigest: plan.planDigest,
|
|
223
|
+
sourceRevision: plan.sourceRevision,
|
|
224
|
+
destination: clone(plan.destination),
|
|
225
|
+
mode: plan.mode,
|
|
226
|
+
state: rejections.length ? "degraded" : "committed",
|
|
227
|
+
effects,
|
|
228
|
+
acceptedRecords: accepted.length,
|
|
229
|
+
rejectedRecords: rejections.length,
|
|
230
|
+
rejections,
|
|
231
|
+
outputDigest: digest(effects),
|
|
232
|
+
...batch.checkpointBefore ? { checkpointBefore: clone(batch.checkpointBefore) } : {},
|
|
233
|
+
checkpointAfter: clone(batch.checkpointAfter),
|
|
234
|
+
verification: "verified"
|
|
235
|
+
};
|
|
236
|
+
await options.hooks?.beforeCommit?.();
|
|
237
|
+
cancel();
|
|
238
|
+
transaction.setReceipt(receipt2);
|
|
239
|
+
transaction.setCheckpoint(batch.checkpointAfter);
|
|
240
|
+
return receipt2;
|
|
241
|
+
});
|
|
242
|
+
this.successes.set(scopeKey, clone(receipt));
|
|
243
|
+
this.attempts.set(scopeKey, { runId: receipt.runId, state: receipt.state, verification: "verified", idempotencyKey });
|
|
244
|
+
await options.hooks?.afterCommit?.(clone(receipt));
|
|
245
|
+
return receipt;
|
|
246
|
+
} catch (error) {
|
|
247
|
+
const code = error instanceof DatasetIngestError ? error.code : void 0;
|
|
248
|
+
this.attempts.set(scopeKey, { runId, state: code === "INGEST_CANCELLED" ? "cancelled" : "failed", verification: "failed", idempotencyKey, ...code ? { errorCode: code } : {} });
|
|
249
|
+
throw error;
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
async resolveAmbiguousCommit(plan, batch) {
|
|
253
|
+
return this.store.getReceipt(datasetDestinationScopeKey(plan.destination), deriveDatasetIngestIdempotencyKey(plan, batch));
|
|
254
|
+
}
|
|
255
|
+
status(scope, expectedSourceRevision) {
|
|
256
|
+
const key = datasetDestinationScopeKey(scope);
|
|
257
|
+
const lastSuccessful = this.successes.get(key);
|
|
258
|
+
return {
|
|
259
|
+
scope: clone(scope),
|
|
260
|
+
...this.attempts.get(key) ? { lastAttempt: clone(this.attempts.get(key)) } : {},
|
|
261
|
+
...lastSuccessful ? { lastSuccessful: clone(lastSuccessful) } : {},
|
|
262
|
+
freshness: lastSuccessful ? expectedSourceRevision && expectedSourceRevision !== lastSuccessful.sourceRevision ? "stale" : "current" : "never"
|
|
263
|
+
};
|
|
264
|
+
}
|
|
265
|
+
};
|
|
266
|
+
|
|
267
|
+
// src/dataset-lineage.ts
|
|
268
|
+
var DATASET_LINEAGE_CONTRACT = "fortemi.dataset-lineage/v1";
|
|
269
|
+
var DATASET_LINEAGE_SCHEMA_VERSION = "1.0.0";
|
|
270
|
+
var LINEAGE_ENTITY_KINDS = [
|
|
271
|
+
"dataset",
|
|
272
|
+
"dataset-revision",
|
|
273
|
+
"distribution",
|
|
274
|
+
"record",
|
|
275
|
+
"field",
|
|
276
|
+
"chunk",
|
|
277
|
+
"index",
|
|
278
|
+
"embedding-set",
|
|
279
|
+
"graph-artifact",
|
|
280
|
+
"community-artifact",
|
|
281
|
+
"processing-plan",
|
|
282
|
+
"run"
|
|
283
|
+
];
|
|
284
|
+
var LINEAGE_RELATIONSHIP_KINDS = [
|
|
285
|
+
"derived-from",
|
|
286
|
+
"field-derived-from",
|
|
287
|
+
"extracted-from",
|
|
288
|
+
"chunk-of",
|
|
289
|
+
"indexed-from",
|
|
290
|
+
"embedded-from",
|
|
291
|
+
"graph-derived-from",
|
|
292
|
+
"community-derived-from",
|
|
293
|
+
"revision-of",
|
|
294
|
+
"distributed-as",
|
|
295
|
+
"join-influence",
|
|
296
|
+
"filter-influence",
|
|
297
|
+
"aggregation-influence",
|
|
298
|
+
"ordering-influence",
|
|
299
|
+
"similarity-influence"
|
|
300
|
+
];
|
|
301
|
+
var LineageValidationError = class extends Error {
|
|
302
|
+
constructor(code, message) {
|
|
303
|
+
super(message);
|
|
304
|
+
this.code = code;
|
|
305
|
+
this.name = "LineageValidationError";
|
|
306
|
+
}
|
|
307
|
+
};
|
|
308
|
+
function canonicalJson2(value) {
|
|
309
|
+
if (value === void 0) return "null";
|
|
310
|
+
if (value === null || typeof value !== "object") return JSON.stringify(value);
|
|
311
|
+
if (Array.isArray(value)) return `[${value.map(canonicalJson2).join(",")}]`;
|
|
312
|
+
const entries = Object.entries(value).filter(([, item]) => item !== void 0).sort(([left], [right]) => left.localeCompare(right));
|
|
313
|
+
return `{${entries.map(([key, item]) => `${JSON.stringify(key)}:${canonicalJson2(item)}`).join(",")}}`;
|
|
314
|
+
}
|
|
315
|
+
function computeLineageDigest(value) {
|
|
316
|
+
return computeHash(new TextEncoder().encode(canonicalJson2(value)));
|
|
317
|
+
}
|
|
318
|
+
function clone2(value) {
|
|
319
|
+
return structuredClone(value);
|
|
320
|
+
}
|
|
321
|
+
function requireIdentity(id, label) {
|
|
322
|
+
if (!id || id.trim() !== id) throw new LineageValidationError("IDENTITY_REQUIRED", `${label} must be a non-empty canonical identity`);
|
|
323
|
+
}
|
|
324
|
+
function requireTimestamp(value, label) {
|
|
325
|
+
if (!/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{3})?Z$/.test(value) || !Number.isFinite(Date.parse(value))) {
|
|
326
|
+
throw new LineageValidationError("VALUE_INVALID", `${label} must be an RFC 3339 UTC timestamp`);
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
var ENTITY_KIND_SET = new Set(LINEAGE_ENTITY_KINDS);
|
|
330
|
+
var RELATIONSHIP_KIND_SET = new Set(LINEAGE_RELATIONSHIP_KINDS);
|
|
331
|
+
var RELATIONSHIP_ENDPOINTS = {
|
|
332
|
+
"field-derived-from": [["field"], ["field"]],
|
|
333
|
+
"chunk-of": [["chunk"], ["record", "distribution"]],
|
|
334
|
+
"indexed-from": [["index"], ["dataset", "dataset-revision", "distribution", "record", "field", "chunk"]],
|
|
335
|
+
"embedded-from": [["embedding-set"], ["dataset", "dataset-revision", "record", "field", "chunk"]],
|
|
336
|
+
"graph-derived-from": [["graph-artifact"], ["dataset", "dataset-revision", "index", "embedding-set"]],
|
|
337
|
+
"community-derived-from": [["community-artifact"], ["graph-artifact"]],
|
|
338
|
+
"revision-of": [["dataset-revision"], ["dataset"]],
|
|
339
|
+
"distributed-as": [["dataset", "dataset-revision"], ["distribution"]]
|
|
340
|
+
};
|
|
341
|
+
var DatasetLineageLedger = class _DatasetLineageLedger {
|
|
342
|
+
sequence = 0;
|
|
343
|
+
entities = /* @__PURE__ */ new Map();
|
|
344
|
+
agents = /* @__PURE__ */ new Map();
|
|
345
|
+
activities = /* @__PURE__ */ new Map();
|
|
346
|
+
evidence = /* @__PURE__ */ new Map();
|
|
347
|
+
assertions = /* @__PURE__ */ new Map();
|
|
348
|
+
corrections = /* @__PURE__ */ new Map();
|
|
349
|
+
maximumTraversalDepth;
|
|
350
|
+
maximumTraversalResults;
|
|
351
|
+
maximumPageSize;
|
|
352
|
+
constructor(options = {}) {
|
|
353
|
+
this.maximumTraversalDepth = options.maximumTraversalDepth ?? 16;
|
|
354
|
+
this.maximumTraversalResults = options.maximumTraversalResults ?? 1e4;
|
|
355
|
+
this.maximumPageSize = options.maximumPageSize ?? 500;
|
|
356
|
+
}
|
|
357
|
+
get snapshot() {
|
|
358
|
+
return this.sequence;
|
|
359
|
+
}
|
|
360
|
+
appendEntity(entity) {
|
|
361
|
+
requireIdentity(entity.id, "entity.id");
|
|
362
|
+
if (!ENTITY_KIND_SET.has(entity.kind)) throw new LineageValidationError("VALUE_INVALID", `Unsupported entity kind ${String(entity.kind)}`);
|
|
363
|
+
requireIdentity(entity.schemaId, "entity.schemaId");
|
|
364
|
+
requireIdentity(entity.schemaVersion, "entity.schemaVersion");
|
|
365
|
+
requireTimestamp(entity.createdAt, "entity.createdAt");
|
|
366
|
+
this.ensureUnique(this.entities, entity.id, "entity");
|
|
367
|
+
return this.append(this.entities, entity.id, entity);
|
|
368
|
+
}
|
|
369
|
+
appendAgent(agent) {
|
|
370
|
+
requireIdentity(agent.id, "agent.id");
|
|
371
|
+
requireIdentity(agent.name, "agent.name");
|
|
372
|
+
this.ensureUnique(this.agents, agent.id, "agent");
|
|
373
|
+
return this.append(this.agents, agent.id, agent);
|
|
374
|
+
}
|
|
375
|
+
appendActivity(activity) {
|
|
376
|
+
requireIdentity(activity.id, "activity.id");
|
|
377
|
+
requireTimestamp(activity.startedAt, "activity.startedAt");
|
|
378
|
+
if (activity.endedAt) requireTimestamp(activity.endedAt, "activity.endedAt");
|
|
379
|
+
for (const agentId of activity.agentIds) {
|
|
380
|
+
if (!this.agents.has(agentId)) throw new LineageValidationError("AGENT_DANGLING", `Activity references unknown agent ${agentId}`);
|
|
381
|
+
}
|
|
382
|
+
this.ensureUnique(this.activities, activity.id, "activity");
|
|
383
|
+
return this.append(this.activities, activity.id, activity);
|
|
384
|
+
}
|
|
385
|
+
appendEvidence(item) {
|
|
386
|
+
requireIdentity(item.id, "evidence.id");
|
|
387
|
+
requireIdentity(item.revision, "evidence.revision");
|
|
388
|
+
requireIdentity(item.locator, "evidence.locator");
|
|
389
|
+
requireTimestamp(item.capturedAt, "evidence.capturedAt");
|
|
390
|
+
if (item.payload !== void 0 && computeLineageDigest(item.payload) !== item.digest) {
|
|
391
|
+
throw new LineageValidationError("EVIDENCE_DIGEST_MISMATCH", `Evidence ${item.id} payload does not match ${item.digest}`);
|
|
392
|
+
}
|
|
393
|
+
const key = `${item.id}@${item.revision}`;
|
|
394
|
+
this.ensureUnique(this.evidence, key, "evidence revision");
|
|
395
|
+
return this.append(this.evidence, key, item);
|
|
396
|
+
}
|
|
397
|
+
appendAssertion(assertion) {
|
|
398
|
+
requireIdentity(assertion.id, "assertion.id");
|
|
399
|
+
requireIdentity(assertion.revision, "assertion.revision");
|
|
400
|
+
requireIdentity(assertion.method, "assertion.method");
|
|
401
|
+
requireIdentity(assertion.schemaId, "assertion.schemaId");
|
|
402
|
+
requireIdentity(assertion.schemaVersion, "assertion.schemaVersion");
|
|
403
|
+
requireTimestamp(assertion.assertedAt, "assertion.assertedAt");
|
|
404
|
+
if (!RELATIONSHIP_KIND_SET.has(assertion.relationship)) throw new LineageValidationError("VALUE_INVALID", `Unsupported relationship ${String(assertion.relationship)}`);
|
|
405
|
+
if (!Number.isFinite(assertion.confidence) || assertion.confidence < 0 || assertion.confidence > 1) {
|
|
406
|
+
throw new LineageValidationError("VALUE_INVALID", "assertion.confidence must be between 0 and 1");
|
|
407
|
+
}
|
|
408
|
+
const source = this.entities.get(assertion.sourceEntityId)?.value;
|
|
409
|
+
const target = this.entities.get(assertion.targetEntityId)?.value;
|
|
410
|
+
if (!source || !target) throw new LineageValidationError("IDENTITY_DANGLING", "Assertion source and target must both exist");
|
|
411
|
+
const endpointRule = RELATIONSHIP_ENDPOINTS[assertion.relationship];
|
|
412
|
+
if (endpointRule && (!endpointRule[0].includes(source.kind) || !endpointRule[1].includes(target.kind))) {
|
|
413
|
+
throw new LineageValidationError("TYPE_DIRECTION_INVALID", `${assertion.relationship} does not allow ${source.kind} -> ${target.kind}`);
|
|
414
|
+
}
|
|
415
|
+
if (!this.agents.has(assertion.issuerAgentId)) throw new LineageValidationError("AGENT_DANGLING", `Unknown issuer agent ${assertion.issuerAgentId}`);
|
|
416
|
+
if (assertion.producingActivityId && !this.activities.has(assertion.producingActivityId)) {
|
|
417
|
+
throw new LineageValidationError("IDENTITY_DANGLING", `Unknown producing activity ${assertion.producingActivityId}`);
|
|
418
|
+
}
|
|
419
|
+
if (assertion.assertionKind === "observed" && !assertion.producingActivityId) {
|
|
420
|
+
throw new LineageValidationError("ACTIVITY_REQUIRED", "Observed assertions require a producing activity");
|
|
421
|
+
}
|
|
422
|
+
if (assertion.assertionKind === "observed" && assertion.evidence.length === 0) {
|
|
423
|
+
throw new LineageValidationError("EVIDENCE_DANGLING", "Observed assertions require evidence");
|
|
424
|
+
}
|
|
425
|
+
for (const reference of assertion.evidence) {
|
|
426
|
+
const item = this.evidence.get(`${reference.evidenceId}@${reference.revision}`)?.value;
|
|
427
|
+
if (!item) throw new LineageValidationError("EVIDENCE_DANGLING", `Unknown evidence ${reference.evidenceId}@${reference.revision}`);
|
|
428
|
+
if (item.digest !== reference.digest) throw new LineageValidationError("EVIDENCE_DIGEST_MISMATCH", `Evidence reference digest differs for ${reference.evidenceId}`);
|
|
429
|
+
if (reference.locator && reference.locator !== item.locator) throw new LineageValidationError("EVIDENCE_DIGEST_MISMATCH", `Evidence locator differs for ${reference.evidenceId}`);
|
|
430
|
+
}
|
|
431
|
+
const key = `${assertion.id}@${assertion.revision}`;
|
|
432
|
+
if (this.assertions.has(key)) throw new LineageValidationError("IDENTITY_DUPLICATE", `Duplicate assertion revision ${key}`);
|
|
433
|
+
const prior = [...this.assertions.values()].some((entry) => entry.value.id === assertion.id);
|
|
434
|
+
if (prior && !this.hasReplacementPermission(assertion.id, assertion.revision)) {
|
|
435
|
+
throw new LineageValidationError("CORRECTION_INVALID", `Assertion ${assertion.id} can only receive revision ${assertion.revision} after an explicit correction`);
|
|
436
|
+
}
|
|
437
|
+
return this.append(this.assertions, key, assertion);
|
|
438
|
+
}
|
|
439
|
+
appendCorrection(correction) {
|
|
440
|
+
requireIdentity(correction.id, "correction.id");
|
|
441
|
+
requireIdentity(correction.reason, "correction.reason");
|
|
442
|
+
requireTimestamp(correction.recordedAt, "correction.recordedAt");
|
|
443
|
+
if (!this.assertions.has(`${correction.assertionId}@${correction.assertionRevision}`)) throw new LineageValidationError("IDENTITY_DANGLING", `Unknown assertion ${correction.assertionId}@${correction.assertionRevision}`);
|
|
444
|
+
if (!this.agents.has(correction.issuerAgentId)) throw new LineageValidationError("AGENT_DANGLING", `Unknown correction issuer ${correction.issuerAgentId}`);
|
|
445
|
+
const activity = this.activities.get(correction.activityId)?.value;
|
|
446
|
+
if (!activity || activity.kind !== "correction") throw new LineageValidationError("CORRECTION_INVALID", "Correction requires a correction activity");
|
|
447
|
+
if (correction.action === "retract" && (correction.replacementAssertionId || correction.replacementRevision)) throw new LineageValidationError("CORRECTION_INVALID", "Retraction cannot name a replacement");
|
|
448
|
+
if (correction.action !== "retract" && (!correction.replacementAssertionId || !correction.replacementRevision)) throw new LineageValidationError("CORRECTION_INVALID", `${correction.action} requires a replacement assertion identity and revision`);
|
|
449
|
+
if (correction.replacementAssertionId === correction.assertionId && correction.action === "supersede") {
|
|
450
|
+
throw new LineageValidationError("CORRECTION_INVALID", "Supersession must identify a different assertion");
|
|
451
|
+
}
|
|
452
|
+
this.ensureUnique(this.corrections, correction.id, "correction");
|
|
453
|
+
return this.append(this.corrections, correction.id, correction);
|
|
454
|
+
}
|
|
455
|
+
traverse(request, policy) {
|
|
456
|
+
this.validateTraversalRequest(request);
|
|
457
|
+
const snapshot = request.snapshot ?? this.sequence;
|
|
458
|
+
if (snapshot < 0 || snapshot > this.sequence) throw new LineageValidationError("SNAPSHOT_UNAVAILABLE", `Snapshot ${snapshot} is unavailable`);
|
|
459
|
+
const entityMap = this.visibleAt(this.entities, snapshot);
|
|
460
|
+
const assertions = [...this.visibleAt(this.assertions, snapshot).values()].filter((assertion) => !request.relationshipKinds || request.relationshipKinds.includes(assertion.relationship)).filter((assertion) => !request.assertionKinds || request.assertionKinds.includes(assertion.assertionKind)).filter((assertion) => policy.canReadAssertion(clone2(assertion))).filter((assertion) => {
|
|
461
|
+
const source = entityMap.get(assertion.sourceEntityId);
|
|
462
|
+
const target = entityMap.get(assertion.targetEntityId);
|
|
463
|
+
return Boolean(source && target && policy.canReadEntity(clone2(source)) && policy.canReadEntity(clone2(target)));
|
|
464
|
+
}).sort(compareAssertions);
|
|
465
|
+
for (const id of request.startEntityIds) {
|
|
466
|
+
const entity = entityMap.get(id);
|
|
467
|
+
if (!entity || !policy.canReadEntity(clone2(entity))) throw new LineageValidationError("IDENTITY_DANGLING", `Start entity ${id} is unavailable`);
|
|
468
|
+
}
|
|
469
|
+
const queue = [...new Set(request.startEntityIds)].sort().map((id) => ({ id, depth: 0, path: [] }));
|
|
470
|
+
const visitedDepth = /* @__PURE__ */ new Map();
|
|
471
|
+
const nodeMap = /* @__PURE__ */ new Map();
|
|
472
|
+
const edgeMap = /* @__PURE__ */ new Map();
|
|
473
|
+
while (queue.length > 0) {
|
|
474
|
+
const current = queue.shift();
|
|
475
|
+
const priorDepth = visitedDepth.get(current.id);
|
|
476
|
+
if (priorDepth !== void 0 && priorDepth <= current.depth) continue;
|
|
477
|
+
visitedDepth.set(current.id, current.depth);
|
|
478
|
+
const entity = entityMap.get(current.id);
|
|
479
|
+
if (!request.entityKinds || request.entityKinds.includes(entity.kind) || current.depth === 0) {
|
|
480
|
+
nodeMap.set(current.id, { entity: clone2(entity), depth: current.depth, pathAssertionIds: [...current.path] });
|
|
481
|
+
}
|
|
482
|
+
if (current.depth >= request.maximumDepth) continue;
|
|
483
|
+
for (const assertion of assertions) {
|
|
484
|
+
const downstream = assertion.sourceEntityId === current.id;
|
|
485
|
+
const upstream = assertion.targetEntityId === current.id;
|
|
486
|
+
if (request.direction === "downstream" && !downstream || request.direction === "upstream" && !upstream || request.direction === "both" && !downstream && !upstream) continue;
|
|
487
|
+
const nextId = downstream ? assertion.targetEntityId : assertion.sourceEntityId;
|
|
488
|
+
edgeMap.set(`${assertion.id}@${assertion.revision}`, {
|
|
489
|
+
assertion: clone2(assertion),
|
|
490
|
+
status: this.statusAt(assertion.id, assertion.revision, snapshot),
|
|
491
|
+
...request.includeEvidence ? { evidence: assertion.evidence.map((reference) => this.evidence.get(`${reference.evidenceId}@${reference.revision}`)).filter((entry) => Boolean(entry && entry.sequence <= snapshot)).map((entry) => entry.value).filter((item) => policy.canReadEvidence(clone2(item))).map(clone2) } : {}
|
|
492
|
+
});
|
|
493
|
+
queue.push({ id: nextId, depth: current.depth + 1, path: [...current.path, assertion.id] });
|
|
494
|
+
}
|
|
495
|
+
if (nodeMap.size + edgeMap.size > request.maximumResults) break;
|
|
496
|
+
}
|
|
497
|
+
const allNodes = [...nodeMap.values()].sort((left, right) => left.depth - right.depth || left.entity.id.localeCompare(right.entity.id));
|
|
498
|
+
const allEdges = [...edgeMap.values()].sort((left, right) => compareAssertions(left.assertion, right.assertion));
|
|
499
|
+
const rows = [
|
|
500
|
+
...allNodes.map((value) => ({ kind: "node", key: `n:${String(value.depth).padStart(8, "0")}:${value.entity.id}`, value })),
|
|
501
|
+
...allEdges.map((value) => ({ kind: "edge", key: `e:${value.assertion.sourceEntityId}:${value.assertion.targetEntityId}:${value.assertion.relationship}:${value.assertion.id}`, value }))
|
|
502
|
+
].sort((left, right) => left.key.localeCompare(right.key));
|
|
503
|
+
const requestFingerprint = this.traversalFingerprint(request, snapshot);
|
|
504
|
+
const offset = request.cursor ? this.decodeCursor(request.cursor, requestFingerprint, snapshot) : 0;
|
|
505
|
+
const bounded = rows.slice(0, request.maximumResults);
|
|
506
|
+
const page = bounded.slice(offset, offset + request.pageSize);
|
|
507
|
+
const nextOffset = offset + page.length;
|
|
508
|
+
const truncated = rows.length > request.maximumResults;
|
|
509
|
+
return {
|
|
510
|
+
contract: DATASET_LINEAGE_CONTRACT,
|
|
511
|
+
schemaVersion: DATASET_LINEAGE_SCHEMA_VERSION,
|
|
512
|
+
snapshot,
|
|
513
|
+
nodes: page.filter((row) => row.kind === "node").map((row) => clone2(row.value)),
|
|
514
|
+
edges: page.filter((row) => row.kind === "edge").map((row) => clone2(row.value)),
|
|
515
|
+
...nextOffset < bounded.length ? { nextCursor: `${snapshot}.${nextOffset}.${requestFingerprint}` } : {},
|
|
516
|
+
truncated
|
|
517
|
+
};
|
|
518
|
+
}
|
|
519
|
+
exportArchive(snapshot = this.sequence) {
|
|
520
|
+
if (snapshot < 0 || snapshot > this.sequence) throw new LineageValidationError("SNAPSHOT_UNAVAILABLE", `Snapshot ${snapshot} is unavailable`);
|
|
521
|
+
const content = {
|
|
522
|
+
contract: DATASET_LINEAGE_CONTRACT,
|
|
523
|
+
schemaVersion: DATASET_LINEAGE_SCHEMA_VERSION,
|
|
524
|
+
snapshot,
|
|
525
|
+
entities: this.sorted(this.visibleAt(this.entities, snapshot)),
|
|
526
|
+
agents: this.sorted(this.visibleAt(this.agents, snapshot)),
|
|
527
|
+
activities: this.sorted(this.visibleAt(this.activities, snapshot)),
|
|
528
|
+
evidence: this.sorted(this.visibleAt(this.evidence, snapshot), (item) => `${item.id}@${item.revision}`),
|
|
529
|
+
assertions: this.sorted(this.visibleAt(this.assertions, snapshot)),
|
|
530
|
+
corrections: this.sorted(this.visibleAt(this.corrections, snapshot))
|
|
531
|
+
};
|
|
532
|
+
return clone2({ ...content, digest: computeLineageDigest(content) });
|
|
533
|
+
}
|
|
534
|
+
static importArchive(archive, options = {}) {
|
|
535
|
+
if (archive.contract !== DATASET_LINEAGE_CONTRACT || !archive.schemaVersion.startsWith("1.")) throw new LineageValidationError("VALUE_INVALID", "Unsupported lineage archive contract");
|
|
536
|
+
const { digest: digest2, ...content } = archive;
|
|
537
|
+
if (computeLineageDigest(content) !== digest2) throw new LineageValidationError("EVIDENCE_DIGEST_MISMATCH", "Lineage archive digest does not match canonical content");
|
|
538
|
+
const ledger = new _DatasetLineageLedger(options);
|
|
539
|
+
for (const item of archive.agents) ledger.appendAgent(item);
|
|
540
|
+
for (const item of archive.entities) ledger.appendEntity(item);
|
|
541
|
+
for (const item of archive.activities) ledger.appendActivity(item);
|
|
542
|
+
for (const item of archive.evidence) ledger.appendEvidence(item);
|
|
543
|
+
const replacementKeys = new Set(archive.corrections.flatMap((item) => item.replacementAssertionId && item.replacementRevision ? [`${item.replacementAssertionId}@${item.replacementRevision}`] : []));
|
|
544
|
+
for (const item of archive.assertions.filter((item2) => !replacementKeys.has(`${item2.id}@${item2.revision}`))) ledger.appendAssertion(item);
|
|
545
|
+
const pending = [...archive.corrections];
|
|
546
|
+
while (pending.length > 0) {
|
|
547
|
+
const index = pending.findIndex((item) => ledger.assertions.has(`${item.assertionId}@${item.assertionRevision}`));
|
|
548
|
+
if (index < 0) throw new LineageValidationError("CORRECTION_INVALID", "Correction history contains an unreachable assertion revision");
|
|
549
|
+
const correction = pending.splice(index, 1)[0];
|
|
550
|
+
ledger.appendCorrection(correction);
|
|
551
|
+
if (correction.replacementAssertionId && correction.replacementRevision) {
|
|
552
|
+
const key = `${correction.replacementAssertionId}@${correction.replacementRevision}`;
|
|
553
|
+
if (!ledger.assertions.has(key)) {
|
|
554
|
+
const replacement = archive.assertions.find((item) => `${item.id}@${item.revision}` === key);
|
|
555
|
+
if (!replacement) throw new LineageValidationError("IDENTITY_DANGLING", `Missing replacement assertion ${key}`);
|
|
556
|
+
ledger.appendAssertion(replacement);
|
|
557
|
+
}
|
|
558
|
+
}
|
|
559
|
+
}
|
|
560
|
+
return ledger;
|
|
561
|
+
}
|
|
562
|
+
project(capabilities, snapshot = this.sequence) {
|
|
563
|
+
const source = this.exportArchive(snapshot);
|
|
564
|
+
const losses = [];
|
|
565
|
+
const entities = source.entities.filter((item, index) => {
|
|
566
|
+
const keep = capabilities.entityKinds.includes(item.kind);
|
|
567
|
+
if (!keep) losses.push({ path: `/entities/${index}`, reason: "unsupported-entity-kind", canonicalDigest: computeLineageDigest(item) });
|
|
568
|
+
return keep;
|
|
569
|
+
});
|
|
570
|
+
const entityIds = new Set(entities.map((item) => item.id));
|
|
571
|
+
const assertions = source.assertions.filter((item, index) => {
|
|
572
|
+
const reason = !capabilities.relationshipKinds.includes(item.relationship) ? "unsupported-relationship-kind" : !capabilities.assertionKinds.includes(item.assertionKind) ? "unsupported-assertion-kind" : !entityIds.has(item.sourceEntityId) || !entityIds.has(item.targetEntityId) ? "unsupported-entity-kind" : void 0;
|
|
573
|
+
if (reason) losses.push({ path: `/assertions/${index}`, reason, canonicalDigest: computeLineageDigest(item) });
|
|
574
|
+
return !reason;
|
|
575
|
+
});
|
|
576
|
+
const evidenceIds = new Set(assertions.flatMap((item) => item.evidence.map((reference) => `${reference.evidenceId}@${reference.revision}`)));
|
|
577
|
+
const evidence = capabilities.preservesEvidence ? source.evidence.filter((item) => evidenceIds.has(`${item.id}@${item.revision}`)) : [];
|
|
578
|
+
if (!capabilities.preservesEvidence && evidenceIds.size > 0) losses.push({ path: "/evidence", reason: "evidence-omitted", canonicalDigest: computeLineageDigest(source.evidence) });
|
|
579
|
+
const assertionIds = new Set(assertions.map((item) => item.id));
|
|
580
|
+
const corrections = capabilities.preservesCorrections ? source.corrections.filter((item) => assertionIds.has(item.assertionId)) : [];
|
|
581
|
+
if (!capabilities.preservesCorrections && source.corrections.length > 0) losses.push({ path: "/corrections", reason: "corrections-omitted", canonicalDigest: computeLineageDigest(source.corrections) });
|
|
582
|
+
losses.sort((left, right) => left.path.localeCompare(right.path) || left.reason.localeCompare(right.reason));
|
|
583
|
+
const projectionContent = { entities, assertions, evidence, corrections };
|
|
584
|
+
const projectionDigest = computeLineageDigest(projectionContent);
|
|
585
|
+
return clone2({
|
|
586
|
+
contract: DATASET_LINEAGE_CONTRACT,
|
|
587
|
+
schemaVersion: DATASET_LINEAGE_SCHEMA_VERSION,
|
|
588
|
+
canonical: false,
|
|
589
|
+
regenerable: true,
|
|
590
|
+
sourceDigest: source.digest,
|
|
591
|
+
...projectionContent,
|
|
592
|
+
digest: projectionDigest,
|
|
593
|
+
lossReceipt: {
|
|
594
|
+
contract: DATASET_LINEAGE_CONTRACT,
|
|
595
|
+
schemaVersion: DATASET_LINEAGE_SCHEMA_VERSION,
|
|
596
|
+
sourceDigest: source.digest,
|
|
597
|
+
projectionDigest,
|
|
598
|
+
lossless: losses.length === 0,
|
|
599
|
+
losses
|
|
600
|
+
}
|
|
601
|
+
});
|
|
602
|
+
}
|
|
603
|
+
append(map, key, value) {
|
|
604
|
+
this.sequence += 1;
|
|
605
|
+
map.set(key, { sequence: this.sequence, value: clone2(value) });
|
|
606
|
+
return this.sequence;
|
|
607
|
+
}
|
|
608
|
+
ensureUnique(map, key, label) {
|
|
609
|
+
if (map.has(key)) throw new LineageValidationError("IDENTITY_DUPLICATE", `Duplicate ${label} identity ${key}`);
|
|
610
|
+
}
|
|
611
|
+
visibleAt(map, snapshot) {
|
|
612
|
+
return new Map([...map].filter(([, entry]) => entry.sequence <= snapshot).map(([key, entry]) => [key, clone2(entry.value)]));
|
|
613
|
+
}
|
|
614
|
+
sorted(map, key = (item) => item.id) {
|
|
615
|
+
return [...map.values()].sort((left, right) => key(left).localeCompare(key(right))).map(clone2);
|
|
616
|
+
}
|
|
617
|
+
hasReplacementPermission(assertionId, revision) {
|
|
618
|
+
return [...this.corrections.values()].some((entry) => entry.value.action === "correct" && entry.value.replacementAssertionId === assertionId && entry.value.replacementRevision === revision);
|
|
619
|
+
}
|
|
620
|
+
statusAt(assertionId, revision, snapshot) {
|
|
621
|
+
const latest = [...this.corrections.values()].filter((entry) => entry.sequence <= snapshot && entry.value.assertionId === assertionId && entry.value.assertionRevision === revision).sort((left, right) => right.sequence - left.sequence)[0]?.value;
|
|
622
|
+
if (!latest) return "active";
|
|
623
|
+
return latest.action === "correct" ? "corrected" : latest.action === "retract" ? "retracted" : "superseded";
|
|
624
|
+
}
|
|
625
|
+
validateTraversalRequest(request) {
|
|
626
|
+
if (request.startEntityIds.length === 0) throw new LineageValidationError("IDENTITY_REQUIRED", "At least one start entity is required");
|
|
627
|
+
if (!Number.isSafeInteger(request.maximumDepth) || request.maximumDepth < 0 || request.maximumDepth > this.maximumTraversalDepth) throw new LineageValidationError("TRAVERSAL_LIMIT_EXCEEDED", `maximumDepth exceeds ${this.maximumTraversalDepth}`);
|
|
628
|
+
if (!Number.isSafeInteger(request.maximumResults) || request.maximumResults < 1 || request.maximumResults > this.maximumTraversalResults) throw new LineageValidationError("TRAVERSAL_LIMIT_EXCEEDED", `maximumResults exceeds ${this.maximumTraversalResults}`);
|
|
629
|
+
if (!Number.isSafeInteger(request.pageSize) || request.pageSize < 1 || request.pageSize > this.maximumPageSize || request.pageSize > request.maximumResults) throw new LineageValidationError("TRAVERSAL_LIMIT_EXCEEDED", `pageSize exceeds ${this.maximumPageSize}`);
|
|
630
|
+
}
|
|
631
|
+
traversalFingerprint(request, snapshot) {
|
|
632
|
+
const parameters = { ...request };
|
|
633
|
+
delete parameters.cursor;
|
|
634
|
+
delete parameters.snapshot;
|
|
635
|
+
return computeLineageDigest({ ...parameters, startEntityIds: [...new Set(parameters.startEntityIds)].sort(), snapshot });
|
|
636
|
+
}
|
|
637
|
+
decodeCursor(cursor, fingerprint, snapshot) {
|
|
638
|
+
const match = /^(\d+)\.(\d+)\.(sha256:[0-9a-f]{64})$/.exec(cursor);
|
|
639
|
+
if (!match || Number(match[1]) !== snapshot || match[3] !== fingerprint) throw new LineageValidationError("CURSOR_INVALID", "Cursor does not match this query and snapshot");
|
|
640
|
+
const offset = Number(match[2]);
|
|
641
|
+
if (!Number.isSafeInteger(offset) || offset < 0) throw new LineageValidationError("CURSOR_INVALID", "Cursor offset is invalid");
|
|
642
|
+
return offset;
|
|
643
|
+
}
|
|
644
|
+
};
|
|
645
|
+
function compareAssertions(left, right) {
|
|
646
|
+
return left.sourceEntityId.localeCompare(right.sourceEntityId) || left.targetEntityId.localeCompare(right.targetEntityId) || left.relationship.localeCompare(right.relationship) || left.id.localeCompare(right.id);
|
|
647
|
+
}
|
|
648
|
+
|
|
649
|
+
// src/dataset-execution-capabilities.ts
|
|
650
|
+
var DATASET_EXECUTION_CONTRACT = "fortemi.dataset-execution-capabilities/v1";
|
|
651
|
+
var DATASET_EXECUTION_SCHEMA_VERSION = "1.0.0";
|
|
652
|
+
var DATASET_EXECUTION_CAPABILITY_IDS = [
|
|
653
|
+
"ingest.full",
|
|
654
|
+
"ingest.snapshot",
|
|
655
|
+
"ingest.incremental",
|
|
656
|
+
"ingest.stream",
|
|
657
|
+
"schema.inspect",
|
|
658
|
+
"identity.stable-revision",
|
|
659
|
+
"identity.record",
|
|
660
|
+
"mutation.upsert",
|
|
661
|
+
"mutation.tombstone",
|
|
662
|
+
"mutation.reconcile",
|
|
663
|
+
"checkpoint.read",
|
|
664
|
+
"checkpoint.write",
|
|
665
|
+
"execution.cancel",
|
|
666
|
+
"rejection.record",
|
|
667
|
+
"index.lexical",
|
|
668
|
+
"index.chunk",
|
|
669
|
+
"index.vector",
|
|
670
|
+
"index.hybrid",
|
|
671
|
+
"index.rerank",
|
|
672
|
+
"index.graph",
|
|
673
|
+
"index.community",
|
|
674
|
+
"lineage.dataset",
|
|
675
|
+
"lineage.record",
|
|
676
|
+
"lineage.field",
|
|
677
|
+
"lineage.relationship-evidence",
|
|
678
|
+
"transaction.atomic-batch",
|
|
679
|
+
"privacy.pre-materialization-filter",
|
|
680
|
+
"pagination.cursor",
|
|
681
|
+
"ordering.deterministic"
|
|
682
|
+
];
|
|
683
|
+
var ID_SET = new Set(DATASET_EXECUTION_CAPABILITY_IDS);
|
|
684
|
+
var LIMIT_KEYS = ["maxInputBytes", "maxRecordBytes", "maxBatchRecords", "maxConcurrency", "maxPageSize", "maxTraversalDepth"];
|
|
685
|
+
function major(version) {
|
|
686
|
+
const match = /^(\d+)\.(\d+)\.(\d+)(?:[-+].*)?$/.exec(version);
|
|
687
|
+
return match ? Number(match[1]) : null;
|
|
688
|
+
}
|
|
689
|
+
function compareVersions(left, right) {
|
|
690
|
+
const parse = (value) => {
|
|
691
|
+
const match = /^(\d+)\.(\d+)\.(\d+)(?:[-+].*)?$/.exec(value);
|
|
692
|
+
return match ? [Number(match[1]), Number(match[2]), Number(match[3])] : null;
|
|
693
|
+
};
|
|
694
|
+
const a = parse(left);
|
|
695
|
+
const b = parse(right);
|
|
696
|
+
if (!a || !b) return null;
|
|
697
|
+
for (let index = 0; index < 3; index++) {
|
|
698
|
+
if (a[index] !== b[index]) return a[index] > b[index] ? 1 : -1;
|
|
699
|
+
}
|
|
700
|
+
return 0;
|
|
701
|
+
}
|
|
702
|
+
function isSupported(capability) {
|
|
703
|
+
return capability !== void 0 && capability.status !== "unsupported";
|
|
704
|
+
}
|
|
705
|
+
function validateDatasetExecutionDescriptor(descriptor) {
|
|
706
|
+
const diagnostics = [];
|
|
707
|
+
if (descriptor.contract !== DATASET_EXECUTION_CONTRACT) {
|
|
708
|
+
diagnostics.push({ code: "CONTRACT_MAJOR_UNSUPPORTED", path: "/contract", message: `Unsupported contract ${String(descriptor.contract)}` });
|
|
709
|
+
}
|
|
710
|
+
if (major(descriptor.schemaVersion) !== 1) {
|
|
711
|
+
diagnostics.push({ code: "SCHEMA_VERSION_UNSUPPORTED", path: "/schemaVersion", message: `Unsupported descriptor schema version ${descriptor.schemaVersion}` });
|
|
712
|
+
}
|
|
713
|
+
const capabilities = /* @__PURE__ */ new Map();
|
|
714
|
+
descriptor.capabilities.forEach((capability, index) => {
|
|
715
|
+
if (!ID_SET.has(capability.id) || compareVersions(capability.version, capability.version) === null) {
|
|
716
|
+
diagnostics.push({ code: "DESCRIPTOR_INVALID", path: `/capabilities/${index}`, message: `Invalid capability declaration ${capability.id}` });
|
|
717
|
+
return;
|
|
718
|
+
}
|
|
719
|
+
if (capabilities.has(capability.id)) {
|
|
720
|
+
diagnostics.push({ code: "CAPABILITY_DUPLICATE", capability: capability.id, path: `/capabilities/${index}/id`, message: `Capability ${capability.id} is declared more than once` });
|
|
721
|
+
}
|
|
722
|
+
capabilities.set(capability.id, capability);
|
|
723
|
+
for (const key of LIMIT_KEYS) {
|
|
724
|
+
const value = capability.limits?.[key];
|
|
725
|
+
if (value !== void 0 && (!Number.isSafeInteger(value) || value < 0)) {
|
|
726
|
+
diagnostics.push({ code: "DESCRIPTOR_INVALID", capability: capability.id, path: `/capabilities/${index}/limits/${key}`, message: `${key} must be a non-negative safe integer` });
|
|
727
|
+
}
|
|
728
|
+
}
|
|
729
|
+
if (capability.status !== "unsupported" && capability.evidence.length === 0) {
|
|
730
|
+
diagnostics.push({ code: "DESCRIPTOR_INVALID", capability: capability.id, path: `/capabilities/${index}/evidence`, message: `Supported capability ${capability.id} requires evidence` });
|
|
731
|
+
}
|
|
732
|
+
for (const evidenceId of capability.evidence) {
|
|
733
|
+
if (!descriptor.evidence.some((evidence) => evidence.id === evidenceId)) {
|
|
734
|
+
diagnostics.push({ code: "DESCRIPTOR_INVALID", capability: capability.id, path: `/capabilities/${index}/evidence`, message: `Unknown evidence ${evidenceId}` });
|
|
735
|
+
}
|
|
736
|
+
}
|
|
737
|
+
});
|
|
738
|
+
const requireTogether = (source, requirements) => {
|
|
739
|
+
if (!isSupported(capabilities.get(source))) return;
|
|
740
|
+
for (const requirement of requirements) {
|
|
741
|
+
if (!isSupported(capabilities.get(requirement))) {
|
|
742
|
+
diagnostics.push({ code: "CAPABILITY_INCONSISTENT", capability: source, message: `${source} requires ${requirement}` });
|
|
743
|
+
}
|
|
744
|
+
}
|
|
745
|
+
};
|
|
746
|
+
requireTogether("ingest.incremental", ["identity.stable-revision", "checkpoint.read", "checkpoint.write"]);
|
|
747
|
+
requireTogether("lineage.field", ["lineage.relationship-evidence"]);
|
|
748
|
+
requireTogether("mutation.reconcile", ["mutation.upsert", "mutation.tombstone"]);
|
|
749
|
+
requireTogether("index.hybrid", ["index.lexical", "index.vector"]);
|
|
750
|
+
if (descriptor.guarantees.transaction === "atomic-batch" && !isSupported(capabilities.get("transaction.atomic-batch"))) {
|
|
751
|
+
diagnostics.push({ code: "CAPABILITY_INCONSISTENT", capability: "transaction.atomic-batch", message: "Atomic-batch guarantee requires transaction.atomic-batch capability" });
|
|
752
|
+
}
|
|
753
|
+
if (descriptor.runtime.plane === "static-cache" && descriptor.runtime.dataClass !== "static-cache") {
|
|
754
|
+
diagnostics.push({ code: "CAPABILITY_INCONSISTENT", message: "Static-cache execution plane must declare static-cache data class" });
|
|
755
|
+
}
|
|
756
|
+
if (descriptor.runtime.plane === "live-remote-persistence" && descriptor.runtime.maturity === "stable" && !descriptor.evidence.some((item) => item.kind === "live-qualification")) {
|
|
757
|
+
diagnostics.push({ code: "CAPABILITY_INCONSISTENT", message: "Stable live remote persistence requires live qualification evidence" });
|
|
758
|
+
}
|
|
759
|
+
return diagnostics;
|
|
760
|
+
}
|
|
761
|
+
function assessRequirement(requirement, capabilities) {
|
|
762
|
+
const capability = capabilities.get(requirement.id);
|
|
763
|
+
if (!isSupported(capability)) {
|
|
764
|
+
return { ok: false, reason: "unsupported", diagnostics: [{ code: "REQUIRED_CAPABILITY_MISSING", capability: requirement.id, message: `Capability ${requirement.id} is unsupported` }] };
|
|
765
|
+
}
|
|
766
|
+
if (requirement.minimumVersion) {
|
|
767
|
+
const comparison = compareVersions(capability.version, requirement.minimumVersion);
|
|
768
|
+
if (comparison === null || comparison < 0) {
|
|
769
|
+
return { ok: false, reason: "version-insufficient", diagnostics: [{ code: "CAPABILITY_VERSION_INSUFFICIENT", capability: requirement.id, message: `${capability.version} does not satisfy ${requirement.minimumVersion}` }] };
|
|
770
|
+
}
|
|
771
|
+
}
|
|
772
|
+
for (const key of LIMIT_KEYS) {
|
|
773
|
+
const required = requirement.minimumLimits?.[key];
|
|
774
|
+
if (required !== void 0 && (capability.limits?.[key] ?? -1) < required) {
|
|
775
|
+
return { ok: false, reason: "limit-insufficient", diagnostics: [{ code: "CAPABILITY_LIMIT_INSUFFICIENT", capability: requirement.id, path: `/minimumLimits/${key}`, message: `${key} does not satisfy ${required}` }] };
|
|
776
|
+
}
|
|
777
|
+
}
|
|
778
|
+
return { ok: true, diagnostics: [] };
|
|
779
|
+
}
|
|
780
|
+
function negotiateDatasetExecutionCapabilities(descriptor, request) {
|
|
781
|
+
const diagnostics = validateDatasetExecutionDescriptor(descriptor);
|
|
782
|
+
if (request.contract !== DATASET_EXECUTION_CONTRACT) {
|
|
783
|
+
diagnostics.push({ code: "CONTRACT_MAJOR_UNSUPPORTED", path: "/contract", message: `Unsupported request contract ${String(request.contract)}` });
|
|
784
|
+
}
|
|
785
|
+
const capabilities = new Map(descriptor.capabilities.map((capability) => [capability.id, capability]));
|
|
786
|
+
const selected = [];
|
|
787
|
+
const degradations = [];
|
|
788
|
+
if (diagnostics.length === 0) {
|
|
789
|
+
for (const requirement of request.required) {
|
|
790
|
+
const assessment = assessRequirement(requirement, capabilities);
|
|
791
|
+
if (assessment.ok) selected.push(requirement.id);
|
|
792
|
+
else diagnostics.push(...assessment.diagnostics);
|
|
793
|
+
}
|
|
794
|
+
for (const requirement of request.optional ?? []) {
|
|
795
|
+
const assessment = assessRequirement(requirement, capabilities);
|
|
796
|
+
if (assessment.ok) {
|
|
797
|
+
selected.push(requirement.id);
|
|
798
|
+
continue;
|
|
799
|
+
}
|
|
800
|
+
const fallback = requirement.fallback?.find((id) => isSupported(capabilities.get(id)));
|
|
801
|
+
if (fallback) selected.push(fallback);
|
|
802
|
+
degradations.push({
|
|
803
|
+
requested: requirement.id,
|
|
804
|
+
...fallback ? { selected: fallback } : {},
|
|
805
|
+
reason: assessment.reason,
|
|
806
|
+
changedGuarantees: fallback ? [`${requirement.id} replaced by ${fallback}`] : [`${requirement.id} omitted`]
|
|
807
|
+
});
|
|
808
|
+
}
|
|
809
|
+
}
|
|
810
|
+
return {
|
|
811
|
+
contract: DATASET_EXECUTION_CONTRACT,
|
|
812
|
+
accepted: diagnostics.length === 0,
|
|
813
|
+
runtime: { ...descriptor.runtime },
|
|
814
|
+
selected: [...new Set(selected)],
|
|
815
|
+
degradations,
|
|
816
|
+
diagnostics
|
|
817
|
+
};
|
|
818
|
+
}
|
|
819
|
+
|
|
820
|
+
// src/dataset-execution-descriptors.ts
|
|
821
|
+
var FORTEMI_BROWSER_LOCAL_DATASET_EXECUTION_DESCRIPTOR = {
|
|
822
|
+
contract: DATASET_EXECUTION_CONTRACT,
|
|
823
|
+
schemaVersion: DATASET_EXECUTION_SCHEMA_VERSION,
|
|
824
|
+
runtime: { id: "fortemi-browser", version: "2026.8.0", plane: "browser-local-archive", dataClass: "canonical", maturity: "stable" },
|
|
825
|
+
guarantees: { transaction: "atomic-batch", isolation: "serializable", durability: "wal", availability: "local-process", ordering: "backend-cursor" },
|
|
826
|
+
capabilities: [
|
|
827
|
+
{ id: "ingest.full", version: "1.0.0", status: "supported", limits: { maxBatchRecords: 1e3, maxConcurrency: 1 }, evidence: ["browser-conformance"] },
|
|
828
|
+
{ id: "identity.record", version: "1.0.0", status: "supported", evidence: ["browser-conformance"] },
|
|
829
|
+
{ id: "mutation.upsert", version: "1.0.0", status: "supported", evidence: ["browser-conformance"] },
|
|
830
|
+
{ id: "transaction.atomic-batch", version: "1.0.0", status: "supported", evidence: ["browser-conformance"] },
|
|
831
|
+
{ id: "ordering.deterministic", version: "1.0.0", status: "supported", evidence: ["browser-conformance"] }
|
|
832
|
+
],
|
|
833
|
+
evidence: [{ id: "browser-conformance", kind: "conformance-report", uri: "fortemi://conformance/browser-local/v1" }]
|
|
834
|
+
};
|
|
835
|
+
var FORTEMI_STATIC_CACHE_DATASET_EXECUTION_DESCRIPTOR = {
|
|
836
|
+
contract: DATASET_EXECUTION_CONTRACT,
|
|
837
|
+
schemaVersion: DATASET_EXECUTION_SCHEMA_VERSION,
|
|
838
|
+
runtime: { id: "fortemi-static", version: "2026.8.0", plane: "static-cache", dataClass: "static-cache", maturity: "stable" },
|
|
839
|
+
guarantees: { transaction: "none", isolation: "snapshot", durability: "filesystem", availability: "local-process", ordering: "stable-identity" },
|
|
840
|
+
capabilities: [
|
|
841
|
+
{ id: "index.lexical", version: "1.0.0", status: "supported", limits: { maxPageSize: 1e3 }, evidence: ["static-conformance"] },
|
|
842
|
+
{ id: "pagination.cursor", version: "1.0.0", status: "supported", evidence: ["static-conformance"] },
|
|
843
|
+
{ id: "ordering.deterministic", version: "1.0.0", status: "supported", evidence: ["static-conformance"] }
|
|
844
|
+
],
|
|
845
|
+
evidence: [{ id: "static-conformance", kind: "conformance-report", uri: "fortemi://conformance/static-cache/v1" }]
|
|
846
|
+
};
|
|
847
|
+
var FORTEMI_PORTABLE_SHARD_DATASET_EXECUTION_DESCRIPTOR = {
|
|
848
|
+
contract: DATASET_EXECUTION_CONTRACT,
|
|
849
|
+
schemaVersion: DATASET_EXECUTION_SCHEMA_VERSION,
|
|
850
|
+
runtime: { id: "fortemi-shard", version: "2026.8.0", plane: "portable-shard", dataClass: "portable-projection", maturity: "stable" },
|
|
851
|
+
guarantees: { transaction: "none", isolation: "snapshot", durability: "filesystem", availability: "local-process", ordering: "stable-identity" },
|
|
852
|
+
capabilities: [
|
|
853
|
+
{ id: "schema.inspect", version: "1.0.0", status: "supported", evidence: ["shard-conformance"] },
|
|
854
|
+
{ id: "lineage.dataset", version: "1.0.0", status: "experimental", evidence: ["shard-conformance"] },
|
|
855
|
+
{ id: "ordering.deterministic", version: "1.0.0", status: "supported", evidence: ["shard-conformance"] }
|
|
856
|
+
],
|
|
857
|
+
evidence: [{ id: "shard-conformance", kind: "conformance-report", uri: "fortemi://conformance/knowledge-shard/v1" }]
|
|
858
|
+
};
|
|
859
|
+
var DATASET_MATERIALIZATION_CONTRACT = "fortemi.dataset-materialization-profile/v1";
|
|
860
|
+
var DATASET_MATERIALIZATION_SCHEMA_VERSION = "1.0.0";
|
|
861
|
+
var DATASET_MATERIALIZATION_KINDS = [
|
|
862
|
+
"chunking",
|
|
863
|
+
"lexical",
|
|
864
|
+
"vector",
|
|
865
|
+
"hybrid",
|
|
866
|
+
"rerank",
|
|
867
|
+
"entity-relationship-extraction",
|
|
868
|
+
"graph-retrieval",
|
|
869
|
+
"community"
|
|
870
|
+
];
|
|
871
|
+
var DatasetMaterializationError = class extends Error {
|
|
872
|
+
constructor(code, message) {
|
|
873
|
+
super(message);
|
|
874
|
+
this.code = code;
|
|
875
|
+
this.name = "DatasetMaterializationError";
|
|
876
|
+
}
|
|
877
|
+
};
|
|
878
|
+
var encoder = new TextEncoder();
|
|
879
|
+
function canonicalJson3(value) {
|
|
880
|
+
if (value === null || typeof value !== "object") return JSON.stringify(value);
|
|
881
|
+
if (Array.isArray(value)) return `[${value.map(canonicalJson3).join(",")}]`;
|
|
882
|
+
return `{${Object.entries(value).filter(([, item]) => item !== void 0).sort(([left], [right]) => left.localeCompare(right)).map(([key, item]) => `${JSON.stringify(key)}:${canonicalJson3(item)}`).join(",")}}`;
|
|
883
|
+
}
|
|
884
|
+
function digestDatasetMaterializationValue(value) {
|
|
885
|
+
return computeHash(encoder.encode(canonicalJson3(value)));
|
|
886
|
+
}
|
|
887
|
+
function profileDiagnostics(profile) {
|
|
888
|
+
const diagnostics = [];
|
|
889
|
+
const requiredBoundaries = ["chunking", "model-invocation", "index-persistence"];
|
|
890
|
+
if (profile.contract !== DATASET_MATERIALIZATION_CONTRACT || !/^1\./.test(profile.schemaVersion)) {
|
|
891
|
+
diagnostics.push({ code: "PROFILE_INVALID", path: "/contract", message: "Unsupported profile contract or schema major" });
|
|
892
|
+
}
|
|
893
|
+
if (profile.output.dataClass !== "regenerable-index" || profile.output.canonicalMutation !== false) {
|
|
894
|
+
diagnostics.push({ code: "PROFILE_INVALID", path: "/output", message: "Profiles may only produce derived, regenerable artifacts" });
|
|
895
|
+
}
|
|
896
|
+
for (const boundary of requiredBoundaries) {
|
|
897
|
+
if (!profile.privacy.filtersBefore.includes(boundary)) {
|
|
898
|
+
diagnostics.push({ code: "PROFILE_INVALID", path: "/privacy/filtersBefore", message: `Privacy filtering must precede ${boundary}` });
|
|
899
|
+
}
|
|
900
|
+
}
|
|
901
|
+
if (profile.determinism.class === "seeded" && profile.determinism.seedRequired !== true) {
|
|
902
|
+
diagnostics.push({ code: "PROFILE_INVALID", path: "/determinism/seedRequired", message: "Seeded profiles must require a seed" });
|
|
903
|
+
}
|
|
904
|
+
if (profile.resourceLimits.maxInputBytes < 0 || profile.resourceLimits.maxRecords < 0 || profile.resourceLimits.maxConcurrency < 1) {
|
|
905
|
+
diagnostics.push({ code: "PROFILE_INVALID", path: "/resourceLimits", message: "Resource limits must be non-negative and concurrency must be positive" });
|
|
906
|
+
}
|
|
907
|
+
return diagnostics;
|
|
908
|
+
}
|
|
909
|
+
function negotiateDatasetMaterializationProfile(request) {
|
|
910
|
+
const candidates = [request.profile, ...request.fallbackProfiles ?? []];
|
|
911
|
+
const primaryDiagnostics = profileDiagnostics(request.profile);
|
|
912
|
+
if (!request.profile.operations.includes(request.operation)) {
|
|
913
|
+
primaryDiagnostics.push({ code: "OPERATION_UNSUPPORTED", path: "/operations", message: `${request.operation} is not supported by ${request.profile.id}` });
|
|
914
|
+
}
|
|
915
|
+
if (request.profile.status === "unsupported") {
|
|
916
|
+
primaryDiagnostics.push({ code: "PROFILE_UNSUPPORTED", path: "/status", message: `${request.profile.id} is unsupported` });
|
|
917
|
+
}
|
|
918
|
+
for (let index = 0; index < candidates.length; index += 1) {
|
|
919
|
+
const candidate = candidates[index];
|
|
920
|
+
const diagnostics = index === 0 ? primaryDiagnostics : profileDiagnostics(candidate);
|
|
921
|
+
if (!candidate.operations.includes(request.operation) || candidate.status === "unsupported" || diagnostics.length > 0) continue;
|
|
922
|
+
const capability = negotiateDatasetExecutionCapabilities(request.runtime, {
|
|
923
|
+
contract: DATASET_EXECUTION_CONTRACT,
|
|
924
|
+
required: candidate.requiredRuntimeCapabilities,
|
|
925
|
+
optional: candidate.optionalRuntimeCapabilities
|
|
926
|
+
});
|
|
927
|
+
if (capability.accepted) {
|
|
928
|
+
const fallback = index > 0;
|
|
929
|
+
return {
|
|
930
|
+
accepted: true,
|
|
931
|
+
requestedProfile: request.profile.id,
|
|
932
|
+
selectedProfile: candidate.id,
|
|
933
|
+
runtime: { ...request.runtime.runtime },
|
|
934
|
+
degraded: fallback || capability.degradations.length > 0,
|
|
935
|
+
degradations: capability.degradations,
|
|
936
|
+
diagnostics: fallback ? primaryDiagnostics : []
|
|
937
|
+
};
|
|
938
|
+
}
|
|
939
|
+
if (index === 0) primaryDiagnostics.push(...capability.diagnostics);
|
|
940
|
+
}
|
|
941
|
+
return {
|
|
942
|
+
accepted: false,
|
|
943
|
+
requestedProfile: request.profile.id,
|
|
944
|
+
runtime: { ...request.runtime.runtime },
|
|
945
|
+
degraded: false,
|
|
946
|
+
degradations: [],
|
|
947
|
+
diagnostics: [...primaryDiagnostics, { code: "NO_FALLBACK_PROFILE", message: `No compatible ${request.operation} profile is available` }]
|
|
948
|
+
};
|
|
949
|
+
}
|
|
950
|
+
function clone3(value) {
|
|
951
|
+
return structuredClone(value);
|
|
952
|
+
}
|
|
953
|
+
function assertResources(snapshot, profile) {
|
|
954
|
+
const inputBytes = encoder.encode(canonicalJson3(snapshot.records)).byteLength;
|
|
955
|
+
if (snapshot.records.length > profile.resourceLimits.maxRecords || inputBytes > profile.resourceLimits.maxInputBytes) {
|
|
956
|
+
throw new DatasetMaterializationError("RESOURCE_LIMIT_EXCEEDED", `Input exceeds profile ${profile.id} resource limits`);
|
|
957
|
+
}
|
|
958
|
+
return inputBytes;
|
|
959
|
+
}
|
|
960
|
+
function stableArtifacts(artifacts) {
|
|
961
|
+
return [...artifacts].sort((left, right) => {
|
|
962
|
+
const score = (right.score ?? Number.NEGATIVE_INFINITY) - (left.score ?? Number.NEGATIVE_INFINITY);
|
|
963
|
+
return score || left.logicalId.localeCompare(right.logicalId) || left.digest.localeCompare(right.digest);
|
|
964
|
+
});
|
|
965
|
+
}
|
|
966
|
+
async function executeDatasetMaterialization(request, runtime, adapter, authorize, options = {}) {
|
|
967
|
+
const negotiation = negotiateDatasetMaterializationProfile({ operation: "build", profile: request.profile, runtime, fallbackProfiles: options.fallbackProfiles });
|
|
968
|
+
if (!negotiation.accepted || !negotiation.selectedProfile) {
|
|
969
|
+
throw new DatasetMaterializationError("NEGOTIATION_FAILED", negotiation.diagnostics.map((item) => item.message).join("; "));
|
|
970
|
+
}
|
|
971
|
+
const selected = [request.profile, ...options.fallbackProfiles ?? []].find((item) => item.id === negotiation.selectedProfile);
|
|
972
|
+
try {
|
|
973
|
+
const validateConfiguration = new Ajv20202({ strict: true, allErrors: true }).compile(selected.configurationSchema);
|
|
974
|
+
if (!validateConfiguration(request.configuration)) {
|
|
975
|
+
throw new DatasetMaterializationError("CONFIGURATION_INVALID", `Configuration does not satisfy ${selected.id}: ${JSON.stringify(validateConfiguration.errors)}`);
|
|
976
|
+
}
|
|
977
|
+
} catch (error) {
|
|
978
|
+
if (error instanceof DatasetMaterializationError) throw error;
|
|
979
|
+
throw new DatasetMaterializationError("PROFILE_INVALID", `Invalid configuration schema for ${selected.id}: ${error instanceof Error ? error.message : String(error)}`);
|
|
980
|
+
}
|
|
981
|
+
const inputBytes = assertResources(request.snapshot, selected);
|
|
982
|
+
const allowed = [];
|
|
983
|
+
const denied = [];
|
|
984
|
+
for (const record of request.snapshot.records) {
|
|
985
|
+
if (await authorize(Object.freeze(clone3(record)))) allowed.push(clone3(record));
|
|
986
|
+
else denied.push(record.digest);
|
|
987
|
+
}
|
|
988
|
+
const privacy = {
|
|
989
|
+
policyId: "caller-authorization",
|
|
990
|
+
policyVersion: "1.0.0",
|
|
991
|
+
policyDigest: digestDatasetMaterializationValue({ policy: "caller-authorization", version: "1.0.0" }),
|
|
992
|
+
allowedRecordDigests: allowed.map((item) => item.digest).sort(),
|
|
993
|
+
deniedRecordDigests: denied.sort(),
|
|
994
|
+
evaluatedAt: (options.now ?? (() => (/* @__PURE__ */ new Date()).toISOString()))()
|
|
995
|
+
};
|
|
996
|
+
const affected = request.affected ?? {
|
|
997
|
+
sourceRevisions: [request.snapshot.revision],
|
|
998
|
+
recordDigests: allowed.map((item) => item.digest).sort(),
|
|
999
|
+
chunkDigests: []
|
|
1000
|
+
};
|
|
1001
|
+
const detachedSnapshot = clone3({ ...request.snapshot, records: allowed });
|
|
1002
|
+
const result = await adapter.materialize(Object.freeze({
|
|
1003
|
+
snapshot: detachedSnapshot,
|
|
1004
|
+
profile: clone3(selected),
|
|
1005
|
+
configuration: clone3(request.configuration),
|
|
1006
|
+
mode: request.mode,
|
|
1007
|
+
affected: clone3(affected)
|
|
1008
|
+
}));
|
|
1009
|
+
const artifacts = selected.determinism.class === "deterministic" ? stableArtifacts(result.artifacts) : clone3(result.artifacts);
|
|
1010
|
+
const digests = artifacts.map((item) => item.digest).sort();
|
|
1011
|
+
const counts = artifacts.reduce((all, item) => {
|
|
1012
|
+
all[item.kind] = (all[item.kind] ?? 0) + 1;
|
|
1013
|
+
return all;
|
|
1014
|
+
}, {});
|
|
1015
|
+
const profileDigest = digestDatasetMaterializationValue(selected);
|
|
1016
|
+
const receiptBase = {
|
|
1017
|
+
contract: DATASET_MATERIALIZATION_CONTRACT,
|
|
1018
|
+
schemaVersion: DATASET_MATERIALIZATION_SCHEMA_VERSION,
|
|
1019
|
+
runId: request.runId,
|
|
1020
|
+
processingRunId: request.processingRunId,
|
|
1021
|
+
source: { datasetId: request.snapshot.datasetId, revision: request.snapshot.revision, digests: [...request.snapshot.sourceDigests].sort() },
|
|
1022
|
+
schema: { id: request.snapshot.schemaId, version: request.snapshot.schemaVersion, digest: request.snapshot.schemaDigest },
|
|
1023
|
+
profile: { id: selected.id, version: selected.version, digest: profileDigest, configurationDigest: digestDatasetMaterializationValue(request.configuration), implementation: clone3(selected.implementation) },
|
|
1024
|
+
runtime: clone3(runtime.runtime),
|
|
1025
|
+
negotiation: { requestedProfile: negotiation.requestedProfile, selectedProfile: negotiation.selectedProfile, degraded: negotiation.degraded, degradations: clone3(negotiation.degradations) },
|
|
1026
|
+
mode: request.mode,
|
|
1027
|
+
affected: clone3(affected),
|
|
1028
|
+
output: { counts, digests, aggregateDigest: digestDatasetMaterializationValue(digests) },
|
|
1029
|
+
privacy,
|
|
1030
|
+
resources: { ...clone3(result.resources), inputBytes },
|
|
1031
|
+
createdAt: (options.now ?? (() => (/* @__PURE__ */ new Date()).toISOString()))()
|
|
1032
|
+
};
|
|
1033
|
+
const receipt = {
|
|
1034
|
+
...receiptBase,
|
|
1035
|
+
receiptId: digestDatasetMaterializationValue(receiptBase)
|
|
1036
|
+
};
|
|
1037
|
+
return { artifacts, receipt };
|
|
1038
|
+
}
|
|
1039
|
+
async function executeDatasetRetrieval(request, runtime, adapter, receiptId, fallbackProfiles = []) {
|
|
1040
|
+
const negotiation = negotiateDatasetMaterializationProfile({ operation: "query", profile: request.profile, runtime, fallbackProfiles });
|
|
1041
|
+
if (!negotiation.accepted || !negotiation.selectedProfile) throw new DatasetMaterializationError("NEGOTIATION_FAILED", negotiation.diagnostics.map((item) => item.message).join("; "));
|
|
1042
|
+
if (!adapter.retrieve) throw new DatasetMaterializationError("ADAPTER_MISMATCH", `Backend ${adapter.backend.id} cannot query`);
|
|
1043
|
+
const selected = [request.profile, ...fallbackProfiles].find((item) => item.id === negotiation.selectedProfile);
|
|
1044
|
+
const results = await adapter.retrieve(Object.freeze({ request: clone3(request), profile: clone3(selected) }));
|
|
1045
|
+
return {
|
|
1046
|
+
contract: DATASET_MATERIALIZATION_CONTRACT,
|
|
1047
|
+
schemaVersion: DATASET_MATERIALIZATION_SCHEMA_VERSION,
|
|
1048
|
+
queryId: request.queryId,
|
|
1049
|
+
requestedProfile: request.profile.id,
|
|
1050
|
+
actualProfile: selected.id,
|
|
1051
|
+
actualBackend: { ...adapter.backend, plane: runtime.runtime.plane },
|
|
1052
|
+
degraded: negotiation.degraded,
|
|
1053
|
+
...negotiation.degraded ? { fallbackReason: negotiation.diagnostics.map((item) => item.message).join("; ") || "optional capability degradation" } : {},
|
|
1054
|
+
scoreSemantics: { implementationScoped: true, comparableAcrossImplementations: false },
|
|
1055
|
+
results: selected.determinism.class === "deterministic" ? stableArtifacts(results).slice(0, request.limit) : clone3(results).slice(0, request.limit),
|
|
1056
|
+
receiptId
|
|
1057
|
+
};
|
|
1058
|
+
}
|
|
1059
|
+
function compareDatasetIncrementalParity(full, incremental) {
|
|
1060
|
+
const select = (items, kind) => items.filter((item) => !kind || item.kind === kind).map((item) => `${item.logicalId}:${item.digest}`).sort();
|
|
1061
|
+
const mismatches = [];
|
|
1062
|
+
if (canonicalJson3(select(full)) !== canonicalJson3(select(incremental))) mismatches.push("identities");
|
|
1063
|
+
if (canonicalJson3(select(full, "chunk")) !== canonicalJson3(select(incremental, "chunk"))) mismatches.push("chunks");
|
|
1064
|
+
if (canonicalJson3(select(full, "relationship")) !== canonicalJson3(select(incremental, "relationship"))) mismatches.push("relationships");
|
|
1065
|
+
if (canonicalJson3(select(full, "community")) !== canonicalJson3(select(incremental, "community"))) mismatches.push("communities");
|
|
1066
|
+
if (canonicalJson3(full.map((item) => item.digest).sort()) !== canonicalJson3(incremental.map((item) => item.digest).sort())) mismatches.push("digests");
|
|
1067
|
+
const order = (items) => items.map((item) => `${item.logicalId}:${item.digest}`);
|
|
1068
|
+
if (canonicalJson3(order(full)) !== canonicalJson3(order(incremental))) mismatches.push("ordering");
|
|
1069
|
+
return { equivalent: mismatches.length === 0, mismatches };
|
|
1070
|
+
}
|
|
1071
|
+
function validateDatasetBenchmarkEvidence(evidence) {
|
|
1072
|
+
const errors = [];
|
|
1073
|
+
if (evidence.contract !== DATASET_MATERIALIZATION_CONTRACT || !/^1\./.test(evidence.schemaVersion)) errors.push("unsupported benchmark contract");
|
|
1074
|
+
if (!evidence.correctness.passed) errors.push("correctness suite did not pass");
|
|
1075
|
+
if (evidence.freshness.sourceRevision !== evidence.corpus.revision) errors.push("benchmark evidence is stale");
|
|
1076
|
+
if (!evidence.claims.corpusScoped || evidence.claims.universalScaleLimit) errors.push("benchmark claims must remain corpus-scoped");
|
|
1077
|
+
if (evidence.measurements.length === 0) errors.push("benchmark has no measurements");
|
|
1078
|
+
return errors;
|
|
1079
|
+
}
|
|
13
1080
|
|
|
14
1081
|
// src/event-bus.ts
|
|
15
1082
|
var TypedEventBus = class {
|
|
@@ -926,6 +1993,85 @@ var migration0022 = {
|
|
|
926
1993
|
`
|
|
927
1994
|
};
|
|
928
1995
|
|
|
1996
|
+
// src/migrations/0023_source_metadata_purge.ts
|
|
1997
|
+
var migration0023 = {
|
|
1998
|
+
version: 23,
|
|
1999
|
+
name: "0023_source_metadata_purge",
|
|
2000
|
+
sql: `
|
|
2001
|
+
CREATE TABLE IF NOT EXISTS source_identity (
|
|
2002
|
+
id TEXT PRIMARY KEY,
|
|
2003
|
+
tenant_id TEXT NOT NULL DEFAULT 'default',
|
|
2004
|
+
archive_id TEXT,
|
|
2005
|
+
namespace TEXT NOT NULL,
|
|
2006
|
+
external_id TEXT NOT NULL,
|
|
2007
|
+
external_id_hash TEXT NOT NULL,
|
|
2008
|
+
source_schema_version TEXT NOT NULL,
|
|
2009
|
+
content_digest TEXT NOT NULL,
|
|
2010
|
+
import_run_id TEXT NOT NULL,
|
|
2011
|
+
caller_stable_id TEXT,
|
|
2012
|
+
note_id TEXT NOT NULL REFERENCES note(id) ON DELETE CASCADE,
|
|
2013
|
+
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
2014
|
+
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
2015
|
+
UNIQUE (tenant_id, archive_id, namespace, external_id)
|
|
2016
|
+
);
|
|
2017
|
+
|
|
2018
|
+
CREATE UNIQUE INDEX IF NOT EXISTS idx_source_identity_scope_key
|
|
2019
|
+
ON source_identity(tenant_id, COALESCE(archive_id, ''), namespace, external_id);
|
|
2020
|
+
CREATE INDEX IF NOT EXISTS idx_source_identity_note ON source_identity(note_id);
|
|
2021
|
+
CREATE INDEX IF NOT EXISTS idx_source_identity_import_run ON source_identity(import_run_id);
|
|
2022
|
+
CREATE INDEX IF NOT EXISTS idx_source_identity_hash ON source_identity(external_id_hash);
|
|
2023
|
+
|
|
2024
|
+
CREATE TABLE IF NOT EXISTS source_import_run (
|
|
2025
|
+
id TEXT PRIMARY KEY,
|
|
2026
|
+
tenant_id TEXT NOT NULL DEFAULT 'default',
|
|
2027
|
+
archive_id TEXT,
|
|
2028
|
+
namespace TEXT NOT NULL,
|
|
2029
|
+
started_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
2030
|
+
completed_at TIMESTAMPTZ,
|
|
2031
|
+
checkpoint JSONB NOT NULL DEFAULT '{}',
|
|
2032
|
+
receipt JSONB NOT NULL DEFAULT '{}'
|
|
2033
|
+
);
|
|
2034
|
+
|
|
2035
|
+
CREATE TABLE IF NOT EXISTS metadata_index_path (
|
|
2036
|
+
path TEXT PRIMARY KEY,
|
|
2037
|
+
value_type TEXT NOT NULL,
|
|
2038
|
+
indexed_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
|
2039
|
+
);
|
|
2040
|
+
|
|
2041
|
+
INSERT INTO metadata_index_path (path, value_type) VALUES
|
|
2042
|
+
('provider', 'string'),
|
|
2043
|
+
('model', 'string'),
|
|
2044
|
+
('role', 'string'),
|
|
2045
|
+
('event_kind', 'string'),
|
|
2046
|
+
('sensitivity', 'string'),
|
|
2047
|
+
('import_run_id', 'string')
|
|
2048
|
+
ON CONFLICT (path) DO NOTHING;
|
|
2049
|
+
|
|
2050
|
+
CREATE INDEX IF NOT EXISTS idx_note_metadata_provider
|
|
2051
|
+
ON note_revised_current ((ai_metadata ->> 'provider'));
|
|
2052
|
+
CREATE INDEX IF NOT EXISTS idx_note_metadata_model
|
|
2053
|
+
ON note_revised_current ((ai_metadata ->> 'model'));
|
|
2054
|
+
CREATE INDEX IF NOT EXISTS idx_note_metadata_role
|
|
2055
|
+
ON note_revised_current ((ai_metadata ->> 'role'));
|
|
2056
|
+
CREATE INDEX IF NOT EXISTS idx_note_metadata_event_kind
|
|
2057
|
+
ON note_revised_current ((ai_metadata ->> 'event_kind'));
|
|
2058
|
+
CREATE INDEX IF NOT EXISTS idx_note_metadata_sensitivity
|
|
2059
|
+
ON note_revised_current ((ai_metadata ->> 'sensitivity'));
|
|
2060
|
+
|
|
2061
|
+
CREATE TABLE IF NOT EXISTS deletion_receipt (
|
|
2062
|
+
id TEXT PRIMARY KEY,
|
|
2063
|
+
operation_key TEXT NOT NULL UNIQUE,
|
|
2064
|
+
tenant_id TEXT NOT NULL DEFAULT 'default',
|
|
2065
|
+
archive_id TEXT,
|
|
2066
|
+
selector_hash TEXT NOT NULL,
|
|
2067
|
+
outcome TEXT NOT NULL,
|
|
2068
|
+
counts JSONB NOT NULL,
|
|
2069
|
+
completed_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
2070
|
+
policy JSONB NOT NULL DEFAULT '{}'
|
|
2071
|
+
);
|
|
2072
|
+
`
|
|
2073
|
+
};
|
|
2074
|
+
|
|
929
2075
|
// src/migrations/index.ts
|
|
930
2076
|
var allMigrations = [
|
|
931
2077
|
migration0001,
|
|
@@ -949,7 +2095,8 @@ var allMigrations = [
|
|
|
949
2095
|
migration0019,
|
|
950
2096
|
migration0020,
|
|
951
2097
|
migration0021,
|
|
952
|
-
migration0022
|
|
2098
|
+
migration0022,
|
|
2099
|
+
migration0023
|
|
953
2100
|
];
|
|
954
2101
|
|
|
955
2102
|
// src/data-archive.ts
|
|
@@ -1055,13 +2202,6 @@ async function restoreDbSnapshot(source, options = {}) {
|
|
|
1055
2202
|
const createOptions = { loadDataDir: data };
|
|
1056
2203
|
return createPGliteInstance(options.persistence ?? "memory", options.archiveName ?? "default", createOptions);
|
|
1057
2204
|
}
|
|
1058
|
-
function computeHash(data) {
|
|
1059
|
-
const digest = sha256(data);
|
|
1060
|
-
return `sha256:${bytesToHex(digest)}`;
|
|
1061
|
-
}
|
|
1062
|
-
function computeBlobHash(data) {
|
|
1063
|
-
return `blake3:${bytesToHex(blake3(data))}`;
|
|
1064
|
-
}
|
|
1065
2205
|
|
|
1066
2206
|
// src/repositories/notes-repository.ts
|
|
1067
2207
|
var NotesRepository = class {
|
|
@@ -1732,7 +2872,7 @@ var EmbeddingSetsRepository = class {
|
|
|
1732
2872
|
const set = await this.get(setId);
|
|
1733
2873
|
if (set.kind !== "virtual") throw new Error(`Embedding set is not virtual: ${setId}`);
|
|
1734
2874
|
const definition = this.definitionFromRow(set);
|
|
1735
|
-
const
|
|
2875
|
+
const now2 = (/* @__PURE__ */ new Date()).toISOString();
|
|
1736
2876
|
const materialization = definition.materialization ? { ...definition.materialization, freshness: "stale" } : null;
|
|
1737
2877
|
await this.db.query(
|
|
1738
2878
|
`UPDATE embedding_set
|
|
@@ -1741,7 +2881,7 @@ var EmbeddingSetsRepository = class {
|
|
|
1741
2881
|
[
|
|
1742
2882
|
setId,
|
|
1743
2883
|
jsonParam(materialization),
|
|
1744
|
-
jsonParam({ status: "stale", sourceHash: definition.materialization?.inputHash, checkedAt:
|
|
2884
|
+
jsonParam({ status: "stale", sourceHash: definition.materialization?.inputHash, checkedAt: now2, reason })
|
|
1745
2885
|
]
|
|
1746
2886
|
);
|
|
1747
2887
|
}
|
|
@@ -2004,6 +3144,90 @@ var EmbeddingSetsRepository = class {
|
|
|
2004
3144
|
}
|
|
2005
3145
|
};
|
|
2006
3146
|
|
|
3147
|
+
// src/repositories/metadata-predicates.ts
|
|
3148
|
+
var REGISTERED_METADATA_PATHS = [
|
|
3149
|
+
"provider",
|
|
3150
|
+
"model",
|
|
3151
|
+
"role",
|
|
3152
|
+
"event_kind",
|
|
3153
|
+
"sensitivity",
|
|
3154
|
+
"import_run_id"
|
|
3155
|
+
];
|
|
3156
|
+
var REGISTERED_SET = new Set(REGISTERED_METADATA_PATHS);
|
|
3157
|
+
var MAX_PREDICATES = 8;
|
|
3158
|
+
var MAX_IN_VALUES = 32;
|
|
3159
|
+
var MAX_VALUE_LENGTH = 256;
|
|
3160
|
+
function assertRegisteredPath(path) {
|
|
3161
|
+
if (!REGISTERED_SET.has(path) || path.includes(".") || path.includes("/")) {
|
|
3162
|
+
throw new Error(`Unsupported metadata predicate path: ${path}`);
|
|
3163
|
+
}
|
|
3164
|
+
}
|
|
3165
|
+
function assertBoundedValue(value) {
|
|
3166
|
+
if (typeof value === "string" && value.length > MAX_VALUE_LENGTH) {
|
|
3167
|
+
throw new Error("Metadata predicate value exceeds the 256 character bound");
|
|
3168
|
+
}
|
|
3169
|
+
}
|
|
3170
|
+
function jsonAccessor(path) {
|
|
3171
|
+
if (path === "import_run_id") return "si.import_run_id";
|
|
3172
|
+
return `c.ai_metadata ->> '${path}'`;
|
|
3173
|
+
}
|
|
3174
|
+
function buildMetadataPredicateConditions(options, startIdx) {
|
|
3175
|
+
const predicates = options.metadataPredicates ?? [];
|
|
3176
|
+
if (predicates.length > MAX_PREDICATES) {
|
|
3177
|
+
throw new Error(`Metadata predicate count exceeds the ${MAX_PREDICATES} predicate bound`);
|
|
3178
|
+
}
|
|
3179
|
+
const conditions = [];
|
|
3180
|
+
const params = [];
|
|
3181
|
+
const joins = [];
|
|
3182
|
+
let idx = startIdx;
|
|
3183
|
+
let needsSourceJoin = options.tenant_id !== void 0 || options.archive_id !== void 0;
|
|
3184
|
+
if (options.tenant_id !== void 0) {
|
|
3185
|
+
conditions.push(`COALESCE(si.tenant_id, 'default') = $${idx++}`);
|
|
3186
|
+
params.push(options.tenant_id);
|
|
3187
|
+
}
|
|
3188
|
+
if (options.archive_id !== void 0) {
|
|
3189
|
+
conditions.push(`si.archive_id IS NOT DISTINCT FROM $${idx++}`);
|
|
3190
|
+
params.push(options.archive_id);
|
|
3191
|
+
}
|
|
3192
|
+
for (const predicate of predicates) {
|
|
3193
|
+
assertRegisteredPath(predicate.path);
|
|
3194
|
+
const lhs = jsonAccessor(predicate.path);
|
|
3195
|
+
if (predicate.path === "import_run_id") needsSourceJoin = true;
|
|
3196
|
+
if (predicate.op === "eq") {
|
|
3197
|
+
assertBoundedValue(predicate.value);
|
|
3198
|
+
conditions.push(`${lhs} IS NOT DISTINCT FROM $${idx++}`);
|
|
3199
|
+
params.push(predicate.value == null ? null : String(predicate.value));
|
|
3200
|
+
} else if (predicate.op === "in") {
|
|
3201
|
+
if (predicate.value.length > MAX_IN_VALUES) {
|
|
3202
|
+
throw new Error(`Metadata predicate membership exceeds the ${MAX_IN_VALUES} value bound`);
|
|
3203
|
+
}
|
|
3204
|
+
for (const value of predicate.value) assertBoundedValue(value);
|
|
3205
|
+
conditions.push(`${lhs} = ANY($${idx++})`);
|
|
3206
|
+
params.push(predicate.value.map((value) => value == null ? null : String(value)));
|
|
3207
|
+
} else if (predicate.op === "range") {
|
|
3208
|
+
if (predicate.gte === void 0 && predicate.lte === void 0) {
|
|
3209
|
+
throw new Error("Metadata range predicate requires gte or lte");
|
|
3210
|
+
}
|
|
3211
|
+
if (predicate.gte !== void 0) {
|
|
3212
|
+
assertBoundedValue(predicate.gte);
|
|
3213
|
+
conditions.push(`${lhs} >= $${idx++}`);
|
|
3214
|
+
params.push(String(predicate.gte));
|
|
3215
|
+
}
|
|
3216
|
+
if (predicate.lte !== void 0) {
|
|
3217
|
+
assertBoundedValue(predicate.lte);
|
|
3218
|
+
conditions.push(`${lhs} <= $${idx++}`);
|
|
3219
|
+
params.push(String(predicate.lte));
|
|
3220
|
+
}
|
|
3221
|
+
} else {
|
|
3222
|
+
conditions.push(predicate.value === false ? `${lhs} IS NULL` : `${lhs} IS NOT NULL`);
|
|
3223
|
+
}
|
|
3224
|
+
}
|
|
3225
|
+
if (needsSourceJoin) {
|
|
3226
|
+
joins.push("LEFT JOIN source_identity si ON si.note_id = n.id");
|
|
3227
|
+
}
|
|
3228
|
+
return { conditions, joins, params, nextIdx: idx };
|
|
3229
|
+
}
|
|
3230
|
+
|
|
2007
3231
|
// src/repositories/search-repository.ts
|
|
2008
3232
|
var ATTACHMENT_TEXT_JOIN2 = `
|
|
2009
3233
|
LEFT JOIN (
|
|
@@ -2072,6 +3296,41 @@ var SearchRepository = class {
|
|
|
2072
3296
|
attachEmbeddingStatus(results, embeddingSet) {
|
|
2073
3297
|
return results.map((r) => ({ ...r, has_embedding: embeddingSet.has(r.id) }));
|
|
2074
3298
|
}
|
|
3299
|
+
async fetchLocatorMap(noteIds, metadataPaths = []) {
|
|
3300
|
+
const locators = /* @__PURE__ */ new Map();
|
|
3301
|
+
if (noteIds.length === 0) return locators;
|
|
3302
|
+
const result = await this.db.query(
|
|
3303
|
+
`SELECT note_id, namespace, external_id_hash, import_run_id, source_schema_version
|
|
3304
|
+
FROM source_identity
|
|
3305
|
+
WHERE note_id = ANY($1)
|
|
3306
|
+
ORDER BY created_at ASC`,
|
|
3307
|
+
[noteIds]
|
|
3308
|
+
);
|
|
3309
|
+
for (const row of result.rows) {
|
|
3310
|
+
const existing = locators.get(row.note_id) ?? [];
|
|
3311
|
+
existing.push({
|
|
3312
|
+
note_id: row.note_id,
|
|
3313
|
+
chunk: { kind: "current", index: 0 },
|
|
3314
|
+
source: {
|
|
3315
|
+
namespace: row.namespace,
|
|
3316
|
+
external_id_hash: row.external_id_hash,
|
|
3317
|
+
import_run_id: row.import_run_id,
|
|
3318
|
+
schema_version: row.source_schema_version
|
|
3319
|
+
},
|
|
3320
|
+
metadata_paths: [...metadataPaths]
|
|
3321
|
+
});
|
|
3322
|
+
locators.set(row.note_id, existing);
|
|
3323
|
+
}
|
|
3324
|
+
for (const noteId of noteIds) {
|
|
3325
|
+
if (!locators.has(noteId)) {
|
|
3326
|
+
locators.set(noteId, [{ note_id: noteId, chunk: { kind: "current", index: 0 }, metadata_paths: [...metadataPaths] }]);
|
|
3327
|
+
}
|
|
3328
|
+
}
|
|
3329
|
+
return locators;
|
|
3330
|
+
}
|
|
3331
|
+
metadataPaths(options) {
|
|
3332
|
+
return [...new Set((options.metadataPredicates ?? []).map((predicate) => predicate.path))];
|
|
3333
|
+
}
|
|
2075
3334
|
async search(query, options = {}, queryEmbedding) {
|
|
2076
3335
|
const { limit = 20, offset = 0 } = options;
|
|
2077
3336
|
const mode = options.mode ?? "auto";
|
|
@@ -2098,17 +3357,21 @@ var SearchRepository = class {
|
|
|
2098
3357
|
const resolvedEmbeddingSet = await this.resolveEmbeddingSet(options);
|
|
2099
3358
|
const tsqFn = this.tsqueryFn(query);
|
|
2100
3359
|
const { conditions, params, nextIdx } = buildNoteConditions(options, 2);
|
|
3360
|
+
const metadata = buildMetadataPredicateConditions(options, nextIdx);
|
|
3361
|
+
conditions.push(...metadata.conditions);
|
|
3362
|
+
params.push(...metadata.params);
|
|
2101
3363
|
conditions.unshift(
|
|
2102
3364
|
`(n.tsv @@ ${tsqFn}('english', $1) OR
|
|
2103
3365
|
${COMBINED_TEXT_VECTOR_SQL2} @@ ${tsqFn}('english', $1))`
|
|
2104
3366
|
);
|
|
2105
|
-
let paramIdx = this.scopeToResolvedEmbeddingSet(conditions, params, nextIdx, resolvedEmbeddingSet);
|
|
3367
|
+
let paramIdx = this.scopeToResolvedEmbeddingSet(conditions, params, metadata.nextIdx, resolvedEmbeddingSet);
|
|
2106
3368
|
const allParams = [query, ...params];
|
|
2107
3369
|
const where = conditions.join(" AND ");
|
|
2108
3370
|
const countResult = await this.db.query(
|
|
2109
3371
|
`SELECT COUNT(*) as count
|
|
2110
3372
|
FROM note n
|
|
2111
3373
|
LEFT JOIN note_revised_current c ON c.note_id = n.id
|
|
3374
|
+
${metadata.joins.join("\n")}
|
|
2112
3375
|
${ATTACHMENT_TEXT_JOIN2}
|
|
2113
3376
|
WHERE ${where}`,
|
|
2114
3377
|
allParams
|
|
@@ -2129,6 +3392,7 @@ var SearchRepository = class {
|
|
|
2129
3392
|
) as snippet
|
|
2130
3393
|
FROM note n
|
|
2131
3394
|
LEFT JOIN note_revised_current c ON c.note_id = n.id
|
|
3395
|
+
${metadata.joins.join("\n")}
|
|
2132
3396
|
${ATTACHMENT_TEXT_JOIN2}
|
|
2133
3397
|
WHERE ${where}
|
|
2134
3398
|
ORDER BY rank DESC, n.created_at DESC
|
|
@@ -2145,12 +3409,14 @@ var SearchRepository = class {
|
|
|
2145
3409
|
const idsResult = await this.db.query(
|
|
2146
3410
|
`SELECT n.id FROM note n
|
|
2147
3411
|
LEFT JOIN note_revised_current c ON c.note_id = n.id
|
|
3412
|
+
${metadata.joins.join("\n")}
|
|
2148
3413
|
${ATTACHMENT_TEXT_JOIN2}
|
|
2149
3414
|
WHERE ${where}`,
|
|
2150
3415
|
allParams
|
|
2151
3416
|
);
|
|
2152
3417
|
facets = await this.fetchFacets(idsResult.rows.map((r) => r.id));
|
|
2153
3418
|
}
|
|
3419
|
+
const locatorMap = await this.fetchLocatorMap(resultIds, this.metadataPaths(options));
|
|
2154
3420
|
const baseResults = result.rows.map((r) => ({
|
|
2155
3421
|
id: r.id,
|
|
2156
3422
|
title: r.title,
|
|
@@ -2158,7 +3424,8 @@ var SearchRepository = class {
|
|
|
2158
3424
|
rank: r.rank,
|
|
2159
3425
|
created_at: r.created_at,
|
|
2160
3426
|
updated_at: r.updated_at,
|
|
2161
|
-
tags: tagMap.get(r.id) ?? []
|
|
3427
|
+
tags: tagMap.get(r.id) ?? [],
|
|
3428
|
+
locators: locatorMap.get(r.id) ?? []
|
|
2162
3429
|
}));
|
|
2163
3430
|
return {
|
|
2164
3431
|
results: this.attachEmbeddingStatus(baseResults, embeddingSet),
|
|
@@ -2176,12 +3443,17 @@ var SearchRepository = class {
|
|
|
2176
3443
|
const vectorStr = `[${queryEmbedding.join(",")}]`;
|
|
2177
3444
|
const resolvedEmbeddingSet = await this.resolveEmbeddingSet(options);
|
|
2178
3445
|
const { conditions, params, nextIdx } = buildNoteConditions(options, 1);
|
|
2179
|
-
|
|
3446
|
+
const metadata = buildMetadataPredicateConditions(options, nextIdx);
|
|
3447
|
+
conditions.push(...metadata.conditions);
|
|
3448
|
+
params.push(...metadata.params);
|
|
3449
|
+
let paramIdx = this.scopeToResolvedEmbeddingRows(conditions, params, metadata.nextIdx, resolvedEmbeddingSet);
|
|
2180
3450
|
const where = conditions.join(" AND ");
|
|
2181
3451
|
const countResult = await this.db.query(
|
|
2182
3452
|
`SELECT COUNT(*) as count
|
|
2183
3453
|
FROM embedding e
|
|
2184
3454
|
JOIN note n ON n.id = e.note_id
|
|
3455
|
+
LEFT JOIN note_revised_current c ON c.note_id = n.id
|
|
3456
|
+
${metadata.joins.join("\n")}
|
|
2185
3457
|
WHERE ${where}`,
|
|
2186
3458
|
params
|
|
2187
3459
|
);
|
|
@@ -2196,6 +3468,7 @@ var SearchRepository = class {
|
|
|
2196
3468
|
FROM embedding e
|
|
2197
3469
|
JOIN note n ON n.id = e.note_id
|
|
2198
3470
|
LEFT JOIN note_revised_current c ON c.note_id = n.id
|
|
3471
|
+
${metadata.joins.join("\n")}
|
|
2199
3472
|
${ATTACHMENT_TEXT_JOIN2}
|
|
2200
3473
|
WHERE ${where}
|
|
2201
3474
|
ORDER BY e.vector <=> $${vecIdx}::vector ASC
|
|
@@ -2204,9 +3477,15 @@ var SearchRepository = class {
|
|
|
2204
3477
|
);
|
|
2205
3478
|
const tagMap = await this.fetchTagMap(result.rows.map((r) => r.id));
|
|
2206
3479
|
const facets = options.include_facets ? await this.fetchFacets((await this.db.query(
|
|
2207
|
-
`SELECT n.id
|
|
3480
|
+
`SELECT n.id
|
|
3481
|
+
FROM embedding e
|
|
3482
|
+
JOIN note n ON n.id = e.note_id
|
|
3483
|
+
LEFT JOIN note_revised_current c ON c.note_id = n.id
|
|
3484
|
+
${metadata.joins.join("\n")}
|
|
3485
|
+
WHERE ${where}`,
|
|
2208
3486
|
params
|
|
2209
3487
|
)).rows.map((r) => r.id)) : void 0;
|
|
3488
|
+
const locatorMap = await this.fetchLocatorMap(result.rows.map((r) => r.id), this.metadataPaths(options));
|
|
2210
3489
|
return {
|
|
2211
3490
|
results: result.rows.map((r) => ({
|
|
2212
3491
|
id: r.id,
|
|
@@ -2216,7 +3495,8 @@ var SearchRepository = class {
|
|
|
2216
3495
|
created_at: r.created_at,
|
|
2217
3496
|
updated_at: r.updated_at,
|
|
2218
3497
|
tags: tagMap.get(r.id) ?? [],
|
|
2219
|
-
has_embedding: true
|
|
3498
|
+
has_embedding: true,
|
|
3499
|
+
locators: locatorMap.get(r.id) ?? []
|
|
2220
3500
|
})),
|
|
2221
3501
|
total,
|
|
2222
3502
|
query: "",
|
|
@@ -2234,12 +3514,15 @@ var SearchRepository = class {
|
|
|
2234
3514
|
const tsqFn = this.tsqueryFn(query);
|
|
2235
3515
|
const resolvedEmbeddingSet = await this.resolveEmbeddingSet(options);
|
|
2236
3516
|
const textCond = buildNoteConditions(options, 2);
|
|
3517
|
+
const textMeta = buildMetadataPredicateConditions(options, textCond.nextIdx);
|
|
2237
3518
|
const textConditions = [
|
|
2238
3519
|
...textCond.conditions,
|
|
3520
|
+
...textMeta.conditions,
|
|
2239
3521
|
`(n.tsv @@ ${tsqFn}('english', $1) OR
|
|
2240
3522
|
${COMBINED_TEXT_VECTOR_SQL2} @@ ${tsqFn}('english', $1))`
|
|
2241
3523
|
];
|
|
2242
|
-
|
|
3524
|
+
textCond.params.push(...textMeta.params);
|
|
3525
|
+
this.scopeToResolvedEmbeddingSet(textConditions, textCond.params, textMeta.nextIdx, resolvedEmbeddingSet);
|
|
2243
3526
|
const textWhere = textConditions.join(" AND ");
|
|
2244
3527
|
const textParams = [query, ...textCond.params];
|
|
2245
3528
|
const textResult = await this.db.query(
|
|
@@ -2250,6 +3533,7 @@ var SearchRepository = class {
|
|
|
2250
3533
|
) as rank
|
|
2251
3534
|
FROM note n
|
|
2252
3535
|
LEFT JOIN note_revised_current c ON c.note_id = n.id
|
|
3536
|
+
${textMeta.joins.join("\n")}
|
|
2253
3537
|
${ATTACHMENT_TEXT_JOIN2}
|
|
2254
3538
|
WHERE ${textWhere}
|
|
2255
3539
|
ORDER BY rank DESC
|
|
@@ -2257,13 +3541,18 @@ var SearchRepository = class {
|
|
|
2257
3541
|
textParams
|
|
2258
3542
|
);
|
|
2259
3543
|
const vecCond = buildNoteConditions(options, 1);
|
|
2260
|
-
|
|
3544
|
+
const vecMeta = buildMetadataPredicateConditions(options, vecCond.nextIdx);
|
|
3545
|
+
vecCond.conditions.push(...vecMeta.conditions);
|
|
3546
|
+
vecCond.params.push(...vecMeta.params);
|
|
3547
|
+
vecCond.nextIdx = this.scopeToResolvedEmbeddingRows(vecCond.conditions, vecCond.params, vecMeta.nextIdx, resolvedEmbeddingSet);
|
|
2261
3548
|
const vecWhere = vecCond.conditions.join(" AND ");
|
|
2262
3549
|
const vecVecIdx = vecCond.nextIdx;
|
|
2263
3550
|
const vectorResult = await this.db.query(
|
|
2264
3551
|
`SELECT n.id, (e.vector <=> $${vecVecIdx}::vector) as distance
|
|
2265
3552
|
FROM embedding e
|
|
2266
3553
|
JOIN note n ON n.id = e.note_id
|
|
3554
|
+
LEFT JOIN note_revised_current c ON c.note_id = n.id
|
|
3555
|
+
${vecMeta.joins.join("\n")}
|
|
2267
3556
|
WHERE ${vecWhere}
|
|
2268
3557
|
ORDER BY e.vector <=> $${vecVecIdx}::vector ASC
|
|
2269
3558
|
LIMIT 100`,
|
|
@@ -2296,6 +3585,7 @@ var SearchRepository = class {
|
|
|
2296
3585
|
this.fetchTagMap(pageIds),
|
|
2297
3586
|
this.fetchEmbeddingStatus(pageIds, resolvedEmbeddingSet, options.embeddingSetId)
|
|
2298
3587
|
]);
|
|
3588
|
+
const locatorMap = await this.fetchLocatorMap(pageIds, this.metadataPaths(options));
|
|
2299
3589
|
const facets = options.include_facets ? await this.fetchFacets(sortedIds) : void 0;
|
|
2300
3590
|
return {
|
|
2301
3591
|
results: pageIds.map((id) => {
|
|
@@ -2309,7 +3599,8 @@ var SearchRepository = class {
|
|
|
2309
3599
|
created_at: r.created_at,
|
|
2310
3600
|
updated_at: r.updated_at,
|
|
2311
3601
|
tags: tagMap.get(id) ?? [],
|
|
2312
|
-
has_embedding: embeddingSet.has(id)
|
|
3602
|
+
has_embedding: embeddingSet.has(id),
|
|
3603
|
+
locators: locatorMap.get(id) ?? []
|
|
2313
3604
|
};
|
|
2314
3605
|
}).filter((r) => r !== null),
|
|
2315
3606
|
total,
|
|
@@ -2325,10 +3616,17 @@ var SearchRepository = class {
|
|
|
2325
3616
|
const { limit = 20, offset = 0 } = options;
|
|
2326
3617
|
const resolvedEmbeddingSet = await this.resolveEmbeddingSet(options);
|
|
2327
3618
|
const { conditions, params, nextIdx } = buildNoteConditions(options, 1);
|
|
2328
|
-
|
|
3619
|
+
const metadata = buildMetadataPredicateConditions(options, nextIdx);
|
|
3620
|
+
conditions.push(...metadata.conditions);
|
|
3621
|
+
params.push(...metadata.params);
|
|
3622
|
+
let paramIdx = this.scopeToResolvedEmbeddingSet(conditions, params, metadata.nextIdx, resolvedEmbeddingSet);
|
|
2329
3623
|
const where = conditions.join(" AND ");
|
|
2330
3624
|
const countResult = await this.db.query(
|
|
2331
|
-
`SELECT COUNT(*) as count
|
|
3625
|
+
`SELECT COUNT(*) as count
|
|
3626
|
+
FROM note n
|
|
3627
|
+
LEFT JOIN note_revised_current c ON c.note_id = n.id
|
|
3628
|
+
${metadata.joins.join("\n")}
|
|
3629
|
+
WHERE ${where}`,
|
|
2332
3630
|
params
|
|
2333
3631
|
);
|
|
2334
3632
|
const total = parseInt(countResult.rows[0].count, 10);
|
|
@@ -2338,6 +3636,7 @@ var SearchRepository = class {
|
|
|
2338
3636
|
LEFT(${COMBINED_TEXT_SQL}, 200) as snippet
|
|
2339
3637
|
FROM note n
|
|
2340
3638
|
LEFT JOIN note_revised_current c ON c.note_id = n.id
|
|
3639
|
+
${metadata.joins.join("\n")}
|
|
2341
3640
|
${ATTACHMENT_TEXT_JOIN2}
|
|
2342
3641
|
WHERE ${where}
|
|
2343
3642
|
ORDER BY n.created_at DESC
|
|
@@ -2346,6 +3645,7 @@ var SearchRepository = class {
|
|
|
2346
3645
|
);
|
|
2347
3646
|
const resultIds = result.rows.map((r) => r.id);
|
|
2348
3647
|
const embeddingSet = await this.fetchEmbeddingStatus(resultIds, resolvedEmbeddingSet, options.embeddingSetId);
|
|
3648
|
+
const locatorMap = await this.fetchLocatorMap(resultIds, this.metadataPaths(options));
|
|
2349
3649
|
return {
|
|
2350
3650
|
results: result.rows.map((r) => ({
|
|
2351
3651
|
id: r.id,
|
|
@@ -2355,7 +3655,8 @@ var SearchRepository = class {
|
|
|
2355
3655
|
created_at: r.created_at,
|
|
2356
3656
|
updated_at: r.updated_at,
|
|
2357
3657
|
tags: [],
|
|
2358
|
-
has_embedding: embeddingSet.has(r.id)
|
|
3658
|
+
has_embedding: embeddingSet.has(r.id),
|
|
3659
|
+
locators: locatorMap.get(r.id) ?? []
|
|
2359
3660
|
})),
|
|
2360
3661
|
total,
|
|
2361
3662
|
query: "",
|
|
@@ -3945,8 +5246,8 @@ async function migrateLegacyBlobStore(archiveName, target, indexedDbFactory) {
|
|
|
3945
5246
|
|
|
3946
5247
|
// src/blob-store.ts
|
|
3947
5248
|
var MemoryBlobStore = class {
|
|
3948
|
-
constructor(
|
|
3949
|
-
this.now =
|
|
5249
|
+
constructor(now2 = Date.now) {
|
|
5250
|
+
this.now = now2;
|
|
3950
5251
|
}
|
|
3951
5252
|
entries = /* @__PURE__ */ new Map();
|
|
3952
5253
|
async put(bytes) {
|
|
@@ -4019,12 +5320,12 @@ function toChecksum(hex) {
|
|
|
4019
5320
|
return CHECKSUM_PREFIX + hex;
|
|
4020
5321
|
}
|
|
4021
5322
|
var BytecaskBlobStore = class {
|
|
4022
|
-
constructor(facade, adapter, index, probe,
|
|
5323
|
+
constructor(facade, adapter, index, probe, now2 = Date.now) {
|
|
4023
5324
|
this.facade = facade;
|
|
4024
5325
|
this.adapter = adapter;
|
|
4025
5326
|
this.index = index;
|
|
4026
5327
|
this.probe = probe;
|
|
4027
|
-
this.now =
|
|
5328
|
+
this.now = now2;
|
|
4028
5329
|
}
|
|
4029
5330
|
async put(bytes) {
|
|
4030
5331
|
return toChecksum(await this.facade.put(bytes));
|
|
@@ -4159,6 +5460,482 @@ function createLazyBlobStore(archiveName, options) {
|
|
|
4159
5460
|
};
|
|
4160
5461
|
}
|
|
4161
5462
|
|
|
5463
|
+
// src/repositories/source-upsert-repository.ts
|
|
5464
|
+
var DEFAULT_MAX_ITEMS = 500;
|
|
5465
|
+
function assertSource(input) {
|
|
5466
|
+
if (!input.namespace || input.namespace.length > 128) throw new Error("Source namespace is required and must be <= 128 characters");
|
|
5467
|
+
if (!input.external_id || input.external_id.length > 1024) throw new Error("Source external_id is required and must be <= 1024 characters");
|
|
5468
|
+
if (!input.source_schema_version || input.source_schema_version.length > 64) {
|
|
5469
|
+
throw new Error("Source schema version is required and must be <= 64 characters");
|
|
5470
|
+
}
|
|
5471
|
+
if (!input.import_run_id || input.import_run_id.length > 128) throw new Error("Source import_run_id is required and must be <= 128 characters");
|
|
5472
|
+
}
|
|
5473
|
+
function sourceHash(source) {
|
|
5474
|
+
return computeHash(new TextEncoder().encode([
|
|
5475
|
+
source.tenant_id ?? "default",
|
|
5476
|
+
source.archive_id ?? "",
|
|
5477
|
+
source.namespace,
|
|
5478
|
+
source.external_id
|
|
5479
|
+
].join("\0")));
|
|
5480
|
+
}
|
|
5481
|
+
function contentDigest(content) {
|
|
5482
|
+
return computeHash(new TextEncoder().encode(content));
|
|
5483
|
+
}
|
|
5484
|
+
async function insertNote(tx, input, noteId, digest2) {
|
|
5485
|
+
const originalId = generateId();
|
|
5486
|
+
if (input.source.archive_id) {
|
|
5487
|
+
await tx.query(
|
|
5488
|
+
`INSERT INTO archive (id, name)
|
|
5489
|
+
VALUES ($1, $2)
|
|
5490
|
+
ON CONFLICT (id) DO NOTHING`,
|
|
5491
|
+
[input.source.archive_id, input.source.archive_id]
|
|
5492
|
+
);
|
|
5493
|
+
}
|
|
5494
|
+
await tx.query(
|
|
5495
|
+
`INSERT INTO note (id, archive_id, title, format, source, visibility)
|
|
5496
|
+
VALUES ($1, $2, $3, $4, $5, $6)`,
|
|
5497
|
+
[
|
|
5498
|
+
noteId,
|
|
5499
|
+
input.source.archive_id ?? null,
|
|
5500
|
+
input.title ?? null,
|
|
5501
|
+
input.format ?? "markdown",
|
|
5502
|
+
`source:${input.source.namespace}`,
|
|
5503
|
+
input.visibility ?? "private"
|
|
5504
|
+
]
|
|
5505
|
+
);
|
|
5506
|
+
await tx.query(
|
|
5507
|
+
`INSERT INTO note_original (id, note_id, content, content_hash)
|
|
5508
|
+
VALUES ($1, $2, $3, $4)`,
|
|
5509
|
+
[originalId, noteId, input.content, digest2]
|
|
5510
|
+
);
|
|
5511
|
+
await tx.query(
|
|
5512
|
+
`INSERT INTO note_revised_current (note_id, content, ai_metadata)
|
|
5513
|
+
VALUES ($1, $2, $3::jsonb)`,
|
|
5514
|
+
[noteId, input.content, JSON.stringify(input.metadata ?? null)]
|
|
5515
|
+
);
|
|
5516
|
+
}
|
|
5517
|
+
async function updateNote(tx, input, noteId, outcome) {
|
|
5518
|
+
if (input.source.archive_id) {
|
|
5519
|
+
await tx.query(
|
|
5520
|
+
`INSERT INTO archive (id, name)
|
|
5521
|
+
VALUES ($1, $2)
|
|
5522
|
+
ON CONFLICT (id) DO NOTHING`,
|
|
5523
|
+
[input.source.archive_id, input.source.archive_id]
|
|
5524
|
+
);
|
|
5525
|
+
}
|
|
5526
|
+
if (outcome === "versioned") {
|
|
5527
|
+
const count2 = await tx.query(
|
|
5528
|
+
`SELECT COUNT(*) AS count FROM note_revision WHERE note_id = $1`,
|
|
5529
|
+
[noteId]
|
|
5530
|
+
);
|
|
5531
|
+
const current = await tx.query(
|
|
5532
|
+
`SELECT content, ai_metadata FROM note_revised_current WHERE note_id = $1`,
|
|
5533
|
+
[noteId]
|
|
5534
|
+
);
|
|
5535
|
+
const nextRevision = Number.parseInt(count2.rows[0]?.count ?? "0", 10) + 1;
|
|
5536
|
+
if (current.rows[0]) {
|
|
5537
|
+
await tx.query(
|
|
5538
|
+
`INSERT INTO note_revision (id, note_id, revision_number, type, content, ai_metadata)
|
|
5539
|
+
VALUES ($1, $2, $3, 'source-import', $4, $5::jsonb)`,
|
|
5540
|
+
[
|
|
5541
|
+
generateId(),
|
|
5542
|
+
noteId,
|
|
5543
|
+
nextRevision,
|
|
5544
|
+
current.rows[0].content,
|
|
5545
|
+
JSON.stringify(current.rows[0].ai_metadata ?? null)
|
|
5546
|
+
]
|
|
5547
|
+
);
|
|
5548
|
+
}
|
|
5549
|
+
}
|
|
5550
|
+
await tx.query(
|
|
5551
|
+
`UPDATE note
|
|
5552
|
+
SET title = $1, format = $2, visibility = $3, archive_id = $4, updated_at = now(), deleted_at = NULL
|
|
5553
|
+
WHERE id = $5`,
|
|
5554
|
+
[
|
|
5555
|
+
input.title ?? null,
|
|
5556
|
+
input.format ?? "markdown",
|
|
5557
|
+
input.visibility ?? "private",
|
|
5558
|
+
input.source.archive_id ?? null,
|
|
5559
|
+
noteId
|
|
5560
|
+
]
|
|
5561
|
+
);
|
|
5562
|
+
await tx.query(
|
|
5563
|
+
`UPDATE note_revised_current
|
|
5564
|
+
SET content = $1, ai_metadata = $2::jsonb, is_user_edited = false, updated_at = now()
|
|
5565
|
+
WHERE note_id = $3`,
|
|
5566
|
+
[input.content, JSON.stringify(input.metadata ?? null), noteId]
|
|
5567
|
+
);
|
|
5568
|
+
}
|
|
5569
|
+
var SourceUpsertRepository = class {
|
|
5570
|
+
constructor(db, events) {
|
|
5571
|
+
this.db = db;
|
|
5572
|
+
this.events = events;
|
|
5573
|
+
}
|
|
5574
|
+
async upsertBatch(items, options = {}) {
|
|
5575
|
+
const maxItems = options.maxItems ?? DEFAULT_MAX_ITEMS;
|
|
5576
|
+
if (items.length > maxItems) throw new Error(`Source upsert batch exceeds the ${maxItems} item bound`);
|
|
5577
|
+
if (items.length === 0) {
|
|
5578
|
+
return {
|
|
5579
|
+
import_run_id: "",
|
|
5580
|
+
dry_run: options.dryRun === true,
|
|
5581
|
+
outcomes: [],
|
|
5582
|
+
counts: { inserted: 0, unchanged: 0, versioned: 0, replaced: 0, conflict: 0, rejected: 0 }
|
|
5583
|
+
};
|
|
5584
|
+
}
|
|
5585
|
+
const outcomes = [];
|
|
5586
|
+
for (const [index, item] of items.entries()) {
|
|
5587
|
+
try {
|
|
5588
|
+
assertSource(item.source);
|
|
5589
|
+
outcomes.push({
|
|
5590
|
+
index,
|
|
5591
|
+
outcome: "rejected",
|
|
5592
|
+
external_id_hash: sourceHash(item.source),
|
|
5593
|
+
content_digest: contentDigest(item.content)
|
|
5594
|
+
});
|
|
5595
|
+
} catch (error) {
|
|
5596
|
+
outcomes.push({
|
|
5597
|
+
index,
|
|
5598
|
+
outcome: "rejected",
|
|
5599
|
+
external_id_hash: item.source ? sourceHash({ ...item.source, external_id: item.source.external_id ?? "" }) : "",
|
|
5600
|
+
content_digest: contentDigest(item.content ?? ""),
|
|
5601
|
+
reason: error instanceof Error ? error.message : String(error)
|
|
5602
|
+
});
|
|
5603
|
+
}
|
|
5604
|
+
}
|
|
5605
|
+
if (outcomes.some((outcome) => outcome.reason)) {
|
|
5606
|
+
return this.finish(items[0].source?.import_run_id ?? "", options.dryRun === true, outcomes);
|
|
5607
|
+
}
|
|
5608
|
+
if (options.dryRun) {
|
|
5609
|
+
const preview = [];
|
|
5610
|
+
for (const [index, item] of items.entries()) {
|
|
5611
|
+
const externalIdHash = sourceHash(item.source);
|
|
5612
|
+
const digest2 = contentDigest(item.content);
|
|
5613
|
+
const existing = await this.db.query(
|
|
5614
|
+
`SELECT note_id, content_digest
|
|
5615
|
+
FROM source_identity
|
|
5616
|
+
WHERE tenant_id = $1
|
|
5617
|
+
AND archive_id IS NOT DISTINCT FROM $2
|
|
5618
|
+
AND namespace = $3
|
|
5619
|
+
AND external_id = $4
|
|
5620
|
+
LIMIT 1`,
|
|
5621
|
+
[item.source.tenant_id ?? "default", item.source.archive_id ?? null, item.source.namespace, item.source.external_id]
|
|
5622
|
+
);
|
|
5623
|
+
if (existing.rows.length === 0) {
|
|
5624
|
+
preview.push({ index, outcome: "inserted", external_id_hash: externalIdHash, content_digest: digest2 });
|
|
5625
|
+
} else if (existing.rows[0].content_digest === digest2) {
|
|
5626
|
+
preview.push({ index, outcome: "unchanged", note_id: existing.rows[0].note_id, external_id_hash: externalIdHash, content_digest: digest2 });
|
|
5627
|
+
} else if ((item.policy ?? "version") === "conflict") {
|
|
5628
|
+
preview.push({ index, outcome: "conflict", note_id: existing.rows[0].note_id, external_id_hash: externalIdHash, content_digest: digest2 });
|
|
5629
|
+
} else {
|
|
5630
|
+
preview.push({
|
|
5631
|
+
index,
|
|
5632
|
+
outcome: item.policy === "replace" ? "replaced" : "versioned",
|
|
5633
|
+
note_id: existing.rows[0].note_id,
|
|
5634
|
+
external_id_hash: externalIdHash,
|
|
5635
|
+
content_digest: digest2
|
|
5636
|
+
});
|
|
5637
|
+
}
|
|
5638
|
+
}
|
|
5639
|
+
return this.finish(items[0].source.import_run_id, true, preview);
|
|
5640
|
+
}
|
|
5641
|
+
await this.db.transaction(async (tx) => {
|
|
5642
|
+
for (const [index, item] of items.entries()) {
|
|
5643
|
+
const externalIdHash = sourceHash(item.source);
|
|
5644
|
+
const digest2 = contentDigest(item.content);
|
|
5645
|
+
const existing = await tx.query(
|
|
5646
|
+
`SELECT note_id, content_digest
|
|
5647
|
+
FROM source_identity
|
|
5648
|
+
WHERE tenant_id = $1
|
|
5649
|
+
AND archive_id IS NOT DISTINCT FROM $2
|
|
5650
|
+
AND namespace = $3
|
|
5651
|
+
AND external_id = $4
|
|
5652
|
+
LIMIT 1`,
|
|
5653
|
+
[item.source.tenant_id ?? "default", item.source.archive_id ?? null, item.source.namespace, item.source.external_id]
|
|
5654
|
+
);
|
|
5655
|
+
if (existing.rows.length === 0) {
|
|
5656
|
+
const noteId = item.source.caller_stable_id ?? generateId();
|
|
5657
|
+
await insertNote(tx, item, noteId, digest2);
|
|
5658
|
+
await tx.query(
|
|
5659
|
+
`INSERT INTO source_identity
|
|
5660
|
+
(id, tenant_id, archive_id, namespace, external_id, external_id_hash,
|
|
5661
|
+
source_schema_version, content_digest, import_run_id, caller_stable_id, note_id)
|
|
5662
|
+
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)`,
|
|
5663
|
+
[
|
|
5664
|
+
generateId(),
|
|
5665
|
+
item.source.tenant_id ?? "default",
|
|
5666
|
+
item.source.archive_id ?? null,
|
|
5667
|
+
item.source.namespace,
|
|
5668
|
+
item.source.external_id,
|
|
5669
|
+
externalIdHash,
|
|
5670
|
+
item.source.source_schema_version,
|
|
5671
|
+
digest2,
|
|
5672
|
+
item.source.import_run_id,
|
|
5673
|
+
item.source.caller_stable_id ?? null,
|
|
5674
|
+
noteId
|
|
5675
|
+
]
|
|
5676
|
+
);
|
|
5677
|
+
outcomes[index] = { index, outcome: "inserted", note_id: noteId, external_id_hash: externalIdHash, content_digest: digest2 };
|
|
5678
|
+
continue;
|
|
5679
|
+
}
|
|
5680
|
+
const row = existing.rows[0];
|
|
5681
|
+
if (row.content_digest === digest2) {
|
|
5682
|
+
outcomes[index] = { index, outcome: "unchanged", note_id: row.note_id, external_id_hash: externalIdHash, content_digest: digest2 };
|
|
5683
|
+
continue;
|
|
5684
|
+
}
|
|
5685
|
+
const policy = item.policy ?? "version";
|
|
5686
|
+
if (policy === "conflict") {
|
|
5687
|
+
outcomes[index] = { index, outcome: "conflict", note_id: row.note_id, external_id_hash: externalIdHash, content_digest: digest2 };
|
|
5688
|
+
continue;
|
|
5689
|
+
}
|
|
5690
|
+
const outcome = policy === "replace" ? "replaced" : "versioned";
|
|
5691
|
+
await updateNote(tx, item, row.note_id, outcome);
|
|
5692
|
+
await tx.query(
|
|
5693
|
+
`UPDATE source_identity
|
|
5694
|
+
SET source_schema_version = $1, content_digest = $2, import_run_id = $3, updated_at = now()
|
|
5695
|
+
WHERE note_id = $4
|
|
5696
|
+
AND tenant_id = $5
|
|
5697
|
+
AND archive_id IS NOT DISTINCT FROM $6
|
|
5698
|
+
AND namespace = $7
|
|
5699
|
+
AND external_id = $8`,
|
|
5700
|
+
[
|
|
5701
|
+
item.source.source_schema_version,
|
|
5702
|
+
digest2,
|
|
5703
|
+
item.source.import_run_id,
|
|
5704
|
+
row.note_id,
|
|
5705
|
+
item.source.tenant_id ?? "default",
|
|
5706
|
+
item.source.archive_id ?? null,
|
|
5707
|
+
item.source.namespace,
|
|
5708
|
+
item.source.external_id
|
|
5709
|
+
]
|
|
5710
|
+
);
|
|
5711
|
+
outcomes[index] = { index, outcome, note_id: row.note_id, external_id_hash: externalIdHash, content_digest: digest2 };
|
|
5712
|
+
}
|
|
5713
|
+
if (hasMaterialChange(outcomes)) {
|
|
5714
|
+
await tx.query(
|
|
5715
|
+
`INSERT INTO source_import_run (id, tenant_id, archive_id, namespace, completed_at, checkpoint, receipt)
|
|
5716
|
+
VALUES ($1, $2, $3, $4, now(), $5::jsonb, $6::jsonb)
|
|
5717
|
+
ON CONFLICT (id) DO UPDATE
|
|
5718
|
+
SET completed_at = EXCLUDED.completed_at,
|
|
5719
|
+
checkpoint = EXCLUDED.checkpoint,
|
|
5720
|
+
receipt = EXCLUDED.receipt`,
|
|
5721
|
+
[
|
|
5722
|
+
items[0].source.import_run_id,
|
|
5723
|
+
items[0].source.tenant_id ?? "default",
|
|
5724
|
+
items[0].source.archive_id ?? null,
|
|
5725
|
+
items[0].source.namespace,
|
|
5726
|
+
JSON.stringify({ item_count: items.length }),
|
|
5727
|
+
JSON.stringify({ counts: countOutcomes(outcomes) })
|
|
5728
|
+
]
|
|
5729
|
+
);
|
|
5730
|
+
}
|
|
5731
|
+
});
|
|
5732
|
+
if (hasMaterialChange(outcomes)) {
|
|
5733
|
+
this.events?.emit("source.upserted", { importRunId: items[0].source.import_run_id, counts: countOutcomes(outcomes) });
|
|
5734
|
+
}
|
|
5735
|
+
return this.finish(items[0].source.import_run_id, false, outcomes);
|
|
5736
|
+
}
|
|
5737
|
+
finish(importRunId, dryRun, outcomes) {
|
|
5738
|
+
return { import_run_id: importRunId, dry_run: dryRun, outcomes, counts: countOutcomes(outcomes) };
|
|
5739
|
+
}
|
|
5740
|
+
};
|
|
5741
|
+
function hasMaterialChange(outcomes) {
|
|
5742
|
+
return outcomes.some((outcome) => outcome.outcome === "inserted" || outcome.outcome === "versioned" || outcome.outcome === "replaced");
|
|
5743
|
+
}
|
|
5744
|
+
function countOutcomes(outcomes) {
|
|
5745
|
+
return {
|
|
5746
|
+
inserted: outcomes.filter((outcome) => outcome.outcome === "inserted").length,
|
|
5747
|
+
unchanged: outcomes.filter((outcome) => outcome.outcome === "unchanged").length,
|
|
5748
|
+
versioned: outcomes.filter((outcome) => outcome.outcome === "versioned").length,
|
|
5749
|
+
replaced: outcomes.filter((outcome) => outcome.outcome === "replaced").length,
|
|
5750
|
+
conflict: outcomes.filter((outcome) => outcome.outcome === "conflict").length,
|
|
5751
|
+
rejected: outcomes.filter((outcome) => outcome.outcome === "rejected").length
|
|
5752
|
+
};
|
|
5753
|
+
}
|
|
5754
|
+
|
|
5755
|
+
// src/repositories/lifecycle-purge-repository.ts
|
|
5756
|
+
function zeroCounts() {
|
|
5757
|
+
return {
|
|
5758
|
+
notes: 0,
|
|
5759
|
+
revisions: 0,
|
|
5760
|
+
links: 0,
|
|
5761
|
+
tags: 0,
|
|
5762
|
+
embeddings: 0,
|
|
5763
|
+
attachments: 0,
|
|
5764
|
+
blobs: 0,
|
|
5765
|
+
graph_edges: 0,
|
|
5766
|
+
provenance_edges: 0,
|
|
5767
|
+
source_identities: 0
|
|
5768
|
+
};
|
|
5769
|
+
}
|
|
5770
|
+
function selectorHash(selector) {
|
|
5771
|
+
return computeHash(new TextEncoder().encode(JSON.stringify({
|
|
5772
|
+
tenant_id: selector.tenant_id ?? "default",
|
|
5773
|
+
archive_id: selector.archive_id ?? null,
|
|
5774
|
+
note_ids: [...selector.note_ids ?? []].sort(),
|
|
5775
|
+
source: selector.source ? {
|
|
5776
|
+
namespace: selector.source.namespace,
|
|
5777
|
+
external_id_hash: selector.source.external_id ? computeHash(new TextEncoder().encode(selector.source.external_id)) : null
|
|
5778
|
+
} : null
|
|
5779
|
+
})));
|
|
5780
|
+
}
|
|
5781
|
+
function buildSelectorWhere(selector, startIdx) {
|
|
5782
|
+
const clauses = [];
|
|
5783
|
+
const params = [];
|
|
5784
|
+
let idx = startIdx;
|
|
5785
|
+
if (selector.note_ids?.length) {
|
|
5786
|
+
clauses.push(`n.id = ANY($${idx++})`);
|
|
5787
|
+
params.push([...selector.note_ids]);
|
|
5788
|
+
}
|
|
5789
|
+
if (selector.tenant_id !== void 0 || selector.archive_id !== void 0 || selector.source) {
|
|
5790
|
+
clauses.push(`EXISTS (
|
|
5791
|
+
SELECT 1 FROM source_identity si
|
|
5792
|
+
WHERE si.note_id = n.id
|
|
5793
|
+
AND si.tenant_id = $${idx++}
|
|
5794
|
+
AND si.archive_id IS NOT DISTINCT FROM $${idx++}
|
|
5795
|
+
${selector.source ? `AND si.namespace = $${idx++}` : ""}
|
|
5796
|
+
${selector.source?.external_id ? `AND si.external_id = $${idx++}` : ""}
|
|
5797
|
+
)`);
|
|
5798
|
+
params.push(selector.tenant_id ?? "default", selector.archive_id ?? null);
|
|
5799
|
+
if (selector.source) params.push(selector.source.namespace);
|
|
5800
|
+
if (selector.source?.external_id) params.push(selector.source.external_id);
|
|
5801
|
+
}
|
|
5802
|
+
if (clauses.length === 0) throw new Error("Purge selector must target note_ids or source identity");
|
|
5803
|
+
return { sql: clauses.join(" AND "), params };
|
|
5804
|
+
}
|
|
5805
|
+
async function selectedNoteIds(db, selector) {
|
|
5806
|
+
const where = buildSelectorWhere(selector, 1);
|
|
5807
|
+
const result = await db.query(
|
|
5808
|
+
`SELECT n.id FROM note n WHERE ${where.sql} ORDER BY n.id`,
|
|
5809
|
+
where.params
|
|
5810
|
+
);
|
|
5811
|
+
return result.rows.map((row) => row.id);
|
|
5812
|
+
}
|
|
5813
|
+
var LifecyclePurgeRepository = class {
|
|
5814
|
+
constructor(db, events) {
|
|
5815
|
+
this.db = db;
|
|
5816
|
+
this.events = events;
|
|
5817
|
+
}
|
|
5818
|
+
async preview(selector) {
|
|
5819
|
+
const noteIds = await selectedNoteIds(this.db, selector);
|
|
5820
|
+
return { selector_hash: selectorHash(selector), counts: await this.count(noteIds) };
|
|
5821
|
+
}
|
|
5822
|
+
async purge(selector, operationKey) {
|
|
5823
|
+
const existing = await this.db.query(
|
|
5824
|
+
`SELECT id, operation_key, tenant_id, archive_id, selector_hash, outcome, counts, completed_at, policy
|
|
5825
|
+
FROM deletion_receipt
|
|
5826
|
+
WHERE operation_key = $1`,
|
|
5827
|
+
[operationKey]
|
|
5828
|
+
);
|
|
5829
|
+
if (existing.rows[0]) return existing.rows[0];
|
|
5830
|
+
const hash = selectorHash(selector);
|
|
5831
|
+
let receipt;
|
|
5832
|
+
await this.db.transaction(async (tx) => {
|
|
5833
|
+
const noteIds = await selectedNoteIds(tx, selector);
|
|
5834
|
+
const counts = await this.count(noteIds, tx);
|
|
5835
|
+
await this.deleteSelected(tx, noteIds);
|
|
5836
|
+
receipt = {
|
|
5837
|
+
id: generateId(),
|
|
5838
|
+
operation_key: operationKey,
|
|
5839
|
+
tenant_id: selector.tenant_id ?? "default",
|
|
5840
|
+
archive_id: selector.archive_id ?? null,
|
|
5841
|
+
selector_hash: hash,
|
|
5842
|
+
outcome: "completed",
|
|
5843
|
+
counts,
|
|
5844
|
+
completed_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
5845
|
+
policy: {
|
|
5846
|
+
authority: "fortemi#1092",
|
|
5847
|
+
mode: "terminal-purge",
|
|
5848
|
+
receipt_contains_content: false
|
|
5849
|
+
}
|
|
5850
|
+
};
|
|
5851
|
+
await tx.query(
|
|
5852
|
+
`INSERT INTO deletion_receipt
|
|
5853
|
+
(id, operation_key, tenant_id, archive_id, selector_hash, outcome, counts, completed_at, policy)
|
|
5854
|
+
VALUES ($1, $2, $3, $4, $5, 'completed', $6::jsonb, $7, $8::jsonb)`,
|
|
5855
|
+
[
|
|
5856
|
+
receipt.id,
|
|
5857
|
+
receipt.operation_key,
|
|
5858
|
+
receipt.tenant_id,
|
|
5859
|
+
receipt.archive_id,
|
|
5860
|
+
receipt.selector_hash,
|
|
5861
|
+
JSON.stringify(receipt.counts),
|
|
5862
|
+
receipt.completed_at,
|
|
5863
|
+
JSON.stringify(receipt.policy)
|
|
5864
|
+
]
|
|
5865
|
+
);
|
|
5866
|
+
});
|
|
5867
|
+
const completedReceipt = receipt;
|
|
5868
|
+
if (!completedReceipt) throw new Error("Purge transaction did not produce a receipt");
|
|
5869
|
+
this.events?.emit("purge.completed", { receiptId: completedReceipt.id, counts: completedReceipt.counts });
|
|
5870
|
+
return completedReceipt;
|
|
5871
|
+
}
|
|
5872
|
+
async count(noteIds, db = this.db) {
|
|
5873
|
+
const counts = zeroCounts();
|
|
5874
|
+
if (noteIds.length === 0) return counts;
|
|
5875
|
+
const params = [noteIds];
|
|
5876
|
+
const rows = await Promise.all([
|
|
5877
|
+
db.query("SELECT COUNT(*) AS count FROM note WHERE id = ANY($1)", params),
|
|
5878
|
+
db.query("SELECT COUNT(*) AS count FROM note_revision WHERE note_id = ANY($1)", params),
|
|
5879
|
+
db.query("SELECT COUNT(*) AS count FROM link WHERE source_note_id = ANY($1) OR target_note_id = ANY($1)", params),
|
|
5880
|
+
db.query("SELECT COUNT(*) AS count FROM note_tag WHERE note_id = ANY($1)", params),
|
|
5881
|
+
db.query("SELECT COUNT(*) AS count FROM embedding WHERE note_id = ANY($1)", params),
|
|
5882
|
+
db.query("SELECT COUNT(*) AS count FROM attachment WHERE note_id = ANY($1)", params),
|
|
5883
|
+
db.query(
|
|
5884
|
+
`SELECT COUNT(*) AS count FROM attachment_blob ab
|
|
5885
|
+
WHERE EXISTS (SELECT 1 FROM attachment a WHERE a.blob_id = ab.id AND a.note_id = ANY($1))`,
|
|
5886
|
+
params
|
|
5887
|
+
),
|
|
5888
|
+
db.query("SELECT COUNT(*) AS count FROM graph_edge_artifact WHERE from_note_id = ANY($1) OR to_note_id = ANY($1)", params),
|
|
5889
|
+
db.query(
|
|
5890
|
+
`SELECT COUNT(*) AS count FROM provenance_edge
|
|
5891
|
+
WHERE (entity_type = 'note' AND entity_id = ANY($1))
|
|
5892
|
+
OR (attributes ->> 'note_id') = ANY($1)`,
|
|
5893
|
+
params
|
|
5894
|
+
),
|
|
5895
|
+
db.query("SELECT COUNT(*) AS count FROM source_identity WHERE note_id = ANY($1)", params)
|
|
5896
|
+
]);
|
|
5897
|
+
const values = rows.map((row) => Number.parseInt(row.rows[0]?.count ?? "0", 10));
|
|
5898
|
+
[
|
|
5899
|
+
counts.notes,
|
|
5900
|
+
counts.revisions,
|
|
5901
|
+
counts.links,
|
|
5902
|
+
counts.tags,
|
|
5903
|
+
counts.embeddings,
|
|
5904
|
+
counts.attachments,
|
|
5905
|
+
counts.blobs,
|
|
5906
|
+
counts.graph_edges,
|
|
5907
|
+
counts.provenance_edges,
|
|
5908
|
+
counts.source_identities
|
|
5909
|
+
] = values;
|
|
5910
|
+
return counts;
|
|
5911
|
+
}
|
|
5912
|
+
async deleteSelected(tx, noteIds) {
|
|
5913
|
+
if (noteIds.length === 0) return;
|
|
5914
|
+
const params = [noteIds];
|
|
5915
|
+
await tx.query("DELETE FROM community_assignment WHERE note_id = ANY($1)", params);
|
|
5916
|
+
await tx.query("DELETE FROM graph_edge_artifact WHERE from_note_id = ANY($1) OR to_note_id = ANY($1)", params);
|
|
5917
|
+
await tx.query("DELETE FROM embedding_set_member WHERE note_id = ANY($1)", params);
|
|
5918
|
+
await tx.query("DELETE FROM embedding WHERE note_id = ANY($1)", params);
|
|
5919
|
+
await tx.query("DELETE FROM attachment_embedding WHERE attachment_id IN (SELECT id FROM attachment WHERE note_id = ANY($1))", params);
|
|
5920
|
+
await tx.query("DELETE FROM attachment WHERE note_id = ANY($1)", params);
|
|
5921
|
+
await tx.query(
|
|
5922
|
+
`DELETE FROM attachment_blob ab
|
|
5923
|
+
WHERE NOT EXISTS (SELECT 1 FROM attachment a WHERE a.blob_id = ab.id)`
|
|
5924
|
+
);
|
|
5925
|
+
await tx.query("DELETE FROM source_identity WHERE note_id = ANY($1)", params);
|
|
5926
|
+
await tx.query("DELETE FROM provenance_edge WHERE (entity_type = $2 AND entity_id = ANY($1)) OR (attributes ->> $3) = ANY($1)", [noteIds, "note", "note_id"]);
|
|
5927
|
+
await tx.query("DELETE FROM job_queue WHERE note_id = ANY($1)", params);
|
|
5928
|
+
await tx.query("DELETE FROM collection_note WHERE note_id = ANY($1)", params);
|
|
5929
|
+
await tx.query("DELETE FROM note_tag WHERE note_id = ANY($1)", params);
|
|
5930
|
+
await tx.query("DELETE FROM link WHERE source_note_id = ANY($1) OR target_note_id = ANY($1)", params);
|
|
5931
|
+
await tx.query("DELETE FROM note_revision WHERE note_id = ANY($1)", params);
|
|
5932
|
+
await tx.query("DELETE FROM note_revised_current WHERE note_id = ANY($1)", params);
|
|
5933
|
+
await tx.query("DELETE FROM note_original WHERE note_id = ANY($1)", params);
|
|
5934
|
+
await tx.query("DELETE FROM shard_field_presence WHERE component = $2 AND record_id = ANY($1)", [noteIds, "notes"]);
|
|
5935
|
+
await tx.query("DELETE FROM note WHERE id = ANY($1)", params);
|
|
5936
|
+
}
|
|
5937
|
+
};
|
|
5938
|
+
|
|
4162
5939
|
// src/repositories/graph-repository.ts
|
|
4163
5940
|
var SIMILARITY_GRAPH_ALGORITHM = "knn-batched-v1";
|
|
4164
5941
|
var DEFAULT_GRAPH_BATCH_SIZE = 64;
|
|
@@ -4874,7 +6651,7 @@ function titleGenerationHandler(job, db) {
|
|
|
4874
6651
|
|
|
4875
6652
|
Note content:
|
|
4876
6653
|
${content.slice(0, 1e3)}`;
|
|
4877
|
-
const llmTitle = (await llmFn2(prompt, { maxTokens: 60, temperature: 0.3 })).trim();
|
|
6654
|
+
const llmTitle = (await llmFn2(prompt, { maxTokens: 60, temperature: 0.3, task: "chat.general" })).trim();
|
|
4878
6655
|
if (llmTitle) {
|
|
4879
6656
|
const title2 = llmTitle.length > 200 ? llmTitle.slice(0, 197) + "..." : llmTitle;
|
|
4880
6657
|
await db.query(
|
|
@@ -4915,7 +6692,7 @@ Respond with ONLY the enhanced note content, no explanation.
|
|
|
4915
6692
|
|
|
4916
6693
|
Original note:
|
|
4917
6694
|
${content}`;
|
|
4918
|
-
const revised = (await llmFn2(prompt, { maxTokens: 2e3, temperature: 0.4 })).trim();
|
|
6695
|
+
const revised = (await llmFn2(prompt, { maxTokens: 2e3, temperature: 0.4, task: "chat.revision" })).trim();
|
|
4919
6696
|
if (!revised || revised === content) return { skipped: true, reason: "no changes from LLM" };
|
|
4920
6697
|
const revResult = await db.query(
|
|
4921
6698
|
`SELECT COALESCE(MAX(revision_number), 0) as max_rev FROM note_revision WHERE note_id = $1`,
|
|
@@ -4956,7 +6733,7 @@ Text:
|
|
|
4956
6733
|
${content.slice(0, 1500)}
|
|
4957
6734
|
|
|
4958
6735
|
Tags:`;
|
|
4959
|
-
const response = (await llmFn2(prompt, { maxTokens: 60, temperature: 0.1 })).trim();
|
|
6736
|
+
const response = (await llmFn2(prompt, { maxTokens: 60, temperature: 0.1, task: "chat.tagging" })).trim();
|
|
4960
6737
|
const tags = response.split(/[,\n]/).map((t) => t.trim().toLowerCase().replace(/^[-*\d.]+\s*/, "").replace(/['"]/g, "")).filter((t) => {
|
|
4961
6738
|
if (t.length < 2 || t.length > 40) return false;
|
|
4962
6739
|
if (/^\d+$/.test(t)) return false;
|
|
@@ -6239,10 +8016,24 @@ function chunkText(text, maxChars = 800, overlap = 100) {
|
|
|
6239
8016
|
}
|
|
6240
8017
|
|
|
6241
8018
|
// src/capabilities/embedding-handler.ts
|
|
8019
|
+
var DEFAULT_LARGE_DOCUMENT_CHARS = 12e3;
|
|
8020
|
+
var DEFAULT_LARGE_DOCUMENT_CHUNKS = 12;
|
|
6242
8021
|
var embedFn = null;
|
|
8022
|
+
var embeddingTaskSelectionOptions = {};
|
|
6243
8023
|
function setEmbedFunction(fn) {
|
|
6244
8024
|
embedFn = fn;
|
|
6245
8025
|
}
|
|
8026
|
+
function setEmbeddingTaskSelectionOptions(options = {}) {
|
|
8027
|
+
embeddingTaskSelectionOptions = { ...options };
|
|
8028
|
+
}
|
|
8029
|
+
function getEmbeddingTaskSelectionOptions() {
|
|
8030
|
+
return { ...embeddingTaskSelectionOptions };
|
|
8031
|
+
}
|
|
8032
|
+
function selectEmbeddingTask(content, chunks, options = {}) {
|
|
8033
|
+
const largeDocumentChars = options.largeDocumentChars ?? DEFAULT_LARGE_DOCUMENT_CHARS;
|
|
8034
|
+
const largeDocumentChunks = options.largeDocumentChunks ?? DEFAULT_LARGE_DOCUMENT_CHUNKS;
|
|
8035
|
+
return content.length >= largeDocumentChars || chunks.length >= largeDocumentChunks ? "embedding.large-document" : "embedding.document";
|
|
8036
|
+
}
|
|
6246
8037
|
function getEmbedFunction() {
|
|
6247
8038
|
return embedFn;
|
|
6248
8039
|
}
|
|
@@ -6268,7 +8059,8 @@ async function embeddingGenerationHandler(job, db) {
|
|
|
6268
8059
|
if (!noteText) return { skipped: true, reason: "note missing, deleted, or has no content" };
|
|
6269
8060
|
const content = noteText.combined;
|
|
6270
8061
|
const chunks = chunkText(content);
|
|
6271
|
-
const
|
|
8062
|
+
const task = selectEmbeddingTask(content, chunks, embeddingTaskSelectionOptions);
|
|
8063
|
+
const embeddings = await fn(chunks, { task });
|
|
6272
8064
|
const vector = averageEmbeddings(embeddings);
|
|
6273
8065
|
const embeddingSets = new EmbeddingSetsRepository(db);
|
|
6274
8066
|
const set = await embeddingSets.ensureDefault();
|
|
@@ -6277,7 +8069,7 @@ async function embeddingGenerationHandler(job, db) {
|
|
|
6277
8069
|
embedding_set_id: set.id,
|
|
6278
8070
|
vector
|
|
6279
8071
|
});
|
|
6280
|
-
return { chunks: chunks.length, embeddings: embeddings.length, setId: set.id };
|
|
8072
|
+
return { chunks: chunks.length, embeddings: embeddings.length, setId: set.id, task };
|
|
6281
8073
|
}
|
|
6282
8074
|
|
|
6283
8075
|
// src/capabilities/auto-tag.ts
|
|
@@ -6436,6 +8228,226 @@ function unregisterLlmCapability() {
|
|
|
6436
8228
|
setLlmFunction(null);
|
|
6437
8229
|
}
|
|
6438
8230
|
|
|
8231
|
+
// src/capabilities/fallback-router.ts
|
|
8232
|
+
var DEFAULT_COOLDOWNS = {
|
|
8233
|
+
rateLimit: 3e4,
|
|
8234
|
+
serverError: 6e4,
|
|
8235
|
+
connectionFailure: 3e5,
|
|
8236
|
+
contentPolicy: 0
|
|
8237
|
+
};
|
|
8238
|
+
function classifyError(error) {
|
|
8239
|
+
const msg = error instanceof Error ? error.message : String(error);
|
|
8240
|
+
const lower = msg.toLowerCase();
|
|
8241
|
+
if (lower.includes("429") || lower.includes("rate limit")) return "rate_limit";
|
|
8242
|
+
if (lower.includes("500") || lower.includes("502") || lower.includes("503") || lower.includes("504")) return "server_error";
|
|
8243
|
+
if (lower.includes("content") && (lower.includes("policy") || lower.includes("filter"))) return "content_policy";
|
|
8244
|
+
if (lower.includes("context") && (lower.includes("window") || lower.includes("length") || lower.includes("too long"))) return "context_window";
|
|
8245
|
+
if (lower.includes("fetch") || lower.includes("network") || lower.includes("connection") || lower.includes("econnrefused") || lower.includes("timeout")) return "connection_failure";
|
|
8246
|
+
return "unknown";
|
|
8247
|
+
}
|
|
8248
|
+
var FallbackRouter = class {
|
|
8249
|
+
id = "fallback-router";
|
|
8250
|
+
name = "Fallback Router";
|
|
8251
|
+
tier = "remote";
|
|
8252
|
+
providers;
|
|
8253
|
+
cooldowns;
|
|
8254
|
+
cooldownMap = /* @__PURE__ */ new Map();
|
|
8255
|
+
events;
|
|
8256
|
+
get capabilities() {
|
|
8257
|
+
const available = this.getAvailableProviders();
|
|
8258
|
+
return {
|
|
8259
|
+
embeddings: available.some((p) => p.capabilities.embeddings),
|
|
8260
|
+
chat: available.some((p) => p.capabilities.chat),
|
|
8261
|
+
streaming: available.some((p) => p.capabilities.streaming),
|
|
8262
|
+
vision: available.some((p) => p.capabilities.vision),
|
|
8263
|
+
toolCalling: available.some((p) => p.capabilities.toolCalling),
|
|
8264
|
+
structuredOutput: available.some((p) => p.capabilities.structuredOutput),
|
|
8265
|
+
maxContextTokens: Math.max(
|
|
8266
|
+
...available.map((p) => p.capabilities.maxContextTokens ?? 0),
|
|
8267
|
+
0
|
|
8268
|
+
)
|
|
8269
|
+
};
|
|
8270
|
+
}
|
|
8271
|
+
constructor(config) {
|
|
8272
|
+
this.providers = [...config.providers];
|
|
8273
|
+
this.cooldowns = { ...DEFAULT_COOLDOWNS, ...config.cooldowns };
|
|
8274
|
+
this.events = config.events;
|
|
8275
|
+
}
|
|
8276
|
+
// -------------------------------------------------------------------------
|
|
8277
|
+
// Provider management
|
|
8278
|
+
// -------------------------------------------------------------------------
|
|
8279
|
+
/** Get providers not currently in cooldown */
|
|
8280
|
+
getAvailableProviders() {
|
|
8281
|
+
const now2 = Date.now();
|
|
8282
|
+
return this.providers.filter((p) => {
|
|
8283
|
+
const cd = this.cooldownMap.get(p.id);
|
|
8284
|
+
if (!cd) return true;
|
|
8285
|
+
if (now2 >= cd.expiresAt) {
|
|
8286
|
+
this.cooldownMap.delete(p.id);
|
|
8287
|
+
return true;
|
|
8288
|
+
}
|
|
8289
|
+
return false;
|
|
8290
|
+
});
|
|
8291
|
+
}
|
|
8292
|
+
/** Get providers in cooldown with their expiry info */
|
|
8293
|
+
getCoolingDown() {
|
|
8294
|
+
const now2 = Date.now();
|
|
8295
|
+
const result = [];
|
|
8296
|
+
for (const [id, entry] of this.cooldownMap) {
|
|
8297
|
+
if (now2 < entry.expiresAt) {
|
|
8298
|
+
result.push({ providerId: id, category: entry.category, expiresAt: entry.expiresAt });
|
|
8299
|
+
}
|
|
8300
|
+
}
|
|
8301
|
+
return result;
|
|
8302
|
+
}
|
|
8303
|
+
/** Manually clear cooldown for a provider */
|
|
8304
|
+
clearCooldown(providerId) {
|
|
8305
|
+
this.cooldownMap.delete(providerId);
|
|
8306
|
+
}
|
|
8307
|
+
/** Clear all cooldowns */
|
|
8308
|
+
clearAllCooldowns() {
|
|
8309
|
+
this.cooldownMap.clear();
|
|
8310
|
+
}
|
|
8311
|
+
/** Add a provider to the chain (appended at lowest priority) */
|
|
8312
|
+
addProvider(provider) {
|
|
8313
|
+
this.providers.push(provider);
|
|
8314
|
+
}
|
|
8315
|
+
/** Remove a provider from the chain */
|
|
8316
|
+
removeProvider(id) {
|
|
8317
|
+
this.providers = this.providers.filter((p) => p.id !== id);
|
|
8318
|
+
this.cooldownMap.delete(id);
|
|
8319
|
+
}
|
|
8320
|
+
/** Reorder providers (new priority order) */
|
|
8321
|
+
setOrder(ids) {
|
|
8322
|
+
const byId = new Map(this.providers.map((p) => [p.id, p]));
|
|
8323
|
+
const reordered = [];
|
|
8324
|
+
for (const id of ids) {
|
|
8325
|
+
const p = byId.get(id);
|
|
8326
|
+
if (p) reordered.push(p);
|
|
8327
|
+
}
|
|
8328
|
+
for (const p of this.providers) {
|
|
8329
|
+
if (!ids.includes(p.id)) reordered.push(p);
|
|
8330
|
+
}
|
|
8331
|
+
this.providers = reordered;
|
|
8332
|
+
}
|
|
8333
|
+
// -------------------------------------------------------------------------
|
|
8334
|
+
// InferenceProvider interface — with fallback
|
|
8335
|
+
// -------------------------------------------------------------------------
|
|
8336
|
+
async embed(request) {
|
|
8337
|
+
return this.withFallback(
|
|
8338
|
+
(p) => p.capabilities.embeddings && !!p.embed,
|
|
8339
|
+
(p) => p.embed(request)
|
|
8340
|
+
);
|
|
8341
|
+
}
|
|
8342
|
+
async complete(request) {
|
|
8343
|
+
return this.withFallback(
|
|
8344
|
+
(p) => p.capabilities.chat && !!p.complete,
|
|
8345
|
+
(p) => p.complete(request)
|
|
8346
|
+
);
|
|
8347
|
+
}
|
|
8348
|
+
async *stream(request) {
|
|
8349
|
+
const candidates = this.getAvailableProviders().filter((p) => p.capabilities.streaming && p.stream);
|
|
8350
|
+
if (candidates.length === 0) {
|
|
8351
|
+
throw new Error("No available providers with streaming capability");
|
|
8352
|
+
}
|
|
8353
|
+
yield* candidates[0].stream(request);
|
|
8354
|
+
}
|
|
8355
|
+
async listModels() {
|
|
8356
|
+
const available = this.getAvailableProviders();
|
|
8357
|
+
const results = await Promise.allSettled(
|
|
8358
|
+
available.map((p) => p.listModels())
|
|
8359
|
+
);
|
|
8360
|
+
const models = [];
|
|
8361
|
+
for (const r of results) {
|
|
8362
|
+
if (r.status === "fulfilled") models.push(...r.value);
|
|
8363
|
+
}
|
|
8364
|
+
return models;
|
|
8365
|
+
}
|
|
8366
|
+
async probe() {
|
|
8367
|
+
const available = this.getAvailableProviders();
|
|
8368
|
+
if (available.length === 0) {
|
|
8369
|
+
return { status: "down", latencyMs: 0, message: "All providers in cooldown" };
|
|
8370
|
+
}
|
|
8371
|
+
const start = Date.now();
|
|
8372
|
+
const results = await Promise.allSettled(
|
|
8373
|
+
available.map((p) => p.probe())
|
|
8374
|
+
);
|
|
8375
|
+
const okCount = results.filter(
|
|
8376
|
+
(r) => r.status === "fulfilled" && r.value.status === "ok"
|
|
8377
|
+
).length;
|
|
8378
|
+
return {
|
|
8379
|
+
status: okCount === available.length ? "ok" : okCount > 0 ? "degraded" : "down",
|
|
8380
|
+
latencyMs: Date.now() - start,
|
|
8381
|
+
message: `${okCount}/${available.length} providers healthy`
|
|
8382
|
+
};
|
|
8383
|
+
}
|
|
8384
|
+
dispose() {
|
|
8385
|
+
for (const p of this.providers) {
|
|
8386
|
+
p.dispose();
|
|
8387
|
+
}
|
|
8388
|
+
this.providers = [];
|
|
8389
|
+
this.cooldownMap.clear();
|
|
8390
|
+
}
|
|
8391
|
+
// -------------------------------------------------------------------------
|
|
8392
|
+
// Core fallback logic
|
|
8393
|
+
// -------------------------------------------------------------------------
|
|
8394
|
+
async withFallback(filter, execute) {
|
|
8395
|
+
const candidates = this.getAvailableProviders().filter(filter);
|
|
8396
|
+
if (candidates.length === 0) {
|
|
8397
|
+
throw new Error("No available providers for this request");
|
|
8398
|
+
}
|
|
8399
|
+
let lastError;
|
|
8400
|
+
for (const provider of candidates) {
|
|
8401
|
+
try {
|
|
8402
|
+
return await execute(provider);
|
|
8403
|
+
} catch (err) {
|
|
8404
|
+
lastError = err instanceof Error ? err : new Error(String(err));
|
|
8405
|
+
const category = classifyError(err);
|
|
8406
|
+
this.applyCooldown(provider.id, category);
|
|
8407
|
+
const nextCandidate = candidates[candidates.indexOf(provider) + 1];
|
|
8408
|
+
if (nextCandidate) {
|
|
8409
|
+
this.events?.emit("provider.fallback", {
|
|
8410
|
+
fromProvider: provider.id,
|
|
8411
|
+
toProvider: nextCandidate.id,
|
|
8412
|
+
errorCategory: category,
|
|
8413
|
+
error: lastError.message
|
|
8414
|
+
});
|
|
8415
|
+
}
|
|
8416
|
+
}
|
|
8417
|
+
}
|
|
8418
|
+
throw lastError ?? new Error("All providers failed");
|
|
8419
|
+
}
|
|
8420
|
+
applyCooldown(providerId, category) {
|
|
8421
|
+
let cooldownMs;
|
|
8422
|
+
switch (category) {
|
|
8423
|
+
case "rate_limit":
|
|
8424
|
+
cooldownMs = this.cooldowns.rateLimit;
|
|
8425
|
+
break;
|
|
8426
|
+
case "server_error":
|
|
8427
|
+
cooldownMs = this.cooldowns.serverError;
|
|
8428
|
+
break;
|
|
8429
|
+
case "connection_failure":
|
|
8430
|
+
cooldownMs = this.cooldowns.connectionFailure;
|
|
8431
|
+
break;
|
|
8432
|
+
case "content_policy":
|
|
8433
|
+
cooldownMs = this.cooldowns.contentPolicy;
|
|
8434
|
+
break;
|
|
8435
|
+
default:
|
|
8436
|
+
cooldownMs = this.cooldowns.serverError;
|
|
8437
|
+
}
|
|
8438
|
+
if (cooldownMs > 0) {
|
|
8439
|
+
const expiresAt = Date.now() + cooldownMs;
|
|
8440
|
+
this.cooldownMap.set(providerId, { expiresAt, category });
|
|
8441
|
+
this.events?.emit("provider.cooldown", {
|
|
8442
|
+
providerId,
|
|
8443
|
+
errorCategory: category,
|
|
8444
|
+
cooldownMs,
|
|
8445
|
+
expiresAt
|
|
8446
|
+
});
|
|
8447
|
+
}
|
|
8448
|
+
}
|
|
8449
|
+
};
|
|
8450
|
+
|
|
6439
8451
|
// src/capabilities/provider-registry.ts
|
|
6440
8452
|
var ProviderRegistry = class {
|
|
6441
8453
|
constructor(events) {
|
|
@@ -6443,6 +8455,7 @@ var ProviderRegistry = class {
|
|
|
6443
8455
|
}
|
|
6444
8456
|
providers = /* @__PURE__ */ new Map();
|
|
6445
8457
|
activeId = null;
|
|
8458
|
+
routes = /* @__PURE__ */ new Map();
|
|
6446
8459
|
/** Register a provider. First provider with embedding capability becomes active. */
|
|
6447
8460
|
add(provider) {
|
|
6448
8461
|
if (this.providers.has(provider.id)) {
|
|
@@ -6480,6 +8493,36 @@ var ProviderRegistry = class {
|
|
|
6480
8493
|
this.syncLegacyFunctions();
|
|
6481
8494
|
this.events?.emit("provider.active", { id, name: provider.name });
|
|
6482
8495
|
}
|
|
8496
|
+
setRoute(task, policy) {
|
|
8497
|
+
const cloned = cloneProviderRoutePolicy(policy);
|
|
8498
|
+
this.routes.set(task, cloned);
|
|
8499
|
+
this.events?.emit("provider.route.configured", {
|
|
8500
|
+
task,
|
|
8501
|
+
providerIds: cloned.providerIds ?? [],
|
|
8502
|
+
model: cloned.model,
|
|
8503
|
+
fallback: cloned.fallback,
|
|
8504
|
+
hasRequirements: Boolean(cloned.requirements)
|
|
8505
|
+
});
|
|
8506
|
+
}
|
|
8507
|
+
getRoute(task) {
|
|
8508
|
+
const route = this.routes.get(task);
|
|
8509
|
+
return route ? cloneProviderRoutePolicy(route) : void 0;
|
|
8510
|
+
}
|
|
8511
|
+
clearRoute(task) {
|
|
8512
|
+
this.routes.delete(task);
|
|
8513
|
+
this.events?.emit("provider.route.cleared", { task });
|
|
8514
|
+
}
|
|
8515
|
+
clearRoutes() {
|
|
8516
|
+
const tasks = Array.from(this.routes.keys());
|
|
8517
|
+
this.routes.clear();
|
|
8518
|
+
if (tasks.length === 0) {
|
|
8519
|
+
this.events?.emit("provider.route.cleared", {});
|
|
8520
|
+
return;
|
|
8521
|
+
}
|
|
8522
|
+
for (const task of tasks) {
|
|
8523
|
+
this.events?.emit("provider.route.cleared", { task });
|
|
8524
|
+
}
|
|
8525
|
+
}
|
|
6483
8526
|
/** Get the currently active provider */
|
|
6484
8527
|
getActive() {
|
|
6485
8528
|
if (!this.activeId) return null;
|
|
@@ -6511,27 +8554,51 @@ var ProviderRegistry = class {
|
|
|
6511
8554
|
}
|
|
6512
8555
|
/** Convenience: embed using active provider */
|
|
6513
8556
|
async embed(request) {
|
|
6514
|
-
const
|
|
6515
|
-
|
|
6516
|
-
throw new Error("No
|
|
6517
|
-
|
|
6518
|
-
|
|
8557
|
+
const route = request.task ? this.routes.get(request.task) : void 0;
|
|
8558
|
+
return this.withRouteFallback(request.task, "embeddings", request.model, async (selection) => {
|
|
8559
|
+
if (!selection.provider.embed) throw new Error("No routed provider with embedding capability");
|
|
8560
|
+
return selection.provider.embed(withResolvedModel(request, selection.model));
|
|
8561
|
+
}, route?.fallback !== false);
|
|
6519
8562
|
}
|
|
6520
8563
|
/** Convenience: complete using active provider */
|
|
6521
8564
|
async complete(request) {
|
|
6522
|
-
const
|
|
6523
|
-
|
|
6524
|
-
throw new Error("No
|
|
6525
|
-
|
|
6526
|
-
|
|
8565
|
+
const route = request.task ? this.routes.get(request.task) : void 0;
|
|
8566
|
+
return this.withRouteFallback(request.task, "chat", request.model, async (selection) => {
|
|
8567
|
+
if (!selection.provider.complete) throw new Error("No routed provider with chat capability");
|
|
8568
|
+
return selection.provider.complete(withResolvedModel(request, selection.model));
|
|
8569
|
+
}, route?.fallback !== false);
|
|
6527
8570
|
}
|
|
6528
8571
|
/** Convenience: stream using active provider */
|
|
6529
8572
|
stream(request) {
|
|
6530
|
-
const provider = this.
|
|
6531
|
-
if (!provider
|
|
6532
|
-
|
|
6533
|
-
|
|
6534
|
-
|
|
8573
|
+
const { provider, model } = this.resolveProvider(request.task, "streaming", request.model, true);
|
|
8574
|
+
if (!provider.stream) throw new Error("No routed provider with streaming capability");
|
|
8575
|
+
return provider.stream(withResolvedModel(request, model));
|
|
8576
|
+
}
|
|
8577
|
+
previewRoute(task, capability, requestModel) {
|
|
8578
|
+
const { provider, model, routeMatched } = this.resolveProvider(task, capability, requestModel, false);
|
|
8579
|
+
return {
|
|
8580
|
+
provider,
|
|
8581
|
+
providerId: provider.id,
|
|
8582
|
+
providerName: provider.name,
|
|
8583
|
+
tier: provider.tier,
|
|
8584
|
+
capability,
|
|
8585
|
+
task,
|
|
8586
|
+
model,
|
|
8587
|
+
routeMatched
|
|
8588
|
+
};
|
|
8589
|
+
}
|
|
8590
|
+
async probeRoute(task, capability, requestModel) {
|
|
8591
|
+
const selection = this.previewRoute(task, capability, requestModel);
|
|
8592
|
+
const probe = await selection.provider.probe();
|
|
8593
|
+
return { ...selection, probe };
|
|
8594
|
+
}
|
|
8595
|
+
validateRoute(task, capability = inferInferenceTaskCapability(task), policy = this.routes.get(task)) {
|
|
8596
|
+
return validateProviderRoute(task, capability, policy, this.list());
|
|
8597
|
+
}
|
|
8598
|
+
validateRoutes() {
|
|
8599
|
+
return Array.from(this.routes.entries()).map(
|
|
8600
|
+
([task, policy]) => this.validateRoute(task, inferInferenceTaskCapability(task), policy)
|
|
8601
|
+
);
|
|
6535
8602
|
}
|
|
6536
8603
|
/** Dispose all providers */
|
|
6537
8604
|
dispose() {
|
|
@@ -6553,29 +8620,357 @@ var ProviderRegistry = class {
|
|
|
6553
8620
|
syncLegacyFunctions() {
|
|
6554
8621
|
const active = this.getActive();
|
|
6555
8622
|
if (active?.embed && active.capabilities.embeddings) {
|
|
6556
|
-
const embedBridge = (texts) =>
|
|
8623
|
+
const embedBridge = (texts, options) => {
|
|
8624
|
+
const request = {
|
|
8625
|
+
texts,
|
|
8626
|
+
task: options?.task ?? "embedding.document"
|
|
8627
|
+
};
|
|
8628
|
+
if (options?.model) request.model = options.model;
|
|
8629
|
+
return this.embed(request).then((r) => r.vectors);
|
|
8630
|
+
};
|
|
6557
8631
|
setEmbedFunction(embedBridge);
|
|
6558
8632
|
} else {
|
|
6559
8633
|
setEmbedFunction(null);
|
|
6560
8634
|
}
|
|
6561
8635
|
if (active?.complete && active.capabilities.chat) {
|
|
6562
|
-
const llmBridge = (prompt, options) =>
|
|
6563
|
-
|
|
6564
|
-
|
|
6565
|
-
|
|
6566
|
-
|
|
8636
|
+
const llmBridge = (prompt, options) => {
|
|
8637
|
+
const request = {
|
|
8638
|
+
prompt,
|
|
8639
|
+
task: options?.task ?? "chat.general"
|
|
8640
|
+
};
|
|
8641
|
+
if (options?.model) request.model = options.model;
|
|
8642
|
+
if (options?.maxTokens !== void 0) request.maxTokens = options.maxTokens;
|
|
8643
|
+
if (options?.temperature !== void 0) request.temperature = options.temperature;
|
|
8644
|
+
return this.complete(request).then((r) => r.text);
|
|
8645
|
+
};
|
|
6567
8646
|
setLlmFunction(llmBridge);
|
|
6568
8647
|
} else {
|
|
6569
8648
|
setLlmFunction(null);
|
|
6570
8649
|
}
|
|
6571
8650
|
}
|
|
8651
|
+
resolveProvider(task, capability, requestModel, emitSelection = false) {
|
|
8652
|
+
const candidates = this.resolveRouteSelections(task, capability, requestModel);
|
|
8653
|
+
const selection = candidates[0];
|
|
8654
|
+
if (!selection) {
|
|
8655
|
+
const label = task ? ` for task '${task}'` : "";
|
|
8656
|
+
throw new Error(`No provider satisfies ${String(capability)} route${label}`);
|
|
8657
|
+
}
|
|
8658
|
+
if (emitSelection) {
|
|
8659
|
+
this.emitRouteSelected(selection);
|
|
8660
|
+
}
|
|
8661
|
+
return selection;
|
|
8662
|
+
}
|
|
8663
|
+
resolveRouteSelections(task, capability, requestModel) {
|
|
8664
|
+
const route = task ? this.routes.get(task) : void 0;
|
|
8665
|
+
const model = requestModel ?? route?.model;
|
|
8666
|
+
const routeMatched = Boolean(route);
|
|
8667
|
+
return this.resolveCandidates(route, capability).filter((provider) => Boolean(provider.capabilities[capability])).filter((provider) => capability !== "embeddings" || Boolean(provider.embed)).filter((provider) => capability !== "chat" || Boolean(provider.complete)).filter((provider) => capability !== "streaming" || Boolean(provider.stream)).map((provider) => ({
|
|
8668
|
+
provider,
|
|
8669
|
+
providerId: provider.id,
|
|
8670
|
+
providerName: provider.name,
|
|
8671
|
+
tier: provider.tier,
|
|
8672
|
+
capability,
|
|
8673
|
+
task,
|
|
8674
|
+
model,
|
|
8675
|
+
routeMatched
|
|
8676
|
+
}));
|
|
8677
|
+
}
|
|
8678
|
+
emitRouteSelected(selection) {
|
|
8679
|
+
this.events?.emit("provider.route.selected", {
|
|
8680
|
+
providerId: selection.providerId,
|
|
8681
|
+
providerName: selection.providerName,
|
|
8682
|
+
tier: selection.tier,
|
|
8683
|
+
capability: String(selection.capability),
|
|
8684
|
+
task: selection.task,
|
|
8685
|
+
model: selection.model,
|
|
8686
|
+
routeMatched: selection.routeMatched
|
|
8687
|
+
});
|
|
8688
|
+
}
|
|
8689
|
+
async withRouteFallback(task, capability, requestModel, execute, allowFallback) {
|
|
8690
|
+
const selections = this.resolveRouteSelections(task, capability, requestModel);
|
|
8691
|
+
const candidates = allowFallback ? selections : selections.slice(0, 1);
|
|
8692
|
+
if (!candidates.length) {
|
|
8693
|
+
const label = task ? ` for task '${task}'` : "";
|
|
8694
|
+
const route = task ? this.routes.get(task) : void 0;
|
|
8695
|
+
this.emitRouteFailed(
|
|
8696
|
+
void 0,
|
|
8697
|
+
capability,
|
|
8698
|
+
task,
|
|
8699
|
+
requestModel ?? route?.model,
|
|
8700
|
+
Boolean(route),
|
|
8701
|
+
0,
|
|
8702
|
+
0,
|
|
8703
|
+
0,
|
|
8704
|
+
"no_provider",
|
|
8705
|
+
`No provider satisfies ${String(capability)} route${label}`
|
|
8706
|
+
);
|
|
8707
|
+
throw new Error(`No provider satisfies ${String(capability)} route${label}`);
|
|
8708
|
+
}
|
|
8709
|
+
let lastError;
|
|
8710
|
+
let lastSelection;
|
|
8711
|
+
let lastErrorCategory = "unknown";
|
|
8712
|
+
const start = Date.now();
|
|
8713
|
+
for (let index = 0; index < candidates.length; index++) {
|
|
8714
|
+
const selection = candidates[index];
|
|
8715
|
+
try {
|
|
8716
|
+
this.emitRouteSelected(selection);
|
|
8717
|
+
const result = await execute(selection);
|
|
8718
|
+
this.emitRouteCompleted(selection, index + 1, index, Date.now() - start);
|
|
8719
|
+
return result;
|
|
8720
|
+
} catch (err) {
|
|
8721
|
+
lastSelection = selection;
|
|
8722
|
+
lastError = err instanceof Error ? err : new Error(String(err));
|
|
8723
|
+
lastErrorCategory = classifyError(err);
|
|
8724
|
+
const next = candidates[index + 1];
|
|
8725
|
+
if (next) {
|
|
8726
|
+
this.events?.emit("provider.fallback", {
|
|
8727
|
+
fromProvider: selection.providerId,
|
|
8728
|
+
toProvider: next.providerId,
|
|
8729
|
+
errorCategory: lastErrorCategory,
|
|
8730
|
+
error: lastError.message
|
|
8731
|
+
});
|
|
8732
|
+
}
|
|
8733
|
+
}
|
|
8734
|
+
}
|
|
8735
|
+
this.emitRouteFailed(
|
|
8736
|
+
lastSelection,
|
|
8737
|
+
capability,
|
|
8738
|
+
task,
|
|
8739
|
+
requestModel,
|
|
8740
|
+
candidates[0]?.routeMatched ?? Boolean(task && this.routes.get(task)),
|
|
8741
|
+
candidates.length,
|
|
8742
|
+
Math.max(candidates.length - 1, 0),
|
|
8743
|
+
Date.now() - start,
|
|
8744
|
+
lastErrorCategory,
|
|
8745
|
+
lastError?.message ?? "All routed providers failed"
|
|
8746
|
+
);
|
|
8747
|
+
throw lastError ?? new Error("All routed providers failed");
|
|
8748
|
+
}
|
|
8749
|
+
emitRouteCompleted(selection, attempt, fallbackCount, latencyMs) {
|
|
8750
|
+
this.events?.emit("provider.route.completed", {
|
|
8751
|
+
providerId: selection.providerId,
|
|
8752
|
+
providerName: selection.providerName,
|
|
8753
|
+
tier: selection.tier,
|
|
8754
|
+
capability: String(selection.capability),
|
|
8755
|
+
task: selection.task,
|
|
8756
|
+
model: selection.model,
|
|
8757
|
+
routeMatched: selection.routeMatched,
|
|
8758
|
+
attempt,
|
|
8759
|
+
fallbackCount,
|
|
8760
|
+
latencyMs
|
|
8761
|
+
});
|
|
8762
|
+
}
|
|
8763
|
+
emitRouteFailed(selection, capability, task, model, routeMatched, attempt, fallbackCount, latencyMs, errorCategory, error) {
|
|
8764
|
+
this.events?.emit("provider.route.failed", {
|
|
8765
|
+
providerId: selection?.providerId,
|
|
8766
|
+
providerName: selection?.providerName,
|
|
8767
|
+
tier: selection?.tier,
|
|
8768
|
+
capability: String(capability),
|
|
8769
|
+
task,
|
|
8770
|
+
model: selection?.model ?? model,
|
|
8771
|
+
routeMatched,
|
|
8772
|
+
attempt,
|
|
8773
|
+
fallbackCount,
|
|
8774
|
+
latencyMs,
|
|
8775
|
+
errorCategory,
|
|
8776
|
+
error
|
|
8777
|
+
});
|
|
8778
|
+
}
|
|
8779
|
+
resolveCandidates(route, capability) {
|
|
8780
|
+
const all = this.list();
|
|
8781
|
+
if (!route) {
|
|
8782
|
+
const active = this.getActive();
|
|
8783
|
+
return active ? [active, ...all.filter((provider) => provider.id !== active.id)] : all;
|
|
8784
|
+
}
|
|
8785
|
+
const byId = new Map(all.map((provider) => [provider.id, provider]));
|
|
8786
|
+
const selected = [];
|
|
8787
|
+
for (const id of route.providerIds ?? []) {
|
|
8788
|
+
const provider = byId.get(id);
|
|
8789
|
+
if (provider) selected.push(provider);
|
|
8790
|
+
}
|
|
8791
|
+
const hasExplicitProviderIds = Boolean(route.providerIds?.length);
|
|
8792
|
+
const remaining = hasExplicitProviderIds || route.fallback === false ? selected : all;
|
|
8793
|
+
return remaining.filter((provider) => {
|
|
8794
|
+
if (route.tiers?.length && !route.tiers.includes(provider.tier)) return false;
|
|
8795
|
+
if (!providerSatisfiesRouteRequirements(provider, route.requirements, capability)) return false;
|
|
8796
|
+
return true;
|
|
8797
|
+
});
|
|
8798
|
+
}
|
|
6572
8799
|
};
|
|
8800
|
+
function withResolvedModel(request, model) {
|
|
8801
|
+
return model === void 0 ? request : { ...request, model };
|
|
8802
|
+
}
|
|
8803
|
+
function inferInferenceTaskCapability(task) {
|
|
8804
|
+
if (task.startsWith("embedding.")) return "embeddings";
|
|
8805
|
+
if (task.startsWith("vision.")) return "vision";
|
|
8806
|
+
return "chat";
|
|
8807
|
+
}
|
|
8808
|
+
function validateProviderRoute(task, capability, policy, providers) {
|
|
8809
|
+
const byId = new Map(providers.map((provider) => [provider.id, provider]));
|
|
8810
|
+
const providerIds = policy?.providerIds ?? [];
|
|
8811
|
+
const issues = [];
|
|
8812
|
+
const eligibleProviderIds = [];
|
|
8813
|
+
if (policy && providerIds.length === 0 && policy.fallback === false) {
|
|
8814
|
+
issues.push({
|
|
8815
|
+
severity: "error",
|
|
8816
|
+
code: "empty-explicit-chain",
|
|
8817
|
+
message: `Route '${task}' disables fallback but does not name a provider.`
|
|
8818
|
+
});
|
|
8819
|
+
}
|
|
8820
|
+
const candidates = providerIds.length ? providerIds.map((id) => byId.get(id)).filter((provider) => Boolean(provider)) : providers;
|
|
8821
|
+
for (const providerId of providerIds) {
|
|
8822
|
+
if (!byId.has(providerId)) {
|
|
8823
|
+
issues.push({
|
|
8824
|
+
severity: "error",
|
|
8825
|
+
code: "missing-provider",
|
|
8826
|
+
providerId,
|
|
8827
|
+
message: `Route '${task}' references missing provider '${providerId}'.`
|
|
8828
|
+
});
|
|
8829
|
+
}
|
|
8830
|
+
}
|
|
8831
|
+
for (const provider of candidates) {
|
|
8832
|
+
if (policy?.tiers?.length && !policy.tiers.includes(provider.tier)) continue;
|
|
8833
|
+
if (!provider.capabilities[capability]) {
|
|
8834
|
+
issues.push({
|
|
8835
|
+
severity: "error",
|
|
8836
|
+
code: "unsupported-capability",
|
|
8837
|
+
providerId: provider.id,
|
|
8838
|
+
message: `Provider '${provider.id}' does not support ${String(capability)} for route '${task}'.`
|
|
8839
|
+
});
|
|
8840
|
+
continue;
|
|
8841
|
+
}
|
|
8842
|
+
if (capability === "embeddings" && !provider.embed) {
|
|
8843
|
+
issues.push({
|
|
8844
|
+
severity: "error",
|
|
8845
|
+
code: "missing-handler",
|
|
8846
|
+
providerId: provider.id,
|
|
8847
|
+
message: `Provider '${provider.id}' has no embedding handler for route '${task}'.`
|
|
8848
|
+
});
|
|
8849
|
+
continue;
|
|
8850
|
+
}
|
|
8851
|
+
if (capability === "chat" && !provider.complete) {
|
|
8852
|
+
issues.push({
|
|
8853
|
+
severity: "error",
|
|
8854
|
+
code: "missing-handler",
|
|
8855
|
+
providerId: provider.id,
|
|
8856
|
+
message: `Provider '${provider.id}' has no chat handler for route '${task}'.`
|
|
8857
|
+
});
|
|
8858
|
+
continue;
|
|
8859
|
+
}
|
|
8860
|
+
if (capability === "streaming" && !provider.stream) {
|
|
8861
|
+
issues.push({
|
|
8862
|
+
severity: "error",
|
|
8863
|
+
code: "missing-handler",
|
|
8864
|
+
providerId: provider.id,
|
|
8865
|
+
message: `Provider '${provider.id}' has no streaming handler for route '${task}'.`
|
|
8866
|
+
});
|
|
8867
|
+
continue;
|
|
8868
|
+
}
|
|
8869
|
+
const requirementIssue = getProviderRouteRequirementIssue(provider, policy?.requirements, capability);
|
|
8870
|
+
if (requirementIssue) {
|
|
8871
|
+
issues.push({
|
|
8872
|
+
severity: "error",
|
|
8873
|
+
code: "profile-requirement",
|
|
8874
|
+
providerId: provider.id,
|
|
8875
|
+
message: requirementIssue
|
|
8876
|
+
});
|
|
8877
|
+
continue;
|
|
8878
|
+
}
|
|
8879
|
+
eligibleProviderIds.push(provider.id);
|
|
8880
|
+
}
|
|
8881
|
+
if (!eligibleProviderIds.length) {
|
|
8882
|
+
issues.push({
|
|
8883
|
+
severity: "error",
|
|
8884
|
+
code: "no-eligible-provider",
|
|
8885
|
+
message: `Route '${task}' has no eligible ${String(capability)} provider.`
|
|
8886
|
+
});
|
|
8887
|
+
}
|
|
8888
|
+
return {
|
|
8889
|
+
task,
|
|
8890
|
+
capability,
|
|
8891
|
+
policy: policy ? cloneProviderRoutePolicy(policy) : void 0,
|
|
8892
|
+
providerIds,
|
|
8893
|
+
eligibleProviderIds,
|
|
8894
|
+
issues,
|
|
8895
|
+
ok: !issues.some((issue) => issue.severity === "error")
|
|
8896
|
+
};
|
|
8897
|
+
}
|
|
8898
|
+
function cloneProviderRoutePolicy(policy) {
|
|
8899
|
+
return {
|
|
8900
|
+
...policy,
|
|
8901
|
+
providerIds: policy.providerIds ? [...policy.providerIds] : void 0,
|
|
8902
|
+
tiers: policy.tiers ? [...policy.tiers] : void 0,
|
|
8903
|
+
requirements: policy.requirements ? {
|
|
8904
|
+
...policy.requirements,
|
|
8905
|
+
privacyTiers: policy.requirements.privacyTiers ? [...policy.requirements.privacyTiers] : void 0
|
|
8906
|
+
} : void 0
|
|
8907
|
+
};
|
|
8908
|
+
}
|
|
8909
|
+
var COST_ORDER = ["free", "low", "medium", "high"];
|
|
8910
|
+
function providerSatisfiesRouteRequirements(provider, requirements, capability) {
|
|
8911
|
+
return getProviderRouteRequirementIssue(provider, requirements, capability) === void 0;
|
|
8912
|
+
}
|
|
8913
|
+
function getProviderRouteRequirementIssue(provider, requirements, capability) {
|
|
8914
|
+
if (!requirements) return void 0;
|
|
8915
|
+
const profile = provider.profile;
|
|
8916
|
+
if (requirements.privacyTiers?.length) {
|
|
8917
|
+
const privacy = profile?.privacyTier ?? privacyTierFromProviderTier(provider.tier);
|
|
8918
|
+
if (!requirements.privacyTiers.includes(privacy)) {
|
|
8919
|
+
return `Provider '${provider.id}' does not match required privacy tier`;
|
|
8920
|
+
}
|
|
8921
|
+
}
|
|
8922
|
+
if (requirements.maxCostTier) {
|
|
8923
|
+
const cost = profile?.costTier;
|
|
8924
|
+
if (!cost || COST_ORDER.indexOf(cost) > COST_ORDER.indexOf(requirements.maxCostTier)) {
|
|
8925
|
+
return `Provider '${provider.id}' does not match required cost tier`;
|
|
8926
|
+
}
|
|
8927
|
+
}
|
|
8928
|
+
if (requirements.minContextTokens) {
|
|
8929
|
+
const context = provider.capabilities.maxContextTokens;
|
|
8930
|
+
if (!context || context < requirements.minContextTokens) {
|
|
8931
|
+
return `Provider '${provider.id}' does not advertise enough context`;
|
|
8932
|
+
}
|
|
8933
|
+
}
|
|
8934
|
+
if (capability === "embeddings" && requirements.minEmbeddingDimensions) {
|
|
8935
|
+
const dimensions = profile?.embeddingDimensions ?? [];
|
|
8936
|
+
if (!dimensions.some((dimension) => dimension >= requirements.minEmbeddingDimensions)) {
|
|
8937
|
+
return `Provider '${provider.id}' does not advertise enough embedding dimensions`;
|
|
8938
|
+
}
|
|
8939
|
+
}
|
|
8940
|
+
if (requirements.dataClass) {
|
|
8941
|
+
const allowed = profile?.dataClasses ?? defaultDataClasses(provider.tier);
|
|
8942
|
+
if (!allowed.includes(requirements.dataClass)) {
|
|
8943
|
+
return `Provider '${provider.id}' does not allow required data class`;
|
|
8944
|
+
}
|
|
8945
|
+
}
|
|
8946
|
+
if (requirements.maxInputChars) {
|
|
8947
|
+
const maxInputChars = profile?.maxInputChars;
|
|
8948
|
+
if (!maxInputChars || maxInputChars < requirements.maxInputChars) {
|
|
8949
|
+
return `Provider '${provider.id}' does not advertise enough input capacity`;
|
|
8950
|
+
}
|
|
8951
|
+
}
|
|
8952
|
+
return void 0;
|
|
8953
|
+
}
|
|
8954
|
+
function privacyTierFromProviderTier(tier) {
|
|
8955
|
+
if (tier === "remote") return "external";
|
|
8956
|
+
if (tier === "chrome-ai") return "host-managed";
|
|
8957
|
+
return "local";
|
|
8958
|
+
}
|
|
8959
|
+
function defaultDataClasses(tier) {
|
|
8960
|
+
if (tier === "remote") return ["public"];
|
|
8961
|
+
return ["public", "private", "sensitive"];
|
|
8962
|
+
}
|
|
6573
8963
|
function createLegacyProvider(options) {
|
|
6574
8964
|
const { embedFn: embedFn2, llmFn: llmFn2, id = "legacy", name = "Legacy Provider" } = options;
|
|
6575
8965
|
return {
|
|
6576
8966
|
id,
|
|
6577
8967
|
name,
|
|
6578
8968
|
tier: "in-browser",
|
|
8969
|
+
profile: options.profile ?? {
|
|
8970
|
+
privacyTier: "local",
|
|
8971
|
+
costTier: "free",
|
|
8972
|
+
dataClasses: ["public", "private", "sensitive"]
|
|
8973
|
+
},
|
|
6579
8974
|
capabilities: {
|
|
6580
8975
|
embeddings: !!embedFn2,
|
|
6581
8976
|
chat: !!llmFn2,
|
|
@@ -6585,14 +8980,11 @@ function createLegacyProvider(options) {
|
|
|
6585
8980
|
structuredOutput: false
|
|
6586
8981
|
},
|
|
6587
8982
|
embed: embedFn2 ? async (request) => ({
|
|
6588
|
-
vectors: await embedFn2(request.texts),
|
|
8983
|
+
vectors: await embedFn2(request.texts, embedOptionsFromRequest(request)),
|
|
6589
8984
|
model: "legacy"
|
|
6590
8985
|
}) : void 0,
|
|
6591
8986
|
complete: llmFn2 ? async (request) => ({
|
|
6592
|
-
text: await llmFn2(request.prompt,
|
|
6593
|
-
maxTokens: request.maxTokens,
|
|
6594
|
-
temperature: request.temperature
|
|
6595
|
-
}),
|
|
8987
|
+
text: await llmFn2(request.prompt, llmOptionsFromRequest(request)),
|
|
6596
8988
|
model: "legacy"
|
|
6597
8989
|
}) : void 0,
|
|
6598
8990
|
async listModels() {
|
|
@@ -6608,6 +9000,114 @@ function createLegacyProvider(options) {
|
|
|
6608
9000
|
}
|
|
6609
9001
|
};
|
|
6610
9002
|
}
|
|
9003
|
+
function embedOptionsFromRequest(request) {
|
|
9004
|
+
const options = {};
|
|
9005
|
+
if (request.task) options.task = request.task;
|
|
9006
|
+
if (request.model) options.model = request.model;
|
|
9007
|
+
return Object.keys(options).length ? options : void 0;
|
|
9008
|
+
}
|
|
9009
|
+
function llmOptionsFromRequest(request) {
|
|
9010
|
+
const options = {};
|
|
9011
|
+
if (request.maxTokens !== void 0) options.maxTokens = request.maxTokens;
|
|
9012
|
+
if (request.temperature !== void 0) options.temperature = request.temperature;
|
|
9013
|
+
if (request.task) options.task = request.task;
|
|
9014
|
+
if (request.model) options.model = request.model;
|
|
9015
|
+
return Object.keys(options).length ? options : void 0;
|
|
9016
|
+
}
|
|
9017
|
+
|
|
9018
|
+
// src/capabilities/local-discovery.ts
|
|
9019
|
+
var LOCAL_ENDPOINTS = [
|
|
9020
|
+
{ id: "ollama", name: "Ollama", baseURL: "http://localhost:11434/v1", defaultPort: 11434 },
|
|
9021
|
+
{ id: "lm-studio", name: "LM Studio", baseURL: "http://localhost:1234/v1", defaultPort: 1234 },
|
|
9022
|
+
{ id: "llama-cpp", name: "llama.cpp", baseURL: "http://localhost:8080/v1", defaultPort: 8080 },
|
|
9023
|
+
{ id: "vllm", name: "vLLM", baseURL: "http://localhost:8000/v1", defaultPort: 8e3 },
|
|
9024
|
+
{ id: "jan", name: "Jan", baseURL: "http://localhost:1337/v1", defaultPort: 1337 },
|
|
9025
|
+
{ id: "localai", name: "LocalAI", baseURL: "http://localhost:8080/v1", defaultPort: 8080 }
|
|
9026
|
+
];
|
|
9027
|
+
function classifyModel(modelId) {
|
|
9028
|
+
const lower = modelId.toLowerCase();
|
|
9029
|
+
if (lower.includes("embed") || lower.includes("e5-") || lower.includes("bge-") || lower.includes("nomic-") || lower.includes("mxbai-") || lower.includes("all-minilm") || lower.includes("gte-")) {
|
|
9030
|
+
return "embedding";
|
|
9031
|
+
}
|
|
9032
|
+
if (lower.includes("vision") || lower.includes("llava") || lower.includes("moondream") || lower.includes("minicpm-v") || lower.includes("bakllava")) {
|
|
9033
|
+
return "vision";
|
|
9034
|
+
}
|
|
9035
|
+
return "chat";
|
|
9036
|
+
}
|
|
9037
|
+
function inferLocalEmbeddingDimensions(models) {
|
|
9038
|
+
const embeddingModel = models.find((model) => model.capabilities.embeddings)?.id.toLowerCase();
|
|
9039
|
+
if (!embeddingModel) return void 0;
|
|
9040
|
+
if (embeddingModel.includes("nomic-embed")) return [768];
|
|
9041
|
+
if (embeddingModel.includes("bge-large")) return [1024];
|
|
9042
|
+
if (embeddingModel.includes("bge-base")) return [768];
|
|
9043
|
+
if (embeddingModel.includes("bge-small")) return [384];
|
|
9044
|
+
if (embeddingModel.includes("all-minilm")) return [384];
|
|
9045
|
+
if (embeddingModel.includes("mxbai-embed-large")) return [1024];
|
|
9046
|
+
return void 0;
|
|
9047
|
+
}
|
|
9048
|
+
function createLocalProviderProfile(models = []) {
|
|
9049
|
+
const embeddingDimensions = inferLocalEmbeddingDimensions(models);
|
|
9050
|
+
return {
|
|
9051
|
+
privacyTier: "local",
|
|
9052
|
+
costTier: "free",
|
|
9053
|
+
embeddingDimensions,
|
|
9054
|
+
dataClasses: ["public", "private", "sensitive"]
|
|
9055
|
+
};
|
|
9056
|
+
}
|
|
9057
|
+
async function probeEndpoint(endpoint, timeoutMs) {
|
|
9058
|
+
try {
|
|
9059
|
+
const response = await globalThis.fetch(`${endpoint.baseURL}/models`, {
|
|
9060
|
+
signal: AbortSignal.timeout(timeoutMs)
|
|
9061
|
+
});
|
|
9062
|
+
if (!response.ok) return null;
|
|
9063
|
+
const data = await response.json();
|
|
9064
|
+
let modelIds;
|
|
9065
|
+
if (data.data && Array.isArray(data.data)) {
|
|
9066
|
+
modelIds = data.data.map((m) => m.id);
|
|
9067
|
+
} else if (data.models && Array.isArray(data.models)) {
|
|
9068
|
+
modelIds = data.models.map((m) => m.name ?? m.model ?? "");
|
|
9069
|
+
} else {
|
|
9070
|
+
modelIds = [];
|
|
9071
|
+
}
|
|
9072
|
+
const models = modelIds.filter(Boolean).map((id) => {
|
|
9073
|
+
const category = classifyModel(id);
|
|
9074
|
+
return {
|
|
9075
|
+
id,
|
|
9076
|
+
name: id,
|
|
9077
|
+
capabilities: {
|
|
9078
|
+
embeddings: category === "embedding",
|
|
9079
|
+
chat: category === "chat" || category === "vision",
|
|
9080
|
+
vision: category === "vision"
|
|
9081
|
+
}
|
|
9082
|
+
};
|
|
9083
|
+
});
|
|
9084
|
+
return {
|
|
9085
|
+
id: endpoint.id,
|
|
9086
|
+
name: endpoint.name,
|
|
9087
|
+
baseURL: endpoint.baseURL,
|
|
9088
|
+
models
|
|
9089
|
+
};
|
|
9090
|
+
} catch {
|
|
9091
|
+
return null;
|
|
9092
|
+
}
|
|
9093
|
+
}
|
|
9094
|
+
async function discoverLocalProviders(options = {}) {
|
|
9095
|
+
const { extraEndpoints = [], timeoutMs = 2e3, skipPorts = [] } = options;
|
|
9096
|
+
const allEndpoints = [...LOCAL_ENDPOINTS, ...extraEndpoints];
|
|
9097
|
+
const seen = /* @__PURE__ */ new Set();
|
|
9098
|
+
const uniqueEndpoints = allEndpoints.filter((ep) => {
|
|
9099
|
+
if (seen.has(ep.baseURL)) return false;
|
|
9100
|
+
if (skipPorts.includes(ep.defaultPort)) return false;
|
|
9101
|
+
seen.add(ep.baseURL);
|
|
9102
|
+
return true;
|
|
9103
|
+
});
|
|
9104
|
+
const results = await Promise.allSettled(
|
|
9105
|
+
uniqueEndpoints.map((ep) => probeEndpoint(ep, timeoutMs))
|
|
9106
|
+
);
|
|
9107
|
+
return results.filter(
|
|
9108
|
+
(r) => r.status === "fulfilled" && r.value !== null
|
|
9109
|
+
).map((r) => r.value);
|
|
9110
|
+
}
|
|
6611
9111
|
|
|
6612
9112
|
// src/capabilities/openai-provider.ts
|
|
6613
9113
|
var OpenAICompatibleProvider = class {
|
|
@@ -6615,6 +9115,7 @@ var OpenAICompatibleProvider = class {
|
|
|
6615
9115
|
name;
|
|
6616
9116
|
tier;
|
|
6617
9117
|
capabilities;
|
|
9118
|
+
profile;
|
|
6618
9119
|
baseURL;
|
|
6619
9120
|
apiKey;
|
|
6620
9121
|
defaultModel;
|
|
@@ -6640,6 +9141,7 @@ var OpenAICompatibleProvider = class {
|
|
|
6640
9141
|
toolCalling: false,
|
|
6641
9142
|
structuredOutput: false
|
|
6642
9143
|
};
|
|
9144
|
+
this.profile = config.profile ?? (this.tier === "local-server" ? { privacyTier: "local", costTier: "free" } : { privacyTier: "external" });
|
|
6643
9145
|
}
|
|
6644
9146
|
// -------------------------------------------------------------------------
|
|
6645
9147
|
// InferenceProvider interface
|
|
@@ -6734,15 +9236,19 @@ var OpenAICompatibleProvider = class {
|
|
|
6734
9236
|
try {
|
|
6735
9237
|
const response = await this.fetch("/models", void 0, "GET");
|
|
6736
9238
|
const data = response;
|
|
6737
|
-
return data.data.map((m) =>
|
|
6738
|
-
|
|
6739
|
-
|
|
6740
|
-
|
|
6741
|
-
|
|
6742
|
-
|
|
6743
|
-
|
|
6744
|
-
|
|
6745
|
-
|
|
9239
|
+
return data.data.map((m) => {
|
|
9240
|
+
const category = classifyModel(m.id);
|
|
9241
|
+
return {
|
|
9242
|
+
id: m.id,
|
|
9243
|
+
name: m.id,
|
|
9244
|
+
capabilities: {
|
|
9245
|
+
chat: category === "chat" || category === "vision",
|
|
9246
|
+
embeddings: category === "embedding",
|
|
9247
|
+
vision: category === "vision"
|
|
9248
|
+
},
|
|
9249
|
+
owned_by: m.owned_by
|
|
9250
|
+
};
|
|
9251
|
+
});
|
|
6746
9252
|
} catch {
|
|
6747
9253
|
return [];
|
|
6748
9254
|
}
|
|
@@ -6790,10 +9296,6 @@ var OpenAICompatibleProvider = class {
|
|
|
6790
9296
|
return void 0;
|
|
6791
9297
|
}
|
|
6792
9298
|
}
|
|
6793
|
-
isEmbeddingModel(id) {
|
|
6794
|
-
const lower = id.toLowerCase();
|
|
6795
|
-
return lower.includes("embed") || lower.includes("e5-") || lower.includes("bge-") || lower.includes("nomic-") || lower.includes("mxbai-") || lower.includes("all-minilm");
|
|
6796
|
-
}
|
|
6797
9299
|
isLocalURL() {
|
|
6798
9300
|
try {
|
|
6799
9301
|
const url = new URL(this.baseURL);
|
|
@@ -6834,299 +9336,77 @@ var OpenAICompatibleProvider = class {
|
|
|
6834
9336
|
}
|
|
6835
9337
|
};
|
|
6836
9338
|
|
|
6837
|
-
// src/capabilities/
|
|
6838
|
-
|
|
6839
|
-
{
|
|
6840
|
-
|
|
6841
|
-
|
|
6842
|
-
|
|
6843
|
-
|
|
6844
|
-
|
|
6845
|
-
|
|
6846
|
-
|
|
6847
|
-
const lower = modelId.toLowerCase();
|
|
6848
|
-
if (lower.includes("embed") || lower.includes("e5-") || lower.includes("bge-") || lower.includes("nomic-") || lower.includes("mxbai-") || lower.includes("all-minilm") || lower.includes("gte-")) {
|
|
6849
|
-
return "embedding";
|
|
6850
|
-
}
|
|
6851
|
-
if (lower.includes("vision") || lower.includes("llava") || lower.includes("moondream") || lower.includes("minicpm-v") || lower.includes("bakllava")) {
|
|
6852
|
-
return "vision";
|
|
6853
|
-
}
|
|
6854
|
-
return "chat";
|
|
9339
|
+
// src/capabilities/bridge-provider.ts
|
|
9340
|
+
function capabilitiesFromBridge(info) {
|
|
9341
|
+
return {
|
|
9342
|
+
embeddings: Boolean(info.capabilities.embeddings),
|
|
9343
|
+
chat: Boolean(info.capabilities.chat),
|
|
9344
|
+
streaming: Boolean(info.capabilities.streaming),
|
|
9345
|
+
vision: false,
|
|
9346
|
+
toolCalling: false,
|
|
9347
|
+
structuredOutput: false
|
|
9348
|
+
};
|
|
6855
9349
|
}
|
|
6856
|
-
|
|
6857
|
-
|
|
6858
|
-
|
|
6859
|
-
|
|
6860
|
-
|
|
6861
|
-
|
|
6862
|
-
|
|
6863
|
-
|
|
6864
|
-
|
|
6865
|
-
modelIds = data.data.map((m) => m.id);
|
|
6866
|
-
} else if (data.models && Array.isArray(data.models)) {
|
|
6867
|
-
modelIds = data.models.map((m) => m.name ?? m.model ?? "");
|
|
6868
|
-
} else {
|
|
6869
|
-
modelIds = [];
|
|
6870
|
-
}
|
|
6871
|
-
const models = modelIds.filter(Boolean).map((id) => {
|
|
6872
|
-
const category = classifyModel(id);
|
|
6873
|
-
return {
|
|
6874
|
-
id,
|
|
6875
|
-
name: id,
|
|
6876
|
-
capabilities: {
|
|
6877
|
-
embeddings: category === "embedding",
|
|
6878
|
-
chat: category === "chat" || category === "vision",
|
|
6879
|
-
vision: category === "vision"
|
|
6880
|
-
}
|
|
6881
|
-
};
|
|
6882
|
-
});
|
|
6883
|
-
return {
|
|
6884
|
-
id: endpoint.id,
|
|
6885
|
-
name: endpoint.name,
|
|
6886
|
-
baseURL: endpoint.baseURL,
|
|
6887
|
-
models
|
|
9350
|
+
var BridgeInferenceProvider = class {
|
|
9351
|
+
constructor(bridge, info) {
|
|
9352
|
+
this.bridge = bridge;
|
|
9353
|
+
this.id = info.id;
|
|
9354
|
+
this.name = info.name;
|
|
9355
|
+
this.tier = info.tier;
|
|
9356
|
+
this.capabilities = capabilitiesFromBridge(info);
|
|
9357
|
+
this.profile = info.profile ?? {
|
|
9358
|
+
privacyTier: "host-managed"
|
|
6888
9359
|
};
|
|
6889
|
-
} catch {
|
|
6890
|
-
return null;
|
|
6891
9360
|
}
|
|
6892
|
-
|
|
6893
|
-
|
|
6894
|
-
|
|
6895
|
-
|
|
6896
|
-
|
|
6897
|
-
|
|
6898
|
-
if (
|
|
6899
|
-
|
|
6900
|
-
seen.add(ep.baseURL);
|
|
6901
|
-
return true;
|
|
6902
|
-
});
|
|
6903
|
-
const results = await Promise.allSettled(
|
|
6904
|
-
uniqueEndpoints.map((ep) => probeEndpoint(ep, timeoutMs))
|
|
6905
|
-
);
|
|
6906
|
-
return results.filter(
|
|
6907
|
-
(r) => r.status === "fulfilled" && r.value !== null
|
|
6908
|
-
).map((r) => r.value);
|
|
6909
|
-
}
|
|
6910
|
-
|
|
6911
|
-
// src/capabilities/fallback-router.ts
|
|
6912
|
-
var DEFAULT_COOLDOWNS = {
|
|
6913
|
-
rateLimit: 3e4,
|
|
6914
|
-
serverError: 6e4,
|
|
6915
|
-
connectionFailure: 3e5,
|
|
6916
|
-
contentPolicy: 0
|
|
6917
|
-
};
|
|
6918
|
-
function classifyError(error) {
|
|
6919
|
-
const msg = error instanceof Error ? error.message : String(error);
|
|
6920
|
-
const lower = msg.toLowerCase();
|
|
6921
|
-
if (lower.includes("429") || lower.includes("rate limit")) return "rate_limit";
|
|
6922
|
-
if (lower.includes("500") || lower.includes("502") || lower.includes("503") || lower.includes("504")) return "server_error";
|
|
6923
|
-
if (lower.includes("content") && (lower.includes("policy") || lower.includes("filter"))) return "content_policy";
|
|
6924
|
-
if (lower.includes("context") && (lower.includes("window") || lower.includes("length") || lower.includes("too long"))) return "context_window";
|
|
6925
|
-
if (lower.includes("fetch") || lower.includes("network") || lower.includes("connection") || lower.includes("econnrefused") || lower.includes("timeout")) return "connection_failure";
|
|
6926
|
-
return "unknown";
|
|
6927
|
-
}
|
|
6928
|
-
var FallbackRouter = class {
|
|
6929
|
-
id = "fallback-router";
|
|
6930
|
-
name = "Fallback Router";
|
|
6931
|
-
tier = "remote";
|
|
6932
|
-
providers;
|
|
6933
|
-
cooldowns;
|
|
6934
|
-
cooldownMap = /* @__PURE__ */ new Map();
|
|
6935
|
-
events;
|
|
6936
|
-
get capabilities() {
|
|
6937
|
-
const available = this.getAvailableProviders();
|
|
6938
|
-
return {
|
|
6939
|
-
embeddings: available.some((p) => p.capabilities.embeddings),
|
|
6940
|
-
chat: available.some((p) => p.capabilities.chat),
|
|
6941
|
-
streaming: available.some((p) => p.capabilities.streaming),
|
|
6942
|
-
vision: available.some((p) => p.capabilities.vision),
|
|
6943
|
-
toolCalling: available.some((p) => p.capabilities.toolCalling),
|
|
6944
|
-
structuredOutput: available.some((p) => p.capabilities.structuredOutput),
|
|
6945
|
-
maxContextTokens: Math.max(
|
|
6946
|
-
...available.map((p) => p.capabilities.maxContextTokens ?? 0),
|
|
6947
|
-
0
|
|
6948
|
-
)
|
|
6949
|
-
};
|
|
6950
|
-
}
|
|
6951
|
-
constructor(config) {
|
|
6952
|
-
this.providers = [...config.providers];
|
|
6953
|
-
this.cooldowns = { ...DEFAULT_COOLDOWNS, ...config.cooldowns };
|
|
6954
|
-
this.events = config.events;
|
|
6955
|
-
}
|
|
6956
|
-
// -------------------------------------------------------------------------
|
|
6957
|
-
// Provider management
|
|
6958
|
-
// -------------------------------------------------------------------------
|
|
6959
|
-
/** Get providers not currently in cooldown */
|
|
6960
|
-
getAvailableProviders() {
|
|
6961
|
-
const now = Date.now();
|
|
6962
|
-
return this.providers.filter((p) => {
|
|
6963
|
-
const cd = this.cooldownMap.get(p.id);
|
|
6964
|
-
if (!cd) return true;
|
|
6965
|
-
if (now >= cd.expiresAt) {
|
|
6966
|
-
this.cooldownMap.delete(p.id);
|
|
6967
|
-
return true;
|
|
6968
|
-
}
|
|
6969
|
-
return false;
|
|
6970
|
-
});
|
|
6971
|
-
}
|
|
6972
|
-
/** Get providers in cooldown with their expiry info */
|
|
6973
|
-
getCoolingDown() {
|
|
6974
|
-
const now = Date.now();
|
|
6975
|
-
const result = [];
|
|
6976
|
-
for (const [id, entry] of this.cooldownMap) {
|
|
6977
|
-
if (now < entry.expiresAt) {
|
|
6978
|
-
result.push({ providerId: id, category: entry.category, expiresAt: entry.expiresAt });
|
|
6979
|
-
}
|
|
6980
|
-
}
|
|
6981
|
-
return result;
|
|
6982
|
-
}
|
|
6983
|
-
/** Manually clear cooldown for a provider */
|
|
6984
|
-
clearCooldown(providerId) {
|
|
6985
|
-
this.cooldownMap.delete(providerId);
|
|
6986
|
-
}
|
|
6987
|
-
/** Clear all cooldowns */
|
|
6988
|
-
clearAllCooldowns() {
|
|
6989
|
-
this.cooldownMap.clear();
|
|
6990
|
-
}
|
|
6991
|
-
/** Add a provider to the chain (appended at lowest priority) */
|
|
6992
|
-
addProvider(provider) {
|
|
6993
|
-
this.providers.push(provider);
|
|
6994
|
-
}
|
|
6995
|
-
/** Remove a provider from the chain */
|
|
6996
|
-
removeProvider(id) {
|
|
6997
|
-
this.providers = this.providers.filter((p) => p.id !== id);
|
|
6998
|
-
this.cooldownMap.delete(id);
|
|
6999
|
-
}
|
|
7000
|
-
/** Reorder providers (new priority order) */
|
|
7001
|
-
setOrder(ids) {
|
|
7002
|
-
const byId = new Map(this.providers.map((p) => [p.id, p]));
|
|
7003
|
-
const reordered = [];
|
|
7004
|
-
for (const id of ids) {
|
|
7005
|
-
const p = byId.get(id);
|
|
7006
|
-
if (p) reordered.push(p);
|
|
9361
|
+
id;
|
|
9362
|
+
name;
|
|
9363
|
+
tier;
|
|
9364
|
+
capabilities;
|
|
9365
|
+
profile;
|
|
9366
|
+
async embed(request) {
|
|
9367
|
+
if (!this.capabilities.embeddings) {
|
|
9368
|
+
throw new Error(`Bridge provider '${this.id}' does not support embeddings`);
|
|
7007
9369
|
}
|
|
7008
|
-
|
|
7009
|
-
|
|
9370
|
+
if (!this.bridge.inference) {
|
|
9371
|
+
throw new Error("Fortemi bridge inference router is unavailable");
|
|
7010
9372
|
}
|
|
7011
|
-
this.
|
|
7012
|
-
}
|
|
7013
|
-
// -------------------------------------------------------------------------
|
|
7014
|
-
// InferenceProvider interface — with fallback
|
|
7015
|
-
// -------------------------------------------------------------------------
|
|
7016
|
-
async embed(request) {
|
|
7017
|
-
return this.withFallback(
|
|
7018
|
-
(p) => p.capabilities.embeddings && !!p.embed,
|
|
7019
|
-
(p) => p.embed(request)
|
|
7020
|
-
);
|
|
9373
|
+
return this.bridge.inference.embed(this.id, request);
|
|
7021
9374
|
}
|
|
7022
9375
|
async complete(request) {
|
|
7023
|
-
|
|
7024
|
-
(
|
|
7025
|
-
|
|
7026
|
-
)
|
|
9376
|
+
if (!this.capabilities.chat) {
|
|
9377
|
+
throw new Error(`Bridge provider '${this.id}' does not support chat`);
|
|
9378
|
+
}
|
|
9379
|
+
if (!this.bridge.inference) {
|
|
9380
|
+
throw new Error("Fortemi bridge inference router is unavailable");
|
|
9381
|
+
}
|
|
9382
|
+
return this.bridge.inference.complete(this.id, request);
|
|
7027
9383
|
}
|
|
7028
|
-
|
|
7029
|
-
|
|
7030
|
-
|
|
7031
|
-
throw new Error("No available providers with streaming capability");
|
|
9384
|
+
stream(request) {
|
|
9385
|
+
if (!this.capabilities.streaming) {
|
|
9386
|
+
throw new Error(`Bridge provider '${this.id}' does not support streaming`);
|
|
7032
9387
|
}
|
|
7033
|
-
|
|
9388
|
+
if (!this.bridge.inference?.stream) {
|
|
9389
|
+
throw new Error("Fortemi bridge streaming router is unavailable");
|
|
9390
|
+
}
|
|
9391
|
+
return this.bridge.inference.stream(this.id, request);
|
|
7034
9392
|
}
|
|
7035
9393
|
async listModels() {
|
|
7036
|
-
|
|
7037
|
-
const results = await Promise.allSettled(
|
|
7038
|
-
available.map((p) => p.listModels())
|
|
7039
|
-
);
|
|
7040
|
-
const models = [];
|
|
7041
|
-
for (const r of results) {
|
|
7042
|
-
if (r.status === "fulfilled") models.push(...r.value);
|
|
7043
|
-
}
|
|
7044
|
-
return models;
|
|
9394
|
+
return [];
|
|
7045
9395
|
}
|
|
7046
9396
|
async probe() {
|
|
7047
|
-
|
|
7048
|
-
|
|
7049
|
-
return { status: "down", latencyMs: 0, message: "All providers in cooldown" };
|
|
9397
|
+
if (!this.bridge.inference) {
|
|
9398
|
+
return { status: "down", latencyMs: 0, message: "Fortemi bridge inference router is unavailable" };
|
|
7050
9399
|
}
|
|
7051
|
-
|
|
7052
|
-
const results = await Promise.allSettled(
|
|
7053
|
-
available.map((p) => p.probe())
|
|
7054
|
-
);
|
|
7055
|
-
const okCount = results.filter(
|
|
7056
|
-
(r) => r.status === "fulfilled" && r.value.status === "ok"
|
|
7057
|
-
).length;
|
|
7058
|
-
return {
|
|
7059
|
-
status: okCount === available.length ? "ok" : okCount > 0 ? "degraded" : "down",
|
|
7060
|
-
latencyMs: Date.now() - start,
|
|
7061
|
-
message: `${okCount}/${available.length} providers healthy`
|
|
7062
|
-
};
|
|
9400
|
+
return this.bridge.inference.probeProvider(this.id);
|
|
7063
9401
|
}
|
|
7064
9402
|
dispose() {
|
|
7065
|
-
for (const p of this.providers) {
|
|
7066
|
-
p.dispose();
|
|
7067
|
-
}
|
|
7068
|
-
this.providers = [];
|
|
7069
|
-
this.cooldownMap.clear();
|
|
7070
|
-
}
|
|
7071
|
-
// -------------------------------------------------------------------------
|
|
7072
|
-
// Core fallback logic
|
|
7073
|
-
// -------------------------------------------------------------------------
|
|
7074
|
-
async withFallback(filter, execute) {
|
|
7075
|
-
const candidates = this.getAvailableProviders().filter(filter);
|
|
7076
|
-
if (candidates.length === 0) {
|
|
7077
|
-
throw new Error("No available providers for this request");
|
|
7078
|
-
}
|
|
7079
|
-
let lastError;
|
|
7080
|
-
for (const provider of candidates) {
|
|
7081
|
-
try {
|
|
7082
|
-
return await execute(provider);
|
|
7083
|
-
} catch (err) {
|
|
7084
|
-
lastError = err instanceof Error ? err : new Error(String(err));
|
|
7085
|
-
const category = classifyError(err);
|
|
7086
|
-
this.applyCooldown(provider.id, category);
|
|
7087
|
-
const nextCandidate = candidates[candidates.indexOf(provider) + 1];
|
|
7088
|
-
if (nextCandidate) {
|
|
7089
|
-
this.events?.emit("provider.fallback", {
|
|
7090
|
-
fromProvider: provider.id,
|
|
7091
|
-
toProvider: nextCandidate.id,
|
|
7092
|
-
errorCategory: category,
|
|
7093
|
-
error: lastError.message
|
|
7094
|
-
});
|
|
7095
|
-
}
|
|
7096
|
-
}
|
|
7097
|
-
}
|
|
7098
|
-
throw lastError ?? new Error("All providers failed");
|
|
7099
|
-
}
|
|
7100
|
-
applyCooldown(providerId, category) {
|
|
7101
|
-
let cooldownMs;
|
|
7102
|
-
switch (category) {
|
|
7103
|
-
case "rate_limit":
|
|
7104
|
-
cooldownMs = this.cooldowns.rateLimit;
|
|
7105
|
-
break;
|
|
7106
|
-
case "server_error":
|
|
7107
|
-
cooldownMs = this.cooldowns.serverError;
|
|
7108
|
-
break;
|
|
7109
|
-
case "connection_failure":
|
|
7110
|
-
cooldownMs = this.cooldowns.connectionFailure;
|
|
7111
|
-
break;
|
|
7112
|
-
case "content_policy":
|
|
7113
|
-
cooldownMs = this.cooldowns.contentPolicy;
|
|
7114
|
-
break;
|
|
7115
|
-
default:
|
|
7116
|
-
cooldownMs = this.cooldowns.serverError;
|
|
7117
|
-
}
|
|
7118
|
-
if (cooldownMs > 0) {
|
|
7119
|
-
const expiresAt = Date.now() + cooldownMs;
|
|
7120
|
-
this.cooldownMap.set(providerId, { expiresAt, category });
|
|
7121
|
-
this.events?.emit("provider.cooldown", {
|
|
7122
|
-
providerId,
|
|
7123
|
-
errorCategory: category,
|
|
7124
|
-
cooldownMs,
|
|
7125
|
-
expiresAt
|
|
7126
|
-
});
|
|
7127
|
-
}
|
|
7128
9403
|
}
|
|
7129
9404
|
};
|
|
9405
|
+
async function createBridgeInferenceProviders(bridge) {
|
|
9406
|
+
if (!bridge.inference) return [];
|
|
9407
|
+
const providers = await bridge.inference.listProviders();
|
|
9408
|
+
return providers.map((info) => new BridgeInferenceProvider(bridge, info));
|
|
9409
|
+
}
|
|
7130
9410
|
|
|
7131
9411
|
// src/fortemi-bridge.ts
|
|
7132
9412
|
function getFortemiBridge(host = globalThis) {
|
|
@@ -7155,6 +9435,159 @@ async function hasFortemiSecureSecrets(host = globalThis) {
|
|
|
7155
9435
|
}
|
|
7156
9436
|
}
|
|
7157
9437
|
|
|
9438
|
+
// src/capabilities/inference-runtime.ts
|
|
9439
|
+
function defineInferenceRuntime(config) {
|
|
9440
|
+
return config;
|
|
9441
|
+
}
|
|
9442
|
+
function defineInferenceProvider(provider) {
|
|
9443
|
+
return { kind: "provider", provider };
|
|
9444
|
+
}
|
|
9445
|
+
function defineOpenAICompatibleProvider(config) {
|
|
9446
|
+
return { kind: "openai-compatible", config };
|
|
9447
|
+
}
|
|
9448
|
+
function defineLegacyInferenceProvider(config) {
|
|
9449
|
+
return { kind: "legacy", ...config };
|
|
9450
|
+
}
|
|
9451
|
+
function mergeInferenceRuntimeConfigs(...configs) {
|
|
9452
|
+
const merged = {};
|
|
9453
|
+
for (const config of configs) {
|
|
9454
|
+
if (!config) continue;
|
|
9455
|
+
if (config.providers?.length) {
|
|
9456
|
+
merged.providers = mergeConfiguredProviders(merged.providers, config.providers);
|
|
9457
|
+
}
|
|
9458
|
+
if (config.routes) {
|
|
9459
|
+
merged.routes = { ...merged.routes ?? {}, ...config.routes };
|
|
9460
|
+
}
|
|
9461
|
+
if (config.activeProviderId !== void 0) {
|
|
9462
|
+
merged.activeProviderId = config.activeProviderId;
|
|
9463
|
+
}
|
|
9464
|
+
if (config.bridgeHost !== void 0) {
|
|
9465
|
+
merged.bridgeHost = config.bridgeHost;
|
|
9466
|
+
}
|
|
9467
|
+
if (config.includeBridgeProviders !== void 0) {
|
|
9468
|
+
merged.includeBridgeProviders = config.includeBridgeProviders;
|
|
9469
|
+
}
|
|
9470
|
+
if (config.discoverLocal !== void 0) {
|
|
9471
|
+
merged.discoverLocal = config.discoverLocal;
|
|
9472
|
+
}
|
|
9473
|
+
if (config.embeddingTaskSelection) {
|
|
9474
|
+
merged.embeddingTaskSelection = {
|
|
9475
|
+
...merged.embeddingTaskSelection ?? {},
|
|
9476
|
+
...config.embeddingTaskSelection
|
|
9477
|
+
};
|
|
9478
|
+
}
|
|
9479
|
+
}
|
|
9480
|
+
return merged;
|
|
9481
|
+
}
|
|
9482
|
+
function getConfiguredInferenceProviderId(config) {
|
|
9483
|
+
switch (config.kind) {
|
|
9484
|
+
case "provider":
|
|
9485
|
+
return config.provider.id;
|
|
9486
|
+
case "openai-compatible":
|
|
9487
|
+
return config.config.id;
|
|
9488
|
+
case "legacy":
|
|
9489
|
+
return config.id ?? "legacy";
|
|
9490
|
+
}
|
|
9491
|
+
}
|
|
9492
|
+
function mergeConfiguredProviders(previous, next) {
|
|
9493
|
+
const byId = /* @__PURE__ */ new Map();
|
|
9494
|
+
for (const provider of previous ?? []) {
|
|
9495
|
+
byId.set(getConfiguredInferenceProviderId(provider), provider);
|
|
9496
|
+
}
|
|
9497
|
+
for (const provider of next) {
|
|
9498
|
+
const id = getConfiguredInferenceProviderId(provider);
|
|
9499
|
+
byId.delete(id);
|
|
9500
|
+
byId.set(id, provider);
|
|
9501
|
+
}
|
|
9502
|
+
return Array.from(byId.values());
|
|
9503
|
+
}
|
|
9504
|
+
async function configureInferenceRuntime(options = {}) {
|
|
9505
|
+
const registry = options.registry ?? new ProviderRegistry(options.events);
|
|
9506
|
+
setEmbeddingTaskSelectionOptions(options.embeddingTaskSelection);
|
|
9507
|
+
for (const providerConfig of options.providers ?? []) {
|
|
9508
|
+
registry.add(createConfiguredProvider(providerConfig));
|
|
9509
|
+
}
|
|
9510
|
+
if (options.includeBridgeProviders !== false) {
|
|
9511
|
+
const bridge = getFortemiBridge(options.bridgeHost);
|
|
9512
|
+
if (bridge) {
|
|
9513
|
+
for (const provider of await createBridgeInferenceProviders(bridge)) {
|
|
9514
|
+
if (!registry.get(provider.id)) registry.add(provider);
|
|
9515
|
+
}
|
|
9516
|
+
}
|
|
9517
|
+
}
|
|
9518
|
+
if (options.discoverLocal) {
|
|
9519
|
+
const discoveryOptions = options.discoverLocal === true ? {} : options.discoverLocal;
|
|
9520
|
+
const discovered = await discoverLocalProviders(discoveryOptions);
|
|
9521
|
+
for (const provider of discovered) {
|
|
9522
|
+
const id = `local:${provider.id}`;
|
|
9523
|
+
if (registry.get(id)) continue;
|
|
9524
|
+
registry.add(new OpenAICompatibleProvider({
|
|
9525
|
+
id,
|
|
9526
|
+
name: provider.name,
|
|
9527
|
+
baseURL: provider.baseURL,
|
|
9528
|
+
tier: "local-server",
|
|
9529
|
+
defaultModel: provider.models.find((model) => model.capabilities.chat)?.id,
|
|
9530
|
+
defaultEmbeddingModel: provider.models.find((model) => model.capabilities.embeddings)?.id,
|
|
9531
|
+
profile: createLocalProviderProfile(provider.models)
|
|
9532
|
+
}));
|
|
9533
|
+
}
|
|
9534
|
+
}
|
|
9535
|
+
for (const [task, route] of Object.entries(options.routes ?? {})) {
|
|
9536
|
+
if (route) registry.setRoute(task, route);
|
|
9537
|
+
}
|
|
9538
|
+
if (options.activeProviderId) {
|
|
9539
|
+
registry.setActive(options.activeProviderId);
|
|
9540
|
+
}
|
|
9541
|
+
if (options.capabilityManager) {
|
|
9542
|
+
wireCapabilities(options.capabilityManager, registry);
|
|
9543
|
+
}
|
|
9544
|
+
const routeValidation = registry.validateRoutes();
|
|
9545
|
+
return {
|
|
9546
|
+
registry,
|
|
9547
|
+
providers: registry.list(),
|
|
9548
|
+
routeValidation,
|
|
9549
|
+
routeIssues: routeValidation.flatMap((route) => route.issues)
|
|
9550
|
+
};
|
|
9551
|
+
}
|
|
9552
|
+
function createConfiguredProvider(config) {
|
|
9553
|
+
switch (config.kind) {
|
|
9554
|
+
case "provider":
|
|
9555
|
+
return config.provider;
|
|
9556
|
+
case "openai-compatible":
|
|
9557
|
+
return new OpenAICompatibleProvider(config.config);
|
|
9558
|
+
case "legacy":
|
|
9559
|
+
return createLegacyProvider(config);
|
|
9560
|
+
}
|
|
9561
|
+
}
|
|
9562
|
+
function wireCapabilities(manager, registry) {
|
|
9563
|
+
if (registry.hasEmbeddings()) {
|
|
9564
|
+
manager.registerLoader("semantic", async () => {
|
|
9565
|
+
setEmbedFunction((texts, options) => {
|
|
9566
|
+
const request = {
|
|
9567
|
+
texts,
|
|
9568
|
+
task: options?.task ?? "embedding.document"
|
|
9569
|
+
};
|
|
9570
|
+
if (options?.model) Object.assign(request, { model: options.model });
|
|
9571
|
+
return registry.embed(request).then((result) => result.vectors);
|
|
9572
|
+
});
|
|
9573
|
+
});
|
|
9574
|
+
}
|
|
9575
|
+
if (registry.hasChat()) {
|
|
9576
|
+
manager.registerLoader("llm", async () => {
|
|
9577
|
+
setLlmFunction((prompt, options) => {
|
|
9578
|
+
const request = {
|
|
9579
|
+
prompt,
|
|
9580
|
+
task: options?.task ?? "chat.general"
|
|
9581
|
+
};
|
|
9582
|
+
if (options?.model) Object.assign(request, { model: options.model });
|
|
9583
|
+
if (options?.maxTokens !== void 0) Object.assign(request, { maxTokens: options.maxTokens });
|
|
9584
|
+
if (options?.temperature !== void 0) Object.assign(request, { temperature: options.temperature });
|
|
9585
|
+
return registry.complete(request).then((result) => result.text);
|
|
9586
|
+
});
|
|
9587
|
+
});
|
|
9588
|
+
}
|
|
9589
|
+
}
|
|
9590
|
+
|
|
7158
9591
|
// src/security/plugin-content.ts
|
|
7159
9592
|
var DEFAULT_DIRECTIVES = {
|
|
7160
9593
|
"default-src": ["'self'"],
|
|
@@ -7196,12 +9629,12 @@ function parseSriToken(integrity) {
|
|
|
7196
9629
|
const separator = token.indexOf("-");
|
|
7197
9630
|
if (separator <= 0) throw new Error("Invalid SRI token");
|
|
7198
9631
|
const algorithm = token.slice(0, separator).toLowerCase();
|
|
7199
|
-
const
|
|
9632
|
+
const digest2 = token.slice(separator + 1);
|
|
7200
9633
|
if (!SUPPORTED_SRI_ALGORITHMS.has(algorithm)) {
|
|
7201
9634
|
throw new Error("Unsupported SRI algorithm: " + algorithm);
|
|
7202
9635
|
}
|
|
7203
|
-
if (!
|
|
7204
|
-
return { algorithm, digest };
|
|
9636
|
+
if (!digest2) throw new Error("Invalid SRI digest");
|
|
9637
|
+
return { algorithm, digest: digest2 };
|
|
7205
9638
|
}
|
|
7206
9639
|
function toBase64(bytes) {
|
|
7207
9640
|
const data = new Uint8Array(bytes);
|
|
@@ -7218,8 +9651,8 @@ async function computeSri(data, algorithm = "sha384") {
|
|
|
7218
9651
|
if (!SUPPORTED_SRI_ALGORITHMS.has(normalized)) {
|
|
7219
9652
|
throw new Error("Unsupported SRI algorithm: " + algorithm);
|
|
7220
9653
|
}
|
|
7221
|
-
const
|
|
7222
|
-
return normalized + "-" + toBase64(
|
|
9654
|
+
const digest2 = await crypto.subtle.digest(normalized.toUpperCase().replace("SHA", "SHA-"), toArrayBuffer(data));
|
|
9655
|
+
return normalized + "-" + toBase64(digest2);
|
|
7223
9656
|
}
|
|
7224
9657
|
async function verifySri(data, integrity) {
|
|
7225
9658
|
const expected = parseSriToken(integrity);
|
|
@@ -8251,12 +10684,12 @@ async function verifyShardSignature(input) {
|
|
|
8251
10684
|
false,
|
|
8252
10685
|
["verify"]
|
|
8253
10686
|
);
|
|
8254
|
-
const
|
|
10687
|
+
const digest2 = await sha256Hex(canonicalPayloadBytes(payload));
|
|
8255
10688
|
signatureValid = await globalThis.crypto.subtle.verify(
|
|
8256
10689
|
"Ed25519",
|
|
8257
10690
|
key,
|
|
8258
10691
|
toBufferSource(base64urlToBytes(envelope.signature)),
|
|
8259
|
-
toBufferSource(new TextEncoder().encode(
|
|
10692
|
+
toBufferSource(new TextEncoder().encode(digest2))
|
|
8260
10693
|
);
|
|
8261
10694
|
} catch (err) {
|
|
8262
10695
|
return { ok: false, reason: "malformed", detail: err instanceof Error ? err.message : String(err) };
|
|
@@ -8301,11 +10734,11 @@ async function signShard(input) {
|
|
|
8301
10734
|
manifest_digest: await sha256Hex(manifest),
|
|
8302
10735
|
blob_digests: sidecarBlobDigests(input.files)
|
|
8303
10736
|
};
|
|
8304
|
-
const
|
|
10737
|
+
const digest2 = await sha256Hex(canonicalPayloadBytes(payload));
|
|
8305
10738
|
const signature = await globalThis.crypto.subtle.sign(
|
|
8306
10739
|
"Ed25519",
|
|
8307
10740
|
input.privateKey,
|
|
8308
|
-
toBufferSource(new TextEncoder().encode(
|
|
10741
|
+
toBufferSource(new TextEncoder().encode(digest2))
|
|
8309
10742
|
);
|
|
8310
10743
|
const envelope = {
|
|
8311
10744
|
...payload,
|
|
@@ -26850,7 +29283,7 @@ function addCanonicalFormats(ajv) {
|
|
|
26850
29283
|
}
|
|
26851
29284
|
function getLegacyAjv() {
|
|
26852
29285
|
if (!legacyAjvInstance) {
|
|
26853
|
-
legacyAjvInstance = new
|
|
29286
|
+
legacyAjvInstance = new Ajv20202({
|
|
26854
29287
|
allErrors: true,
|
|
26855
29288
|
strict: true,
|
|
26856
29289
|
validateFormats: false
|
|
@@ -26861,7 +29294,7 @@ function getLegacyAjv() {
|
|
|
26861
29294
|
}
|
|
26862
29295
|
function getCoreAjv() {
|
|
26863
29296
|
if (!coreAjvInstance) {
|
|
26864
|
-
coreAjvInstance = new
|
|
29297
|
+
coreAjvInstance = new Ajv20202({
|
|
26865
29298
|
allErrors: true,
|
|
26866
29299
|
strict: true,
|
|
26867
29300
|
validateFormats: true
|
|
@@ -26881,7 +29314,7 @@ function getCoreAjv() {
|
|
|
26881
29314
|
}
|
|
26882
29315
|
function getRecordAjv() {
|
|
26883
29316
|
if (!recordAjvInstance) {
|
|
26884
|
-
recordAjvInstance = new
|
|
29317
|
+
recordAjvInstance = new Ajv20202({
|
|
26885
29318
|
allErrors: true,
|
|
26886
29319
|
strict: true,
|
|
26887
29320
|
validateFormats: true
|
|
@@ -26901,7 +29334,7 @@ function getRecordAjv() {
|
|
|
26901
29334
|
}
|
|
26902
29335
|
function getFullAjv() {
|
|
26903
29336
|
if (!fullAjvInstance) {
|
|
26904
|
-
fullAjvInstance = new
|
|
29337
|
+
fullAjvInstance = new Ajv20202({
|
|
26905
29338
|
allErrors: true,
|
|
26906
29339
|
strict: true,
|
|
26907
29340
|
validateFormats: true
|
|
@@ -27561,7 +29994,7 @@ async function promoteBlobs(blobStore, blobs) {
|
|
|
27561
29994
|
|
|
27562
29995
|
// src/shard/full-v1-store.ts
|
|
27563
29996
|
var decoder3 = new TextDecoder();
|
|
27564
|
-
var
|
|
29997
|
+
var encoder2 = new TextEncoder();
|
|
27565
29998
|
function emptyCounts() {
|
|
27566
29999
|
return {
|
|
27567
30000
|
notes: 0,
|
|
@@ -27612,15 +30045,15 @@ function componentRecords(component, bytes) {
|
|
|
27612
30045
|
const parsed = spec.encoding === "json-array" ? parseJsonArrayBytes(bytes) : parseJsonlBytes(bytes);
|
|
27613
30046
|
return parsed;
|
|
27614
30047
|
}
|
|
27615
|
-
function
|
|
27616
|
-
if (Array.isArray(value)) return `[${value.map(
|
|
30048
|
+
function canonicalJson4(value) {
|
|
30049
|
+
if (Array.isArray(value)) return `[${value.map(canonicalJson4).join(",")}]`;
|
|
27617
30050
|
if (value && typeof value === "object") {
|
|
27618
|
-
return `{${Object.entries(value).sort(([left], [right]) => left.localeCompare(right)).map(([key, item]) => `${JSON.stringify(key)}:${
|
|
30051
|
+
return `{${Object.entries(value).sort(([left], [right]) => left.localeCompare(right)).map(([key, item]) => `${JSON.stringify(key)}:${canonicalJson4(item)}`).join(",")}}`;
|
|
27619
30052
|
}
|
|
27620
30053
|
return JSON.stringify(value);
|
|
27621
30054
|
}
|
|
27622
30055
|
function encodeComponentRecords(component, records) {
|
|
27623
|
-
return
|
|
30056
|
+
return encoder2.encode(FULL_V1_COMPONENT_FILES[component].encoding === "json-array" ? JSON.stringify(records) : records.map((record) => JSON.stringify(record)).join("\n"));
|
|
27624
30057
|
}
|
|
27625
30058
|
function attachmentBlobReferences(files) {
|
|
27626
30059
|
const refs = /* @__PURE__ */ new Map();
|
|
@@ -27871,7 +30304,7 @@ async function exportFullV1Snapshot(db, blobStore) {
|
|
|
27871
30304
|
for (const row of persisted.rows) currentRecords.get(row.component)?.push(row.record_json);
|
|
27872
30305
|
const changed = [...currentRecords].some(([component, records]) => {
|
|
27873
30306
|
const original = componentRecords(component, files.get(FULL_V1_COMPONENT_FILES[component].file));
|
|
27874
|
-
return
|
|
30307
|
+
return canonicalJson4(records) !== canonicalJson4(original);
|
|
27875
30308
|
});
|
|
27876
30309
|
let blobRefs;
|
|
27877
30310
|
if (changed) {
|
|
@@ -27890,7 +30323,7 @@ async function exportFullV1Snapshot(db, blobStore) {
|
|
|
27890
30323
|
name: "fortemi-react-full-v1-store",
|
|
27891
30324
|
version: manifest.producer?.version ?? "unknown"
|
|
27892
30325
|
};
|
|
27893
|
-
files.set("manifest.json",
|
|
30326
|
+
files.set("manifest.json", encoder2.encode(JSON.stringify(manifest, null, 2)));
|
|
27894
30327
|
files.delete("signature.json");
|
|
27895
30328
|
blobRefs = attachmentBlobReferences(files);
|
|
27896
30329
|
} else {
|
|
@@ -27914,7 +30347,7 @@ async function exportFullV1Snapshot(db, blobStore) {
|
|
|
27914
30347
|
}
|
|
27915
30348
|
return { success: true, archive: packTarGz(files), errors: [], capability_report: capability };
|
|
27916
30349
|
}
|
|
27917
|
-
var
|
|
30350
|
+
var encoder3 = new TextEncoder();
|
|
27918
30351
|
var decoder4 = new TextDecoder();
|
|
27919
30352
|
function iso2(value) {
|
|
27920
30353
|
if (value === null) return null;
|
|
@@ -27943,7 +30376,7 @@ function readRecords(files, component) {
|
|
|
27943
30376
|
function writeRecords(files, component, records) {
|
|
27944
30377
|
const spec = FULL_V1_COMPONENT_FILES[component];
|
|
27945
30378
|
const text = spec.encoding === "json-array" ? JSON.stringify(records) : records.map((record) => JSON.stringify(record)).join("\n");
|
|
27946
|
-
files.set(spec.file,
|
|
30379
|
+
files.set(spec.file, encoder3.encode(text));
|
|
27947
30380
|
}
|
|
27948
30381
|
async function liveRepresentationLosses(db) {
|
|
27949
30382
|
const tables = [
|
|
@@ -27959,13 +30392,13 @@ async function liveRepresentationLosses(db) {
|
|
|
27959
30392
|
const result = await db.query(
|
|
27960
30393
|
`SELECT COUNT(*) AS count FROM ${table} WHERE deleted_at IS NOT NULL`
|
|
27961
30394
|
);
|
|
27962
|
-
const
|
|
27963
|
-
if (
|
|
30395
|
+
const count2 = Number(result.rows[0]?.count ?? 0);
|
|
30396
|
+
if (count2 > 0) {
|
|
27964
30397
|
losses.push({
|
|
27965
30398
|
code: "unrepresentable-live-tombstone",
|
|
27966
30399
|
component,
|
|
27967
|
-
count,
|
|
27968
|
-
message: `${
|
|
30400
|
+
count: count2,
|
|
30401
|
+
message: `${count2} ${table} tombstone(s) have no full-v1 wire field`,
|
|
27969
30402
|
action: "reject",
|
|
27970
30403
|
reason: "full-v1-live-production"
|
|
27971
30404
|
});
|
|
@@ -27997,15 +30430,15 @@ async function liveRepresentationLosses(db) {
|
|
|
27997
30430
|
for (const row of vectorDimensions.rows) {
|
|
27998
30431
|
const dimension = Number(row.dimension);
|
|
27999
30432
|
if (dimension === 768) continue;
|
|
28000
|
-
const
|
|
30433
|
+
const count2 = Number(row.count);
|
|
28001
30434
|
losses.push({
|
|
28002
30435
|
code: "unrepresentable-live-embedding-dimension",
|
|
28003
30436
|
component: "embeddings",
|
|
28004
|
-
count,
|
|
30437
|
+
count: count2,
|
|
28005
30438
|
field_path: "/vector",
|
|
28006
30439
|
source_state: "value",
|
|
28007
30440
|
destination_capability: "full-v1 requires exactly 768 vector dimensions",
|
|
28008
|
-
message: `${
|
|
30441
|
+
message: `${count2} embedding vector(s) have ${dimension} dimensions`,
|
|
28009
30442
|
action: "reject",
|
|
28010
30443
|
reason: "full-v1-live-production"
|
|
28011
30444
|
});
|
|
@@ -28410,7 +30843,7 @@ async function exportLiveFullV1(db, coreArchive, legacyArchive, options) {
|
|
|
28410
30843
|
capability_report: capability
|
|
28411
30844
|
};
|
|
28412
30845
|
}
|
|
28413
|
-
files.set("manifest.json",
|
|
30846
|
+
files.set("manifest.json", encoder3.encode(JSON.stringify(manifest, null, 2)));
|
|
28414
30847
|
for (const note of records.notes) {
|
|
28415
30848
|
const attachments = Array.isArray(note.attachments) ? note.attachments : [];
|
|
28416
30849
|
for (const projection of attachments) {
|
|
@@ -28455,7 +30888,7 @@ async function exportLiveFullV1(db, coreArchive, legacyArchive, options) {
|
|
|
28455
30888
|
}
|
|
28456
30889
|
|
|
28457
30890
|
// src/shard/shard-export.ts
|
|
28458
|
-
var
|
|
30891
|
+
var encoder4 = new TextEncoder();
|
|
28459
30892
|
var CORE_V1_FILES = /* @__PURE__ */ new Set([
|
|
28460
30893
|
"notes.jsonl",
|
|
28461
30894
|
"collections.json",
|
|
@@ -28565,16 +30998,27 @@ async function collectCoreV1Losses(db, options) {
|
|
|
28565
30998
|
{ component: "community_assignments", table: "community_assignment" }
|
|
28566
30999
|
];
|
|
28567
31000
|
for (const { component, table } of componentCounts) {
|
|
28568
|
-
const
|
|
28569
|
-
if (
|
|
31001
|
+
const count2 = await rowCount(db, `SELECT COUNT(*) AS count FROM ${table}`);
|
|
31002
|
+
if (count2 > 0) {
|
|
28570
31003
|
losses.push({
|
|
28571
31004
|
code: "component-outside-profile",
|
|
28572
31005
|
component,
|
|
28573
|
-
count,
|
|
28574
|
-
message: `${
|
|
31006
|
+
count: count2,
|
|
31007
|
+
message: `${count2} ${component} record(s) are outside core-v1 and were omitted.`
|
|
28575
31008
|
});
|
|
28576
31009
|
}
|
|
28577
31010
|
}
|
|
31011
|
+
const sourceIdentities = await rowCount(db, "SELECT COUNT(*) AS count FROM source_identity");
|
|
31012
|
+
if (sourceIdentities > 0) {
|
|
31013
|
+
losses.push({
|
|
31014
|
+
code: "source-identity-outside-profile",
|
|
31015
|
+
count: sourceIdentities,
|
|
31016
|
+
field_path: "source_identity",
|
|
31017
|
+
action: "omit",
|
|
31018
|
+
destination_capability: "core-v1 does not declare source-addressed identity mappings",
|
|
31019
|
+
message: `${sourceIdentities} source identity mapping(s) are outside core-v1 and were omitted.`
|
|
31020
|
+
});
|
|
31021
|
+
}
|
|
28578
31022
|
const nullRevisions = await rowCount(
|
|
28579
31023
|
db,
|
|
28580
31024
|
`SELECT COUNT(*) AS count
|
|
@@ -28863,11 +31307,11 @@ async function exportShardBytes(db, options, mode) {
|
|
|
28863
31307
|
const slice = shardNotes.slice(offset, offset + clusterSize);
|
|
28864
31308
|
const href = `notes/${String(offset).padStart(6, "0")}.jsonl`;
|
|
28865
31309
|
clusters.push({ href, offset });
|
|
28866
|
-
files.set(href,
|
|
31310
|
+
files.set(href, encoder4.encode(slice.map((n) => JSON.stringify(n)).join("\n")));
|
|
28867
31311
|
}
|
|
28868
31312
|
layout = { clusters: { notes: clusters } };
|
|
28869
31313
|
} else {
|
|
28870
|
-
files.set("notes.jsonl",
|
|
31314
|
+
files.set("notes.jsonl", encoder4.encode(shardNotes.map((n) => JSON.stringify(n)).join("\n")));
|
|
28871
31315
|
}
|
|
28872
31316
|
components.push("notes");
|
|
28873
31317
|
counts.notes = notes.length;
|
|
@@ -28895,7 +31339,7 @@ async function exportShardBytes(db, options, mode) {
|
|
|
28895
31339
|
mode?.nativeSchema2Presence
|
|
28896
31340
|
);
|
|
28897
31341
|
}
|
|
28898
|
-
files.set("collections.json",
|
|
31342
|
+
files.set("collections.json", encoder4.encode(JSON.stringify(shardCollections)));
|
|
28899
31343
|
components.push("collections");
|
|
28900
31344
|
counts.collections = shardCollections.length;
|
|
28901
31345
|
const allTagRows = await db.query(
|
|
@@ -28911,7 +31355,7 @@ async function exportShardBytes(db, options, mode) {
|
|
|
28911
31355
|
const shardTags = tagsToShard(
|
|
28912
31356
|
relevantTags.map((r) => ({ name: r.tag, created_at: r.created_at }))
|
|
28913
31357
|
);
|
|
28914
|
-
files.set("tags.json",
|
|
31358
|
+
files.set("tags.json", encoder4.encode(JSON.stringify(shardTags)));
|
|
28915
31359
|
components.push("tags");
|
|
28916
31360
|
counts.tags = shardTags.length;
|
|
28917
31361
|
const templateRows = await db.query(`SELECT * FROM template ORDER BY created_at, id`);
|
|
@@ -28927,7 +31371,7 @@ async function exportShardBytes(db, options, mode) {
|
|
|
28927
31371
|
mode?.nativeSchema2Presence
|
|
28928
31372
|
);
|
|
28929
31373
|
}
|
|
28930
|
-
files.set("templates.json",
|
|
31374
|
+
files.set("templates.json", encoder4.encode(JSON.stringify(shardTemplates)));
|
|
28931
31375
|
components.push("templates");
|
|
28932
31376
|
counts.templates = shardTemplates.length;
|
|
28933
31377
|
}
|
|
@@ -28952,7 +31396,7 @@ async function exportShardBytes(db, options, mode) {
|
|
|
28952
31396
|
);
|
|
28953
31397
|
}
|
|
28954
31398
|
const linksJsonl = shardLinks.map((l) => JSON.stringify(l)).join("\n");
|
|
28955
|
-
files.set("links.jsonl",
|
|
31399
|
+
files.set("links.jsonl", encoder4.encode(linksJsonl));
|
|
28956
31400
|
components.push("links");
|
|
28957
31401
|
counts.links = shardLinks.length;
|
|
28958
31402
|
const allNoteSkosRows = await db.query(`SELECT * FROM note_skos_tag ORDER BY created_at`);
|
|
@@ -28969,25 +31413,25 @@ async function exportShardBytes(db, options, mode) {
|
|
|
28969
31413
|
(row) => exportedConceptIds.has(row.source_concept_id) && exportedConceptIds.has(row.target_concept_id)
|
|
28970
31414
|
) : allRelationRows.rows;
|
|
28971
31415
|
const shardSkosSchemes = filteredSchemeRows.map(skosSchemeToShard);
|
|
28972
|
-
files.set("skos_schemes.json",
|
|
31416
|
+
files.set("skos_schemes.json", encoder4.encode(JSON.stringify(shardSkosSchemes)));
|
|
28973
31417
|
components.push("skos_schemes");
|
|
28974
31418
|
counts.skos_schemes = shardSkosSchemes.length;
|
|
28975
31419
|
const shardSkosConcepts = filteredConceptRows.map(skosConceptToShard);
|
|
28976
|
-
files.set("skos_concepts.json",
|
|
31420
|
+
files.set("skos_concepts.json", encoder4.encode(JSON.stringify(shardSkosConcepts)));
|
|
28977
31421
|
components.push("skos_concepts");
|
|
28978
31422
|
counts.skos_concepts = shardSkosConcepts.length;
|
|
28979
31423
|
const skosRelationsJsonl = filteredRelationRows.map((row) => JSON.stringify(skosRelationToShard(row))).join("\n");
|
|
28980
|
-
files.set("skos_relations.jsonl",
|
|
31424
|
+
files.set("skos_relations.jsonl", encoder4.encode(skosRelationsJsonl));
|
|
28981
31425
|
components.push("skos_relations");
|
|
28982
31426
|
counts.skos_relations = filteredRelationRows.length;
|
|
28983
31427
|
const noteSkosJsonl = filteredNoteSkosRows.map((row) => JSON.stringify(noteSkosTagToShard(row))).join("\n");
|
|
28984
|
-
files.set("note_skos_tags.jsonl",
|
|
31428
|
+
files.set("note_skos_tags.jsonl", encoder4.encode(noteSkosJsonl));
|
|
28985
31429
|
components.push("note_skos_tags");
|
|
28986
31430
|
counts.note_skos_tags = filteredNoteSkosRows.length;
|
|
28987
31431
|
const provenanceRows = await db.query(`SELECT * FROM provenance_edge ORDER BY started_at`);
|
|
28988
31432
|
const filteredProvenanceRows = isFiltered ? provenanceRows.rows.filter((row) => row.entity_type !== "note" || exportedNoteIds.has(row.entity_id)) : provenanceRows.rows;
|
|
28989
31433
|
const provenanceJsonl = filteredProvenanceRows.map((row) => JSON.stringify(provenanceEdgeToShard(row))).join("\n");
|
|
28990
|
-
files.set("provenance_edges.jsonl",
|
|
31434
|
+
files.set("provenance_edges.jsonl", encoder4.encode(provenanceJsonl));
|
|
28991
31435
|
components.push("provenance_edges");
|
|
28992
31436
|
counts.provenance_edges = filteredProvenanceRows.length;
|
|
28993
31437
|
if (options?.includeEmbeddings) {
|
|
@@ -29027,7 +31471,7 @@ async function exportShardBytes(db, options, mode) {
|
|
|
29027
31471
|
freshness_json: { status: "unknown" }
|
|
29028
31472
|
} : row
|
|
29029
31473
|
));
|
|
29030
|
-
files.set("embedding_sets.json",
|
|
31474
|
+
files.set("embedding_sets.json", encoder4.encode(JSON.stringify(shardEmbSets)));
|
|
29031
31475
|
components.push("embedding_sets");
|
|
29032
31476
|
counts.embedding_sets = shardEmbSets.length;
|
|
29033
31477
|
const embeddingConfigRows = await db.query(
|
|
@@ -29037,7 +31481,7 @@ async function exportShardBytes(db, options, mode) {
|
|
|
29037
31481
|
);
|
|
29038
31482
|
if (embeddingConfigRows.rows.length > 0) {
|
|
29039
31483
|
const shardEmbeddingConfigs = embeddingConfigRows.rows.map((row) => embeddingConfigToShard(row));
|
|
29040
|
-
files.set("embedding_configs.json",
|
|
31484
|
+
files.set("embedding_configs.json", encoder4.encode(JSON.stringify(shardEmbeddingConfigs)));
|
|
29041
31485
|
components.push("embedding_configs");
|
|
29042
31486
|
counts.embedding_configs = shardEmbeddingConfigs.length;
|
|
29043
31487
|
}
|
|
@@ -29050,7 +31494,7 @@ async function exportShardBytes(db, options, mode) {
|
|
|
29050
31494
|
(member) => exportedSetIds.has(member.embedding_set_id) && exportedNoteIds.has(member.note_id) && (includeMaterializedSelectors || !virtualSetIds.has(member.embedding_set_id))
|
|
29051
31495
|
);
|
|
29052
31496
|
const membersJsonl = scopedEmbMemberRows.map((m) => JSON.stringify(embeddingSetMemberToShard(m))).join("\n");
|
|
29053
|
-
files.set("embedding_set_members.jsonl",
|
|
31497
|
+
files.set("embedding_set_members.jsonl", encoder4.encode(membersJsonl));
|
|
29054
31498
|
components.push("embedding_set_members");
|
|
29055
31499
|
counts.embedding_set_members = scopedEmbMemberRows.length;
|
|
29056
31500
|
const embRows = await db.query(
|
|
@@ -29076,7 +31520,7 @@ async function exportShardBytes(db, options, mode) {
|
|
|
29076
31520
|
(embedding) => exportedSetIds.has(embedding.embedding_set_id) && exportedNoteIds.has(embedding.note_id) && (memberEmbeddingIds.size === 0 || memberEmbeddingIds.has(embedding.id))
|
|
29077
31521
|
);
|
|
29078
31522
|
const embJsonl = scopedEmbRows.map((e) => JSON.stringify(embeddingToShard(e))).join("\n");
|
|
29079
|
-
files.set("embeddings.jsonl",
|
|
31523
|
+
files.set("embeddings.jsonl", encoder4.encode(embJsonl));
|
|
29080
31524
|
components.push("embeddings");
|
|
29081
31525
|
counts.embeddings = scopedEmbRows.length;
|
|
29082
31526
|
}
|
|
@@ -29102,7 +31546,7 @@ async function exportShardBytes(db, options, mode) {
|
|
|
29102
31546
|
freshness: jsonObject4(row.freshness_json) ?? { status: "unknown" },
|
|
29103
31547
|
created_at: iso3(row.created_at)
|
|
29104
31548
|
}));
|
|
29105
|
-
files.set("graph_sources.json",
|
|
31549
|
+
files.set("graph_sources.json", encoder4.encode(JSON.stringify(shardGraphSources)));
|
|
29106
31550
|
components.push("graph_sources");
|
|
29107
31551
|
counts.graph_sources = shardGraphSources.length;
|
|
29108
31552
|
}
|
|
@@ -29118,7 +31562,7 @@ async function exportShardBytes(db, options, mode) {
|
|
|
29118
31562
|
rank: row.rank,
|
|
29119
31563
|
metadata: jsonObject4(row.metadata_json)
|
|
29120
31564
|
})).join("\n");
|
|
29121
|
-
files.set("graph_edges.jsonl",
|
|
31565
|
+
files.set("graph_edges.jsonl", encoder4.encode(graphEdgesJsonl));
|
|
29122
31566
|
components.push("graph_edges");
|
|
29123
31567
|
counts.graph_edges = scopedGraphEdgeRows.length;
|
|
29124
31568
|
}
|
|
@@ -29154,7 +31598,7 @@ async function exportShardBytes(db, options, mode) {
|
|
|
29154
31598
|
})),
|
|
29155
31599
|
created_at: iso3(row.created_at)
|
|
29156
31600
|
}));
|
|
29157
|
-
files.set("communities.json",
|
|
31601
|
+
files.set("communities.json", encoder4.encode(JSON.stringify(shardCommunitySets)));
|
|
29158
31602
|
components.push("communities");
|
|
29159
31603
|
counts.community_sets = shardCommunitySets.length;
|
|
29160
31604
|
counts.communities = scopedCommunityRows.length;
|
|
@@ -29170,7 +31614,7 @@ async function exportShardBytes(db, options, mode) {
|
|
|
29170
31614
|
source_type: row.source_type,
|
|
29171
31615
|
metadata: jsonObject4(row.metadata_json)
|
|
29172
31616
|
})).join("\n");
|
|
29173
|
-
files.set("community_assignments.jsonl",
|
|
31617
|
+
files.set("community_assignments.jsonl", encoder4.encode(assignmentsJsonl));
|
|
29174
31618
|
components.push("community_assignments");
|
|
29175
31619
|
counts.community_assignments = scopedAssignmentRows.length;
|
|
29176
31620
|
}
|
|
@@ -29193,7 +31637,7 @@ async function exportShardBytes(db, options, mode) {
|
|
|
29193
31637
|
const coreTags = [...tagsByName.values()].sort(
|
|
29194
31638
|
(left, right) => left.name.localeCompare(right.name)
|
|
29195
31639
|
);
|
|
29196
|
-
files.set("tags.json",
|
|
31640
|
+
files.set("tags.json", encoder4.encode(JSON.stringify(coreTags)));
|
|
29197
31641
|
components.splice(0, components.length, ...CORE_V1_COMPONENTS);
|
|
29198
31642
|
for (const key of Object.keys(counts)) {
|
|
29199
31643
|
if (!CORE_V1_COMPONENTS.includes(key)) delete counts[key];
|
|
@@ -29233,7 +31677,7 @@ async function exportShardBytes(db, options, mode) {
|
|
|
29233
31677
|
...!coreV1 ? { migrated_from: null } : {},
|
|
29234
31678
|
...!coreV1 && layout ? { layout } : {}
|
|
29235
31679
|
};
|
|
29236
|
-
files.set("manifest.json",
|
|
31680
|
+
files.set("manifest.json", encoder4.encode(JSON.stringify(manifest, null, 2)));
|
|
29237
31681
|
if (options?.includeBlobs && options.blobStore) {
|
|
29238
31682
|
const packed = /* @__PURE__ */ new Set();
|
|
29239
31683
|
for (const row of attachmentRows.rows) {
|
|
@@ -30718,13 +33162,13 @@ function noteSearchText(note) {
|
|
|
30718
33162
|
}
|
|
30719
33163
|
function countOccurrences(haystack, needle) {
|
|
30720
33164
|
if (!needle) return 0;
|
|
30721
|
-
let
|
|
33165
|
+
let count2 = 0;
|
|
30722
33166
|
let index = haystack.indexOf(needle);
|
|
30723
33167
|
while (index !== -1) {
|
|
30724
|
-
|
|
33168
|
+
count2 += 1;
|
|
30725
33169
|
index = haystack.indexOf(needle, index + needle.length);
|
|
30726
33170
|
}
|
|
30727
|
-
return
|
|
33171
|
+
return count2;
|
|
30728
33172
|
}
|
|
30729
33173
|
function noteMatchesTokens(note, tokens) {
|
|
30730
33174
|
if (tokens.length === 0) return true;
|
|
@@ -31202,7 +33646,7 @@ function hasPositiveInteger(value) {
|
|
|
31202
33646
|
}
|
|
31203
33647
|
function isFacetCounts(value) {
|
|
31204
33648
|
if (!value || typeof value !== "object" || Array.isArray(value)) return false;
|
|
31205
|
-
return Object.values(value).every((counts) => !!counts && typeof counts === "object" && !Array.isArray(counts) && Object.values(counts).every((
|
|
33649
|
+
return Object.values(value).every((counts) => !!counts && typeof counts === "object" && !Array.isArray(counts) && Object.values(counts).every((count2) => hasNonNegativeInteger(count2)));
|
|
31206
33650
|
}
|
|
31207
33651
|
function isPlainRecord(value) {
|
|
31208
33652
|
return !!value && typeof value === "object" && !Array.isArray(value);
|
|
@@ -33739,7 +36183,7 @@ function getAiwgFortemiIndexExportSchema() {
|
|
|
33739
36183
|
}
|
|
33740
36184
|
function getAjv() {
|
|
33741
36185
|
if (!ajvInstance) {
|
|
33742
|
-
ajvInstance = new
|
|
36186
|
+
ajvInstance = new Ajv20202({
|
|
33743
36187
|
allErrors: true,
|
|
33744
36188
|
strict: false,
|
|
33745
36189
|
validateFormats: false
|
|
@@ -33802,7 +36246,7 @@ function validateAiwgFortemiProjectedRecordSchema(value) {
|
|
|
33802
36246
|
const valid = validate(value);
|
|
33803
36247
|
return { valid, errors: formatErrors2(validate.errors) };
|
|
33804
36248
|
}
|
|
33805
|
-
var
|
|
36249
|
+
var encoder5 = new TextEncoder();
|
|
33806
36250
|
var UUID_NAMESPACE = "7ab5d1f8-29d2-5e35-9e2f-3a45de171a9e";
|
|
33807
36251
|
function uuid(kind, id) {
|
|
33808
36252
|
return v5(`${kind}:${id}`, UUID_NAMESPACE);
|
|
@@ -33818,7 +36262,7 @@ function addLoss(losses, code, message, details = {}) {
|
|
|
33818
36262
|
losses.push({ code, message, ...details });
|
|
33819
36263
|
}
|
|
33820
36264
|
function encode(values, encoding) {
|
|
33821
|
-
return
|
|
36265
|
+
return encoder5.encode(encoding === "json-array" ? JSON.stringify(values) : values.map((value) => JSON.stringify(value)).join("\n"));
|
|
33822
36266
|
}
|
|
33823
36267
|
function noteTitle(record, losses) {
|
|
33824
36268
|
if (own(record, "title")) return record.title ?? null;
|
|
@@ -33986,7 +36430,7 @@ async function convertAiwgIndexToFullV1(index, options = {}) {
|
|
|
33986
36430
|
tags: recordTags,
|
|
33987
36431
|
attachments: []
|
|
33988
36432
|
});
|
|
33989
|
-
const hash = await sha256Hex(
|
|
36433
|
+
const hash = await sha256Hex(encoder5.encode(content));
|
|
33990
36434
|
rows.get("note_originals").push({
|
|
33991
36435
|
id: uuid("note-original", record.id),
|
|
33992
36436
|
note_id: noteId,
|
|
@@ -34078,7 +36522,7 @@ async function convertAiwgIndexToFullV1(index, options = {}) {
|
|
|
34078
36522
|
metric: null,
|
|
34079
36523
|
algorithm: null,
|
|
34080
36524
|
parameters: null,
|
|
34081
|
-
input_hash: `sha256:${await sha256Hex(
|
|
36525
|
+
input_hash: `sha256:${await sha256Hex(encoder5.encode(JSON.stringify(relationshipInput)))}`,
|
|
34082
36526
|
freshness: { status: "fresh", checked_at: exportedAt },
|
|
34083
36527
|
created_at: exportedAt
|
|
34084
36528
|
});
|
|
@@ -34536,7 +36980,7 @@ async function convertAiwgIndexToFullV1(index, options = {}) {
|
|
|
34536
36980
|
checksums,
|
|
34537
36981
|
min_reader_version: "2.0.0"
|
|
34538
36982
|
};
|
|
34539
|
-
const manifestBytes =
|
|
36983
|
+
const manifestBytes = encoder5.encode(JSON.stringify(manifest, null, 2));
|
|
34540
36984
|
files.set("manifest.json", manifestBytes);
|
|
34541
36985
|
const validation = await validateFullV1ShardArchive(files);
|
|
34542
36986
|
if (!validation.valid) {
|
|
@@ -34797,13 +37241,20 @@ var RECORD_COLLECTIONS = [
|
|
|
34797
37241
|
"collection_note",
|
|
34798
37242
|
"attachment",
|
|
34799
37243
|
"attachment_blob",
|
|
34800
|
-
"shard_manifest"
|
|
37244
|
+
"shard_manifest",
|
|
37245
|
+
"source_identity",
|
|
37246
|
+
"source_import_run",
|
|
37247
|
+
"deletion_receipt"
|
|
34801
37248
|
];
|
|
34802
37249
|
var RECORD_STORE_CAPABILITIES = {
|
|
34803
37250
|
crud: true,
|
|
34804
37251
|
journal: true,
|
|
34805
37252
|
atomicBatch: true,
|
|
34806
37253
|
boundedTextScan: true,
|
|
37254
|
+
sourceAddressedUpsert: true,
|
|
37255
|
+
deletionReceipts: true,
|
|
37256
|
+
typedMetadataPredicates: false,
|
|
37257
|
+
evidenceLocators: true,
|
|
34807
37258
|
fullTextSearch: false,
|
|
34808
37259
|
vectorSearch: false,
|
|
34809
37260
|
sqlJoins: false
|
|
@@ -35764,7 +38215,7 @@ function createRecordBackend(store, options = {}) {
|
|
|
35764
38215
|
}
|
|
35765
38216
|
|
|
35766
38217
|
// src/records/record-shard.ts
|
|
35767
|
-
var
|
|
38218
|
+
var encoder6 = new TextEncoder();
|
|
35768
38219
|
var decoder8 = new TextDecoder();
|
|
35769
38220
|
function emptyCounts3() {
|
|
35770
38221
|
return {
|
|
@@ -35992,7 +38443,7 @@ async function buildRecordShardArchive(store, options, profile) {
|
|
|
35992
38443
|
(attachment) => exportedNoteIds.has(attachment.note_id)
|
|
35993
38444
|
);
|
|
35994
38445
|
const projectedAttachmentCount = browserNotes.reduce(
|
|
35995
|
-
(
|
|
38446
|
+
(count2, note) => count2 + (note.attachments?.length ?? 0),
|
|
35996
38447
|
0
|
|
35997
38448
|
);
|
|
35998
38449
|
const sourceNoteById = new Map(notes.map((note) => [note.id, note]));
|
|
@@ -36022,11 +38473,11 @@ async function buildRecordShardArchive(store, options, profile) {
|
|
|
36022
38473
|
const slice = shardNotes.slice(offset, offset + clusterSize);
|
|
36023
38474
|
const href = `notes/${String(offset).padStart(6, "0")}.jsonl`;
|
|
36024
38475
|
clusters.push({ href, offset });
|
|
36025
|
-
files.set(href,
|
|
38476
|
+
files.set(href, encoder6.encode(slice.map((n) => JSON.stringify(n)).join("\n")));
|
|
36026
38477
|
}
|
|
36027
38478
|
layout = { clusters: { notes: clusters } };
|
|
36028
38479
|
} else {
|
|
36029
|
-
files.set("notes.jsonl",
|
|
38480
|
+
files.set("notes.jsonl", encoder6.encode(shardNotes.map((n) => JSON.stringify(n)).join("\n")));
|
|
36030
38481
|
}
|
|
36031
38482
|
components.push("notes");
|
|
36032
38483
|
counts.notes = shardNotes.length;
|
|
@@ -36054,7 +38505,7 @@ async function buildRecordShardArchive(store, options, profile) {
|
|
|
36054
38505
|
note_count: mapped.note_count ?? 0
|
|
36055
38506
|
};
|
|
36056
38507
|
});
|
|
36057
|
-
files.set("collections.json",
|
|
38508
|
+
files.set("collections.json", encoder6.encode(JSON.stringify(shardCollections)));
|
|
36058
38509
|
components.push("collections");
|
|
36059
38510
|
counts.collections = shardCollections.length;
|
|
36060
38511
|
const distinctTags = [...new Set(
|
|
@@ -36071,7 +38522,7 @@ async function buildRecordShardArchive(store, options, profile) {
|
|
|
36071
38522
|
created_at: tagCreatedAt.get(name) ?? (/* @__PURE__ */ new Date(0)).toISOString()
|
|
36072
38523
|
}))
|
|
36073
38524
|
);
|
|
36074
|
-
files.set("tags.json",
|
|
38525
|
+
files.set("tags.json", encoder6.encode(JSON.stringify(shardTags)));
|
|
36075
38526
|
components.push("tags");
|
|
36076
38527
|
counts.tags = shardTags.length;
|
|
36077
38528
|
const links = await store.list("link");
|
|
@@ -36093,7 +38544,7 @@ async function buildRecordShardArchive(store, options, profile) {
|
|
|
36093
38544
|
}
|
|
36094
38545
|
return shard;
|
|
36095
38546
|
});
|
|
36096
|
-
files.set("links.jsonl",
|
|
38547
|
+
files.set("links.jsonl", encoder6.encode(shardLinks.map((l) => JSON.stringify(l)).join("\n")));
|
|
36097
38548
|
components.push("links");
|
|
36098
38549
|
counts.links = shardLinks.length;
|
|
36099
38550
|
const checksums = {};
|
|
@@ -36146,6 +38597,17 @@ async function buildRecordShardArchive(store, options, profile) {
|
|
|
36146
38597
|
message: `${collectionStateLosses} collection lifecycle state(s) are outside record-v1 and were omitted.`
|
|
36147
38598
|
});
|
|
36148
38599
|
}
|
|
38600
|
+
const sourceIdentities = await store.list("source_identity");
|
|
38601
|
+
if (sourceIdentities.length > 0) {
|
|
38602
|
+
losses.push({
|
|
38603
|
+
code: "source-identity-outside-profile",
|
|
38604
|
+
count: sourceIdentities.length,
|
|
38605
|
+
field_path: "source_identity",
|
|
38606
|
+
action: "omit",
|
|
38607
|
+
destination_capability: "record-v1 does not declare source-addressed identity mappings",
|
|
38608
|
+
message: `${sourceIdentities.length} source identity mapping(s) are outside record-v1 and were omitted.`
|
|
38609
|
+
});
|
|
38610
|
+
}
|
|
36149
38611
|
}
|
|
36150
38612
|
let manifest = isRecordV1 ? {
|
|
36151
38613
|
version: recordSchemaVersion2,
|
|
@@ -36212,7 +38674,7 @@ async function buildRecordShardArchive(store, options, profile) {
|
|
|
36212
38674
|
);
|
|
36213
38675
|
}
|
|
36214
38676
|
}
|
|
36215
|
-
files.set("manifest.json",
|
|
38677
|
+
files.set("manifest.json", encoder6.encode(JSON.stringify(manifest, null, 2)));
|
|
36216
38678
|
if (options?.includeBlobs && options.blobStore) {
|
|
36217
38679
|
const packed = /* @__PURE__ */ new Set();
|
|
36218
38680
|
for (const checksum of exportedBlobChecksums) {
|
|
@@ -36428,9 +38890,9 @@ async function importShardToRecords(store, data, options) {
|
|
|
36428
38890
|
}
|
|
36429
38891
|
for (const component of manifest.components ?? []) {
|
|
36430
38892
|
if (UNSUPPORTED_COMPONENTS.includes(component)) {
|
|
36431
|
-
const
|
|
38893
|
+
const count2 = manifest.counts?.[component];
|
|
36432
38894
|
const key = component === "communities" ? "communities" : component;
|
|
36433
|
-
skipped[key] = (skipped[key] ?? 0) + (typeof
|
|
38895
|
+
skipped[key] = (skipped[key] ?? 0) + (typeof count2 === "number" ? count2 : 0);
|
|
36434
38896
|
warnings.push(
|
|
36435
38897
|
`Shard component '${component}' is not supported by the canonical record tier and was skipped. Import into a PGlite-backed store to preserve it.`
|
|
36436
38898
|
);
|
|
@@ -36598,7 +39060,7 @@ async function importShardToRecords(store, data, options) {
|
|
|
36598
39060
|
id: existingOriginal?.id ?? generateId(),
|
|
36599
39061
|
note_id: note.id,
|
|
36600
39062
|
content: note.original_content,
|
|
36601
|
-
content_hash: computeHash(
|
|
39063
|
+
content_hash: computeHash(encoder6.encode(note.original_content)),
|
|
36602
39064
|
created_at: createdAt2
|
|
36603
39065
|
};
|
|
36604
39066
|
mutations.push({ op: "put", collection: "note_original", record: originalRecord });
|
|
@@ -36830,9 +39292,302 @@ async function importShardToRecords(store, data, options) {
|
|
|
36830
39292
|
};
|
|
36831
39293
|
}
|
|
36832
39294
|
|
|
39295
|
+
// src/records/source-upsert.ts
|
|
39296
|
+
function now() {
|
|
39297
|
+
return (/* @__PURE__ */ new Date()).toISOString();
|
|
39298
|
+
}
|
|
39299
|
+
function contentDigest2(content) {
|
|
39300
|
+
return computeHash(new TextEncoder().encode(content));
|
|
39301
|
+
}
|
|
39302
|
+
function sourceHash2(source) {
|
|
39303
|
+
return computeHash(new TextEncoder().encode([
|
|
39304
|
+
source.tenant_id ?? "default",
|
|
39305
|
+
source.archive_id ?? "",
|
|
39306
|
+
source.namespace,
|
|
39307
|
+
source.external_id
|
|
39308
|
+
].join("\0")));
|
|
39309
|
+
}
|
|
39310
|
+
function countOutcomes2(outcomes) {
|
|
39311
|
+
return {
|
|
39312
|
+
inserted: outcomes.filter((outcome) => outcome.outcome === "inserted").length,
|
|
39313
|
+
unchanged: outcomes.filter((outcome) => outcome.outcome === "unchanged").length,
|
|
39314
|
+
versioned: outcomes.filter((outcome) => outcome.outcome === "versioned").length,
|
|
39315
|
+
replaced: outcomes.filter((outcome) => outcome.outcome === "replaced").length,
|
|
39316
|
+
conflict: outcomes.filter((outcome) => outcome.outcome === "conflict").length,
|
|
39317
|
+
rejected: outcomes.filter((outcome) => outcome.outcome === "rejected").length
|
|
39318
|
+
};
|
|
39319
|
+
}
|
|
39320
|
+
async function findSource(store, source) {
|
|
39321
|
+
const identities = await store.list("source_identity");
|
|
39322
|
+
return identities.find((identity) => identity.tenant_id === (source.tenant_id ?? "default") && identity.archive_id === (source.archive_id ?? null) && identity.namespace === source.namespace && identity.external_id === source.external_id) ?? null;
|
|
39323
|
+
}
|
|
39324
|
+
async function upsertRecordStoreSources(store, items, options = {}) {
|
|
39325
|
+
const maxItems = options.maxItems ?? 500;
|
|
39326
|
+
if (items.length > maxItems) throw new Error(`Source upsert batch exceeds the ${maxItems} item bound`);
|
|
39327
|
+
if (!store.applyBatch) throw new Error("RecordStore source upsert requires atomic applyBatch() support");
|
|
39328
|
+
if (items.length === 0) {
|
|
39329
|
+
return {
|
|
39330
|
+
import_run_id: "",
|
|
39331
|
+
dry_run: options.dryRun === true,
|
|
39332
|
+
outcomes: [],
|
|
39333
|
+
counts: { inserted: 0, unchanged: 0, versioned: 0, replaced: 0, conflict: 0, rejected: 0 }
|
|
39334
|
+
};
|
|
39335
|
+
}
|
|
39336
|
+
const outcomes = [];
|
|
39337
|
+
const mutations = [];
|
|
39338
|
+
const stamp = now();
|
|
39339
|
+
for (const [index, item] of items.entries()) {
|
|
39340
|
+
const external_id_hash = sourceHash2(item.source);
|
|
39341
|
+
const content_digest = contentDigest2(item.content);
|
|
39342
|
+
const existing = await findSource(store, item.source);
|
|
39343
|
+
if (!existing) {
|
|
39344
|
+
const noteId = item.source.caller_stable_id ?? generateId();
|
|
39345
|
+
outcomes.push({ index, outcome: "inserted", note_id: noteId, external_id_hash, content_digest });
|
|
39346
|
+
if (!options.dryRun) {
|
|
39347
|
+
const note2 = {
|
|
39348
|
+
id: noteId,
|
|
39349
|
+
archive_id: item.source.archive_id ?? null,
|
|
39350
|
+
title: item.title ?? null,
|
|
39351
|
+
format: item.format ?? "markdown",
|
|
39352
|
+
source: `source:${item.source.namespace}`,
|
|
39353
|
+
visibility: item.visibility ?? "private",
|
|
39354
|
+
revision_mode: "standard",
|
|
39355
|
+
is_starred: false,
|
|
39356
|
+
is_pinned: false,
|
|
39357
|
+
is_archived: false,
|
|
39358
|
+
created_at: stamp,
|
|
39359
|
+
updated_at: stamp,
|
|
39360
|
+
deleted_at: null
|
|
39361
|
+
};
|
|
39362
|
+
const original = {
|
|
39363
|
+
id: generateId(),
|
|
39364
|
+
note_id: noteId,
|
|
39365
|
+
content: item.content,
|
|
39366
|
+
content_hash: content_digest,
|
|
39367
|
+
created_at: stamp
|
|
39368
|
+
};
|
|
39369
|
+
const current2 = {
|
|
39370
|
+
id: noteId,
|
|
39371
|
+
content: item.content,
|
|
39372
|
+
ai_metadata: item.metadata ?? null,
|
|
39373
|
+
generation_count: 0,
|
|
39374
|
+
model: null,
|
|
39375
|
+
is_user_edited: false,
|
|
39376
|
+
updated_at: stamp
|
|
39377
|
+
};
|
|
39378
|
+
const identity = {
|
|
39379
|
+
id: generateId(),
|
|
39380
|
+
tenant_id: item.source.tenant_id ?? "default",
|
|
39381
|
+
archive_id: item.source.archive_id ?? null,
|
|
39382
|
+
namespace: item.source.namespace,
|
|
39383
|
+
external_id: item.source.external_id,
|
|
39384
|
+
external_id_hash,
|
|
39385
|
+
source_schema_version: item.source.source_schema_version,
|
|
39386
|
+
content_digest,
|
|
39387
|
+
import_run_id: item.source.import_run_id,
|
|
39388
|
+
caller_stable_id: item.source.caller_stable_id ?? null,
|
|
39389
|
+
note_id: noteId,
|
|
39390
|
+
created_at: stamp,
|
|
39391
|
+
updated_at: stamp
|
|
39392
|
+
};
|
|
39393
|
+
mutations.push(
|
|
39394
|
+
{ op: "put", collection: "note", record: note2 },
|
|
39395
|
+
{ op: "put", collection: "note_original", record: original },
|
|
39396
|
+
{ op: "put", collection: "note_revised_current", record: current2 },
|
|
39397
|
+
{ op: "put", collection: "source_identity", record: identity }
|
|
39398
|
+
);
|
|
39399
|
+
}
|
|
39400
|
+
continue;
|
|
39401
|
+
}
|
|
39402
|
+
if (existing.content_digest === content_digest) {
|
|
39403
|
+
outcomes.push({ index, outcome: "unchanged", note_id: existing.note_id, external_id_hash, content_digest });
|
|
39404
|
+
continue;
|
|
39405
|
+
}
|
|
39406
|
+
const policy = item.policy ?? "version";
|
|
39407
|
+
if (policy === "conflict") {
|
|
39408
|
+
outcomes.push({ index, outcome: "conflict", note_id: existing.note_id, external_id_hash, content_digest });
|
|
39409
|
+
continue;
|
|
39410
|
+
}
|
|
39411
|
+
const note = await store.get("note", existing.note_id);
|
|
39412
|
+
const current = await store.get("note_revised_current", existing.note_id);
|
|
39413
|
+
if (!note || !current) {
|
|
39414
|
+
outcomes.push({ index, outcome: "rejected", note_id: existing.note_id, external_id_hash, content_digest, reason: "source identity points to a missing note" });
|
|
39415
|
+
continue;
|
|
39416
|
+
}
|
|
39417
|
+
const outcome = policy === "replace" ? "replaced" : "versioned";
|
|
39418
|
+
outcomes.push({ index, outcome, note_id: existing.note_id, external_id_hash, content_digest });
|
|
39419
|
+
if (!options.dryRun) {
|
|
39420
|
+
mutations.push(
|
|
39421
|
+
{
|
|
39422
|
+
op: "put",
|
|
39423
|
+
collection: "note",
|
|
39424
|
+
record: {
|
|
39425
|
+
...note,
|
|
39426
|
+
title: item.title ?? null,
|
|
39427
|
+
archive_id: item.source.archive_id ?? null,
|
|
39428
|
+
format: item.format ?? "markdown",
|
|
39429
|
+
visibility: item.visibility ?? "private",
|
|
39430
|
+
deleted_at: null,
|
|
39431
|
+
updated_at: stamp
|
|
39432
|
+
}
|
|
39433
|
+
},
|
|
39434
|
+
{
|
|
39435
|
+
op: "put",
|
|
39436
|
+
collection: "note_revised_current",
|
|
39437
|
+
record: { ...current, content: item.content, ai_metadata: item.metadata ?? null, is_user_edited: false, updated_at: stamp }
|
|
39438
|
+
},
|
|
39439
|
+
{
|
|
39440
|
+
op: "put",
|
|
39441
|
+
collection: "source_identity",
|
|
39442
|
+
record: { ...existing, source_schema_version: item.source.source_schema_version, content_digest, import_run_id: item.source.import_run_id, updated_at: stamp }
|
|
39443
|
+
}
|
|
39444
|
+
);
|
|
39445
|
+
}
|
|
39446
|
+
}
|
|
39447
|
+
if (!options.dryRun && hasMaterialChange2(outcomes)) {
|
|
39448
|
+
const run = {
|
|
39449
|
+
id: items[0].source.import_run_id,
|
|
39450
|
+
tenant_id: items[0].source.tenant_id ?? "default",
|
|
39451
|
+
archive_id: items[0].source.archive_id ?? null,
|
|
39452
|
+
namespace: items[0].source.namespace,
|
|
39453
|
+
started_at: stamp,
|
|
39454
|
+
completed_at: stamp,
|
|
39455
|
+
checkpoint: { item_count: items.length },
|
|
39456
|
+
receipt: { counts: countOutcomes2(outcomes) }
|
|
39457
|
+
};
|
|
39458
|
+
mutations.push({ op: "put", collection: "source_import_run", record: run });
|
|
39459
|
+
await store.applyBatch(mutations);
|
|
39460
|
+
}
|
|
39461
|
+
return {
|
|
39462
|
+
import_run_id: items[0].source.import_run_id,
|
|
39463
|
+
dry_run: options.dryRun === true,
|
|
39464
|
+
outcomes,
|
|
39465
|
+
counts: countOutcomes2(outcomes)
|
|
39466
|
+
};
|
|
39467
|
+
}
|
|
39468
|
+
function hasMaterialChange2(outcomes) {
|
|
39469
|
+
return outcomes.some((outcome) => outcome.outcome === "inserted" || outcome.outcome === "versioned" || outcome.outcome === "replaced");
|
|
39470
|
+
}
|
|
39471
|
+
|
|
39472
|
+
// src/records/lifecycle-purge.ts
|
|
39473
|
+
function hashSelector(selector) {
|
|
39474
|
+
return computeHash(new TextEncoder().encode(JSON.stringify({
|
|
39475
|
+
tenant_id: selector.tenant_id ?? "default",
|
|
39476
|
+
archive_id: selector.archive_id ?? null,
|
|
39477
|
+
note_ids: [...selector.note_ids ?? []].sort(),
|
|
39478
|
+
source: selector.source ? {
|
|
39479
|
+
namespace: selector.source.namespace,
|
|
39480
|
+
external_id_hash: selector.source.external_id ? computeHash(new TextEncoder().encode(selector.source.external_id)) : null
|
|
39481
|
+
} : null
|
|
39482
|
+
})));
|
|
39483
|
+
}
|
|
39484
|
+
function zeroCounts2() {
|
|
39485
|
+
return {
|
|
39486
|
+
notes: 0,
|
|
39487
|
+
revisions: 0,
|
|
39488
|
+
links: 0,
|
|
39489
|
+
tags: 0,
|
|
39490
|
+
embeddings: 0,
|
|
39491
|
+
attachments: 0,
|
|
39492
|
+
blobs: 0,
|
|
39493
|
+
graph_edges: 0,
|
|
39494
|
+
provenance_edges: 0,
|
|
39495
|
+
source_identities: 0
|
|
39496
|
+
};
|
|
39497
|
+
}
|
|
39498
|
+
async function selectedNoteIds2(store, selector) {
|
|
39499
|
+
const notes = await store.list("note");
|
|
39500
|
+
if (selector.note_ids?.length) return notes.filter((note) => selector.note_ids.includes(note.id)).map((note) => note.id);
|
|
39501
|
+
if (!selector.source) throw new Error("Purge selector must target note_ids or source identity");
|
|
39502
|
+
const identities = await store.list("source_identity");
|
|
39503
|
+
return identities.filter((identity) => identity.tenant_id === (selector.tenant_id ?? "default") && identity.archive_id === (selector.archive_id ?? null) && identity.namespace === selector.source.namespace && (selector.source.external_id === void 0 || identity.external_id === selector.source.external_id)).map((identity) => identity.note_id);
|
|
39504
|
+
}
|
|
39505
|
+
async function count(store, noteIds) {
|
|
39506
|
+
const counts = zeroCounts2();
|
|
39507
|
+
if (noteIds.length === 0) return counts;
|
|
39508
|
+
const noteSet = new Set(noteIds);
|
|
39509
|
+
counts.notes = (await store.list("note")).filter((record) => noteSet.has(record.id)).length;
|
|
39510
|
+
counts.revisions = 0;
|
|
39511
|
+
counts.links = (await store.list("link")).filter((record) => noteSet.has(record.source_note_id) || noteSet.has(record.target_note_id)).length;
|
|
39512
|
+
counts.tags = (await store.list("note_tag")).filter((record) => noteSet.has(record.note_id)).length;
|
|
39513
|
+
counts.attachments = (await store.list("attachment")).filter((record) => noteSet.has(record.note_id)).length;
|
|
39514
|
+
const purgedBlobIds = new Set((await store.list("attachment")).filter((record) => noteSet.has(record.note_id)).map((record) => record.blob_id));
|
|
39515
|
+
counts.blobs = (await store.list("attachment_blob")).filter((record) => purgedBlobIds.has(record.id)).length;
|
|
39516
|
+
counts.source_identities = (await store.list("source_identity")).filter((record) => noteSet.has(record.note_id)).length;
|
|
39517
|
+
return counts;
|
|
39518
|
+
}
|
|
39519
|
+
async function previewRecordStorePurge(store, selector) {
|
|
39520
|
+
return { selector_hash: hashSelector(selector), counts: await count(store, await selectedNoteIds2(store, selector)) };
|
|
39521
|
+
}
|
|
39522
|
+
async function purgeRecordStoreGraph(store, selector, operationKey) {
|
|
39523
|
+
if (!store.applyBatch) throw new Error("RecordStore purge requires atomic applyBatch() support");
|
|
39524
|
+
const prior = (await store.list("deletion_receipt")).find((receipt2) => receipt2.operation_key === operationKey);
|
|
39525
|
+
if (prior) return {
|
|
39526
|
+
id: prior.id,
|
|
39527
|
+
operation_key: prior.operation_key,
|
|
39528
|
+
tenant_id: prior.tenant_id,
|
|
39529
|
+
archive_id: prior.archive_id,
|
|
39530
|
+
selector_hash: prior.selector_hash,
|
|
39531
|
+
outcome: "completed",
|
|
39532
|
+
counts: prior.counts,
|
|
39533
|
+
completed_at: prior.completed_at,
|
|
39534
|
+
policy: prior.policy
|
|
39535
|
+
};
|
|
39536
|
+
const noteIds = await selectedNoteIds2(store, selector);
|
|
39537
|
+
const noteSet = new Set(noteIds);
|
|
39538
|
+
const counts = await count(store, noteIds);
|
|
39539
|
+
const mutations = [];
|
|
39540
|
+
for (const collection of ["note_revised_current", "note_original", "note_tag", "collection_note", "attachment", "source_identity"]) {
|
|
39541
|
+
for (const record of await store.list(collection)) {
|
|
39542
|
+
const noteId = collection === "note_revised_current" ? record.id : "note_id" in record && typeof record.note_id === "string" ? record.note_id : null;
|
|
39543
|
+
if (noteId && noteSet.has(noteId)) {
|
|
39544
|
+
mutations.push({ op: "delete", collection, id: record.id });
|
|
39545
|
+
}
|
|
39546
|
+
}
|
|
39547
|
+
}
|
|
39548
|
+
for (const record of await store.list("link")) {
|
|
39549
|
+
if (noteSet.has(record.source_note_id) || noteSet.has(record.target_note_id)) mutations.push({ op: "delete", collection: "link", id: record.id });
|
|
39550
|
+
}
|
|
39551
|
+
for (const record of await store.list("note")) {
|
|
39552
|
+
if (noteSet.has(record.id)) mutations.push({ op: "delete", collection: "note", id: record.id });
|
|
39553
|
+
}
|
|
39554
|
+
const liveBlobIds = new Set((await store.list("attachment")).filter((record) => !noteSet.has(record.note_id)).map((record) => record.blob_id));
|
|
39555
|
+
for (const record of await store.list("attachment_blob")) {
|
|
39556
|
+
if (!liveBlobIds.has(record.id)) mutations.push({ op: "delete", collection: "attachment_blob", id: record.id });
|
|
39557
|
+
}
|
|
39558
|
+
const receipt = {
|
|
39559
|
+
id: generateId(),
|
|
39560
|
+
operation_key: operationKey,
|
|
39561
|
+
tenant_id: selector.tenant_id ?? "default",
|
|
39562
|
+
archive_id: selector.archive_id ?? null,
|
|
39563
|
+
selector_hash: hashSelector(selector),
|
|
39564
|
+
outcome: "completed",
|
|
39565
|
+
counts,
|
|
39566
|
+
completed_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
39567
|
+
policy: {
|
|
39568
|
+
authority: "fortemi#1092",
|
|
39569
|
+
mode: "terminal-purge",
|
|
39570
|
+
receipt_contains_content: false
|
|
39571
|
+
}
|
|
39572
|
+
};
|
|
39573
|
+
mutations.push({ op: "put", collection: "deletion_receipt", record: receipt });
|
|
39574
|
+
await store.applyBatch(mutations);
|
|
39575
|
+
return {
|
|
39576
|
+
id: receipt.id,
|
|
39577
|
+
operation_key: receipt.operation_key,
|
|
39578
|
+
tenant_id: receipt.tenant_id,
|
|
39579
|
+
archive_id: receipt.archive_id,
|
|
39580
|
+
selector_hash: receipt.selector_hash,
|
|
39581
|
+
outcome: "completed",
|
|
39582
|
+
counts,
|
|
39583
|
+
completed_at: receipt.completed_at,
|
|
39584
|
+
policy: receipt.policy
|
|
39585
|
+
};
|
|
39586
|
+
}
|
|
39587
|
+
|
|
36833
39588
|
// src/index.ts
|
|
36834
|
-
var VERSION = "2026.
|
|
39589
|
+
var VERSION = "2026.9.0";
|
|
36835
39590
|
|
|
36836
|
-
export { AIWG_SCAN_REQUIRED_FIELDS, AllowlistTrustStore, ArchiveManager, AttachmentsRepository, CORE_V1_COMPONENTS, CURRENT_MIGRATION_HEAD, CURRENT_SHARD_VERSION, CanonicalAttachmentsRepository, CanonicalNotesRepository, CapabilityManager, CaptureKnowledgeInputSchema, CollectionsRepository, CommunitiesRepository, DB_SNAPSHOT_SCHEMA_VERSION, DbSnapshotVersionError, EMBED_REQUEST_KIND, EMBED_RESPONSE_KIND, EmbeddingSetsRepository, FORTEMI_COMPATIBILITY_PATH, FORTEMI_COMPATIBILITY_STATES, FORTEMI_REQUIRED_COMPATIBILITY_CAPABILITIES, FORTEMI_SERVER_COMPATIBILITY_REVISION, FallbackRouter, FortemiToolManifest, GetNoteInputSchema, GraphRepository, IdbRecordStore, JOB_CAPABILITIES, JOB_PRIORITIES, JobQueueWorker, LOCAL_ENDPOINTS, LinksRepository, ListNotesInputSchema, ManageArchiveInputSchema, ManageAttachmentsInputSchema, ManageCapabilitiesInputSchema, ManageCollectionsInputSchema, ManageLinksInputSchema, ManageNoteInputSchema, ManageTagsInputSchema, MemoryBlobStore, MemoryRecordStore, MigrationRunner, NotesRepository, OpenAICompatibleProvider, PGliteStorageBackend, PGliteStorageBackendFactory, PGliteWorkerClient, PGliteWorkerStorageBackend, PGliteWorkerStorageBackendFactory, ProvenanceRepository, ProviderRegistry, RECORD_COLLECTIONS, RECORD_SCHEMA_VERSION, RECORD_STORE_CAPABILITIES, SHARD_FORMAT, SIGNATURE_ENTRY, SIGNING_ENVELOPE_VERSION, SUPPORTED_PGLITE_VERSION, SearchInputSchema, SearchRepository, SkosRepository, TagsRepository, TransactionProxy, TypedEventBus, VERSION, aiRevisionHandler, aiwgFortemiIndexFromKnowledgeShard, aiwgFortemiIndexToCommunityGraph, aiwgFortemiIndexToKnowledgeShard, aiwgFortemiIndexToKnowledgeShardWithReport, allMigrations, appendPluginScript, assertAiwgFortemiChunkManifest, assertAiwgFortemiChunkPart, assertAiwgFortemiIndexExport, assertAiwgStaticEmbeddingSet, assertShardComponentRecord, buildAiwgChunkedIndex, buildAiwgStaticEmbeddingSet, buildNoteConditions, buildPluginCsp, captureKnowledge, chunkText, classifyError, classifyModel, clearPrefetchedShard, collectionFromShard, collectionToShard, computeBlobHash, computeHash, computeSri, conceptTaggingHandler, cosineSimilarity, createAiwgFetchChunkLoader, createAiwgFetchDetailLoader, createAiwgIndexController, createAiwgReviewDecisionExport, createBlobStore, createCosineSemanticProvider, createCspReportHandler, createFortemi, createLazyBlobStore, createLegacyProvider, createPGliteBackend, createPGliteInstance, createRecordBackend, createRecordStore, createRemoteBackend, createRoutes, createShardBackend, createShardCapabilityReport, createWorkerEmbedFunction, defaultStorageBackendFactory, detectCommunities, detectGpuCapabilities, detectInferenceCapabilities, discoverLocalProviders, dropAttachmentProjection, dropNoteProjection, dumpDbSnapshot, embeddingConfigToShard, embeddingFromShard, embeddingGenerationHandler, embeddingSetFromShard, embeddingSetMemberToShard, embeddingSetToShard, embeddingToShard, enqueueFullWorkflow, enqueueJob, enqueueNoteCreationJobs, estimateModelFit, estimateVramMB, estimateVramTier, exportShard, exportShardFromRecords, exportShardFromRecordsWithReport, exportShardWithReport, fetchAndValidateFortemiCompatibility, fetchPluginScript, filterAiwgRecordsByPrivacy, findAiwgStaticDuplicatePairs, formatFortemiCompatibilitySummary, fortemiCompatibilityUrl, fortemiManifest, fromPrefetched, generateId, getAiwgFortemiFacets, getAiwgFortemiIndexExportSchema, getEmbedFunction, getFortemiBridge, getFortemiSecretStore, getJobQueueStatus, getKnowledgeShardContractReceipt, getKnowledgeShardProfileRegistry, getKnowledgeShardSchema, getLlmFunction, getNote, getPrefetchedSha256, handleEmbedRequests, hasFortemiSecureSecrets, importShard, importShardToRecords, isPluginScriptAllowed, isShardPrefetched, isShardSigningSupported, linkFromShard, linkToShard, linkingHandler, listNotes, manageArchive, manageAttachments, manageCapabilities, manageCollections, manageLinks, manageNote, manageTags, matchRoute, migrateLegacyBlobStore, noteFromShard, noteSkosTagToShard, noteToShard, openShard, packTarGz, parseCspReport, prefetchShard, profileSupportError, projectAttachments, projectNotes, projectRecords, provenanceEdgeToShard, queryAiwgFortemiIndex, queryAiwgHybridIndex, queryAiwgSemanticIndex, registerLlmCapability, registerSemanticCapability, registerSemanticCapabilityWorker, registerServiceWorker, restoreDbSnapshot, searchTool, selectBackend, selectLlmModel, setEmbedFunction, setLlmFunction, sha256Hex, sidecarBlobDigests, signShard, skosConceptToShard, skosRelationToShard, skosSchemeToShard, suggestTags, tagsToShard, templateToShard, titleGenerationHandler, unpackTarGz, unregisterLlmCapability, unregisterSemanticCapability, urlLinkToShard, validateAiwgFortemiChunkManifest, validateAiwgFortemiChunkPart, validateAiwgFortemiIndexExport, validateAiwgFortemiIndexExportSchema, validateAiwgFortemiProjectedRecordSchema, validateAiwgStaticEmbeddingSet, validateChecksums, validateCoreV1ShardArchive, validateFortemiCompatibilityResponse, validateFullV1ShardArchive, validateRecordV1ShardArchive, validateShardArchive, validateShardComponentRecord, validateShardManifest, verifyDbSnapshotMeta, verifyShardSignature, verifySri };
|
|
39591
|
+
export { AIWG_SCAN_REQUIRED_FIELDS, AllowlistTrustStore, ArchiveManager, AttachmentsRepository, BridgeInferenceProvider, CORE_V1_COMPONENTS, CURRENT_MIGRATION_HEAD, CURRENT_SHARD_VERSION, CanonicalAttachmentsRepository, CanonicalNotesRepository, CapabilityManager, CaptureKnowledgeInputSchema, CollectionsRepository, CommunitiesRepository, DATASET_EXECUTION_CAPABILITY_IDS, DATASET_EXECUTION_CONTRACT, DATASET_EXECUTION_SCHEMA_VERSION, DATASET_INGEST_CONTRACT, DATASET_INGEST_SCHEMA_VERSION, DATASET_LINEAGE_CONTRACT, DATASET_LINEAGE_SCHEMA_VERSION, DATASET_MATERIALIZATION_CONTRACT, DATASET_MATERIALIZATION_KINDS, DATASET_MATERIALIZATION_SCHEMA_VERSION, DB_SNAPSHOT_SCHEMA_VERSION, DEFAULT_LARGE_DOCUMENT_CHARS, DEFAULT_LARGE_DOCUMENT_CHUNKS, DatasetIngestError, DatasetIngestExecutor, DatasetLineageLedger, DatasetMaterializationError, DbSnapshotVersionError, EMBED_REQUEST_KIND, EMBED_RESPONSE_KIND, EmbeddingSetsRepository, FORTEMI_BROWSER_LOCAL_DATASET_EXECUTION_DESCRIPTOR, FORTEMI_COMPATIBILITY_PATH, FORTEMI_COMPATIBILITY_STATES, FORTEMI_PORTABLE_SHARD_DATASET_EXECUTION_DESCRIPTOR, FORTEMI_REQUIRED_COMPATIBILITY_CAPABILITIES, FORTEMI_SERVER_COMPATIBILITY_REVISION, FORTEMI_STATIC_CACHE_DATASET_EXECUTION_DESCRIPTOR, FallbackRouter, FortemiToolManifest, GetNoteInputSchema, GraphRepository, IdbRecordStore, JOB_CAPABILITIES, JOB_PRIORITIES, JobQueueWorker, LINEAGE_ENTITY_KINDS, LINEAGE_RELATIONSHIP_KINDS, LOCAL_ENDPOINTS, LifecyclePurgeRepository, LineageValidationError, LinksRepository, ListNotesInputSchema, ManageArchiveInputSchema, ManageAttachmentsInputSchema, ManageCapabilitiesInputSchema, ManageCollectionsInputSchema, ManageLinksInputSchema, ManageNoteInputSchema, ManageTagsInputSchema, MemoryBlobStore, MemoryDatasetIngestStore, MemoryRecordStore, MigrationRunner, NotesRepository, OpenAICompatibleProvider, PGliteStorageBackend, PGliteStorageBackendFactory, PGliteWorkerClient, PGliteWorkerStorageBackend, PGliteWorkerStorageBackendFactory, ProvenanceRepository, ProviderRegistry, RECORD_COLLECTIONS, RECORD_SCHEMA_VERSION, RECORD_STORE_CAPABILITIES, REGISTERED_METADATA_PATHS, SHARD_FORMAT, SIGNATURE_ENTRY, SIGNING_ENVELOPE_VERSION, SUPPORTED_PGLITE_VERSION, SearchInputSchema, SearchRepository, SkosRepository, SourceUpsertRepository, TagsRepository, TransactionProxy, TypedEventBus, VERSION, aiRevisionHandler, aiwgFortemiIndexFromKnowledgeShard, aiwgFortemiIndexToCommunityGraph, aiwgFortemiIndexToKnowledgeShard, aiwgFortemiIndexToKnowledgeShardWithReport, allMigrations, appendPluginScript, assertAiwgFortemiChunkManifest, assertAiwgFortemiChunkPart, assertAiwgFortemiIndexExport, assertAiwgStaticEmbeddingSet, assertShardComponentRecord, buildAiwgChunkedIndex, buildAiwgStaticEmbeddingSet, buildMetadataPredicateConditions, buildNoteConditions, buildPluginCsp, captureKnowledge, chunkText, classifyError, classifyModel, clearPrefetchedShard, collectionFromShard, collectionToShard, compareDatasetIncrementalParity, computeBlobHash, computeHash, computeLineageDigest, computeSri, conceptTaggingHandler, configureInferenceRuntime, cosineSimilarity, createAiwgFetchChunkLoader, createAiwgFetchDetailLoader, createAiwgIndexController, createAiwgReviewDecisionExport, createBlobStore, createBridgeInferenceProviders, createCosineSemanticProvider, createCspReportHandler, createFortemi, createLazyBlobStore, createLegacyProvider, createLocalProviderProfile, createPGliteBackend, createPGliteInstance, createRecordBackend, createRecordStore, createRemoteBackend, createRoutes, createShardBackend, createShardCapabilityReport, createWorkerEmbedFunction, datasetDestinationScopeKey, defaultStorageBackendFactory, defineInferenceProvider, defineInferenceRuntime, defineLegacyInferenceProvider, defineOpenAICompatibleProvider, deriveDatasetIngestIdempotencyKey, detectCommunities, detectGpuCapabilities, detectInferenceCapabilities, digestDatasetMaterializationValue, discoverLocalProviders, dropAttachmentProjection, dropNoteProjection, dumpDbSnapshot, embeddingConfigToShard, embeddingFromShard, embeddingGenerationHandler, embeddingSetFromShard, embeddingSetMemberToShard, embeddingSetToShard, embeddingToShard, enqueueFullWorkflow, enqueueJob, enqueueNoteCreationJobs, estimateModelFit, estimateVramMB, estimateVramTier, executeDatasetMaterialization, executeDatasetRetrieval, exportShard, exportShardFromRecords, exportShardFromRecordsWithReport, exportShardWithReport, fetchAndValidateFortemiCompatibility, fetchPluginScript, filterAiwgRecordsByPrivacy, findAiwgStaticDuplicatePairs, formatFortemiCompatibilitySummary, fortemiCompatibilityUrl, fortemiManifest, fromPrefetched, generateId, getAiwgFortemiFacets, getAiwgFortemiIndexExportSchema, getConfiguredInferenceProviderId, getEmbedFunction, getEmbeddingTaskSelectionOptions, getFortemiBridge, getFortemiSecretStore, getJobQueueStatus, getKnowledgeShardContractReceipt, getKnowledgeShardProfileRegistry, getKnowledgeShardSchema, getLlmFunction, getNote, getPrefetchedSha256, getProviderRouteRequirementIssue, handleEmbedRequests, hasFortemiSecureSecrets, importShard, importShardToRecords, inferInferenceTaskCapability, inferLocalEmbeddingDimensions, isPluginScriptAllowed, isShardPrefetched, isShardSigningSupported, linkFromShard, linkToShard, linkingHandler, listNotes, manageArchive, manageAttachments, manageCapabilities, manageCollections, manageLinks, manageNote, manageTags, matchRoute, mergeInferenceRuntimeConfigs, migrateLegacyBlobStore, negotiateDatasetExecutionCapabilities, negotiateDatasetMaterializationProfile, noteFromShard, noteSkosTagToShard, noteToShard, openShard, packTarGz, parseCspReport, prefetchShard, previewRecordStorePurge, profileSupportError, projectAttachments, projectNotes, projectRecords, provenanceEdgeToShard, providerSatisfiesRouteRequirements, purgeRecordStoreGraph, queryAiwgFortemiIndex, queryAiwgHybridIndex, queryAiwgSemanticIndex, registerLlmCapability, registerSemanticCapability, registerSemanticCapabilityWorker, registerServiceWorker, restoreDbSnapshot, searchTool, selectBackend, selectEmbeddingTask, selectLlmModel, setEmbedFunction, setEmbeddingTaskSelectionOptions, setLlmFunction, sha256Hex, sidecarBlobDigests, signShard, skosConceptToShard, skosRelationToShard, skosSchemeToShard, suggestTags, tagsToShard, templateToShard, titleGenerationHandler, unpackTarGz, unregisterLlmCapability, unregisterSemanticCapability, upsertRecordStoreSources, urlLinkToShard, validateAiwgFortemiChunkManifest, validateAiwgFortemiChunkPart, validateAiwgFortemiIndexExport, validateAiwgFortemiIndexExportSchema, validateAiwgFortemiProjectedRecordSchema, validateAiwgStaticEmbeddingSet, validateChecksums, validateCoreV1ShardArchive, validateDatasetBenchmarkEvidence, validateDatasetExecutionDescriptor, validateFortemiCompatibilityResponse, validateFullV1ShardArchive, validateProviderRoute, validateRecordV1ShardArchive, validateShardArchive, validateShardComponentRecord, validateShardManifest, verifyDbSnapshotMeta, verifyShardSignature, verifySri };
|
|
36837
39592
|
//# sourceMappingURL=index.js.map
|
|
36838
39593
|
//# sourceMappingURL=index.js.map
|