@fortemi/core 2026.6.0 → 2026.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +60 -4
- package/dist/aiwg-index.d.ts +212 -0
- package/dist/aiwg-index.js +550 -0
- package/dist/aiwg-index.js.map +1 -0
- package/dist/index.d.ts +182 -163
- package/dist/index.js +976 -171
- package/dist/index.js.map +1 -1
- package/package.json +5 -1
package/dist/index.js
CHANGED
|
@@ -150,6 +150,103 @@ async function createPGliteInstance(persistence, archiveName = "default") {
|
|
|
150
150
|
return db;
|
|
151
151
|
}
|
|
152
152
|
|
|
153
|
+
// src/worker/worker-client.ts
|
|
154
|
+
var PGliteWorkerClient = class {
|
|
155
|
+
constructor(worker) {
|
|
156
|
+
this.worker = worker;
|
|
157
|
+
this.readyPromise = new Promise((resolve) => {
|
|
158
|
+
this.resolveReady = resolve;
|
|
159
|
+
});
|
|
160
|
+
this.worker.addEventListener("message", (e) => {
|
|
161
|
+
const msg = e.data;
|
|
162
|
+
if (msg.type === "READY") {
|
|
163
|
+
this.resolveReady();
|
|
164
|
+
return;
|
|
165
|
+
}
|
|
166
|
+
if (!("id" in msg)) return;
|
|
167
|
+
const pending = this.pending.get(msg.id);
|
|
168
|
+
if (!pending) return;
|
|
169
|
+
this.pending.delete(msg.id);
|
|
170
|
+
if (msg.type === "ERROR") {
|
|
171
|
+
pending.reject(new Error(msg.error));
|
|
172
|
+
} else {
|
|
173
|
+
pending.resolve(msg);
|
|
174
|
+
}
|
|
175
|
+
});
|
|
176
|
+
}
|
|
177
|
+
pending = /* @__PURE__ */ new Map();
|
|
178
|
+
readyPromise;
|
|
179
|
+
resolveReady;
|
|
180
|
+
/** Resolves when the worker broadcasts READY after database initialisation. */
|
|
181
|
+
async waitReady() {
|
|
182
|
+
return this.readyPromise;
|
|
183
|
+
}
|
|
184
|
+
send(request) {
|
|
185
|
+
const id = generateId();
|
|
186
|
+
return new Promise((resolve, reject) => {
|
|
187
|
+
this.pending.set(id, {
|
|
188
|
+
resolve,
|
|
189
|
+
reject
|
|
190
|
+
});
|
|
191
|
+
this.worker.postMessage({ ...request, id });
|
|
192
|
+
});
|
|
193
|
+
}
|
|
194
|
+
async query(sql, params) {
|
|
195
|
+
const resp = await this.send({ type: "QUERY", sql, params });
|
|
196
|
+
return { rows: resp.rows, fields: resp.fields };
|
|
197
|
+
}
|
|
198
|
+
async exec(sql) {
|
|
199
|
+
await this.send({ type: "EXEC", sql });
|
|
200
|
+
}
|
|
201
|
+
async transaction(fn) {
|
|
202
|
+
const resp = await this.send({ type: "BEGIN" });
|
|
203
|
+
const txId = resp.txId;
|
|
204
|
+
const proxy = new TransactionProxy(this, txId);
|
|
205
|
+
try {
|
|
206
|
+
const result = await fn(proxy);
|
|
207
|
+
await this.send({ type: "COMMIT", txId });
|
|
208
|
+
return result;
|
|
209
|
+
} catch (err) {
|
|
210
|
+
await this.send({ type: "ROLLBACK", txId }).catch(() => {
|
|
211
|
+
});
|
|
212
|
+
throw err;
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
/** Forward TX_QUERY for TransactionProxy — not part of the public surface. */
|
|
216
|
+
async _txQuery(txId, sql, params) {
|
|
217
|
+
const resp = await this.send({
|
|
218
|
+
type: "TX_QUERY",
|
|
219
|
+
txId,
|
|
220
|
+
sql,
|
|
221
|
+
params
|
|
222
|
+
});
|
|
223
|
+
return { rows: resp.rows };
|
|
224
|
+
}
|
|
225
|
+
/** Forward TX_EXEC for TransactionProxy — not part of the public surface. */
|
|
226
|
+
async _txExec(txId, sql) {
|
|
227
|
+
await this.send({ type: "TX_EXEC", txId, sql });
|
|
228
|
+
}
|
|
229
|
+
async ping() {
|
|
230
|
+
await this.send({ type: "PING" });
|
|
231
|
+
}
|
|
232
|
+
async close() {
|
|
233
|
+
await this.send({ type: "CLOSE" });
|
|
234
|
+
this.worker.terminate();
|
|
235
|
+
}
|
|
236
|
+
};
|
|
237
|
+
var TransactionProxy = class {
|
|
238
|
+
constructor(client, txId) {
|
|
239
|
+
this.client = client;
|
|
240
|
+
this.txId = txId;
|
|
241
|
+
}
|
|
242
|
+
async query(sql, params) {
|
|
243
|
+
return this.client._txQuery(this.txId, sql, params);
|
|
244
|
+
}
|
|
245
|
+
async exec(sql) {
|
|
246
|
+
return this.client._txExec(this.txId, sql);
|
|
247
|
+
}
|
|
248
|
+
};
|
|
249
|
+
|
|
153
250
|
// src/storage-backend.ts
|
|
154
251
|
var PGliteStorageBackend = class {
|
|
155
252
|
constructor(id, db) {
|
|
@@ -177,6 +274,41 @@ var PGliteStorageBackendFactory = class {
|
|
|
177
274
|
}
|
|
178
275
|
};
|
|
179
276
|
var defaultStorageBackendFactory = new PGliteStorageBackendFactory();
|
|
277
|
+
var PGliteWorkerStorageBackend = class {
|
|
278
|
+
constructor(id, client) {
|
|
279
|
+
this.id = id;
|
|
280
|
+
this.client = client;
|
|
281
|
+
}
|
|
282
|
+
mode = "readwrite";
|
|
283
|
+
query(sql, params) {
|
|
284
|
+
return this.client.query(sql, params);
|
|
285
|
+
}
|
|
286
|
+
exec(sql) {
|
|
287
|
+
return this.client.exec(sql);
|
|
288
|
+
}
|
|
289
|
+
transaction(fn) {
|
|
290
|
+
return this.client.transaction(fn);
|
|
291
|
+
}
|
|
292
|
+
close() {
|
|
293
|
+
return this.client.close();
|
|
294
|
+
}
|
|
295
|
+
};
|
|
296
|
+
var PGliteWorkerStorageBackendFactory = class {
|
|
297
|
+
constructor(options) {
|
|
298
|
+
this.options = options;
|
|
299
|
+
}
|
|
300
|
+
async open(input) {
|
|
301
|
+
const worker = this.options.createWorker();
|
|
302
|
+
const client = new PGliteWorkerClient(worker);
|
|
303
|
+
worker.postMessage({
|
|
304
|
+
type: "INIT",
|
|
305
|
+
persistence: input.persistence,
|
|
306
|
+
archiveName: input.archiveName
|
|
307
|
+
});
|
|
308
|
+
await client.waitReady();
|
|
309
|
+
return new PGliteWorkerStorageBackend(`pglite-worker:${input.persistence}:${input.archiveName}`, client);
|
|
310
|
+
}
|
|
311
|
+
};
|
|
180
312
|
|
|
181
313
|
// src/capability-manager.ts
|
|
182
314
|
var VALID_TRANSITIONS = {
|
|
@@ -785,6 +917,28 @@ var migration0008 = {
|
|
|
785
917
|
`
|
|
786
918
|
};
|
|
787
919
|
|
|
920
|
+
// src/migrations/0009_vector_selector_performance.ts
|
|
921
|
+
var migration0009 = {
|
|
922
|
+
version: 9,
|
|
923
|
+
name: "0009_vector_selector_performance",
|
|
924
|
+
sql: `
|
|
925
|
+
CREATE INDEX IF NOT EXISTS idx_embedding_set_kind ON embedding_set(kind);
|
|
926
|
+
CREATE INDEX IF NOT EXISTS idx_embedding_set_member_set ON embedding_set_member(embedding_set_id);
|
|
927
|
+
CREATE INDEX IF NOT EXISTS idx_embedding_set_member_note ON embedding_set_member(note_id);
|
|
928
|
+
CREATE INDEX IF NOT EXISTS idx_embedding_set_member_embedding ON embedding_set_member(embedding_id);
|
|
929
|
+
|
|
930
|
+
CREATE INDEX IF NOT EXISTS idx_note_source ON note(source);
|
|
931
|
+
CREATE INDEX IF NOT EXISTS idx_note_format ON note(format);
|
|
932
|
+
CREATE INDEX IF NOT EXISTS idx_note_visibility ON note(visibility);
|
|
933
|
+
CREATE INDEX IF NOT EXISTS idx_note_starred ON note(is_starred);
|
|
934
|
+
CREATE INDEX IF NOT EXISTS idx_note_archived ON note(is_archived);
|
|
935
|
+
CREATE INDEX IF NOT EXISTS idx_note_updated_at ON note(updated_at);
|
|
936
|
+
|
|
937
|
+
CREATE INDEX IF NOT EXISTS idx_note_revised_current_user_edited ON note_revised_current(is_user_edited);
|
|
938
|
+
CREATE INDEX IF NOT EXISTS idx_note_revised_current_generation_count ON note_revised_current(generation_count);
|
|
939
|
+
`
|
|
940
|
+
};
|
|
941
|
+
|
|
788
942
|
// src/migrations/index.ts
|
|
789
943
|
var allMigrations = [
|
|
790
944
|
migration0001,
|
|
@@ -794,18 +948,19 @@ var allMigrations = [
|
|
|
794
948
|
migration0005,
|
|
795
949
|
migration0006,
|
|
796
950
|
migration0007,
|
|
797
|
-
migration0008
|
|
951
|
+
migration0008,
|
|
952
|
+
migration0009
|
|
798
953
|
];
|
|
799
954
|
|
|
800
955
|
// src/archive-manager.ts
|
|
801
956
|
var ArchiveManager = class {
|
|
802
|
-
constructor(persistenceOrFactory, events) {
|
|
957
|
+
constructor(persistenceOrFactory, events, persistenceOverride) {
|
|
803
958
|
this.events = events;
|
|
804
959
|
if (typeof persistenceOrFactory === "string") {
|
|
805
960
|
this.persistence = persistenceOrFactory;
|
|
806
961
|
this.backendFactory = defaultStorageBackendFactory;
|
|
807
962
|
} else {
|
|
808
|
-
this.persistence = "memory";
|
|
963
|
+
this.persistence = persistenceOverride ?? "memory";
|
|
809
964
|
this.backendFactory = persistenceOrFactory;
|
|
810
965
|
}
|
|
811
966
|
this.archives.set("default", {
|
|
@@ -1170,103 +1325,6 @@ function createBlobStore(archiveName) {
|
|
|
1170
1325
|
return new IdbBlobStore(archiveName);
|
|
1171
1326
|
}
|
|
1172
1327
|
|
|
1173
|
-
// src/worker/worker-client.ts
|
|
1174
|
-
var PGliteWorkerClient = class {
|
|
1175
|
-
constructor(worker) {
|
|
1176
|
-
this.worker = worker;
|
|
1177
|
-
this.readyPromise = new Promise((resolve) => {
|
|
1178
|
-
this.resolveReady = resolve;
|
|
1179
|
-
});
|
|
1180
|
-
this.worker.addEventListener("message", (e) => {
|
|
1181
|
-
const msg = e.data;
|
|
1182
|
-
if (msg.type === "READY") {
|
|
1183
|
-
this.resolveReady();
|
|
1184
|
-
return;
|
|
1185
|
-
}
|
|
1186
|
-
if (!("id" in msg)) return;
|
|
1187
|
-
const pending = this.pending.get(msg.id);
|
|
1188
|
-
if (!pending) return;
|
|
1189
|
-
this.pending.delete(msg.id);
|
|
1190
|
-
if (msg.type === "ERROR") {
|
|
1191
|
-
pending.reject(new Error(msg.error));
|
|
1192
|
-
} else {
|
|
1193
|
-
pending.resolve(msg);
|
|
1194
|
-
}
|
|
1195
|
-
});
|
|
1196
|
-
}
|
|
1197
|
-
pending = /* @__PURE__ */ new Map();
|
|
1198
|
-
readyPromise;
|
|
1199
|
-
resolveReady;
|
|
1200
|
-
/** Resolves when the worker broadcasts READY after database initialisation. */
|
|
1201
|
-
async waitReady() {
|
|
1202
|
-
return this.readyPromise;
|
|
1203
|
-
}
|
|
1204
|
-
send(request) {
|
|
1205
|
-
const id = generateId();
|
|
1206
|
-
return new Promise((resolve, reject) => {
|
|
1207
|
-
this.pending.set(id, {
|
|
1208
|
-
resolve,
|
|
1209
|
-
reject
|
|
1210
|
-
});
|
|
1211
|
-
this.worker.postMessage({ ...request, id });
|
|
1212
|
-
});
|
|
1213
|
-
}
|
|
1214
|
-
async query(sql, params) {
|
|
1215
|
-
const resp = await this.send({ type: "QUERY", sql, params });
|
|
1216
|
-
return { rows: resp.rows, fields: resp.fields };
|
|
1217
|
-
}
|
|
1218
|
-
async exec(sql) {
|
|
1219
|
-
await this.send({ type: "EXEC", sql });
|
|
1220
|
-
}
|
|
1221
|
-
async transaction(fn) {
|
|
1222
|
-
const resp = await this.send({ type: "BEGIN" });
|
|
1223
|
-
const txId = resp.txId;
|
|
1224
|
-
const proxy = new TransactionProxy(this, txId);
|
|
1225
|
-
try {
|
|
1226
|
-
const result = await fn(proxy);
|
|
1227
|
-
await this.send({ type: "COMMIT", txId });
|
|
1228
|
-
return result;
|
|
1229
|
-
} catch (err) {
|
|
1230
|
-
await this.send({ type: "ROLLBACK", txId }).catch(() => {
|
|
1231
|
-
});
|
|
1232
|
-
throw err;
|
|
1233
|
-
}
|
|
1234
|
-
}
|
|
1235
|
-
/** Forward TX_QUERY for TransactionProxy — not part of the public surface. */
|
|
1236
|
-
async _txQuery(txId, sql, params) {
|
|
1237
|
-
const resp = await this.send({
|
|
1238
|
-
type: "TX_QUERY",
|
|
1239
|
-
txId,
|
|
1240
|
-
sql,
|
|
1241
|
-
params
|
|
1242
|
-
});
|
|
1243
|
-
return { rows: resp.rows };
|
|
1244
|
-
}
|
|
1245
|
-
/** Forward TX_EXEC for TransactionProxy — not part of the public surface. */
|
|
1246
|
-
async _txExec(txId, sql) {
|
|
1247
|
-
await this.send({ type: "TX_EXEC", txId, sql });
|
|
1248
|
-
}
|
|
1249
|
-
async ping() {
|
|
1250
|
-
await this.send({ type: "PING" });
|
|
1251
|
-
}
|
|
1252
|
-
async close() {
|
|
1253
|
-
await this.send({ type: "CLOSE" });
|
|
1254
|
-
this.worker.terminate();
|
|
1255
|
-
}
|
|
1256
|
-
};
|
|
1257
|
-
var TransactionProxy = class {
|
|
1258
|
-
constructor(client, txId) {
|
|
1259
|
-
this.client = client;
|
|
1260
|
-
this.txId = txId;
|
|
1261
|
-
}
|
|
1262
|
-
async query(sql, params) {
|
|
1263
|
-
return this.client._txQuery(this.txId, sql, params);
|
|
1264
|
-
}
|
|
1265
|
-
async exec(sql) {
|
|
1266
|
-
return this.client._txExec(this.txId, sql);
|
|
1267
|
-
}
|
|
1268
|
-
};
|
|
1269
|
-
|
|
1270
1328
|
// src/repositories/notes-repository.ts
|
|
1271
1329
|
var NotesRepository = class {
|
|
1272
1330
|
constructor(db, events) {
|
|
@@ -1690,6 +1748,9 @@ function dateString(value) {
|
|
|
1690
1748
|
function dateMillis(value) {
|
|
1691
1749
|
return value instanceof Date ? value.getTime() : new Date(value).getTime();
|
|
1692
1750
|
}
|
|
1751
|
+
function hashJson(value) {
|
|
1752
|
+
return computeHash(new TextEncoder().encode(JSON.stringify(value)));
|
|
1753
|
+
}
|
|
1693
1754
|
var EmbeddingSetsRepository = class {
|
|
1694
1755
|
constructor(db) {
|
|
1695
1756
|
this.db = db;
|
|
@@ -1737,7 +1798,12 @@ var EmbeddingSetsRepository = class {
|
|
|
1737
1798
|
input.updatedAt ?? null
|
|
1738
1799
|
]
|
|
1739
1800
|
);
|
|
1740
|
-
|
|
1801
|
+
const row = await this.get(id);
|
|
1802
|
+
if (input.materialization?.allowed) {
|
|
1803
|
+
await this.refreshMaterializedVirtualSet(id);
|
|
1804
|
+
return this.get(id);
|
|
1805
|
+
}
|
|
1806
|
+
return row;
|
|
1741
1807
|
}
|
|
1742
1808
|
async ensureDefault() {
|
|
1743
1809
|
const existing = await this.db.query(
|
|
@@ -1829,14 +1895,97 @@ var EmbeddingSetsRepository = class {
|
|
|
1829
1895
|
const set = await this.get(selector.embeddingSetId);
|
|
1830
1896
|
if (set.kind === "virtual") {
|
|
1831
1897
|
const definition = this.definitionFromRow(set);
|
|
1832
|
-
return this.resolveDefinition({ kind: "embedding-set", embeddingSetId: set.id }, definition);
|
|
1898
|
+
return this.resolveDefinition({ kind: "embedding-set", embeddingSetId: set.id }, definition, set);
|
|
1833
1899
|
}
|
|
1834
1900
|
return this.resolvePhysicalSet(selector, set.id);
|
|
1835
1901
|
}
|
|
1836
1902
|
if (!selector.definition) throw new Error("virtual-definition selector requires definition");
|
|
1837
1903
|
return this.resolveDefinition(selector, selector.definition);
|
|
1838
1904
|
}
|
|
1839
|
-
async
|
|
1905
|
+
async refreshMaterializedVirtualSet(setId) {
|
|
1906
|
+
const set = await this.get(setId);
|
|
1907
|
+
if (set.kind !== "virtual") throw new Error(`Embedding set is not virtual: ${setId}`);
|
|
1908
|
+
const definition = this.definitionFromRow(set);
|
|
1909
|
+
if (!definition.materialization?.allowed) {
|
|
1910
|
+
throw new Error(`Virtual embedding set does not allow materialization: ${setId}`);
|
|
1911
|
+
}
|
|
1912
|
+
const live = await this.resolveDefinition(
|
|
1913
|
+
{ kind: "embedding-set", embeddingSetId: setId },
|
|
1914
|
+
definition,
|
|
1915
|
+
set,
|
|
1916
|
+
{ forceLive: true }
|
|
1917
|
+
);
|
|
1918
|
+
await this.db.query(`DELETE FROM embedding_set_member WHERE embedding_set_id = $1`, [setId]);
|
|
1919
|
+
for (const row of live.rows) {
|
|
1920
|
+
await this.db.query(
|
|
1921
|
+
`INSERT INTO embedding_set_member (embedding_set_id, note_id, embedding_id)
|
|
1922
|
+
VALUES ($1, $2, $3)
|
|
1923
|
+
ON CONFLICT DO NOTHING`,
|
|
1924
|
+
[setId, row.note_id, row.embedding_id]
|
|
1925
|
+
);
|
|
1926
|
+
}
|
|
1927
|
+
const inputHash = this.resolutionInputHash(definition, live.rows);
|
|
1928
|
+
const generatedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
1929
|
+
const materialization = {
|
|
1930
|
+
...definition.materialization,
|
|
1931
|
+
allowed: true,
|
|
1932
|
+
includeResolvedMembers: true,
|
|
1933
|
+
freshness: "fresh",
|
|
1934
|
+
inputHash,
|
|
1935
|
+
generatedAt,
|
|
1936
|
+
resolvedMemberCount: live.rows.length
|
|
1937
|
+
};
|
|
1938
|
+
const freshness = {
|
|
1939
|
+
status: "fresh",
|
|
1940
|
+
sourceHash: inputHash,
|
|
1941
|
+
checkedAt: generatedAt
|
|
1942
|
+
};
|
|
1943
|
+
await this.db.query(
|
|
1944
|
+
`UPDATE embedding_set
|
|
1945
|
+
SET materialization_json = $2::jsonb, freshness_json = $3::jsonb, updated_at = now()
|
|
1946
|
+
WHERE id = $1`,
|
|
1947
|
+
[setId, jsonParam(materialization), jsonParam(freshness)]
|
|
1948
|
+
);
|
|
1949
|
+
return this.finalizeResolution(
|
|
1950
|
+
{ kind: "embedding-set", embeddingSetId: setId },
|
|
1951
|
+
live.rows,
|
|
1952
|
+
live.errors,
|
|
1953
|
+
definition.compatibility,
|
|
1954
|
+
"fresh",
|
|
1955
|
+
"materialized"
|
|
1956
|
+
);
|
|
1957
|
+
}
|
|
1958
|
+
async markVirtualSetStale(setId, reason) {
|
|
1959
|
+
const set = await this.get(setId);
|
|
1960
|
+
if (set.kind !== "virtual") throw new Error(`Embedding set is not virtual: ${setId}`);
|
|
1961
|
+
const definition = this.definitionFromRow(set);
|
|
1962
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
1963
|
+
const materialization = definition.materialization ? { ...definition.materialization, freshness: "stale" } : null;
|
|
1964
|
+
await this.db.query(
|
|
1965
|
+
`UPDATE embedding_set
|
|
1966
|
+
SET materialization_json = $2::jsonb, freshness_json = $3::jsonb, updated_at = now()
|
|
1967
|
+
WHERE id = $1`,
|
|
1968
|
+
[
|
|
1969
|
+
setId,
|
|
1970
|
+
jsonParam(materialization),
|
|
1971
|
+
jsonParam({ status: "stale", sourceHash: definition.materialization?.inputHash, checkedAt: now, reason })
|
|
1972
|
+
]
|
|
1973
|
+
);
|
|
1974
|
+
}
|
|
1975
|
+
async resolveDefinition(selector, definition, set, options = {}) {
|
|
1976
|
+
if (!options.forceLive && set && definition.materialization?.allowed && definition.materialization.freshness === "fresh") {
|
|
1977
|
+
const materialized = await this.resolveMaterializedRows(set.id);
|
|
1978
|
+
if (materialized.length > 0 || definition.materialization.resolvedMemberCount === 0) {
|
|
1979
|
+
return this.finalizeResolution(
|
|
1980
|
+
selector,
|
|
1981
|
+
materialized,
|
|
1982
|
+
[],
|
|
1983
|
+
definition.compatibility,
|
|
1984
|
+
"fresh",
|
|
1985
|
+
"materialized"
|
|
1986
|
+
);
|
|
1987
|
+
}
|
|
1988
|
+
}
|
|
1840
1989
|
let rows;
|
|
1841
1990
|
const errors = [];
|
|
1842
1991
|
switch (definition.source.type) {
|
|
@@ -1858,10 +2007,10 @@ var EmbeddingSetsRepository = class {
|
|
|
1858
2007
|
default:
|
|
1859
2008
|
rows = [];
|
|
1860
2009
|
}
|
|
1861
|
-
return this.finalizeResolution(selector, rows, errors, definition.compatibility, definition.materialization?.freshness ?? "unknown");
|
|
2010
|
+
return this.finalizeResolution(selector, rows, errors, definition.compatibility, definition.materialization?.freshness ?? "unknown", "live");
|
|
1862
2011
|
}
|
|
1863
2012
|
async resolvePhysicalSet(selector, setId) {
|
|
1864
|
-
return this.finalizeResolution(selector, await this.resolvePhysicalRows(setId), [], DEFAULT_COMPATIBILITY, "fresh");
|
|
2013
|
+
return this.finalizeResolution(selector, await this.resolvePhysicalRows(setId), [], DEFAULT_COMPATIBILITY, "fresh", "live");
|
|
1865
2014
|
}
|
|
1866
2015
|
async resolvePhysicalRows(setId) {
|
|
1867
2016
|
const result = await this.db.query(
|
|
@@ -1873,6 +2022,17 @@ var EmbeddingSetsRepository = class {
|
|
|
1873
2022
|
);
|
|
1874
2023
|
return result.rows;
|
|
1875
2024
|
}
|
|
2025
|
+
async resolveMaterializedRows(setId) {
|
|
2026
|
+
const result = await this.db.query(
|
|
2027
|
+
`SELECT e.note_id, e.embedding_set_id, e.id as embedding_id, e.vector::text as vector, e.created_at
|
|
2028
|
+
FROM embedding_set_member m
|
|
2029
|
+
JOIN embedding e ON e.id = m.embedding_id
|
|
2030
|
+
WHERE m.embedding_set_id = $1
|
|
2031
|
+
ORDER BY e.note_id, e.created_at DESC`,
|
|
2032
|
+
[setId]
|
|
2033
|
+
);
|
|
2034
|
+
return result.rows;
|
|
2035
|
+
}
|
|
1876
2036
|
async resolveCriteriaSource(source) {
|
|
1877
2037
|
const criteria = source.criteria;
|
|
1878
2038
|
if (criteria.conceptIds && criteria.conceptIds.length > 0) {
|
|
@@ -1893,6 +2053,50 @@ var EmbeddingSetsRepository = class {
|
|
|
1893
2053
|
conditions.push(`EXISTS (SELECT 1 FROM collection_note cn WHERE cn.note_id = n.id AND cn.collection_id = ANY($${idx++}))`);
|
|
1894
2054
|
params.push(criteria.collectionIds);
|
|
1895
2055
|
}
|
|
2056
|
+
if (criteria.sources?.length) {
|
|
2057
|
+
conditions.push(`n.source = ANY($${idx++})`);
|
|
2058
|
+
params.push(criteria.sources);
|
|
2059
|
+
}
|
|
2060
|
+
if (criteria.formats?.length) {
|
|
2061
|
+
conditions.push(`n.format = ANY($${idx++})`);
|
|
2062
|
+
params.push(criteria.formats);
|
|
2063
|
+
}
|
|
2064
|
+
if (criteria.visibilities?.length) {
|
|
2065
|
+
conditions.push(`n.visibility = ANY($${idx++})`);
|
|
2066
|
+
params.push(criteria.visibilities);
|
|
2067
|
+
}
|
|
2068
|
+
if (criteria.isStarred !== void 0) {
|
|
2069
|
+
conditions.push(`n.is_starred = $${idx++}`);
|
|
2070
|
+
params.push(criteria.isStarred);
|
|
2071
|
+
}
|
|
2072
|
+
if (criteria.isArchived !== void 0) {
|
|
2073
|
+
conditions.push(`n.is_archived = $${idx++}`);
|
|
2074
|
+
params.push(criteria.isArchived);
|
|
2075
|
+
}
|
|
2076
|
+
if (criteria.hasTitle !== void 0) {
|
|
2077
|
+
conditions.push(criteria.hasTitle ? `n.title IS NOT NULL AND n.title <> ''` : `(n.title IS NULL OR n.title = '')`);
|
|
2078
|
+
}
|
|
2079
|
+
if (criteria.hasEmbedding === false) {
|
|
2080
|
+
conditions.push("FALSE");
|
|
2081
|
+
}
|
|
2082
|
+
if (criteria.isUserEdited !== void 0) {
|
|
2083
|
+
conditions.push(`COALESCE(c.is_user_edited, false) = $${idx++}`);
|
|
2084
|
+
params.push(criteria.isUserEdited);
|
|
2085
|
+
}
|
|
2086
|
+
if (criteria.hasAiMetadata !== void 0) {
|
|
2087
|
+
conditions.push(criteria.hasAiMetadata ? `c.ai_metadata IS NOT NULL` : `c.ai_metadata IS NULL`);
|
|
2088
|
+
}
|
|
2089
|
+
if (criteria.hasRevisions !== void 0) {
|
|
2090
|
+
conditions.push(criteria.hasRevisions ? `EXISTS (SELECT 1 FROM note_revision nr WHERE nr.note_id = n.id)` : `NOT EXISTS (SELECT 1 FROM note_revision nr WHERE nr.note_id = n.id)`);
|
|
2091
|
+
}
|
|
2092
|
+
if (criteria.minGenerationCount !== void 0) {
|
|
2093
|
+
conditions.push(`COALESCE(c.generation_count, 0) >= $${idx++}`);
|
|
2094
|
+
params.push(criteria.minGenerationCount);
|
|
2095
|
+
}
|
|
2096
|
+
if (criteria.maxGenerationCount !== void 0) {
|
|
2097
|
+
conditions.push(`COALESCE(c.generation_count, 0) <= $${idx++}`);
|
|
2098
|
+
params.push(criteria.maxGenerationCount);
|
|
2099
|
+
}
|
|
1896
2100
|
if (criteria.updatedAfter) {
|
|
1897
2101
|
conditions.push(`n.updated_at >= $${idx++}`);
|
|
1898
2102
|
params.push(criteria.updatedAfter);
|
|
@@ -1983,7 +2187,7 @@ var EmbeddingSetsRepository = class {
|
|
|
1983
2187
|
}
|
|
1984
2188
|
return resolved.sort((a, b) => a.note_id.localeCompare(b.note_id));
|
|
1985
2189
|
}
|
|
1986
|
-
finalizeResolution(selector, rows, errors, compatibility, freshness) {
|
|
2190
|
+
finalizeResolution(selector, rows, errors, compatibility, freshness, resolutionSource) {
|
|
1987
2191
|
const deduped = this.resolveDuplicateRows(rows, compatibility, errors);
|
|
1988
2192
|
return {
|
|
1989
2193
|
selector,
|
|
@@ -1991,9 +2195,17 @@ var EmbeddingSetsRepository = class {
|
|
|
1991
2195
|
noteIds: deduped.map((row) => row.note_id),
|
|
1992
2196
|
embeddingIds: deduped.map((row) => row.embedding_id),
|
|
1993
2197
|
errors,
|
|
1994
|
-
freshness: { status: freshness }
|
|
2198
|
+
freshness: { status: freshness },
|
|
2199
|
+
resolutionSource
|
|
1995
2200
|
};
|
|
1996
2201
|
}
|
|
2202
|
+
resolutionInputHash(definition, rows) {
|
|
2203
|
+
return hashJson({
|
|
2204
|
+
source: definition.source,
|
|
2205
|
+
compatibility: definition.compatibility,
|
|
2206
|
+
members: rows.map((row) => [row.note_id, row.embedding_set_id, row.embedding_id])
|
|
2207
|
+
});
|
|
2208
|
+
}
|
|
1997
2209
|
definitionFromRow(row) {
|
|
1998
2210
|
const source = asObject(row.source_json);
|
|
1999
2211
|
if (!source) throw new Error(`Virtual embedding set has no source definition: ${row.id}`);
|
|
@@ -2398,8 +2610,9 @@ var SearchRepository = class {
|
|
|
2398
2610
|
};
|
|
2399
2611
|
|
|
2400
2612
|
// src/repositories/graph-repository.ts
|
|
2401
|
-
var SIMILARITY_GRAPH_ALGORITHM = "knn-v1";
|
|
2402
|
-
|
|
2613
|
+
var SIMILARITY_GRAPH_ALGORITHM = "knn-batched-v1";
|
|
2614
|
+
var DEFAULT_GRAPH_BATCH_SIZE = 64;
|
|
2615
|
+
function hashJson2(value) {
|
|
2403
2616
|
return computeHash(new TextEncoder().encode(JSON.stringify(value)));
|
|
2404
2617
|
}
|
|
2405
2618
|
function sourceIdFor(inputHash) {
|
|
@@ -2410,6 +2623,45 @@ function jsonObject(value) {
|
|
|
2410
2623
|
if (typeof value === "string") return JSON.parse(value);
|
|
2411
2624
|
return value;
|
|
2412
2625
|
}
|
|
2626
|
+
async function yieldToEventLoop() {
|
|
2627
|
+
const scheduler = globalThis.scheduler;
|
|
2628
|
+
if (scheduler?.yield) {
|
|
2629
|
+
await scheduler.yield();
|
|
2630
|
+
return;
|
|
2631
|
+
}
|
|
2632
|
+
await new Promise((resolve) => setTimeout(resolve, 0));
|
|
2633
|
+
}
|
|
2634
|
+
async function maybeYield(done, every) {
|
|
2635
|
+
if (every > 0 && done > 0 && done % every === 0) {
|
|
2636
|
+
await yieldToEventLoop();
|
|
2637
|
+
}
|
|
2638
|
+
}
|
|
2639
|
+
function parseVector(value) {
|
|
2640
|
+
return value.replace(/^\[/, "").replace(/\]$/, "").split(",").filter((part) => part.length > 0).map(Number);
|
|
2641
|
+
}
|
|
2642
|
+
function vectorScore(left, right, metric) {
|
|
2643
|
+
let dot = 0;
|
|
2644
|
+
let leftNorm = 0;
|
|
2645
|
+
let rightNorm = 0;
|
|
2646
|
+
let squaredDistance = 0;
|
|
2647
|
+
for (let i = 0; i < Math.min(left.length, right.length); i++) {
|
|
2648
|
+
dot += left[i] * right[i];
|
|
2649
|
+
leftNorm += left[i] * left[i];
|
|
2650
|
+
rightNorm += right[i] * right[i];
|
|
2651
|
+
const diff = left[i] - right[i];
|
|
2652
|
+
squaredDistance += diff * diff;
|
|
2653
|
+
}
|
|
2654
|
+
if (metric === "inner_product") {
|
|
2655
|
+
return { distance: -dot, similarity: dot };
|
|
2656
|
+
}
|
|
2657
|
+
if (metric === "l2") {
|
|
2658
|
+
const distance = Math.sqrt(squaredDistance);
|
|
2659
|
+
return { distance, similarity: -distance };
|
|
2660
|
+
}
|
|
2661
|
+
const denominator = Math.sqrt(leftNorm) * Math.sqrt(rightNorm);
|
|
2662
|
+
const similarity = denominator === 0 ? 0 : dot / denominator;
|
|
2663
|
+
return { distance: 1 - similarity, similarity };
|
|
2664
|
+
}
|
|
2413
2665
|
function detectCommunities(edges, nodes = [], options = {}) {
|
|
2414
2666
|
const nodeIds = new Set(nodes.map((n) => n.id));
|
|
2415
2667
|
for (const edge of edges) {
|
|
@@ -2486,7 +2738,7 @@ var GraphRepository = class {
|
|
|
2486
2738
|
const normalized = this.normalizeSimilarityRequest(request);
|
|
2487
2739
|
const resolved = await new EmbeddingSetsRepository(this.db).resolveSelector(normalized.selector);
|
|
2488
2740
|
const cacheKey = await this.computeSimilarityGraphCacheKey(normalized, resolved);
|
|
2489
|
-
const inputHash =
|
|
2741
|
+
const inputHash = hashJson2(cacheKey);
|
|
2490
2742
|
const source = await this.findGraphSource(inputHash);
|
|
2491
2743
|
if (!source) return null;
|
|
2492
2744
|
const graph = await this.graphFromArtifact(source.id, resolved.noteIds);
|
|
@@ -2501,7 +2753,7 @@ var GraphRepository = class {
|
|
|
2501
2753
|
const normalized = this.normalizeSimilarityRequest(request);
|
|
2502
2754
|
const resolved = await new EmbeddingSetsRepository(this.db).resolveSelector(normalized.selector);
|
|
2503
2755
|
const cacheKey = await this.computeSimilarityGraphCacheKey(normalized, resolved);
|
|
2504
|
-
const inputHash =
|
|
2756
|
+
const inputHash = hashJson2(cacheKey);
|
|
2505
2757
|
if (normalized.source !== "live-only") {
|
|
2506
2758
|
const cached = await this.findGraphSource(inputHash);
|
|
2507
2759
|
if (cached?.freshness === "fresh") {
|
|
@@ -2534,7 +2786,7 @@ var GraphRepository = class {
|
|
|
2534
2786
|
};
|
|
2535
2787
|
}
|
|
2536
2788
|
async saveSimilarityGraphArtifact(input) {
|
|
2537
|
-
const inputHash =
|
|
2789
|
+
const inputHash = hashJson2(input.cacheKey);
|
|
2538
2790
|
const id = sourceIdFor(inputHash);
|
|
2539
2791
|
const parameters = {
|
|
2540
2792
|
k: input.request.k,
|
|
@@ -2585,27 +2837,32 @@ var GraphRepository = class {
|
|
|
2585
2837
|
async buildSimilarityGraphFromResolved(resolved, options) {
|
|
2586
2838
|
const k = options.k ?? 5;
|
|
2587
2839
|
const minSimilarity = options.minSimilarity ?? options.threshold ?? -1;
|
|
2840
|
+
const metric = options.metric ?? "cosine";
|
|
2841
|
+
const yieldEvery = options.yieldEvery ?? options.batchSize ?? DEFAULT_GRAPH_BATCH_SIZE;
|
|
2588
2842
|
const embeddings = resolved.rows;
|
|
2589
2843
|
const nodes = embeddings.map((row) => ({ id: row.note_id }));
|
|
2590
2844
|
const edgeMap = /* @__PURE__ */ new Map();
|
|
2591
|
-
|
|
2592
|
-
|
|
2593
|
-
|
|
2594
|
-
|
|
2595
|
-
|
|
2596
|
-
|
|
2597
|
-
|
|
2598
|
-
|
|
2599
|
-
|
|
2600
|
-
|
|
2845
|
+
const vectors = embeddings.map((row) => ({
|
|
2846
|
+
row,
|
|
2847
|
+
vector: parseVector(row.vector)
|
|
2848
|
+
}));
|
|
2849
|
+
options.onProgress?.({ phase: "prepare", done: vectors.length, total: vectors.length });
|
|
2850
|
+
for (const [index, item] of vectors.entries()) {
|
|
2851
|
+
const neighbors = vectors.filter((candidate) => candidate.row.note_id !== item.row.note_id).map((candidate) => {
|
|
2852
|
+
const score = vectorScore(item.vector, candidate.vector, metric);
|
|
2853
|
+
return { noteId: candidate.row.note_id, ...score };
|
|
2854
|
+
}).sort((a, b) => a.distance - b.distance || a.noteId.localeCompare(b.noteId)).slice(0, k);
|
|
2855
|
+
for (const neighbor of neighbors) {
|
|
2601
2856
|
if (neighbor.similarity < minSimilarity) continue;
|
|
2602
|
-
const [source, target] = [row.note_id, neighbor.
|
|
2857
|
+
const [source, target] = [item.row.note_id, neighbor.noteId].sort();
|
|
2603
2858
|
const id = `${source}\0${target}`;
|
|
2604
2859
|
const existing = edgeMap.get(id);
|
|
2605
2860
|
if (!existing || neighbor.similarity > existing.weight) {
|
|
2606
2861
|
edgeMap.set(id, { source, target, weight: neighbor.similarity, kind: "similarity" });
|
|
2607
2862
|
}
|
|
2608
2863
|
}
|
|
2864
|
+
options.onProgress?.({ phase: "neighbors", done: index + 1, total: vectors.length });
|
|
2865
|
+
await maybeYield(index + 1, yieldEvery);
|
|
2609
2866
|
}
|
|
2610
2867
|
const edges = Array.from(edgeMap.values()).sort(
|
|
2611
2868
|
(a, b) => a.source.localeCompare(b.source) || a.target.localeCompare(b.target)
|
|
@@ -2616,7 +2873,7 @@ var GraphRepository = class {
|
|
|
2616
2873
|
const firstSetId = resolved.rows[0]?.embedding_set_id;
|
|
2617
2874
|
const set = firstSetId ? await new EmbeddingSetsRepository(this.db).get(firstSetId) : null;
|
|
2618
2875
|
return {
|
|
2619
|
-
selectorHash:
|
|
2876
|
+
selectorHash: hashJson2(request.selector),
|
|
2620
2877
|
resolvedEmbeddingSetId: request.selector.kind === "embedding-set" ? request.selector.embeddingSetId : void 0,
|
|
2621
2878
|
virtualSetId: request.selector.kind === "virtual-definition" ? request.selector.definition?.id : void 0,
|
|
2622
2879
|
k: request.k,
|
|
@@ -2625,9 +2882,9 @@ var GraphRepository = class {
|
|
|
2625
2882
|
model: set?.model_name ?? "unknown",
|
|
2626
2883
|
dimension: set?.dimensions ?? 0,
|
|
2627
2884
|
truncateDimension: set?.truncate_dimension ?? null,
|
|
2628
|
-
memberHash:
|
|
2629
|
-
vectorHash:
|
|
2630
|
-
parameterHash:
|
|
2885
|
+
memberHash: hashJson2(resolved.rows.map((row) => [row.note_id, row.embedding_set_id, row.embedding_id])),
|
|
2886
|
+
vectorHash: hashJson2(resolved.rows.map((row) => [row.embedding_id, row.vector])),
|
|
2887
|
+
parameterHash: hashJson2({ k: request.k, minSimilarity: request.minSimilarity, metric: request.metric, algorithm: SIMILARITY_GRAPH_ALGORITHM })
|
|
2631
2888
|
};
|
|
2632
2889
|
}
|
|
2633
2890
|
async findGraphSource(inputHash) {
|
|
@@ -5171,6 +5428,33 @@ var FallbackRouter = class {
|
|
|
5171
5428
|
}
|
|
5172
5429
|
};
|
|
5173
5430
|
|
|
5431
|
+
// src/fortemi-bridge.ts
|
|
5432
|
+
function getFortemiBridge(host = globalThis) {
|
|
5433
|
+
return host?.fortemiBridge ?? null;
|
|
5434
|
+
}
|
|
5435
|
+
function getFortemiSecretStore(host = globalThis) {
|
|
5436
|
+
return host?.fortemiBridge?.secrets ?? host?.fortemiSecureStorage ?? null;
|
|
5437
|
+
}
|
|
5438
|
+
async function hasFortemiSecureSecrets(host = globalThis) {
|
|
5439
|
+
const bridge = getFortemiBridge(host);
|
|
5440
|
+
if (bridge) {
|
|
5441
|
+
try {
|
|
5442
|
+
const capabilities = await bridge.capabilities();
|
|
5443
|
+
if (!capabilities.secureSecrets) return false;
|
|
5444
|
+
return Boolean(await bridge.secrets.isAvailable());
|
|
5445
|
+
} catch {
|
|
5446
|
+
return false;
|
|
5447
|
+
}
|
|
5448
|
+
}
|
|
5449
|
+
const legacySecrets = host?.fortemiSecureStorage;
|
|
5450
|
+
if (!legacySecrets) return false;
|
|
5451
|
+
try {
|
|
5452
|
+
return Boolean(await legacySecrets.isAvailable());
|
|
5453
|
+
} catch {
|
|
5454
|
+
return false;
|
|
5455
|
+
}
|
|
5456
|
+
}
|
|
5457
|
+
|
|
5174
5458
|
// src/security/plugin-content.ts
|
|
5175
5459
|
var DEFAULT_DIRECTIVES = {
|
|
5176
5460
|
"default-src": ["'self'"],
|
|
@@ -5589,7 +5873,7 @@ function embeddingToShard(emb) {
|
|
|
5589
5873
|
id: emb.id,
|
|
5590
5874
|
note_id: emb.note_id,
|
|
5591
5875
|
embedding_set_id: emb.embedding_set_id,
|
|
5592
|
-
vector: typeof emb.vector === "string" ?
|
|
5876
|
+
vector: typeof emb.vector === "string" ? parseVector2(emb.vector) : emb.vector,
|
|
5593
5877
|
created_at: toISOString(emb.created_at)
|
|
5594
5878
|
};
|
|
5595
5879
|
}
|
|
@@ -5655,7 +5939,7 @@ function toISOString(date) {
|
|
|
5655
5939
|
if (date instanceof Date) return date.toISOString();
|
|
5656
5940
|
return date;
|
|
5657
5941
|
}
|
|
5658
|
-
function
|
|
5942
|
+
function parseVector2(vectorStr) {
|
|
5659
5943
|
const inner = vectorStr.replace(/^\[/, "").replace(/\]$/, "");
|
|
5660
5944
|
return inner.split(",").map(Number);
|
|
5661
5945
|
}
|
|
@@ -5827,25 +6111,60 @@ async function exportShard(db, options) {
|
|
|
5827
6111
|
components.push("provenance_edges");
|
|
5828
6112
|
counts.provenance_edges = filteredProvenanceRows.length;
|
|
5829
6113
|
if (options?.includeEmbeddings) {
|
|
5830
|
-
const
|
|
5831
|
-
const
|
|
6114
|
+
const embeddingSetIds = options.embeddingSetIds?.filter(Boolean) ?? [];
|
|
6115
|
+
const setScoped = embeddingSetIds.length > 0;
|
|
6116
|
+
const includeMaterializedSelectors = options.includeMaterializedSelectors === true;
|
|
6117
|
+
const embSetRows = await db.query(
|
|
6118
|
+
`SELECT * FROM embedding_set
|
|
6119
|
+
${setScoped ? "WHERE id = ANY($1)" : ""}
|
|
6120
|
+
ORDER BY created_at`,
|
|
6121
|
+
setScoped ? [embeddingSetIds] : []
|
|
6122
|
+
);
|
|
6123
|
+
const exportedSetIds = new Set(embSetRows.rows.map((row) => row.id));
|
|
6124
|
+
const virtualSetIds = new Set(embSetRows.rows.filter((row) => row.kind === "virtual").map((row) => row.id));
|
|
6125
|
+
const shardEmbSets = embSetRows.rows.map((row) => embeddingSetToShard(
|
|
6126
|
+
row.kind === "virtual" && !includeMaterializedSelectors ? {
|
|
6127
|
+
...row,
|
|
6128
|
+
materialization_json: void 0,
|
|
6129
|
+
freshness_json: { status: "unknown" }
|
|
6130
|
+
} : row
|
|
6131
|
+
));
|
|
5832
6132
|
files.set("embedding_sets.json", encoder.encode(JSON.stringify(shardEmbSets)));
|
|
5833
6133
|
components.push("embedding_sets");
|
|
5834
6134
|
counts.embedding_sets = shardEmbSets.length;
|
|
5835
|
-
const embMemberRows = await db.query(
|
|
5836
|
-
|
|
6135
|
+
const embMemberRows = await db.query(
|
|
6136
|
+
`SELECT * FROM embedding_set_member
|
|
6137
|
+
${setScoped ? "WHERE embedding_set_id = ANY($1)" : ""}`,
|
|
6138
|
+
setScoped ? [embeddingSetIds] : []
|
|
6139
|
+
);
|
|
6140
|
+
const scopedEmbMemberRows = embMemberRows.rows.filter(
|
|
6141
|
+
(member) => exportedSetIds.has(member.embedding_set_id) && exportedNoteIds.has(member.note_id) && (includeMaterializedSelectors || !virtualSetIds.has(member.embedding_set_id))
|
|
6142
|
+
);
|
|
6143
|
+
const membersJsonl = scopedEmbMemberRows.map((m) => JSON.stringify(embeddingSetMemberToShard(m))).join("\n");
|
|
5837
6144
|
files.set("embedding_set_members.jsonl", encoder.encode(membersJsonl));
|
|
5838
6145
|
components.push("embedding_set_members");
|
|
5839
|
-
counts.embedding_set_members =
|
|
5840
|
-
const embRows = await db.query(
|
|
5841
|
-
|
|
6146
|
+
counts.embedding_set_members = scopedEmbMemberRows.length;
|
|
6147
|
+
const embRows = await db.query(
|
|
6148
|
+
`SELECT * FROM embedding
|
|
6149
|
+
${setScoped ? "WHERE embedding_set_id = ANY($1)" : ""}
|
|
6150
|
+
ORDER BY created_at`,
|
|
6151
|
+
setScoped ? [embeddingSetIds] : []
|
|
6152
|
+
);
|
|
6153
|
+
const memberEmbeddingIds = new Set(scopedEmbMemberRows.map((member) => member.embedding_id));
|
|
6154
|
+
const scopedEmbRows = embRows.rows.filter(
|
|
6155
|
+
(embedding) => exportedSetIds.has(embedding.embedding_set_id) && exportedNoteIds.has(embedding.note_id) && (memberEmbeddingIds.size === 0 || memberEmbeddingIds.has(embedding.id))
|
|
6156
|
+
);
|
|
6157
|
+
const embJsonl = scopedEmbRows.map((e) => JSON.stringify(embeddingToShard(e))).join("\n");
|
|
5842
6158
|
files.set("embeddings.jsonl", encoder.encode(embJsonl));
|
|
5843
6159
|
components.push("embeddings");
|
|
5844
|
-
counts.embeddings =
|
|
6160
|
+
counts.embeddings = scopedEmbRows.length;
|
|
5845
6161
|
}
|
|
5846
6162
|
const graphSourceRows = await db.query(`SELECT * FROM graph_source ORDER BY created_at, id`);
|
|
5847
|
-
|
|
5848
|
-
|
|
6163
|
+
const graphScoped = !!options?.embeddingSetIds?.length;
|
|
6164
|
+
const scopedGraphSourceRows = graphScoped ? graphSourceRows.rows.filter((row) => !row.embedding_set_id || options.embeddingSetIds?.includes(row.embedding_set_id)) : graphSourceRows.rows;
|
|
6165
|
+
const exportedGraphSourceIds = new Set(scopedGraphSourceRows.map((row) => row.id));
|
|
6166
|
+
if (scopedGraphSourceRows.length > 0) {
|
|
6167
|
+
const shardGraphSources = scopedGraphSourceRows.map((row) => ({
|
|
5849
6168
|
id: row.id,
|
|
5850
6169
|
name: row.name,
|
|
5851
6170
|
kind: row.kind,
|
|
@@ -5867,8 +6186,9 @@ async function exportShard(db, options) {
|
|
|
5867
6186
|
counts.graph_sources = shardGraphSources.length;
|
|
5868
6187
|
}
|
|
5869
6188
|
const graphEdgeRows = await db.query(`SELECT * FROM graph_edge_artifact ORDER BY graph_source_id, from_note_id, to_note_id, kind`);
|
|
5870
|
-
|
|
5871
|
-
|
|
6189
|
+
const scopedGraphEdgeRows = graphScoped ? graphEdgeRows.rows.filter((row) => exportedGraphSourceIds.has(row.graph_source_id)) : graphEdgeRows.rows;
|
|
6190
|
+
if (scopedGraphEdgeRows.length > 0) {
|
|
6191
|
+
const graphEdgesJsonl = scopedGraphEdgeRows.map((row) => JSON.stringify({
|
|
5872
6192
|
graph_source_id: row.graph_source_id,
|
|
5873
6193
|
from_note_id: row.from_note_id,
|
|
5874
6194
|
to_note_id: row.to_note_id,
|
|
@@ -5879,18 +6199,21 @@ async function exportShard(db, options) {
|
|
|
5879
6199
|
})).join("\n");
|
|
5880
6200
|
files.set("graph_edges.jsonl", encoder.encode(graphEdgesJsonl));
|
|
5881
6201
|
components.push("graph_edges");
|
|
5882
|
-
counts.graph_edges =
|
|
6202
|
+
counts.graph_edges = scopedGraphEdgeRows.length;
|
|
5883
6203
|
}
|
|
5884
6204
|
const communitySetRows = await db.query(`SELECT * FROM community_set ORDER BY created_at, id`);
|
|
5885
6205
|
const communityRows = await db.query(`SELECT * FROM community ORDER BY community_set_id, rank NULLS LAST, id`);
|
|
5886
|
-
|
|
6206
|
+
const scopedCommunitySetRows = graphScoped ? communitySetRows.rows.filter((row) => exportedGraphSourceIds.has(row.graph_source_id)) : communitySetRows.rows;
|
|
6207
|
+
const exportedCommunitySetIds = new Set(scopedCommunitySetRows.map((row) => row.id));
|
|
6208
|
+
const scopedCommunityRows = graphScoped ? communityRows.rows.filter((row) => exportedCommunitySetIds.has(row.community_set_id)) : communityRows.rows;
|
|
6209
|
+
if (scopedCommunitySetRows.length > 0) {
|
|
5887
6210
|
const communitiesBySet = /* @__PURE__ */ new Map();
|
|
5888
|
-
for (const row of
|
|
6211
|
+
for (const row of scopedCommunityRows) {
|
|
5889
6212
|
const rows = communitiesBySet.get(row.community_set_id) ?? [];
|
|
5890
6213
|
rows.push(row);
|
|
5891
6214
|
communitiesBySet.set(row.community_set_id, rows);
|
|
5892
6215
|
}
|
|
5893
|
-
const shardCommunitySets =
|
|
6216
|
+
const shardCommunitySets = scopedCommunitySetRows.map((row) => ({
|
|
5894
6217
|
id: row.id,
|
|
5895
6218
|
graph_source_id: row.graph_source_id,
|
|
5896
6219
|
name: row.name,
|
|
@@ -5913,11 +6236,12 @@ async function exportShard(db, options) {
|
|
|
5913
6236
|
files.set("communities.json", encoder.encode(JSON.stringify(shardCommunitySets)));
|
|
5914
6237
|
components.push("communities");
|
|
5915
6238
|
counts.community_sets = shardCommunitySets.length;
|
|
5916
|
-
counts.communities =
|
|
6239
|
+
counts.communities = scopedCommunityRows.length;
|
|
5917
6240
|
}
|
|
5918
6241
|
const assignmentRows = await db.query(`SELECT * FROM community_assignment ORDER BY community_set_id, community_id, note_id`);
|
|
5919
|
-
|
|
5920
|
-
|
|
6242
|
+
const scopedAssignmentRows = graphScoped ? assignmentRows.rows.filter((row) => exportedCommunitySetIds.has(row.community_set_id)) : assignmentRows.rows;
|
|
6243
|
+
if (scopedAssignmentRows.length > 0) {
|
|
6244
|
+
const assignmentsJsonl = scopedAssignmentRows.map((row) => JSON.stringify({
|
|
5921
6245
|
community_set_id: row.community_set_id,
|
|
5922
6246
|
community_id: row.community_id,
|
|
5923
6247
|
note_id: row.note_id,
|
|
@@ -5927,7 +6251,7 @@ async function exportShard(db, options) {
|
|
|
5927
6251
|
})).join("\n");
|
|
5928
6252
|
files.set("community_assignments.jsonl", encoder.encode(assignmentsJsonl));
|
|
5929
6253
|
components.push("community_assignments");
|
|
5930
|
-
counts.community_assignments =
|
|
6254
|
+
counts.community_assignments = scopedAssignmentRows.length;
|
|
5931
6255
|
}
|
|
5932
6256
|
const checksums = {};
|
|
5933
6257
|
for (const [filename, data] of files) {
|
|
@@ -5949,9 +6273,25 @@ async function exportShard(db, options) {
|
|
|
5949
6273
|
|
|
5950
6274
|
// src/shard/shard-import.ts
|
|
5951
6275
|
var decoder = new TextDecoder();
|
|
6276
|
+
var DEFAULT_BATCH_SIZE = 250;
|
|
6277
|
+
async function yieldToEventLoop2() {
|
|
6278
|
+
const scheduler = globalThis.scheduler;
|
|
6279
|
+
if (scheduler?.yield) {
|
|
6280
|
+
await scheduler.yield();
|
|
6281
|
+
return;
|
|
6282
|
+
}
|
|
6283
|
+
await new Promise((resolve) => setTimeout(resolve, 0));
|
|
6284
|
+
}
|
|
6285
|
+
async function maybeYield2(done, batchSize) {
|
|
6286
|
+
if (batchSize > 0 && done > 0 && done % batchSize === 0) {
|
|
6287
|
+
await yieldToEventLoop2();
|
|
6288
|
+
}
|
|
6289
|
+
}
|
|
5952
6290
|
async function importShard(db, data, options) {
|
|
5953
6291
|
const start = performance.now();
|
|
5954
6292
|
const strategy = options?.conflictStrategy ?? "skip";
|
|
6293
|
+
const batchSize = options?.batchSize ?? DEFAULT_BATCH_SIZE;
|
|
6294
|
+
const report = options?.onProgress;
|
|
5955
6295
|
const warnings = [];
|
|
5956
6296
|
const errors = [];
|
|
5957
6297
|
const counts = {
|
|
@@ -5975,9 +6315,11 @@ async function importShard(db, data, options) {
|
|
|
5975
6315
|
};
|
|
5976
6316
|
const skipped = {};
|
|
5977
6317
|
const inputData = data instanceof ArrayBuffer ? new Uint8Array(data) : data;
|
|
6318
|
+
report?.({ phase: "unpack", done: 0, total: 1 });
|
|
5978
6319
|
let files;
|
|
5979
6320
|
try {
|
|
5980
6321
|
files = unpackTarGz(inputData);
|
|
6322
|
+
report?.({ phase: "unpack", done: 1, total: 1 });
|
|
5981
6323
|
} catch (err) {
|
|
5982
6324
|
return {
|
|
5983
6325
|
success: false,
|
|
@@ -6024,6 +6366,7 @@ async function importShard(db, data, options) {
|
|
|
6024
6366
|
duration_ms: performance.now() - start
|
|
6025
6367
|
};
|
|
6026
6368
|
}
|
|
6369
|
+
report?.({ phase: "validate", done: 0, total: 1 });
|
|
6027
6370
|
const checksumResult = await validateChecksums(manifest.checksums, files);
|
|
6028
6371
|
if (!checksumResult.valid) {
|
|
6029
6372
|
return {
|
|
@@ -6035,6 +6378,7 @@ async function importShard(db, data, options) {
|
|
|
6035
6378
|
duration_ms: performance.now() - start
|
|
6036
6379
|
};
|
|
6037
6380
|
}
|
|
6381
|
+
report?.({ phase: "validate", done: 1, total: 1 });
|
|
6038
6382
|
const parsedNotes = parseJsonl(files.get("notes.jsonl"));
|
|
6039
6383
|
const parsedCollections = parseJsonArray(files.get("collections.json"));
|
|
6040
6384
|
parseJsonArray(files.get("tags.json"));
|
|
@@ -6085,7 +6429,8 @@ async function importShard(db, data, options) {
|
|
|
6085
6429
|
const conflictClause = strategy === "skip" ? "ON CONFLICT DO NOTHING" : "";
|
|
6086
6430
|
try {
|
|
6087
6431
|
await db.transaction(async (tx) => {
|
|
6088
|
-
|
|
6432
|
+
report?.({ phase: "collections", done: 0, total: parsedCollections.length });
|
|
6433
|
+
for (const [index, shardCol] of parsedCollections.entries()) {
|
|
6089
6434
|
const col = collectionFromShard(shardCol);
|
|
6090
6435
|
if (strategy === "replace") {
|
|
6091
6436
|
await tx.query(
|
|
@@ -6102,8 +6447,11 @@ async function importShard(db, data, options) {
|
|
|
6102
6447
|
);
|
|
6103
6448
|
}
|
|
6104
6449
|
counts.collections++;
|
|
6450
|
+
report?.({ phase: "collections", done: index + 1, total: parsedCollections.length });
|
|
6451
|
+
await maybeYield2(index + 1, batchSize);
|
|
6105
6452
|
}
|
|
6106
|
-
|
|
6453
|
+
report?.({ phase: "notes", done: 0, total: parsedNotes.length });
|
|
6454
|
+
for (const [index, shardNote] of parsedNotes.entries()) {
|
|
6107
6455
|
const note = noteFromShard(shardNote);
|
|
6108
6456
|
const contentHash = computeHash(new TextEncoder().encode(note.original_content));
|
|
6109
6457
|
if (strategy === "replace") {
|
|
@@ -6180,7 +6528,12 @@ async function importShard(db, data, options) {
|
|
|
6180
6528
|
);
|
|
6181
6529
|
}
|
|
6182
6530
|
counts.notes++;
|
|
6531
|
+
report?.({ phase: "notes", done: index + 1, total: parsedNotes.length });
|
|
6532
|
+
await maybeYield2(index + 1, batchSize);
|
|
6183
6533
|
}
|
|
6534
|
+
const totalSkos = parsedSkosSchemes.length + parsedSkosConcepts.length + parsedSkosRelations.length + parsedNoteSkosTags.length;
|
|
6535
|
+
let doneSkos = 0;
|
|
6536
|
+
report?.({ phase: "skos", done: doneSkos, total: totalSkos });
|
|
6184
6537
|
for (const scheme of parsedSkosSchemes) {
|
|
6185
6538
|
if (strategy === "replace") {
|
|
6186
6539
|
await tx.query(
|
|
@@ -6197,6 +6550,8 @@ async function importShard(db, data, options) {
|
|
|
6197
6550
|
);
|
|
6198
6551
|
}
|
|
6199
6552
|
counts.skos_schemes++;
|
|
6553
|
+
report?.({ phase: "skos", done: ++doneSkos, total: totalSkos });
|
|
6554
|
+
await maybeYield2(doneSkos, batchSize);
|
|
6200
6555
|
}
|
|
6201
6556
|
for (const concept of parsedSkosConcepts) {
|
|
6202
6557
|
const altLabels = JSON.stringify(concept.alt_labels ?? []);
|
|
@@ -6215,8 +6570,11 @@ async function importShard(db, data, options) {
|
|
|
6215
6570
|
);
|
|
6216
6571
|
}
|
|
6217
6572
|
counts.skos_concepts++;
|
|
6573
|
+
report?.({ phase: "skos", done: ++doneSkos, total: totalSkos });
|
|
6574
|
+
await maybeYield2(doneSkos, batchSize);
|
|
6218
6575
|
}
|
|
6219
|
-
|
|
6576
|
+
report?.({ phase: "links", done: 0, total: parsedLinks.length });
|
|
6577
|
+
for (const [index, shardLink] of parsedLinks.entries()) {
|
|
6220
6578
|
const link = linkFromShard(shardLink);
|
|
6221
6579
|
if (strategy === "replace") {
|
|
6222
6580
|
await tx.query(
|
|
@@ -6233,6 +6591,8 @@ async function importShard(db, data, options) {
|
|
|
6233
6591
|
);
|
|
6234
6592
|
}
|
|
6235
6593
|
counts.links++;
|
|
6594
|
+
report?.({ phase: "links", done: index + 1, total: parsedLinks.length });
|
|
6595
|
+
await maybeYield2(index + 1, batchSize);
|
|
6236
6596
|
}
|
|
6237
6597
|
for (const relation of parsedSkosRelations) {
|
|
6238
6598
|
if (strategy === "replace") {
|
|
@@ -6250,6 +6610,8 @@ async function importShard(db, data, options) {
|
|
|
6250
6610
|
);
|
|
6251
6611
|
}
|
|
6252
6612
|
counts.skos_relations++;
|
|
6613
|
+
report?.({ phase: "skos", done: ++doneSkos, total: totalSkos });
|
|
6614
|
+
await maybeYield2(doneSkos, batchSize);
|
|
6253
6615
|
}
|
|
6254
6616
|
for (const tag of parsedNoteSkosTags) {
|
|
6255
6617
|
await tx.query(
|
|
@@ -6259,8 +6621,11 @@ async function importShard(db, data, options) {
|
|
|
6259
6621
|
[tag.id, tag.note_id, tag.concept_id, tag.created_at]
|
|
6260
6622
|
);
|
|
6261
6623
|
counts.note_skos_tags++;
|
|
6624
|
+
report?.({ phase: "skos", done: ++doneSkos, total: totalSkos });
|
|
6625
|
+
await maybeYield2(doneSkos, batchSize);
|
|
6262
6626
|
}
|
|
6263
|
-
|
|
6627
|
+
report?.({ phase: "provenance", done: 0, total: parsedProvenanceEdges.length });
|
|
6628
|
+
for (const [index, edge] of parsedProvenanceEdges.entries()) {
|
|
6264
6629
|
const attributes = edge.attributes === null ? null : JSON.stringify(edge.attributes);
|
|
6265
6630
|
if (strategy === "replace") {
|
|
6266
6631
|
await tx.query(
|
|
@@ -6277,8 +6642,11 @@ async function importShard(db, data, options) {
|
|
|
6277
6642
|
);
|
|
6278
6643
|
}
|
|
6279
6644
|
counts.provenance_edges++;
|
|
6645
|
+
report?.({ phase: "provenance", done: index + 1, total: parsedProvenanceEdges.length });
|
|
6646
|
+
await maybeYield2(index + 1, batchSize);
|
|
6280
6647
|
}
|
|
6281
|
-
|
|
6648
|
+
report?.({ phase: "embedding_sets", done: 0, total: parsedEmbSets.length });
|
|
6649
|
+
for (const [index, shardSet] of parsedEmbSets.entries()) {
|
|
6282
6650
|
const set = embeddingSetFromShard(shardSet);
|
|
6283
6651
|
if (strategy === "replace") {
|
|
6284
6652
|
await tx.query(
|
|
@@ -6301,8 +6669,11 @@ async function importShard(db, data, options) {
|
|
|
6301
6669
|
);
|
|
6302
6670
|
}
|
|
6303
6671
|
counts.embedding_sets++;
|
|
6672
|
+
report?.({ phase: "embedding_sets", done: index + 1, total: parsedEmbSets.length });
|
|
6673
|
+
await maybeYield2(index + 1, batchSize);
|
|
6304
6674
|
}
|
|
6305
|
-
|
|
6675
|
+
report?.({ phase: "embeddings", done: 0, total: parsedEmbeddings.length });
|
|
6676
|
+
for (const [index, shardEmb] of parsedEmbeddings.entries()) {
|
|
6306
6677
|
const emb = embeddingFromShard(shardEmb);
|
|
6307
6678
|
if (strategy === "replace") {
|
|
6308
6679
|
await tx.query(
|
|
@@ -6319,7 +6690,12 @@ async function importShard(db, data, options) {
|
|
|
6319
6690
|
);
|
|
6320
6691
|
}
|
|
6321
6692
|
counts.embeddings++;
|
|
6693
|
+
report?.({ phase: "embeddings", done: index + 1, total: parsedEmbeddings.length });
|
|
6694
|
+
await maybeYield2(index + 1, batchSize);
|
|
6322
6695
|
}
|
|
6696
|
+
const totalGraph = parsedGraphSources.length + parsedGraphEdges.length;
|
|
6697
|
+
let doneGraph = 0;
|
|
6698
|
+
report?.({ phase: "graph", done: doneGraph, total: totalGraph });
|
|
6323
6699
|
for (const source of parsedGraphSources) {
|
|
6324
6700
|
const parameters = source.parameters == null ? null : JSON.stringify(source.parameters);
|
|
6325
6701
|
const freshness = JSON.stringify({ ...source.freshness ?? {}, status: "unknown" });
|
|
@@ -6332,6 +6708,8 @@ async function importShard(db, data, options) {
|
|
|
6332
6708
|
[source.id, source.name, source.kind, source.source_table ?? null, source.embedding_set_id ?? null, source.virtual_set_id ?? null, source.model ?? null, source.dimension ?? null, source.truncate_dimension ?? null, source.metric ?? null, source.algorithm ?? null, parameters, source.input_hash, freshness, source.created_at]
|
|
6333
6709
|
);
|
|
6334
6710
|
counts.graph_sources++;
|
|
6711
|
+
report?.({ phase: "graph", done: ++doneGraph, total: totalGraph });
|
|
6712
|
+
await maybeYield2(doneGraph, batchSize);
|
|
6335
6713
|
}
|
|
6336
6714
|
for (const edge of parsedGraphEdges) {
|
|
6337
6715
|
const metadata = edge.metadata == null ? null : JSON.stringify(edge.metadata);
|
|
@@ -6342,7 +6720,12 @@ async function importShard(db, data, options) {
|
|
|
6342
6720
|
[edge.graph_source_id, edge.from_note_id, edge.to_note_id, edge.weight, edge.kind, edge.rank ?? null, metadata]
|
|
6343
6721
|
);
|
|
6344
6722
|
counts.graph_edges++;
|
|
6723
|
+
report?.({ phase: "graph", done: ++doneGraph, total: totalGraph });
|
|
6724
|
+
await maybeYield2(doneGraph, batchSize);
|
|
6345
6725
|
}
|
|
6726
|
+
const totalCommunities = parsedCommunitySets.length + parsedCommunitySets.reduce((sum, set) => sum + (set.communities?.length ?? 0), 0) + parsedCommunityAssignments.length;
|
|
6727
|
+
let doneCommunities = 0;
|
|
6728
|
+
report?.({ phase: "communities", done: doneCommunities, total: totalCommunities });
|
|
6346
6729
|
for (const set of parsedCommunitySets) {
|
|
6347
6730
|
const parameters = set.parameters == null ? null : JSON.stringify(set.parameters);
|
|
6348
6731
|
const freshness = JSON.stringify({ ...set.freshness ?? {}, status: "unknown" });
|
|
@@ -6353,6 +6736,8 @@ async function importShard(db, data, options) {
|
|
|
6353
6736
|
[set.id, set.graph_source_id, set.name, set.source_type, set.algorithm ?? null, parameters, set.input_hash, freshness, set.created_at]
|
|
6354
6737
|
);
|
|
6355
6738
|
counts.community_sets++;
|
|
6739
|
+
report?.({ phase: "communities", done: ++doneCommunities, total: totalCommunities });
|
|
6740
|
+
await maybeYield2(doneCommunities, batchSize);
|
|
6356
6741
|
for (const community of set.communities ?? []) {
|
|
6357
6742
|
const metadata = community.metadata == null ? null : JSON.stringify(community.metadata);
|
|
6358
6743
|
await tx.query(
|
|
@@ -6362,6 +6747,8 @@ async function importShard(db, data, options) {
|
|
|
6362
6747
|
[set.id, community.id, community.label ?? null, community.rank ?? null, community.size ?? null, community.confidence ?? null, community.representative_note_ids ?? [], metadata]
|
|
6363
6748
|
);
|
|
6364
6749
|
counts.communities++;
|
|
6750
|
+
report?.({ phase: "communities", done: ++doneCommunities, total: totalCommunities });
|
|
6751
|
+
await maybeYield2(doneCommunities, batchSize);
|
|
6365
6752
|
}
|
|
6366
6753
|
}
|
|
6367
6754
|
for (const assignment of parsedCommunityAssignments) {
|
|
@@ -6373,16 +6760,22 @@ async function importShard(db, data, options) {
|
|
|
6373
6760
|
[assignment.community_set_id, assignment.community_id, assignment.note_id, assignment.confidence ?? null, assignment.source_type, metadata]
|
|
6374
6761
|
);
|
|
6375
6762
|
counts.community_assignments++;
|
|
6763
|
+
report?.({ phase: "communities", done: ++doneCommunities, total: totalCommunities });
|
|
6764
|
+
await maybeYield2(doneCommunities, batchSize);
|
|
6376
6765
|
}
|
|
6377
|
-
|
|
6766
|
+
report?.({ phase: "embedding_set_members", done: 0, total: parsedEmbMembers.length });
|
|
6767
|
+
for (const [index, member] of parsedEmbMembers.entries()) {
|
|
6378
6768
|
await tx.query(
|
|
6379
6769
|
`INSERT INTO embedding_set_member (embedding_set_id, note_id, embedding_id)
|
|
6380
6770
|
VALUES ($1, $2, $3) ON CONFLICT DO NOTHING`,
|
|
6381
6771
|
[member.embedding_set_id, member.note_id, member.embedding_id]
|
|
6382
6772
|
);
|
|
6383
6773
|
counts.embedding_set_members++;
|
|
6774
|
+
report?.({ phase: "embedding_set_members", done: index + 1, total: parsedEmbMembers.length });
|
|
6775
|
+
await maybeYield2(index + 1, batchSize);
|
|
6384
6776
|
}
|
|
6385
6777
|
});
|
|
6778
|
+
report?.({ phase: "index", done: 1, total: 1 });
|
|
6386
6779
|
} catch (err) {
|
|
6387
6780
|
return {
|
|
6388
6781
|
success: false,
|
|
@@ -6433,8 +6826,15 @@ var VALID_TYPES = /* @__PURE__ */ new Set([
|
|
|
6433
6826
|
"crm.organization",
|
|
6434
6827
|
"crm.event",
|
|
6435
6828
|
"crm.interaction",
|
|
6436
|
-
"aiwg.artifact"
|
|
6829
|
+
"aiwg.artifact",
|
|
6830
|
+
"docs.page"
|
|
6437
6831
|
]);
|
|
6832
|
+
var DEFAULT_QUERY_WEIGHTS = {
|
|
6833
|
+
title: 4,
|
|
6834
|
+
tag: 3,
|
|
6835
|
+
concept: 2,
|
|
6836
|
+
text: 1
|
|
6837
|
+
};
|
|
6438
6838
|
function hasString(value) {
|
|
6439
6839
|
return typeof value === "string" && value.length > 0;
|
|
6440
6840
|
}
|
|
@@ -6442,6 +6842,16 @@ function pushFacet(counts, name, value) {
|
|
|
6442
6842
|
counts[name] ??= {};
|
|
6443
6843
|
counts[name][value] = (counts[name][value] ?? 0) + 1;
|
|
6444
6844
|
}
|
|
6845
|
+
function hasNonNegativeInteger(value) {
|
|
6846
|
+
return Number.isInteger(value) && typeof value === "number" && value >= 0;
|
|
6847
|
+
}
|
|
6848
|
+
function hasPositiveInteger(value) {
|
|
6849
|
+
return Number.isInteger(value) && typeof value === "number" && value > 0;
|
|
6850
|
+
}
|
|
6851
|
+
function isFacetCounts(value) {
|
|
6852
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return false;
|
|
6853
|
+
return Object.values(value).every((counts) => !!counts && typeof counts === "object" && !Array.isArray(counts) && Object.values(counts).every((count) => hasNonNegativeInteger(count)));
|
|
6854
|
+
}
|
|
6445
6855
|
function validateAiwgFortemiIndexExport(value) {
|
|
6446
6856
|
const errors = [];
|
|
6447
6857
|
const counts = {};
|
|
@@ -6493,6 +6903,87 @@ function assertAiwgFortemiIndexExport(value) {
|
|
|
6493
6903
|
}
|
|
6494
6904
|
return value;
|
|
6495
6905
|
}
|
|
6906
|
+
function validateAiwgFortemiChunkManifest(value) {
|
|
6907
|
+
const errors = [];
|
|
6908
|
+
const data = value;
|
|
6909
|
+
if (data?.schema_version !== "aiwg.fortemi.index.chunk-manifest.v1") {
|
|
6910
|
+
errors.push("schema_version must be aiwg.fortemi.index.chunk-manifest.v1");
|
|
6911
|
+
}
|
|
6912
|
+
if (!hasString(data?.generated_at)) errors.push("generated_at is required");
|
|
6913
|
+
if (!hasString(data?.source?.repo)) errors.push("source.repo is required");
|
|
6914
|
+
if (!hasString(data?.source?.privacy)) errors.push("source.privacy is required");
|
|
6915
|
+
if (!hasNonNegativeInteger(data?.total)) errors.push("total must be a non-negative integer");
|
|
6916
|
+
if (!hasPositiveInteger(data?.part_size)) errors.push("part_size must be a positive integer");
|
|
6917
|
+
if (data.facets !== void 0 && !isFacetCounts(data.facets)) {
|
|
6918
|
+
errors.push("facets must be a nested string-to-number count object");
|
|
6919
|
+
}
|
|
6920
|
+
if (!Array.isArray(data?.parts)) errors.push("parts must be an array");
|
|
6921
|
+
let expectedOffset = 0;
|
|
6922
|
+
const parts = Array.isArray(data?.parts) ? data.parts : [];
|
|
6923
|
+
for (const [index, part] of parts.entries()) {
|
|
6924
|
+
if (!hasString(part.href)) errors.push("parts[" + index + "].href is required");
|
|
6925
|
+
if (!hasNonNegativeInteger(part.offset)) errors.push("parts[" + index + "].offset must be a non-negative integer");
|
|
6926
|
+
if (!hasNonNegativeInteger(part.count)) errors.push("parts[" + index + "].count must be a non-negative integer");
|
|
6927
|
+
if (hasNonNegativeInteger(part.offset) && part.offset !== expectedOffset) {
|
|
6928
|
+
errors.push("parts[" + index + "].offset must be " + expectedOffset);
|
|
6929
|
+
}
|
|
6930
|
+
if (hasNonNegativeInteger(part.count)) expectedOffset += part.count;
|
|
6931
|
+
}
|
|
6932
|
+
if (hasNonNegativeInteger(data?.total) && expectedOffset !== data.total) {
|
|
6933
|
+
errors.push("parts counts must add up to total");
|
|
6934
|
+
}
|
|
6935
|
+
return { valid: errors.length === 0, errors };
|
|
6936
|
+
}
|
|
6937
|
+
function assertAiwgFortemiChunkManifest(value) {
|
|
6938
|
+
const result = validateAiwgFortemiChunkManifest(value);
|
|
6939
|
+
if (!result.valid) {
|
|
6940
|
+
throw new Error("Invalid AIWG Fortemi chunk manifest:\n" + result.errors.join("\n"));
|
|
6941
|
+
}
|
|
6942
|
+
return value;
|
|
6943
|
+
}
|
|
6944
|
+
function validateAiwgFortemiChunkPart(value, partRef, manifest) {
|
|
6945
|
+
const errors = [];
|
|
6946
|
+
const data = value;
|
|
6947
|
+
if (data?.schema_version !== "aiwg.fortemi.index.chunk.v1") {
|
|
6948
|
+
errors.push("schema_version must be aiwg.fortemi.index.chunk.v1");
|
|
6949
|
+
}
|
|
6950
|
+
if (data?.manifest_schema_version !== "aiwg.fortemi.index.chunk-manifest.v1") {
|
|
6951
|
+
errors.push("manifest_schema_version must be aiwg.fortemi.index.chunk-manifest.v1");
|
|
6952
|
+
}
|
|
6953
|
+
if (!hasNonNegativeInteger(data?.offset)) errors.push("offset must be a non-negative integer");
|
|
6954
|
+
if (!Array.isArray(data?.items)) errors.push("items must be an array");
|
|
6955
|
+
if (partRef && hasNonNegativeInteger(data?.offset) && data.offset !== partRef.offset) {
|
|
6956
|
+
errors.push("offset must match manifest part offset " + partRef.offset);
|
|
6957
|
+
}
|
|
6958
|
+
if (partRef && Array.isArray(data?.items) && data.items.length !== partRef.count) {
|
|
6959
|
+
errors.push("items length must match manifest part count " + partRef.count);
|
|
6960
|
+
}
|
|
6961
|
+
if (Array.isArray(data?.items)) {
|
|
6962
|
+
const validation = validateAiwgFortemiIndexExport({
|
|
6963
|
+
schema_version: "aiwg.fortemi.index.export.v1",
|
|
6964
|
+
generated_at: manifest?.generated_at ?? "1970-01-01T00:00:00.000Z",
|
|
6965
|
+
source: manifest?.source ?? { repo: "chunk", privacy: "public" },
|
|
6966
|
+
items: data.items
|
|
6967
|
+
});
|
|
6968
|
+
errors.push(...validation.errors.map((error) => "items." + error));
|
|
6969
|
+
}
|
|
6970
|
+
return { valid: errors.length === 0, errors };
|
|
6971
|
+
}
|
|
6972
|
+
function assertAiwgFortemiChunkPart(value, partRef, manifest) {
|
|
6973
|
+
const result = validateAiwgFortemiChunkPart(value, partRef, manifest);
|
|
6974
|
+
if (!result.valid) {
|
|
6975
|
+
throw new Error("Invalid AIWG Fortemi chunk part:\n" + result.errors.join("\n"));
|
|
6976
|
+
}
|
|
6977
|
+
return value;
|
|
6978
|
+
}
|
|
6979
|
+
function createAiwgFetchChunkLoader(baseUrl) {
|
|
6980
|
+
return async (part) => {
|
|
6981
|
+
const href = baseUrl ? new URL(part.href, baseUrl).toString() : part.href;
|
|
6982
|
+
const response = await fetch(href);
|
|
6983
|
+
if (!response.ok) throw new Error("Failed to fetch AIWG index chunk " + href + ": " + response.status);
|
|
6984
|
+
return response.json();
|
|
6985
|
+
};
|
|
6986
|
+
}
|
|
6496
6987
|
function getAiwgFortemiFacets(items) {
|
|
6497
6988
|
const result = {};
|
|
6498
6989
|
for (const item of items) {
|
|
@@ -6515,27 +7006,167 @@ function matchesFacetFilters(item, filters) {
|
|
|
6515
7006
|
if (!filters) return true;
|
|
6516
7007
|
return Object.entries(filters).every(([name, expected]) => includesAll(item.facets[name] ?? [], expected));
|
|
6517
7008
|
}
|
|
6518
|
-
function
|
|
6519
|
-
|
|
6520
|
-
const
|
|
6521
|
-
|
|
6522
|
-
|
|
6523
|
-
|
|
6524
|
-
}
|
|
7009
|
+
function queryMatches(item, q) {
|
|
7010
|
+
if (!q) return [];
|
|
7011
|
+
const matches = [];
|
|
7012
|
+
if (item.title.toLowerCase().includes(q)) matches.push({ field: "title", value: item.title });
|
|
7013
|
+
if (item.text.toLowerCase().includes(q)) matches.push({ field: "text", value: item.text });
|
|
7014
|
+
for (const tag of item.tags) {
|
|
7015
|
+
if (tag.toLowerCase().includes(q)) matches.push({ field: "tag", value: tag });
|
|
7016
|
+
}
|
|
7017
|
+
for (const concept of item.concepts) {
|
|
7018
|
+
if (concept.toLowerCase().includes(q)) matches.push({ field: "concept", value: concept });
|
|
7019
|
+
}
|
|
7020
|
+
return matches;
|
|
7021
|
+
}
|
|
7022
|
+
function rankMatches(matches, weights) {
|
|
7023
|
+
return matches.reduce((total, match) => total + weights[match.field], 0);
|
|
7024
|
+
}
|
|
7025
|
+
function clipSnippet(value, q, maxLength) {
|
|
7026
|
+
const normalizedLength = Math.max(20, maxLength);
|
|
7027
|
+
if (!value) return "";
|
|
7028
|
+
if (!q) return value.length > normalizedLength ? `${value.slice(0, normalizedLength).trimEnd()}...` : value;
|
|
7029
|
+
const lower = value.toLowerCase();
|
|
7030
|
+
const index = lower.indexOf(q);
|
|
7031
|
+
if (index < 0) return value.length > normalizedLength ? `${value.slice(0, normalizedLength).trimEnd()}...` : value;
|
|
7032
|
+
const context = Math.max(0, Math.floor((normalizedLength - q.length) / 2));
|
|
7033
|
+
const start = Math.max(0, index - context);
|
|
7034
|
+
const end = Math.min(value.length, start + normalizedLength);
|
|
7035
|
+
const prefix = start > 0 ? "..." : "";
|
|
7036
|
+
const suffix = end < value.length ? "..." : "";
|
|
7037
|
+
return `${prefix}${value.slice(start, end).trim()}${suffix}`;
|
|
7038
|
+
}
|
|
7039
|
+
function createSnippet(item, matches, q, maxLength) {
|
|
7040
|
+
const textMatch = matches.find((match) => match.field === "text");
|
|
7041
|
+
const titleMatch = matches.find((match) => match.field === "title");
|
|
7042
|
+
const firstMatch = textMatch ?? titleMatch ?? matches[0];
|
|
7043
|
+
return clipSnippet(firstMatch?.value ?? item.text, q, maxLength);
|
|
7044
|
+
}
|
|
7045
|
+
function createRankedEntries(items, q, options, ordinalBase = 0) {
|
|
7046
|
+
const weights = { ...DEFAULT_QUERY_WEIGHTS, ...options.weights };
|
|
7047
|
+
return items.map((item, ordinal) => ({ item, ordinal: ordinalBase + ordinal, matches: queryMatches(item, q) })).filter(({ item, matches }) => {
|
|
7048
|
+
if (q && matches.length === 0) return false;
|
|
6525
7049
|
if (options.types && !options.types.includes(item.type)) return false;
|
|
6526
7050
|
if (options.privacy && !options.privacy.includes(item.privacy.classification)) return false;
|
|
6527
7051
|
if (!includesAll(item.tags, options.tags)) return false;
|
|
6528
7052
|
if (!includesAll(item.concepts, options.concepts)) return false;
|
|
6529
7053
|
if (!matchesFacetFilters(item, options.facets)) return false;
|
|
6530
|
-
if (options.relationshipTargetId && !item.relationships.some((rel) => rel.target_id === options.relationshipTargetId))
|
|
7054
|
+
if (options.relationshipTargetId && !item.relationships.some((rel) => rel.target_id === options.relationshipTargetId)) {
|
|
7055
|
+
return false;
|
|
7056
|
+
}
|
|
6531
7057
|
return true;
|
|
7058
|
+
}).map(({ item, ordinal, matches }) => ({
|
|
7059
|
+
item,
|
|
7060
|
+
ordinal,
|
|
7061
|
+
rank: rankMatches(matches, weights),
|
|
7062
|
+
matches
|
|
7063
|
+
}));
|
|
7064
|
+
}
|
|
7065
|
+
function sortRankedEntries(entries, rank) {
|
|
7066
|
+
return [...entries].sort((left, right) => {
|
|
7067
|
+
if (rank) return right.rank - left.rank || left.ordinal - right.ordinal;
|
|
7068
|
+
return left.ordinal - right.ordinal;
|
|
6532
7069
|
});
|
|
7070
|
+
}
|
|
7071
|
+
function createQueryResultFromRankedEntries(entries, query, options) {
|
|
7072
|
+
const ranked = sortRankedEntries(entries, options.rank);
|
|
6533
7073
|
const offset = options.offset ?? 0;
|
|
6534
|
-
const limit = options.limit ??
|
|
7074
|
+
const limit = options.limit ?? ranked.length;
|
|
7075
|
+
const page = ranked.slice(offset, offset + limit);
|
|
7076
|
+
const result = {
|
|
7077
|
+
items: page.map((entry) => entry.item),
|
|
7078
|
+
total: ranked.length,
|
|
7079
|
+
facets: getAiwgFortemiFacets(ranked.map((entry) => entry.item))
|
|
7080
|
+
};
|
|
7081
|
+
if (options.rank || options.snippets || options.includeMatches) {
|
|
7082
|
+
const snippetLength = options.snippetLength ?? 160;
|
|
7083
|
+
result.rankedItems = page.map((entry) => ({
|
|
7084
|
+
item: entry.item,
|
|
7085
|
+
rank: entry.rank,
|
|
7086
|
+
...options.snippets ? { snippet: createSnippet(entry.item, entry.matches, query, snippetLength) } : {},
|
|
7087
|
+
...options.includeMatches ? { matches: entry.matches } : {}
|
|
7088
|
+
}));
|
|
7089
|
+
}
|
|
7090
|
+
return result;
|
|
7091
|
+
}
|
|
7092
|
+
function queryAiwgFortemiIndex(index, query = "", options = {}) {
|
|
7093
|
+
const q = query.trim().toLowerCase();
|
|
7094
|
+
return createQueryResultFromRankedEntries(createRankedEntries(index.items, q, options), q, options);
|
|
7095
|
+
}
|
|
7096
|
+
function chunkPartCacheKey(part) {
|
|
7097
|
+
return `${part.offset}:${part.href}`;
|
|
7098
|
+
}
|
|
7099
|
+
function clampMaxCachedParts(value) {
|
|
7100
|
+
if (!hasPositiveInteger(value)) return 3;
|
|
7101
|
+
return value;
|
|
7102
|
+
}
|
|
7103
|
+
function isDirectChunkBrowse(query, options) {
|
|
7104
|
+
return query.trim() === "" && !options.rank && !options.snippets && !options.includeMatches && !options.types && !options.facets && !options.tags && !options.concepts && !options.privacy && !options.relationshipTargetId;
|
|
7105
|
+
}
|
|
7106
|
+
function getPartsForRange(manifest, offset, limit) {
|
|
7107
|
+
const end = offset + limit;
|
|
7108
|
+
return manifest.parts.filter((part) => part.count > 0 && part.offset < end && part.offset + part.count > offset);
|
|
7109
|
+
}
|
|
7110
|
+
async function loadChunkPart(runtime, part) {
|
|
7111
|
+
const key = chunkPartCacheKey(part);
|
|
7112
|
+
const cached = runtime.partCache.get(key);
|
|
7113
|
+
if (cached) {
|
|
7114
|
+
runtime.partCache.delete(key);
|
|
7115
|
+
runtime.partCache.set(key, cached);
|
|
7116
|
+
return { part: cached, fetched: false };
|
|
7117
|
+
}
|
|
7118
|
+
const parsed = assertAiwgFortemiChunkPart(await runtime.loader(part, runtime.manifest), part, runtime.manifest);
|
|
7119
|
+
runtime.partCache.set(key, parsed);
|
|
7120
|
+
while (runtime.partCache.size > runtime.maxCachedParts) {
|
|
7121
|
+
const oldest = runtime.partCache.keys().next().value;
|
|
7122
|
+
if (oldest === void 0) break;
|
|
7123
|
+
runtime.partCache.delete(oldest);
|
|
7124
|
+
}
|
|
7125
|
+
return { part: parsed, fetched: true };
|
|
7126
|
+
}
|
|
7127
|
+
async function queryChunkedAiwgFortemiIndex(runtime, query = "", options = {}) {
|
|
7128
|
+
const q = query.trim().toLowerCase();
|
|
7129
|
+
let scannedParts = 0;
|
|
7130
|
+
let fetchedParts = 0;
|
|
7131
|
+
if (isDirectChunkBrowse(query, options)) {
|
|
7132
|
+
const offset = options.offset ?? 0;
|
|
7133
|
+
const limit = options.limit ?? runtime.manifest.total;
|
|
7134
|
+
const parts = getPartsForRange(runtime.manifest, offset, limit);
|
|
7135
|
+
const items = [];
|
|
7136
|
+
for (const partRef of parts) {
|
|
7137
|
+
const loaded = await loadChunkPart(runtime, partRef);
|
|
7138
|
+
if (loaded.fetched) fetchedParts += 1;
|
|
7139
|
+
scannedParts += 1;
|
|
7140
|
+
options.onProgress?.({ phase: "part", done: scannedParts, total: parts.length, href: partRef.href });
|
|
7141
|
+
const start = Math.max(0, offset - partRef.offset);
|
|
7142
|
+
const end = Math.min(loaded.part.items.length, offset + limit - partRef.offset);
|
|
7143
|
+
items.push(...loaded.part.items.slice(start, end));
|
|
7144
|
+
}
|
|
7145
|
+
return {
|
|
7146
|
+
items,
|
|
7147
|
+
total: runtime.manifest.total,
|
|
7148
|
+
facets: runtime.manifest.facets ?? {},
|
|
7149
|
+
manifestTotal: runtime.manifest.total,
|
|
7150
|
+
scannedParts,
|
|
7151
|
+
fetchedParts,
|
|
7152
|
+
complete: true
|
|
7153
|
+
};
|
|
7154
|
+
}
|
|
7155
|
+
const entries = [];
|
|
7156
|
+
for (const partRef of runtime.manifest.parts) {
|
|
7157
|
+
const loaded = await loadChunkPart(runtime, partRef);
|
|
7158
|
+
if (loaded.fetched) fetchedParts += 1;
|
|
7159
|
+
scannedParts += 1;
|
|
7160
|
+
options.onProgress?.({ phase: "part", done: scannedParts, total: runtime.manifest.parts.length, href: partRef.href });
|
|
7161
|
+
entries.push(...createRankedEntries(loaded.part.items, q, options, partRef.offset));
|
|
7162
|
+
options.onProgress?.({ phase: "query", done: scannedParts, total: runtime.manifest.parts.length, href: partRef.href });
|
|
7163
|
+
}
|
|
6535
7164
|
return {
|
|
6536
|
-
|
|
6537
|
-
|
|
6538
|
-
|
|
7165
|
+
...createQueryResultFromRankedEntries(entries, q, options),
|
|
7166
|
+
manifestTotal: runtime.manifest.total,
|
|
7167
|
+
scannedParts,
|
|
7168
|
+
fetchedParts,
|
|
7169
|
+
complete: true
|
|
6539
7170
|
};
|
|
6540
7171
|
}
|
|
6541
7172
|
function createAiwgReviewDecisionExport(source, decisions, generatedAt = (/* @__PURE__ */ new Date()).toISOString()) {
|
|
@@ -6546,10 +7177,184 @@ function createAiwgReviewDecisionExport(source, decisions, generatedAt = (/* @__
|
|
|
6546
7177
|
decisions: [...decisions].sort((left, right) => left.item_id.localeCompare(right.item_id))
|
|
6547
7178
|
};
|
|
6548
7179
|
}
|
|
7180
|
+
function createAiwgIndexController(initialIndex) {
|
|
7181
|
+
let index = initialIndex ?? null;
|
|
7182
|
+
let chunked = null;
|
|
7183
|
+
let data = null;
|
|
7184
|
+
let error = null;
|
|
7185
|
+
let reviewDecisions = [];
|
|
7186
|
+
const listeners = /* @__PURE__ */ new Set();
|
|
7187
|
+
const snapshot = () => ({
|
|
7188
|
+
index,
|
|
7189
|
+
chunked: chunked ? {
|
|
7190
|
+
manifest: chunked.manifest,
|
|
7191
|
+
cachedParts: chunked.partCache.size,
|
|
7192
|
+
maxCachedParts: chunked.maxCachedParts
|
|
7193
|
+
} : null,
|
|
7194
|
+
data,
|
|
7195
|
+
error,
|
|
7196
|
+
reviewDecisions: [...reviewDecisions]
|
|
7197
|
+
});
|
|
7198
|
+
const notify = () => {
|
|
7199
|
+
const current = snapshot();
|
|
7200
|
+
for (const listener of listeners) listener(current);
|
|
7201
|
+
};
|
|
7202
|
+
const requireIndex = () => {
|
|
7203
|
+
if (!index) throw new Error("No AIWG index export loaded");
|
|
7204
|
+
return index;
|
|
7205
|
+
};
|
|
7206
|
+
return {
|
|
7207
|
+
loadIndex(value) {
|
|
7208
|
+
try {
|
|
7209
|
+
const parsed = assertAiwgFortemiIndexExport(value);
|
|
7210
|
+
index = parsed;
|
|
7211
|
+
chunked = null;
|
|
7212
|
+
data = null;
|
|
7213
|
+
reviewDecisions = [];
|
|
7214
|
+
error = null;
|
|
7215
|
+
notify();
|
|
7216
|
+
return parsed;
|
|
7217
|
+
} catch (err) {
|
|
7218
|
+
error = err instanceof Error ? err : new Error(String(err));
|
|
7219
|
+
notify();
|
|
7220
|
+
throw error;
|
|
7221
|
+
}
|
|
7222
|
+
},
|
|
7223
|
+
loadChunkedIndex(manifest, loader, options = {}) {
|
|
7224
|
+
try {
|
|
7225
|
+
const parsed = assertAiwgFortemiChunkManifest(manifest);
|
|
7226
|
+
index = null;
|
|
7227
|
+
chunked = {
|
|
7228
|
+
manifest: parsed,
|
|
7229
|
+
loader,
|
|
7230
|
+
maxCachedParts: clampMaxCachedParts(options.maxCachedParts),
|
|
7231
|
+
partCache: /* @__PURE__ */ new Map()
|
|
7232
|
+
};
|
|
7233
|
+
data = null;
|
|
7234
|
+
reviewDecisions = [];
|
|
7235
|
+
error = null;
|
|
7236
|
+
notify();
|
|
7237
|
+
return parsed;
|
|
7238
|
+
} catch (err) {
|
|
7239
|
+
error = err instanceof Error ? err : new Error(String(err));
|
|
7240
|
+
notify();
|
|
7241
|
+
throw error;
|
|
7242
|
+
}
|
|
7243
|
+
},
|
|
7244
|
+
getIndex() {
|
|
7245
|
+
return index;
|
|
7246
|
+
},
|
|
7247
|
+
getChunkedManifest() {
|
|
7248
|
+
return chunked?.manifest ?? null;
|
|
7249
|
+
},
|
|
7250
|
+
getSnapshot() {
|
|
7251
|
+
return snapshot();
|
|
7252
|
+
},
|
|
7253
|
+
query(query = "", options) {
|
|
7254
|
+
const result = queryAiwgFortemiIndex(requireIndex(), query, options);
|
|
7255
|
+
data = result;
|
|
7256
|
+
error = null;
|
|
7257
|
+
notify();
|
|
7258
|
+
return result;
|
|
7259
|
+
},
|
|
7260
|
+
async queryChunked(query = "", options) {
|
|
7261
|
+
if (!chunked) throw new Error("No AIWG chunked index manifest loaded");
|
|
7262
|
+
try {
|
|
7263
|
+
const result = await queryChunkedAiwgFortemiIndex(chunked, query, options);
|
|
7264
|
+
data = result;
|
|
7265
|
+
error = null;
|
|
7266
|
+
notify();
|
|
7267
|
+
return result;
|
|
7268
|
+
} catch (err) {
|
|
7269
|
+
error = err instanceof Error ? err : new Error(String(err));
|
|
7270
|
+
notify();
|
|
7271
|
+
throw error;
|
|
7272
|
+
}
|
|
7273
|
+
},
|
|
7274
|
+
clearChunkCache() {
|
|
7275
|
+
chunked?.partCache.clear();
|
|
7276
|
+
error = null;
|
|
7277
|
+
notify();
|
|
7278
|
+
},
|
|
7279
|
+
toCommunityGraph(options) {
|
|
7280
|
+
return aiwgFortemiIndexToCommunityGraph(requireIndex(), options);
|
|
7281
|
+
},
|
|
7282
|
+
setReviewDecision(input) {
|
|
7283
|
+
const decision = {
|
|
7284
|
+
...input,
|
|
7285
|
+
updated_at: (/* @__PURE__ */ new Date()).toISOString()
|
|
7286
|
+
};
|
|
7287
|
+
reviewDecisions = [
|
|
7288
|
+
...reviewDecisions.filter((item) => item.item_id !== decision.item_id),
|
|
7289
|
+
decision
|
|
7290
|
+
].sort((left, right) => left.item_id.localeCompare(right.item_id));
|
|
7291
|
+
error = null;
|
|
7292
|
+
notify();
|
|
7293
|
+
return decision;
|
|
7294
|
+
},
|
|
7295
|
+
clearReviewDecision(itemId) {
|
|
7296
|
+
reviewDecisions = reviewDecisions.filter((item) => item.item_id !== itemId);
|
|
7297
|
+
error = null;
|
|
7298
|
+
notify();
|
|
7299
|
+
},
|
|
7300
|
+
createReviewDecisionExport(generatedAt) {
|
|
7301
|
+
return createAiwgReviewDecisionExport(requireIndex(), reviewDecisions, generatedAt);
|
|
7302
|
+
},
|
|
7303
|
+
subscribe(listener) {
|
|
7304
|
+
listeners.add(listener);
|
|
7305
|
+
return () => {
|
|
7306
|
+
listeners.delete(listener);
|
|
7307
|
+
};
|
|
7308
|
+
}
|
|
7309
|
+
};
|
|
7310
|
+
}
|
|
7311
|
+
function aiwgFortemiIndexToCommunityGraph(index, options = {}) {
|
|
7312
|
+
const ids = new Set(index.items.map((item) => item.id));
|
|
7313
|
+
const relationshipWeights = options.relationshipWeights ?? {};
|
|
7314
|
+
const edgeCounts = /* @__PURE__ */ new Map();
|
|
7315
|
+
for (const item of index.items) {
|
|
7316
|
+
for (const relationship of item.relationships) {
|
|
7317
|
+
if (!ids.has(relationship.target_id) && !options.includeDanglingRelationships) continue;
|
|
7318
|
+
const kind = relationship.type;
|
|
7319
|
+
const baseWeight = relationshipWeights[kind] ?? 1;
|
|
7320
|
+
const key = `${item.id}\0${relationship.target_id}\0${kind}`;
|
|
7321
|
+
const existing = edgeCounts.get(key);
|
|
7322
|
+
if (existing) existing.weight += baseWeight;
|
|
7323
|
+
else edgeCounts.set(key, { source: item.id, target: relationship.target_id, kind, weight: baseWeight });
|
|
7324
|
+
}
|
|
7325
|
+
}
|
|
7326
|
+
const communities = /* @__PURE__ */ new Map();
|
|
7327
|
+
for (const item of index.items) {
|
|
7328
|
+
const communityIds = communityIdsFor(item, options);
|
|
7329
|
+
for (const communityId of communityIds) {
|
|
7330
|
+
const nodes = communities.get(communityId) ?? [];
|
|
7331
|
+
nodes.push(item.id);
|
|
7332
|
+
communities.set(communityId, nodes);
|
|
7333
|
+
}
|
|
7334
|
+
}
|
|
7335
|
+
return {
|
|
7336
|
+
nodes: index.items.map((item) => ({ id: item.id })),
|
|
7337
|
+
edges: Array.from(edgeCounts.values()).sort((left, right) => left.source.localeCompare(right.source) || left.target.localeCompare(right.target) || left.kind.localeCompare(right.kind)),
|
|
7338
|
+
communities: Array.from(communities.entries()).map(([id, nodes]) => ({ id, nodes: [...new Set(nodes)].sort() })).sort((left, right) => left.id.localeCompare(right.id))
|
|
7339
|
+
};
|
|
7340
|
+
}
|
|
7341
|
+
function communityIdsFor(item, options) {
|
|
7342
|
+
if (options.communityFacet) {
|
|
7343
|
+
const values = item.facets[options.communityFacet] ?? [];
|
|
7344
|
+
if (values.length > 0) return values.map((value) => `${options.communityFacet}:${value}`);
|
|
7345
|
+
}
|
|
7346
|
+
if (options.communityTagPrefix) {
|
|
7347
|
+
const prefix = options.communityTagPrefix;
|
|
7348
|
+
const tags = item.tags.filter((tag) => tag.startsWith(prefix));
|
|
7349
|
+
if (tags.length > 0) return tags;
|
|
7350
|
+
}
|
|
7351
|
+
if (item.concepts.length > 0) return item.concepts.map((concept) => `concept:${concept}`);
|
|
7352
|
+
return [`type:${item.type}`];
|
|
7353
|
+
}
|
|
6549
7354
|
|
|
6550
7355
|
// src/index.ts
|
|
6551
|
-
var VERSION = "2026.6.
|
|
7356
|
+
var VERSION = "2026.6.2";
|
|
6552
7357
|
|
|
6553
|
-
export { ArchiveManager, AttachmentsRepository, CURRENT_SHARD_VERSION, CapabilityManager, CaptureKnowledgeInputSchema, CollectionsRepository, CommunitiesRepository, EmbeddingSetsRepository, FallbackRouter, FortemiToolManifest, GetNoteInputSchema, GraphRepository, JOB_CAPABILITIES, JOB_PRIORITIES, JobQueueWorker, LOCAL_ENDPOINTS, LinksRepository, ListNotesInputSchema, ManageArchiveInputSchema, ManageAttachmentsInputSchema, ManageCapabilitiesInputSchema, ManageCollectionsInputSchema, ManageLinksInputSchema, ManageNoteInputSchema, ManageTagsInputSchema, MemoryBlobStore, MigrationRunner, NotesRepository, OpenAICompatibleProvider, PGliteStorageBackend, PGliteStorageBackendFactory, PGliteWorkerClient, ProviderRegistry, SHARD_FORMAT, SearchInputSchema, SearchRepository, SkosRepository, TagsRepository, TransactionProxy, TypedEventBus, VERSION, aiRevisionHandler, allMigrations, appendPluginScript, assertAiwgFortemiIndexExport, buildNoteConditions, buildPluginCsp, captureKnowledge, chunkText, classifyError, classifyModel, collectionFromShard, collectionToShard, computeHash, computeSri, conceptTaggingHandler, cosineSimilarity, createAiwgReviewDecisionExport, createBlobStore, createCspReportHandler, createFortemi, createLegacyProvider, createPGliteInstance, createRoutes, defaultStorageBackendFactory, detectCommunities, detectGpuCapabilities, detectInferenceCapabilities, discoverLocalProviders, embeddingFromShard, embeddingGenerationHandler, embeddingSetFromShard, embeddingSetMemberToShard, embeddingSetToShard, embeddingToShard, enqueueFullWorkflow, enqueueJob, enqueueNoteCreationJobs, estimateModelFit, estimateVramMB, estimateVramTier, exportShard, fetchPluginScript, fortemiManifest, generateId, getAiwgFortemiFacets, getEmbedFunction, getJobQueueStatus, getLlmFunction, getNote, importShard, isPluginScriptAllowed, linkFromShard, linkToShard, linkingHandler, listNotes, manageArchive, manageAttachments, manageCapabilities, manageCollections, manageLinks, manageNote, manageTags, matchRoute, noteFromShard, noteSkosTagToShard, noteToShard, packTarGz, parseCspReport, provenanceEdgeToShard, queryAiwgFortemiIndex, registerLlmCapability, registerSemanticCapability, registerServiceWorker, searchTool, selectLlmModel, setEmbedFunction, setLlmFunction, sha256Hex, skosConceptToShard, skosRelationToShard, skosSchemeToShard, suggestTags, tagsFromShard, tagsToShard, titleGenerationHandler, unpackTarGz, unregisterLlmCapability, unregisterSemanticCapability, validateAiwgFortemiIndexExport, validateChecksums, verifySri };
|
|
7358
|
+
export { ArchiveManager, AttachmentsRepository, CURRENT_SHARD_VERSION, CapabilityManager, CaptureKnowledgeInputSchema, CollectionsRepository, CommunitiesRepository, EmbeddingSetsRepository, FallbackRouter, FortemiToolManifest, GetNoteInputSchema, GraphRepository, JOB_CAPABILITIES, JOB_PRIORITIES, JobQueueWorker, LOCAL_ENDPOINTS, LinksRepository, ListNotesInputSchema, ManageArchiveInputSchema, ManageAttachmentsInputSchema, ManageCapabilitiesInputSchema, ManageCollectionsInputSchema, ManageLinksInputSchema, ManageNoteInputSchema, ManageTagsInputSchema, MemoryBlobStore, MigrationRunner, NotesRepository, OpenAICompatibleProvider, PGliteStorageBackend, PGliteStorageBackendFactory, PGliteWorkerClient, PGliteWorkerStorageBackend, PGliteWorkerStorageBackendFactory, ProviderRegistry, SHARD_FORMAT, SearchInputSchema, SearchRepository, SkosRepository, TagsRepository, TransactionProxy, TypedEventBus, VERSION, aiRevisionHandler, aiwgFortemiIndexToCommunityGraph, allMigrations, appendPluginScript, assertAiwgFortemiChunkManifest, assertAiwgFortemiChunkPart, assertAiwgFortemiIndexExport, buildNoteConditions, buildPluginCsp, captureKnowledge, chunkText, classifyError, classifyModel, collectionFromShard, collectionToShard, computeHash, computeSri, conceptTaggingHandler, cosineSimilarity, createAiwgFetchChunkLoader, createAiwgIndexController, createAiwgReviewDecisionExport, createBlobStore, createCspReportHandler, createFortemi, createLegacyProvider, createPGliteInstance, createRoutes, defaultStorageBackendFactory, detectCommunities, detectGpuCapabilities, detectInferenceCapabilities, discoverLocalProviders, embeddingFromShard, embeddingGenerationHandler, embeddingSetFromShard, embeddingSetMemberToShard, embeddingSetToShard, embeddingToShard, enqueueFullWorkflow, enqueueJob, enqueueNoteCreationJobs, estimateModelFit, estimateVramMB, estimateVramTier, exportShard, fetchPluginScript, fortemiManifest, generateId, getAiwgFortemiFacets, getEmbedFunction, getFortemiBridge, getFortemiSecretStore, getJobQueueStatus, getLlmFunction, getNote, hasFortemiSecureSecrets, importShard, isPluginScriptAllowed, linkFromShard, linkToShard, linkingHandler, listNotes, manageArchive, manageAttachments, manageCapabilities, manageCollections, manageLinks, manageNote, manageTags, matchRoute, noteFromShard, noteSkosTagToShard, noteToShard, packTarGz, parseCspReport, provenanceEdgeToShard, queryAiwgFortemiIndex, registerLlmCapability, registerSemanticCapability, registerServiceWorker, searchTool, selectLlmModel, setEmbedFunction, setLlmFunction, sha256Hex, skosConceptToShard, skosRelationToShard, skosSchemeToShard, suggestTags, tagsFromShard, tagsToShard, titleGenerationHandler, unpackTarGz, unregisterLlmCapability, unregisterSemanticCapability, validateAiwgFortemiChunkManifest, validateAiwgFortemiChunkPart, validateAiwgFortemiIndexExport, validateChecksums, verifySri };
|
|
6554
7359
|
//# sourceMappingURL=index.js.map
|
|
6555
7360
|
//# sourceMappingURL=index.js.map
|