@fortemi/core 2026.6.0 → 2026.6.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -150,6 +150,103 @@ async function createPGliteInstance(persistence, archiveName = "default") {
150
150
  return db;
151
151
  }
152
152
 
153
+ // src/worker/worker-client.ts
154
+ var PGliteWorkerClient = class {
155
+ constructor(worker) {
156
+ this.worker = worker;
157
+ this.readyPromise = new Promise((resolve) => {
158
+ this.resolveReady = resolve;
159
+ });
160
+ this.worker.addEventListener("message", (e) => {
161
+ const msg = e.data;
162
+ if (msg.type === "READY") {
163
+ this.resolveReady();
164
+ return;
165
+ }
166
+ if (!("id" in msg)) return;
167
+ const pending = this.pending.get(msg.id);
168
+ if (!pending) return;
169
+ this.pending.delete(msg.id);
170
+ if (msg.type === "ERROR") {
171
+ pending.reject(new Error(msg.error));
172
+ } else {
173
+ pending.resolve(msg);
174
+ }
175
+ });
176
+ }
177
+ pending = /* @__PURE__ */ new Map();
178
+ readyPromise;
179
+ resolveReady;
180
+ /** Resolves when the worker broadcasts READY after database initialisation. */
181
+ async waitReady() {
182
+ return this.readyPromise;
183
+ }
184
+ send(request) {
185
+ const id = generateId();
186
+ return new Promise((resolve, reject) => {
187
+ this.pending.set(id, {
188
+ resolve,
189
+ reject
190
+ });
191
+ this.worker.postMessage({ ...request, id });
192
+ });
193
+ }
194
+ async query(sql, params) {
195
+ const resp = await this.send({ type: "QUERY", sql, params });
196
+ return { rows: resp.rows, fields: resp.fields };
197
+ }
198
+ async exec(sql) {
199
+ await this.send({ type: "EXEC", sql });
200
+ }
201
+ async transaction(fn) {
202
+ const resp = await this.send({ type: "BEGIN" });
203
+ const txId = resp.txId;
204
+ const proxy = new TransactionProxy(this, txId);
205
+ try {
206
+ const result = await fn(proxy);
207
+ await this.send({ type: "COMMIT", txId });
208
+ return result;
209
+ } catch (err) {
210
+ await this.send({ type: "ROLLBACK", txId }).catch(() => {
211
+ });
212
+ throw err;
213
+ }
214
+ }
215
+ /** Forward TX_QUERY for TransactionProxy — not part of the public surface. */
216
+ async _txQuery(txId, sql, params) {
217
+ const resp = await this.send({
218
+ type: "TX_QUERY",
219
+ txId,
220
+ sql,
221
+ params
222
+ });
223
+ return { rows: resp.rows };
224
+ }
225
+ /** Forward TX_EXEC for TransactionProxy — not part of the public surface. */
226
+ async _txExec(txId, sql) {
227
+ await this.send({ type: "TX_EXEC", txId, sql });
228
+ }
229
+ async ping() {
230
+ await this.send({ type: "PING" });
231
+ }
232
+ async close() {
233
+ await this.send({ type: "CLOSE" });
234
+ this.worker.terminate();
235
+ }
236
+ };
237
+ var TransactionProxy = class {
238
+ constructor(client, txId) {
239
+ this.client = client;
240
+ this.txId = txId;
241
+ }
242
+ async query(sql, params) {
243
+ return this.client._txQuery(this.txId, sql, params);
244
+ }
245
+ async exec(sql) {
246
+ return this.client._txExec(this.txId, sql);
247
+ }
248
+ };
249
+
153
250
  // src/storage-backend.ts
154
251
  var PGliteStorageBackend = class {
155
252
  constructor(id, db) {
@@ -177,6 +274,41 @@ var PGliteStorageBackendFactory = class {
177
274
  }
178
275
  };
179
276
  var defaultStorageBackendFactory = new PGliteStorageBackendFactory();
277
+ var PGliteWorkerStorageBackend = class {
278
+ constructor(id, client) {
279
+ this.id = id;
280
+ this.client = client;
281
+ }
282
+ mode = "readwrite";
283
+ query(sql, params) {
284
+ return this.client.query(sql, params);
285
+ }
286
+ exec(sql) {
287
+ return this.client.exec(sql);
288
+ }
289
+ transaction(fn) {
290
+ return this.client.transaction(fn);
291
+ }
292
+ close() {
293
+ return this.client.close();
294
+ }
295
+ };
296
+ var PGliteWorkerStorageBackendFactory = class {
297
+ constructor(options) {
298
+ this.options = options;
299
+ }
300
+ async open(input) {
301
+ const worker = this.options.createWorker();
302
+ const client = new PGliteWorkerClient(worker);
303
+ worker.postMessage({
304
+ type: "INIT",
305
+ persistence: input.persistence,
306
+ archiveName: input.archiveName
307
+ });
308
+ await client.waitReady();
309
+ return new PGliteWorkerStorageBackend(`pglite-worker:${input.persistence}:${input.archiveName}`, client);
310
+ }
311
+ };
180
312
 
181
313
  // src/capability-manager.ts
182
314
  var VALID_TRANSITIONS = {
@@ -785,6 +917,28 @@ var migration0008 = {
785
917
  `
786
918
  };
787
919
 
920
+ // src/migrations/0009_vector_selector_performance.ts
921
+ var migration0009 = {
922
+ version: 9,
923
+ name: "0009_vector_selector_performance",
924
+ sql: `
925
+ CREATE INDEX IF NOT EXISTS idx_embedding_set_kind ON embedding_set(kind);
926
+ CREATE INDEX IF NOT EXISTS idx_embedding_set_member_set ON embedding_set_member(embedding_set_id);
927
+ CREATE INDEX IF NOT EXISTS idx_embedding_set_member_note ON embedding_set_member(note_id);
928
+ CREATE INDEX IF NOT EXISTS idx_embedding_set_member_embedding ON embedding_set_member(embedding_id);
929
+
930
+ CREATE INDEX IF NOT EXISTS idx_note_source ON note(source);
931
+ CREATE INDEX IF NOT EXISTS idx_note_format ON note(format);
932
+ CREATE INDEX IF NOT EXISTS idx_note_visibility ON note(visibility);
933
+ CREATE INDEX IF NOT EXISTS idx_note_starred ON note(is_starred);
934
+ CREATE INDEX IF NOT EXISTS idx_note_archived ON note(is_archived);
935
+ CREATE INDEX IF NOT EXISTS idx_note_updated_at ON note(updated_at);
936
+
937
+ CREATE INDEX IF NOT EXISTS idx_note_revised_current_user_edited ON note_revised_current(is_user_edited);
938
+ CREATE INDEX IF NOT EXISTS idx_note_revised_current_generation_count ON note_revised_current(generation_count);
939
+ `
940
+ };
941
+
788
942
  // src/migrations/index.ts
789
943
  var allMigrations = [
790
944
  migration0001,
@@ -794,18 +948,19 @@ var allMigrations = [
794
948
  migration0005,
795
949
  migration0006,
796
950
  migration0007,
797
- migration0008
951
+ migration0008,
952
+ migration0009
798
953
  ];
799
954
 
800
955
  // src/archive-manager.ts
801
956
  var ArchiveManager = class {
802
- constructor(persistenceOrFactory, events) {
957
+ constructor(persistenceOrFactory, events, persistenceOverride) {
803
958
  this.events = events;
804
959
  if (typeof persistenceOrFactory === "string") {
805
960
  this.persistence = persistenceOrFactory;
806
961
  this.backendFactory = defaultStorageBackendFactory;
807
962
  } else {
808
- this.persistence = "memory";
963
+ this.persistence = persistenceOverride ?? "memory";
809
964
  this.backendFactory = persistenceOrFactory;
810
965
  }
811
966
  this.archives.set("default", {
@@ -1170,103 +1325,6 @@ function createBlobStore(archiveName) {
1170
1325
  return new IdbBlobStore(archiveName);
1171
1326
  }
1172
1327
 
1173
- // src/worker/worker-client.ts
1174
- var PGliteWorkerClient = class {
1175
- constructor(worker) {
1176
- this.worker = worker;
1177
- this.readyPromise = new Promise((resolve) => {
1178
- this.resolveReady = resolve;
1179
- });
1180
- this.worker.addEventListener("message", (e) => {
1181
- const msg = e.data;
1182
- if (msg.type === "READY") {
1183
- this.resolveReady();
1184
- return;
1185
- }
1186
- if (!("id" in msg)) return;
1187
- const pending = this.pending.get(msg.id);
1188
- if (!pending) return;
1189
- this.pending.delete(msg.id);
1190
- if (msg.type === "ERROR") {
1191
- pending.reject(new Error(msg.error));
1192
- } else {
1193
- pending.resolve(msg);
1194
- }
1195
- });
1196
- }
1197
- pending = /* @__PURE__ */ new Map();
1198
- readyPromise;
1199
- resolveReady;
1200
- /** Resolves when the worker broadcasts READY after database initialisation. */
1201
- async waitReady() {
1202
- return this.readyPromise;
1203
- }
1204
- send(request) {
1205
- const id = generateId();
1206
- return new Promise((resolve, reject) => {
1207
- this.pending.set(id, {
1208
- resolve,
1209
- reject
1210
- });
1211
- this.worker.postMessage({ ...request, id });
1212
- });
1213
- }
1214
- async query(sql, params) {
1215
- const resp = await this.send({ type: "QUERY", sql, params });
1216
- return { rows: resp.rows, fields: resp.fields };
1217
- }
1218
- async exec(sql) {
1219
- await this.send({ type: "EXEC", sql });
1220
- }
1221
- async transaction(fn) {
1222
- const resp = await this.send({ type: "BEGIN" });
1223
- const txId = resp.txId;
1224
- const proxy = new TransactionProxy(this, txId);
1225
- try {
1226
- const result = await fn(proxy);
1227
- await this.send({ type: "COMMIT", txId });
1228
- return result;
1229
- } catch (err) {
1230
- await this.send({ type: "ROLLBACK", txId }).catch(() => {
1231
- });
1232
- throw err;
1233
- }
1234
- }
1235
- /** Forward TX_QUERY for TransactionProxy — not part of the public surface. */
1236
- async _txQuery(txId, sql, params) {
1237
- const resp = await this.send({
1238
- type: "TX_QUERY",
1239
- txId,
1240
- sql,
1241
- params
1242
- });
1243
- return { rows: resp.rows };
1244
- }
1245
- /** Forward TX_EXEC for TransactionProxy — not part of the public surface. */
1246
- async _txExec(txId, sql) {
1247
- await this.send({ type: "TX_EXEC", txId, sql });
1248
- }
1249
- async ping() {
1250
- await this.send({ type: "PING" });
1251
- }
1252
- async close() {
1253
- await this.send({ type: "CLOSE" });
1254
- this.worker.terminate();
1255
- }
1256
- };
1257
- var TransactionProxy = class {
1258
- constructor(client, txId) {
1259
- this.client = client;
1260
- this.txId = txId;
1261
- }
1262
- async query(sql, params) {
1263
- return this.client._txQuery(this.txId, sql, params);
1264
- }
1265
- async exec(sql) {
1266
- return this.client._txExec(this.txId, sql);
1267
- }
1268
- };
1269
-
1270
1328
  // src/repositories/notes-repository.ts
1271
1329
  var NotesRepository = class {
1272
1330
  constructor(db, events) {
@@ -1690,6 +1748,9 @@ function dateString(value) {
1690
1748
  function dateMillis(value) {
1691
1749
  return value instanceof Date ? value.getTime() : new Date(value).getTime();
1692
1750
  }
1751
+ function hashJson(value) {
1752
+ return computeHash(new TextEncoder().encode(JSON.stringify(value)));
1753
+ }
1693
1754
  var EmbeddingSetsRepository = class {
1694
1755
  constructor(db) {
1695
1756
  this.db = db;
@@ -1737,7 +1798,12 @@ var EmbeddingSetsRepository = class {
1737
1798
  input.updatedAt ?? null
1738
1799
  ]
1739
1800
  );
1740
- return this.get(id);
1801
+ const row = await this.get(id);
1802
+ if (input.materialization?.allowed) {
1803
+ await this.refreshMaterializedVirtualSet(id);
1804
+ return this.get(id);
1805
+ }
1806
+ return row;
1741
1807
  }
1742
1808
  async ensureDefault() {
1743
1809
  const existing = await this.db.query(
@@ -1829,14 +1895,97 @@ var EmbeddingSetsRepository = class {
1829
1895
  const set = await this.get(selector.embeddingSetId);
1830
1896
  if (set.kind === "virtual") {
1831
1897
  const definition = this.definitionFromRow(set);
1832
- return this.resolveDefinition({ kind: "embedding-set", embeddingSetId: set.id }, definition);
1898
+ return this.resolveDefinition({ kind: "embedding-set", embeddingSetId: set.id }, definition, set);
1833
1899
  }
1834
1900
  return this.resolvePhysicalSet(selector, set.id);
1835
1901
  }
1836
1902
  if (!selector.definition) throw new Error("virtual-definition selector requires definition");
1837
1903
  return this.resolveDefinition(selector, selector.definition);
1838
1904
  }
1839
- async resolveDefinition(selector, definition) {
1905
+ async refreshMaterializedVirtualSet(setId) {
1906
+ const set = await this.get(setId);
1907
+ if (set.kind !== "virtual") throw new Error(`Embedding set is not virtual: ${setId}`);
1908
+ const definition = this.definitionFromRow(set);
1909
+ if (!definition.materialization?.allowed) {
1910
+ throw new Error(`Virtual embedding set does not allow materialization: ${setId}`);
1911
+ }
1912
+ const live = await this.resolveDefinition(
1913
+ { kind: "embedding-set", embeddingSetId: setId },
1914
+ definition,
1915
+ set,
1916
+ { forceLive: true }
1917
+ );
1918
+ await this.db.query(`DELETE FROM embedding_set_member WHERE embedding_set_id = $1`, [setId]);
1919
+ for (const row of live.rows) {
1920
+ await this.db.query(
1921
+ `INSERT INTO embedding_set_member (embedding_set_id, note_id, embedding_id)
1922
+ VALUES ($1, $2, $3)
1923
+ ON CONFLICT DO NOTHING`,
1924
+ [setId, row.note_id, row.embedding_id]
1925
+ );
1926
+ }
1927
+ const inputHash = this.resolutionInputHash(definition, live.rows);
1928
+ const generatedAt = (/* @__PURE__ */ new Date()).toISOString();
1929
+ const materialization = {
1930
+ ...definition.materialization,
1931
+ allowed: true,
1932
+ includeResolvedMembers: true,
1933
+ freshness: "fresh",
1934
+ inputHash,
1935
+ generatedAt,
1936
+ resolvedMemberCount: live.rows.length
1937
+ };
1938
+ const freshness = {
1939
+ status: "fresh",
1940
+ sourceHash: inputHash,
1941
+ checkedAt: generatedAt
1942
+ };
1943
+ await this.db.query(
1944
+ `UPDATE embedding_set
1945
+ SET materialization_json = $2::jsonb, freshness_json = $3::jsonb, updated_at = now()
1946
+ WHERE id = $1`,
1947
+ [setId, jsonParam(materialization), jsonParam(freshness)]
1948
+ );
1949
+ return this.finalizeResolution(
1950
+ { kind: "embedding-set", embeddingSetId: setId },
1951
+ live.rows,
1952
+ live.errors,
1953
+ definition.compatibility,
1954
+ "fresh",
1955
+ "materialized"
1956
+ );
1957
+ }
1958
+ async markVirtualSetStale(setId, reason) {
1959
+ const set = await this.get(setId);
1960
+ if (set.kind !== "virtual") throw new Error(`Embedding set is not virtual: ${setId}`);
1961
+ const definition = this.definitionFromRow(set);
1962
+ const now = (/* @__PURE__ */ new Date()).toISOString();
1963
+ const materialization = definition.materialization ? { ...definition.materialization, freshness: "stale" } : null;
1964
+ await this.db.query(
1965
+ `UPDATE embedding_set
1966
+ SET materialization_json = $2::jsonb, freshness_json = $3::jsonb, updated_at = now()
1967
+ WHERE id = $1`,
1968
+ [
1969
+ setId,
1970
+ jsonParam(materialization),
1971
+ jsonParam({ status: "stale", sourceHash: definition.materialization?.inputHash, checkedAt: now, reason })
1972
+ ]
1973
+ );
1974
+ }
1975
+ async resolveDefinition(selector, definition, set, options = {}) {
1976
+ if (!options.forceLive && set && definition.materialization?.allowed && definition.materialization.freshness === "fresh") {
1977
+ const materialized = await this.resolveMaterializedRows(set.id);
1978
+ if (materialized.length > 0 || definition.materialization.resolvedMemberCount === 0) {
1979
+ return this.finalizeResolution(
1980
+ selector,
1981
+ materialized,
1982
+ [],
1983
+ definition.compatibility,
1984
+ "fresh",
1985
+ "materialized"
1986
+ );
1987
+ }
1988
+ }
1840
1989
  let rows;
1841
1990
  const errors = [];
1842
1991
  switch (definition.source.type) {
@@ -1858,10 +2007,10 @@ var EmbeddingSetsRepository = class {
1858
2007
  default:
1859
2008
  rows = [];
1860
2009
  }
1861
- return this.finalizeResolution(selector, rows, errors, definition.compatibility, definition.materialization?.freshness ?? "unknown");
2010
+ return this.finalizeResolution(selector, rows, errors, definition.compatibility, definition.materialization?.freshness ?? "unknown", "live");
1862
2011
  }
1863
2012
  async resolvePhysicalSet(selector, setId) {
1864
- return this.finalizeResolution(selector, await this.resolvePhysicalRows(setId), [], DEFAULT_COMPATIBILITY, "fresh");
2013
+ return this.finalizeResolution(selector, await this.resolvePhysicalRows(setId), [], DEFAULT_COMPATIBILITY, "fresh", "live");
1865
2014
  }
1866
2015
  async resolvePhysicalRows(setId) {
1867
2016
  const result = await this.db.query(
@@ -1873,6 +2022,17 @@ var EmbeddingSetsRepository = class {
1873
2022
  );
1874
2023
  return result.rows;
1875
2024
  }
2025
+ async resolveMaterializedRows(setId) {
2026
+ const result = await this.db.query(
2027
+ `SELECT e.note_id, e.embedding_set_id, e.id as embedding_id, e.vector::text as vector, e.created_at
2028
+ FROM embedding_set_member m
2029
+ JOIN embedding e ON e.id = m.embedding_id
2030
+ WHERE m.embedding_set_id = $1
2031
+ ORDER BY e.note_id, e.created_at DESC`,
2032
+ [setId]
2033
+ );
2034
+ return result.rows;
2035
+ }
1876
2036
  async resolveCriteriaSource(source) {
1877
2037
  const criteria = source.criteria;
1878
2038
  if (criteria.conceptIds && criteria.conceptIds.length > 0) {
@@ -1893,6 +2053,50 @@ var EmbeddingSetsRepository = class {
1893
2053
  conditions.push(`EXISTS (SELECT 1 FROM collection_note cn WHERE cn.note_id = n.id AND cn.collection_id = ANY($${idx++}))`);
1894
2054
  params.push(criteria.collectionIds);
1895
2055
  }
2056
+ if (criteria.sources?.length) {
2057
+ conditions.push(`n.source = ANY($${idx++})`);
2058
+ params.push(criteria.sources);
2059
+ }
2060
+ if (criteria.formats?.length) {
2061
+ conditions.push(`n.format = ANY($${idx++})`);
2062
+ params.push(criteria.formats);
2063
+ }
2064
+ if (criteria.visibilities?.length) {
2065
+ conditions.push(`n.visibility = ANY($${idx++})`);
2066
+ params.push(criteria.visibilities);
2067
+ }
2068
+ if (criteria.isStarred !== void 0) {
2069
+ conditions.push(`n.is_starred = $${idx++}`);
2070
+ params.push(criteria.isStarred);
2071
+ }
2072
+ if (criteria.isArchived !== void 0) {
2073
+ conditions.push(`n.is_archived = $${idx++}`);
2074
+ params.push(criteria.isArchived);
2075
+ }
2076
+ if (criteria.hasTitle !== void 0) {
2077
+ conditions.push(criteria.hasTitle ? `n.title IS NOT NULL AND n.title <> ''` : `(n.title IS NULL OR n.title = '')`);
2078
+ }
2079
+ if (criteria.hasEmbedding === false) {
2080
+ conditions.push("FALSE");
2081
+ }
2082
+ if (criteria.isUserEdited !== void 0) {
2083
+ conditions.push(`COALESCE(c.is_user_edited, false) = $${idx++}`);
2084
+ params.push(criteria.isUserEdited);
2085
+ }
2086
+ if (criteria.hasAiMetadata !== void 0) {
2087
+ conditions.push(criteria.hasAiMetadata ? `c.ai_metadata IS NOT NULL` : `c.ai_metadata IS NULL`);
2088
+ }
2089
+ if (criteria.hasRevisions !== void 0) {
2090
+ conditions.push(criteria.hasRevisions ? `EXISTS (SELECT 1 FROM note_revision nr WHERE nr.note_id = n.id)` : `NOT EXISTS (SELECT 1 FROM note_revision nr WHERE nr.note_id = n.id)`);
2091
+ }
2092
+ if (criteria.minGenerationCount !== void 0) {
2093
+ conditions.push(`COALESCE(c.generation_count, 0) >= $${idx++}`);
2094
+ params.push(criteria.minGenerationCount);
2095
+ }
2096
+ if (criteria.maxGenerationCount !== void 0) {
2097
+ conditions.push(`COALESCE(c.generation_count, 0) <= $${idx++}`);
2098
+ params.push(criteria.maxGenerationCount);
2099
+ }
1896
2100
  if (criteria.updatedAfter) {
1897
2101
  conditions.push(`n.updated_at >= $${idx++}`);
1898
2102
  params.push(criteria.updatedAfter);
@@ -1983,7 +2187,7 @@ var EmbeddingSetsRepository = class {
1983
2187
  }
1984
2188
  return resolved.sort((a, b) => a.note_id.localeCompare(b.note_id));
1985
2189
  }
1986
- finalizeResolution(selector, rows, errors, compatibility, freshness) {
2190
+ finalizeResolution(selector, rows, errors, compatibility, freshness, resolutionSource) {
1987
2191
  const deduped = this.resolveDuplicateRows(rows, compatibility, errors);
1988
2192
  return {
1989
2193
  selector,
@@ -1991,9 +2195,17 @@ var EmbeddingSetsRepository = class {
1991
2195
  noteIds: deduped.map((row) => row.note_id),
1992
2196
  embeddingIds: deduped.map((row) => row.embedding_id),
1993
2197
  errors,
1994
- freshness: { status: freshness }
2198
+ freshness: { status: freshness },
2199
+ resolutionSource
1995
2200
  };
1996
2201
  }
2202
+ resolutionInputHash(definition, rows) {
2203
+ return hashJson({
2204
+ source: definition.source,
2205
+ compatibility: definition.compatibility,
2206
+ members: rows.map((row) => [row.note_id, row.embedding_set_id, row.embedding_id])
2207
+ });
2208
+ }
1997
2209
  definitionFromRow(row) {
1998
2210
  const source = asObject(row.source_json);
1999
2211
  if (!source) throw new Error(`Virtual embedding set has no source definition: ${row.id}`);
@@ -2398,8 +2610,9 @@ var SearchRepository = class {
2398
2610
  };
2399
2611
 
2400
2612
  // src/repositories/graph-repository.ts
2401
- var SIMILARITY_GRAPH_ALGORITHM = "knn-v1";
2402
- function hashJson(value) {
2613
+ var SIMILARITY_GRAPH_ALGORITHM = "knn-batched-v1";
2614
+ var DEFAULT_GRAPH_BATCH_SIZE = 64;
2615
+ function hashJson2(value) {
2403
2616
  return computeHash(new TextEncoder().encode(JSON.stringify(value)));
2404
2617
  }
2405
2618
  function sourceIdFor(inputHash) {
@@ -2410,6 +2623,45 @@ function jsonObject(value) {
2410
2623
  if (typeof value === "string") return JSON.parse(value);
2411
2624
  return value;
2412
2625
  }
2626
+ async function yieldToEventLoop() {
2627
+ const scheduler = globalThis.scheduler;
2628
+ if (scheduler?.yield) {
2629
+ await scheduler.yield();
2630
+ return;
2631
+ }
2632
+ await new Promise((resolve) => setTimeout(resolve, 0));
2633
+ }
2634
+ async function maybeYield(done, every) {
2635
+ if (every > 0 && done > 0 && done % every === 0) {
2636
+ await yieldToEventLoop();
2637
+ }
2638
+ }
2639
+ function parseVector(value) {
2640
+ return value.replace(/^\[/, "").replace(/\]$/, "").split(",").filter((part) => part.length > 0).map(Number);
2641
+ }
2642
+ function vectorScore(left, right, metric) {
2643
+ let dot = 0;
2644
+ let leftNorm = 0;
2645
+ let rightNorm = 0;
2646
+ let squaredDistance = 0;
2647
+ for (let i = 0; i < Math.min(left.length, right.length); i++) {
2648
+ dot += left[i] * right[i];
2649
+ leftNorm += left[i] * left[i];
2650
+ rightNorm += right[i] * right[i];
2651
+ const diff = left[i] - right[i];
2652
+ squaredDistance += diff * diff;
2653
+ }
2654
+ if (metric === "inner_product") {
2655
+ return { distance: -dot, similarity: dot };
2656
+ }
2657
+ if (metric === "l2") {
2658
+ const distance = Math.sqrt(squaredDistance);
2659
+ return { distance, similarity: -distance };
2660
+ }
2661
+ const denominator = Math.sqrt(leftNorm) * Math.sqrt(rightNorm);
2662
+ const similarity = denominator === 0 ? 0 : dot / denominator;
2663
+ return { distance: 1 - similarity, similarity };
2664
+ }
2413
2665
  function detectCommunities(edges, nodes = [], options = {}) {
2414
2666
  const nodeIds = new Set(nodes.map((n) => n.id));
2415
2667
  for (const edge of edges) {
@@ -2486,7 +2738,7 @@ var GraphRepository = class {
2486
2738
  const normalized = this.normalizeSimilarityRequest(request);
2487
2739
  const resolved = await new EmbeddingSetsRepository(this.db).resolveSelector(normalized.selector);
2488
2740
  const cacheKey = await this.computeSimilarityGraphCacheKey(normalized, resolved);
2489
- const inputHash = hashJson(cacheKey);
2741
+ const inputHash = hashJson2(cacheKey);
2490
2742
  const source = await this.findGraphSource(inputHash);
2491
2743
  if (!source) return null;
2492
2744
  const graph = await this.graphFromArtifact(source.id, resolved.noteIds);
@@ -2501,7 +2753,7 @@ var GraphRepository = class {
2501
2753
  const normalized = this.normalizeSimilarityRequest(request);
2502
2754
  const resolved = await new EmbeddingSetsRepository(this.db).resolveSelector(normalized.selector);
2503
2755
  const cacheKey = await this.computeSimilarityGraphCacheKey(normalized, resolved);
2504
- const inputHash = hashJson(cacheKey);
2756
+ const inputHash = hashJson2(cacheKey);
2505
2757
  if (normalized.source !== "live-only") {
2506
2758
  const cached = await this.findGraphSource(inputHash);
2507
2759
  if (cached?.freshness === "fresh") {
@@ -2534,7 +2786,7 @@ var GraphRepository = class {
2534
2786
  };
2535
2787
  }
2536
2788
  async saveSimilarityGraphArtifact(input) {
2537
- const inputHash = hashJson(input.cacheKey);
2789
+ const inputHash = hashJson2(input.cacheKey);
2538
2790
  const id = sourceIdFor(inputHash);
2539
2791
  const parameters = {
2540
2792
  k: input.request.k,
@@ -2585,27 +2837,32 @@ var GraphRepository = class {
2585
2837
  async buildSimilarityGraphFromResolved(resolved, options) {
2586
2838
  const k = options.k ?? 5;
2587
2839
  const minSimilarity = options.minSimilarity ?? options.threshold ?? -1;
2840
+ const metric = options.metric ?? "cosine";
2841
+ const yieldEvery = options.yieldEvery ?? options.batchSize ?? DEFAULT_GRAPH_BATCH_SIZE;
2588
2842
  const embeddings = resolved.rows;
2589
2843
  const nodes = embeddings.map((row) => ({ id: row.note_id }));
2590
2844
  const edgeMap = /* @__PURE__ */ new Map();
2591
- for (const row of embeddings) {
2592
- const neighbors = await this.db.query(
2593
- `SELECT note_id, 1 - (vector <=> $2::vector) as similarity
2594
- FROM embedding
2595
- WHERE id = ANY($1) AND note_id != $3
2596
- ORDER BY vector <=> $2::vector ASC
2597
- LIMIT $4`,
2598
- [resolved.embeddingIds, row.vector, row.note_id, k]
2599
- );
2600
- for (const neighbor of neighbors.rows) {
2845
+ const vectors = embeddings.map((row) => ({
2846
+ row,
2847
+ vector: parseVector(row.vector)
2848
+ }));
2849
+ options.onProgress?.({ phase: "prepare", done: vectors.length, total: vectors.length });
2850
+ for (const [index, item] of vectors.entries()) {
2851
+ const neighbors = vectors.filter((candidate) => candidate.row.note_id !== item.row.note_id).map((candidate) => {
2852
+ const score = vectorScore(item.vector, candidate.vector, metric);
2853
+ return { noteId: candidate.row.note_id, ...score };
2854
+ }).sort((a, b) => a.distance - b.distance || a.noteId.localeCompare(b.noteId)).slice(0, k);
2855
+ for (const neighbor of neighbors) {
2601
2856
  if (neighbor.similarity < minSimilarity) continue;
2602
- const [source, target] = [row.note_id, neighbor.note_id].sort();
2857
+ const [source, target] = [item.row.note_id, neighbor.noteId].sort();
2603
2858
  const id = `${source}\0${target}`;
2604
2859
  const existing = edgeMap.get(id);
2605
2860
  if (!existing || neighbor.similarity > existing.weight) {
2606
2861
  edgeMap.set(id, { source, target, weight: neighbor.similarity, kind: "similarity" });
2607
2862
  }
2608
2863
  }
2864
+ options.onProgress?.({ phase: "neighbors", done: index + 1, total: vectors.length });
2865
+ await maybeYield(index + 1, yieldEvery);
2609
2866
  }
2610
2867
  const edges = Array.from(edgeMap.values()).sort(
2611
2868
  (a, b) => a.source.localeCompare(b.source) || a.target.localeCompare(b.target)
@@ -2616,7 +2873,7 @@ var GraphRepository = class {
2616
2873
  const firstSetId = resolved.rows[0]?.embedding_set_id;
2617
2874
  const set = firstSetId ? await new EmbeddingSetsRepository(this.db).get(firstSetId) : null;
2618
2875
  return {
2619
- selectorHash: hashJson(request.selector),
2876
+ selectorHash: hashJson2(request.selector),
2620
2877
  resolvedEmbeddingSetId: request.selector.kind === "embedding-set" ? request.selector.embeddingSetId : void 0,
2621
2878
  virtualSetId: request.selector.kind === "virtual-definition" ? request.selector.definition?.id : void 0,
2622
2879
  k: request.k,
@@ -2625,9 +2882,9 @@ var GraphRepository = class {
2625
2882
  model: set?.model_name ?? "unknown",
2626
2883
  dimension: set?.dimensions ?? 0,
2627
2884
  truncateDimension: set?.truncate_dimension ?? null,
2628
- memberHash: hashJson(resolved.rows.map((row) => [row.note_id, row.embedding_set_id, row.embedding_id])),
2629
- vectorHash: hashJson(resolved.rows.map((row) => [row.embedding_id, row.vector])),
2630
- parameterHash: hashJson({ k: request.k, minSimilarity: request.minSimilarity, metric: request.metric, algorithm: SIMILARITY_GRAPH_ALGORITHM })
2885
+ memberHash: hashJson2(resolved.rows.map((row) => [row.note_id, row.embedding_set_id, row.embedding_id])),
2886
+ vectorHash: hashJson2(resolved.rows.map((row) => [row.embedding_id, row.vector])),
2887
+ parameterHash: hashJson2({ k: request.k, minSimilarity: request.minSimilarity, metric: request.metric, algorithm: SIMILARITY_GRAPH_ALGORITHM })
2631
2888
  };
2632
2889
  }
2633
2890
  async findGraphSource(inputHash) {
@@ -5171,6 +5428,33 @@ var FallbackRouter = class {
5171
5428
  }
5172
5429
  };
5173
5430
 
5431
+ // src/fortemi-bridge.ts
5432
+ function getFortemiBridge(host = globalThis) {
5433
+ return host?.fortemiBridge ?? null;
5434
+ }
5435
+ function getFortemiSecretStore(host = globalThis) {
5436
+ return host?.fortemiBridge?.secrets ?? host?.fortemiSecureStorage ?? null;
5437
+ }
5438
+ async function hasFortemiSecureSecrets(host = globalThis) {
5439
+ const bridge = getFortemiBridge(host);
5440
+ if (bridge) {
5441
+ try {
5442
+ const capabilities = await bridge.capabilities();
5443
+ if (!capabilities.secureSecrets) return false;
5444
+ return Boolean(await bridge.secrets.isAvailable());
5445
+ } catch {
5446
+ return false;
5447
+ }
5448
+ }
5449
+ const legacySecrets = host?.fortemiSecureStorage;
5450
+ if (!legacySecrets) return false;
5451
+ try {
5452
+ return Boolean(await legacySecrets.isAvailable());
5453
+ } catch {
5454
+ return false;
5455
+ }
5456
+ }
5457
+
5174
5458
  // src/security/plugin-content.ts
5175
5459
  var DEFAULT_DIRECTIVES = {
5176
5460
  "default-src": ["'self'"],
@@ -5589,7 +5873,7 @@ function embeddingToShard(emb) {
5589
5873
  id: emb.id,
5590
5874
  note_id: emb.note_id,
5591
5875
  embedding_set_id: emb.embedding_set_id,
5592
- vector: typeof emb.vector === "string" ? parseVector(emb.vector) : emb.vector,
5876
+ vector: typeof emb.vector === "string" ? parseVector2(emb.vector) : emb.vector,
5593
5877
  created_at: toISOString(emb.created_at)
5594
5878
  };
5595
5879
  }
@@ -5655,7 +5939,7 @@ function toISOString(date) {
5655
5939
  if (date instanceof Date) return date.toISOString();
5656
5940
  return date;
5657
5941
  }
5658
- function parseVector(vectorStr) {
5942
+ function parseVector2(vectorStr) {
5659
5943
  const inner = vectorStr.replace(/^\[/, "").replace(/\]$/, "");
5660
5944
  return inner.split(",").map(Number);
5661
5945
  }
@@ -5827,25 +6111,60 @@ async function exportShard(db, options) {
5827
6111
  components.push("provenance_edges");
5828
6112
  counts.provenance_edges = filteredProvenanceRows.length;
5829
6113
  if (options?.includeEmbeddings) {
5830
- const embSetRows = await db.query(`SELECT * FROM embedding_set ORDER BY created_at`);
5831
- const shardEmbSets = embSetRows.rows.map(embeddingSetToShard);
6114
+ const embeddingSetIds = options.embeddingSetIds?.filter(Boolean) ?? [];
6115
+ const setScoped = embeddingSetIds.length > 0;
6116
+ const includeMaterializedSelectors = options.includeMaterializedSelectors === true;
6117
+ const embSetRows = await db.query(
6118
+ `SELECT * FROM embedding_set
6119
+ ${setScoped ? "WHERE id = ANY($1)" : ""}
6120
+ ORDER BY created_at`,
6121
+ setScoped ? [embeddingSetIds] : []
6122
+ );
6123
+ const exportedSetIds = new Set(embSetRows.rows.map((row) => row.id));
6124
+ const virtualSetIds = new Set(embSetRows.rows.filter((row) => row.kind === "virtual").map((row) => row.id));
6125
+ const shardEmbSets = embSetRows.rows.map((row) => embeddingSetToShard(
6126
+ row.kind === "virtual" && !includeMaterializedSelectors ? {
6127
+ ...row,
6128
+ materialization_json: void 0,
6129
+ freshness_json: { status: "unknown" }
6130
+ } : row
6131
+ ));
5832
6132
  files.set("embedding_sets.json", encoder.encode(JSON.stringify(shardEmbSets)));
5833
6133
  components.push("embedding_sets");
5834
6134
  counts.embedding_sets = shardEmbSets.length;
5835
- const embMemberRows = await db.query(`SELECT * FROM embedding_set_member`);
5836
- const membersJsonl = embMemberRows.rows.map((m) => JSON.stringify(embeddingSetMemberToShard(m))).join("\n");
6135
+ const embMemberRows = await db.query(
6136
+ `SELECT * FROM embedding_set_member
6137
+ ${setScoped ? "WHERE embedding_set_id = ANY($1)" : ""}`,
6138
+ setScoped ? [embeddingSetIds] : []
6139
+ );
6140
+ const scopedEmbMemberRows = embMemberRows.rows.filter(
6141
+ (member) => exportedSetIds.has(member.embedding_set_id) && exportedNoteIds.has(member.note_id) && (includeMaterializedSelectors || !virtualSetIds.has(member.embedding_set_id))
6142
+ );
6143
+ const membersJsonl = scopedEmbMemberRows.map((m) => JSON.stringify(embeddingSetMemberToShard(m))).join("\n");
5837
6144
  files.set("embedding_set_members.jsonl", encoder.encode(membersJsonl));
5838
6145
  components.push("embedding_set_members");
5839
- counts.embedding_set_members = embMemberRows.rows.length;
5840
- const embRows = await db.query(`SELECT * FROM embedding ORDER BY created_at`);
5841
- const embJsonl = embRows.rows.map((e) => JSON.stringify(embeddingToShard(e))).join("\n");
6146
+ counts.embedding_set_members = scopedEmbMemberRows.length;
6147
+ const embRows = await db.query(
6148
+ `SELECT * FROM embedding
6149
+ ${setScoped ? "WHERE embedding_set_id = ANY($1)" : ""}
6150
+ ORDER BY created_at`,
6151
+ setScoped ? [embeddingSetIds] : []
6152
+ );
6153
+ const memberEmbeddingIds = new Set(scopedEmbMemberRows.map((member) => member.embedding_id));
6154
+ const scopedEmbRows = embRows.rows.filter(
6155
+ (embedding) => exportedSetIds.has(embedding.embedding_set_id) && exportedNoteIds.has(embedding.note_id) && (memberEmbeddingIds.size === 0 || memberEmbeddingIds.has(embedding.id))
6156
+ );
6157
+ const embJsonl = scopedEmbRows.map((e) => JSON.stringify(embeddingToShard(e))).join("\n");
5842
6158
  files.set("embeddings.jsonl", encoder.encode(embJsonl));
5843
6159
  components.push("embeddings");
5844
- counts.embeddings = embRows.rows.length;
6160
+ counts.embeddings = scopedEmbRows.length;
5845
6161
  }
5846
6162
  const graphSourceRows = await db.query(`SELECT * FROM graph_source ORDER BY created_at, id`);
5847
- if (graphSourceRows.rows.length > 0) {
5848
- const shardGraphSources = graphSourceRows.rows.map((row) => ({
6163
+ const graphScoped = !!options?.embeddingSetIds?.length;
6164
+ const scopedGraphSourceRows = graphScoped ? graphSourceRows.rows.filter((row) => !row.embedding_set_id || options.embeddingSetIds?.includes(row.embedding_set_id)) : graphSourceRows.rows;
6165
+ const exportedGraphSourceIds = new Set(scopedGraphSourceRows.map((row) => row.id));
6166
+ if (scopedGraphSourceRows.length > 0) {
6167
+ const shardGraphSources = scopedGraphSourceRows.map((row) => ({
5849
6168
  id: row.id,
5850
6169
  name: row.name,
5851
6170
  kind: row.kind,
@@ -5867,8 +6186,9 @@ async function exportShard(db, options) {
5867
6186
  counts.graph_sources = shardGraphSources.length;
5868
6187
  }
5869
6188
  const graphEdgeRows = await db.query(`SELECT * FROM graph_edge_artifact ORDER BY graph_source_id, from_note_id, to_note_id, kind`);
5870
- if (graphEdgeRows.rows.length > 0) {
5871
- const graphEdgesJsonl = graphEdgeRows.rows.map((row) => JSON.stringify({
6189
+ const scopedGraphEdgeRows = graphScoped ? graphEdgeRows.rows.filter((row) => exportedGraphSourceIds.has(row.graph_source_id)) : graphEdgeRows.rows;
6190
+ if (scopedGraphEdgeRows.length > 0) {
6191
+ const graphEdgesJsonl = scopedGraphEdgeRows.map((row) => JSON.stringify({
5872
6192
  graph_source_id: row.graph_source_id,
5873
6193
  from_note_id: row.from_note_id,
5874
6194
  to_note_id: row.to_note_id,
@@ -5879,18 +6199,21 @@ async function exportShard(db, options) {
5879
6199
  })).join("\n");
5880
6200
  files.set("graph_edges.jsonl", encoder.encode(graphEdgesJsonl));
5881
6201
  components.push("graph_edges");
5882
- counts.graph_edges = graphEdgeRows.rows.length;
6202
+ counts.graph_edges = scopedGraphEdgeRows.length;
5883
6203
  }
5884
6204
  const communitySetRows = await db.query(`SELECT * FROM community_set ORDER BY created_at, id`);
5885
6205
  const communityRows = await db.query(`SELECT * FROM community ORDER BY community_set_id, rank NULLS LAST, id`);
5886
- if (communitySetRows.rows.length > 0) {
6206
+ const scopedCommunitySetRows = graphScoped ? communitySetRows.rows.filter((row) => exportedGraphSourceIds.has(row.graph_source_id)) : communitySetRows.rows;
6207
+ const exportedCommunitySetIds = new Set(scopedCommunitySetRows.map((row) => row.id));
6208
+ const scopedCommunityRows = graphScoped ? communityRows.rows.filter((row) => exportedCommunitySetIds.has(row.community_set_id)) : communityRows.rows;
6209
+ if (scopedCommunitySetRows.length > 0) {
5887
6210
  const communitiesBySet = /* @__PURE__ */ new Map();
5888
- for (const row of communityRows.rows) {
6211
+ for (const row of scopedCommunityRows) {
5889
6212
  const rows = communitiesBySet.get(row.community_set_id) ?? [];
5890
6213
  rows.push(row);
5891
6214
  communitiesBySet.set(row.community_set_id, rows);
5892
6215
  }
5893
- const shardCommunitySets = communitySetRows.rows.map((row) => ({
6216
+ const shardCommunitySets = scopedCommunitySetRows.map((row) => ({
5894
6217
  id: row.id,
5895
6218
  graph_source_id: row.graph_source_id,
5896
6219
  name: row.name,
@@ -5913,11 +6236,12 @@ async function exportShard(db, options) {
5913
6236
  files.set("communities.json", encoder.encode(JSON.stringify(shardCommunitySets)));
5914
6237
  components.push("communities");
5915
6238
  counts.community_sets = shardCommunitySets.length;
5916
- counts.communities = communityRows.rows.length;
6239
+ counts.communities = scopedCommunityRows.length;
5917
6240
  }
5918
6241
  const assignmentRows = await db.query(`SELECT * FROM community_assignment ORDER BY community_set_id, community_id, note_id`);
5919
- if (assignmentRows.rows.length > 0) {
5920
- const assignmentsJsonl = assignmentRows.rows.map((row) => JSON.stringify({
6242
+ const scopedAssignmentRows = graphScoped ? assignmentRows.rows.filter((row) => exportedCommunitySetIds.has(row.community_set_id)) : assignmentRows.rows;
6243
+ if (scopedAssignmentRows.length > 0) {
6244
+ const assignmentsJsonl = scopedAssignmentRows.map((row) => JSON.stringify({
5921
6245
  community_set_id: row.community_set_id,
5922
6246
  community_id: row.community_id,
5923
6247
  note_id: row.note_id,
@@ -5927,7 +6251,7 @@ async function exportShard(db, options) {
5927
6251
  })).join("\n");
5928
6252
  files.set("community_assignments.jsonl", encoder.encode(assignmentsJsonl));
5929
6253
  components.push("community_assignments");
5930
- counts.community_assignments = assignmentRows.rows.length;
6254
+ counts.community_assignments = scopedAssignmentRows.length;
5931
6255
  }
5932
6256
  const checksums = {};
5933
6257
  for (const [filename, data] of files) {
@@ -5949,9 +6273,25 @@ async function exportShard(db, options) {
5949
6273
 
5950
6274
  // src/shard/shard-import.ts
5951
6275
  var decoder = new TextDecoder();
6276
+ var DEFAULT_BATCH_SIZE = 250;
6277
+ async function yieldToEventLoop2() {
6278
+ const scheduler = globalThis.scheduler;
6279
+ if (scheduler?.yield) {
6280
+ await scheduler.yield();
6281
+ return;
6282
+ }
6283
+ await new Promise((resolve) => setTimeout(resolve, 0));
6284
+ }
6285
+ async function maybeYield2(done, batchSize) {
6286
+ if (batchSize > 0 && done > 0 && done % batchSize === 0) {
6287
+ await yieldToEventLoop2();
6288
+ }
6289
+ }
5952
6290
  async function importShard(db, data, options) {
5953
6291
  const start = performance.now();
5954
6292
  const strategy = options?.conflictStrategy ?? "skip";
6293
+ const batchSize = options?.batchSize ?? DEFAULT_BATCH_SIZE;
6294
+ const report = options?.onProgress;
5955
6295
  const warnings = [];
5956
6296
  const errors = [];
5957
6297
  const counts = {
@@ -5975,9 +6315,11 @@ async function importShard(db, data, options) {
5975
6315
  };
5976
6316
  const skipped = {};
5977
6317
  const inputData = data instanceof ArrayBuffer ? new Uint8Array(data) : data;
6318
+ report?.({ phase: "unpack", done: 0, total: 1 });
5978
6319
  let files;
5979
6320
  try {
5980
6321
  files = unpackTarGz(inputData);
6322
+ report?.({ phase: "unpack", done: 1, total: 1 });
5981
6323
  } catch (err) {
5982
6324
  return {
5983
6325
  success: false,
@@ -6024,6 +6366,7 @@ async function importShard(db, data, options) {
6024
6366
  duration_ms: performance.now() - start
6025
6367
  };
6026
6368
  }
6369
+ report?.({ phase: "validate", done: 0, total: 1 });
6027
6370
  const checksumResult = await validateChecksums(manifest.checksums, files);
6028
6371
  if (!checksumResult.valid) {
6029
6372
  return {
@@ -6035,6 +6378,7 @@ async function importShard(db, data, options) {
6035
6378
  duration_ms: performance.now() - start
6036
6379
  };
6037
6380
  }
6381
+ report?.({ phase: "validate", done: 1, total: 1 });
6038
6382
  const parsedNotes = parseJsonl(files.get("notes.jsonl"));
6039
6383
  const parsedCollections = parseJsonArray(files.get("collections.json"));
6040
6384
  parseJsonArray(files.get("tags.json"));
@@ -6085,7 +6429,8 @@ async function importShard(db, data, options) {
6085
6429
  const conflictClause = strategy === "skip" ? "ON CONFLICT DO NOTHING" : "";
6086
6430
  try {
6087
6431
  await db.transaction(async (tx) => {
6088
- for (const shardCol of parsedCollections) {
6432
+ report?.({ phase: "collections", done: 0, total: parsedCollections.length });
6433
+ for (const [index, shardCol] of parsedCollections.entries()) {
6089
6434
  const col = collectionFromShard(shardCol);
6090
6435
  if (strategy === "replace") {
6091
6436
  await tx.query(
@@ -6102,8 +6447,11 @@ async function importShard(db, data, options) {
6102
6447
  );
6103
6448
  }
6104
6449
  counts.collections++;
6450
+ report?.({ phase: "collections", done: index + 1, total: parsedCollections.length });
6451
+ await maybeYield2(index + 1, batchSize);
6105
6452
  }
6106
- for (const shardNote of parsedNotes) {
6453
+ report?.({ phase: "notes", done: 0, total: parsedNotes.length });
6454
+ for (const [index, shardNote] of parsedNotes.entries()) {
6107
6455
  const note = noteFromShard(shardNote);
6108
6456
  const contentHash = computeHash(new TextEncoder().encode(note.original_content));
6109
6457
  if (strategy === "replace") {
@@ -6180,7 +6528,12 @@ async function importShard(db, data, options) {
6180
6528
  );
6181
6529
  }
6182
6530
  counts.notes++;
6531
+ report?.({ phase: "notes", done: index + 1, total: parsedNotes.length });
6532
+ await maybeYield2(index + 1, batchSize);
6183
6533
  }
6534
+ const totalSkos = parsedSkosSchemes.length + parsedSkosConcepts.length + parsedSkosRelations.length + parsedNoteSkosTags.length;
6535
+ let doneSkos = 0;
6536
+ report?.({ phase: "skos", done: doneSkos, total: totalSkos });
6184
6537
  for (const scheme of parsedSkosSchemes) {
6185
6538
  if (strategy === "replace") {
6186
6539
  await tx.query(
@@ -6197,6 +6550,8 @@ async function importShard(db, data, options) {
6197
6550
  );
6198
6551
  }
6199
6552
  counts.skos_schemes++;
6553
+ report?.({ phase: "skos", done: ++doneSkos, total: totalSkos });
6554
+ await maybeYield2(doneSkos, batchSize);
6200
6555
  }
6201
6556
  for (const concept of parsedSkosConcepts) {
6202
6557
  const altLabels = JSON.stringify(concept.alt_labels ?? []);
@@ -6215,8 +6570,11 @@ async function importShard(db, data, options) {
6215
6570
  );
6216
6571
  }
6217
6572
  counts.skos_concepts++;
6573
+ report?.({ phase: "skos", done: ++doneSkos, total: totalSkos });
6574
+ await maybeYield2(doneSkos, batchSize);
6218
6575
  }
6219
- for (const shardLink of parsedLinks) {
6576
+ report?.({ phase: "links", done: 0, total: parsedLinks.length });
6577
+ for (const [index, shardLink] of parsedLinks.entries()) {
6220
6578
  const link = linkFromShard(shardLink);
6221
6579
  if (strategy === "replace") {
6222
6580
  await tx.query(
@@ -6233,6 +6591,8 @@ async function importShard(db, data, options) {
6233
6591
  );
6234
6592
  }
6235
6593
  counts.links++;
6594
+ report?.({ phase: "links", done: index + 1, total: parsedLinks.length });
6595
+ await maybeYield2(index + 1, batchSize);
6236
6596
  }
6237
6597
  for (const relation of parsedSkosRelations) {
6238
6598
  if (strategy === "replace") {
@@ -6250,6 +6610,8 @@ async function importShard(db, data, options) {
6250
6610
  );
6251
6611
  }
6252
6612
  counts.skos_relations++;
6613
+ report?.({ phase: "skos", done: ++doneSkos, total: totalSkos });
6614
+ await maybeYield2(doneSkos, batchSize);
6253
6615
  }
6254
6616
  for (const tag of parsedNoteSkosTags) {
6255
6617
  await tx.query(
@@ -6259,8 +6621,11 @@ async function importShard(db, data, options) {
6259
6621
  [tag.id, tag.note_id, tag.concept_id, tag.created_at]
6260
6622
  );
6261
6623
  counts.note_skos_tags++;
6624
+ report?.({ phase: "skos", done: ++doneSkos, total: totalSkos });
6625
+ await maybeYield2(doneSkos, batchSize);
6262
6626
  }
6263
- for (const edge of parsedProvenanceEdges) {
6627
+ report?.({ phase: "provenance", done: 0, total: parsedProvenanceEdges.length });
6628
+ for (const [index, edge] of parsedProvenanceEdges.entries()) {
6264
6629
  const attributes = edge.attributes === null ? null : JSON.stringify(edge.attributes);
6265
6630
  if (strategy === "replace") {
6266
6631
  await tx.query(
@@ -6277,8 +6642,11 @@ async function importShard(db, data, options) {
6277
6642
  );
6278
6643
  }
6279
6644
  counts.provenance_edges++;
6645
+ report?.({ phase: "provenance", done: index + 1, total: parsedProvenanceEdges.length });
6646
+ await maybeYield2(index + 1, batchSize);
6280
6647
  }
6281
- for (const shardSet of parsedEmbSets) {
6648
+ report?.({ phase: "embedding_sets", done: 0, total: parsedEmbSets.length });
6649
+ for (const [index, shardSet] of parsedEmbSets.entries()) {
6282
6650
  const set = embeddingSetFromShard(shardSet);
6283
6651
  if (strategy === "replace") {
6284
6652
  await tx.query(
@@ -6301,8 +6669,11 @@ async function importShard(db, data, options) {
6301
6669
  );
6302
6670
  }
6303
6671
  counts.embedding_sets++;
6672
+ report?.({ phase: "embedding_sets", done: index + 1, total: parsedEmbSets.length });
6673
+ await maybeYield2(index + 1, batchSize);
6304
6674
  }
6305
- for (const shardEmb of parsedEmbeddings) {
6675
+ report?.({ phase: "embeddings", done: 0, total: parsedEmbeddings.length });
6676
+ for (const [index, shardEmb] of parsedEmbeddings.entries()) {
6306
6677
  const emb = embeddingFromShard(shardEmb);
6307
6678
  if (strategy === "replace") {
6308
6679
  await tx.query(
@@ -6319,7 +6690,12 @@ async function importShard(db, data, options) {
6319
6690
  );
6320
6691
  }
6321
6692
  counts.embeddings++;
6693
+ report?.({ phase: "embeddings", done: index + 1, total: parsedEmbeddings.length });
6694
+ await maybeYield2(index + 1, batchSize);
6322
6695
  }
6696
+ const totalGraph = parsedGraphSources.length + parsedGraphEdges.length;
6697
+ let doneGraph = 0;
6698
+ report?.({ phase: "graph", done: doneGraph, total: totalGraph });
6323
6699
  for (const source of parsedGraphSources) {
6324
6700
  const parameters = source.parameters == null ? null : JSON.stringify(source.parameters);
6325
6701
  const freshness = JSON.stringify({ ...source.freshness ?? {}, status: "unknown" });
@@ -6332,6 +6708,8 @@ async function importShard(db, data, options) {
6332
6708
  [source.id, source.name, source.kind, source.source_table ?? null, source.embedding_set_id ?? null, source.virtual_set_id ?? null, source.model ?? null, source.dimension ?? null, source.truncate_dimension ?? null, source.metric ?? null, source.algorithm ?? null, parameters, source.input_hash, freshness, source.created_at]
6333
6709
  );
6334
6710
  counts.graph_sources++;
6711
+ report?.({ phase: "graph", done: ++doneGraph, total: totalGraph });
6712
+ await maybeYield2(doneGraph, batchSize);
6335
6713
  }
6336
6714
  for (const edge of parsedGraphEdges) {
6337
6715
  const metadata = edge.metadata == null ? null : JSON.stringify(edge.metadata);
@@ -6342,7 +6720,12 @@ async function importShard(db, data, options) {
6342
6720
  [edge.graph_source_id, edge.from_note_id, edge.to_note_id, edge.weight, edge.kind, edge.rank ?? null, metadata]
6343
6721
  );
6344
6722
  counts.graph_edges++;
6723
+ report?.({ phase: "graph", done: ++doneGraph, total: totalGraph });
6724
+ await maybeYield2(doneGraph, batchSize);
6345
6725
  }
6726
+ const totalCommunities = parsedCommunitySets.length + parsedCommunitySets.reduce((sum, set) => sum + (set.communities?.length ?? 0), 0) + parsedCommunityAssignments.length;
6727
+ let doneCommunities = 0;
6728
+ report?.({ phase: "communities", done: doneCommunities, total: totalCommunities });
6346
6729
  for (const set of parsedCommunitySets) {
6347
6730
  const parameters = set.parameters == null ? null : JSON.stringify(set.parameters);
6348
6731
  const freshness = JSON.stringify({ ...set.freshness ?? {}, status: "unknown" });
@@ -6353,6 +6736,8 @@ async function importShard(db, data, options) {
6353
6736
  [set.id, set.graph_source_id, set.name, set.source_type, set.algorithm ?? null, parameters, set.input_hash, freshness, set.created_at]
6354
6737
  );
6355
6738
  counts.community_sets++;
6739
+ report?.({ phase: "communities", done: ++doneCommunities, total: totalCommunities });
6740
+ await maybeYield2(doneCommunities, batchSize);
6356
6741
  for (const community of set.communities ?? []) {
6357
6742
  const metadata = community.metadata == null ? null : JSON.stringify(community.metadata);
6358
6743
  await tx.query(
@@ -6362,6 +6747,8 @@ async function importShard(db, data, options) {
6362
6747
  [set.id, community.id, community.label ?? null, community.rank ?? null, community.size ?? null, community.confidence ?? null, community.representative_note_ids ?? [], metadata]
6363
6748
  );
6364
6749
  counts.communities++;
6750
+ report?.({ phase: "communities", done: ++doneCommunities, total: totalCommunities });
6751
+ await maybeYield2(doneCommunities, batchSize);
6365
6752
  }
6366
6753
  }
6367
6754
  for (const assignment of parsedCommunityAssignments) {
@@ -6373,16 +6760,22 @@ async function importShard(db, data, options) {
6373
6760
  [assignment.community_set_id, assignment.community_id, assignment.note_id, assignment.confidence ?? null, assignment.source_type, metadata]
6374
6761
  );
6375
6762
  counts.community_assignments++;
6763
+ report?.({ phase: "communities", done: ++doneCommunities, total: totalCommunities });
6764
+ await maybeYield2(doneCommunities, batchSize);
6376
6765
  }
6377
- for (const member of parsedEmbMembers) {
6766
+ report?.({ phase: "embedding_set_members", done: 0, total: parsedEmbMembers.length });
6767
+ for (const [index, member] of parsedEmbMembers.entries()) {
6378
6768
  await tx.query(
6379
6769
  `INSERT INTO embedding_set_member (embedding_set_id, note_id, embedding_id)
6380
6770
  VALUES ($1, $2, $3) ON CONFLICT DO NOTHING`,
6381
6771
  [member.embedding_set_id, member.note_id, member.embedding_id]
6382
6772
  );
6383
6773
  counts.embedding_set_members++;
6774
+ report?.({ phase: "embedding_set_members", done: index + 1, total: parsedEmbMembers.length });
6775
+ await maybeYield2(index + 1, batchSize);
6384
6776
  }
6385
6777
  });
6778
+ report?.({ phase: "index", done: 1, total: 1 });
6386
6779
  } catch (err) {
6387
6780
  return {
6388
6781
  success: false,
@@ -6433,8 +6826,15 @@ var VALID_TYPES = /* @__PURE__ */ new Set([
6433
6826
  "crm.organization",
6434
6827
  "crm.event",
6435
6828
  "crm.interaction",
6436
- "aiwg.artifact"
6829
+ "aiwg.artifact",
6830
+ "docs.page"
6437
6831
  ]);
6832
+ var DEFAULT_QUERY_WEIGHTS = {
6833
+ title: 4,
6834
+ tag: 3,
6835
+ concept: 2,
6836
+ text: 1
6837
+ };
6438
6838
  function hasString(value) {
6439
6839
  return typeof value === "string" && value.length > 0;
6440
6840
  }
@@ -6442,6 +6842,16 @@ function pushFacet(counts, name, value) {
6442
6842
  counts[name] ??= {};
6443
6843
  counts[name][value] = (counts[name][value] ?? 0) + 1;
6444
6844
  }
6845
+ function hasNonNegativeInteger(value) {
6846
+ return Number.isInteger(value) && typeof value === "number" && value >= 0;
6847
+ }
6848
+ function hasPositiveInteger(value) {
6849
+ return Number.isInteger(value) && typeof value === "number" && value > 0;
6850
+ }
6851
+ function isFacetCounts(value) {
6852
+ if (!value || typeof value !== "object" || Array.isArray(value)) return false;
6853
+ return Object.values(value).every((counts) => !!counts && typeof counts === "object" && !Array.isArray(counts) && Object.values(counts).every((count) => hasNonNegativeInteger(count)));
6854
+ }
6445
6855
  function validateAiwgFortemiIndexExport(value) {
6446
6856
  const errors = [];
6447
6857
  const counts = {};
@@ -6493,6 +6903,87 @@ function assertAiwgFortemiIndexExport(value) {
6493
6903
  }
6494
6904
  return value;
6495
6905
  }
6906
+ function validateAiwgFortemiChunkManifest(value) {
6907
+ const errors = [];
6908
+ const data = value;
6909
+ if (data?.schema_version !== "aiwg.fortemi.index.chunk-manifest.v1") {
6910
+ errors.push("schema_version must be aiwg.fortemi.index.chunk-manifest.v1");
6911
+ }
6912
+ if (!hasString(data?.generated_at)) errors.push("generated_at is required");
6913
+ if (!hasString(data?.source?.repo)) errors.push("source.repo is required");
6914
+ if (!hasString(data?.source?.privacy)) errors.push("source.privacy is required");
6915
+ if (!hasNonNegativeInteger(data?.total)) errors.push("total must be a non-negative integer");
6916
+ if (!hasPositiveInteger(data?.part_size)) errors.push("part_size must be a positive integer");
6917
+ if (data.facets !== void 0 && !isFacetCounts(data.facets)) {
6918
+ errors.push("facets must be a nested string-to-number count object");
6919
+ }
6920
+ if (!Array.isArray(data?.parts)) errors.push("parts must be an array");
6921
+ let expectedOffset = 0;
6922
+ const parts = Array.isArray(data?.parts) ? data.parts : [];
6923
+ for (const [index, part] of parts.entries()) {
6924
+ if (!hasString(part.href)) errors.push("parts[" + index + "].href is required");
6925
+ if (!hasNonNegativeInteger(part.offset)) errors.push("parts[" + index + "].offset must be a non-negative integer");
6926
+ if (!hasNonNegativeInteger(part.count)) errors.push("parts[" + index + "].count must be a non-negative integer");
6927
+ if (hasNonNegativeInteger(part.offset) && part.offset !== expectedOffset) {
6928
+ errors.push("parts[" + index + "].offset must be " + expectedOffset);
6929
+ }
6930
+ if (hasNonNegativeInteger(part.count)) expectedOffset += part.count;
6931
+ }
6932
+ if (hasNonNegativeInteger(data?.total) && expectedOffset !== data.total) {
6933
+ errors.push("parts counts must add up to total");
6934
+ }
6935
+ return { valid: errors.length === 0, errors };
6936
+ }
6937
+ function assertAiwgFortemiChunkManifest(value) {
6938
+ const result = validateAiwgFortemiChunkManifest(value);
6939
+ if (!result.valid) {
6940
+ throw new Error("Invalid AIWG Fortemi chunk manifest:\n" + result.errors.join("\n"));
6941
+ }
6942
+ return value;
6943
+ }
6944
+ function validateAiwgFortemiChunkPart(value, partRef, manifest) {
6945
+ const errors = [];
6946
+ const data = value;
6947
+ if (data?.schema_version !== "aiwg.fortemi.index.chunk.v1") {
6948
+ errors.push("schema_version must be aiwg.fortemi.index.chunk.v1");
6949
+ }
6950
+ if (data?.manifest_schema_version !== "aiwg.fortemi.index.chunk-manifest.v1") {
6951
+ errors.push("manifest_schema_version must be aiwg.fortemi.index.chunk-manifest.v1");
6952
+ }
6953
+ if (!hasNonNegativeInteger(data?.offset)) errors.push("offset must be a non-negative integer");
6954
+ if (!Array.isArray(data?.items)) errors.push("items must be an array");
6955
+ if (partRef && hasNonNegativeInteger(data?.offset) && data.offset !== partRef.offset) {
6956
+ errors.push("offset must match manifest part offset " + partRef.offset);
6957
+ }
6958
+ if (partRef && Array.isArray(data?.items) && data.items.length !== partRef.count) {
6959
+ errors.push("items length must match manifest part count " + partRef.count);
6960
+ }
6961
+ if (Array.isArray(data?.items)) {
6962
+ const validation = validateAiwgFortemiIndexExport({
6963
+ schema_version: "aiwg.fortemi.index.export.v1",
6964
+ generated_at: manifest?.generated_at ?? "1970-01-01T00:00:00.000Z",
6965
+ source: manifest?.source ?? { repo: "chunk", privacy: "public" },
6966
+ items: data.items
6967
+ });
6968
+ errors.push(...validation.errors.map((error) => "items." + error));
6969
+ }
6970
+ return { valid: errors.length === 0, errors };
6971
+ }
6972
+ function assertAiwgFortemiChunkPart(value, partRef, manifest) {
6973
+ const result = validateAiwgFortemiChunkPart(value, partRef, manifest);
6974
+ if (!result.valid) {
6975
+ throw new Error("Invalid AIWG Fortemi chunk part:\n" + result.errors.join("\n"));
6976
+ }
6977
+ return value;
6978
+ }
6979
+ function createAiwgFetchChunkLoader(baseUrl) {
6980
+ return async (part) => {
6981
+ const href = baseUrl ? new URL(part.href, baseUrl).toString() : part.href;
6982
+ const response = await fetch(href);
6983
+ if (!response.ok) throw new Error("Failed to fetch AIWG index chunk " + href + ": " + response.status);
6984
+ return response.json();
6985
+ };
6986
+ }
6496
6987
  function getAiwgFortemiFacets(items) {
6497
6988
  const result = {};
6498
6989
  for (const item of items) {
@@ -6515,27 +7006,167 @@ function matchesFacetFilters(item, filters) {
6515
7006
  if (!filters) return true;
6516
7007
  return Object.entries(filters).every(([name, expected]) => includesAll(item.facets[name] ?? [], expected));
6517
7008
  }
6518
- function queryAiwgFortemiIndex(index, query = "", options = {}) {
6519
- const q = query.trim().toLowerCase();
6520
- const filtered = index.items.filter((item) => {
6521
- if (q) {
6522
- const haystack = [item.title, item.text, ...item.tags, ...item.concepts].join("\n").toLowerCase();
6523
- if (!haystack.includes(q)) return false;
6524
- }
7009
+ function queryMatches(item, q) {
7010
+ if (!q) return [];
7011
+ const matches = [];
7012
+ if (item.title.toLowerCase().includes(q)) matches.push({ field: "title", value: item.title });
7013
+ if (item.text.toLowerCase().includes(q)) matches.push({ field: "text", value: item.text });
7014
+ for (const tag of item.tags) {
7015
+ if (tag.toLowerCase().includes(q)) matches.push({ field: "tag", value: tag });
7016
+ }
7017
+ for (const concept of item.concepts) {
7018
+ if (concept.toLowerCase().includes(q)) matches.push({ field: "concept", value: concept });
7019
+ }
7020
+ return matches;
7021
+ }
7022
+ function rankMatches(matches, weights) {
7023
+ return matches.reduce((total, match) => total + weights[match.field], 0);
7024
+ }
7025
+ function clipSnippet(value, q, maxLength) {
7026
+ const normalizedLength = Math.max(20, maxLength);
7027
+ if (!value) return "";
7028
+ if (!q) return value.length > normalizedLength ? `${value.slice(0, normalizedLength).trimEnd()}...` : value;
7029
+ const lower = value.toLowerCase();
7030
+ const index = lower.indexOf(q);
7031
+ if (index < 0) return value.length > normalizedLength ? `${value.slice(0, normalizedLength).trimEnd()}...` : value;
7032
+ const context = Math.max(0, Math.floor((normalizedLength - q.length) / 2));
7033
+ const start = Math.max(0, index - context);
7034
+ const end = Math.min(value.length, start + normalizedLength);
7035
+ const prefix = start > 0 ? "..." : "";
7036
+ const suffix = end < value.length ? "..." : "";
7037
+ return `${prefix}${value.slice(start, end).trim()}${suffix}`;
7038
+ }
7039
+ function createSnippet(item, matches, q, maxLength) {
7040
+ const textMatch = matches.find((match) => match.field === "text");
7041
+ const titleMatch = matches.find((match) => match.field === "title");
7042
+ const firstMatch = textMatch ?? titleMatch ?? matches[0];
7043
+ return clipSnippet(firstMatch?.value ?? item.text, q, maxLength);
7044
+ }
7045
+ function createRankedEntries(items, q, options, ordinalBase = 0) {
7046
+ const weights = { ...DEFAULT_QUERY_WEIGHTS, ...options.weights };
7047
+ return items.map((item, ordinal) => ({ item, ordinal: ordinalBase + ordinal, matches: queryMatches(item, q) })).filter(({ item, matches }) => {
7048
+ if (q && matches.length === 0) return false;
6525
7049
  if (options.types && !options.types.includes(item.type)) return false;
6526
7050
  if (options.privacy && !options.privacy.includes(item.privacy.classification)) return false;
6527
7051
  if (!includesAll(item.tags, options.tags)) return false;
6528
7052
  if (!includesAll(item.concepts, options.concepts)) return false;
6529
7053
  if (!matchesFacetFilters(item, options.facets)) return false;
6530
- if (options.relationshipTargetId && !item.relationships.some((rel) => rel.target_id === options.relationshipTargetId)) return false;
7054
+ if (options.relationshipTargetId && !item.relationships.some((rel) => rel.target_id === options.relationshipTargetId)) {
7055
+ return false;
7056
+ }
6531
7057
  return true;
7058
+ }).map(({ item, ordinal, matches }) => ({
7059
+ item,
7060
+ ordinal,
7061
+ rank: rankMatches(matches, weights),
7062
+ matches
7063
+ }));
7064
+ }
7065
+ function sortRankedEntries(entries, rank) {
7066
+ return [...entries].sort((left, right) => {
7067
+ if (rank) return right.rank - left.rank || left.ordinal - right.ordinal;
7068
+ return left.ordinal - right.ordinal;
6532
7069
  });
7070
+ }
7071
+ function createQueryResultFromRankedEntries(entries, query, options) {
7072
+ const ranked = sortRankedEntries(entries, options.rank);
6533
7073
  const offset = options.offset ?? 0;
6534
- const limit = options.limit ?? filtered.length;
7074
+ const limit = options.limit ?? ranked.length;
7075
+ const page = ranked.slice(offset, offset + limit);
7076
+ const result = {
7077
+ items: page.map((entry) => entry.item),
7078
+ total: ranked.length,
7079
+ facets: getAiwgFortemiFacets(ranked.map((entry) => entry.item))
7080
+ };
7081
+ if (options.rank || options.snippets || options.includeMatches) {
7082
+ const snippetLength = options.snippetLength ?? 160;
7083
+ result.rankedItems = page.map((entry) => ({
7084
+ item: entry.item,
7085
+ rank: entry.rank,
7086
+ ...options.snippets ? { snippet: createSnippet(entry.item, entry.matches, query, snippetLength) } : {},
7087
+ ...options.includeMatches ? { matches: entry.matches } : {}
7088
+ }));
7089
+ }
7090
+ return result;
7091
+ }
7092
+ function queryAiwgFortemiIndex(index, query = "", options = {}) {
7093
+ const q = query.trim().toLowerCase();
7094
+ return createQueryResultFromRankedEntries(createRankedEntries(index.items, q, options), q, options);
7095
+ }
7096
+ function chunkPartCacheKey(part) {
7097
+ return `${part.offset}:${part.href}`;
7098
+ }
7099
+ function clampMaxCachedParts(value) {
7100
+ if (!hasPositiveInteger(value)) return 3;
7101
+ return value;
7102
+ }
7103
+ function isDirectChunkBrowse(query, options) {
7104
+ return query.trim() === "" && !options.rank && !options.snippets && !options.includeMatches && !options.types && !options.facets && !options.tags && !options.concepts && !options.privacy && !options.relationshipTargetId;
7105
+ }
7106
+ function getPartsForRange(manifest, offset, limit) {
7107
+ const end = offset + limit;
7108
+ return manifest.parts.filter((part) => part.count > 0 && part.offset < end && part.offset + part.count > offset);
7109
+ }
7110
+ async function loadChunkPart(runtime, part) {
7111
+ const key = chunkPartCacheKey(part);
7112
+ const cached = runtime.partCache.get(key);
7113
+ if (cached) {
7114
+ runtime.partCache.delete(key);
7115
+ runtime.partCache.set(key, cached);
7116
+ return { part: cached, fetched: false };
7117
+ }
7118
+ const parsed = assertAiwgFortemiChunkPart(await runtime.loader(part, runtime.manifest), part, runtime.manifest);
7119
+ runtime.partCache.set(key, parsed);
7120
+ while (runtime.partCache.size > runtime.maxCachedParts) {
7121
+ const oldest = runtime.partCache.keys().next().value;
7122
+ if (oldest === void 0) break;
7123
+ runtime.partCache.delete(oldest);
7124
+ }
7125
+ return { part: parsed, fetched: true };
7126
+ }
7127
+ async function queryChunkedAiwgFortemiIndex(runtime, query = "", options = {}) {
7128
+ const q = query.trim().toLowerCase();
7129
+ let scannedParts = 0;
7130
+ let fetchedParts = 0;
7131
+ if (isDirectChunkBrowse(query, options)) {
7132
+ const offset = options.offset ?? 0;
7133
+ const limit = options.limit ?? runtime.manifest.total;
7134
+ const parts = getPartsForRange(runtime.manifest, offset, limit);
7135
+ const items = [];
7136
+ for (const partRef of parts) {
7137
+ const loaded = await loadChunkPart(runtime, partRef);
7138
+ if (loaded.fetched) fetchedParts += 1;
7139
+ scannedParts += 1;
7140
+ options.onProgress?.({ phase: "part", done: scannedParts, total: parts.length, href: partRef.href });
7141
+ const start = Math.max(0, offset - partRef.offset);
7142
+ const end = Math.min(loaded.part.items.length, offset + limit - partRef.offset);
7143
+ items.push(...loaded.part.items.slice(start, end));
7144
+ }
7145
+ return {
7146
+ items,
7147
+ total: runtime.manifest.total,
7148
+ facets: runtime.manifest.facets ?? {},
7149
+ manifestTotal: runtime.manifest.total,
7150
+ scannedParts,
7151
+ fetchedParts,
7152
+ complete: true
7153
+ };
7154
+ }
7155
+ const entries = [];
7156
+ for (const partRef of runtime.manifest.parts) {
7157
+ const loaded = await loadChunkPart(runtime, partRef);
7158
+ if (loaded.fetched) fetchedParts += 1;
7159
+ scannedParts += 1;
7160
+ options.onProgress?.({ phase: "part", done: scannedParts, total: runtime.manifest.parts.length, href: partRef.href });
7161
+ entries.push(...createRankedEntries(loaded.part.items, q, options, partRef.offset));
7162
+ options.onProgress?.({ phase: "query", done: scannedParts, total: runtime.manifest.parts.length, href: partRef.href });
7163
+ }
6535
7164
  return {
6536
- items: filtered.slice(offset, offset + limit),
6537
- total: filtered.length,
6538
- facets: getAiwgFortemiFacets(filtered)
7165
+ ...createQueryResultFromRankedEntries(entries, q, options),
7166
+ manifestTotal: runtime.manifest.total,
7167
+ scannedParts,
7168
+ fetchedParts,
7169
+ complete: true
6539
7170
  };
6540
7171
  }
6541
7172
  function createAiwgReviewDecisionExport(source, decisions, generatedAt = (/* @__PURE__ */ new Date()).toISOString()) {
@@ -6546,10 +7177,184 @@ function createAiwgReviewDecisionExport(source, decisions, generatedAt = (/* @__
6546
7177
  decisions: [...decisions].sort((left, right) => left.item_id.localeCompare(right.item_id))
6547
7178
  };
6548
7179
  }
7180
+ function createAiwgIndexController(initialIndex) {
7181
+ let index = initialIndex ?? null;
7182
+ let chunked = null;
7183
+ let data = null;
7184
+ let error = null;
7185
+ let reviewDecisions = [];
7186
+ const listeners = /* @__PURE__ */ new Set();
7187
+ const snapshot = () => ({
7188
+ index,
7189
+ chunked: chunked ? {
7190
+ manifest: chunked.manifest,
7191
+ cachedParts: chunked.partCache.size,
7192
+ maxCachedParts: chunked.maxCachedParts
7193
+ } : null,
7194
+ data,
7195
+ error,
7196
+ reviewDecisions: [...reviewDecisions]
7197
+ });
7198
+ const notify = () => {
7199
+ const current = snapshot();
7200
+ for (const listener of listeners) listener(current);
7201
+ };
7202
+ const requireIndex = () => {
7203
+ if (!index) throw new Error("No AIWG index export loaded");
7204
+ return index;
7205
+ };
7206
+ return {
7207
+ loadIndex(value) {
7208
+ try {
7209
+ const parsed = assertAiwgFortemiIndexExport(value);
7210
+ index = parsed;
7211
+ chunked = null;
7212
+ data = null;
7213
+ reviewDecisions = [];
7214
+ error = null;
7215
+ notify();
7216
+ return parsed;
7217
+ } catch (err) {
7218
+ error = err instanceof Error ? err : new Error(String(err));
7219
+ notify();
7220
+ throw error;
7221
+ }
7222
+ },
7223
+ loadChunkedIndex(manifest, loader, options = {}) {
7224
+ try {
7225
+ const parsed = assertAiwgFortemiChunkManifest(manifest);
7226
+ index = null;
7227
+ chunked = {
7228
+ manifest: parsed,
7229
+ loader,
7230
+ maxCachedParts: clampMaxCachedParts(options.maxCachedParts),
7231
+ partCache: /* @__PURE__ */ new Map()
7232
+ };
7233
+ data = null;
7234
+ reviewDecisions = [];
7235
+ error = null;
7236
+ notify();
7237
+ return parsed;
7238
+ } catch (err) {
7239
+ error = err instanceof Error ? err : new Error(String(err));
7240
+ notify();
7241
+ throw error;
7242
+ }
7243
+ },
7244
+ getIndex() {
7245
+ return index;
7246
+ },
7247
+ getChunkedManifest() {
7248
+ return chunked?.manifest ?? null;
7249
+ },
7250
+ getSnapshot() {
7251
+ return snapshot();
7252
+ },
7253
+ query(query = "", options) {
7254
+ const result = queryAiwgFortemiIndex(requireIndex(), query, options);
7255
+ data = result;
7256
+ error = null;
7257
+ notify();
7258
+ return result;
7259
+ },
7260
+ async queryChunked(query = "", options) {
7261
+ if (!chunked) throw new Error("No AIWG chunked index manifest loaded");
7262
+ try {
7263
+ const result = await queryChunkedAiwgFortemiIndex(chunked, query, options);
7264
+ data = result;
7265
+ error = null;
7266
+ notify();
7267
+ return result;
7268
+ } catch (err) {
7269
+ error = err instanceof Error ? err : new Error(String(err));
7270
+ notify();
7271
+ throw error;
7272
+ }
7273
+ },
7274
+ clearChunkCache() {
7275
+ chunked?.partCache.clear();
7276
+ error = null;
7277
+ notify();
7278
+ },
7279
+ toCommunityGraph(options) {
7280
+ return aiwgFortemiIndexToCommunityGraph(requireIndex(), options);
7281
+ },
7282
+ setReviewDecision(input) {
7283
+ const decision = {
7284
+ ...input,
7285
+ updated_at: (/* @__PURE__ */ new Date()).toISOString()
7286
+ };
7287
+ reviewDecisions = [
7288
+ ...reviewDecisions.filter((item) => item.item_id !== decision.item_id),
7289
+ decision
7290
+ ].sort((left, right) => left.item_id.localeCompare(right.item_id));
7291
+ error = null;
7292
+ notify();
7293
+ return decision;
7294
+ },
7295
+ clearReviewDecision(itemId) {
7296
+ reviewDecisions = reviewDecisions.filter((item) => item.item_id !== itemId);
7297
+ error = null;
7298
+ notify();
7299
+ },
7300
+ createReviewDecisionExport(generatedAt) {
7301
+ return createAiwgReviewDecisionExport(requireIndex(), reviewDecisions, generatedAt);
7302
+ },
7303
+ subscribe(listener) {
7304
+ listeners.add(listener);
7305
+ return () => {
7306
+ listeners.delete(listener);
7307
+ };
7308
+ }
7309
+ };
7310
+ }
7311
+ function aiwgFortemiIndexToCommunityGraph(index, options = {}) {
7312
+ const ids = new Set(index.items.map((item) => item.id));
7313
+ const relationshipWeights = options.relationshipWeights ?? {};
7314
+ const edgeCounts = /* @__PURE__ */ new Map();
7315
+ for (const item of index.items) {
7316
+ for (const relationship of item.relationships) {
7317
+ if (!ids.has(relationship.target_id) && !options.includeDanglingRelationships) continue;
7318
+ const kind = relationship.type;
7319
+ const baseWeight = relationshipWeights[kind] ?? 1;
7320
+ const key = `${item.id}\0${relationship.target_id}\0${kind}`;
7321
+ const existing = edgeCounts.get(key);
7322
+ if (existing) existing.weight += baseWeight;
7323
+ else edgeCounts.set(key, { source: item.id, target: relationship.target_id, kind, weight: baseWeight });
7324
+ }
7325
+ }
7326
+ const communities = /* @__PURE__ */ new Map();
7327
+ for (const item of index.items) {
7328
+ const communityIds = communityIdsFor(item, options);
7329
+ for (const communityId of communityIds) {
7330
+ const nodes = communities.get(communityId) ?? [];
7331
+ nodes.push(item.id);
7332
+ communities.set(communityId, nodes);
7333
+ }
7334
+ }
7335
+ return {
7336
+ nodes: index.items.map((item) => ({ id: item.id })),
7337
+ edges: Array.from(edgeCounts.values()).sort((left, right) => left.source.localeCompare(right.source) || left.target.localeCompare(right.target) || left.kind.localeCompare(right.kind)),
7338
+ communities: Array.from(communities.entries()).map(([id, nodes]) => ({ id, nodes: [...new Set(nodes)].sort() })).sort((left, right) => left.id.localeCompare(right.id))
7339
+ };
7340
+ }
7341
+ function communityIdsFor(item, options) {
7342
+ if (options.communityFacet) {
7343
+ const values = item.facets[options.communityFacet] ?? [];
7344
+ if (values.length > 0) return values.map((value) => `${options.communityFacet}:${value}`);
7345
+ }
7346
+ if (options.communityTagPrefix) {
7347
+ const prefix = options.communityTagPrefix;
7348
+ const tags = item.tags.filter((tag) => tag.startsWith(prefix));
7349
+ if (tags.length > 0) return tags;
7350
+ }
7351
+ if (item.concepts.length > 0) return item.concepts.map((concept) => `concept:${concept}`);
7352
+ return [`type:${item.type}`];
7353
+ }
6549
7354
 
6550
7355
  // src/index.ts
6551
- var VERSION = "2026.6.0";
7356
+ var VERSION = "2026.6.2";
6552
7357
 
6553
- export { ArchiveManager, AttachmentsRepository, CURRENT_SHARD_VERSION, CapabilityManager, CaptureKnowledgeInputSchema, CollectionsRepository, CommunitiesRepository, EmbeddingSetsRepository, FallbackRouter, FortemiToolManifest, GetNoteInputSchema, GraphRepository, JOB_CAPABILITIES, JOB_PRIORITIES, JobQueueWorker, LOCAL_ENDPOINTS, LinksRepository, ListNotesInputSchema, ManageArchiveInputSchema, ManageAttachmentsInputSchema, ManageCapabilitiesInputSchema, ManageCollectionsInputSchema, ManageLinksInputSchema, ManageNoteInputSchema, ManageTagsInputSchema, MemoryBlobStore, MigrationRunner, NotesRepository, OpenAICompatibleProvider, PGliteStorageBackend, PGliteStorageBackendFactory, PGliteWorkerClient, ProviderRegistry, SHARD_FORMAT, SearchInputSchema, SearchRepository, SkosRepository, TagsRepository, TransactionProxy, TypedEventBus, VERSION, aiRevisionHandler, allMigrations, appendPluginScript, assertAiwgFortemiIndexExport, buildNoteConditions, buildPluginCsp, captureKnowledge, chunkText, classifyError, classifyModel, collectionFromShard, collectionToShard, computeHash, computeSri, conceptTaggingHandler, cosineSimilarity, createAiwgReviewDecisionExport, createBlobStore, createCspReportHandler, createFortemi, createLegacyProvider, createPGliteInstance, createRoutes, defaultStorageBackendFactory, detectCommunities, detectGpuCapabilities, detectInferenceCapabilities, discoverLocalProviders, embeddingFromShard, embeddingGenerationHandler, embeddingSetFromShard, embeddingSetMemberToShard, embeddingSetToShard, embeddingToShard, enqueueFullWorkflow, enqueueJob, enqueueNoteCreationJobs, estimateModelFit, estimateVramMB, estimateVramTier, exportShard, fetchPluginScript, fortemiManifest, generateId, getAiwgFortemiFacets, getEmbedFunction, getJobQueueStatus, getLlmFunction, getNote, importShard, isPluginScriptAllowed, linkFromShard, linkToShard, linkingHandler, listNotes, manageArchive, manageAttachments, manageCapabilities, manageCollections, manageLinks, manageNote, manageTags, matchRoute, noteFromShard, noteSkosTagToShard, noteToShard, packTarGz, parseCspReport, provenanceEdgeToShard, queryAiwgFortemiIndex, registerLlmCapability, registerSemanticCapability, registerServiceWorker, searchTool, selectLlmModel, setEmbedFunction, setLlmFunction, sha256Hex, skosConceptToShard, skosRelationToShard, skosSchemeToShard, suggestTags, tagsFromShard, tagsToShard, titleGenerationHandler, unpackTarGz, unregisterLlmCapability, unregisterSemanticCapability, validateAiwgFortemiIndexExport, validateChecksums, verifySri };
7358
+ export { ArchiveManager, AttachmentsRepository, CURRENT_SHARD_VERSION, CapabilityManager, CaptureKnowledgeInputSchema, CollectionsRepository, CommunitiesRepository, EmbeddingSetsRepository, FallbackRouter, FortemiToolManifest, GetNoteInputSchema, GraphRepository, JOB_CAPABILITIES, JOB_PRIORITIES, JobQueueWorker, LOCAL_ENDPOINTS, LinksRepository, ListNotesInputSchema, ManageArchiveInputSchema, ManageAttachmentsInputSchema, ManageCapabilitiesInputSchema, ManageCollectionsInputSchema, ManageLinksInputSchema, ManageNoteInputSchema, ManageTagsInputSchema, MemoryBlobStore, MigrationRunner, NotesRepository, OpenAICompatibleProvider, PGliteStorageBackend, PGliteStorageBackendFactory, PGliteWorkerClient, PGliteWorkerStorageBackend, PGliteWorkerStorageBackendFactory, ProviderRegistry, SHARD_FORMAT, SearchInputSchema, SearchRepository, SkosRepository, TagsRepository, TransactionProxy, TypedEventBus, VERSION, aiRevisionHandler, aiwgFortemiIndexToCommunityGraph, allMigrations, appendPluginScript, assertAiwgFortemiChunkManifest, assertAiwgFortemiChunkPart, assertAiwgFortemiIndexExport, buildNoteConditions, buildPluginCsp, captureKnowledge, chunkText, classifyError, classifyModel, collectionFromShard, collectionToShard, computeHash, computeSri, conceptTaggingHandler, cosineSimilarity, createAiwgFetchChunkLoader, createAiwgIndexController, createAiwgReviewDecisionExport, createBlobStore, createCspReportHandler, createFortemi, createLegacyProvider, createPGliteInstance, createRoutes, defaultStorageBackendFactory, detectCommunities, detectGpuCapabilities, detectInferenceCapabilities, discoverLocalProviders, embeddingFromShard, embeddingGenerationHandler, embeddingSetFromShard, embeddingSetMemberToShard, embeddingSetToShard, embeddingToShard, enqueueFullWorkflow, enqueueJob, enqueueNoteCreationJobs, estimateModelFit, estimateVramMB, estimateVramTier, exportShard, fetchPluginScript, fortemiManifest, generateId, getAiwgFortemiFacets, getEmbedFunction, getFortemiBridge, getFortemiSecretStore, getJobQueueStatus, getLlmFunction, getNote, hasFortemiSecureSecrets, importShard, isPluginScriptAllowed, linkFromShard, linkToShard, linkingHandler, listNotes, manageArchive, manageAttachments, manageCapabilities, manageCollections, manageLinks, manageNote, manageTags, matchRoute, noteFromShard, noteSkosTagToShard, noteToShard, packTarGz, parseCspReport, provenanceEdgeToShard, queryAiwgFortemiIndex, registerLlmCapability, registerSemanticCapability, registerServiceWorker, searchTool, selectLlmModel, setEmbedFunction, setLlmFunction, sha256Hex, skosConceptToShard, skosRelationToShard, skosSchemeToShard, suggestTags, tagsFromShard, tagsToShard, titleGenerationHandler, unpackTarGz, unregisterLlmCapability, unregisterSemanticCapability, validateAiwgFortemiChunkManifest, validateAiwgFortemiChunkPart, validateAiwgFortemiIndexExport, validateChecksums, verifySri };
6554
7359
  //# sourceMappingURL=index.js.map
6555
7360
  //# sourceMappingURL=index.js.map