@fortemi/core 2026.6.1 → 2026.6.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -917,6 +917,28 @@ var migration0008 = {
917
917
  `
918
918
  };
919
919
 
920
+ // src/migrations/0009_vector_selector_performance.ts
921
+ var migration0009 = {
922
+ version: 9,
923
+ name: "0009_vector_selector_performance",
924
+ sql: `
925
+ CREATE INDEX IF NOT EXISTS idx_embedding_set_kind ON embedding_set(kind);
926
+ CREATE INDEX IF NOT EXISTS idx_embedding_set_member_set ON embedding_set_member(embedding_set_id);
927
+ CREATE INDEX IF NOT EXISTS idx_embedding_set_member_note ON embedding_set_member(note_id);
928
+ CREATE INDEX IF NOT EXISTS idx_embedding_set_member_embedding ON embedding_set_member(embedding_id);
929
+
930
+ CREATE INDEX IF NOT EXISTS idx_note_source ON note(source);
931
+ CREATE INDEX IF NOT EXISTS idx_note_format ON note(format);
932
+ CREATE INDEX IF NOT EXISTS idx_note_visibility ON note(visibility);
933
+ CREATE INDEX IF NOT EXISTS idx_note_starred ON note(is_starred);
934
+ CREATE INDEX IF NOT EXISTS idx_note_archived ON note(is_archived);
935
+ CREATE INDEX IF NOT EXISTS idx_note_updated_at ON note(updated_at);
936
+
937
+ CREATE INDEX IF NOT EXISTS idx_note_revised_current_user_edited ON note_revised_current(is_user_edited);
938
+ CREATE INDEX IF NOT EXISTS idx_note_revised_current_generation_count ON note_revised_current(generation_count);
939
+ `
940
+ };
941
+
920
942
  // src/migrations/index.ts
921
943
  var allMigrations = [
922
944
  migration0001,
@@ -926,7 +948,8 @@ var allMigrations = [
926
948
  migration0005,
927
949
  migration0006,
928
950
  migration0007,
929
- migration0008
951
+ migration0008,
952
+ migration0009
930
953
  ];
931
954
 
932
955
  // src/archive-manager.ts
@@ -1725,6 +1748,9 @@ function dateString(value) {
1725
1748
  function dateMillis(value) {
1726
1749
  return value instanceof Date ? value.getTime() : new Date(value).getTime();
1727
1750
  }
1751
+ function hashJson(value) {
1752
+ return computeHash(new TextEncoder().encode(JSON.stringify(value)));
1753
+ }
1728
1754
  var EmbeddingSetsRepository = class {
1729
1755
  constructor(db) {
1730
1756
  this.db = db;
@@ -1772,7 +1798,12 @@ var EmbeddingSetsRepository = class {
1772
1798
  input.updatedAt ?? null
1773
1799
  ]
1774
1800
  );
1775
- return this.get(id);
1801
+ const row = await this.get(id);
1802
+ if (input.materialization?.allowed) {
1803
+ await this.refreshMaterializedVirtualSet(id);
1804
+ return this.get(id);
1805
+ }
1806
+ return row;
1776
1807
  }
1777
1808
  async ensureDefault() {
1778
1809
  const existing = await this.db.query(
@@ -1864,14 +1895,97 @@ var EmbeddingSetsRepository = class {
1864
1895
  const set = await this.get(selector.embeddingSetId);
1865
1896
  if (set.kind === "virtual") {
1866
1897
  const definition = this.definitionFromRow(set);
1867
- return this.resolveDefinition({ kind: "embedding-set", embeddingSetId: set.id }, definition);
1898
+ return this.resolveDefinition({ kind: "embedding-set", embeddingSetId: set.id }, definition, set);
1868
1899
  }
1869
1900
  return this.resolvePhysicalSet(selector, set.id);
1870
1901
  }
1871
1902
  if (!selector.definition) throw new Error("virtual-definition selector requires definition");
1872
1903
  return this.resolveDefinition(selector, selector.definition);
1873
1904
  }
1874
- async resolveDefinition(selector, definition) {
1905
+ async refreshMaterializedVirtualSet(setId) {
1906
+ const set = await this.get(setId);
1907
+ if (set.kind !== "virtual") throw new Error(`Embedding set is not virtual: ${setId}`);
1908
+ const definition = this.definitionFromRow(set);
1909
+ if (!definition.materialization?.allowed) {
1910
+ throw new Error(`Virtual embedding set does not allow materialization: ${setId}`);
1911
+ }
1912
+ const live = await this.resolveDefinition(
1913
+ { kind: "embedding-set", embeddingSetId: setId },
1914
+ definition,
1915
+ set,
1916
+ { forceLive: true }
1917
+ );
1918
+ await this.db.query(`DELETE FROM embedding_set_member WHERE embedding_set_id = $1`, [setId]);
1919
+ for (const row of live.rows) {
1920
+ await this.db.query(
1921
+ `INSERT INTO embedding_set_member (embedding_set_id, note_id, embedding_id)
1922
+ VALUES ($1, $2, $3)
1923
+ ON CONFLICT DO NOTHING`,
1924
+ [setId, row.note_id, row.embedding_id]
1925
+ );
1926
+ }
1927
+ const inputHash = this.resolutionInputHash(definition, live.rows);
1928
+ const generatedAt = (/* @__PURE__ */ new Date()).toISOString();
1929
+ const materialization = {
1930
+ ...definition.materialization,
1931
+ allowed: true,
1932
+ includeResolvedMembers: true,
1933
+ freshness: "fresh",
1934
+ inputHash,
1935
+ generatedAt,
1936
+ resolvedMemberCount: live.rows.length
1937
+ };
1938
+ const freshness = {
1939
+ status: "fresh",
1940
+ sourceHash: inputHash,
1941
+ checkedAt: generatedAt
1942
+ };
1943
+ await this.db.query(
1944
+ `UPDATE embedding_set
1945
+ SET materialization_json = $2::jsonb, freshness_json = $3::jsonb, updated_at = now()
1946
+ WHERE id = $1`,
1947
+ [setId, jsonParam(materialization), jsonParam(freshness)]
1948
+ );
1949
+ return this.finalizeResolution(
1950
+ { kind: "embedding-set", embeddingSetId: setId },
1951
+ live.rows,
1952
+ live.errors,
1953
+ definition.compatibility,
1954
+ "fresh",
1955
+ "materialized"
1956
+ );
1957
+ }
1958
+ async markVirtualSetStale(setId, reason) {
1959
+ const set = await this.get(setId);
1960
+ if (set.kind !== "virtual") throw new Error(`Embedding set is not virtual: ${setId}`);
1961
+ const definition = this.definitionFromRow(set);
1962
+ const now = (/* @__PURE__ */ new Date()).toISOString();
1963
+ const materialization = definition.materialization ? { ...definition.materialization, freshness: "stale" } : null;
1964
+ await this.db.query(
1965
+ `UPDATE embedding_set
1966
+ SET materialization_json = $2::jsonb, freshness_json = $3::jsonb, updated_at = now()
1967
+ WHERE id = $1`,
1968
+ [
1969
+ setId,
1970
+ jsonParam(materialization),
1971
+ jsonParam({ status: "stale", sourceHash: definition.materialization?.inputHash, checkedAt: now, reason })
1972
+ ]
1973
+ );
1974
+ }
1975
+ async resolveDefinition(selector, definition, set, options = {}) {
1976
+ if (!options.forceLive && set && definition.materialization?.allowed && definition.materialization.freshness === "fresh") {
1977
+ const materialized = await this.resolveMaterializedRows(set.id);
1978
+ if (materialized.length > 0 || definition.materialization.resolvedMemberCount === 0) {
1979
+ return this.finalizeResolution(
1980
+ selector,
1981
+ materialized,
1982
+ [],
1983
+ definition.compatibility,
1984
+ "fresh",
1985
+ "materialized"
1986
+ );
1987
+ }
1988
+ }
1875
1989
  let rows;
1876
1990
  const errors = [];
1877
1991
  switch (definition.source.type) {
@@ -1893,10 +2007,10 @@ var EmbeddingSetsRepository = class {
1893
2007
  default:
1894
2008
  rows = [];
1895
2009
  }
1896
- return this.finalizeResolution(selector, rows, errors, definition.compatibility, definition.materialization?.freshness ?? "unknown");
2010
+ return this.finalizeResolution(selector, rows, errors, definition.compatibility, definition.materialization?.freshness ?? "unknown", "live");
1897
2011
  }
1898
2012
  async resolvePhysicalSet(selector, setId) {
1899
- return this.finalizeResolution(selector, await this.resolvePhysicalRows(setId), [], DEFAULT_COMPATIBILITY, "fresh");
2013
+ return this.finalizeResolution(selector, await this.resolvePhysicalRows(setId), [], DEFAULT_COMPATIBILITY, "fresh", "live");
1900
2014
  }
1901
2015
  async resolvePhysicalRows(setId) {
1902
2016
  const result = await this.db.query(
@@ -1908,6 +2022,17 @@ var EmbeddingSetsRepository = class {
1908
2022
  );
1909
2023
  return result.rows;
1910
2024
  }
2025
+ async resolveMaterializedRows(setId) {
2026
+ const result = await this.db.query(
2027
+ `SELECT e.note_id, e.embedding_set_id, e.id as embedding_id, e.vector::text as vector, e.created_at
2028
+ FROM embedding_set_member m
2029
+ JOIN embedding e ON e.id = m.embedding_id
2030
+ WHERE m.embedding_set_id = $1
2031
+ ORDER BY e.note_id, e.created_at DESC`,
2032
+ [setId]
2033
+ );
2034
+ return result.rows;
2035
+ }
1911
2036
  async resolveCriteriaSource(source) {
1912
2037
  const criteria = source.criteria;
1913
2038
  if (criteria.conceptIds && criteria.conceptIds.length > 0) {
@@ -1928,6 +2053,50 @@ var EmbeddingSetsRepository = class {
1928
2053
  conditions.push(`EXISTS (SELECT 1 FROM collection_note cn WHERE cn.note_id = n.id AND cn.collection_id = ANY($${idx++}))`);
1929
2054
  params.push(criteria.collectionIds);
1930
2055
  }
2056
+ if (criteria.sources?.length) {
2057
+ conditions.push(`n.source = ANY($${idx++})`);
2058
+ params.push(criteria.sources);
2059
+ }
2060
+ if (criteria.formats?.length) {
2061
+ conditions.push(`n.format = ANY($${idx++})`);
2062
+ params.push(criteria.formats);
2063
+ }
2064
+ if (criteria.visibilities?.length) {
2065
+ conditions.push(`n.visibility = ANY($${idx++})`);
2066
+ params.push(criteria.visibilities);
2067
+ }
2068
+ if (criteria.isStarred !== void 0) {
2069
+ conditions.push(`n.is_starred = $${idx++}`);
2070
+ params.push(criteria.isStarred);
2071
+ }
2072
+ if (criteria.isArchived !== void 0) {
2073
+ conditions.push(`n.is_archived = $${idx++}`);
2074
+ params.push(criteria.isArchived);
2075
+ }
2076
+ if (criteria.hasTitle !== void 0) {
2077
+ conditions.push(criteria.hasTitle ? `n.title IS NOT NULL AND n.title <> ''` : `(n.title IS NULL OR n.title = '')`);
2078
+ }
2079
+ if (criteria.hasEmbedding === false) {
2080
+ conditions.push("FALSE");
2081
+ }
2082
+ if (criteria.isUserEdited !== void 0) {
2083
+ conditions.push(`COALESCE(c.is_user_edited, false) = $${idx++}`);
2084
+ params.push(criteria.isUserEdited);
2085
+ }
2086
+ if (criteria.hasAiMetadata !== void 0) {
2087
+ conditions.push(criteria.hasAiMetadata ? `c.ai_metadata IS NOT NULL` : `c.ai_metadata IS NULL`);
2088
+ }
2089
+ if (criteria.hasRevisions !== void 0) {
2090
+ conditions.push(criteria.hasRevisions ? `EXISTS (SELECT 1 FROM note_revision nr WHERE nr.note_id = n.id)` : `NOT EXISTS (SELECT 1 FROM note_revision nr WHERE nr.note_id = n.id)`);
2091
+ }
2092
+ if (criteria.minGenerationCount !== void 0) {
2093
+ conditions.push(`COALESCE(c.generation_count, 0) >= $${idx++}`);
2094
+ params.push(criteria.minGenerationCount);
2095
+ }
2096
+ if (criteria.maxGenerationCount !== void 0) {
2097
+ conditions.push(`COALESCE(c.generation_count, 0) <= $${idx++}`);
2098
+ params.push(criteria.maxGenerationCount);
2099
+ }
1931
2100
  if (criteria.updatedAfter) {
1932
2101
  conditions.push(`n.updated_at >= $${idx++}`);
1933
2102
  params.push(criteria.updatedAfter);
@@ -2018,7 +2187,7 @@ var EmbeddingSetsRepository = class {
2018
2187
  }
2019
2188
  return resolved.sort((a, b) => a.note_id.localeCompare(b.note_id));
2020
2189
  }
2021
- finalizeResolution(selector, rows, errors, compatibility, freshness) {
2190
+ finalizeResolution(selector, rows, errors, compatibility, freshness, resolutionSource) {
2022
2191
  const deduped = this.resolveDuplicateRows(rows, compatibility, errors);
2023
2192
  return {
2024
2193
  selector,
@@ -2026,9 +2195,17 @@ var EmbeddingSetsRepository = class {
2026
2195
  noteIds: deduped.map((row) => row.note_id),
2027
2196
  embeddingIds: deduped.map((row) => row.embedding_id),
2028
2197
  errors,
2029
- freshness: { status: freshness }
2198
+ freshness: { status: freshness },
2199
+ resolutionSource
2030
2200
  };
2031
2201
  }
2202
+ resolutionInputHash(definition, rows) {
2203
+ return hashJson({
2204
+ source: definition.source,
2205
+ compatibility: definition.compatibility,
2206
+ members: rows.map((row) => [row.note_id, row.embedding_set_id, row.embedding_id])
2207
+ });
2208
+ }
2032
2209
  definitionFromRow(row) {
2033
2210
  const source = asObject(row.source_json);
2034
2211
  if (!source) throw new Error(`Virtual embedding set has no source definition: ${row.id}`);
@@ -2433,8 +2610,9 @@ var SearchRepository = class {
2433
2610
  };
2434
2611
 
2435
2612
  // src/repositories/graph-repository.ts
2436
- var SIMILARITY_GRAPH_ALGORITHM = "knn-v1";
2437
- function hashJson(value) {
2613
+ var SIMILARITY_GRAPH_ALGORITHM = "knn-batched-v1";
2614
+ var DEFAULT_GRAPH_BATCH_SIZE = 64;
2615
+ function hashJson2(value) {
2438
2616
  return computeHash(new TextEncoder().encode(JSON.stringify(value)));
2439
2617
  }
2440
2618
  function sourceIdFor(inputHash) {
@@ -2445,6 +2623,45 @@ function jsonObject(value) {
2445
2623
  if (typeof value === "string") return JSON.parse(value);
2446
2624
  return value;
2447
2625
  }
2626
+ async function yieldToEventLoop() {
2627
+ const scheduler = globalThis.scheduler;
2628
+ if (scheduler?.yield) {
2629
+ await scheduler.yield();
2630
+ return;
2631
+ }
2632
+ await new Promise((resolve) => setTimeout(resolve, 0));
2633
+ }
2634
+ async function maybeYield(done, every) {
2635
+ if (every > 0 && done > 0 && done % every === 0) {
2636
+ await yieldToEventLoop();
2637
+ }
2638
+ }
2639
+ function parseVector(value) {
2640
+ return value.replace(/^\[/, "").replace(/\]$/, "").split(",").filter((part) => part.length > 0).map(Number);
2641
+ }
2642
+ function vectorScore(left, right, metric) {
2643
+ let dot = 0;
2644
+ let leftNorm = 0;
2645
+ let rightNorm = 0;
2646
+ let squaredDistance = 0;
2647
+ for (let i = 0; i < Math.min(left.length, right.length); i++) {
2648
+ dot += left[i] * right[i];
2649
+ leftNorm += left[i] * left[i];
2650
+ rightNorm += right[i] * right[i];
2651
+ const diff = left[i] - right[i];
2652
+ squaredDistance += diff * diff;
2653
+ }
2654
+ if (metric === "inner_product") {
2655
+ return { distance: -dot, similarity: dot };
2656
+ }
2657
+ if (metric === "l2") {
2658
+ const distance = Math.sqrt(squaredDistance);
2659
+ return { distance, similarity: -distance };
2660
+ }
2661
+ const denominator = Math.sqrt(leftNorm) * Math.sqrt(rightNorm);
2662
+ const similarity = denominator === 0 ? 0 : dot / denominator;
2663
+ return { distance: 1 - similarity, similarity };
2664
+ }
2448
2665
  function detectCommunities(edges, nodes = [], options = {}) {
2449
2666
  const nodeIds = new Set(nodes.map((n) => n.id));
2450
2667
  for (const edge of edges) {
@@ -2521,7 +2738,7 @@ var GraphRepository = class {
2521
2738
  const normalized = this.normalizeSimilarityRequest(request);
2522
2739
  const resolved = await new EmbeddingSetsRepository(this.db).resolveSelector(normalized.selector);
2523
2740
  const cacheKey = await this.computeSimilarityGraphCacheKey(normalized, resolved);
2524
- const inputHash = hashJson(cacheKey);
2741
+ const inputHash = hashJson2(cacheKey);
2525
2742
  const source = await this.findGraphSource(inputHash);
2526
2743
  if (!source) return null;
2527
2744
  const graph = await this.graphFromArtifact(source.id, resolved.noteIds);
@@ -2536,7 +2753,7 @@ var GraphRepository = class {
2536
2753
  const normalized = this.normalizeSimilarityRequest(request);
2537
2754
  const resolved = await new EmbeddingSetsRepository(this.db).resolveSelector(normalized.selector);
2538
2755
  const cacheKey = await this.computeSimilarityGraphCacheKey(normalized, resolved);
2539
- const inputHash = hashJson(cacheKey);
2756
+ const inputHash = hashJson2(cacheKey);
2540
2757
  if (normalized.source !== "live-only") {
2541
2758
  const cached = await this.findGraphSource(inputHash);
2542
2759
  if (cached?.freshness === "fresh") {
@@ -2569,7 +2786,7 @@ var GraphRepository = class {
2569
2786
  };
2570
2787
  }
2571
2788
  async saveSimilarityGraphArtifact(input) {
2572
- const inputHash = hashJson(input.cacheKey);
2789
+ const inputHash = hashJson2(input.cacheKey);
2573
2790
  const id = sourceIdFor(inputHash);
2574
2791
  const parameters = {
2575
2792
  k: input.request.k,
@@ -2620,27 +2837,32 @@ var GraphRepository = class {
2620
2837
  async buildSimilarityGraphFromResolved(resolved, options) {
2621
2838
  const k = options.k ?? 5;
2622
2839
  const minSimilarity = options.minSimilarity ?? options.threshold ?? -1;
2840
+ const metric = options.metric ?? "cosine";
2841
+ const yieldEvery = options.yieldEvery ?? options.batchSize ?? DEFAULT_GRAPH_BATCH_SIZE;
2623
2842
  const embeddings = resolved.rows;
2624
2843
  const nodes = embeddings.map((row) => ({ id: row.note_id }));
2625
2844
  const edgeMap = /* @__PURE__ */ new Map();
2626
- for (const row of embeddings) {
2627
- const neighbors = await this.db.query(
2628
- `SELECT note_id, 1 - (vector <=> $2::vector) as similarity
2629
- FROM embedding
2630
- WHERE id = ANY($1) AND note_id != $3
2631
- ORDER BY vector <=> $2::vector ASC
2632
- LIMIT $4`,
2633
- [resolved.embeddingIds, row.vector, row.note_id, k]
2634
- );
2635
- for (const neighbor of neighbors.rows) {
2845
+ const vectors = embeddings.map((row) => ({
2846
+ row,
2847
+ vector: parseVector(row.vector)
2848
+ }));
2849
+ options.onProgress?.({ phase: "prepare", done: vectors.length, total: vectors.length });
2850
+ for (const [index, item] of vectors.entries()) {
2851
+ const neighbors = vectors.filter((candidate) => candidate.row.note_id !== item.row.note_id).map((candidate) => {
2852
+ const score = vectorScore(item.vector, candidate.vector, metric);
2853
+ return { noteId: candidate.row.note_id, ...score };
2854
+ }).sort((a, b) => a.distance - b.distance || a.noteId.localeCompare(b.noteId)).slice(0, k);
2855
+ for (const neighbor of neighbors) {
2636
2856
  if (neighbor.similarity < minSimilarity) continue;
2637
- const [source, target] = [row.note_id, neighbor.note_id].sort();
2857
+ const [source, target] = [item.row.note_id, neighbor.noteId].sort();
2638
2858
  const id = `${source}\0${target}`;
2639
2859
  const existing = edgeMap.get(id);
2640
2860
  if (!existing || neighbor.similarity > existing.weight) {
2641
2861
  edgeMap.set(id, { source, target, weight: neighbor.similarity, kind: "similarity" });
2642
2862
  }
2643
2863
  }
2864
+ options.onProgress?.({ phase: "neighbors", done: index + 1, total: vectors.length });
2865
+ await maybeYield(index + 1, yieldEvery);
2644
2866
  }
2645
2867
  const edges = Array.from(edgeMap.values()).sort(
2646
2868
  (a, b) => a.source.localeCompare(b.source) || a.target.localeCompare(b.target)
@@ -2651,7 +2873,7 @@ var GraphRepository = class {
2651
2873
  const firstSetId = resolved.rows[0]?.embedding_set_id;
2652
2874
  const set = firstSetId ? await new EmbeddingSetsRepository(this.db).get(firstSetId) : null;
2653
2875
  return {
2654
- selectorHash: hashJson(request.selector),
2876
+ selectorHash: hashJson2(request.selector),
2655
2877
  resolvedEmbeddingSetId: request.selector.kind === "embedding-set" ? request.selector.embeddingSetId : void 0,
2656
2878
  virtualSetId: request.selector.kind === "virtual-definition" ? request.selector.definition?.id : void 0,
2657
2879
  k: request.k,
@@ -2660,9 +2882,9 @@ var GraphRepository = class {
2660
2882
  model: set?.model_name ?? "unknown",
2661
2883
  dimension: set?.dimensions ?? 0,
2662
2884
  truncateDimension: set?.truncate_dimension ?? null,
2663
- memberHash: hashJson(resolved.rows.map((row) => [row.note_id, row.embedding_set_id, row.embedding_id])),
2664
- vectorHash: hashJson(resolved.rows.map((row) => [row.embedding_id, row.vector])),
2665
- parameterHash: hashJson({ k: request.k, minSimilarity: request.minSimilarity, metric: request.metric, algorithm: SIMILARITY_GRAPH_ALGORITHM })
2885
+ memberHash: hashJson2(resolved.rows.map((row) => [row.note_id, row.embedding_set_id, row.embedding_id])),
2886
+ vectorHash: hashJson2(resolved.rows.map((row) => [row.embedding_id, row.vector])),
2887
+ parameterHash: hashJson2({ k: request.k, minSimilarity: request.minSimilarity, metric: request.metric, algorithm: SIMILARITY_GRAPH_ALGORITHM })
2666
2888
  };
2667
2889
  }
2668
2890
  async findGraphSource(inputHash) {
@@ -5651,7 +5873,7 @@ function embeddingToShard(emb) {
5651
5873
  id: emb.id,
5652
5874
  note_id: emb.note_id,
5653
5875
  embedding_set_id: emb.embedding_set_id,
5654
- vector: typeof emb.vector === "string" ? parseVector(emb.vector) : emb.vector,
5876
+ vector: typeof emb.vector === "string" ? parseVector2(emb.vector) : emb.vector,
5655
5877
  created_at: toISOString(emb.created_at)
5656
5878
  };
5657
5879
  }
@@ -5717,7 +5939,7 @@ function toISOString(date) {
5717
5939
  if (date instanceof Date) return date.toISOString();
5718
5940
  return date;
5719
5941
  }
5720
- function parseVector(vectorStr) {
5942
+ function parseVector2(vectorStr) {
5721
5943
  const inner = vectorStr.replace(/^\[/, "").replace(/\]$/, "");
5722
5944
  return inner.split(",").map(Number);
5723
5945
  }
@@ -5891,6 +6113,7 @@ async function exportShard(db, options) {
5891
6113
  if (options?.includeEmbeddings) {
5892
6114
  const embeddingSetIds = options.embeddingSetIds?.filter(Boolean) ?? [];
5893
6115
  const setScoped = embeddingSetIds.length > 0;
6116
+ const includeMaterializedSelectors = options.includeMaterializedSelectors === true;
5894
6117
  const embSetRows = await db.query(
5895
6118
  `SELECT * FROM embedding_set
5896
6119
  ${setScoped ? "WHERE id = ANY($1)" : ""}
@@ -5898,7 +6121,14 @@ async function exportShard(db, options) {
5898
6121
  setScoped ? [embeddingSetIds] : []
5899
6122
  );
5900
6123
  const exportedSetIds = new Set(embSetRows.rows.map((row) => row.id));
5901
- const shardEmbSets = embSetRows.rows.map(embeddingSetToShard);
6124
+ const virtualSetIds = new Set(embSetRows.rows.filter((row) => row.kind === "virtual").map((row) => row.id));
6125
+ const shardEmbSets = embSetRows.rows.map((row) => embeddingSetToShard(
6126
+ row.kind === "virtual" && !includeMaterializedSelectors ? {
6127
+ ...row,
6128
+ materialization_json: void 0,
6129
+ freshness_json: { status: "unknown" }
6130
+ } : row
6131
+ ));
5902
6132
  files.set("embedding_sets.json", encoder.encode(JSON.stringify(shardEmbSets)));
5903
6133
  components.push("embedding_sets");
5904
6134
  counts.embedding_sets = shardEmbSets.length;
@@ -5908,7 +6138,7 @@ async function exportShard(db, options) {
5908
6138
  setScoped ? [embeddingSetIds] : []
5909
6139
  );
5910
6140
  const scopedEmbMemberRows = embMemberRows.rows.filter(
5911
- (member) => exportedSetIds.has(member.embedding_set_id) && exportedNoteIds.has(member.note_id)
6141
+ (member) => exportedSetIds.has(member.embedding_set_id) && exportedNoteIds.has(member.note_id) && (includeMaterializedSelectors || !virtualSetIds.has(member.embedding_set_id))
5912
6142
  );
5913
6143
  const membersJsonl = scopedEmbMemberRows.map((m) => JSON.stringify(embeddingSetMemberToShard(m))).join("\n");
5914
6144
  files.set("embedding_set_members.jsonl", encoder.encode(membersJsonl));
@@ -6044,7 +6274,7 @@ async function exportShard(db, options) {
6044
6274
  // src/shard/shard-import.ts
6045
6275
  var decoder = new TextDecoder();
6046
6276
  var DEFAULT_BATCH_SIZE = 250;
6047
- async function yieldToEventLoop() {
6277
+ async function yieldToEventLoop2() {
6048
6278
  const scheduler = globalThis.scheduler;
6049
6279
  if (scheduler?.yield) {
6050
6280
  await scheduler.yield();
@@ -6052,9 +6282,9 @@ async function yieldToEventLoop() {
6052
6282
  }
6053
6283
  await new Promise((resolve) => setTimeout(resolve, 0));
6054
6284
  }
6055
- async function maybeYield(done, batchSize) {
6285
+ async function maybeYield2(done, batchSize) {
6056
6286
  if (batchSize > 0 && done > 0 && done % batchSize === 0) {
6057
- await yieldToEventLoop();
6287
+ await yieldToEventLoop2();
6058
6288
  }
6059
6289
  }
6060
6290
  async function importShard(db, data, options) {
@@ -6218,7 +6448,7 @@ async function importShard(db, data, options) {
6218
6448
  }
6219
6449
  counts.collections++;
6220
6450
  report?.({ phase: "collections", done: index + 1, total: parsedCollections.length });
6221
- await maybeYield(index + 1, batchSize);
6451
+ await maybeYield2(index + 1, batchSize);
6222
6452
  }
6223
6453
  report?.({ phase: "notes", done: 0, total: parsedNotes.length });
6224
6454
  for (const [index, shardNote] of parsedNotes.entries()) {
@@ -6299,7 +6529,7 @@ async function importShard(db, data, options) {
6299
6529
  }
6300
6530
  counts.notes++;
6301
6531
  report?.({ phase: "notes", done: index + 1, total: parsedNotes.length });
6302
- await maybeYield(index + 1, batchSize);
6532
+ await maybeYield2(index + 1, batchSize);
6303
6533
  }
6304
6534
  const totalSkos = parsedSkosSchemes.length + parsedSkosConcepts.length + parsedSkosRelations.length + parsedNoteSkosTags.length;
6305
6535
  let doneSkos = 0;
@@ -6321,7 +6551,7 @@ async function importShard(db, data, options) {
6321
6551
  }
6322
6552
  counts.skos_schemes++;
6323
6553
  report?.({ phase: "skos", done: ++doneSkos, total: totalSkos });
6324
- await maybeYield(doneSkos, batchSize);
6554
+ await maybeYield2(doneSkos, batchSize);
6325
6555
  }
6326
6556
  for (const concept of parsedSkosConcepts) {
6327
6557
  const altLabels = JSON.stringify(concept.alt_labels ?? []);
@@ -6341,7 +6571,7 @@ async function importShard(db, data, options) {
6341
6571
  }
6342
6572
  counts.skos_concepts++;
6343
6573
  report?.({ phase: "skos", done: ++doneSkos, total: totalSkos });
6344
- await maybeYield(doneSkos, batchSize);
6574
+ await maybeYield2(doneSkos, batchSize);
6345
6575
  }
6346
6576
  report?.({ phase: "links", done: 0, total: parsedLinks.length });
6347
6577
  for (const [index, shardLink] of parsedLinks.entries()) {
@@ -6362,7 +6592,7 @@ async function importShard(db, data, options) {
6362
6592
  }
6363
6593
  counts.links++;
6364
6594
  report?.({ phase: "links", done: index + 1, total: parsedLinks.length });
6365
- await maybeYield(index + 1, batchSize);
6595
+ await maybeYield2(index + 1, batchSize);
6366
6596
  }
6367
6597
  for (const relation of parsedSkosRelations) {
6368
6598
  if (strategy === "replace") {
@@ -6381,7 +6611,7 @@ async function importShard(db, data, options) {
6381
6611
  }
6382
6612
  counts.skos_relations++;
6383
6613
  report?.({ phase: "skos", done: ++doneSkos, total: totalSkos });
6384
- await maybeYield(doneSkos, batchSize);
6614
+ await maybeYield2(doneSkos, batchSize);
6385
6615
  }
6386
6616
  for (const tag of parsedNoteSkosTags) {
6387
6617
  await tx.query(
@@ -6392,7 +6622,7 @@ async function importShard(db, data, options) {
6392
6622
  );
6393
6623
  counts.note_skos_tags++;
6394
6624
  report?.({ phase: "skos", done: ++doneSkos, total: totalSkos });
6395
- await maybeYield(doneSkos, batchSize);
6625
+ await maybeYield2(doneSkos, batchSize);
6396
6626
  }
6397
6627
  report?.({ phase: "provenance", done: 0, total: parsedProvenanceEdges.length });
6398
6628
  for (const [index, edge] of parsedProvenanceEdges.entries()) {
@@ -6413,7 +6643,7 @@ async function importShard(db, data, options) {
6413
6643
  }
6414
6644
  counts.provenance_edges++;
6415
6645
  report?.({ phase: "provenance", done: index + 1, total: parsedProvenanceEdges.length });
6416
- await maybeYield(index + 1, batchSize);
6646
+ await maybeYield2(index + 1, batchSize);
6417
6647
  }
6418
6648
  report?.({ phase: "embedding_sets", done: 0, total: parsedEmbSets.length });
6419
6649
  for (const [index, shardSet] of parsedEmbSets.entries()) {
@@ -6440,7 +6670,7 @@ async function importShard(db, data, options) {
6440
6670
  }
6441
6671
  counts.embedding_sets++;
6442
6672
  report?.({ phase: "embedding_sets", done: index + 1, total: parsedEmbSets.length });
6443
- await maybeYield(index + 1, batchSize);
6673
+ await maybeYield2(index + 1, batchSize);
6444
6674
  }
6445
6675
  report?.({ phase: "embeddings", done: 0, total: parsedEmbeddings.length });
6446
6676
  for (const [index, shardEmb] of parsedEmbeddings.entries()) {
@@ -6461,7 +6691,7 @@ async function importShard(db, data, options) {
6461
6691
  }
6462
6692
  counts.embeddings++;
6463
6693
  report?.({ phase: "embeddings", done: index + 1, total: parsedEmbeddings.length });
6464
- await maybeYield(index + 1, batchSize);
6694
+ await maybeYield2(index + 1, batchSize);
6465
6695
  }
6466
6696
  const totalGraph = parsedGraphSources.length + parsedGraphEdges.length;
6467
6697
  let doneGraph = 0;
@@ -6479,7 +6709,7 @@ async function importShard(db, data, options) {
6479
6709
  );
6480
6710
  counts.graph_sources++;
6481
6711
  report?.({ phase: "graph", done: ++doneGraph, total: totalGraph });
6482
- await maybeYield(doneGraph, batchSize);
6712
+ await maybeYield2(doneGraph, batchSize);
6483
6713
  }
6484
6714
  for (const edge of parsedGraphEdges) {
6485
6715
  const metadata = edge.metadata == null ? null : JSON.stringify(edge.metadata);
@@ -6491,7 +6721,7 @@ async function importShard(db, data, options) {
6491
6721
  );
6492
6722
  counts.graph_edges++;
6493
6723
  report?.({ phase: "graph", done: ++doneGraph, total: totalGraph });
6494
- await maybeYield(doneGraph, batchSize);
6724
+ await maybeYield2(doneGraph, batchSize);
6495
6725
  }
6496
6726
  const totalCommunities = parsedCommunitySets.length + parsedCommunitySets.reduce((sum, set) => sum + (set.communities?.length ?? 0), 0) + parsedCommunityAssignments.length;
6497
6727
  let doneCommunities = 0;
@@ -6507,7 +6737,7 @@ async function importShard(db, data, options) {
6507
6737
  );
6508
6738
  counts.community_sets++;
6509
6739
  report?.({ phase: "communities", done: ++doneCommunities, total: totalCommunities });
6510
- await maybeYield(doneCommunities, batchSize);
6740
+ await maybeYield2(doneCommunities, batchSize);
6511
6741
  for (const community of set.communities ?? []) {
6512
6742
  const metadata = community.metadata == null ? null : JSON.stringify(community.metadata);
6513
6743
  await tx.query(
@@ -6518,7 +6748,7 @@ async function importShard(db, data, options) {
6518
6748
  );
6519
6749
  counts.communities++;
6520
6750
  report?.({ phase: "communities", done: ++doneCommunities, total: totalCommunities });
6521
- await maybeYield(doneCommunities, batchSize);
6751
+ await maybeYield2(doneCommunities, batchSize);
6522
6752
  }
6523
6753
  }
6524
6754
  for (const assignment of parsedCommunityAssignments) {
@@ -6531,7 +6761,7 @@ async function importShard(db, data, options) {
6531
6761
  );
6532
6762
  counts.community_assignments++;
6533
6763
  report?.({ phase: "communities", done: ++doneCommunities, total: totalCommunities });
6534
- await maybeYield(doneCommunities, batchSize);
6764
+ await maybeYield2(doneCommunities, batchSize);
6535
6765
  }
6536
6766
  report?.({ phase: "embedding_set_members", done: 0, total: parsedEmbMembers.length });
6537
6767
  for (const [index, member] of parsedEmbMembers.entries()) {
@@ -6542,7 +6772,7 @@ async function importShard(db, data, options) {
6542
6772
  );
6543
6773
  counts.embedding_set_members++;
6544
6774
  report?.({ phase: "embedding_set_members", done: index + 1, total: parsedEmbMembers.length });
6545
- await maybeYield(index + 1, batchSize);
6775
+ await maybeYield2(index + 1, batchSize);
6546
6776
  }
6547
6777
  });
6548
6778
  report?.({ phase: "index", done: 1, total: 1 });
@@ -6596,8 +6826,15 @@ var VALID_TYPES = /* @__PURE__ */ new Set([
6596
6826
  "crm.organization",
6597
6827
  "crm.event",
6598
6828
  "crm.interaction",
6599
- "aiwg.artifact"
6829
+ "aiwg.artifact",
6830
+ "docs.page"
6600
6831
  ]);
6832
+ var DEFAULT_QUERY_WEIGHTS = {
6833
+ title: 4,
6834
+ tag: 3,
6835
+ concept: 2,
6836
+ text: 1
6837
+ };
6601
6838
  function hasString(value) {
6602
6839
  return typeof value === "string" && value.length > 0;
6603
6840
  }
@@ -6605,6 +6842,16 @@ function pushFacet(counts, name, value) {
6605
6842
  counts[name] ??= {};
6606
6843
  counts[name][value] = (counts[name][value] ?? 0) + 1;
6607
6844
  }
6845
+ function hasNonNegativeInteger(value) {
6846
+ return Number.isInteger(value) && typeof value === "number" && value >= 0;
6847
+ }
6848
+ function hasPositiveInteger(value) {
6849
+ return Number.isInteger(value) && typeof value === "number" && value > 0;
6850
+ }
6851
+ function isFacetCounts(value) {
6852
+ if (!value || typeof value !== "object" || Array.isArray(value)) return false;
6853
+ return Object.values(value).every((counts) => !!counts && typeof counts === "object" && !Array.isArray(counts) && Object.values(counts).every((count) => hasNonNegativeInteger(count)));
6854
+ }
6608
6855
  function validateAiwgFortemiIndexExport(value) {
6609
6856
  const errors = [];
6610
6857
  const counts = {};
@@ -6656,6 +6903,87 @@ function assertAiwgFortemiIndexExport(value) {
6656
6903
  }
6657
6904
  return value;
6658
6905
  }
6906
+ function validateAiwgFortemiChunkManifest(value) {
6907
+ const errors = [];
6908
+ const data = value;
6909
+ if (data?.schema_version !== "aiwg.fortemi.index.chunk-manifest.v1") {
6910
+ errors.push("schema_version must be aiwg.fortemi.index.chunk-manifest.v1");
6911
+ }
6912
+ if (!hasString(data?.generated_at)) errors.push("generated_at is required");
6913
+ if (!hasString(data?.source?.repo)) errors.push("source.repo is required");
6914
+ if (!hasString(data?.source?.privacy)) errors.push("source.privacy is required");
6915
+ if (!hasNonNegativeInteger(data?.total)) errors.push("total must be a non-negative integer");
6916
+ if (!hasPositiveInteger(data?.part_size)) errors.push("part_size must be a positive integer");
6917
+ if (data.facets !== void 0 && !isFacetCounts(data.facets)) {
6918
+ errors.push("facets must be a nested string-to-number count object");
6919
+ }
6920
+ if (!Array.isArray(data?.parts)) errors.push("parts must be an array");
6921
+ let expectedOffset = 0;
6922
+ const parts = Array.isArray(data?.parts) ? data.parts : [];
6923
+ for (const [index, part] of parts.entries()) {
6924
+ if (!hasString(part.href)) errors.push("parts[" + index + "].href is required");
6925
+ if (!hasNonNegativeInteger(part.offset)) errors.push("parts[" + index + "].offset must be a non-negative integer");
6926
+ if (!hasNonNegativeInteger(part.count)) errors.push("parts[" + index + "].count must be a non-negative integer");
6927
+ if (hasNonNegativeInteger(part.offset) && part.offset !== expectedOffset) {
6928
+ errors.push("parts[" + index + "].offset must be " + expectedOffset);
6929
+ }
6930
+ if (hasNonNegativeInteger(part.count)) expectedOffset += part.count;
6931
+ }
6932
+ if (hasNonNegativeInteger(data?.total) && expectedOffset !== data.total) {
6933
+ errors.push("parts counts must add up to total");
6934
+ }
6935
+ return { valid: errors.length === 0, errors };
6936
+ }
6937
+ function assertAiwgFortemiChunkManifest(value) {
6938
+ const result = validateAiwgFortemiChunkManifest(value);
6939
+ if (!result.valid) {
6940
+ throw new Error("Invalid AIWG Fortemi chunk manifest:\n" + result.errors.join("\n"));
6941
+ }
6942
+ return value;
6943
+ }
6944
+ function validateAiwgFortemiChunkPart(value, partRef, manifest) {
6945
+ const errors = [];
6946
+ const data = value;
6947
+ if (data?.schema_version !== "aiwg.fortemi.index.chunk.v1") {
6948
+ errors.push("schema_version must be aiwg.fortemi.index.chunk.v1");
6949
+ }
6950
+ if (data?.manifest_schema_version !== "aiwg.fortemi.index.chunk-manifest.v1") {
6951
+ errors.push("manifest_schema_version must be aiwg.fortemi.index.chunk-manifest.v1");
6952
+ }
6953
+ if (!hasNonNegativeInteger(data?.offset)) errors.push("offset must be a non-negative integer");
6954
+ if (!Array.isArray(data?.items)) errors.push("items must be an array");
6955
+ if (partRef && hasNonNegativeInteger(data?.offset) && data.offset !== partRef.offset) {
6956
+ errors.push("offset must match manifest part offset " + partRef.offset);
6957
+ }
6958
+ if (partRef && Array.isArray(data?.items) && data.items.length !== partRef.count) {
6959
+ errors.push("items length must match manifest part count " + partRef.count);
6960
+ }
6961
+ if (Array.isArray(data?.items)) {
6962
+ const validation = validateAiwgFortemiIndexExport({
6963
+ schema_version: "aiwg.fortemi.index.export.v1",
6964
+ generated_at: manifest?.generated_at ?? "1970-01-01T00:00:00.000Z",
6965
+ source: manifest?.source ?? { repo: "chunk", privacy: "public" },
6966
+ items: data.items
6967
+ });
6968
+ errors.push(...validation.errors.map((error) => "items." + error));
6969
+ }
6970
+ return { valid: errors.length === 0, errors };
6971
+ }
6972
+ function assertAiwgFortemiChunkPart(value, partRef, manifest) {
6973
+ const result = validateAiwgFortemiChunkPart(value, partRef, manifest);
6974
+ if (!result.valid) {
6975
+ throw new Error("Invalid AIWG Fortemi chunk part:\n" + result.errors.join("\n"));
6976
+ }
6977
+ return value;
6978
+ }
6979
+ function createAiwgFetchChunkLoader(baseUrl) {
6980
+ return async (part) => {
6981
+ const href = baseUrl ? new URL(part.href, baseUrl).toString() : part.href;
6982
+ const response = await fetch(href);
6983
+ if (!response.ok) throw new Error("Failed to fetch AIWG index chunk " + href + ": " + response.status);
6984
+ return response.json();
6985
+ };
6986
+ }
6659
6987
  function getAiwgFortemiFacets(items) {
6660
6988
  const result = {};
6661
6989
  for (const item of items) {
@@ -6678,27 +7006,167 @@ function matchesFacetFilters(item, filters) {
6678
7006
  if (!filters) return true;
6679
7007
  return Object.entries(filters).every(([name, expected]) => includesAll(item.facets[name] ?? [], expected));
6680
7008
  }
6681
- function queryAiwgFortemiIndex(index, query = "", options = {}) {
6682
- const q = query.trim().toLowerCase();
6683
- const filtered = index.items.filter((item) => {
6684
- if (q) {
6685
- const haystack = [item.title, item.text, ...item.tags, ...item.concepts].join("\n").toLowerCase();
6686
- if (!haystack.includes(q)) return false;
6687
- }
7009
+ function queryMatches(item, q) {
7010
+ if (!q) return [];
7011
+ const matches = [];
7012
+ if (item.title.toLowerCase().includes(q)) matches.push({ field: "title", value: item.title });
7013
+ if (item.text.toLowerCase().includes(q)) matches.push({ field: "text", value: item.text });
7014
+ for (const tag of item.tags) {
7015
+ if (tag.toLowerCase().includes(q)) matches.push({ field: "tag", value: tag });
7016
+ }
7017
+ for (const concept of item.concepts) {
7018
+ if (concept.toLowerCase().includes(q)) matches.push({ field: "concept", value: concept });
7019
+ }
7020
+ return matches;
7021
+ }
7022
+ function rankMatches(matches, weights) {
7023
+ return matches.reduce((total, match) => total + weights[match.field], 0);
7024
+ }
7025
+ function clipSnippet(value, q, maxLength) {
7026
+ const normalizedLength = Math.max(20, maxLength);
7027
+ if (!value) return "";
7028
+ if (!q) return value.length > normalizedLength ? `${value.slice(0, normalizedLength).trimEnd()}...` : value;
7029
+ const lower = value.toLowerCase();
7030
+ const index = lower.indexOf(q);
7031
+ if (index < 0) return value.length > normalizedLength ? `${value.slice(0, normalizedLength).trimEnd()}...` : value;
7032
+ const context = Math.max(0, Math.floor((normalizedLength - q.length) / 2));
7033
+ const start = Math.max(0, index - context);
7034
+ const end = Math.min(value.length, start + normalizedLength);
7035
+ const prefix = start > 0 ? "..." : "";
7036
+ const suffix = end < value.length ? "..." : "";
7037
+ return `${prefix}${value.slice(start, end).trim()}${suffix}`;
7038
+ }
7039
+ function createSnippet(item, matches, q, maxLength) {
7040
+ const textMatch = matches.find((match) => match.field === "text");
7041
+ const titleMatch = matches.find((match) => match.field === "title");
7042
+ const firstMatch = textMatch ?? titleMatch ?? matches[0];
7043
+ return clipSnippet(firstMatch?.value ?? item.text, q, maxLength);
7044
+ }
7045
+ function createRankedEntries(items, q, options, ordinalBase = 0) {
7046
+ const weights = { ...DEFAULT_QUERY_WEIGHTS, ...options.weights };
7047
+ return items.map((item, ordinal) => ({ item, ordinal: ordinalBase + ordinal, matches: queryMatches(item, q) })).filter(({ item, matches }) => {
7048
+ if (q && matches.length === 0) return false;
6688
7049
  if (options.types && !options.types.includes(item.type)) return false;
6689
7050
  if (options.privacy && !options.privacy.includes(item.privacy.classification)) return false;
6690
7051
  if (!includesAll(item.tags, options.tags)) return false;
6691
7052
  if (!includesAll(item.concepts, options.concepts)) return false;
6692
7053
  if (!matchesFacetFilters(item, options.facets)) return false;
6693
- if (options.relationshipTargetId && !item.relationships.some((rel) => rel.target_id === options.relationshipTargetId)) return false;
7054
+ if (options.relationshipTargetId && !item.relationships.some((rel) => rel.target_id === options.relationshipTargetId)) {
7055
+ return false;
7056
+ }
6694
7057
  return true;
7058
+ }).map(({ item, ordinal, matches }) => ({
7059
+ item,
7060
+ ordinal,
7061
+ rank: rankMatches(matches, weights),
7062
+ matches
7063
+ }));
7064
+ }
7065
+ function sortRankedEntries(entries, rank) {
7066
+ return [...entries].sort((left, right) => {
7067
+ if (rank) return right.rank - left.rank || left.ordinal - right.ordinal;
7068
+ return left.ordinal - right.ordinal;
6695
7069
  });
7070
+ }
7071
+ function createQueryResultFromRankedEntries(entries, query, options) {
7072
+ const ranked = sortRankedEntries(entries, options.rank);
6696
7073
  const offset = options.offset ?? 0;
6697
- const limit = options.limit ?? filtered.length;
7074
+ const limit = options.limit ?? ranked.length;
7075
+ const page = ranked.slice(offset, offset + limit);
7076
+ const result = {
7077
+ items: page.map((entry) => entry.item),
7078
+ total: ranked.length,
7079
+ facets: getAiwgFortemiFacets(ranked.map((entry) => entry.item))
7080
+ };
7081
+ if (options.rank || options.snippets || options.includeMatches) {
7082
+ const snippetLength = options.snippetLength ?? 160;
7083
+ result.rankedItems = page.map((entry) => ({
7084
+ item: entry.item,
7085
+ rank: entry.rank,
7086
+ ...options.snippets ? { snippet: createSnippet(entry.item, entry.matches, query, snippetLength) } : {},
7087
+ ...options.includeMatches ? { matches: entry.matches } : {}
7088
+ }));
7089
+ }
7090
+ return result;
7091
+ }
7092
+ function queryAiwgFortemiIndex(index, query = "", options = {}) {
7093
+ const q = query.trim().toLowerCase();
7094
+ return createQueryResultFromRankedEntries(createRankedEntries(index.items, q, options), q, options);
7095
+ }
7096
+ function chunkPartCacheKey(part) {
7097
+ return `${part.offset}:${part.href}`;
7098
+ }
7099
+ function clampMaxCachedParts(value) {
7100
+ if (!hasPositiveInteger(value)) return 3;
7101
+ return value;
7102
+ }
7103
+ function isDirectChunkBrowse(query, options) {
7104
+ return query.trim() === "" && !options.rank && !options.snippets && !options.includeMatches && !options.types && !options.facets && !options.tags && !options.concepts && !options.privacy && !options.relationshipTargetId;
7105
+ }
7106
+ function getPartsForRange(manifest, offset, limit) {
7107
+ const end = offset + limit;
7108
+ return manifest.parts.filter((part) => part.count > 0 && part.offset < end && part.offset + part.count > offset);
7109
+ }
7110
+ async function loadChunkPart(runtime, part) {
7111
+ const key = chunkPartCacheKey(part);
7112
+ const cached = runtime.partCache.get(key);
7113
+ if (cached) {
7114
+ runtime.partCache.delete(key);
7115
+ runtime.partCache.set(key, cached);
7116
+ return { part: cached, fetched: false };
7117
+ }
7118
+ const parsed = assertAiwgFortemiChunkPart(await runtime.loader(part, runtime.manifest), part, runtime.manifest);
7119
+ runtime.partCache.set(key, parsed);
7120
+ while (runtime.partCache.size > runtime.maxCachedParts) {
7121
+ const oldest = runtime.partCache.keys().next().value;
7122
+ if (oldest === void 0) break;
7123
+ runtime.partCache.delete(oldest);
7124
+ }
7125
+ return { part: parsed, fetched: true };
7126
+ }
7127
+ async function queryChunkedAiwgFortemiIndex(runtime, query = "", options = {}) {
7128
+ const q = query.trim().toLowerCase();
7129
+ let scannedParts = 0;
7130
+ let fetchedParts = 0;
7131
+ if (isDirectChunkBrowse(query, options)) {
7132
+ const offset = options.offset ?? 0;
7133
+ const limit = options.limit ?? runtime.manifest.total;
7134
+ const parts = getPartsForRange(runtime.manifest, offset, limit);
7135
+ const items = [];
7136
+ for (const partRef of parts) {
7137
+ const loaded = await loadChunkPart(runtime, partRef);
7138
+ if (loaded.fetched) fetchedParts += 1;
7139
+ scannedParts += 1;
7140
+ options.onProgress?.({ phase: "part", done: scannedParts, total: parts.length, href: partRef.href });
7141
+ const start = Math.max(0, offset - partRef.offset);
7142
+ const end = Math.min(loaded.part.items.length, offset + limit - partRef.offset);
7143
+ items.push(...loaded.part.items.slice(start, end));
7144
+ }
7145
+ return {
7146
+ items,
7147
+ total: runtime.manifest.total,
7148
+ facets: runtime.manifest.facets ?? {},
7149
+ manifestTotal: runtime.manifest.total,
7150
+ scannedParts,
7151
+ fetchedParts,
7152
+ complete: true
7153
+ };
7154
+ }
7155
+ const entries = [];
7156
+ for (const partRef of runtime.manifest.parts) {
7157
+ const loaded = await loadChunkPart(runtime, partRef);
7158
+ if (loaded.fetched) fetchedParts += 1;
7159
+ scannedParts += 1;
7160
+ options.onProgress?.({ phase: "part", done: scannedParts, total: runtime.manifest.parts.length, href: partRef.href });
7161
+ entries.push(...createRankedEntries(loaded.part.items, q, options, partRef.offset));
7162
+ options.onProgress?.({ phase: "query", done: scannedParts, total: runtime.manifest.parts.length, href: partRef.href });
7163
+ }
6698
7164
  return {
6699
- items: filtered.slice(offset, offset + limit),
6700
- total: filtered.length,
6701
- facets: getAiwgFortemiFacets(filtered)
7165
+ ...createQueryResultFromRankedEntries(entries, q, options),
7166
+ manifestTotal: runtime.manifest.total,
7167
+ scannedParts,
7168
+ fetchedParts,
7169
+ complete: true
6702
7170
  };
6703
7171
  }
6704
7172
  function createAiwgReviewDecisionExport(source, decisions, generatedAt = (/* @__PURE__ */ new Date()).toISOString()) {
@@ -6709,6 +7177,137 @@ function createAiwgReviewDecisionExport(source, decisions, generatedAt = (/* @__
6709
7177
  decisions: [...decisions].sort((left, right) => left.item_id.localeCompare(right.item_id))
6710
7178
  };
6711
7179
  }
7180
+ function createAiwgIndexController(initialIndex) {
7181
+ let index = initialIndex ?? null;
7182
+ let chunked = null;
7183
+ let data = null;
7184
+ let error = null;
7185
+ let reviewDecisions = [];
7186
+ const listeners = /* @__PURE__ */ new Set();
7187
+ const snapshot = () => ({
7188
+ index,
7189
+ chunked: chunked ? {
7190
+ manifest: chunked.manifest,
7191
+ cachedParts: chunked.partCache.size,
7192
+ maxCachedParts: chunked.maxCachedParts
7193
+ } : null,
7194
+ data,
7195
+ error,
7196
+ reviewDecisions: [...reviewDecisions]
7197
+ });
7198
+ const notify = () => {
7199
+ const current = snapshot();
7200
+ for (const listener of listeners) listener(current);
7201
+ };
7202
+ const requireIndex = () => {
7203
+ if (!index) throw new Error("No AIWG index export loaded");
7204
+ return index;
7205
+ };
7206
+ return {
7207
+ loadIndex(value) {
7208
+ try {
7209
+ const parsed = assertAiwgFortemiIndexExport(value);
7210
+ index = parsed;
7211
+ chunked = null;
7212
+ data = null;
7213
+ reviewDecisions = [];
7214
+ error = null;
7215
+ notify();
7216
+ return parsed;
7217
+ } catch (err) {
7218
+ error = err instanceof Error ? err : new Error(String(err));
7219
+ notify();
7220
+ throw error;
7221
+ }
7222
+ },
7223
+ loadChunkedIndex(manifest, loader, options = {}) {
7224
+ try {
7225
+ const parsed = assertAiwgFortemiChunkManifest(manifest);
7226
+ index = null;
7227
+ chunked = {
7228
+ manifest: parsed,
7229
+ loader,
7230
+ maxCachedParts: clampMaxCachedParts(options.maxCachedParts),
7231
+ partCache: /* @__PURE__ */ new Map()
7232
+ };
7233
+ data = null;
7234
+ reviewDecisions = [];
7235
+ error = null;
7236
+ notify();
7237
+ return parsed;
7238
+ } catch (err) {
7239
+ error = err instanceof Error ? err : new Error(String(err));
7240
+ notify();
7241
+ throw error;
7242
+ }
7243
+ },
7244
+ getIndex() {
7245
+ return index;
7246
+ },
7247
+ getChunkedManifest() {
7248
+ return chunked?.manifest ?? null;
7249
+ },
7250
+ getSnapshot() {
7251
+ return snapshot();
7252
+ },
7253
+ query(query = "", options) {
7254
+ const result = queryAiwgFortemiIndex(requireIndex(), query, options);
7255
+ data = result;
7256
+ error = null;
7257
+ notify();
7258
+ return result;
7259
+ },
7260
+ async queryChunked(query = "", options) {
7261
+ if (!chunked) throw new Error("No AIWG chunked index manifest loaded");
7262
+ try {
7263
+ const result = await queryChunkedAiwgFortemiIndex(chunked, query, options);
7264
+ data = result;
7265
+ error = null;
7266
+ notify();
7267
+ return result;
7268
+ } catch (err) {
7269
+ error = err instanceof Error ? err : new Error(String(err));
7270
+ notify();
7271
+ throw error;
7272
+ }
7273
+ },
7274
+ clearChunkCache() {
7275
+ chunked?.partCache.clear();
7276
+ error = null;
7277
+ notify();
7278
+ },
7279
+ toCommunityGraph(options) {
7280
+ return aiwgFortemiIndexToCommunityGraph(requireIndex(), options);
7281
+ },
7282
+ setReviewDecision(input) {
7283
+ const decision = {
7284
+ ...input,
7285
+ updated_at: (/* @__PURE__ */ new Date()).toISOString()
7286
+ };
7287
+ reviewDecisions = [
7288
+ ...reviewDecisions.filter((item) => item.item_id !== decision.item_id),
7289
+ decision
7290
+ ].sort((left, right) => left.item_id.localeCompare(right.item_id));
7291
+ error = null;
7292
+ notify();
7293
+ return decision;
7294
+ },
7295
+ clearReviewDecision(itemId) {
7296
+ reviewDecisions = reviewDecisions.filter((item) => item.item_id !== itemId);
7297
+ error = null;
7298
+ notify();
7299
+ },
7300
+ createReviewDecisionExport(generatedAt) {
7301
+ return createAiwgReviewDecisionExport(requireIndex(), reviewDecisions, generatedAt);
7302
+ },
7303
+ subscribe(listener) {
7304
+ listeners.add(listener);
7305
+ return () => {
7306
+ listeners.delete(listener);
7307
+ };
7308
+ }
7309
+ };
7310
+ }
6712
7311
  function aiwgFortemiIndexToCommunityGraph(index, options = {}) {
6713
7312
  const ids = new Set(index.items.map((item) => item.id));
6714
7313
  const relationshipWeights = options.relationshipWeights ?? {};
@@ -6754,8 +7353,8 @@ function communityIdsFor(item, options) {
6754
7353
  }
6755
7354
 
6756
7355
  // src/index.ts
6757
- var VERSION = "2026.6.1";
7356
+ var VERSION = "2026.6.2";
6758
7357
 
6759
- export { ArchiveManager, AttachmentsRepository, CURRENT_SHARD_VERSION, CapabilityManager, CaptureKnowledgeInputSchema, CollectionsRepository, CommunitiesRepository, EmbeddingSetsRepository, FallbackRouter, FortemiToolManifest, GetNoteInputSchema, GraphRepository, JOB_CAPABILITIES, JOB_PRIORITIES, JobQueueWorker, LOCAL_ENDPOINTS, LinksRepository, ListNotesInputSchema, ManageArchiveInputSchema, ManageAttachmentsInputSchema, ManageCapabilitiesInputSchema, ManageCollectionsInputSchema, ManageLinksInputSchema, ManageNoteInputSchema, ManageTagsInputSchema, MemoryBlobStore, MigrationRunner, NotesRepository, OpenAICompatibleProvider, PGliteStorageBackend, PGliteStorageBackendFactory, PGliteWorkerClient, PGliteWorkerStorageBackend, PGliteWorkerStorageBackendFactory, ProviderRegistry, SHARD_FORMAT, SearchInputSchema, SearchRepository, SkosRepository, TagsRepository, TransactionProxy, TypedEventBus, VERSION, aiRevisionHandler, aiwgFortemiIndexToCommunityGraph, allMigrations, appendPluginScript, assertAiwgFortemiIndexExport, buildNoteConditions, buildPluginCsp, captureKnowledge, chunkText, classifyError, classifyModel, collectionFromShard, collectionToShard, computeHash, computeSri, conceptTaggingHandler, cosineSimilarity, createAiwgReviewDecisionExport, createBlobStore, createCspReportHandler, createFortemi, createLegacyProvider, createPGliteInstance, createRoutes, defaultStorageBackendFactory, detectCommunities, detectGpuCapabilities, detectInferenceCapabilities, discoverLocalProviders, embeddingFromShard, embeddingGenerationHandler, embeddingSetFromShard, embeddingSetMemberToShard, embeddingSetToShard, embeddingToShard, enqueueFullWorkflow, enqueueJob, enqueueNoteCreationJobs, estimateModelFit, estimateVramMB, estimateVramTier, exportShard, fetchPluginScript, fortemiManifest, generateId, getAiwgFortemiFacets, getEmbedFunction, getFortemiBridge, getFortemiSecretStore, getJobQueueStatus, getLlmFunction, getNote, hasFortemiSecureSecrets, importShard, isPluginScriptAllowed, linkFromShard, linkToShard, linkingHandler, listNotes, manageArchive, manageAttachments, manageCapabilities, manageCollections, manageLinks, manageNote, manageTags, matchRoute, noteFromShard, noteSkosTagToShard, noteToShard, packTarGz, parseCspReport, provenanceEdgeToShard, queryAiwgFortemiIndex, registerLlmCapability, registerSemanticCapability, registerServiceWorker, searchTool, selectLlmModel, setEmbedFunction, setLlmFunction, sha256Hex, skosConceptToShard, skosRelationToShard, skosSchemeToShard, suggestTags, tagsFromShard, tagsToShard, titleGenerationHandler, unpackTarGz, unregisterLlmCapability, unregisterSemanticCapability, validateAiwgFortemiIndexExport, validateChecksums, verifySri };
7358
+ export { ArchiveManager, AttachmentsRepository, CURRENT_SHARD_VERSION, CapabilityManager, CaptureKnowledgeInputSchema, CollectionsRepository, CommunitiesRepository, EmbeddingSetsRepository, FallbackRouter, FortemiToolManifest, GetNoteInputSchema, GraphRepository, JOB_CAPABILITIES, JOB_PRIORITIES, JobQueueWorker, LOCAL_ENDPOINTS, LinksRepository, ListNotesInputSchema, ManageArchiveInputSchema, ManageAttachmentsInputSchema, ManageCapabilitiesInputSchema, ManageCollectionsInputSchema, ManageLinksInputSchema, ManageNoteInputSchema, ManageTagsInputSchema, MemoryBlobStore, MigrationRunner, NotesRepository, OpenAICompatibleProvider, PGliteStorageBackend, PGliteStorageBackendFactory, PGliteWorkerClient, PGliteWorkerStorageBackend, PGliteWorkerStorageBackendFactory, ProviderRegistry, SHARD_FORMAT, SearchInputSchema, SearchRepository, SkosRepository, TagsRepository, TransactionProxy, TypedEventBus, VERSION, aiRevisionHandler, aiwgFortemiIndexToCommunityGraph, allMigrations, appendPluginScript, assertAiwgFortemiChunkManifest, assertAiwgFortemiChunkPart, assertAiwgFortemiIndexExport, buildNoteConditions, buildPluginCsp, captureKnowledge, chunkText, classifyError, classifyModel, collectionFromShard, collectionToShard, computeHash, computeSri, conceptTaggingHandler, cosineSimilarity, createAiwgFetchChunkLoader, createAiwgIndexController, createAiwgReviewDecisionExport, createBlobStore, createCspReportHandler, createFortemi, createLegacyProvider, createPGliteInstance, createRoutes, defaultStorageBackendFactory, detectCommunities, detectGpuCapabilities, detectInferenceCapabilities, discoverLocalProviders, embeddingFromShard, embeddingGenerationHandler, embeddingSetFromShard, embeddingSetMemberToShard, embeddingSetToShard, embeddingToShard, enqueueFullWorkflow, enqueueJob, enqueueNoteCreationJobs, estimateModelFit, estimateVramMB, estimateVramTier, exportShard, fetchPluginScript, fortemiManifest, generateId, getAiwgFortemiFacets, getEmbedFunction, getFortemiBridge, getFortemiSecretStore, getJobQueueStatus, getLlmFunction, getNote, hasFortemiSecureSecrets, importShard, isPluginScriptAllowed, linkFromShard, linkToShard, linkingHandler, listNotes, manageArchive, manageAttachments, manageCapabilities, manageCollections, manageLinks, manageNote, manageTags, matchRoute, noteFromShard, noteSkosTagToShard, noteToShard, packTarGz, parseCspReport, provenanceEdgeToShard, queryAiwgFortemiIndex, registerLlmCapability, registerSemanticCapability, registerServiceWorker, searchTool, selectLlmModel, setEmbedFunction, setLlmFunction, sha256Hex, skosConceptToShard, skosRelationToShard, skosSchemeToShard, suggestTags, tagsFromShard, tagsToShard, titleGenerationHandler, unpackTarGz, unregisterLlmCapability, unregisterSemanticCapability, validateAiwgFortemiChunkManifest, validateAiwgFortemiChunkPart, validateAiwgFortemiIndexExport, validateChecksums, verifySri };
6760
7359
  //# sourceMappingURL=index.js.map
6761
7360
  //# sourceMappingURL=index.js.map