@fortemi/core 2026.7.1 → 2026.7.3

package/README.md

@@ -227,17 +227,31 @@ scan projection.
 Static semantic search uses an optional sidecar contract:
 
 ```ts
-import { queryAiwgHybridIndex } from '@fortemi/core/aiwg-index'
+import { buildAiwgStaticEmbeddingSet, queryAiwgHybridIndex } from '@fortemi/core/aiwg-index'
 
 const embeddingSet = await fetch('/search/aiwg-index/embeddings.json').then((res) => res.json())
 const queryEmbedding = await hostEmbed('workspace health check')
 const semanticResults = queryAiwgHybridIndex(index, embeddingSet, 'health check', queryEmbedding)
+
+// Node/CLI generation path. Supply any Node-safe model backend; @fortemi/core
+// only writes the shared embedding-set graph format and does not require DOM or WebGL.
+const cliEmbeddingSet = await buildAiwgStaticEmbeddingSet(index, {
+  id: 'aiwg-cli-embeddings',
+  backend: {
+    model: 'local-node-embedder',
+    dimensions: 384,
+    embed: async (input) => nodeEmbed(input),
+  },
+})
 ```
 
 `aiwg.fortemi.embedding.set.v1` records the model, dimensions, granularity
 (`title-summary`, `body`, `chunked-body`, or a project value), source record IDs,
 and per-vector input hashes. Hosts provide query embeddings; `@fortemi/core` does
-not add a model runtime dependency for static AIWG search.
+not add a model runtime dependency for static AIWG search. The builder uses the
+same record text projection as static search, including attachment
+`extracted_text`, and records only attachment metadata references, never raw blob
+bytes.
 
 ## What You Get
 

package/dist/aiwg-index.d.ts

@@ -1,4 +1,4 @@
-type AiwgFortemiKnownRecordType = 'crm.contact' | 'crm.organization' | 'crm.event' | 'crm.interaction' | 'aiwg.artifact' | 'docs.page';
+type AiwgFortemiKnownRecordType = 'crm.contact' | 'crm.organization' | 'crm.event' | 'crm.interaction' | 'aiwg.artifact' | 'aiwg.kb.page';
 type AiwgFortemiRecordType = AiwgFortemiKnownRecordType | `aiwg.${string}` | `research.${string}` | `docs.${string}` | string;
 type AiwgFortemiRecordSchemaVersion = 'aiwg.fortemi.index.record.v1' | 'aiwg.fortemi.index.record.v2';
 type AiwgFortemiIndexExportSchemaVersion = 'aiwg.fortemi.index.export.v1' | 'aiwg.fortemi.index.export.v2';
@@ -94,6 +94,17 @@ interface AiwgFortemiRecordEmbedding {
     source_path?: string;
     metadata?: Record<string, unknown>;
 }
+interface AiwgFortemiAttachmentReference {
+    id: string;
+    path: string;
+    mime: string | null;
+    checksum: string;
+    bytes: number;
+}
+interface AiwgFortemiBinarySource {
+    extracted_text: string;
+    attachment: AiwgFortemiAttachmentReference;
+}
 interface AiwgFortemiRecord {
     schema_version: AiwgFortemiRecordSchemaVersion;
     id: string;
@@ -108,6 +119,7 @@ interface AiwgFortemiRecord {
     provenance: AiwgFortemiProvenance[];
     search?: AiwgFortemiSearchProjection;
     chunks?: AiwgFortemiChunk[];
+    binary_sources?: AiwgFortemiBinarySource[];
     embeddings?: AiwgFortemiRecordEmbedding[];
     compatibility?: Record<string, unknown>;
     /** Optional rich SKOS metadata for static consumers that need labels/definitions without opening a shard. */
@@ -129,10 +141,7 @@ interface AiwgFortemiIndexExport {
     source: {
         repo: string;
         privacy: AiwgPrivacyClassification;
-        origin?: string;
-        generated?: boolean;
-        checksum?: string;
-        updated_at?: string;
+        graph?: Record<string, unknown>;
     };
     items: AiwgFortemiRecord[];
     compatibility?: Record<string, unknown>;
@@ -153,6 +162,7 @@ interface AiwgFortemiChunkManifest {
     schema_version: 'aiwg.fortemi.index.chunk-manifest.v1';
     generated_at: string;
     source: AiwgFortemiIndexExport['source'];
+    source_export_schema_version?: AiwgFortemiIndexExportSchemaVersion;
     total: number;
     part_size: number;
     facets?: Record<string, Record<string, number>>;
@@ -321,6 +331,30 @@ interface AiwgStaticEmbeddingSet {
     input_hash_algorithm?: string;
     embeddings: AiwgStaticEmbeddingRecord[];
 }
+interface AiwgHeadlessEmbeddingBackend {
+    model: string;
+    dimensions: number;
+    embed(input: string, record: AiwgFortemiRecord): number[] | Promise<number[]>;
+}
+interface AiwgPrivacyFilterOptions {
+    /** Include records classified `private` (default false). */
+    includePrivate?: boolean;
+    /** Include records flagged `pii` (default false). */
+    includePii?: boolean;
+}
+/** Drop `private`/`pii` records unless explicitly opted in (SEC6, default-safe). */
+declare function filterAiwgRecordsByPrivacy(records: AiwgFortemiRecord[], options?: AiwgPrivacyFilterOptions): AiwgFortemiRecord[];
+interface BuildAiwgStaticEmbeddingSetOptions {
+    id: string;
+    backend: AiwgHeadlessEmbeddingBackend;
+    records?: AiwgFortemiRecord[];
+    generatedAt?: string | Date;
+    granularity?: AiwgStaticEmbeddingSet['granularity'];
+    metric?: AiwgStaticEmbeddingSet['metric'];
+    textForRecord?: (record: AiwgFortemiRecord) => string;
+    /** Privacy filtering (SEC6). Default-safe: excludes `private`/`pii` records. */
+    privacy?: AiwgPrivacyFilterOptions;
+}
 interface AiwgStaticSemanticQueryOptions {
     limit?: number;
     offset?: number;
@@ -385,17 +419,21 @@ declare function validateAiwgFortemiChunkManifest(value: unknown): AiwgChunkedIn
 declare function assertAiwgFortemiChunkManifest(value: unknown): AiwgFortemiChunkManifest;
 declare function validateAiwgFortemiChunkPart(value: unknown, partRef?: AiwgFortemiChunkPartRef, manifest?: AiwgFortemiChunkManifest): AiwgChunkedIndexValidationResult;
 declare function assertAiwgFortemiChunkPart(value: unknown, partRef?: AiwgFortemiChunkPartRef, manifest?: AiwgFortemiChunkManifest): AiwgFortemiChunkPart;
+declare function resolveAiwgFetchUrl(href: string, baseUrl?: string | URL): string;
 declare function createAiwgFetchChunkLoader(baseUrl?: string | URL): AiwgChunkedIndexLoader;
 declare function encodeAiwgDetailId(id: string, encoding?: AiwgDetailIdEncoding): string;
 declare function aiwgDetailHrefForId(detail: AiwgFortemiChunkDetailRef, id: string): string;
 declare function createAiwgFetchDetailLoader(baseUrl?: string | URL): AiwgChunkedIndexDetailLoader;
 declare function getAiwgFortemiFacets(items: AiwgFortemiRecord[]): Record<string, Record<string, number>>;
+declare function buildAiwgStaticEmbeddingSet(index: AiwgFortemiIndexExport, options: BuildAiwgStaticEmbeddingSetOptions): Promise<AiwgStaticEmbeddingSet>;
 interface AiwgChunkedIndexBuildOptions {
     partSize?: number;
     projection?: Array<keyof AiwgFortemiRecord>;
     detailHref?: string;
     idEncoding?: AiwgDetailIdEncoding;
     generatedAt?: string;
+    /** Privacy filtering (SEC6). Default-safe: excludes `private`/`pii` records. */
+    privacy?: AiwgPrivacyFilterOptions;
 }
 interface AiwgChunkedIndexBuildResult {
     manifest: AiwgFortemiChunkManifest;
@@ -415,7 +453,10 @@ declare function validateAiwgStaticEmbeddingSet(value: unknown): AiwgChunkedInde
 declare function assertAiwgStaticEmbeddingSet(value: unknown): AiwgStaticEmbeddingSet;
 declare function queryAiwgSemanticIndex(index: AiwgFortemiIndexExport, embeddingSet: AiwgStaticEmbeddingSet, queryEmbedding: number[], options?: AiwgStaticSemanticQueryOptions): AiwgStaticSemanticResult[];
 declare function queryAiwgHybridIndex(index: AiwgFortemiIndexExport, embeddingSet: AiwgStaticEmbeddingSet, query: string, queryEmbedding: number[], options?: AiwgStaticHybridQueryOptions): AiwgStaticSemanticResult[];
-declare function findAiwgStaticDuplicatePairs(index: AiwgFortemiIndexExport, embeddingSet: AiwgStaticEmbeddingSet, threshold?: number): AiwgStaticDuplicatePair[];
+declare const DEFAULT_AIWG_DUPLICATE_SCAN_MAX_EMBEDDINGS = 5000;
+declare function findAiwgStaticDuplicatePairs(index: AiwgFortemiIndexExport, embeddingSet: AiwgStaticEmbeddingSet, threshold?: number, options?: {
+    maxEmbeddings?: number;
+}): AiwgStaticDuplicatePair[];
 declare function createAiwgReviewDecisionExport(source: Pick<AiwgFortemiIndexExport, 'schema_version'>, decisions: AiwgReviewDecision[], generatedAt?: string): AiwgReviewDecisionExport;
 declare function createAiwgIndexController(initialIndex?: AiwgFortemiIndexExport): AiwgIndexController;
 declare function aiwgFortemiIndexToCommunityGraph(index: AiwgFortemiIndexExport, options?: AiwgIndexGraphOptions): {
@@ -434,4 +475,4 @@ declare function aiwgFortemiIndexToCommunityGraph(index: AiwgFortemiIndexExport,
     }[];
 };
 
-export { AIWG_SCAN_REQUIRED_FIELDS, type AiwgChunkedIndexBuildOptions, type AiwgChunkedIndexBuildResult, type AiwgChunkedIndexDetailLoader, type AiwgChunkedIndexLoadOptions, type AiwgChunkedIndexLoader, type AiwgChunkedIndexProgress, type AiwgChunkedIndexProgressPhase, type AiwgChunkedIndexQueryOptions, type AiwgChunkedIndexQueryResult, type AiwgChunkedIndexValidationResult, type AiwgDetailIdEncoding, type AiwgFortemiChunk, type AiwgFortemiChunkDetailRef, type AiwgFortemiChunkManifest, type AiwgFortemiChunkPart, type AiwgFortemiChunkPartRef, type AiwgFortemiIndexExport, type AiwgFortemiIndexExportSchemaVersion, type AiwgFortemiKnownRecordType, type AiwgFortemiProjectedRecord, type AiwgFortemiProvenance, type AiwgFortemiProvenanceEvent, type AiwgFortemiRecord, type AiwgFortemiRecordEmbedding, type AiwgFortemiRecordSchemaVersion, type AiwgFortemiRecordSource, type AiwgFortemiRecordType, type AiwgFortemiRelationship, type AiwgFortemiRelationshipDirection, type AiwgFortemiSearchProjection, type AiwgFortemiSkosConcept, type AiwgFortemiSkosRelation, type AiwgFortemiSkosRelationType, type AiwgIndexController, type AiwgIndexControllerListener, type AiwgIndexControllerSnapshot, type AiwgIndexGraphOptions, type AiwgIndexQueryMatch, type AiwgIndexQueryOptions, type AiwgIndexQueryRankedItem, type AiwgIndexQueryResult, type AiwgIndexQueryWeights, type AiwgIndexValidationResult, type AiwgPrivacyClassification, type AiwgProvenanceConfidence, type AiwgRelationshipDirection, type AiwgRelationshipEdgeSummary, type AiwgRelationshipNodeSummary, type AiwgRelationshipQueryOptions, type AiwgRelationshipSetOperation, type AiwgRelationshipSetOptions, type AiwgRelationshipSetResult, type AiwgRelationshipTraversalOptions, type AiwgRelationshipTraversalResult, type AiwgReviewAction, type AiwgReviewDecision, type AiwgReviewDecisionExport, type AiwgReviewInput, type AiwgStaticDuplicatePair, type AiwgStaticEmbeddingRecord, type AiwgStaticEmbeddingSet, type AiwgStaticHybridQueryOptions, type AiwgStaticSemanticQueryOptions, type AiwgStaticSemanticResult, aiwgDetailHrefForId, aiwgFortemiIndexToCommunityGraph, assertAiwgFortemiChunkManifest, assertAiwgFortemiChunkPart, assertAiwgFortemiIndexExport, assertAiwgStaticEmbeddingSet, buildAiwgChunkedIndex, createAiwgFetchChunkLoader, createAiwgFetchDetailLoader, createAiwgIndexController, createAiwgReviewDecisionExport, encodeAiwgDetailId, findAiwgStaticDuplicatePairs, getAiwgFortemiFacets, queryAiwgFortemiIndex, queryAiwgHybridIndex, queryAiwgSemanticIndex, validateAiwgFortemiChunkManifest, validateAiwgFortemiChunkPart, validateAiwgFortemiIndexExport, validateAiwgStaticEmbeddingSet };
+export { AIWG_SCAN_REQUIRED_FIELDS, type AiwgChunkedIndexBuildOptions, type AiwgChunkedIndexBuildResult, type AiwgChunkedIndexDetailLoader, type AiwgChunkedIndexLoadOptions, type AiwgChunkedIndexLoader, type AiwgChunkedIndexProgress, type AiwgChunkedIndexProgressPhase, type AiwgChunkedIndexQueryOptions, type AiwgChunkedIndexQueryResult, type AiwgChunkedIndexValidationResult, type AiwgDetailIdEncoding, type AiwgFortemiAttachmentReference, type AiwgFortemiBinarySource, type AiwgFortemiChunk, type AiwgFortemiChunkDetailRef, type AiwgFortemiChunkManifest, type AiwgFortemiChunkPart, type AiwgFortemiChunkPartRef, type AiwgFortemiIndexExport, type AiwgFortemiIndexExportSchemaVersion, type AiwgFortemiKnownRecordType, type AiwgFortemiProjectedRecord, type AiwgFortemiProvenance, type AiwgFortemiProvenanceEvent, type AiwgFortemiRecord, type AiwgFortemiRecordEmbedding, type AiwgFortemiRecordSchemaVersion, type AiwgFortemiRecordSource, type AiwgFortemiRecordType, type AiwgFortemiRelationship, type AiwgFortemiRelationshipDirection, type AiwgFortemiSearchProjection, type AiwgFortemiSkosConcept, type AiwgFortemiSkosRelation, type AiwgFortemiSkosRelationType, type AiwgHeadlessEmbeddingBackend, type AiwgIndexController, type AiwgIndexControllerListener, type AiwgIndexControllerSnapshot, type AiwgIndexGraphOptions, type AiwgIndexQueryMatch, type AiwgIndexQueryOptions, type AiwgIndexQueryRankedItem, type AiwgIndexQueryResult, type AiwgIndexQueryWeights, type AiwgIndexValidationResult, type AiwgPrivacyClassification, type AiwgPrivacyFilterOptions, type AiwgProvenanceConfidence, type AiwgRelationshipDirection, type AiwgRelationshipEdgeSummary, type AiwgRelationshipNodeSummary, type AiwgRelationshipQueryOptions, type AiwgRelationshipSetOperation, type AiwgRelationshipSetOptions, type AiwgRelationshipSetResult, type AiwgRelationshipTraversalOptions, type AiwgRelationshipTraversalResult, type AiwgReviewAction, type AiwgReviewDecision, type AiwgReviewDecisionExport, type AiwgReviewInput, type AiwgStaticDuplicatePair, type AiwgStaticEmbeddingRecord, type AiwgStaticEmbeddingSet, type AiwgStaticHybridQueryOptions, type AiwgStaticSemanticQueryOptions, type AiwgStaticSemanticResult, type BuildAiwgStaticEmbeddingSetOptions, DEFAULT_AIWG_DUPLICATE_SCAN_MAX_EMBEDDINGS, aiwgDetailHrefForId, aiwgFortemiIndexToCommunityGraph, assertAiwgFortemiChunkManifest, assertAiwgFortemiChunkPart, assertAiwgFortemiIndexExport, assertAiwgStaticEmbeddingSet, buildAiwgChunkedIndex, buildAiwgStaticEmbeddingSet, createAiwgFetchChunkLoader, createAiwgFetchDetailLoader, createAiwgIndexController, createAiwgReviewDecisionExport, encodeAiwgDetailId, filterAiwgRecordsByPrivacy, findAiwgStaticDuplicatePairs, getAiwgFortemiFacets, queryAiwgFortemiIndex, queryAiwgHybridIndex, queryAiwgSemanticIndex, resolveAiwgFetchUrl, validateAiwgFortemiChunkManifest, validateAiwgFortemiChunkPart, validateAiwgFortemiIndexExport, validateAiwgStaticEmbeddingSet };

package/dist/aiwg-index.js

@@ -1,3 +1,12 @@
+import { sha256 } from '@noble/hashes/sha256';
+import { bytesToHex } from '@noble/hashes/utils';
+
+// src/hash.ts
+function computeHash(data) {
+  const digest = sha256(data);
+  return `sha256:${bytesToHex(digest)}`;
+}
+
 // src/aiwg-index.ts
 var AIWG_SCAN_REQUIRED_FIELDS = [
   "schema_version",
@@ -10,6 +19,16 @@ var AIWG_SCAN_REQUIRED_FIELDS = [
   "concepts",
   "privacy"
 ];
+function isPrivacyExcluded(record, options) {
+  const privacy = record.privacy;
+  if (!privacy) return false;
+  if (privacy.classification === "private" && !options?.includePrivate) return true;
+  if (privacy.pii && !options?.includePii) return true;
+  return false;
+}
+function filterAiwgRecordsByPrivacy(records, options) {
+  return records.filter((record) => !isPrivacyExcluded(record, options));
+}
 var REQUIRED_RECORD_FIELDS = [
   "schema_version",
   "id",
@@ -36,8 +55,12 @@ function hasString(value) {
   return typeof value === "string" && value.length > 0;
 }
 function pushFacet(counts, name, value) {
-  counts[name] ??= {};
-  counts[name][value] = (counts[name][value] ?? 0) + 1;
+  let bucket = counts[name];
+  if (bucket === void 0) {
+    bucket = /* @__PURE__ */ Object.create(null);
+    counts[name] = bucket;
+  }
+  bucket[value] = (bucket[value] ?? 0) + 1;
 }
 function hasNonNegativeInteger(value) {
   return Number.isInteger(value) && typeof value === "number" && value >= 0;
@@ -159,6 +182,57 @@ function validateOptionalRichMetadata(item, index, errors) {
     errors.push("items[" + index + "].compatibility must be an object");
   }
 }
+function isPrivacyClassification(value) {
+  return value === "private" || value === "sanitized" || value === "public";
+}
+function isProvenanceConfidence(value) {
+  return value === "source" || value === "candidate" || value === "reviewed" || value === "rejected";
+}
+function validateProvenanceItems(item, index, errors) {
+  if (!Array.isArray(item.provenance)) return;
+  for (const [provIndex, prov] of item.provenance.entries()) {
+    const at = "items[" + index + "].provenance[" + provIndex + "]";
+    if (!isPlainRecord(prov)) {
+      errors.push(at + " must be an object");
+      continue;
+    }
+    if (!hasString(prov.field)) errors.push(at + ".field is required");
+    if (!hasString(prov.source)) errors.push(at + ".source is required");
+    if (!hasString(prov.path)) errors.push(at + ".path is required");
+    if (!isProvenanceConfidence(prov.confidence)) errors.push(at + ".confidence must be one of source, candidate, reviewed, rejected");
+    if (!isPrivacyClassification(prov.privacy)) errors.push(at + ".privacy must be one of private, sanitized, public");
+  }
+}
+var V2_ONLY_RECORD_FIELDS = ["search", "chunks", "embeddings", "skos_concepts", "skos_relations", "compatibility"];
+var V2_ONLY_SOURCE_FIELDS = ["origin", "generated", "checksum", "updated_at"];
+var V2_ONLY_RELATIONSHIP_FIELDS = ["target_path", "direction", "metadata"];
+function forbidV2FieldsOnV1Record(item, index, errors) {
+  if (item.schema_version !== "aiwg.fortemi.index.record.v1") return;
+  const at = "items[" + index + "]";
+  const bag = item;
+  const v2msg = " is a v2-only field and must be absent on a record.v1 record";
+  for (const field of V2_ONLY_RECORD_FIELDS) {
+    if (bag[field] !== void 0) errors.push(at + "." + field + v2msg);
+  }
+  if (isPlainRecord(item.source)) {
+    const src = item.source;
+    for (const field of V2_ONLY_SOURCE_FIELDS) {
+      if (src[field] !== void 0) errors.push(at + ".source." + field + v2msg);
+    }
+  }
+  if (isPlainRecord(item.privacy) && item.privacy.locality !== void 0) {
+    errors.push(at + ".privacy.locality" + v2msg);
+  }
+  if (Array.isArray(item.relationships)) {
+    for (const [relIndex, rel] of item.relationships.entries()) {
+      if (!isPlainRecord(rel)) continue;
+      const relBag = rel;
+      for (const field of V2_ONLY_RELATIONSHIP_FIELDS) {
+        if (relBag[field] !== void 0) errors.push(at + ".relationships[" + relIndex + "]." + field + v2msg);
+      }
+    }
+  }
+}
 function validateAiwgFortemiIndexExport(value) {
   const errors = [];
   const counts = {};
@@ -169,10 +243,21 @@ function validateAiwgFortemiIndexExport(value) {
   if (!hasString(data?.generated_at)) errors.push("generated_at is required");
   if (!hasString(data?.source?.repo)) errors.push("source.repo is required");
   if (!hasString(data?.source?.privacy)) errors.push("source.privacy is required");
+  else if (!isPrivacyClassification(data?.source?.privacy)) {
+    errors.push("source.privacy must be one of private, sanitized, public");
+  }
   if (!Array.isArray(data?.items)) errors.push("items must be an array");
   if (data.compatibility !== void 0 && !isPlainRecord(data.compatibility)) {
     errors.push("compatibility must be an object");
   }
+  if (data?.schema_version === "aiwg.fortemi.index.export.v1") {
+    if (isPlainRecord(data.source) && data.source.graph !== void 0) {
+      errors.push("source.graph is a v2-only field and must be absent on an export.v1 export");
+    }
+    if (data.compatibility !== void 0) {
+      errors.push("compatibility is a v2-only field and must be absent on an export.v1 export");
+    }
+  }
   const ids = /* @__PURE__ */ new Set();
   let previousId = "";
   for (const [index, item] of (data.items ?? []).entries()) {
@@ -207,8 +292,12 @@ function validateAiwgFortemiIndexExport(value) {
       errors.push("items[" + index + "].provenance must be a non-empty array");
     }
     validateOptionalRichMetadata(item, index, errors);
+    validateProvenanceItems(item, index, errors);
+    forbidV2FieldsOnV1Record(item, index, errors);
     if (!item.privacy || typeof item.privacy.pii !== "boolean" || !hasString(item.privacy.classification)) {
       errors.push("items[" + index + "].privacy requires classification and pii");
+    } else if (!isPrivacyClassification(item.privacy.classification)) {
+      errors.push("items[" + index + "].privacy.classification must be one of private, sanitized, public");
     }
   }
   return { valid: errors.length === 0, errors, counts };
@@ -229,6 +318,9 @@ function validateAiwgFortemiChunkManifest(value) {
   if (!hasString(data?.generated_at)) errors.push("generated_at is required");
   if (!hasString(data?.source?.repo)) errors.push("source.repo is required");
   if (!hasString(data?.source?.privacy)) errors.push("source.privacy is required");
+  if (data?.source_export_schema_version !== void 0 && !isSupportedIndexSchemaVersion(data.source_export_schema_version)) {
+    errors.push("source_export_schema_version must be aiwg.fortemi.index.export.v1 or aiwg.fortemi.index.export.v2 when present");
+  }
   if (!hasNonNegativeInteger(data?.total)) errors.push("total must be a non-negative integer");
   if (!hasPositiveInteger(data?.part_size)) errors.push("part_size must be a positive integer");
   if (data.facets !== void 0 && !isFacetCounts(data.facets)) {
@@ -347,9 +439,35 @@ function assertAiwgFortemiChunkPart(value, partRef, manifest) {
   }
   return value;
 }
+var ALLOWED_AIWG_FETCH_SCHEMES = /* @__PURE__ */ new Set(["http:", "https:", "blob:", "data:"]);
+function tryParseUrl(value) {
+  try {
+    return new URL(value);
+  } catch {
+    return null;
+  }
+}
+function resolveAiwgFetchUrl(href, baseUrl) {
+  if (baseUrl === void 0) {
+    const absolute = tryParseUrl(href);
+    if (absolute && !ALLOWED_AIWG_FETCH_SCHEMES.has(absolute.protocol)) {
+      throw new Error("Refusing AIWG index fetch with disallowed scheme: " + absolute.protocol);
+    }
+    return href;
+  }
+  const base = new URL(baseUrl);
+  const resolved = new URL(href, base);
+  if (!ALLOWED_AIWG_FETCH_SCHEMES.has(resolved.protocol)) {
+    throw new Error("Refusing AIWG index fetch with disallowed scheme: " + resolved.protocol);
+  }
+  if (resolved.origin !== base.origin) {
+    throw new Error("Refusing cross-origin AIWG index fetch: " + resolved.origin + " != " + base.origin);
+  }
+  return resolved.toString();
+}
 function createAiwgFetchChunkLoader(baseUrl) {
   return async (part) => {
-    const href = baseUrl ? new URL(part.href, baseUrl).toString() : part.href;
+    const href = resolveAiwgFetchUrl(part.href, baseUrl);
     const response = await fetch(href);
     if (!response.ok) throw new Error("Failed to fetch AIWG index chunk " + href + ": " + response.status);
     return response.json();
@@ -369,14 +487,14 @@ function createAiwgFetchDetailLoader(baseUrl) {
   return async (id, manifest) => {
     if (!manifest.detail?.href) throw new Error("Manifest has no detail.href for record resolution");
     const relative = aiwgDetailHrefForId(manifest.detail, id);
-    const href = baseUrl ? new URL(relative, baseUrl).toString() : relative;
+    const href = resolveAiwgFetchUrl(relative, baseUrl);
     const response = await fetch(href);
     if (!response.ok) throw new Error("Failed to fetch AIWG index detail " + href + ": " + response.status);
     return response.json();
   };
 }
 function getAiwgFortemiFacets(items) {
-  const result = {};
+  const result = /* @__PURE__ */ Object.create(null);
   for (const item of items) {
     pushFacet(result, "type", item.type);
     pushFacet(result, "privacy", item.privacy.classification);
@@ -392,7 +510,55 @@ function recordTitle(item) {
   return item.title ?? item.search?.title ?? item.search?.name ?? item.id;
 }
 function recordText(item) {
-  return item.text ?? item.search?.body ?? item.search?.summary ?? item.chunks?.map((chunk) => chunk.text ?? chunk.body ?? chunk.summary ?? "").filter(Boolean).join("\n") ?? "";
+  const base = item.text ?? item.search?.body ?? item.search?.summary ?? item.chunks?.map((chunk) => chunk.text ?? chunk.body ?? chunk.summary ?? "").filter(Boolean).join("\n") ?? "";
+  const extractedText = "binary_sources" in item ? item.binary_sources?.map((source) => source.extracted_text).filter(Boolean).join("\n") ?? "" : "";
+  return [base, extractedText].filter(Boolean).join("\n");
+}
+function defaultEmbeddingInput(record, granularity) {
+  const title = recordTitle(record);
+  const text = recordText(record);
+  if (granularity === "title-summary") {
+    const extractedText = record.binary_sources?.map((source) => source.extracted_text).filter(Boolean).join("\n") ?? "";
+    return [title, record.search?.summary ?? "", extractedText].filter(Boolean).join("\n");
+  }
+  return [title, text].filter(Boolean).join("\n");
+}
+function generatedAtString(value) {
+  if (value instanceof Date) return value.toISOString();
+  return value ?? (/* @__PURE__ */ new Date()).toISOString();
+}
+async function buildAiwgStaticEmbeddingSet(index, options) {
+  assertAiwgFortemiIndexExport(index);
+  const granularity = options.granularity ?? "body";
+  const records = filterAiwgRecordsByPrivacy(options.records ?? index.items, options.privacy);
+  const embeddings = [];
+  for (const record of records) {
+    const input = options.textForRecord?.(record) ?? defaultEmbeddingInput(record, granularity);
+    const embedding = await options.backend.embed(input, record);
+    if (embedding.length !== options.backend.dimensions) {
+      throw new Error(`Embedding for ${record.id} has ${embedding.length} dimensions; expected ${options.backend.dimensions}`);
+    }
+    embeddings.push({
+      record_id: record.id,
+      embedding,
+      granularity,
+      input_hash: computeHash(new TextEncoder().encode(input)),
+      source_path: record.source.path
+    });
+  }
+  const embeddingSet = {
+    schema_version: "aiwg.fortemi.embedding.set.v1",
+    id: options.id,
+    model: options.backend.model,
+    dimensions: options.backend.dimensions,
+    generated_at: generatedAtString(options.generatedAt),
+    granularity,
+    ...options.metric ? { metric: options.metric } : {},
+    input_hash_algorithm: "sha256",
+    embeddings
+  };
+  assertAiwgStaticEmbeddingSet(embeddingSet);
+  return embeddingSet;
 }
 function recordSearchValues(item) {
   const search = item.search;
@@ -425,7 +591,7 @@ function buildAiwgChunkedIndex(index, options = {}) {
   const projection = options.projection;
   const idEncoding = options.idEncoding ?? "base64url";
   const detailHref = options.detailHref ?? "detail/{id}.json";
-  const items = index.items;
+  const items = filterAiwgRecordsByPrivacy(index.items, options.privacy);
   const pad = (value) => String(value).padStart(4, "0");
   const project = (record) => {
     if (!projection) return record;
@@ -453,6 +619,7 @@ function buildAiwgChunkedIndex(index, options = {}) {
     schema_version: "aiwg.fortemi.index.chunk-manifest.v1",
     generated_at: options.generatedAt ?? index.generated_at,
     source: index.source,
+    source_export_schema_version: index.schema_version,
     total: items.length,
     part_size: partSize,
     facets: getAiwgFortemiFacets(items),
@@ -763,8 +930,15 @@ function queryAiwgHybridIndex(index, embeddingSet, query, queryEmbedding, option
     };
   }).filter((result) => result !== null && result.score >= (options.minScore ?? -1)).sort((left, right) => right.score - left.score || left.item.id.localeCompare(right.item.id)).slice(offset, offset + limit);
 }
-function findAiwgStaticDuplicatePairs(index, embeddingSet, threshold = 0.9) {
+var DEFAULT_AIWG_DUPLICATE_SCAN_MAX_EMBEDDINGS = 5e3;
+function findAiwgStaticDuplicatePairs(index, embeddingSet, threshold = 0.9, options) {
   assertAiwgStaticEmbeddingSet(embeddingSet);
+  const maxEmbeddings = options?.maxEmbeddings ?? DEFAULT_AIWG_DUPLICATE_SCAN_MAX_EMBEDDINGS;
+  if (embeddingSet.embeddings.length > maxEmbeddings) {
+    throw new Error(
+      "Embedding set too large for duplicate scan: " + embeddingSet.embeddings.length + " > " + maxEmbeddings + " (raise options.maxEmbeddings to override for trusted input)"
+    );
+  }
   const byId = new Map(index.items.map((item) => [item.id, item]));
   const pairs = [];
   for (let leftIndex = 0; leftIndex < embeddingSet.embeddings.length; leftIndex += 1) {
@@ -1229,7 +1403,7 @@ function createAiwgIndexController(initialIndex) {
       notify();
     },
     createReviewDecisionExport(generatedAt) {
-      const source = index ?? (chunked ? { schema_version: "aiwg.fortemi.index.export.v1" } : null);
+      const source = index ?? (chunked ? { schema_version: chunked.manifest.source_export_schema_version ?? "aiwg.fortemi.index.export.v1" } : null);
       if (!source) throw new Error("No AIWG index export or chunked manifest loaded");
       return createAiwgReviewDecisionExport(source, reviewDecisions, generatedAt);
     },
@@ -1285,6 +1459,6 @@ function communityIdsFor(item, options) {
   return [`type:${item.type}`];
 }
 
-export { AIWG_SCAN_REQUIRED_FIELDS, aiwgDetailHrefForId, aiwgFortemiIndexToCommunityGraph, assertAiwgFortemiChunkManifest, assertAiwgFortemiChunkPart, assertAiwgFortemiIndexExport, assertAiwgStaticEmbeddingSet, buildAiwgChunkedIndex, createAiwgFetchChunkLoader, createAiwgFetchDetailLoader, createAiwgIndexController, createAiwgReviewDecisionExport, encodeAiwgDetailId, findAiwgStaticDuplicatePairs, getAiwgFortemiFacets, queryAiwgFortemiIndex, queryAiwgHybridIndex, queryAiwgSemanticIndex, validateAiwgFortemiChunkManifest, validateAiwgFortemiChunkPart, validateAiwgFortemiIndexExport, validateAiwgStaticEmbeddingSet };
+export { AIWG_SCAN_REQUIRED_FIELDS, DEFAULT_AIWG_DUPLICATE_SCAN_MAX_EMBEDDINGS, aiwgDetailHrefForId, aiwgFortemiIndexToCommunityGraph, assertAiwgFortemiChunkManifest, assertAiwgFortemiChunkPart, assertAiwgFortemiIndexExport, assertAiwgStaticEmbeddingSet, buildAiwgChunkedIndex, buildAiwgStaticEmbeddingSet, createAiwgFetchChunkLoader, createAiwgFetchDetailLoader, createAiwgIndexController, createAiwgReviewDecisionExport, encodeAiwgDetailId, filterAiwgRecordsByPrivacy, findAiwgStaticDuplicatePairs, getAiwgFortemiFacets, queryAiwgFortemiIndex, queryAiwgHybridIndex, queryAiwgSemanticIndex, resolveAiwgFetchUrl, validateAiwgFortemiChunkManifest, validateAiwgFortemiChunkPart, validateAiwgFortemiIndexExport, validateAiwgStaticEmbeddingSet };
 //# sourceMappingURL=aiwg-index.js.map
 //# sourceMappingURL=aiwg-index.js.map