@fortemi/core 2026.7.1 → 2026.7.2

package/README.md

@@ -227,17 +227,31 @@ scan projection.
 Static semantic search uses an optional sidecar contract:
 
 ```ts
-import { queryAiwgHybridIndex } from '@fortemi/core/aiwg-index'
+import { buildAiwgStaticEmbeddingSet, queryAiwgHybridIndex } from '@fortemi/core/aiwg-index'
 
 const embeddingSet = await fetch('/search/aiwg-index/embeddings.json').then((res) => res.json())
 const queryEmbedding = await hostEmbed('workspace health check')
 const semanticResults = queryAiwgHybridIndex(index, embeddingSet, 'health check', queryEmbedding)
+
+// Node/CLI generation path. Supply any Node-safe model backend; @fortemi/core
+// only writes the shared embedding-set graph format and does not require DOM or WebGL.
+const cliEmbeddingSet = await buildAiwgStaticEmbeddingSet(index, {
+  id: 'aiwg-cli-embeddings',
+  backend: {
+    model: 'local-node-embedder',
+    dimensions: 384,
+    embed: async (input) => nodeEmbed(input),
+  },
+})
 ```
 
 `aiwg.fortemi.embedding.set.v1` records the model, dimensions, granularity
 (`title-summary`, `body`, `chunked-body`, or a project value), source record IDs,
 and per-vector input hashes. Hosts provide query embeddings; `@fortemi/core` does
-not add a model runtime dependency for static AIWG search.
+not add a model runtime dependency for static AIWG search. The builder uses the
+same record text projection as static search, including attachment
+`extracted_text`, and records only attachment metadata references, never raw blob
+bytes.
 
 ## What You Get
 

package/dist/aiwg-index.d.ts

@@ -94,6 +94,17 @@ interface AiwgFortemiRecordEmbedding {
     source_path?: string;
     metadata?: Record<string, unknown>;
 }
+interface AiwgFortemiAttachmentReference {
+    id: string;
+    path: string;
+    mime: string | null;
+    checksum: string;
+    bytes: number;
+}
+interface AiwgFortemiBinarySource {
+    extracted_text: string;
+    attachment: AiwgFortemiAttachmentReference;
+}
 interface AiwgFortemiRecord {
     schema_version: AiwgFortemiRecordSchemaVersion;
     id: string;
@@ -108,6 +119,7 @@ interface AiwgFortemiRecord {
     provenance: AiwgFortemiProvenance[];
     search?: AiwgFortemiSearchProjection;
     chunks?: AiwgFortemiChunk[];
+    binary_sources?: AiwgFortemiBinarySource[];
     embeddings?: AiwgFortemiRecordEmbedding[];
     compatibility?: Record<string, unknown>;
     /** Optional rich SKOS metadata for static consumers that need labels/definitions without opening a shard. */
@@ -321,6 +333,20 @@ interface AiwgStaticEmbeddingSet {
     input_hash_algorithm?: string;
     embeddings: AiwgStaticEmbeddingRecord[];
 }
+interface AiwgHeadlessEmbeddingBackend {
+    model: string;
+    dimensions: number;
+    embed(input: string, record: AiwgFortemiRecord): number[] | Promise<number[]>;
+}
+interface BuildAiwgStaticEmbeddingSetOptions {
+    id: string;
+    backend: AiwgHeadlessEmbeddingBackend;
+    records?: AiwgFortemiRecord[];
+    generatedAt?: string | Date;
+    granularity?: AiwgStaticEmbeddingSet['granularity'];
+    metric?: AiwgStaticEmbeddingSet['metric'];
+    textForRecord?: (record: AiwgFortemiRecord) => string;
+}
 interface AiwgStaticSemanticQueryOptions {
     limit?: number;
     offset?: number;
@@ -390,6 +416,7 @@ declare function encodeAiwgDetailId(id: string, encoding?: AiwgDetailIdEncoding)
 declare function aiwgDetailHrefForId(detail: AiwgFortemiChunkDetailRef, id: string): string;
 declare function createAiwgFetchDetailLoader(baseUrl?: string | URL): AiwgChunkedIndexDetailLoader;
 declare function getAiwgFortemiFacets(items: AiwgFortemiRecord[]): Record<string, Record<string, number>>;
+declare function buildAiwgStaticEmbeddingSet(index: AiwgFortemiIndexExport, options: BuildAiwgStaticEmbeddingSetOptions): Promise<AiwgStaticEmbeddingSet>;
 interface AiwgChunkedIndexBuildOptions {
     partSize?: number;
     projection?: Array<keyof AiwgFortemiRecord>;
@@ -434,4 +461,4 @@ declare function aiwgFortemiIndexToCommunityGraph(index: AiwgFortemiIndexExport,
     }[];
 };
 
-export { AIWG_SCAN_REQUIRED_FIELDS, type AiwgChunkedIndexBuildOptions, type AiwgChunkedIndexBuildResult, type AiwgChunkedIndexDetailLoader, type AiwgChunkedIndexLoadOptions, type AiwgChunkedIndexLoader, type AiwgChunkedIndexProgress, type AiwgChunkedIndexProgressPhase, type AiwgChunkedIndexQueryOptions, type AiwgChunkedIndexQueryResult, type AiwgChunkedIndexValidationResult, type AiwgDetailIdEncoding, type AiwgFortemiChunk, type AiwgFortemiChunkDetailRef, type AiwgFortemiChunkManifest, type AiwgFortemiChunkPart, type AiwgFortemiChunkPartRef, type AiwgFortemiIndexExport, type AiwgFortemiIndexExportSchemaVersion, type AiwgFortemiKnownRecordType, type AiwgFortemiProjectedRecord, type AiwgFortemiProvenance, type AiwgFortemiProvenanceEvent, type AiwgFortemiRecord, type AiwgFortemiRecordEmbedding, type AiwgFortemiRecordSchemaVersion, type AiwgFortemiRecordSource, type AiwgFortemiRecordType, type AiwgFortemiRelationship, type AiwgFortemiRelationshipDirection, type AiwgFortemiSearchProjection, type AiwgFortemiSkosConcept, type AiwgFortemiSkosRelation, type AiwgFortemiSkosRelationType, type AiwgIndexController, type AiwgIndexControllerListener, type AiwgIndexControllerSnapshot, type AiwgIndexGraphOptions, type AiwgIndexQueryMatch, type AiwgIndexQueryOptions, type AiwgIndexQueryRankedItem, type AiwgIndexQueryResult, type AiwgIndexQueryWeights, type AiwgIndexValidationResult, type AiwgPrivacyClassification, type AiwgProvenanceConfidence, type AiwgRelationshipDirection, type AiwgRelationshipEdgeSummary, type AiwgRelationshipNodeSummary, type AiwgRelationshipQueryOptions, type AiwgRelationshipSetOperation, type AiwgRelationshipSetOptions, type AiwgRelationshipSetResult, type AiwgRelationshipTraversalOptions, type AiwgRelationshipTraversalResult, type AiwgReviewAction, type AiwgReviewDecision, type AiwgReviewDecisionExport, type AiwgReviewInput, type AiwgStaticDuplicatePair, type AiwgStaticEmbeddingRecord, type AiwgStaticEmbeddingSet, type AiwgStaticHybridQueryOptions, type AiwgStaticSemanticQueryOptions, type AiwgStaticSemanticResult, aiwgDetailHrefForId, aiwgFortemiIndexToCommunityGraph, assertAiwgFortemiChunkManifest, assertAiwgFortemiChunkPart, assertAiwgFortemiIndexExport, assertAiwgStaticEmbeddingSet, buildAiwgChunkedIndex, createAiwgFetchChunkLoader, createAiwgFetchDetailLoader, createAiwgIndexController, createAiwgReviewDecisionExport, encodeAiwgDetailId, findAiwgStaticDuplicatePairs, getAiwgFortemiFacets, queryAiwgFortemiIndex, queryAiwgHybridIndex, queryAiwgSemanticIndex, validateAiwgFortemiChunkManifest, validateAiwgFortemiChunkPart, validateAiwgFortemiIndexExport, validateAiwgStaticEmbeddingSet };
+export { AIWG_SCAN_REQUIRED_FIELDS, type AiwgChunkedIndexBuildOptions, type AiwgChunkedIndexBuildResult, type AiwgChunkedIndexDetailLoader, type AiwgChunkedIndexLoadOptions, type AiwgChunkedIndexLoader, type AiwgChunkedIndexProgress, type AiwgChunkedIndexProgressPhase, type AiwgChunkedIndexQueryOptions, type AiwgChunkedIndexQueryResult, type AiwgChunkedIndexValidationResult, type AiwgDetailIdEncoding, type AiwgFortemiAttachmentReference, type AiwgFortemiBinarySource, type AiwgFortemiChunk, type AiwgFortemiChunkDetailRef, type AiwgFortemiChunkManifest, type AiwgFortemiChunkPart, type AiwgFortemiChunkPartRef, type AiwgFortemiIndexExport, type AiwgFortemiIndexExportSchemaVersion, type AiwgFortemiKnownRecordType, type AiwgFortemiProjectedRecord, type AiwgFortemiProvenance, type AiwgFortemiProvenanceEvent, type AiwgFortemiRecord, type AiwgFortemiRecordEmbedding, type AiwgFortemiRecordSchemaVersion, type AiwgFortemiRecordSource, type AiwgFortemiRecordType, type AiwgFortemiRelationship, type AiwgFortemiRelationshipDirection, type AiwgFortemiSearchProjection, type AiwgFortemiSkosConcept, type AiwgFortemiSkosRelation, type AiwgFortemiSkosRelationType, type AiwgHeadlessEmbeddingBackend, type AiwgIndexController, type AiwgIndexControllerListener, type AiwgIndexControllerSnapshot, type AiwgIndexGraphOptions, type AiwgIndexQueryMatch, type AiwgIndexQueryOptions, type AiwgIndexQueryRankedItem, type AiwgIndexQueryResult, type AiwgIndexQueryWeights, type AiwgIndexValidationResult, type AiwgPrivacyClassification, type AiwgProvenanceConfidence, type AiwgRelationshipDirection, type AiwgRelationshipEdgeSummary, type AiwgRelationshipNodeSummary, type AiwgRelationshipQueryOptions, type AiwgRelationshipSetOperation, type AiwgRelationshipSetOptions, type AiwgRelationshipSetResult, type AiwgRelationshipTraversalOptions, type AiwgRelationshipTraversalResult, type AiwgReviewAction, type AiwgReviewDecision, type AiwgReviewDecisionExport, type AiwgReviewInput, type AiwgStaticDuplicatePair, type AiwgStaticEmbeddingRecord, type AiwgStaticEmbeddingSet, type AiwgStaticHybridQueryOptions, type AiwgStaticSemanticQueryOptions, type AiwgStaticSemanticResult, type BuildAiwgStaticEmbeddingSetOptions, aiwgDetailHrefForId, aiwgFortemiIndexToCommunityGraph, assertAiwgFortemiChunkManifest, assertAiwgFortemiChunkPart, assertAiwgFortemiIndexExport, assertAiwgStaticEmbeddingSet, buildAiwgChunkedIndex, buildAiwgStaticEmbeddingSet, createAiwgFetchChunkLoader, createAiwgFetchDetailLoader, createAiwgIndexController, createAiwgReviewDecisionExport, encodeAiwgDetailId, findAiwgStaticDuplicatePairs, getAiwgFortemiFacets, queryAiwgFortemiIndex, queryAiwgHybridIndex, queryAiwgSemanticIndex, validateAiwgFortemiChunkManifest, validateAiwgFortemiChunkPart, validateAiwgFortemiIndexExport, validateAiwgStaticEmbeddingSet };

package/dist/aiwg-index.js

@@ -1,3 +1,12 @@
+import { sha256 } from '@noble/hashes/sha256';
+import { bytesToHex } from '@noble/hashes/utils';
+
+// src/hash.ts
+function computeHash(data) {
+  const digest = sha256(data);
+  return `sha256:${bytesToHex(digest)}`;
+}
+
 // src/aiwg-index.ts
 var AIWG_SCAN_REQUIRED_FIELDS = [
   "schema_version",
@@ -392,7 +401,55 @@ function recordTitle(item) {
   return item.title ?? item.search?.title ?? item.search?.name ?? item.id;
 }
 function recordText(item) {
-  return item.text ?? item.search?.body ?? item.search?.summary ?? item.chunks?.map((chunk) => chunk.text ?? chunk.body ?? chunk.summary ?? "").filter(Boolean).join("\n") ?? "";
+  const base = item.text ?? item.search?.body ?? item.search?.summary ?? item.chunks?.map((chunk) => chunk.text ?? chunk.body ?? chunk.summary ?? "").filter(Boolean).join("\n") ?? "";
+  const extractedText = "binary_sources" in item ? item.binary_sources?.map((source) => source.extracted_text).filter(Boolean).join("\n") ?? "" : "";
+  return [base, extractedText].filter(Boolean).join("\n");
+}
+function defaultEmbeddingInput(record, granularity) {
+  const title = recordTitle(record);
+  const text = recordText(record);
+  if (granularity === "title-summary") {
+    const extractedText = record.binary_sources?.map((source) => source.extracted_text).filter(Boolean).join("\n") ?? "";
+    return [title, record.search?.summary ?? "", extractedText].filter(Boolean).join("\n");
+  }
+  return [title, text].filter(Boolean).join("\n");
+}
+function generatedAtString(value) {
+  if (value instanceof Date) return value.toISOString();
+  return value ?? (/* @__PURE__ */ new Date()).toISOString();
+}
+async function buildAiwgStaticEmbeddingSet(index, options) {
+  assertAiwgFortemiIndexExport(index);
+  const granularity = options.granularity ?? "body";
+  const records = options.records ?? index.items;
+  const embeddings = [];
+  for (const record of records) {
+    const input = options.textForRecord?.(record) ?? defaultEmbeddingInput(record, granularity);
+    const embedding = await options.backend.embed(input, record);
+    if (embedding.length !== options.backend.dimensions) {
+      throw new Error(`Embedding for ${record.id} has ${embedding.length} dimensions; expected ${options.backend.dimensions}`);
+    }
+    embeddings.push({
+      record_id: record.id,
+      embedding,
+      granularity,
+      input_hash: computeHash(new TextEncoder().encode(input)),
+      source_path: record.source.path
+    });
+  }
+  const embeddingSet = {
+    schema_version: "aiwg.fortemi.embedding.set.v1",
+    id: options.id,
+    model: options.backend.model,
+    dimensions: options.backend.dimensions,
+    generated_at: generatedAtString(options.generatedAt),
+    granularity,
+    ...options.metric ? { metric: options.metric } : {},
+    input_hash_algorithm: "sha256",
+    embeddings
+  };
+  assertAiwgStaticEmbeddingSet(embeddingSet);
+  return embeddingSet;
 }
 function recordSearchValues(item) {
   const search = item.search;
@@ -1285,6 +1342,6 @@ function communityIdsFor(item, options) {
   return [`type:${item.type}`];
 }
 
-export { AIWG_SCAN_REQUIRED_FIELDS, aiwgDetailHrefForId, aiwgFortemiIndexToCommunityGraph, assertAiwgFortemiChunkManifest, assertAiwgFortemiChunkPart, assertAiwgFortemiIndexExport, assertAiwgStaticEmbeddingSet, buildAiwgChunkedIndex, createAiwgFetchChunkLoader, createAiwgFetchDetailLoader, createAiwgIndexController, createAiwgReviewDecisionExport, encodeAiwgDetailId, findAiwgStaticDuplicatePairs, getAiwgFortemiFacets, queryAiwgFortemiIndex, queryAiwgHybridIndex, queryAiwgSemanticIndex, validateAiwgFortemiChunkManifest, validateAiwgFortemiChunkPart, validateAiwgFortemiIndexExport, validateAiwgStaticEmbeddingSet };
+export { AIWG_SCAN_REQUIRED_FIELDS, aiwgDetailHrefForId, aiwgFortemiIndexToCommunityGraph, assertAiwgFortemiChunkManifest, assertAiwgFortemiChunkPart, assertAiwgFortemiIndexExport, assertAiwgStaticEmbeddingSet, buildAiwgChunkedIndex, buildAiwgStaticEmbeddingSet, createAiwgFetchChunkLoader, createAiwgFetchDetailLoader, createAiwgIndexController, createAiwgReviewDecisionExport, encodeAiwgDetailId, findAiwgStaticDuplicatePairs, getAiwgFortemiFacets, queryAiwgFortemiIndex, queryAiwgHybridIndex, queryAiwgSemanticIndex, validateAiwgFortemiChunkManifest, validateAiwgFortemiChunkPart, validateAiwgFortemiIndexExport, validateAiwgStaticEmbeddingSet };
 //# sourceMappingURL=aiwg-index.js.map
 //# sourceMappingURL=aiwg-index.js.map