@fortemi/core 2026.6.0 → 2026.6.2

package/README.md

@@ -38,7 +38,7 @@ Most browser note and knowledge apps choose between a thin IndexedDB wrapper, a
 | Queryable knowledge | SQL-backed repositories for notes, links, tags, collections, SKOS concepts, jobs, and search |
 | Retrieval quality | Full-text search, pgvector-backed semantic search, hybrid ranking, snippets, facets, and filters |
 | AI-ready workflows | Optional embeddings, local LLM capability discovery, job provenance, and fallback routing |
-| Portable archives | Knowledge Shard tar.gz import/export with checksums and JSON format parity |
+| Portable archives | Knowledge Shard tar.gz import/export with set-scoped exports, chunked imports, checksums, and JSON format parity |
 | Agent integration | Manifest-backed tools and direct helper functions for bridge adapters and automation |
 | UI freedom | A headless package you can use from React, another framework, a browser extension, or a custom host |
 
@@ -102,6 +102,62 @@ const results = await search.search('hello')
 await registerServiceWorker()
 ```
 
+## Static AIWG Index Search
+
+Static documentation hosts can import the AIWG index helpers without pulling in
+PGlite, workers, shards, or browser storage:
+
+```ts
+import { createAiwgIndexController } from '@fortemi/core/aiwg-index'
+
+const controller = createAiwgIndexController()
+controller.loadIndex(await fetch('/aiwg-index.json').then((res) => res.json()))
+
+const results = controller.query('deployment', {
+  types: ['docs.page'],
+  rank: true,
+  snippets: true,
+  limit: 10,
+})
+```
+
+Use this subpath for Pagenary-style command palettes, static docs search, and
+vanilla JavaScript review surfaces. The top-level `@fortemi/core` export remains
+available for full archive/runtime integrations.
+
+Large static indexes can use the chunked browser path instead of downloading one
+full `aiwg.fortemi.index.export.v1` file. Host a manifest plus deterministic part
+files:
+
+```text
+/search/aiwg-index/manifest.json
+/search/aiwg-index/part-0000.json
+/search/aiwg-index/part-0001.json
+```
+
+```ts
+import {
+  createAiwgFetchChunkLoader,
+  createAiwgIndexController,
+} from '@fortemi/core/aiwg-index'
+
+const controller = createAiwgIndexController()
+const manifest = await fetch('/search/aiwg-index/manifest.json').then((res) => res.json())
+
+controller.loadChunkedIndex(
+  manifest,
+  createAiwgFetchChunkLoader('/search/aiwg-index/'),
+  { maxCachedParts: 3 },
+)
+
+const page = await controller.queryChunked('', { offset: 100, limit: 25 })
+```
+
+Unfiltered browse requests fetch only the part files intersecting the requested
+`offset` and `limit`. Filtered or ranked searches scan part files to compute exact
+results, but the controller keeps only a bounded part cache and never sets a
+materialized full export in `getIndex()`.
+
 ## What You Get
 
 | Surface | Description |
@@ -111,7 +167,7 @@ await registerServiceWorker()
 | Event bus | Typed subscriptions for note, job, archive, and capability events |
 | Capability system | Embeddings, local LLM, GPU detection, local-provider discovery, fallback routing |
 | Job queue | Server-compatible background workflow for revisions, titles, embeddings, concepts, and links |
-| Knowledge Shards | Tar.gz import/export with checksums and JSON format parity |
+| Knowledge Shards | Tar.gz import/export with checksums, progress callbacks, yielding imports, set-scoped embedding exports, and JSON format parity |
 | Service-worker helpers | Route registration primitives for standalone browser integration |
 
 ## Search and Knowledge Model
@@ -144,11 +200,11 @@ Use the manifest when a host needs to advertise Fortemi operations to an agent r
 | `idb` | IndexedDB-backed PGlite data directory | Firefox and broad compatibility |
 | `memory` | In-memory database | Tests, demos, restricted browser contexts |
 
-Data stays in the selected browser storage mode unless your application explicitly exports it, imports it, or wires external providers. Optional AI capabilities are opt-in and can be routed to local WASM, local provider servers, or host-provided integrations depending on your product requirements.
+Data stays in the selected browser storage mode unless your application explicitly exports it, imports it, or wires external providers. Optional AI capabilities are opt-in and can be routed to local WASM, local provider servers, OpenAI-compatible APIs, or host-provided integrations depending on your product requirements. API keys should live only in browser or machine secure storage; if secure storage is unavailable, do not persist them.
 
 ## React Bindings
 
-For React applications, install `@fortemi/react`. It wraps `@fortemi/core` with `FortemiProvider`, context access, and 21 hooks.
+For React applications, install `@fortemi/react`. It wraps `@fortemi/core` with `FortemiProvider`, context access, worker-mode PGlite, graph visualization primitives, and the React hook surface.
 
 ```bash
 pnpm add @fortemi/react @fortemi/core react

package/dist/aiwg-index.d.ts

@@ -0,0 +1,212 @@
+type AiwgFortemiRecordType = 'crm.contact' | 'crm.organization' | 'crm.event' | 'crm.interaction' | 'aiwg.artifact' | 'docs.page';
+type AiwgPrivacyClassification = 'private' | 'sanitized' | 'public';
+type AiwgProvenanceConfidence = 'source' | 'candidate' | 'reviewed' | 'rejected';
+type AiwgReviewAction = 'accept' | 'reject' | 'defer';
+interface AiwgFortemiRecordSource {
+    path: string;
+    repo_relative_path: string;
+    locator: string;
+}
+interface AiwgFortemiRelationship {
+    type: string;
+    target_id: string;
+    source_path?: string;
+}
+interface AiwgFortemiProvenance {
+    field: string;
+    source: string;
+    path: string;
+    confidence: AiwgProvenanceConfidence;
+    privacy: AiwgPrivacyClassification;
+}
+interface AiwgFortemiRecord {
+    schema_version: 'aiwg.fortemi.index.record.v1';
+    id: string;
+    type: AiwgFortemiRecordType;
+    source: AiwgFortemiRecordSource;
+    title: string;
+    text: string;
+    facets: Record<string, string[]>;
+    tags: string[];
+    concepts: string[];
+    relationships: AiwgFortemiRelationship[];
+    provenance: AiwgFortemiProvenance[];
+    privacy: {
+        classification: AiwgPrivacyClassification;
+        pii: boolean;
+    };
+    updated_at: string;
+}
+interface AiwgFortemiIndexExport {
+    schema_version: 'aiwg.fortemi.index.export.v1';
+    generated_at: string;
+    source: {
+        repo: string;
+        privacy: AiwgPrivacyClassification;
+    };
+    items: AiwgFortemiRecord[];
+}
+interface AiwgFortemiChunkPartRef {
+    href: string;
+    offset: number;
+    count: number;
+}
+interface AiwgFortemiChunkManifest {
+    schema_version: 'aiwg.fortemi.index.chunk-manifest.v1';
+    generated_at: string;
+    source: AiwgFortemiIndexExport['source'];
+    total: number;
+    part_size: number;
+    facets?: Record<string, Record<string, number>>;
+    parts: AiwgFortemiChunkPartRef[];
+}
+interface AiwgFortemiChunkPart {
+    schema_version: 'aiwg.fortemi.index.chunk.v1';
+    manifest_schema_version: 'aiwg.fortemi.index.chunk-manifest.v1';
+    offset: number;
+    items: AiwgFortemiRecord[];
+}
+interface AiwgIndexValidationResult {
+    valid: boolean;
+    errors: string[];
+    counts: Partial<Record<AiwgFortemiRecordType, number>>;
+}
+interface AiwgChunkedIndexValidationResult {
+    valid: boolean;
+    errors: string[];
+}
+interface AiwgIndexQueryOptions {
+    types?: AiwgFortemiRecordType[];
+    facets?: Record<string, string[]>;
+    tags?: string[];
+    concepts?: string[];
+    privacy?: AiwgPrivacyClassification[];
+    relationshipTargetId?: string;
+    limit?: number;
+    offset?: number;
+    rank?: boolean;
+    snippets?: boolean;
+    snippetLength?: number;
+    weights?: Partial<AiwgIndexQueryWeights>;
+    includeMatches?: boolean;
+}
+interface AiwgIndexQueryWeights {
+    title: number;
+    text: number;
+    tag: number;
+    concept: number;
+}
+interface AiwgIndexQueryMatch {
+    field: 'title' | 'text' | 'tag' | 'concept';
+    value: string;
+}
+interface AiwgIndexQueryRankedItem {
+    item: AiwgFortemiRecord;
+    rank: number;
+    snippet?: string;
+    matches?: AiwgIndexQueryMatch[];
+}
+interface AiwgIndexQueryResult {
+    items: AiwgFortemiRecord[];
+    total: number;
+    facets: Record<string, Record<string, number>>;
+    rankedItems?: AiwgIndexQueryRankedItem[];
+}
+type AiwgChunkedIndexLoader = (part: AiwgFortemiChunkPartRef, manifest: AiwgFortemiChunkManifest) => Promise<unknown>;
+interface AiwgChunkedIndexLoadOptions {
+    maxCachedParts?: number;
+}
+type AiwgChunkedIndexProgressPhase = 'part' | 'query';
+interface AiwgChunkedIndexProgress {
+    phase: AiwgChunkedIndexProgressPhase;
+    done: number;
+    total: number;
+    href?: string;
+}
+interface AiwgChunkedIndexQueryOptions extends AiwgIndexQueryOptions {
+    onProgress?: (progress: AiwgChunkedIndexProgress) => void;
+}
+interface AiwgChunkedIndexQueryResult extends AiwgIndexQueryResult {
+    manifestTotal: number;
+    scannedParts: number;
+    fetchedParts: number;
+    complete: boolean;
+}
+interface AiwgReviewDecision {
+    item_id: string;
+    action: AiwgReviewAction;
+    reason?: string;
+    updated_at: string;
+}
+interface AiwgReviewDecisionExport {
+    schema_version: 'aiwg.fortemi.review-decisions.v1';
+    generated_at: string;
+    source_export_schema_version: string;
+    decisions: AiwgReviewDecision[];
+}
+interface AiwgIndexGraphOptions {
+    communityFacet?: string;
+    communityTagPrefix?: string;
+    relationshipWeights?: Record<string, number>;
+    includeDanglingRelationships?: boolean;
+}
+interface AiwgReviewInput {
+    item_id: string;
+    action: AiwgReviewAction;
+    reason?: string;
+}
+interface AiwgIndexControllerSnapshot {
+    index: AiwgFortemiIndexExport | null;
+    chunked: {
+        manifest: AiwgFortemiChunkManifest;
+        cachedParts: number;
+        maxCachedParts: number;
+    } | null;
+    data: AiwgIndexQueryResult | null;
+    error: Error | null;
+    reviewDecisions: AiwgReviewDecision[];
+}
+type AiwgIndexControllerListener = (snapshot: AiwgIndexControllerSnapshot) => void;
+interface AiwgIndexController {
+    loadIndex(value: unknown): AiwgFortemiIndexExport;
+    loadChunkedIndex(manifest: unknown, loader: AiwgChunkedIndexLoader, options?: AiwgChunkedIndexLoadOptions): AiwgFortemiChunkManifest;
+    getIndex(): AiwgFortemiIndexExport | null;
+    getChunkedManifest(): AiwgFortemiChunkManifest | null;
+    getSnapshot(): AiwgIndexControllerSnapshot;
+    query(query?: string, options?: AiwgIndexQueryOptions): AiwgIndexQueryResult;
+    queryChunked(query?: string, options?: AiwgChunkedIndexQueryOptions): Promise<AiwgChunkedIndexQueryResult>;
+    clearChunkCache(): void;
+    toCommunityGraph(options?: AiwgIndexGraphOptions): ReturnType<typeof aiwgFortemiIndexToCommunityGraph>;
+    setReviewDecision(input: AiwgReviewInput): AiwgReviewDecision;
+    clearReviewDecision(itemId: string): void;
+    createReviewDecisionExport(generatedAt?: string): AiwgReviewDecisionExport;
+    subscribe(listener: AiwgIndexControllerListener): () => void;
+}
+declare function validateAiwgFortemiIndexExport(value: unknown): AiwgIndexValidationResult;
+declare function assertAiwgFortemiIndexExport(value: unknown): AiwgFortemiIndexExport;
+declare function validateAiwgFortemiChunkManifest(value: unknown): AiwgChunkedIndexValidationResult;
+declare function assertAiwgFortemiChunkManifest(value: unknown): AiwgFortemiChunkManifest;
+declare function validateAiwgFortemiChunkPart(value: unknown, partRef?: AiwgFortemiChunkPartRef, manifest?: AiwgFortemiChunkManifest): AiwgChunkedIndexValidationResult;
+declare function assertAiwgFortemiChunkPart(value: unknown, partRef?: AiwgFortemiChunkPartRef, manifest?: AiwgFortemiChunkManifest): AiwgFortemiChunkPart;
+declare function createAiwgFetchChunkLoader(baseUrl?: string | URL): AiwgChunkedIndexLoader;
+declare function getAiwgFortemiFacets(items: AiwgFortemiRecord[]): Record<string, Record<string, number>>;
+declare function queryAiwgFortemiIndex(index: AiwgFortemiIndexExport, query?: string, options?: AiwgIndexQueryOptions): AiwgIndexQueryResult;
+declare function createAiwgReviewDecisionExport(source: AiwgFortemiIndexExport, decisions: AiwgReviewDecision[], generatedAt?: string): AiwgReviewDecisionExport;
+declare function createAiwgIndexController(initialIndex?: AiwgFortemiIndexExport): AiwgIndexController;
+declare function aiwgFortemiIndexToCommunityGraph(index: AiwgFortemiIndexExport, options?: AiwgIndexGraphOptions): {
+    nodes: {
+        id: string;
+    }[];
+    edges: {
+        source: string;
+        target: string;
+        kind: string;
+        weight: number;
+    }[];
+    communities: {
+        id: string;
+        nodes: string[];
+    }[];
+};
+
+export { type AiwgChunkedIndexLoadOptions, type AiwgChunkedIndexLoader, type AiwgChunkedIndexProgress, type AiwgChunkedIndexProgressPhase, type AiwgChunkedIndexQueryOptions, type AiwgChunkedIndexQueryResult, type AiwgChunkedIndexValidationResult, type AiwgFortemiChunkManifest, type AiwgFortemiChunkPart, type AiwgFortemiChunkPartRef, type AiwgFortemiIndexExport, type AiwgFortemiProvenance, type AiwgFortemiRecord, type AiwgFortemiRecordSource, type AiwgFortemiRecordType, type AiwgFortemiRelationship, type AiwgIndexController, type AiwgIndexControllerListener, type AiwgIndexControllerSnapshot, type AiwgIndexGraphOptions, type AiwgIndexQueryMatch, type AiwgIndexQueryOptions, type AiwgIndexQueryRankedItem, type AiwgIndexQueryResult, type AiwgIndexQueryWeights, type AiwgIndexValidationResult, type AiwgPrivacyClassification, type AiwgProvenanceConfidence, type AiwgReviewAction, type AiwgReviewDecision, type AiwgReviewDecisionExport, type AiwgReviewInput, aiwgFortemiIndexToCommunityGraph, assertAiwgFortemiChunkManifest, assertAiwgFortemiChunkPart, assertAiwgFortemiIndexExport, createAiwgFetchChunkLoader, createAiwgIndexController, createAiwgReviewDecisionExport, getAiwgFortemiFacets, queryAiwgFortemiIndex, validateAiwgFortemiChunkManifest, validateAiwgFortemiChunkPart, validateAiwgFortemiIndexExport };