@swarmvaultai/engine 0.1.3 → 0.1.5

package/README.md

@@ -21,13 +21,18 @@ If you only want to use SwarmVault as a tool, install `@swarmvaultai/cli` instea
 import {
   compileVault,
   createMcpServer,
+  createWebSearchAdapter,
   defaultVaultConfig,
+  defaultVaultSchema,
+  exploreVault,
   importInbox,
   ingestInput,
   initVault,
   installAgent,
+  getWebSearchAdapterForTask,
   lintVault,
   loadVaultConfig,
+  loadVaultSchema,
   queryVault,
   searchVault,
   startGraphServer,
@@ -41,20 +46,37 @@ The engine also exports the main runtime types for providers, graph artifacts, p
 ## Example
 
 ```ts
-import { compileVault, importInbox, initVault, queryVault, watchVault } from "@swarmvaultai/engine";
+import { compileVault, importInbox, initVault, loadVaultSchema, queryVault, watchVault } from "@swarmvaultai/engine";
 
 const rootDir = process.cwd();
 
 await initVault(rootDir);
+const schema = await loadVaultSchema(rootDir);
+console.log(schema.path);
 await importInbox(rootDir);
 await compileVault(rootDir);
 
-const result = await queryVault(rootDir, "What changed most recently?", true);
-console.log(result.answer);
+const saved = await queryVault(rootDir, "What changed most recently?", true);
+console.log(saved.savedTo);
+
+const exploration = await exploreVault(rootDir, "What should I investigate next?", 3);
+console.log(exploration.hubPath);
 
 const watcher = await watchVault(rootDir, { lint: true });
 ```
 
+## Schema Layer
+
+Each workspace carries a root markdown file named `swarmvault.schema.md`.
+
+The engine treats that file as vault-specific operating guidance for compile and query work. Currently:
+
+- `initVault()` creates the default schema file
+- `loadVaultSchema()` resolves the canonical file and legacy `schema.md` fallback
+- compile and query prompts include the schema content
+- generated pages store `schema_hash`
+- `lintVault()` marks generated pages stale when the schema changes
+
 ## Provider Model
 
 The engine supports:
@@ -88,30 +110,37 @@ This matters because many "OpenAI-compatible" backends only implement part of th
 
 ### Compile + Query
 
-- `compileVault(rootDir)` writes wiki pages, graph data, and search state
-- `queryVault(rootDir, question, save)` answers against the compiled vault
+- `compileVault(rootDir)` writes wiki pages, graph data, and search state using the vault schema as guidance
+- `queryVault(rootDir, question, save)` answers against the compiled vault using the same schema layer and can persist a first-class output page
+- `exploreVault(rootDir, question, steps)` runs a save-first multi-step exploration loop and writes a hub page plus step outputs
 - `searchVault(rootDir, query, limit)` searches compiled pages directly
 
 ### Automation
 
 - `watchVault(rootDir, options)` watches the inbox and appends run records to `state/jobs.ndjson`
-- `lintVault(rootDir)` runs health and anti-drift checks
+- `lintVault(rootDir, options)` runs structural lint, optional deep lint, and optional web-augmented evidence gathering
+
+### Web Search Adapters
+
+- `createWebSearchAdapter(rootDir, id, config)` constructs a normalized web search adapter
+- `getWebSearchAdapterForTask(rootDir, "deepLintProvider")` resolves the configured adapter for `lint --deep --web`
 
 ### MCP
 
 - `createMcpServer(rootDir)` creates an MCP server instance
 - `startMcpServer(rootDir)` runs the MCP server over stdio
 
-The MCP surface includes tools for workspace info, page search, page reads, source listing, querying, ingestion, compile, and lint, along with resources for config, graph, manifests, and page content.
+The MCP surface includes tools for workspace info, page search, page reads, source listing, querying, ingestion, compile, and lint, along with resources for config, graph, manifests, schema, and page content.
 
 ## Artifacts
 
 Running the engine produces a local workspace with these main areas:
 
+- `swarmvault.schema.md`: vault-specific compile and query instructions
 - `inbox/`: capture staging area for markdown bundles and imported files
 - `raw/sources/`: immutable source copies
 - `raw/assets/`: copied attachments referenced by ingested markdown bundles
-- `wiki/`: generated markdown pages and saved outputs
+- `wiki/`: generated markdown pages, saved query outputs, and exploration hub pages
 - `state/manifests/`: source manifests
 - `state/extracts/`: extracted text
 - `state/analyses/`: model analysis output
@@ -119,6 +148,8 @@ Running the engine produces a local workspace with these main areas:
 - `state/search.sqlite`: full-text index
 - `state/jobs.ndjson`: watch-mode automation logs
 
+Saved outputs are indexed immediately into the graph page registry and search index, then linked back into compiled source, concept, and entity pages during later compile runs.
+
 ## Notes
 
 - The engine expects Node `>=24`

package/dist/index.d.ts

@@ -2,6 +2,9 @@ import { z } from 'zod';
 import { Readable, Writable } from 'node:stream';
 import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
 
+declare function installAgent(rootDir: string, agent: "codex" | "claude" | "cursor"): Promise<string>;
+declare function installConfiguredAgents(rootDir: string): Promise<string[]>;
+
 declare const providerCapabilitySchema: z.ZodEnum<{
     responses: "responses";
     chat: "chat";
@@ -27,6 +30,12 @@ type PageKind = "index" | "source" | "concept" | "entity" | "output";
 type Freshness = "fresh" | "stale";
 type ClaimStatus = "extracted" | "inferred" | "conflicted" | "stale";
 type Polarity = "positive" | "negative" | "neutral";
+type OutputOrigin = "query" | "explore";
+declare const webSearchProviderTypeSchema: z.ZodEnum<{
+    custom: "custom";
+    "http-json": "http-json";
+}>;
+type WebSearchProviderType = z.infer<typeof webSearchProviderTypeSchema>;
 interface GenerationAttachment {
     mimeType: string;
     filePath: string;
@@ -62,6 +71,32 @@ interface ProviderConfig {
     capabilities?: ProviderCapability[];
     apiStyle?: "responses" | "chat";
 }
+interface WebSearchProviderConfig {
+    type: WebSearchProviderType;
+    endpoint?: string;
+    method?: "GET" | "POST";
+    apiKeyEnv?: string;
+    apiKeyHeader?: string;
+    apiKeyPrefix?: string;
+    headers?: Record<string, string>;
+    queryParam?: string;
+    limitParam?: string;
+    resultsPath?: string;
+    titleField?: string;
+    urlField?: string;
+    snippetField?: string;
+    module?: string;
+}
+interface WebSearchResult {
+    title: string;
+    url: string;
+    snippet: string;
+}
+interface WebSearchAdapter {
+    readonly id: string;
+    readonly type: WebSearchProviderType;
+    search(query: string, limit?: number): Promise<WebSearchResult[]>;
+}
 interface VaultConfig {
     workspace: {
         rawDir: string;
@@ -81,9 +116,16 @@ interface VaultConfig {
         port: number;
     };
     agents: Array<"codex" | "claude" | "cursor">;
+    webSearch?: {
+        providers: Record<string, WebSearchProviderConfig>;
+        tasks: {
+            deepLintProvider: string;
+        };
+    };
 }
 interface ResolvedPaths {
     rootDir: string;
+    schemaPath: string;
     rawDir: string;
     rawSourcesDir: string;
     rawAssetsDir: string;
@@ -137,6 +179,7 @@ interface SourceClaim {
 interface SourceAnalysis {
     sourceId: string;
     sourceHash: string;
+    schemaHash: string;
     title: string;
     summary: string;
     concepts: AnalyzedTerm[];
@@ -173,7 +216,13 @@ interface GraphPage {
     freshness: Freshness;
     confidence: number;
     backlinks: string[];
+    schemaHash: string;
     sourceHashes: Record<string, string>;
+    relatedPageIds: string[];
+    relatedNodeIds: string[];
+    relatedSourceIds: string[];
+    origin?: OutputOrigin;
+    question?: string;
 }
 interface GraphArtifact {
     generatedAt: string;
@@ -198,13 +247,21 @@ interface SearchResult {
 interface QueryResult {
     answer: string;
     savedTo?: string;
+    savedPageId?: string;
     citations: string[];
+    relatedPageIds: string[];
+    relatedNodeIds: string[];
+    relatedSourceIds: string[];
 }
 interface LintFinding {
     severity: "error" | "warning" | "info";
     code: string;
     message: string;
     pagePath?: string;
+    relatedSourceIds?: string[];
+    relatedPageIds?: string[];
+    suggestedQuery?: string;
+    evidence?: WebSearchResult[];
 }
 interface InboxImportSkip {
     path: string;
@@ -238,9 +295,38 @@ interface WatchRunRecord {
 interface WatchController {
     close(): Promise<void>;
 }
+interface CompileState {
+    generatedAt: string;
+    schemaHash: string;
+    analyses: Record<string, string>;
+    sourceHashes: Record<string, string>;
+    outputHashes: Record<string, string>;
+}
+interface LintOptions {
+    deep?: boolean;
+    web?: boolean;
+}
+interface ExploreStepResult {
+    step: number;
+    question: string;
+    answer: string;
+    savedTo: string;
+    savedPageId: string;
+    citations: string[];
+    followUpQuestions: string[];
+}
+interface ExploreResult {
+    rootQuestion: string;
+    hubPath: string;
+    hubPageId: string;
+    stepCount: number;
+    steps: ExploreStepResult[];
+    suggestedQuestions: string[];
+}
 
 declare function defaultVaultConfig(): VaultConfig;
-declare function resolvePaths(rootDir: string, config?: VaultConfig, configPath?: string): ResolvedPaths;
+declare function defaultVaultSchema(): string;
+declare function resolvePaths(rootDir: string, config?: VaultConfig, configPath?: string, schemaPath?: string): ResolvedPaths;
 declare function loadVaultConfig(rootDir: string): Promise<{
     config: VaultConfig;
     paths: ResolvedPaths;
@@ -255,9 +341,27 @@ declare function importInbox(rootDir: string, inputDir?: string): Promise<InboxI
 declare function listManifests(rootDir: string): Promise<SourceManifest[]>;
 declare function readExtractedText(rootDir: string, manifest: SourceManifest): Promise<string | undefined>;
 
+declare function createMcpServer(rootDir: string): Promise<McpServer>;
+declare function startMcpServer(rootDir: string, stdin?: Readable, stdout?: Writable): Promise<{
+    close: () => Promise<void>;
+}>;
+
+declare function createProvider(id: string, config: ProviderConfig, rootDir: string): Promise<ProviderAdapter>;
+declare function getProviderForTask(rootDir: string, task: keyof Awaited<ReturnType<typeof loadVaultConfig>>["config"]["tasks"]): Promise<ProviderAdapter>;
+declare function assertProviderCapability(provider: ProviderAdapter, capability: ProviderCapability): void;
+
+interface VaultSchema {
+    path: string;
+    content: string;
+    hash: string;
+    isLegacyPath: boolean;
+}
+declare function loadVaultSchema(rootDir: string): Promise<VaultSchema>;
+
 declare function initVault(rootDir: string): Promise<void>;
 declare function compileVault(rootDir: string): Promise<CompileResult>;
 declare function queryVault(rootDir: string, question: string, save?: boolean): Promise<QueryResult>;
+declare function exploreVault(rootDir: string, question: string, steps?: number): Promise<ExploreResult>;
 declare function searchVault(rootDir: string, query: string, limit?: number): Promise<SearchResult[]>;
 declare function listPages(rootDir: string): Promise<GraphPage[]>;
 declare function readPage(rootDir: string, relativePath: string): Promise<{
@@ -269,6 +373,7 @@ declare function readPage(rootDir: string, relativePath: string): Promise<{
 declare function getWorkspaceInfo(rootDir: string): Promise<{
     rootDir: string;
     configPath: string;
+    schemaPath: string;
     rawDir: string;
     wikiDir: string;
     stateDir: string;
@@ -277,29 +382,20 @@ declare function getWorkspaceInfo(rootDir: string): Promise<{
     sourceCount: number;
     pageCount: number;
 }>;
-declare function lintVault(rootDir: string): Promise<LintFinding[]>;
+declare function lintVault(rootDir: string, options?: LintOptions): Promise<LintFinding[]>;
 declare function bootstrapDemo(rootDir: string, input?: string): Promise<{
     manifestId?: string;
     compile?: CompileResult;
 }>;
 
-declare function installAgent(rootDir: string, agent: "codex" | "claude" | "cursor"): Promise<string>;
-declare function installConfiguredAgents(rootDir: string): Promise<string[]>;
-
 declare function startGraphServer(rootDir: string, port?: number): Promise<{
     port: number;
     close: () => Promise<void>;
 }>;
 
-declare function createMcpServer(rootDir: string): Promise<McpServer>;
-declare function startMcpServer(rootDir: string, stdin?: Readable, stdout?: Writable): Promise<{
-    close: () => Promise<void>;
-}>;
-
 declare function watchVault(rootDir: string, options?: WatchOptions): Promise<WatchController>;
 
-declare function createProvider(id: string, config: ProviderConfig, rootDir: string): Promise<ProviderAdapter>;
-declare function getProviderForTask(rootDir: string, task: keyof Awaited<ReturnType<typeof loadVaultConfig>>["config"]["tasks"]): Promise<ProviderAdapter>;
-declare function assertProviderCapability(provider: ProviderAdapter, capability: ProviderCapability): void;
+declare function createWebSearchAdapter(id: string, config: WebSearchProviderConfig, rootDir: string): Promise<WebSearchAdapter>;
+declare function getWebSearchAdapterForTask(rootDir: string, task: "deepLintProvider"): Promise<WebSearchAdapter>;
 
-export { type AnalyzedTerm, type ClaimStatus, type CompileResult, type Freshness, type GenerationAttachment, type GenerationRequest, type GenerationResponse, type GraphArtifact, type GraphEdge, type GraphNode, type GraphPage, type InboxImportResult, type InboxImportSkip, type LintFinding, type PageKind, type Polarity, type ProviderAdapter, type ProviderCapability, type ProviderConfig, type ProviderType, type QueryResult, type ResolvedPaths, type SearchResult, type SourceAnalysis, type SourceAttachment, type SourceClaim, type SourceManifest, type VaultConfig, type WatchController, type WatchOptions, type WatchRunRecord, assertProviderCapability, bootstrapDemo, compileVault, createMcpServer, createProvider, defaultVaultConfig, getProviderForTask, getWorkspaceInfo, importInbox, ingestInput, initVault, initWorkspace, installAgent, installConfiguredAgents, lintVault, listManifests, listPages, loadVaultConfig, providerCapabilitySchema, providerTypeSchema, queryVault, readExtractedText, readPage, resolvePaths, searchVault, startGraphServer, startMcpServer, watchVault };
+export { type AnalyzedTerm, type ClaimStatus, type CompileResult, type CompileState, type ExploreResult, type ExploreStepResult, type Freshness, type GenerationAttachment, type GenerationRequest, type GenerationResponse, type GraphArtifact, type GraphEdge, type GraphNode, type GraphPage, type InboxImportResult, type InboxImportSkip, type LintFinding, type LintOptions, type OutputOrigin, type PageKind, type Polarity, type ProviderAdapter, type ProviderCapability, type ProviderConfig, type ProviderType, type QueryResult, type ResolvedPaths, type SearchResult, type SourceAnalysis, type SourceAttachment, type SourceClaim, type SourceManifest, type VaultConfig, type WatchController, type WatchOptions, type WatchRunRecord, type WebSearchAdapter, type WebSearchProviderConfig, type WebSearchProviderType, type WebSearchResult, assertProviderCapability, bootstrapDemo, compileVault, createMcpServer, createProvider, createWebSearchAdapter, defaultVaultConfig, defaultVaultSchema, exploreVault, getProviderForTask, getWebSearchAdapterForTask, getWorkspaceInfo, importInbox, ingestInput, initVault, initWorkspace, installAgent, installConfiguredAgents, lintVault, listManifests, listPages, loadVaultConfig, loadVaultSchema, providerCapabilitySchema, providerTypeSchema, queryVault, readExtractedText, readPage, resolvePaths, searchVault, startGraphServer, startMcpServer, watchVault, webSearchProviderTypeSchema };