@hasna/knowledge 0.2.3 → 0.2.5

package/docs/architecture/hybrid-semantic-search.md

@@ -0,0 +1,135 @@
+# Hybrid Semantic Search Architecture
+
+`open-knowledge` search is hybrid by design. Keyword search, semantic vector
+search, wiki graph traversal, and citation/provenance signals each solve a
+different part of the retrieval problem.
+
+## Ownership Boundary
+
+`open-files` supplies:
+
+- Stable source refs such as `open-files://file/<id>` and
+  `open-files://file/<id>/revision/<revision_id>`.
+- Source ids, file ids, revisions, hashes, MIME metadata, and storage location.
+- Extracted text and extraction status.
+- Read-only content resolution for deeper reads.
+- Change events that trigger incremental reindexing.
+
+`open-knowledge` owns:
+
+- Chunk boundaries and chunk metadata.
+- Embeddings for source chunks, wiki pages, decisions, and durable answers.
+- FTS indexes over chunks and generated wiki artifacts.
+- Wiki backlinks and citation graph signals.
+- Merge, dedupe, rerank, freshness scoring, and context packing.
+- Permission-aware filtering before any content reaches a model.
+
+## Local Indexes
+
+Local mode starts with SQLite:
+
+- `chunks` stores source/wiki text segments and metadata.
+- `chunks_fts` provides keyword search.
+- `chunk_embeddings` stores embedding vectors as JSON until a local vector
+  extension is chosen.
+- `wiki_pages`, `wiki_backlinks`, and `citations` provide graph and provenance
+  signals.
+- `knowledge_indexes` tracks generated machine-readable shards.
+
+The JSON vector representation is intentionally simple for the first local
+implementation. The retrieval interface should hide it so a later vector
+extension or pgvector backend can replace storage without changing CLI/MCP
+contracts.
+
+## Hosted Indexes
+
+Hosted mode may use:
+
+- Postgres plus pgvector for sources/chunks/wiki/runs.
+- Managed vector stores for large corpora.
+- Object storage for generated artifact shards.
+- Worker queues for extraction, embedding, compile, lint, and refresh jobs.
+
+Permission filtering must happen before reranked context is sent to the model.
+Filtering only after retrieval is not enough if the retriever or reranker can see
+unauthorized content.
+
+## Query Pipeline
+
+1. Normalize the query.
+2. Embed the query if a semantic-capable provider is configured.
+3. Run keyword FTS over source chunks and wiki chunks.
+4. Run vector search over source chunks and wiki pages.
+5. Expand candidate pages through backlinks and citations.
+6. Drop stale candidates whose source revision/hash no longer matches
+   `open-files`.
+7. Apply permission filters.
+8. Merge and dedupe by source revision, wiki page, citation, and text hash.
+9. Rerank by relevance, exact-match score, semantic score, freshness, citation
+   quality, and wiki authority.
+10. Return structured results with source refs, citation spans, page refs,
+    scores, and reason codes.
+
+## Result Shape
+
+Search results should be structured enough for CLI, MCP, and agents:
+
+```json
+{
+  "kind": "source_chunk",
+  "id": "chunk_123",
+  "text": "...",
+  "score": 0.87,
+  "scores": {
+    "keyword": 0.72,
+    "semantic": 0.91,
+    "freshness": 1
+  },
+  "source": {
+    "uri": "open-files://file/file_123",
+    "revision": "rev_456",
+    "hash": "..."
+  },
+  "citation": {
+    "start_offset": 120,
+    "end_offset": 260
+  },
+  "reason": ["semantic_match", "exact_term", "fresh_source"]
+}
+```
+
+## Context Packs
+
+`knowledge <prompt>` and MCP `knowledge_ask` should not receive raw search rows.
+They should receive context packs:
+
+- Query and normalized intent.
+- Selected source/wiki excerpts.
+- Citation metadata.
+- Freshness and permission notes.
+- Known uncertainty or conflicting sources.
+- Suggested durable wiki updates.
+
+This keeps agent prompts stable while the retrieval internals evolve.
+
+## Reindexing
+
+Reindexing is driven by source revisions:
+
+- If an `open-files` hash/revision changes, affected chunks and embeddings become
+  stale.
+- If a source is deleted or access changes, affected chunks must be hidden or
+  removed before future retrieval.
+- Wiki pages should track the source revisions they cite so lint can flag stale
+  pages.
+- Embedding refresh jobs should be idempotent and checkpointed in `runs` and
+  `run_events`.
+
+## Acceptance Criteria
+
+- Local search works without network access for keyword-only retrieval.
+- Semantic search is optional and provider-gated.
+- Every returned excerpt can resolve to a source ref or wiki page.
+- Permission filters run before model context assembly.
+- Retrieval internals can swap from JSON vectors to pgvector or managed vector
+  stores without changing CLI/MCP result contracts.

package/package.json

@@ -1,23 +1,23 @@
 {
   "name": "@hasna/knowledge",
-  "version": "0.2.3",
+  "version": "0.2.5",
   "description": "Agent-friendly local knowledge CLI with JSON output, pagination, and safe destructive actions",
   "type": "module",
   "bin": {
-    "open-knowledge": "./bin/open-knowledge.js",
-    "open-knowledge-mcp": "./bin/open-knowledge-mcp.js"
+    "open-knowledge": "bin/open-knowledge.js",
+    "open-knowledge-mcp": "bin/open-knowledge-mcp.js"
   },
   "files": [
     "bin",
-    "src/mcp-http.js",
-    "src/store.ts",
+    "src",
+    "docs",
     "LICENSE",
     "README.md"
   ],
   "scripts": {
     "test": "bun test",
     "test:cli": "bun test tests/cli.test.ts",
-    "build": "bun build --target=bun --outfile=bin/open-knowledge.js --minify src/cli.ts && bun build --target=bun --outfile=bin/open-knowledge-mcp.js --external @modelcontextprotocol/sdk src/mcp.js",
+    "build": "bun build --target=bun --outfile=bin/open-knowledge.js --minify --external @aws-sdk/client-s3 --external @aws-sdk/credential-providers src/cli.ts && bun build --target=bun --outfile=bin/open-knowledge-mcp.js --external @modelcontextprotocol/sdk src/mcp.js",
     "prepublishOnly": "bun run build",
     "postinstall": "bun run build"
   },
@@ -37,7 +37,7 @@
   },
   "repository": {
     "type": "git",
-    "url": "https://github.com/hasna/knowledge"
+    "url": "git+https://github.com/hasna/knowledge.git"
   },
   "bugs": {
     "url": "https://github.com/hasna/knowledge/issues"
@@ -48,7 +48,12 @@
     "node": ">=18"
   },
   "dependencies": {
+    "@aws-sdk/client-s3": "^3.1063.0",
+    "@aws-sdk/credential-providers": "^3.1063.0",
     "@modelcontextprotocol/sdk": "^1.29.0",
     "zod": "^4.3.6"
+  },
+  "devDependencies": {
+    "@types/bun": "^1.3.14"
   }
 }

package/src/artifact-store.ts

@@ -0,0 +1,184 @@
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
+import { dirname, join, relative, sep } from 'node:path';
+import type { KnowledgeConfig, KnowledgeWorkspace } from './workspace';
+
+interface S3ClientLike {
+  send(command: unknown): Promise<any>;
+}
+
+export interface ArtifactWrite {
+  key: string;
+  body: string | Uint8Array;
+  content_type?: string;
+  metadata?: Record<string, string>;
+}
+
+export interface ArtifactStore {
+  readonly type: 'local' | 's3';
+  readonly canRead: boolean;
+  readonly canWrite: boolean;
+  put(entry: ArtifactWrite): Promise<{ key: string; uri: string }>;
+  getText(key: string): Promise<string>;
+  exists(key: string): Promise<boolean>;
+}
+
+export function normalizeArtifactKey(key: string): string {
+  const raw = key.replace(/\\/g, '/').trim();
+  if (!raw || raw.startsWith('/')) {
+    throw new Error(`Invalid artifact key: ${key}`);
+  }
+  const segments = raw.split('/').filter(Boolean);
+  if (segments.length === 0 || segments.some((segment) => segment === '.' || segment === '..')) {
+    throw new Error(`Invalid artifact key: ${key}`);
+  }
+  return segments.join('/');
+}
+
+function assertInside(root: string, target: string): void {
+  const rel = relative(root, target);
+  if (rel.startsWith('..') || rel === '..' || rel.startsWith(`..${sep}`)) {
+    throw new Error(`Artifact path escapes root: ${target}`);
+  }
+}
+
+export class LocalArtifactStore implements ArtifactStore {
+  readonly type = 'local' as const;
+  readonly canRead = true;
+  readonly canWrite = true;
+
+  constructor(private readonly root: string) {
+    mkdirSync(root, { recursive: true });
+  }
+
+  async put(entry: ArtifactWrite): Promise<{ key: string; uri: string }> {
+    const key = normalizeArtifactKey(entry.key);
+    const path = join(this.root, key);
+    assertInside(this.root, path);
+    mkdirSync(dirname(path), { recursive: true });
+    writeFileSync(path, entry.body);
+    return { key, uri: `file://${path}` };
+  }
+
+  async getText(key: string): Promise<string> {
+    const normalizedKey = normalizeArtifactKey(key);
+    const path = join(this.root, normalizedKey);
+    assertInside(this.root, path);
+    return readFileSync(path, 'utf8');
+  }
+
+  async exists(key: string): Promise<boolean> {
+    const normalizedKey = normalizeArtifactKey(key);
+    const path = join(this.root, normalizedKey);
+    assertInside(this.root, path);
+    return existsSync(path);
+  }
+}
+
+export interface S3ArtifactStoreOptions {
+  bucket: string;
+  prefix?: string;
+  region?: string;
+  profile?: string;
+  max_attempts?: number;
+  server_side_encryption?: 'AES256' | 'aws:kms';
+  kms_key_id?: string;
+  client?: S3ClientLike;
+}
+
+export class S3ArtifactStore implements ArtifactStore {
+  readonly type = 's3' as const;
+  readonly canRead = true;
+  readonly canWrite = true;
+  private client?: S3ClientLike;
+
+  constructor(private readonly options: S3ArtifactStoreOptions) {
+    this.client = options.client;
+  }
+
+  private async getClient(): Promise<S3ClientLike> {
+    if (this.client) return this.client;
+    const [{ S3Client }, { fromIni }] = await Promise.all([
+      import('@aws-sdk/client-s3'),
+      import('@aws-sdk/credential-providers'),
+    ]);
+    this.client = new S3Client({
+      region: this.options.region,
+      credentials: this.options.profile ? fromIni({ profile: this.options.profile }) : undefined,
+      maxAttempts: this.options.max_attempts,
+    });
+    return this.client;
+  }
+
+  private objectKey(key: string): string {
+    const normalizedKey = normalizeArtifactKey(key);
+    const prefix = this.options.prefix ? normalizeArtifactKey(this.options.prefix) : '';
+    return prefix ? `${prefix}/${normalizedKey}` : normalizedKey;
+  }
+
+  async put(entry: ArtifactWrite): Promise<{ key: string; uri: string }> {
+    const [{ PutObjectCommand }, client] = await Promise.all([
+      import('@aws-sdk/client-s3'),
+      this.getClient(),
+    ]);
+    const key = this.objectKey(entry.key);
+    await client.send(new PutObjectCommand({
+      Bucket: this.options.bucket,
+      Key: key,
+      Body: entry.body,
+      ContentType: entry.content_type,
+      Metadata: entry.metadata,
+      ServerSideEncryption: this.options.server_side_encryption,
+      SSEKMSKeyId: this.options.kms_key_id,
+    }));
+    return { key, uri: `s3://${this.options.bucket}/${key}` };
+  }
+
+  async getText(key: string): Promise<string> {
+    const [{ GetObjectCommand }, client] = await Promise.all([
+      import('@aws-sdk/client-s3'),
+      this.getClient(),
+    ]);
+    const objectKey = this.objectKey(key);
+    const response = await client.send(new GetObjectCommand({
+      Bucket: this.options.bucket,
+      Key: objectKey,
+    }));
+    if (!response.Body) return '';
+    return await response.Body.transformToString();
+  }
+
+  async exists(key: string): Promise<boolean> {
+    const [{ HeadObjectCommand }, client] = await Promise.all([
+      import('@aws-sdk/client-s3'),
+      this.getClient(),
+    ]);
+    const objectKey = this.objectKey(key);
+    try {
+      await client.send(new HeadObjectCommand({
+        Bucket: this.options.bucket,
+        Key: objectKey,
+      }));
+      return true;
+    } catch (error) {
+      const name = error instanceof Error ? error.name : '';
+      if (name === 'NotFound' || name === 'NoSuchKey' || name === 'NotFoundError') return false;
+      throw error;
+    }
+  }
+}
+
+export function createArtifactStore(config: KnowledgeConfig, workspace: KnowledgeWorkspace): ArtifactStore {
+  if (config.storage.type === 's3') {
+    if (!config.storage.s3?.bucket) throw new Error('S3 artifact storage requires storage.s3.bucket');
+    return new S3ArtifactStore({
+      bucket: config.storage.s3.bucket,
+      prefix: config.storage.s3.prefix,
+      region: config.storage.s3.region,
+      profile: config.storage.s3.profile,
+      max_attempts: config.storage.s3.max_attempts,
+      server_side_encryption: config.storage.s3.server_side_encryption,
+      kms_key_id: config.storage.s3.kms_key_id,
+    });
+  }
+  return new LocalArtifactStore(workspace.artifactsDir);
+}