@hasna/knowledge 0.2.8 → 0.2.10

package/src/cli.ts

@@ -5,14 +5,9 @@
  * Licensed under the Apache License, Version 2.0
  */
 import { defaultStorePath, loadStore, saveStore, withLock, makeId, makeShortId, ensureStore, type KnowledgeItem } from './store';
-import { ensureKnowledgeWorkspace, readKnowledgeConfig, resolveScopedWorkspace } from './workspace';
-import { getKnowledgeDbStats, migrateKnowledgeDb, openKnowledgeDb } from './knowledge-db';
-import { createArtifactStore } from './artifact-store';
-import { initializeWikiLayout } from './wiki-layout';
-import { ingestOpenFilesManifest } from './manifest-ingest';
-import { consumeOpenFilesOutbox } from './outbox-consume';
-import { resolveOpenFilesSource } from './source-resolver';
-import { approvalStatus, assertS3ReadAllowed, assertWebSearchAllowed, createApprovalGate, recordAuditEvent, recordRedactionFindings, redactSecrets, resolveSafetyPolicy } from './safety';
+import { openKnowledgeDb } from './knowledge-db';
+import { createKnowledgeService } from './service';
+import { approvalStatus, assertS3ReadAllowed, assertWebSearchAllowed, createApprovalGate, recordAuditEvent, recordRedactionFindings, redactSecrets } from './safety';
 import pkg from '../package.json' with { type: 'json' };
 
 type LogLevel = 'debug' | 'info' | 'warn' | 'error';
@@ -170,6 +165,7 @@ Commands:
   wiki init                    Initialize scalable wiki/schema/index/log artifacts
   source resolve <source-ref>  Resolve read-only source content and citation evidence
   ingest manifest <file|s3://> Ingest an open-files manifest into knowledge.db
+  ingest source <source-ref>   Ingest a read-only source ref into knowledge.db
   reindex outbox <file|s3://>  Consume open-files change events and invalidate chunks
   safety status|check|approve|audit|redact
   help [command]               Show help
@@ -235,7 +231,7 @@ function printCommandHelp(command: string): void {
   if (command === 'db') { console.log('Usage: open-knowledge db init|stats [--scope local|global|project] [--json]'); return; }
   if (command === 'wiki') { console.log('Usage: open-knowledge wiki init [--scope local|global|project] [--json]'); return; }
   if (command === 'source') { console.log('Usage: open-knowledge source resolve <source-ref> [--purpose knowledge_answer|knowledge_index] [--limit <n>] [--scope local|global|project] [--json]'); return; }
-  if (command === 'ingest') { console.log('Usage: open-knowledge ingest manifest <file|s3://bucket/key> [--scope local|global|project] [--json]'); return; }
+  if (command === 'ingest') { console.log('Usage: open-knowledge ingest manifest <file|s3://bucket/key> | source <source-ref> [--purpose knowledge_index] [--scope local|global|project] [--json]'); return; }
   if (command === 'reindex') { console.log('Usage: open-knowledge reindex outbox <file|s3://bucket/key> [--scope local|global|project] [--json]'); return; }
   if (command === 'safety') { console.log('Usage: open-knowledge safety status|check|approve|audit|redact [args] [--scope local|global|project] [--json]'); return; }
   printGlobalHelp();
@@ -297,71 +293,49 @@ async function run(argv: string[]): Promise<void> {
 
   if (!command || flags.help || command === 'help') { printCommandHelp(positional[1]); return; }
 
-  const workspace = resolveScopedWorkspace(flags.scope);
+  const service = createKnowledgeService({ scope: flags.scope });
   let storePath = flags.store;
   if (!storePath) {
     if (flags.scope === 'project' || flags.scope === 'local') {
-      storePath = ensureKnowledgeWorkspace(workspace.home).jsonStorePath;
+      storePath = service.jsonStorePath();
     } else {
       storePath = defaultStorePath();
     }
   }
 
   if (command === 'paths') {
-    const resolvedWorkspace = ensureKnowledgeWorkspace(workspace.home);
-    output({
-      ok: true,
-      scope: flags.scope ?? 'global',
-      home: resolvedWorkspace.home,
-      config_path: resolvedWorkspace.configPath,
-      json_store_path: resolvedWorkspace.jsonStorePath,
-      knowledge_db_path: resolvedWorkspace.knowledgeDbPath,
-      artifacts_dir: resolvedWorkspace.artifactsDir,
-      indexes_dir: resolvedWorkspace.indexesDir,
-      logs_dir: resolvedWorkspace.logsDir,
-      runs_dir: resolvedWorkspace.runsDir,
-      schemas_dir: resolvedWorkspace.schemasDir,
-      wiki_dir: resolvedWorkspace.wikiDir,
-      config: readKnowledgeConfig(resolvedWorkspace.configPath),
-      message: resolvedWorkspace.home,
-    }, flags.json);
+    output(service.paths(), flags.json);
     return;
   }
 
   if (command === 'db') {
     const action = positional[1] ?? 'init';
-    const resolvedWorkspace = ensureKnowledgeWorkspace(workspace.home);
     if (action !== 'init' && action !== 'stats') {
       throw new Error("Invalid db action. Use 'init' or 'stats'.");
     }
     if (action === 'init') {
-      const result = migrateKnowledgeDb(resolvedWorkspace.knowledgeDbPath);
+      const result = service.initDb();
       output({ ok: true, ...result, message: `Initialized ${result.path}` }, flags.json);
       return;
     }
-    migrateKnowledgeDb(resolvedWorkspace.knowledgeDbPath);
-    const stats = getKnowledgeDbStats(resolvedWorkspace.knowledgeDbPath);
-    output({ ok: true, path: resolvedWorkspace.knowledgeDbPath, ...stats, message: `knowledge.db schema v${stats.schema_version}` }, flags.json);
+    const stats = service.dbStats();
+    output({ ok: true, path: service.workspace.knowledgeDbPath, ...stats, message: `knowledge.db schema v${stats.schema_version}` }, flags.json);
     return;
   }
 
   if (command === 'wiki') {
     const action = positional[1] ?? 'init';
     if (action !== 'init') throw new Error("Invalid wiki action. Use 'init'.");
-    const resolvedWorkspace = ensureKnowledgeWorkspace(workspace.home);
-    const config = readKnowledgeConfig(resolvedWorkspace.configPath);
-    const artifactStore = createArtifactStore(config, resolvedWorkspace);
-    const result = await initializeWikiLayout(artifactStore);
-    output({ ok: true, ...result, message: `Initialized wiki layout in ${resolvedWorkspace.home}` }, flags.json);
+    const result = await service.initWiki();
+    output({ ok: true, ...result, message: `Initialized wiki layout in ${service.workspace.home}` }, flags.json);
     return;
   }
 
   if (command === 'safety') {
     const action = positional[1] ?? 'status';
-    const resolvedWorkspace = ensureKnowledgeWorkspace(workspace.home);
-    const config = readKnowledgeConfig(resolvedWorkspace.configPath);
-    const policy = resolveSafetyPolicy(config, resolvedWorkspace);
-    migrateKnowledgeDb(resolvedWorkspace.knowledgeDbPath);
+    const resolvedWorkspace = service.ensureWorkspace();
+    const policy = service.safetyPolicy();
+    service.initDb();
     const db = openKnowledgeDb(resolvedWorkspace.knowledgeDbPath);
     try {
       if (action === 'status') {
@@ -487,15 +461,9 @@ async function run(argv: string[]): Promise<void> {
     if (action !== 'resolve') throw new Error("Invalid source action. Use 'resolve'.");
     const sourceRef = positional[2];
     if (!sourceRef) throw new Error('Usage: open-knowledge source resolve <source-ref>');
-    const resolvedWorkspace = ensureKnowledgeWorkspace(workspace.home);
-    const config = readKnowledgeConfig(resolvedWorkspace.configPath);
-    const safetyPolicy = resolveSafetyPolicy(config, resolvedWorkspace);
-    const result = await resolveOpenFilesSource({
-      dbPath: resolvedWorkspace.knowledgeDbPath,
-      sourceRef,
+    const result = await service.resolveSource(sourceRef, {
       purpose: flags.purpose,
       limit: flags.limit,
-      safetyPolicy,
     });
     output({
       ok: true,
@@ -509,20 +477,21 @@ async function run(argv: string[]): Promise<void> {
 
   if (command === 'ingest') {
     const action = positional[1] ?? '';
-    if (action !== 'manifest') throw new Error("Invalid ingest action. Use 'manifest'.");
-    const input = positional[2];
-    if (!input) throw new Error('Usage: open-knowledge ingest manifest <file|s3://bucket/key>');
-    const resolvedWorkspace = ensureKnowledgeWorkspace(workspace.home);
-    const config = readKnowledgeConfig(resolvedWorkspace.configPath);
-    const safetyPolicy = resolveSafetyPolicy(config, resolvedWorkspace);
-    const result = await ingestOpenFilesManifest({
-      dbPath: resolvedWorkspace.knowledgeDbPath,
-      input,
-      config,
-      safetyPolicy,
-    });
-    output({ ok: true, ...result, message: `Ingested ${result.items_seen} manifest item(s)` }, flags.json);
-    return;
+    if (action === 'manifest') {
+      const input = positional[2];
+      if (!input) throw new Error('Usage: open-knowledge ingest manifest <file|s3://bucket/key>');
+      const result = await service.ingestManifest(input);
+      output({ ok: true, ...result, message: `Ingested ${result.items_seen} manifest item(s)` }, flags.json);
+      return;
+    }
+    if (action === 'source') {
+      const sourceRef = positional[2];
+      if (!sourceRef) throw new Error('Usage: open-knowledge ingest source <source-ref>');
+      const result = await service.ingestSource(sourceRef, flags.purpose);
+      output({ ok: true, ...result, message: `Ingested source ${result.source_ref} (${result.chunks_inserted} chunks)` }, flags.json);
+      return;
+    }
+    throw new Error("Invalid ingest action. Use 'manifest' or 'source'.");
   }
 
   if (command === 'reindex') {
@@ -530,15 +499,7 @@ async function run(argv: string[]): Promise<void> {
     if (action !== 'outbox') throw new Error("Invalid reindex action. Use 'outbox'.");
     const input = positional[2];
     if (!input) throw new Error('Usage: open-knowledge reindex outbox <file|s3://bucket/key>');
-    const resolvedWorkspace = ensureKnowledgeWorkspace(workspace.home);
-    const config = readKnowledgeConfig(resolvedWorkspace.configPath);
-    const safetyPolicy = resolveSafetyPolicy(config, resolvedWorkspace);
-    const result = await consumeOpenFilesOutbox({
-      dbPath: resolvedWorkspace.knowledgeDbPath,
-      input,
-      config,
-      safetyPolicy,
-    });
+    const result = await service.consumeOutbox(input);
     output({ ok: true, ...result, message: `Consumed ${result.events_seen} outbox event(s)` }, flags.json);
     return;
   }

package/src/manifest-ingest.ts

@@ -24,6 +24,17 @@ export interface ManifestIngestOptions {
   chunkOverlapChars?: number;
 }
 
+export interface ManifestItemsIngestOptions {
+  dbPath: string;
+  items: ManifestObject[];
+  sourceLabel: string;
+  readAction?: string;
+  safetyPolicy?: SafetyPolicy;
+  now?: Date;
+  maxChunkChars?: number;
+  chunkOverlapChars?: number;
+}
+
 export interface ManifestIngestResult {
   path: string;
   db_path: string;
@@ -36,7 +47,7 @@ export interface ManifestIngestResult {
   skipped: number;
 }
 
-type ManifestObject = Record<string, unknown>;
+export type ManifestObject = Record<string, unknown>;
 
 interface NormalizedManifestItem {
   raw: ManifestObject;
@@ -405,6 +416,23 @@ function insertChunks(db: Database, sourceRevisionId: string, item: NormalizedMa
 }
 
 export async function ingestOpenFilesManifest(options: ManifestIngestOptions): Promise<ManifestIngestResult> {
+  const now = options.now ?? new Date();
+  if (options.safetyPolicy) assertWriteAllowed(options.dbPath, options.safetyPolicy);
+  migrateKnowledgeDb(options.dbPath);
+  const text = await readManifestInput(options.input, options.config, options.safetyPolicy);
+  const items = parseManifestText(text);
+  return ingestOpenFilesManifestItems({
+    dbPath: options.dbPath,
+    items,
+    sourceLabel: options.input,
+    safetyPolicy: options.safetyPolicy,
+    now,
+    maxChunkChars: options.maxChunkChars,
+    chunkOverlapChars: options.chunkOverlapChars,
+  });
+}
+
+export async function ingestOpenFilesManifestItems(options: ManifestItemsIngestOptions): Promise<ManifestIngestResult> {
   const now = (options.now ?? new Date()).toISOString();
   const maxChunkChars = options.maxChunkChars ?? 4000;
   const chunkOverlapChars = options.chunkOverlapChars ?? 200;
@@ -413,8 +441,6 @@ export async function ingestOpenFilesManifest(options: ManifestIngestOptions): P
 
   if (options.safetyPolicy) assertWriteAllowed(options.dbPath, options.safetyPolicy);
   migrateKnowledgeDb(options.dbPath);
-  const text = await readManifestInput(options.input, options.config, options.safetyPolicy);
-  const items = parseManifestText(text);
   const db = openKnowledgeDb(options.dbPath);
   try {
     const result = db.transaction(() => {
@@ -426,13 +452,13 @@ export async function ingestOpenFilesManifest(options: ManifestIngestOptions): P
       let skipped = 0;
       recordAuditEvent(db, {
         event_type: 'source_read',
-        action: options.input.startsWith('s3://') ? 's3_manifest_read' : 'local_manifest_read',
-        target_uri: options.input,
+        action: options.readAction ?? (options.sourceLabel.startsWith('s3://') ? 's3_manifest_read' : 'local_manifest_read'),
+        target_uri: options.sourceLabel,
         decision: 'allow',
-        metadata: { items: items.length, read_only: true },
+        metadata: { items: options.items.length, read_only: true },
         created_at: now,
       });
-      for (const raw of items) {
+      for (const raw of options.items) {
         const item = normalizeManifestItem(raw, now);
         const sourceId = upsertSource(db, item, now);
         const revisionId = upsertRevision(db, sourceId, item, now);
@@ -450,13 +476,13 @@ export async function ingestOpenFilesManifest(options: ManifestIngestOptions): P
         action: 'knowledge_manifest_ingest',
         target_uri: options.dbPath,
         decision: 'allow',
-        metadata: { items: items.length, sources: seenSources.size, revisions: seenRevisions.size, chunks_inserted: chunksInserted, redactions },
+        metadata: { items: options.items.length, sources: seenSources.size, revisions: seenRevisions.size, chunks_inserted: chunksInserted, redactions },
         created_at: now,
       });
       return {
-        path: options.input,
+        path: options.sourceLabel,
         db_path: options.dbPath,
-        items_seen: items.length,
+        items_seen: options.items.length,
         sources_upserted: seenSources.size,
         revisions_upserted: seenRevisions.size,
         chunks_inserted: chunksInserted,

package/src/mcp.js

@@ -5,10 +5,8 @@ import { z } from 'zod';
 import { existsSync, readFileSync, writeFileSync } from 'node:fs';
 import pkg from '../package.json' with { type: 'json' };
 import { defaultStorePath, loadStore, saveStore, makeId, withLock } from './store.ts';
-import { ensureKnowledgeWorkspace, readKnowledgeConfig, resolveScopedWorkspace } from './workspace.ts';
 import { parseSourceRef } from './source-ref.ts';
-import { resolveOpenFilesSource } from './source-resolver.ts';
-import { resolveSafetyPolicy } from './safety.ts';
+import { createKnowledgeService } from './service.ts';
 
 const storePathField = z.string().optional().describe('Path to the JSON store file');
 const scopeField = z.enum(['local', 'global', 'project']).optional().describe('Workspace scope');
@@ -28,7 +26,7 @@ function shortIdFor(id) {
 function resolveStorePath(storePath, scope) {
   if (storePath) return storePath;
   if (scope === 'project' || scope === 'local') {
-    return ensureKnowledgeWorkspace(resolveScopedWorkspace(scope).home).jsonStorePath;
+    return createKnowledgeService({ scope }).jsonStorePath();
   }
   return defaultStorePath();
 }
@@ -76,22 +74,7 @@ export function buildServer() {
   registerTool(server, 'ok_paths', 'Knowledge workspace paths', 'Show resolved workspace and store paths', {
     scope: scopeField,
   }, async ({ scope }) => {
-    const workspace = ensureKnowledgeWorkspace(resolveScopedWorkspace(scope).home);
-    return jsonText({
-      ok: true,
-      scope: scope ?? 'global',
-      home: workspace.home,
-      config_path: workspace.configPath,
-      json_store_path: workspace.jsonStorePath,
-      knowledge_db_path: workspace.knowledgeDbPath,
-      artifacts_dir: workspace.artifactsDir,
-      indexes_dir: workspace.indexesDir,
-      logs_dir: workspace.logsDir,
-      runs_dir: workspace.runsDir,
-      schemas_dir: workspace.schemasDir,
-      wiki_dir: workspace.wikiDir,
-      config: readKnowledgeConfig(workspace.configPath),
-    });
+    return jsonText(createKnowledgeService({ scope }).paths());
   });
 
   registerTool(server, 'ok_parse_source_ref', 'Parse source reference', 'Parse and validate an open-files, S3, file, or web source ref', {
@@ -110,16 +93,11 @@ export function buildServer() {
     limit: z.number().optional().describe('Maximum chunks to return, default 10'),
     scope: scopeField,
   }, async ({ source_ref, purpose, limit, scope }) => {
-    const workspace = ensureKnowledgeWorkspace(resolveScopedWorkspace(scope).home);
-    const config = readKnowledgeConfig(workspace.configPath);
-    const safetyPolicy = resolveSafetyPolicy(config, workspace);
+    const service = createKnowledgeService({ scope });
     try {
-      const result = await resolveOpenFilesSource({
-        dbPath: workspace.knowledgeDbPath,
-        sourceRef: source_ref,
+      const result = await service.resolveSource(source_ref, {
         purpose,
         limit,
-        safetyPolicy,
       });
       return jsonText({ ok: true, ...result });
     } catch (error) {

package/src/service.ts

@@ -0,0 +1,157 @@
+import { createArtifactStore } from './artifact-store';
+import { consumeOpenFilesOutbox } from './outbox-consume';
+import { getKnowledgeDbStats, migrateKnowledgeDb } from './knowledge-db';
+import { ingestOpenFilesManifest } from './manifest-ingest';
+import { ingestSourceRef } from './source-ingest';
+import { resolveOpenFilesSource } from './source-resolver';
+import { resolveSafetyPolicy } from './safety';
+import { initializeWikiLayout } from './wiki-layout';
+import {
+  ensureKnowledgeWorkspace,
+  readKnowledgeConfig,
+  resolveScopedWorkspace,
+  type KnowledgeConfig,
+  type KnowledgeWorkspace,
+} from './workspace';
+
+export interface KnowledgeServiceOptions {
+  scope?: string;
+  cwd?: string;
+}
+
+export interface KnowledgePathsResult {
+  ok: true;
+  scope: string;
+  home: string;
+  config_path: string;
+  json_store_path: string;
+  knowledge_db_path: string;
+  artifacts_dir: string;
+  indexes_dir: string;
+  logs_dir: string;
+  runs_dir: string;
+  schemas_dir: string;
+  wiki_dir: string;
+  config: KnowledgeConfig;
+  message: string;
+}
+
+export class KnowledgeService {
+  private ensuredWorkspace?: KnowledgeWorkspace;
+  private cachedConfig?: KnowledgeConfig;
+
+  constructor(private readonly options: KnowledgeServiceOptions = {}) {}
+
+  get scope(): string {
+    return this.options.scope ?? 'global';
+  }
+
+  get workspace(): KnowledgeWorkspace {
+    return this.ensuredWorkspace ?? resolveScopedWorkspace(this.options.scope, this.options.cwd);
+  }
+
+  ensureWorkspace(): KnowledgeWorkspace {
+    if (!this.ensuredWorkspace) this.ensuredWorkspace = ensureKnowledgeWorkspace(this.workspace.home);
+    return this.ensuredWorkspace;
+  }
+
+  jsonStorePath(): string {
+    return this.ensureWorkspace().jsonStorePath;
+  }
+
+  config(): KnowledgeConfig {
+    if (!this.cachedConfig) {
+      const workspace = this.ensureWorkspace();
+      this.cachedConfig = readKnowledgeConfig(workspace.configPath);
+    }
+    return this.cachedConfig;
+  }
+
+  safetyPolicy() {
+    return resolveSafetyPolicy(this.config(), this.ensureWorkspace());
+  }
+
+  artifactStore() {
+    return createArtifactStore(this.config(), this.ensureWorkspace());
+  }
+
+  paths(): KnowledgePathsResult {
+    const workspace = this.ensureWorkspace();
+    return {
+      ok: true,
+      scope: this.scope,
+      home: workspace.home,
+      config_path: workspace.configPath,
+      json_store_path: workspace.jsonStorePath,
+      knowledge_db_path: workspace.knowledgeDbPath,
+      artifacts_dir: workspace.artifactsDir,
+      indexes_dir: workspace.indexesDir,
+      logs_dir: workspace.logsDir,
+      runs_dir: workspace.runsDir,
+      schemas_dir: workspace.schemasDir,
+      wiki_dir: workspace.wikiDir,
+      config: this.config(),
+      message: workspace.home,
+    };
+  }
+
+  initDb() {
+    return migrateKnowledgeDb(this.ensureWorkspace().knowledgeDbPath);
+  }
+
+  dbStats() {
+    const workspace = this.ensureWorkspace();
+    migrateKnowledgeDb(workspace.knowledgeDbPath);
+    return getKnowledgeDbStats(workspace.knowledgeDbPath);
+  }
+
+  async initWiki() {
+    return initializeWikiLayout(this.artifactStore());
+  }
+
+  async ingestManifest(input: string) {
+    const workspace = this.ensureWorkspace();
+    return ingestOpenFilesManifest({
+      dbPath: workspace.knowledgeDbPath,
+      input,
+      config: this.config(),
+      safetyPolicy: this.safetyPolicy(),
+    });
+  }
+
+  async ingestSource(sourceRef: string, purpose?: string) {
+    const workspace = this.ensureWorkspace();
+    return ingestSourceRef({
+      dbPath: workspace.knowledgeDbPath,
+      sourceRef,
+      purpose,
+      config: this.config(),
+      safetyPolicy: this.safetyPolicy(),
+    });
+  }
+
+  async resolveSource(sourceRef: string, options: { purpose?: string; limit?: number } = {}) {
+    const workspace = this.ensureWorkspace();
+    return resolveOpenFilesSource({
+      dbPath: workspace.knowledgeDbPath,
+      sourceRef,
+      purpose: options.purpose,
+      limit: options.limit,
+      safetyPolicy: this.safetyPolicy(),
+    });
+  }
+
+  async consumeOutbox(input: string) {
+    const workspace = this.ensureWorkspace();
+    return consumeOpenFilesOutbox({
+      dbPath: workspace.knowledgeDbPath,
+      input,
+      config: this.config(),
+      safetyPolicy: this.safetyPolicy(),
+    });
+  }
+}
+
+export function createKnowledgeService(options: KnowledgeServiceOptions = {}): KnowledgeService {
+  return new KnowledgeService(options);
+}