@hasna/knowledge 0.2.27 → 0.2.28

package/src/service.ts

@@ -1,443 +0,0 @@
-import { createArtifactStore } from './artifact-store';
-import {
-  clearKnowledgeAuth,
-  knowledgeAuthStatus,
-  normalizeKnowledgeApiOrigin,
-  saveKnowledgeAuth,
-  type KnowledgeAuthStatus,
-} from './auth';
-import { runKnowledgePrompt, type KnowledgePromptOptions } from './agent';
-import {
-  embeddingIndexStatus,
-  indexKnowledgeEmbeddings,
-  searchVectorIndex,
-  type EmbeddingIndexOptions,
-  type EmbeddingSearchOptions,
-} from './embeddings';
-import { consumeOpenFilesOutbox } from './outbox-consume';
-import { getKnowledgeDbStats, migrateKnowledgeDb, openKnowledgeDb } from './knowledge-db';
-import { ingestOpenFilesManifest } from './manifest-ingest';
-import { ingestSourceRef } from './source-ingest';
-import { resolveOpenFilesSource } from './source-resolver';
-import { providerStatus, listModelRegistry, type ProviderStatusResult, type ModelRegistryEntry } from './providers';
-import { enqueueMissingEmbeddings, refreshEmbeddingIndex, reindexHealth, type ReindexRuntimeOptions } from './reindex';
-import { knowledgeRegistryContract, RemoteKnowledgeClient, type RemoteKnowledgeRegistryContract } from './remote-client';
-import { retrieveKnowledgeContext, type RetrievalOptions } from './retrieval';
-import { hybridSearch, type HybridSearchOptions } from './search';
-import { resolveSafetyPolicy } from './safety';
-import { runProviderWebSearch, type WebSearchOptions } from './web-search';
-import { compileWikiPage, fileAnswerToWiki, lintWiki, type WikiCompileOptions } from './wiki-compiler';
-import {
-  recordStorageObjects,
-  resolveStorageContract,
-  validateStorageConfig,
-  type StorageContract,
-  type StorageValidationResult,
-} from './storage-contract';
-import { initializeWikiLayout, recordWikiLayoutCatalog } from './wiki-layout';
-import {
-  canonicalHasnaXyzKnowledgeStorage,
-  ensureKnowledgeWorkspace,
-  readKnowledgeConfig,
-  resolveScopedWorkspace,
-  writeKnowledgeConfig,
-  type KnowledgeConfig,
-  type KnowledgeWorkspace,
-} from './workspace';
-
-export interface KnowledgeServiceOptions {
-  scope?: string;
-  cwd?: string;
-}
-
-export interface KnowledgePathsResult {
-  ok: true;
-  scope: string;
-  home: string;
-  config_path: string;
-  json_store_path: string;
-  knowledge_db_path: string;
-  artifacts_dir: string;
-  indexes_dir: string;
-  logs_dir: string;
-  runs_dir: string;
-  schemas_dir: string;
-  wiki_dir: string;
-  config: KnowledgeConfig;
-  message: string;
-}
-
-export interface KnowledgeSetupResult {
-  ok: true;
-  mode: KnowledgeConfig['mode'];
-  api_url: string | null;
-  storage_type: KnowledgeConfig['storage']['type'];
-  artifact_uri_prefix: string;
-  canonical_hasna_xyz: StorageContract['canonical_hasna_xyz'];
-  config_path: string;
-  next: string[];
-  message: string;
-}
-
-function normalizeMode(value: string | undefined): KnowledgeConfig['mode'] | undefined {
-  if (!value) return undefined;
-  const normalized = value.trim().toLowerCase();
-  if (normalized === 'local' || normalized === 'offline') return 'local';
-  if (normalized === 'hosted' || normalized === 'remote' || normalized === 'knowledge.hasna.xyz') return 'hosted';
-  throw new Error('Invalid setup mode. Use hosted or local.');
-}
-
-export class KnowledgeService {
-  private ensuredWorkspace?: KnowledgeWorkspace;
-  private cachedConfig?: KnowledgeConfig;
-
-  constructor(private readonly options: KnowledgeServiceOptions = {}) {}
-
-  get scope(): string {
-    return this.options.scope ?? 'global';
-  }
-
-  get workspace(): KnowledgeWorkspace {
-    return this.ensuredWorkspace ?? resolveScopedWorkspace(this.options.scope, this.options.cwd);
-  }
-
-  ensureWorkspace(): KnowledgeWorkspace {
-    if (!this.ensuredWorkspace) this.ensuredWorkspace = ensureKnowledgeWorkspace(this.workspace.home);
-    return this.ensuredWorkspace;
-  }
-
-  jsonStorePath(): string {
-    return this.ensureWorkspace().jsonStorePath;
-  }
-
-  config(): KnowledgeConfig {
-    if (!this.cachedConfig) {
-      const workspace = this.ensureWorkspace();
-      this.cachedConfig = readKnowledgeConfig(workspace.configPath);
-    }
-    return this.cachedConfig;
-  }
-
-  safetyPolicy() {
-    return resolveSafetyPolicy(this.config(), this.ensureWorkspace());
-  }
-
-  artifactStore() {
-    return createArtifactStore(this.config(), this.ensureWorkspace());
-  }
-
-  storageContract(): StorageContract {
-    return resolveStorageContract(this.config(), this.ensureWorkspace(), this.scope);
-  }
-
-  validateStorage(): StorageValidationResult {
-    return validateStorageConfig(this.config(), this.ensureWorkspace());
-  }
-
-  setup(options: { mode?: string; apiUrl?: string; canonicalHasnaXyz?: boolean } = {}): KnowledgeSetupResult {
-    const workspace = this.ensureWorkspace();
-    const current = this.config();
-    const mode = normalizeMode(options.mode) ?? current.mode;
-    const apiUrl = options.apiUrl
-      ? normalizeKnowledgeApiOrigin(options.apiUrl)
-      : current.hosted?.api_url
-        ? normalizeKnowledgeApiOrigin(current.hosted.api_url)
-        : null;
-    const nextConfig: KnowledgeConfig = {
-      ...current,
-      mode,
-      hosted: {
-        ...(current.hosted ?? {}),
-        ...(apiUrl ? { api_url: apiUrl } : {}),
-      },
-      storage: options.canonicalHasnaXyz
-        ? canonicalHasnaXyzKnowledgeStorage()
-        : current.storage,
-    };
-    writeKnowledgeConfig(workspace.configPath, nextConfig);
-    this.cachedConfig = nextConfig;
-    const storage = resolveStorageContract(nextConfig, workspace, this.scope);
-    return {
-      ok: true,
-      mode,
-      api_url: nextConfig.hosted?.api_url ?? null,
-      storage_type: nextConfig.storage.type,
-      artifact_uri_prefix: storage.artifact_store.uri_prefix,
-      canonical_hasna_xyz: storage.canonical_hasna_xyz,
-      config_path: workspace.configPath,
-      next: mode === 'hosted'
-        ? ['open-knowledge auth login --api-key <key>', 'open-knowledge storage status --json', 'open-knowledge remote contracts --json']
-        : ['open-knowledge search <query>', 'knowledge <prompt>'],
-      message: `Set knowledge mode to ${mode}`,
-    };
-  }
-
-  authStatus(env: Record<string, string | undefined> = process.env): KnowledgeAuthStatus {
-    return knowledgeAuthStatus(this.config(), env);
-  }
-
-  saveAuth(input: {
-    apiKey: string;
-    email?: string;
-    orgId?: string;
-    orgSlug?: string;
-    userId?: string;
-    apiUrl?: string;
-  }, env: Record<string, string | undefined> = process.env) {
-    const apiUrl = input.apiUrl ?? this.config().hosted?.api_url;
-    return saveKnowledgeAuth({
-      api_key: input.apiKey,
-      email: input.email,
-      org_id: input.orgId,
-      org_slug: input.orgSlug,
-      user_id: input.userId,
-      api_url: apiUrl,
-    }, env);
-  }
-
-  clearAuth(env: Record<string, string | undefined> = process.env) {
-    return clearKnowledgeAuth(env);
-  }
-
-  remoteContract(): RemoteKnowledgeRegistryContract {
-    const storage = this.storageContract();
-    return knowledgeRegistryContract({
-      mode: this.config().mode,
-      sourceSchemes: this.config().sources.allowed_schemes,
-      storageType: storage.artifact_store.type,
-      artifactUriPrefix: storage.artifact_store.uri_prefix,
-    });
-  }
-
-  remoteClient(env: Record<string, string | undefined> = process.env): RemoteKnowledgeClient | null {
-    return RemoteKnowledgeClient.fromConfig(this.config(), env);
-  }
-
-  paths(): KnowledgePathsResult {
-    const workspace = this.ensureWorkspace();
-    return {
-      ok: true,
-      scope: this.scope,
-      home: workspace.home,
-      config_path: workspace.configPath,
-      json_store_path: workspace.jsonStorePath,
-      knowledge_db_path: workspace.knowledgeDbPath,
-      artifacts_dir: workspace.artifactsDir,
-      indexes_dir: workspace.indexesDir,
-      logs_dir: workspace.logsDir,
-      runs_dir: workspace.runsDir,
-      schemas_dir: workspace.schemasDir,
-      wiki_dir: workspace.wikiDir,
-      config: this.config(),
-      message: workspace.home,
-    };
-  }
-
-  initDb() {
-    return migrateKnowledgeDb(this.ensureWorkspace().knowledgeDbPath);
-  }
-
-  dbStats() {
-    const workspace = this.ensureWorkspace();
-    migrateKnowledgeDb(workspace.knowledgeDbPath);
-    return getKnowledgeDbStats(workspace.knowledgeDbPath);
-  }
-
-  async initWiki() {
-    const workspace = this.ensureWorkspace();
-    migrateKnowledgeDb(workspace.knowledgeDbPath);
-    const result = await initializeWikiLayout(this.artifactStore());
-    const db = openKnowledgeDb(workspace.knowledgeDbPath);
-    try {
-      recordStorageObjects(db, result.artifacts);
-      recordWikiLayoutCatalog(db, result.artifacts);
-    } finally {
-      db.close();
-    }
-    return result;
-  }
-
-  async compileWiki(options: Omit<WikiCompileOptions, 'dbPath' | 'store'> = {}) {
-    const workspace = this.ensureWorkspace();
-    return compileWikiPage({
-      ...options,
-      dbPath: workspace.knowledgeDbPath,
-      store: this.artifactStore(),
-    });
-  }
-
-  async fileAnswer(options: {
-    prompt: string;
-    answer: string;
-    approveWrite?: boolean;
-    limit?: number;
-    semantic?: boolean;
-    modelRef?: string;
-    dimensions?: number;
-    fake?: boolean;
-  }) {
-    const workspace = this.ensureWorkspace();
-    const context = await this.retrieveContext({
-      query: options.prompt,
-      limit: options.limit,
-      semantic: options.semantic,
-      modelRef: options.modelRef,
-      dimensions: options.dimensions,
-      fake: options.fake,
-    });
-    return fileAnswerToWiki({
-      dbPath: workspace.knowledgeDbPath,
-      store: this.artifactStore(),
-      prompt: options.prompt,
-      answer: options.answer,
-      context,
-      approveWrite: options.approveWrite,
-    });
-  }
-
-  lintWiki() {
-    const workspace = this.ensureWorkspace();
-    return lintWiki({ dbPath: workspace.knowledgeDbPath });
-  }
-
-  async ingestManifest(input: string) {
-    const workspace = this.ensureWorkspace();
-    return ingestOpenFilesManifest({
-      dbPath: workspace.knowledgeDbPath,
-      input,
-      config: this.config(),
-      safetyPolicy: this.safetyPolicy(),
-    });
-  }
-
-  async ingestSource(sourceRef: string, purpose?: string) {
-    const workspace = this.ensureWorkspace();
-    return ingestSourceRef({
-      dbPath: workspace.knowledgeDbPath,
-      sourceRef,
-      purpose,
-      config: this.config(),
-      safetyPolicy: this.safetyPolicy(),
-    });
-  }
-
-  async resolveSource(sourceRef: string, options: { purpose?: string; limit?: number } = {}) {
-    const workspace = this.ensureWorkspace();
-    return resolveOpenFilesSource({
-      dbPath: workspace.knowledgeDbPath,
-      sourceRef,
-      purpose: options.purpose,
-      limit: options.limit,
-      safetyPolicy: this.safetyPolicy(),
-    });
-  }
-
-  async consumeOutbox(input: string) {
-    const workspace = this.ensureWorkspace();
-    return consumeOpenFilesOutbox({
-      dbPath: workspace.knowledgeDbPath,
-      input,
-      config: this.config(),
-      safetyPolicy: this.safetyPolicy(),
-    });
-  }
-
-  reindexHealth(options: Omit<ReindexRuntimeOptions, 'dbPath' | 'config'> = {}) {
-    const workspace = this.ensureWorkspace();
-    return reindexHealth({
-      ...options,
-      dbPath: workspace.knowledgeDbPath,
-      config: this.config(),
-    });
-  }
-
-  enqueueReindex(options: Omit<ReindexRuntimeOptions, 'dbPath' | 'config'> = {}) {
-    const workspace = this.ensureWorkspace();
-    return enqueueMissingEmbeddings({
-      ...options,
-      dbPath: workspace.knowledgeDbPath,
-      config: this.config(),
-    });
-  }
-
-  async refreshEmbeddings(options: Omit<ReindexRuntimeOptions & { full?: boolean; limit?: number }, 'dbPath' | 'config'> = {}) {
-    const workspace = this.ensureWorkspace();
-    return refreshEmbeddingIndex({
-      ...options,
-      dbPath: workspace.knowledgeDbPath,
-      config: this.config(),
-    });
-  }
-
-  providerStatus(env: Record<string, string | undefined> = process.env): ProviderStatusResult {
-    return providerStatus(this.config(), env);
-  }
-
-  modelRegistry(): ModelRegistryEntry[] {
-    return listModelRegistry(this.config());
-  }
-
-  embeddingStatus() {
-    const workspace = this.ensureWorkspace();
-    return embeddingIndexStatus(workspace.knowledgeDbPath);
-  }
-
-  async indexEmbeddings(options: Omit<EmbeddingIndexOptions, 'dbPath' | 'config'> = {}) {
-    const workspace = this.ensureWorkspace();
-    return indexKnowledgeEmbeddings({
-      ...options,
-      dbPath: workspace.knowledgeDbPath,
-      config: this.config(),
-    });
-  }
-
-  async semanticSearch(options: Omit<EmbeddingSearchOptions, 'dbPath' | 'config'>) {
-    const workspace = this.ensureWorkspace();
-    return searchVectorIndex({
-      ...options,
-      dbPath: workspace.knowledgeDbPath,
-      config: this.config(),
-    });
-  }
-
-  async search(options: Omit<HybridSearchOptions, 'dbPath' | 'config'>) {
-    const workspace = this.ensureWorkspace();
-    return hybridSearch({
-      ...options,
-      dbPath: workspace.knowledgeDbPath,
-      config: this.config(),
-    });
-  }
-
-  async retrieveContext(options: Omit<RetrievalOptions, 'dbPath' | 'config'>) {
-    const workspace = this.ensureWorkspace();
-    return retrieveKnowledgeContext({
-      ...options,
-      dbPath: workspace.knowledgeDbPath,
-      config: this.config(),
-    });
-  }
-
-  async runPrompt(options: Omit<KnowledgePromptOptions, 'dbPath' | 'config'>) {
-    const workspace = this.ensureWorkspace();
-    return runKnowledgePrompt({
-      ...options,
-      dbPath: workspace.knowledgeDbPath,
-      config: this.config(),
-    });
-  }
-
-  async webSearch(options: Omit<WebSearchOptions, 'dbPath' | 'config' | 'safetyPolicy'>) {
-    const workspace = this.ensureWorkspace();
-    return runProviderWebSearch({
-      ...options,
-      dbPath: workspace.knowledgeDbPath,
-      config: this.config(),
-      safetyPolicy: this.safetyPolicy(),
-    });
-  }
-}
-
-export function createKnowledgeService(options: KnowledgeServiceOptions = {}): KnowledgeService {
-  return new KnowledgeService(options);
-}

package/src/source-ingest.ts

@@ -1,268 +0,0 @@
-import { createHash } from 'node:crypto';
-import { existsSync, readFileSync } from 'node:fs';
-import { basename } from 'node:path';
-import { ingestOpenFilesManifestItems, type ManifestIngestResult, type ManifestObject } from './manifest-ingest';
-import { parseSourceRef, type SourceRef } from './source-ref';
-import { resolveOpenFilesSource } from './source-resolver';
-import type { KnowledgeConfig } from './workspace';
-import { assertS3ReadAllowed, assertWebSearchAllowed, type SafetyPolicy } from './safety';
-
-export interface SourceIngestOptions {
-  dbPath: string;
-  sourceRef: string;
-  purpose?: string;
-  config?: KnowledgeConfig;
-  safetyPolicy?: SafetyPolicy;
-  now?: Date;
-}
-
-export interface SourceIngestResult extends ManifestIngestResult {
-  source_ref: string;
-  content_source: 'catalog_chunks' | 'extracted_text_ref' | 'file' | 's3' | 'web';
-  read_only: true;
-  hash: string;
-}
-
-interface ResolvedText {
-  text: string;
-  contentSource: SourceIngestResult['content_source'];
-  title: string | null;
-  mime: string | null;
-  size: number | null;
-  hash: string | null;
-  revision: string | null;
-  extractedTextRef: string | null;
-  metadata: Record<string, unknown>;
-  permissions: Record<string, unknown>;
-}
-
-function sha256Text(text: string): string {
-  return `sha256:${createHash('sha256').update(text).digest('hex')}`;
-}
-
-function stripHtml(html: string): string {
-  return html
-    .replace(/<script[\s\S]*?<\/script>/gi, ' ')
-    .replace(/<style[\s\S]*?<\/style>/gi, ' ')
-    .replace(/<[^>]+>/g, ' ')
-    .replace(/&nbsp;/g, ' ')
-    .replace(/&amp;/g, '&')
-    .replace(/&lt;/g, '<')
-    .replace(/&gt;/g, '>')
-    .replace(/\s+\n/g, '\n')
-    .replace(/\n\s+/g, '\n')
-    .replace(/[ \t]{2,}/g, ' ')
-    .trim();
-}
-
-async function readS3Text(uri: string, config?: KnowledgeConfig, safetyPolicy?: SafetyPolicy): Promise<string> {
-  const parsed = new URL(uri);
-  const bucket = parsed.hostname;
-  const key = decodeURIComponent(parsed.pathname.replace(/^\/+/, ''));
-  if (!bucket || !key) throw new Error(`Invalid S3 source URI: ${uri}`);
-  if (safetyPolicy) assertS3ReadAllowed(uri, safetyPolicy);
-  const [{ S3Client, GetObjectCommand }, { fromIni }] = await Promise.all([
-    import('@aws-sdk/client-s3'),
-    import('@aws-sdk/credential-providers'),
-  ]);
-  const s3Config = config?.storage.type === 's3' && config.storage.s3?.bucket === bucket ? config.storage.s3 : undefined;
-  const client = new S3Client({
-    region: s3Config?.region,
-    credentials: s3Config?.profile ? fromIni({ profile: s3Config.profile }) : undefined,
-    maxAttempts: s3Config?.max_attempts,
-  });
-  const response = await client.send(new GetObjectCommand({ Bucket: bucket, Key: key }));
-  if (!response.Body) return '';
-  return await response.Body.transformToString();
-}
-
-async function readWebText(uri: string, safetyPolicy?: SafetyPolicy): Promise<{ text: string; mime: string | null }> {
-  if (safetyPolicy) assertWebSearchAllowed(safetyPolicy);
-  const response = await fetch(uri, {
-    headers: {
-      accept: 'text/markdown,text/plain,text/html,application/json;q=0.8,*/*;q=0.5',
-      'user-agent': '@hasna/knowledge source-ingest',
-    },
-  });
-  if (!response.ok) throw new Error(`Web source read failed ${response.status}: ${uri}`);
-  const mime = response.headers.get('content-type');
-  const body = await response.text();
-  return { text: mime?.includes('html') ? stripHtml(body) : body, mime };
-}
-
-function titleForRef(parsed: SourceRef): string | null {
-  if (parsed.kind === 'file') return basename(parsed.path);
-  if (parsed.kind === 's3') return basename(parsed.key);
-  if (parsed.kind === 'web') return basename(new URL(parsed.url).pathname) || parsed.url;
-  return parsed.path ? basename(parsed.path) : parsed.id;
-}
-
-async function readDirectSourceText(parsed: SourceRef, config?: KnowledgeConfig, safetyPolicy?: SafetyPolicy): Promise<ResolvedText> {
-  if (parsed.kind === 'file') {
-    if (!existsSync(parsed.path)) throw new Error(`Source file not found: ${parsed.path}`);
-    const text = readFileSync(parsed.path, 'utf8');
-    return {
-      text,
-      contentSource: 'file',
-      title: titleForRef(parsed),
-      mime: 'text/plain',
-      size: text.length,
-      hash: sha256Text(text),
-      revision: null,
-      extractedTextRef: null,
-      metadata: { path: parsed.path },
-      permissions: { mode: 'read_only' },
-    };
-  }
-
-  if (parsed.kind === 's3') {
-    const text = await readS3Text(parsed.uri, config, safetyPolicy);
-    return {
-      text,
-      contentSource: 's3',
-      title: titleForRef(parsed),
-      mime: 'text/plain',
-      size: text.length,
-      hash: sha256Text(text),
-      revision: null,
-      extractedTextRef: null,
-      metadata: { bucket: parsed.bucket, key: parsed.key },
-      permissions: { mode: 'read_only' },
-    };
-  }
-
-  if (parsed.kind === 'web') {
-    const web = await readWebText(parsed.url, safetyPolicy);
-    return {
-      text: web.text,
-      contentSource: 'web',
-      title: titleForRef(parsed),
-      mime: web.mime,
-      size: web.text.length,
-      hash: sha256Text(web.text),
-      revision: null,
-      extractedTextRef: null,
-      metadata: { url: parsed.url },
-      permissions: { mode: 'read_only' },
-    };
-  }
-
-  throw new Error(`Direct source reading is not available for ${parsed.uri}`);
-}
-
-async function readTextRef(uri: string, config?: KnowledgeConfig, safetyPolicy?: SafetyPolicy): Promise<{ text: string; contentSource: SourceIngestResult['content_source'] }> {
-  if (uri.startsWith('open-files://')) {
-    throw new Error('Open-files extracted text refs require an open-files resolver API. Ingest an open-files manifest with extracted_text or an extracted_text_ref using file://, s3://, or https://.');
-  }
-  const parsed = parseSourceRef(uri);
-  const direct = await readDirectSourceText(parsed, config, safetyPolicy);
-  return { text: direct.text, contentSource: 'extracted_text_ref' };
-}
-
-async function readOpenFilesSourceText(options: SourceIngestOptions): Promise<ResolvedText> {
-  const resolved = await resolveOpenFilesSource({
-    dbPath: options.dbPath,
-    sourceRef: options.sourceRef,
-    purpose: options.purpose ?? 'knowledge_index',
-    limit: 100,
-    safetyPolicy: options.safetyPolicy,
-    now: options.now,
-  });
-  if (!resolved.resolved) {
-    throw new Error('Open-files source is not in the local knowledge catalog. Ingest an open-files manifest first or use the open-files resolver API.');
-  }
-  if (resolved.revision?.extracted_text_uri && !resolved.content.text_available) {
-    const textRef = await readTextRef(resolved.revision.extracted_text_uri, options.config, options.safetyPolicy);
-    return {
-      text: textRef.text,
-      contentSource: textRef.contentSource,
-      title: resolved.source?.title ?? null,
-      mime: resolved.content.mime,
-      size: textRef.text.length,
-      hash: resolved.revision.hash ?? sha256Text(textRef.text),
-      revision: resolved.revision.revision,
-      extractedTextRef: resolved.revision.extracted_text_uri,
-      metadata: resolved.source?.metadata ?? {},
-      permissions: resolved.source?.permissions ?? { mode: 'read_only' },
-    };
-  }
-  if (resolved.chunks.length === 0) {
-    throw new Error('Open-files source has no extracted text chunks yet. Ingest an open-files manifest with extracted_text or extracted_text_ref first.');
-  }
-  const text = resolved.chunks.map((chunk) => chunk.text).join('\n\n');
-  return {
-    text,
-    contentSource: 'catalog_chunks',
-    title: resolved.source?.title ?? null,
-    mime: resolved.content.mime,
-    size: text.length,
-    hash: resolved.revision?.hash ?? sha256Text(text),
-    revision: resolved.revision?.revision ?? null,
-    extractedTextRef: resolved.revision?.extracted_text_uri ?? null,
-    metadata: resolved.source?.metadata ?? {},
-    permissions: resolved.source?.permissions ?? { mode: 'read_only' },
-  };
-}
-
-function manifestItemForSource(sourceRef: string, parsed: SourceRef, resolved: ResolvedText, purpose: string): ManifestObject {
-  const hash = resolved.hash ?? sha256Text(resolved.text);
-  const metadata = {
-    ...resolved.metadata,
-    source_ref: sourceRef,
-    content_source: resolved.contentSource,
-    read_only: true,
-  };
-  const item: ManifestObject = {
-    source_ref: sourceRef,
-    name: resolved.title ?? titleForRef(parsed),
-    mime: resolved.mime ?? 'text/plain',
-    size: resolved.size ?? resolved.text.length,
-    hash,
-    revision: resolved.revision ?? hash,
-    status: 'active',
-    updated_at: new Date().toISOString(),
-    permissions: {
-      mode: 'read_only',
-      allowed_purposes: [purpose],
-      ...resolved.permissions,
-    },
-    metadata,
-    extracted_text_ref: resolved.extractedTextRef,
-    extracted_text: resolved.text,
-  };
-  if (parsed.kind === 'open-files') {
-    if (parsed.entity === 'file') item.file_id = parsed.id;
-    if (parsed.entity === 'source') {
-      item.source_id = parsed.id;
-      item.path = parsed.path;
-    }
-  }
-  if (parsed.kind === 'file') item.path = parsed.path;
-  if (parsed.kind === 's3') item.path = parsed.key;
-  if (parsed.kind === 'web') item.url = parsed.url;
-  return item;
-}
-
-export async function ingestSourceRef(options: SourceIngestOptions): Promise<SourceIngestResult> {
-  const purpose = options.purpose ?? 'knowledge_index';
-  const parsed = parseSourceRef(options.sourceRef);
-  const resolved = parsed.kind === 'open-files'
-    ? await readOpenFilesSourceText(options)
-    : await readDirectSourceText(parsed, options.config, options.safetyPolicy);
-  const item = manifestItemForSource(options.sourceRef, parsed, resolved, purpose);
-  const result = await ingestOpenFilesManifestItems({
-    dbPath: options.dbPath,
-    items: [item],
-    sourceLabel: options.sourceRef,
-    readAction: 'source_ref_ingest_read',
-    safetyPolicy: options.safetyPolicy,
-    now: options.now,
-  });
-  return {
-    ...result,
-    source_ref: options.sourceRef,
-    content_source: resolved.contentSource,
-    read_only: true,
-    hash: String(item.hash),
-  };
-}