@hasna/knowledge 0.2.27 → 0.2.28

package/src/providers.ts

@@ -1,308 +0,0 @@
-import { randomUUID } from 'node:crypto';
-import type { Database } from 'bun:sqlite';
-import type { KnowledgeConfig } from './workspace';
-
-export type AiProviderId = 'openai' | 'anthropic' | 'deepseek';
-
-export interface AiProviderSettings {
-  api_key_env: string;
-  base_url?: string;
-  default_model: string;
-}
-
-export interface AiProvidersConfig {
-  default_model?: string;
-  aliases?: Record<string, string>;
-  openai?: Partial<AiProviderSettings>;
-  anthropic?: Partial<AiProviderSettings>;
-  deepseek?: Partial<AiProviderSettings>;
-}
-
-export interface ModelCapabilities {
-  text_generation: boolean;
-  structured_output: boolean;
-  tool_usage: boolean;
-  tool_streaming: boolean;
-  image_input: boolean;
-  native_web_search: boolean;
-  reasoning: boolean;
-  embeddings: boolean;
-}
-
-export interface ModelRegistryEntry {
-  alias: string;
-  model_ref: string;
-  provider: AiProviderId;
-  model: string;
-  default: boolean;
-  capabilities: ModelCapabilities;
-}
-
-export interface ProviderCredentialStatus {
-  provider: AiProviderId;
-  api_key_env: string;
-  configured: boolean;
-  source: 'env' | 'missing';
-  base_url: string | null;
-  default_model: string;
-}
-
-export interface ProviderStatusResult {
-  default_model: string;
-  providers: ProviderCredentialStatus[];
-  models: ModelRegistryEntry[];
-}
-
-export interface NormalizedProviderUsage {
-  provider: string;
-  model: string;
-  input_tokens: number;
-  output_tokens: number;
-  cost_usd: number;
-  metadata: Record<string, unknown>;
-}
-
-type ProviderFactory = (settings: { apiKey: string; baseURL?: string }) => unknown;
-
-export interface AiProviderRuntimeOptions {
-  config?: KnowledgeConfig;
-  env?: Record<string, string | undefined>;
-  factories?: Partial<Record<AiProviderId, ProviderFactory>>;
-}
-
-const DEFAULT_PROVIDER_SETTINGS: Record<AiProviderId, AiProviderSettings> = {
-  openai: {
-    api_key_env: 'OPENAI_API_KEY',
-    default_model: 'gpt-5.2',
-  },
-  anthropic: {
-    api_key_env: 'ANTHROPIC_API_KEY',
-    default_model: 'claude-sonnet-4-6',
-  },
-  deepseek: {
-    api_key_env: 'DEEPSEEK_API_KEY',
-    default_model: 'deepseek-chat',
-  },
-};
-
-const PROVIDER_CAPABILITIES: Record<AiProviderId, ModelCapabilities> = {
-  openai: {
-    text_generation: true,
-    structured_output: true,
-    tool_usage: true,
-    tool_streaming: true,
-    image_input: true,
-    native_web_search: true,
-    reasoning: true,
-    embeddings: true,
-  },
-  anthropic: {
-    text_generation: true,
-    structured_output: true,
-    tool_usage: true,
-    tool_streaming: true,
-    image_input: true,
-    native_web_search: false,
-    reasoning: true,
-    embeddings: false,
-  },
-  deepseek: {
-    text_generation: true,
-    structured_output: true,
-    tool_usage: true,
-    tool_streaming: true,
-    image_input: false,
-    native_web_search: false,
-    reasoning: true,
-    embeddings: false,
-  },
-};
-
-const BUILTIN_ALIASES: Record<string, string> = {
-  default: 'openai:gpt-5.2',
-  fast: 'openai:gpt-5-mini',
-  reasoning: 'anthropic:claude-opus-4-6',
-  sonnet: 'anthropic:claude-sonnet-4-6',
-  deepseek: 'deepseek:deepseek-chat',
-  'deepseek-reasoning': 'deepseek:deepseek-reasoner',
-};
-
-function providerConfig(config?: KnowledgeConfig): AiProvidersConfig {
-  return (config as KnowledgeConfig & { providers?: AiProvidersConfig } | undefined)?.providers ?? {};
-}
-
-export function providerSettings(config: KnowledgeConfig | undefined, provider: AiProviderId): AiProviderSettings {
-  const configured = providerConfig(config)[provider] ?? {};
-  return {
-    ...DEFAULT_PROVIDER_SETTINGS[provider],
-    ...configured,
-  };
-}
-
-export function modelAliases(config?: KnowledgeConfig): Record<string, string> {
-  const configured = providerConfig(config);
-  return {
-    ...BUILTIN_ALIASES,
-    ...(configured.default_model ? { default: configured.default_model } : {}),
-    ...(configured.aliases ?? {}),
-  };
-}
-
-export function parseModelRef(modelRef: string): { provider: AiProviderId; model: string } {
-  const [provider, ...rest] = modelRef.split(':');
-  const model = rest.join(':');
-  if (provider !== 'openai' && provider !== 'anthropic' && provider !== 'deepseek') {
-    throw new Error(`Unsupported AI provider: ${provider}`);
-  }
-  if (!model) throw new Error(`Invalid model ref: ${modelRef}. Expected provider:model.`);
-  return { provider, model };
-}
-
-export function resolveModelRef(aliasOrRef: string, config?: KnowledgeConfig): string {
-  const aliases = modelAliases(config);
-  return aliases[aliasOrRef] ?? aliasOrRef;
-}
-
-export function listModelRegistry(config?: KnowledgeConfig): ModelRegistryEntry[] {
-  const aliases = modelAliases(config);
-  return Object.entries(aliases).map(([alias, modelRef]) => {
-    const parsed = parseModelRef(modelRef);
-    return {
-      alias,
-      model_ref: modelRef,
-      provider: parsed.provider,
-      model: parsed.model,
-      default: alias === 'default',
-      capabilities: PROVIDER_CAPABILITIES[parsed.provider],
-    };
-  });
-}
-
-export function providerCredentialStatus(config: KnowledgeConfig | undefined, env: Record<string, string | undefined> = process.env): ProviderCredentialStatus[] {
-  return (Object.keys(DEFAULT_PROVIDER_SETTINGS) as AiProviderId[]).map((provider) => {
-    const settings = providerSettings(config, provider);
-    const configured = Boolean(env[settings.api_key_env]);
-    return {
-      provider,
-      api_key_env: settings.api_key_env,
-      configured,
-      source: configured ? 'env' : 'missing',
-      base_url: settings.base_url ?? null,
-      default_model: settings.default_model,
-    };
-  });
-}
-
-export function providerStatus(config?: KnowledgeConfig, env: Record<string, string | undefined> = process.env): ProviderStatusResult {
-  return {
-    default_model: resolveModelRef('default', config),
-    providers: providerCredentialStatus(config, env),
-    models: listModelRegistry(config),
-  };
-}
-
-export function assertProviderCredentials(provider: AiProviderId, config?: KnowledgeConfig, env: Record<string, string | undefined> = process.env): ProviderCredentialStatus {
-  const status = providerCredentialStatus(config, env).find((entry) => entry.provider === provider);
-  if (!status) throw new Error(`Unsupported AI provider: ${provider}`);
-  if (!status.configured) throw new Error(`Missing ${status.api_key_env} for ${provider}. Set the env var to use this provider.`);
-  return status;
-}
-
-async function defaultFactory(provider: AiProviderId): Promise<ProviderFactory> {
-  if (provider === 'openai') {
-    const { createOpenAI } = await import('@ai-sdk/openai');
-    return createOpenAI as ProviderFactory;
-  }
-  if (provider === 'anthropic') {
-    const { createAnthropic } = await import('@ai-sdk/anthropic');
-    return createAnthropic as ProviderFactory;
-  }
-  const { createDeepSeek } = await import('@ai-sdk/deepseek');
-  return createDeepSeek as ProviderFactory;
-}
-
-export async function createAiSdkProviderRegistry(options: AiProviderRuntimeOptions = {}) {
-  const { createProviderRegistry } = await import('ai');
-  const env = options.env ?? process.env;
-  const providers: Record<string, unknown> = {};
-  for (const provider of Object.keys(DEFAULT_PROVIDER_SETTINGS) as AiProviderId[]) {
-    const settings = providerSettings(options.config, provider);
-    const apiKey = env[settings.api_key_env];
-    if (!apiKey) continue;
-    const factory = options.factories?.[provider] ?? await defaultFactory(provider);
-    providers[provider] = factory({ apiKey, baseURL: settings.base_url });
-  }
-  return createProviderRegistry(providers as never);
-}
-
-export async function languageModelFor(aliasOrRef: string, options: AiProviderRuntimeOptions = {}) {
-  const modelRef = resolveModelRef(aliasOrRef, options.config);
-  const parsed = parseModelRef(modelRef);
-  assertProviderCredentials(parsed.provider, options.config, options.env);
-  const registry = await createAiSdkProviderRegistry(options);
-  return registry.languageModel(modelRef as `${string}:${string}`);
-}
-
-function usageNumber(usage: Record<string, unknown>, keys: string[]): number {
-  for (const key of keys) {
-    const value = usage[key];
-    if (typeof value === 'number' && Number.isFinite(value)) return value;
-  }
-  return 0;
-}
-
-export function normalizeAiSdkUsage(input: {
-  provider: string;
-  model: string;
-  usage?: Record<string, unknown> | null;
-  providerMetadata?: Record<string, unknown> | null;
-  costUsd?: number;
-}): NormalizedProviderUsage {
-  const usage = input.usage ?? {};
-  return {
-    provider: input.provider,
-    model: input.model,
-    input_tokens: usageNumber(usage, ['inputTokens', 'promptTokens', 'input_tokens', 'prompt_tokens']),
-    output_tokens: usageNumber(usage, ['outputTokens', 'completionTokens', 'output_tokens', 'completion_tokens']),
-    cost_usd: input.costUsd ?? 0,
-    metadata: {
-      usage,
-      provider_metadata: input.providerMetadata ?? {},
-    },
-  };
-}
-
-export function recordProviderUsage(db: Database, input: NormalizedProviderUsage & { run_id?: string | null; created_at?: string }): string {
-  const id = `usage_${randomUUID()}`;
-  db.run(
-    `INSERT INTO provider_usage (id, run_id, provider, model, input_tokens, output_tokens, cost_usd, metadata_json, created_at)
-     VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
-    [
-      id,
-      input.run_id ?? null,
-      input.provider,
-      input.model,
-      input.input_tokens,
-      input.output_tokens,
-      input.cost_usd,
-      JSON.stringify(input.metadata),
-      input.created_at ?? new Date().toISOString(),
-    ],
-  );
-  return id;
-}
-
-export function createDeterministicFakeProvider(provider: AiProviderId): ProviderFactory {
-  return () => ({
-    languageModel: (modelId: string) => ({
-      provider,
-      modelId,
-      specificationVersion: 'v3',
-    }),
-    chat: (modelId: string) => ({
-      provider,
-      modelId,
-      specificationVersion: 'v3',
-    }),
-  });
-}

package/src/reindex.ts

@@ -1,260 +0,0 @@
-import { createHash, randomUUID } from 'node:crypto';
-import { indexKnowledgeEmbeddings, resolveEmbeddingModelRef, type EmbeddingRuntimeOptions } from './embeddings';
-import { migrateKnowledgeDb, openKnowledgeDb } from './knowledge-db';
-import { parseModelRef } from './providers';
-import type { KnowledgeConfig } from './workspace';
-
-export interface ReindexRuntimeOptions extends EmbeddingRuntimeOptions {
-  dbPath: string;
-  config?: KnowledgeConfig;
-  now?: Date;
-}
-
-export interface ReindexHealthResult {
-  schema_version: number;
-  chunks: number;
-  vector_entries: number;
-  missing_embeddings: number;
-  queued: Record<string, number>;
-  stale_revisions: number;
-}
-
-export interface ReindexEnqueueResult {
-  enqueued: number;
-  already_queued: number;
-  reason: string;
-}
-
-export interface ReindexEmbeddingsResult {
-  run_id: string;
-  full: boolean;
-  deleted_embeddings: number;
-  deleted_vector_entries: number;
-  queued: ReindexEnqueueResult;
-  indexed: Awaited<ReturnType<typeof indexKnowledgeEmbeddings>>;
-  completed_queue_items: number;
-}
-
-interface MissingChunkRow {
-  chunk_id: string;
-  source_revision_id: string | null;
-  source_uri: string | null;
-}
-
-function stableId(prefix: string, value: string): string {
-  return `${prefix}_${createHash('sha256').update(value).digest('hex').slice(0, 20)}`;
-}
-
-function queueCounts(dbPath: string): Record<string, number> {
-  const db = openKnowledgeDb(dbPath);
-  try {
-    const rows = db.query<{ status: string; n: number }, []>(
-      `SELECT status, COUNT(*) AS n FROM reindex_queue GROUP BY status ORDER BY status`,
-    ).all();
-    return Object.fromEntries(rows.map((row) => [row.status, row.n]));
-  } finally {
-    db.close();
-  }
-}
-
-function missingEmbeddingRows(dbPath: string, options: ReindexRuntimeOptions): MissingChunkRow[] {
-  const modelRef = resolveEmbeddingModelRef(options.modelRef, options.config);
-  const parsed = parseModelRef(modelRef);
-  const db = openKnowledgeDb(dbPath);
-  try {
-    return db.query<MissingChunkRow, [string, string]>(
-      `SELECT c.id AS chunk_id, c.source_revision_id, s.uri AS source_uri
-       FROM chunks c
-       LEFT JOIN source_revisions sr ON sr.id = c.source_revision_id
-       LEFT JOIN sources s ON s.id = sr.source_id
-       LEFT JOIN vector_index_entries v ON v.chunk_id = c.id AND v.provider = ? AND v.model = ?
-       WHERE v.id IS NULL
-       ORDER BY c.created_at ASC, c.ordinal ASC`,
-    ).all(parsed.provider, parsed.model);
-  } finally {
-    db.close();
-  }
-}
-
-export function reindexHealth(options: ReindexRuntimeOptions): ReindexHealthResult {
-  migrateKnowledgeDb(options.dbPath);
-  const db = openKnowledgeDb(options.dbPath);
-  try {
-    const version = db.query<{ version: number }, []>('SELECT MAX(version) AS version FROM schema_versions').get()?.version ?? 0;
-    const chunks = db.query<{ n: number }, []>('SELECT COUNT(*) AS n FROM chunks').get()?.n ?? 0;
-    const vectorEntries = db.query<{ n: number }, []>('SELECT COUNT(*) AS n FROM vector_index_entries').get()?.n ?? 0;
-    const missing = missingEmbeddingRows(options.dbPath, options).length;
-    const stale = db.query<{ n: number }, []>(
-      `SELECT COUNT(*) AS n FROM source_revisions
-       WHERE metadata_json LIKE '%"reindex_required":true%' OR metadata_json LIKE '%"status":"stale"%'`,
-    ).get()?.n ?? 0;
-    return {
-      schema_version: version,
-      chunks,
-      vector_entries: vectorEntries,
-      missing_embeddings: missing,
-      queued: queueCounts(options.dbPath),
-      stale_revisions: stale,
-    };
-  } finally {
-    db.close();
-  }
-}
-
-export function enqueueMissingEmbeddings(options: ReindexRuntimeOptions & { reason?: string }): ReindexEnqueueResult {
-  migrateKnowledgeDb(options.dbPath);
-  const now = (options.now ?? new Date()).toISOString();
-  const reason = options.reason ?? 'missing_embedding';
-  const rows = missingEmbeddingRows(options.dbPath, options);
-  const db = openKnowledgeDb(options.dbPath);
-  let enqueued = 0;
-  let alreadyQueued = 0;
-  try {
-    const write = db.transaction(() => {
-      for (const row of rows) {
-        const id = stableId('rq', `embedding\u0000${row.chunk_id}\u0000${reason}`);
-        const before = db.query<{ id: string }, [string, string, string]>(
-          'SELECT id FROM reindex_queue WHERE kind = ? AND target_id = ? AND reason = ?',
-        ).get('embedding', row.chunk_id, reason);
-        if (before) {
-          alreadyQueued += 1;
-          continue;
-        }
-        db.run(
-          `INSERT INTO reindex_queue (id, kind, target_id, source_uri, reason, status, metadata_json, created_at, updated_at)
-           VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
-          [
-            id,
-            'embedding',
-            row.chunk_id,
-            row.source_uri,
-            reason,
-            'pending',
-            JSON.stringify({ source_revision_id: row.source_revision_id }),
-            now,
-            now,
-          ],
-        );
-        enqueued += 1;
-      }
-    });
-    write();
-  } finally {
-    db.close();
-  }
-  return { enqueued, already_queued: alreadyQueued, reason };
-}
-
-function clearEmbeddingIndex(dbPath: string): { embeddings: number; vectorEntries: number } {
-  const db = openKnowledgeDb(dbPath);
-  try {
-    const embeddings = db.query<{ n: number }, []>('SELECT COUNT(*) AS n FROM chunk_embeddings').get()?.n ?? 0;
-    const vectorEntries = db.query<{ n: number }, []>('SELECT COUNT(*) AS n FROM vector_index_entries').get()?.n ?? 0;
-    db.run('DELETE FROM vector_index_entries');
-    db.run('DELETE FROM chunk_embeddings');
-    return { embeddings, vectorEntries };
-  } finally {
-    db.close();
-  }
-}
-
-function completeIndexedQueueItems(dbPath: string, options: ReindexRuntimeOptions, now: string): number {
-  const modelRef = resolveEmbeddingModelRef(options.modelRef, options.config);
-  const parsed = parseModelRef(modelRef);
-  const db = openKnowledgeDb(dbPath);
-  try {
-    const result = db.run(
-      `UPDATE reindex_queue
-       SET status = ?, updated_at = ?
-       WHERE kind = ?
-         AND status = ?
-         AND EXISTS (
-           SELECT 1 FROM vector_index_entries v
-           WHERE v.chunk_id = reindex_queue.target_id
-             AND v.provider = ?
-             AND v.model = ?
-         )`,
-      ['completed', now, 'embedding', 'pending', parsed.provider, parsed.model],
-    );
-    return result.changes;
-  } finally {
-    db.close();
-  }
-}
-
-export async function refreshEmbeddingIndex(options: ReindexRuntimeOptions & { full?: boolean; limit?: number }): Promise<ReindexEmbeddingsResult> {
-  migrateKnowledgeDb(options.dbPath);
-  const now = (options.now ?? new Date()).toISOString();
-  const runId = `run_${randomUUID()}`;
-  const deleted = options.full ? clearEmbeddingIndex(options.dbPath) : { embeddings: 0, vectorEntries: 0 };
-  const queued = enqueueMissingEmbeddings({ ...options, reason: options.full ? 'full_embedding_rebuild' : 'missing_embedding' });
-  const db = openKnowledgeDb(options.dbPath);
-  try {
-    db.run(
-      `INSERT INTO runs (id, type, prompt, status, provider, model, metadata_json, created_at, updated_at)
-       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
-      [
-        runId,
-        'embedding-refresh',
-        options.full ? 'full' : 'incremental',
-        'running',
-        'local',
-        resolveEmbeddingModelRef(options.modelRef, options.config),
-        JSON.stringify({ full: options.full === true, queued }),
-        now,
-        now,
-      ],
-    );
-  } finally {
-    db.close();
-  }
-
-  const indexed = await indexKnowledgeEmbeddings({
-    dbPath: options.dbPath,
-    config: options.config,
-    env: options.env,
-    modelRef: options.modelRef,
-    dimensions: options.dimensions,
-    fake: options.fake,
-    limit: options.limit,
-    now: options.now,
-  });
-
-  const completedQueueItems = completeIndexedQueueItems(options.dbPath, options, now);
-  const doneDb = openKnowledgeDb(options.dbPath);
-  try {
-    doneDb.run(
-      `UPDATE runs SET status = ?, metadata_json = ?, updated_at = ? WHERE id = ?`,
-      [
-        'completed',
-        JSON.stringify({ full: options.full === true, queued, indexed, completed_queue_items: completedQueueItems }),
-        now,
-        runId,
-      ],
-    );
-    doneDb.run(
-      `INSERT INTO run_events (id, run_id, level, event, metadata_json, created_at)
-       VALUES (?, ?, ?, ?, ?, ?)`,
-      [
-        `evt_${randomUUID()}`,
-        runId,
-        'info',
-        'embedding_refresh_completed',
-        JSON.stringify({ queued, indexed, completed_queue_items: completedQueueItems }),
-        now,
-      ],
-    );
-  } finally {
-    doneDb.close();
-  }
-
-  return {
-    run_id: runId,
-    full: options.full === true,
-    deleted_embeddings: deleted.embeddings,
-    deleted_vector_entries: deleted.vectorEntries,
-    queued,
-    indexed,
-    completed_queue_items: completedQueueItems,
-  };
-}