@inneranimalmedia/agentsam-sdk 2.6.2 → 2.6.4

package/src/indexing/ingest/discover-models.js

@@ -0,0 +1,298 @@
+/**
+ * Discover embedding models from the user's configured credentials.
+ *
+ * Authority order for the picker:
+ *   1. Credential-scoped cloud providers (OpenAI, Gemini, Cloudflare Workers AI)
+ *   2. Optional local Ollama inventory (offer only — never required, never SSOT)
+ *   3. Always "none" (AST/text only)
+ *
+ * Dimensions are profile fingerprints for the chosen provider/model — not "Ollama truth."
+ * Regex/name heuristics are hints until a real embed/adapter reports length.
+ */
+
+import { collectModelsStatus } from '../../commands/models.js';
+import { resolveOllamaConfig, probeOllama, probeOllamaModel } from '../../commands/ollama.js';
+import { resolveProviderCredential } from '../../lib/provider-credentials.js';
+import { createProviderRegistry } from '../../../packages/agentsam-knowledge/src/providers/index.js';
+
+function clean(value) {
+  return value == null ? '' : String(value).trim();
+}
+
+/** Known Workers AI / Vectorize-friendly embed dims (hint until first embed probes). */
+const WORKERS_AI_EMBED_DIMS = Object.freeze({
+  '@cf/baai/bge-small-en-v1.5': 384,
+  '@cf/baai/bge-base-en-v1.5': 768,
+  '@cf/baai/bge-large-en-v1.5': 1024,
+  '@cf/baai/bge-m3': 1024,
+  '@cf/google/embeddinggemma-300m': 768,
+});
+
+function looksLikeEmbedModel(name) {
+  const id = clean(name).toLowerCase();
+  if (!id) return false;
+  return /embed|bge-|e5-|gte-|minilm|mxbai-embed|nomic-embed|snowflake-arctic-embed/i.test(id);
+}
+
+/**
+ * Hint dimensions for a provider/model. Not authoritative — adapters may probe.
+ */
+function defaultDimensions(provider, model) {
+  const id = clean(model).toLowerCase();
+  if (provider === 'openai') {
+    if (id.includes('large')) return 3072;
+    if (id.includes('small')) return 1536;
+    return 1536;
+  }
+  if (provider === 'gemini') {
+    return 768;
+  }
+  if (provider === 'workers-ai' || provider === 'cloudflare') {
+    if (WORKERS_AI_EMBED_DIMS[clean(model)]) return WORKERS_AI_EMBED_DIMS[clean(model)];
+    if (id.includes('small')) return 384;
+    if (id.includes('large') || id.includes('bge-m3')) return 1024;
+    return 768;
+  }
+  if (provider === 'ollama') {
+    if (id.includes('mxbai-embed-large')) return 1024;
+    if (id.includes('nomic-embed')) return 768;
+    return 1024;
+  }
+  return 768;
+}
+
+/**
+ * Encode a selectable embedding profile for clack / CLI.
+ * Format: provider|model|dimensions  (pipe — models may contain colons)
+ */
+export function encodeEmbeddingChoice(provider, model, dimensions) {
+  return `${provider}|${model}|${dimensions}`;
+}
+
+/**
+ * @param {string} value
+ */
+export function parseEmbeddingChoice(value) {
+  if (!value || value === 'none') {
+    return { provider: 'none', model: 'none', revision: '1', dimensions: 0, parameters: {} };
+  }
+  // New encoding: provider|model|dims
+  if (String(value).includes('|')) {
+    const [provider, model, dims] = String(value).split('|');
+    return {
+      provider: clean(provider),
+      model: clean(model),
+      revision: '1',
+      dimensions: Number(dims) || defaultDimensions(provider, model),
+      parameters: provider === 'gemini' ? { task: 'code retrieval' } : {},
+    };
+  }
+  // Legacy encoding used briefly: provider:model:dims (breaks on model ids with colons)
+  const parts = String(value).split(':');
+  if (parts.length >= 3) {
+    const provider = parts[0];
+    const dims = parts[parts.length - 1];
+    const model = parts.slice(1, -1).join(':');
+    return {
+      provider: clean(provider),
+      model: clean(model),
+      revision: '1',
+      dimensions: Number(dims) || defaultDimensions(provider, model),
+      parameters: provider === 'gemini' ? { task: 'code retrieval' } : {},
+    };
+  }
+  throw new Error(`invalid_embedding_choice:${value}`);
+}
+
+/**
+ * Build embedding select options from live discovery.
+ * Always offers "none" (AST/text only). Ollama entries appear only when online (optional).
+ *
+ * @param {object} [options]
+ * @returns {Promise<{ options: object[], inventory: object, assistModels: object[] }>}
+ */
+export async function discoverIngestModelOptions(options = {}) {
+  const env = options.env || process.env;
+  const status = await collectModelsStatus({
+    env,
+    home: options.home,
+    discoverRemote: options.discoverRemote !== false,
+    includeLocal: true,
+    // Keep CF embed models; chat allowlist must not hide Vectorize options.
+    curateWorkersAi: options.curateWorkersAi !== false,
+    fetchImpl: options.fetchImpl,
+    providerFetchImpl: options.providerFetchImpl,
+  });
+
+  /** @type {Map<string, object>} */
+  const byValue = new Map();
+  const push = (row) => {
+    if (!row?.value || byValue.has(row.value)) return;
+    byValue.set(row.value, row);
+  };
+
+  push({
+    value: 'none',
+    label: 'None — AST/text only ($0 embeddings)',
+    hint: 'deterministic · no provider spend',
+    provider: 'none',
+    source: 'builtin',
+  });
+
+  // Credential-scoped OpenAI embedding models from live /v1/models
+  for (const row of status.providerModels?.openai || []) {
+    const caps = row.capabilities || {};
+    const id = row.provider_model_id || '';
+    if (caps.embeddings === true || /^text-embedding-/i.test(id)) {
+      const dims = defaultDimensions('openai', id);
+      push({
+        value: encodeEmbeddingChoice('openai', id, dims),
+        label: `OpenAI · ${id}`,
+        hint: `credential · ${dims}d`,
+        provider: 'openai',
+        source: 'provider_api',
+        model: id,
+        dimensions: dims,
+      });
+    }
+  }
+
+  // Gemini: if key configured, offer models from knowledge adapter + any discovered embed ids
+  const geminiCred = resolveProviderCredential('gemini', { env, home: options.home });
+  if (geminiCred?.configured) {
+    const registry = createProviderRegistry({
+      gemini: { apiKey: geminiCred.value || env.GEMINI_API_KEY },
+    });
+    const caps = registry.get('gemini').capabilities();
+    for (const id of caps.models || []) {
+      const dims = defaultDimensions('gemini', id);
+      push({
+        value: encodeEmbeddingChoice('gemini', id, dims),
+        label: `Gemini · ${id}`,
+        hint: `credential · ${dims}d`,
+        provider: 'gemini',
+        source: 'provider_credential',
+        model: id,
+        dimensions: dims,
+      });
+    }
+  }
+
+  // Cloudflare Workers AI embedding models — required for Vectorize / CF lanes.
+  // Uses live account discovery (Text Embeddings). Provider id = workers-ai (knowledge adapter).
+  for (const row of status.providerModels?.cloudflare || []) {
+    const caps = row.capabilities || {};
+    const id = row.provider_model_id || '';
+    const task = String(row.metadata?.task || '').toLowerCase();
+    const isEmbed = caps.embeddings === true || task === 'text embeddings' || looksLikeEmbedModel(id);
+    if (!isEmbed || !id) continue;
+    const dims = defaultDimensions('workers-ai', id);
+    push({
+      value: encodeEmbeddingChoice('workers-ai', id, dims),
+      label: `Workers AI · ${id}`,
+      hint: `cloudflare · vectorize-ready · ${dims}d`,
+      provider: 'workers-ai',
+      source: 'provider_api',
+      model: id,
+      dimensions: dims,
+    });
+  }
+
+  // Optional local Ollama — never required; only offered when online.
+  /** @type {object[]} */
+  const assistModels = [];
+  if (status.local?.online) {
+    const ollamaConfig = resolveOllamaConfig({}, env);
+    for (const row of status.local.models || []) {
+      const name = clean(row.name || row.model);
+      if (!name) continue;
+      const probe = options.skipOllamaProbe
+        ? { ok: true, capabilities: [] }
+        : await probeOllamaModel(name, ollamaConfig, options.fetchImpl || fetch);
+      const caps = Array.isArray(probe.capabilities) ? probe.capabilities : [];
+      const isEmbed = looksLikeEmbedModel(name) || caps.includes('embedding') || caps.includes('embed');
+      if (isEmbed) {
+        // Prefer probed dim length when the adapter returns it; else name hint.
+        const probedDims = Number(probe.dimensions) || Number(probe.embedding_length) || 0;
+        const dims = probedDims > 0 ? probedDims : defaultDimensions('ollama', name);
+        push({
+          value: encodeEmbeddingChoice('ollama', name, dims),
+          label: `Ollama · ${name}`,
+          hint: `local optional · ${dims}d`,
+          provider: 'ollama',
+          source: 'ollama_tags',
+          model: name,
+          dimensions: dims,
+        });
+      } else {
+        assistModels.push({
+          value: `ollama|${name}`,
+          label: `Ollama · ${name}`,
+          hint: 'local assist (allowlist suggestions)',
+          provider: 'ollama',
+          model: name,
+          source: 'ollama_tags',
+        });
+      }
+    }
+  }
+
+  return {
+    options: [...byValue.values()],
+    inventory: status,
+    assistModels,
+    ollama: status.local,
+  };
+}
+
+/**
+ * Ask a local Ollama chat model to suggest include/exclude paths.
+ * Optional — never required for ingest.
+ *
+ * @param {{ root: string, model: string, topLevel: string[], fetchImpl?: typeof fetch, env?: object }} opts
+ */
+export async function suggestScopeWithLocalModel(opts) {
+  const env = opts.env || process.env;
+  const config = resolveOllamaConfig({}, env);
+  const endpoint = new URL('/api/generate', config.baseUrl);
+  const listing = (opts.topLevel || []).slice(0, 80).join('\n');
+  const prompt = [
+    'You help configure a repository knowledge allowlist/denylist for AgentSam codebaseindex.',
+    'Return ONLY compact JSON: {"include":["..."],"exclude":["..."],"rationale":"..."}',
+    'include/exclude must be relative path segments (no globs). Prefer source dirs; exclude deps/build caches.',
+    'Machine inventory already classified top-level paths — prefer refining those categories, do not invent unrelated roots.',
+    `Repository root: ${opts.root}`,
+    opts.categories ? `Categories JSON: ${JSON.stringify(opts.categories)}` : '',
+    'Top-level entries:',
+    listing || '(empty)',
+  ].filter(Boolean).join('\n');
+
+  const response = await (opts.fetchImpl || fetch)(endpoint, {
+    method: 'POST',
+    headers: { 'content-type': 'application/json' },
+    body: JSON.stringify({ model: opts.model, prompt, stream: false, format: 'json' }),
+    signal: AbortSignal.timeout(60_000),
+  });
+  if (!response.ok) {
+    throw new Error(`ollama_assist_failed:HTTP ${response.status}`);
+  }
+  const body = await response.json();
+  const raw = clean(body?.response);
+  let parsed;
+  try {
+    parsed = JSON.parse(raw);
+  } catch {
+    throw new Error('ollama_assist_invalid_json');
+  }
+  const include = Array.isArray(parsed.include) ? parsed.include.map(clean).filter(Boolean) : [];
+  const exclude = Array.isArray(parsed.exclude) ? parsed.exclude.map(clean).filter(Boolean) : [];
+  return {
+    include,
+    exclude,
+    rationale: clean(parsed.rationale) || null,
+    model: opts.model,
+    provider: 'ollama',
+  };
+}
+
+export { probeOllama, resolveOllamaConfig, looksLikeEmbedModel, defaultDimensions, WORKERS_AI_EMBED_DIMS };

package/src/indexing/ingest/inventory.js

@@ -0,0 +1,243 @@
+/**
+ * Inventory projection for codebaseindex — structured authority + compact ASCII view.
+ * The ASCII tree is a projection, not the source of truth.
+ *
+ * Scope suggestions categorize paths; they never auto-apply exclusions.
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+
+const EXT_LANG = new Map([
+  ['.js', 'JavaScript'], ['.mjs', 'JavaScript'], ['.cjs', 'JavaScript'],
+  ['.ts', 'TypeScript'], ['.tsx', 'TypeScript'], ['.jsx', 'JavaScript'],
+  ['.py', 'Python'], ['.go', 'Go'], ['.rs', 'Rust'], ['.md', 'Markdown'],
+  ['.json', 'JSON'], ['.css', 'CSS'], ['.html', 'HTML'], ['.htm', 'HTML'],
+  ['.yml', 'YAML'], ['.yaml', 'YAML'], ['.sql', 'SQL'], ['.sh', 'Shell'],
+]);
+
+const IMAGE = new Set(['.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg', '.ico']);
+const MODEL3D = new Set(['.glb', '.gltf', '.obj', '.fbx', '.stl']);
+const ARCHIVE = new Set(['.zip', '.tar', '.tgz', '.gz']);
+
+/** Usually exclude — dependency / build caches (candidates only). */
+const CATEGORY_DEPENDENCIES = new Set([
+  'node_modules', '.git', 'dist', 'build', '.next', 'coverage', '.turbo',
+  '.cache', 'out', 'target', '__pycache__', '.venv', 'venv', '.wrangler',
+]);
+
+/** Primary source candidates. */
+const CATEGORY_SOURCE = new Set([
+  'src', 'apps', 'packages', 'protocol', 'python', 'services', 'lib', 'server',
+  'worker', 'workers', 'runtime', 'cmd', 'internal', 'pkg',
+]);
+
+/** Documentation / examples. */
+const CATEGORY_DOCS = new Set([
+  'docs', 'examples', 'guides', 'handbook', 'README', 'changelog',
+]);
+
+/** Tooling / CI — review before including. */
+const CATEGORY_TOOLING = new Set([
+  'bin', 'scripts', 'tools', '.github', '.changeset', '.husky', '.vscode',
+  '.cursor', 'tags', 'test', 'tests', '__tests__', 'spec', 'e2e',
+]);
+
+/** Generated / historical — review before including. */
+const CATEGORY_GENERATED = new Set([
+  'generated', 'fixtures', 'snapshots', 'artifacts', 'public', 'static',
+  'site', 'sites', 'templates', 'level-1', 'focus-timer',
+]);
+
+/** AgentSam / config control plane — review. */
+const CATEGORY_CONFIG = new Set([
+  '.agentsam', 'migrations', 'registry', 'wrangler', 'deploy',
+]);
+
+const SUGGEST_EXCLUDE_NAMES = CATEGORY_DEPENDENCIES;
+
+function classifyTopLevel(name) {
+  if (CATEGORY_DEPENDENCIES.has(name)) return 'dependencies';
+  if (CATEGORY_SOURCE.has(name)) return 'source';
+  if (CATEGORY_DOCS.has(name)) return 'docs';
+  if (CATEGORY_GENERATED.has(name)) return 'generated';
+  if (CATEGORY_CONFIG.has(name)) return 'config';
+  if (CATEGORY_TOOLING.has(name)) return 'tooling';
+  if (/^(old|legacy|archive|backup|tmp|temp)/i.test(name)) return 'historical';
+  if (name.startsWith('.')) return 'config';
+  return 'unknown';
+}
+
+function walk(root, rel = '', acc = [], depth = 0) {
+  if (depth > 6 || acc.length > 8000) return acc;
+  const abs = rel ? path.join(root, rel) : root;
+  let entries;
+  try { entries = fs.readdirSync(abs, { withFileTypes: true }); }
+  catch { return acc; }
+  for (const ent of entries) {
+    if (ent.name === '.git' || ent.name === 'node_modules') continue;
+    const child = rel ? `${rel}/${ent.name}` : ent.name;
+    const full = path.join(root, child);
+    if (ent.isDirectory()) {
+      acc.push({ path: child, kind: 'directory', name: ent.name });
+      if (!SUGGEST_EXCLUDE_NAMES.has(ent.name)) walk(root, child, acc, depth + 1);
+    } else if (ent.isFile()) {
+      let bytes = 0;
+      try { bytes = fs.statSync(full).size; } catch { /* ignore */ }
+      const ext = path.extname(ent.name).toLowerCase();
+      acc.push({
+        path: child,
+        kind: 'file',
+        name: ent.name,
+        ext,
+        bytes,
+        language: EXT_LANG.get(ext) || null,
+        media: IMAGE.has(ext) ? 'image' : MODEL3D.has(ext) ? 'model3d' : ARCHIVE.has(ext) ? 'archive' : null,
+      });
+    }
+  }
+  return acc;
+}
+
+function countLoc(root, filePath, maxBytes = 256_000) {
+  try {
+    const full = path.join(root, filePath);
+    const st = fs.statSync(full);
+    if (st.size > maxBytes) return 0;
+    const text = fs.readFileSync(full, 'utf8');
+    if (text.includes('\0')) return 0;
+    return text.split(/\r?\n/).length;
+  } catch {
+    return 0;
+  }
+}
+
+/**
+ * @param {{ root: string, materials?: object|null, maxFiles?: number }} opts
+ */
+export function buildInventory(opts) {
+  const root = path.resolve(opts.root);
+  const entries = walk(root);
+  const files = entries.filter((e) => e.kind === 'file');
+  const dirs = entries.filter((e) => e.kind === 'directory');
+  const top = dirs.filter((d) => !d.path.includes('/')).map((d) => d.name);
+  const languages = {};
+  let locTotal = 0;
+  for (const f of files.slice(0, opts.maxFiles || 4000)) {
+    if (!f.language) continue;
+    const loc = countLoc(root, f.path);
+    languages[f.language] = (languages[f.language] || 0) + loc;
+    locTotal += loc;
+  }
+  const images = files.filter((f) => f.media === 'image').length;
+  const models3d = files.filter((f) => f.media === 'model3d').length;
+  const archives = files.filter((f) => f.media === 'archive').length;
+  const packageManifests = files.filter((f) => f.name === 'package.json').length;
+
+  /** @type {Record<string, string[]>} */
+  const categories = {
+    source: [],
+    docs: [],
+    config: [],
+    generated: [],
+    dependencies: [],
+    tooling: [],
+    historical: [],
+    unknown: [],
+  };
+  for (const name of top) {
+    categories[classifyTopLevel(name)].push(name);
+  }
+
+  const primaryInclude = [
+    ...categories.source,
+    ...categories.docs,
+  ];
+  const reviewInclude = [
+    ...categories.config,
+    ...categories.generated,
+    ...categories.tooling.filter((n) => n === 'test' || n === 'tests' || n === 'scripts'),
+  ];
+  const usuallyExclude = [
+    ...categories.dependencies,
+    ...categories.tooling.filter((n) => !reviewInclude.includes(n) && n !== 'scripts' && n !== 'test' && n !== 'tests'),
+    ...categories.historical,
+  ];
+
+  // Default starting include: primary source + docs (not "everything").
+  // Exclusions are candidates only — never auto-applied without user confirm.
+  const suggestedInclude = (primaryInclude.length ? primaryInclude : top.filter((n) => !CATEGORY_DEPENDENCIES.has(n))).slice(0, 24);
+  if (!suggestedInclude.length) suggestedInclude.push('.');
+  const suggestedExclude = [...new Set(usuallyExclude)];
+
+  /** @type {object} */
+  const inventory = {
+    schema: 'agentsam.inventory.v1',
+    root,
+    scanned_at: new Date().toISOString(),
+    counts: {
+      files: files.length,
+      directories: dirs.length,
+      loc_sampled: locTotal,
+      images,
+      models3d,
+      archives,
+      package_manifests: packageManifests,
+    },
+    languages,
+    top_level: top,
+    categories,
+    suggested: {
+      include: suggestedInclude,
+      exclude: suggestedExclude,
+      review: reviewInclude,
+      note: 'Categories are machine inventory. Suggestions are advisory — no exclusions applied automatically. User confirmations are authoritative.',
+    },
+    materials: opts.materials || null,
+    sample_paths: files.slice(0, 40).map((f) => f.path),
+  };
+  return inventory;
+}
+
+/**
+ * Compact human projection of inventory (not authority).
+ * @param {ReturnType<typeof buildInventory>} inventory
+ */
+export function formatInventoryTree(inventory) {
+  const lines = [];
+  lines.push(`repository  ${inventory.root}`);
+  lines.push(`files ${inventory.counts.files} · dirs ${inventory.counts.directories} · LOC~${inventory.counts.loc_sampled}`);
+  lines.push('');
+  for (const name of inventory.top_level.slice(0, 24)) {
+    const cat = inventory.categories
+      ? Object.entries(inventory.categories).find(([, names]) => names.includes(name))?.[0]
+      : null;
+    const mark = cat ? ` · ${cat}` : '';
+    lines.push(`├── ${name}/${mark}`);
+  }
+  if (inventory.top_level.length > 24) lines.push('└── …');
+  lines.push('');
+  lines.push('Languages');
+  const langs = Object.entries(inventory.languages || {}).sort((a, b) => b[1] - a[1]);
+  for (const [lang, loc] of langs.slice(0, 8)) {
+    lines.push(`  ${lang.padEnd(14)} ${String(loc).padStart(8)} LOC`);
+  }
+  lines.push('');
+  lines.push('Detected');
+  lines.push(`  ${inventory.counts.package_manifests} package.json`);
+  lines.push(`  ${inventory.counts.images} images · ${inventory.counts.models3d} 3D · ${inventory.counts.archives} archives`);
+  lines.push('');
+  lines.push('Recommended scope candidates');
+  lines.push(`  Primary source     ${(inventory.categories?.source || []).join(', ') || '(none)'}`);
+  lines.push(`  Documentation      ${(inventory.categories?.docs || []).join(', ') || '(none)'}`);
+  lines.push(`  Review before incl ${(inventory.suggested?.review || []).join(', ') || '(none)'}`);
+  lines.push(`  Usually exclude    ${(inventory.suggested?.exclude || []).join(', ') || '(none)'}`);
+  lines.push('  (No exclusions applied automatically.)');
+  lines.push('');
+  lines.push('Starting suggestion (editable)');
+  lines.push(`  include: ${(inventory.suggested.include || []).join(', ') || '(none)'}`);
+  lines.push(`  exclude: ${(inventory.suggested.exclude || []).join(', ') || '(none)'}`);
+  return lines.join('\n');
+}
+
+export { classifyTopLevel };