ethagent 3.3.3 → 4.0.0

package/src/models/huggingface.ts

@@ -1,712 +0,0 @@
-import fs from 'node:fs/promises'
-import path from 'node:path'
-import { createHash } from 'node:crypto'
-import {
-  findLocalHfModel,
-  getLocalHfCacheDir,
-  getLocalHfModelsPath,
-  loadLocalHfModels,
-  localPathFor,
-  saveLocalHfModels,
-  uninstallLocalHfModel,
-  upsertLocalHfModel,
-} from './huggingfaceStorage.js'
-
-export {
-  findLocalHfModel,
-  getLocalHfCacheDir,
-  getLocalHfModelsPath,
-  loadLocalHfModels,
-  saveLocalHfModels,
-  uninstallLocalHfModel,
-  upsertLocalHfModel,
-} from './huggingfaceStorage.js'
-
-export type HfFileFormat = 'gguf' | 'safetensors' | 'pickle/bin' | 'unknown'
-export type HfRuntime = 'llama.cpp runnable' | 'download-only' | 'unsupported'
-export type HfTask = 'chat/instruct' | 'base' | 'code' | 'embedding' | 'vision' | 'unknown'
-export type HfSizeClass = 'tiny' | 'small' | 'medium' | 'large'
-export type HfRisk = 'low' | 'medium' | 'high'
-export type HfCredibility = 'established' | 'normal' | 'low-signal'
-export type LocalHfStatus = 'ready' | 'incomplete'
-
-export type HuggingFaceRef = {
-  repoId: string
-  revision?: string
-  filename?: string
-}
-
-export type HuggingFaceSibling = {
-  filename: string
-  sizeBytes?: number
-}
-
-export type HuggingFaceRepoInfo = {
-  repoId: string
-  author?: string
-  sha?: string
-  license?: string
-  downloads?: number
-  likes?: number
-  lastModified?: string
-  tags: string[]
-  siblings: HuggingFaceSibling[]
-}
-
-export type HuggingFaceModelSearchItem = {
-  repoId: string
-  downloads?: number
-  likes?: number
-  lastModified?: string
-  tags: string[]
-}
-
-export type HfSafetyReview = {
-  risk: HfRisk
-  credibility: HfCredibility
-  format: HfFileFormat
-  runtime: HfRuntime
-  task: HfTask
-  sizeClass: HfSizeClass
-  quantization?: string
-  reasons: string[]
-}
-
-export type HfMmprojCandidate = {
-  filename: string
-  sizeBytes: number
-  localPath: string
-}
-
-export type HfDownloadPlan = {
-  repo: HuggingFaceRepoInfo
-  repoId: string
-  requestedRevision: string
-  resolvedRevision: string
-  filename: string
-  sizeBytes: number
-  localPath: string
-  displayName: string
-  review: HfSafetyReview
-  mmprojCandidate?: HfMmprojCandidate
-  includeMmproj?: boolean
-}
-
-export type LocalHfModel = {
-  id: string
-  provider: 'llamacpp'
-  repoId: string
-  requestedRevision: string
-  resolvedRevision: string
-  filename: string
-  displayName: string
-  localPath: string
-  sizeBytes: number
-  format: HfFileFormat
-  runtime: HfRuntime
-  task: HfTask
-  sizeClass: HfSizeClass
-  quantization?: string
-  risk: HfRisk
-  credibility: HfCredibility
-  license?: string
-  downloads?: number
-  likes?: number
-  reviewedAt: string
-  installedAt: string
-  status: LocalHfStatus
-  sha256?: string
-  mmprojPath?: string
-  mmprojAvailable?: boolean
-  mmprojSizeBytes?: number
-}
-
-export type HfDownloadProgress = {
-  status: string
-  completed?: number
-  total?: number
-}
-
-type FetchImpl = typeof fetch
-type ModelInfoResponse = {
-  id?: unknown
-  author?: unknown
-  sha?: unknown
-  downloads?: unknown
-  likes?: unknown
-  lastModified?: unknown
-  tags?: unknown
-  cardData?: unknown
-  siblings?: unknown
-}
-
-type ModelSearchResponseItem = {
-  id?: unknown
-  modelId?: unknown
-  downloads?: unknown
-  likes?: unknown
-  lastModified?: unknown
-  tags?: unknown
-}
-
-const HF_BASE_URL = 'https://huggingface.co'
-const DEFAULT_REVISION = 'main'
-const COMMIT_RE = /^[a-f0-9]{40}$/i
-const DOWNLOAD_PROGRESS_MIN_MS = 100
-const DOWNLOAD_PROGRESS_MIN_BYTES = 16 * 1024 * 1024
-
-export function parseHuggingFaceRef(input: string): HuggingFaceRef {
-  const trimmed = input.trim()
-  if (!trimmed) throw new Error('Hugging Face model link or repo id is required')
-
-  if (/^https?:\/\//i.test(trimmed)) {
-    const url = new URL(trimmed)
-    const host = url.hostname.toLowerCase()
-    if (host !== 'huggingface.co' && host !== 'www.huggingface.co') {
-      throw new Error('Expected a huggingface.co model link')
-    }
-    const parts = url.pathname.split('/').filter(Boolean)
-    if (parts.length < 2) throw new Error('Expected a Hugging Face repo link')
-    const repoId = `${decodeURIComponent(parts[0]!)}/${decodeURIComponent(parts[1]!)}`
-    const mode = parts[2]
-    if (mode === 'blob' || mode === 'resolve' || mode === 'tree') {
-      const revision = parts[3] ? decodeURIComponent(parts[3]) : undefined
-      const filename = parts.length > 4
-        ? parts.slice(4).map(part => decodeURIComponent(part)).join('/')
-        : undefined
-      return { repoId, revision, filename }
-    }
-    return { repoId }
-  }
-
-  const withoutPrefix = trimmed.replace(/^hf:\/\//i, '')
-  const parts = withoutPrefix.split('/').filter(Boolean)
-  if (parts.length < 2) throw new Error('Expected repo id like org/model or a huggingface.co link')
-  const repoId = `${parts[0]!}/${parts[1]!}`
-  let fileParts = parts.slice(2)
-  const mode = fileParts[0]
-  if ((mode === 'blob' || mode === 'resolve' || mode === 'tree') && fileParts.length >= 2) {
-    fileParts = fileParts.slice(2)
-  }
-  const filename = fileParts.length > 0 ? fileParts.join('/') : undefined
-  return { repoId, filename }
-}
-
-export async function fetchHuggingFaceRepoInfo(
-  ref: HuggingFaceRef,
-  fetchImpl: FetchImpl = fetch,
-): Promise<HuggingFaceRepoInfo> {
-  const url = new URL(`${HF_BASE_URL}/api/models/${encodeRepoPath(ref.repoId)}`)
-  url.searchParams.set('blobs', 'true')
-  if (ref.revision) url.searchParams.set('revision', ref.revision)
-  const response = await fetchImpl(url, { headers: { Accept: 'application/json' } })
-  if (!response.ok) {
-    if (response.status === 401 || response.status === 403) {
-      throw new Error('Repo is gated or private')
-    }
-    if (response.status === 404) throw new Error('Hugging Face repo not found')
-    throw new Error(`Hugging Face API HTTP ${response.status}`)
-  }
-  const data = await response.json() as ModelInfoResponse
-  const tags = Array.isArray(data.tags)
-    ? data.tags.filter((tag): tag is string => typeof tag === 'string')
-    : []
-  const siblings = Array.isArray(data.siblings)
-    ? data.siblings.flatMap(sibling => parseSibling(sibling))
-    : []
-  return {
-    repoId: typeof data.id === 'string' ? data.id : ref.repoId,
-    author: typeof data.author === 'string' ? data.author : undefined,
-    sha: typeof data.sha === 'string' ? data.sha : undefined,
-    license: licenseFrom(data.cardData, tags),
-    downloads: typeof data.downloads === 'number' ? data.downloads : undefined,
-    likes: typeof data.likes === 'number' ? data.likes : undefined,
-    lastModified: typeof data.lastModified === 'string' ? data.lastModified : undefined,
-    tags,
-    siblings,
-  }
-}
-
-export async function searchHuggingFaceModels(
-  query: string,
-  options: { limit?: number; filter?: string } = {},
-  fetchImpl: FetchImpl = fetch,
-): Promise<HuggingFaceModelSearchItem[]> {
-  const url = new URL(`${HF_BASE_URL}/api/models`)
-  url.searchParams.set('search', query)
-  url.searchParams.set('sort', 'downloads')
-  url.searchParams.set('direction', '-1')
-  url.searchParams.set('limit', String(options.limit ?? 20))
-  if (options.filter) url.searchParams.set('filter', options.filter)
-  const response = await fetchImpl(url, { headers: { Accept: 'application/json' } })
-  if (!response.ok) throw new Error(`Hugging Face catalog HTTP ${response.status}`)
-  const data = await response.json() as unknown
-  if (!Array.isArray(data)) return []
-  return data.flatMap(parseSearchItem)
-}
-
-export function ggufFiles(repo: HuggingFaceRepoInfo): HuggingFaceSibling[] {
-  return repo.siblings
-    .filter(file => file.filename.toLowerCase().endsWith('.gguf'))
-    .sort((a, b) => a.filename.localeCompare(b.filename))
-}
-
-export function isMmprojFilename(filename: string): boolean {
-  return filename.toLowerCase().startsWith('mmproj-') && filename.toLowerCase().endsWith('.gguf')
-}
-
-export function findMmprojSibling(repo: HuggingFaceRepoInfo): HuggingFaceSibling | undefined {
-  return repo.siblings.find(file => isMmprojFilename(file.filename))
-}
-
-export async function createHfDownloadPlan(
-  input: string,
-  filename?: string,
-  deps: { fetchImpl?: FetchImpl; now?: () => Date } = {},
-): Promise<HfDownloadPlan> {
-  const ref = parseHuggingFaceRef(input)
-  const selectedFilename = filename?.trim() || ref.filename
-  const repo = await fetchHuggingFaceRepoInfo(ref, deps.fetchImpl)
-  const files = ggufFiles(repo)
-  if (files.length === 0) {
-    throw new Error('No compatible local model files found for this link')
-  }
-  const selected = selectedFilename
-    ? files.find(file => file.filename === selectedFilename)
-    : files[0]
-  if (!selected) {
-    throw new Error(`compatible file not found: ${selectedFilename}`)
-  }
-
-  const requestedRevision = ref.revision ?? DEFAULT_REVISION
-  const resolvedRevision = repo.sha || requestedRevision
-  const sizeBytes = selected.sizeBytes ?? 0
-  const review = reviewHfModel({
-    repo,
-    filename: selected.filename,
-    sizeBytes,
-    requestedRevision,
-    resolvedRevision,
-  })
-  const mmprojSibling = findMmprojSibling(repo)
-  const mmprojCandidate: HfMmprojCandidate | undefined = mmprojSibling
-    ? {
-      filename: mmprojSibling.filename,
-      sizeBytes: mmprojSibling.sizeBytes ?? 0,
-      localPath: localPathFor(repo.repoId, resolvedRevision, mmprojSibling.filename),
-    }
-    : undefined
-  return {
-    repo,
-    repoId: repo.repoId,
-    requestedRevision,
-    resolvedRevision,
-    filename: selected.filename,
-    sizeBytes,
-    localPath: localPathFor(repo.repoId, resolvedRevision, selected.filename),
-    displayName: displayNameFor(repo.repoId, selected.filename),
-    review,
-    mmprojCandidate,
-  }
-}
-
-export function reviewHfModel(args: {
-  repo: HuggingFaceRepoInfo
-  filename: string
-  sizeBytes: number
-  requestedRevision: string
-  resolvedRevision: string
-}): HfSafetyReview {
-  const format = fileFormat(args.filename)
-  const runtime: HfRuntime =
-    format === 'gguf' ? 'llama.cpp runnable'
-      : format === 'safetensors' ? 'download-only'
-        : 'unsupported'
-  const quantization = quantizationFromFilename(args.filename)
-  const task = taskFor(args.repo, args.filename)
-  const sizeClass = sizeClassFor(args.sizeBytes)
-  const credibility = credibilityFor(args.repo)
-  const pinned = COMMIT_RE.test(args.requestedRevision) || COMMIT_RE.test(args.resolvedRevision)
-  const repoHasRiskyFiles = args.repo.siblings.some(file => fileFormat(file.filename) === 'pickle/bin')
-  const reasons: string[] = []
-
-  if (format !== 'gguf') reasons.push('selected file is not compatible with local chat')
-  if (!pinned) reasons.push('revision is mutable')
-  if (!args.repo.license) reasons.push('license is missing or unknown')
-  if (credibility === 'low-signal') reasons.push('repo has limited public usage signals')
-  if (repoHasRiskyFiles) reasons.push('repo also contains pickle/bin model files')
-
-  const risk: HfRisk =
-    format !== 'gguf'
-      ? 'high'
-      : repoHasRiskyFiles
-        ? 'medium'
-        : pinned && args.repo.license && credibility !== 'low-signal'
-          ? 'low'
-          : 'medium'
-
-  if (reasons.length === 0) reasons.push('compatible local model file with usable repo metadata')
-
-  return {
-    risk,
-    credibility,
-    format,
-    runtime,
-    task,
-    sizeClass,
-    quantization,
-    reasons,
-  }
-}
-
-export async function* downloadHfModel(
-  plan: HfDownloadPlan,
-  signal?: AbortSignal,
-  fetchImpl: FetchImpl = fetch,
-): AsyncIterable<HfDownloadProgress> {
-  if (plan.review.runtime !== 'llama.cpp runnable') {
-    throw new Error('Selected file is not compatible with local chat')
-  }
-
-  await fs.mkdir(path.dirname(plan.localPath), { recursive: true })
-  const partialPath = `${plan.localPath}.partial`
-  const response = await fetchImpl(resolveUrl(plan.repoId, plan.resolvedRevision, plan.filename), { signal })
-  if (!response.ok || !response.body) {
-    throw new Error(response.ok ? 'empty download body' : `download HTTP ${response.status}`)
-  }
-
-  const total = Number.parseInt(response.headers.get('content-length') ?? '', 10)
-  const hash = createHash('sha256')
-  const handle = await fs.open(partialPath, 'w')
-  let completed = 0
-  let complete = false
-  let lastProgressAt = Date.now()
-  let lastProgressBytes = 0
-  yield { status: 'starting', completed, total: Number.isFinite(total) ? total : undefined }
-  try {
-    const reader = response.body.getReader()
-    while (true) {
-      const { done, value } = await reader.read()
-      if (done) break
-      if (signal?.aborted) throw new Error('Cancelled')
-      const buffer = Buffer.from(value)
-      hash.update(buffer)
-      await handle.write(buffer)
-      completed += buffer.byteLength
-      const now = Date.now()
-      if (shouldReportDownloadProgress(completed, lastProgressBytes, now, lastProgressAt)) {
-        lastProgressAt = now
-        lastProgressBytes = completed
-        yield { status: 'downloading', completed, total: Number.isFinite(total) ? total : undefined }
-      }
-    }
-    complete = true
-  } finally {
-    await handle.close()
-    if (!complete) {
-      await fs.unlink(partialPath).catch(() => {})
-    }
-  }
-
-  await fs.rename(partialPath, plan.localPath)
-
-  let mmprojPath: string | undefined
-  if (plan.includeMmproj && plan.mmprojCandidate) {
-    yield* downloadMmprojFile(plan.repoId, plan.resolvedRevision, plan.mmprojCandidate, signal, fetchImpl)
-    mmprojPath = plan.mmprojCandidate.localPath
-  }
-
-  await upsertLocalHfModel(modelFromPlan(plan, hash.digest('hex'), 'ready', mmprojPath))
-  yield { status: 'success', completed, total: Number.isFinite(total) ? total : completed }
-}
-
-async function* downloadMmprojFile(
-  repoId: string,
-  resolvedRevision: string,
-  candidate: HfMmprojCandidate,
-  signal: AbortSignal | undefined,
-  fetchImpl: FetchImpl,
-): AsyncIterable<HfDownloadProgress> {
-  await fs.mkdir(path.dirname(candidate.localPath), { recursive: true })
-  const partialPath = `${candidate.localPath}.partial`
-  const response = await fetchImpl(resolveUrl(repoId, resolvedRevision, candidate.filename), { signal })
-  if (!response.ok || !response.body) {
-    throw new Error(response.ok ? 'empty projector download body' : `projector download HTTP ${response.status}`)
-  }
-
-  const total = Number.parseInt(response.headers.get('content-length') ?? '', 10)
-  const handle = await fs.open(partialPath, 'w')
-  let completed = 0
-  let complete = false
-  let lastProgressAt = Date.now()
-  let lastProgressBytes = 0
-  yield { status: 'downloading-mmproj', completed, total: Number.isFinite(total) ? total : undefined }
-  try {
-    const reader = response.body.getReader()
-    while (true) {
-      const { done, value } = await reader.read()
-      if (done) break
-      if (signal?.aborted) throw new Error('Cancelled')
-      const buffer = Buffer.from(value)
-      await handle.write(buffer)
-      completed += buffer.byteLength
-      const now = Date.now()
-      if (shouldReportDownloadProgress(completed, lastProgressBytes, now, lastProgressAt)) {
-        lastProgressAt = now
-        lastProgressBytes = completed
-        yield { status: 'downloading-mmproj', completed, total: Number.isFinite(total) ? total : undefined }
-      }
-    }
-    complete = true
-  } finally {
-    await handle.close()
-    if (!complete) {
-      await fs.unlink(partialPath).catch(() => {})
-    }
-  }
-
-  await fs.rename(partialPath, candidate.localPath)
-}
-
-export async function backfillMmprojAvailability(
-  model: LocalHfModel,
-  fetchImpl: FetchImpl = fetch,
-): Promise<LocalHfModel> {
-  if (model.mmprojAvailable !== undefined) return model
-  try {
-    const repo = await fetchHuggingFaceRepoInfo({ repoId: model.repoId }, fetchImpl)
-    const sibling = findMmprojSibling(repo)
-    const next: LocalHfModel = {
-      ...model,
-      mmprojAvailable: Boolean(sibling),
-      mmprojSizeBytes: sibling?.sizeBytes,
-    }
-    await upsertLocalHfModel(next)
-    return next
-  } catch {
-    return model
-  }
-}
-
-export async function backfillMmprojForModels(
-  models: LocalHfModel[],
-  fetchImpl: FetchImpl = fetch,
-): Promise<LocalHfModel[]> {
-  const repoIdToProbe = new Map<string, Promise<HuggingFaceRepoInfo | null>>()
-  for (const model of models) {
-    if (model.mmprojAvailable !== undefined) continue
-    if (repoIdToProbe.has(model.repoId)) continue
-    repoIdToProbe.set(
-      model.repoId,
-      fetchHuggingFaceRepoInfo({ repoId: model.repoId }, fetchImpl).catch(() => null),
-    )
-  }
-  if (repoIdToProbe.size === 0) return models
-  const resolved = new Map<string, HuggingFaceRepoInfo | null>()
-  for (const [repoId, promise] of repoIdToProbe) {
-    resolved.set(repoId, await promise)
-  }
-  const out: LocalHfModel[] = []
-  for (const model of models) {
-    if (model.mmprojAvailable !== undefined) {
-      out.push(model)
-      continue
-    }
-    const repo = resolved.get(model.repoId)
-    if (!repo) {
-      out.push(model)
-      continue
-    }
-    const sibling = findMmprojSibling(repo)
-    const next: LocalHfModel = {
-      ...model,
-      mmprojAvailable: Boolean(sibling),
-      mmprojSizeBytes: sibling?.sizeBytes,
-    }
-    await upsertLocalHfModel(next)
-    out.push(next)
-  }
-  return out
-}
-
-export async function* addMmprojToInstalledModel(
-  modelId: string,
-  signal?: AbortSignal,
-  deps: { fetchImpl?: FetchImpl } = {},
-): AsyncIterable<HfDownloadProgress> {
-  const fetchImpl = deps.fetchImpl ?? fetch
-  const existing = await findLocalHfModel(modelId)
-  if (!existing) throw new Error(`model not installed: ${modelId}`)
-  if (existing.mmprojPath) {
-    yield { status: 'success', completed: 0 }
-    return
-  }
-  const repo = await fetchHuggingFaceRepoInfo({ repoId: existing.repoId }, fetchImpl)
-  const sibling = findMmprojSibling(repo)
-  if (!sibling) throw new Error(`no vision encoder available for ${existing.repoId}`)
-  const candidate: HfMmprojCandidate = {
-    filename: sibling.filename,
-    sizeBytes: sibling.sizeBytes ?? 0,
-    localPath: localPathFor(existing.repoId, existing.resolvedRevision, sibling.filename),
-  }
-  yield* downloadMmprojFile(existing.repoId, existing.resolvedRevision, candidate, signal, fetchImpl)
-  await upsertLocalHfModel({ ...existing, mmprojPath: candidate.localPath })
-  yield { status: 'success', completed: candidate.sizeBytes }
-}
-
-export function shouldReportDownloadProgress(
-  completed: number,
-  lastCompleted: number,
-  nowMs: number,
-  lastReportedMs: number,
-): boolean {
-  return nowMs - lastReportedMs >= DOWNLOAD_PROGRESS_MIN_MS
-    || completed - lastCompleted >= DOWNLOAD_PROGRESS_MIN_BYTES
-}
-
-export function modelFromPlan(
-  plan: HfDownloadPlan,
-  sha256: string | undefined,
-  status: LocalHfStatus,
-  mmprojPath?: string,
-): LocalHfModel {
-  const mmprojAvailable = Boolean(plan.mmprojCandidate)
-  const now = new Date().toISOString()
-  return {
-    id: localModelId(plan.repoId, plan.filename),
-    provider: 'llamacpp',
-    repoId: plan.repoId,
-    requestedRevision: plan.requestedRevision,
-    resolvedRevision: plan.resolvedRevision,
-    filename: plan.filename,
-    displayName: plan.displayName,
-    localPath: plan.localPath,
-    sizeBytes: plan.sizeBytes,
-    format: plan.review.format,
-    runtime: plan.review.runtime,
-    task: plan.review.task,
-    sizeClass: plan.review.sizeClass,
-    quantization: plan.review.quantization,
-    risk: plan.review.risk,
-    credibility: plan.review.credibility,
-    license: plan.repo.license,
-    downloads: plan.repo.downloads,
-    likes: plan.repo.likes,
-    reviewedAt: now,
-    installedAt: now,
-    status,
-    sha256,
-    mmprojPath,
-    mmprojAvailable,
-    mmprojSizeBytes: plan.mmprojCandidate?.sizeBytes,
-  }
-}
-
-export function localModelId(repoId: string, filename: string): string {
-  return `${repoId}#${filename}`
-}
-
-export function displayNameFor(repoId: string, filename: string): string {
-  const basename = filename.split('/').pop() ?? filename
-  return `${repoId} / ${basename}`
-}
-
-export function fileFormat(filename: string): HfFileFormat {
-  const lower = filename.toLowerCase()
-  if (lower.endsWith('.gguf')) return 'gguf'
-  if (lower.endsWith('.safetensors')) return 'safetensors'
-  if (/\.(bin|pt|pth|pkl|pickle)$/.test(lower)) return 'pickle/bin'
-  return 'unknown'
-}
-
-export function quantizationFromFilename(filename: string): string | undefined {
-  const match = filename.toUpperCase().match(/(?:^|[-_.])((?:IQ|Q)\d(?:_[A-Z0-9]+)*|F16|BF16|F32)(?=$|[-_.])/)
-  return match?.[1]
-}
-
-function resolveUrl(repoId: string, revision: string, filename: string): string {
-  return `${HF_BASE_URL}/${encodeRepoPath(repoId)}/resolve/${encodeURIComponent(revision)}/${encodePath(filename)}?download=true`
-}
-
-function encodeRepoPath(repoId: string): string {
-  return repoId.split('/').map(part => encodeURIComponent(part)).join('/')
-}
-
-function encodePath(value: string): string {
-  return value.split('/').map(part => encodeURIComponent(part)).join('/')
-}
-
-function parseSibling(value: unknown): HuggingFaceSibling[] {
-  if (!value || typeof value !== 'object') return []
-  const record = value as { rfilename?: unknown; filename?: unknown; size?: unknown; lfs?: unknown }
-  const filename = typeof record.rfilename === 'string'
-    ? record.rfilename
-    : typeof record.filename === 'string' ? record.filename : ''
-  if (!filename) return []
-  return [{
-    filename,
-    sizeBytes: siblingSizeBytes(record),
-  }]
-}
-
-function parseSearchItem(value: unknown): HuggingFaceModelSearchItem[] {
-  if (!value || typeof value !== 'object' || Array.isArray(value)) return []
-  const item = value as ModelSearchResponseItem
-  const repoId = typeof item.id === 'string'
-    ? item.id
-    : typeof item.modelId === 'string' ? item.modelId : ''
-  if (!repoId.includes('/')) return []
-  return [{
-    repoId,
-    downloads: typeof item.downloads === 'number' ? item.downloads : undefined,
-    likes: typeof item.likes === 'number' ? item.likes : undefined,
-    lastModified: typeof item.lastModified === 'string' ? item.lastModified : undefined,
-    tags: Array.isArray(item.tags)
-      ? item.tags.filter((tag): tag is string => typeof tag === 'string')
-      : [],
-  }]
-}
-
-function siblingSizeBytes(record: { size?: unknown; lfs?: unknown }): number | undefined {
-  if (typeof record.size === 'number' && Number.isFinite(record.size) && record.size >= 0) {
-    return record.size
-  }
-  if (!record.lfs || typeof record.lfs !== 'object' || Array.isArray(record.lfs)) return undefined
-  const size = (record.lfs as { size?: unknown }).size
-  return typeof size === 'number' && Number.isFinite(size) && size >= 0 ? size : undefined
-}
-
-function licenseFrom(cardData: unknown, tags: string[]): string | undefined {
-  if (cardData && typeof cardData === 'object' && !Array.isArray(cardData)) {
-    const license = (cardData as { license?: unknown }).license
-    if (typeof license === 'string' && license.trim()) return license.trim()
-  }
-  const tag = tags.find(item => item.startsWith('license:'))
-  return tag ? tag.slice('license:'.length) : undefined
-}
-
-function credibilityFor(repo: HuggingFaceRepoInfo): HfCredibility {
-  const downloads = repo.downloads ?? 0
-  const likes = repo.likes ?? 0
-  if (downloads >= 10_000 || likes >= 100) return 'established'
-  if (downloads >= 100 || likes >= 5 || Boolean(repo.license)) return 'normal'
-  return 'low-signal'
-}
-
-function taskFor(repo: HuggingFaceRepoInfo, filename: string): HfTask {
-  const haystack = [repo.repoId, filename, ...repo.tags].join(' ').toLowerCase()
-  if (/(embed|embedding)/.test(haystack)) return 'embedding'
-  if (/(vision|vlm|multimodal)/.test(haystack)) return 'vision'
-  if (/(code|coder|coding)/.test(haystack)) return 'code'
-  if (/(chat|instruct|assistant)/.test(haystack)) return 'chat/instruct'
-  if (/(base)/.test(haystack)) return 'base'
-  return 'unknown'
-}
-
-function sizeClassFor(sizeBytes: number): HfSizeClass {
-  const gb = sizeBytes / 1e9
-  if (gb < 2) return 'tiny'
-  if (gb < 8) return 'small'
-  if (gb < 24) return 'medium'
-  return 'large'
-}