dsh-plugin-discover 0.0.0-stage → 1.0.0

package/lib/normalize.js

@@ -0,0 +1,369 @@
+/**
+ * Turn the two directory sources into one card per installable target.
+ *
+ * dsh.so wins every field it supplies; the catalog fills title, repository,
+ * `engines.dsh`, publish time and npm versions. Ecosystem apps without an
+ * install command are not installable targets and are left out.
+ */
+import { createHash } from 'node:crypto'
+import { qualityPrior } from './quality.js'
+
+export const DSH_SO_URL = 'https://www.dsh.so/plugins-index.json'
+export const CATALOG_URL = 'https://cdn.jsdelivr.net/npm/dsh-plugin-catalog/index.json'
+export const ATTRIBUTION = 'Powered by dsh.so — DSH Developer Hub'
+
+const NPM_TARGET_RE = /^(@[a-z0-9][a-z0-9._-]*\/)?[a-z0-9][a-z0-9._-]*(@[a-z0-9][a-z0-9._+-]*)?$/i
+const GITHUB_TARGET_RE = /^github:[a-z0-9][a-z0-9._-]*\/[a-z0-9._-]+(#([^\s;|&$`<>()*?\\]+))?$/iu
+const INSTALL_RE = /^dsh\s+plugin\s+--profile\s+[a-z0-9._-]+\s+add\s+(.+)$/i
+
+function asString(value) {
+  return typeof value === 'string' ? value.trim() : ''
+}
+
+function asNumber(value) {
+  return typeof value === 'number' && Number.isFinite(value) ? value : null
+}
+
+function asStringArray(value, limit = 32) {
+  if (!Array.isArray(value)) return []
+  const seen = new Set()
+  for (const item of value) {
+    if (typeof item !== 'string') continue
+    const text = item.trim().toLowerCase()
+    if (text === '' || seen.has(text)) continue
+    seen.add(text)
+    if (seen.size >= limit) break
+  }
+  return [...seen]
+}
+
+/** Normalize a repository URL so `git+https`, `.git`, case, and trailing slash compare equal. */
+export function normalizeRepo(url) {
+  if (typeof url !== 'string') return ''
+  let value = url.trim().toLowerCase()
+  value = value.replace(/^git\+/, '').replace(/^git@([^:]+):/, '$1/')
+  value = value.replace(/^[a-z][a-z0-9+.-]*:\/\//, '')
+  value = value.replace(/^www\./, '')
+  value = value.split(/[?#]/)[0]
+  value = value.replace(/\.git$/, '').replace(/\/+$/, '')
+  return value
+}
+
+/** Whether a target is a plain npm name or a GitHub spec, never a shell fragment. */
+export function isSafeInstallTarget(target) {
+  if (typeof target !== 'string') return false
+  const text = target.trim()
+  if (text === '' || text.length > 200) return false
+  return NPM_TARGET_RE.test(text) || GITHUB_TARGET_RE.test(text)
+}
+
+/** Parse the dsh.so `install` command into its exact target. */
+export function parseInstallTarget(install) {
+  if (typeof install !== 'string') return null
+  const match = INSTALL_RE.exec(install.trim())
+  if (!match) return null
+  const target = match[1].trim()
+  if (!isSafeInstallTarget(target)) return null
+  return { target }
+}
+
+/** Package name inside an npm or GitHub target (version and ref removed). */
+export function packageNameOf(target) {
+  if (typeof target !== 'string') return ''
+  if (target.toLowerCase().startsWith('github:')) return target.split('#')[0]
+  if (target.startsWith('@')) {
+    const slash = target.indexOf('/')
+    if (slash === -1) return target
+    const at = target.indexOf('@', slash)
+    return at === -1 ? target : target.slice(0, at)
+  }
+  const at = target.indexOf('@')
+  return at === -1 ? target : target.slice(0, at)
+}
+
+function latestCatalogVersion(entry) {
+  const versions = Array.isArray(entry?.versions) ? entry.versions : []
+  let best = null
+  for (const version of versions) {
+    if (!version || typeof version !== 'object') continue
+    const time = asNumber(version.publishTimestamp) ?? -1
+    if (!best || time > best.time) best = { value: version, time }
+  }
+  return best?.value ?? null
+}
+
+function catalogText(value) {
+  if (typeof value === 'string') return value.trim()
+  if (value && typeof value === 'object') {
+    return asString(value.en) || asString(value.zh) || ''
+  }
+  return ''
+}
+
+function catalogTopics(entry) {
+  const metadata = entry?.metadata && typeof entry.metadata === 'object' ? entry.metadata : {}
+  const values = []
+  if (typeof metadata.category === 'string') values.push(metadata.category)
+  if (Array.isArray(metadata.keywords)) values.push(...metadata.keywords)
+  if (Array.isArray(metadata.labels)) values.push(...metadata.labels)
+  return asStringArray(values, 16)
+}
+
+function fitsVerification(value) {
+  if (!value || typeof value !== 'object') return null
+  const level = asNumber(value.level)
+  if (level === null || level < 1 || level > 5) return null
+  return {
+    level,
+    label: asString(value.label) || `L${level}`,
+    lastVerifiedAt: asString(value.lastVerifiedAt) || null,
+  }
+}
+
+function fitsSecurity(value) {
+  if (!value || typeof value !== 'object') return null
+  const status = asString(value.status) || 'unknown'
+  const riskLevel = asString(value.riskLevel) || 'unknown'
+  const counts = value.counts && typeof value.counts === 'object'
+    ? {
+        critical: asNumber(value.counts.critical) ?? 0,
+        warning: asNumber(value.counts.warning) ?? 0,
+        info: asNumber(value.counts.info) ?? 0,
+      }
+    : null
+  return {
+    status,
+    riskLevel,
+    scannedAt: asString(value.scannedAt) || null,
+    ...(counts ? { counts } : {}),
+  }
+}
+
+function contentHash(record) {
+  const payload = JSON.stringify([
+    record.name,
+    record.title ?? '',
+    record.description ?? '',
+    record.topics,
+    record.capability,
+    record.install?.target ?? '',
+    record.repo ?? '',
+  ])
+  return createHash('sha256').update(payload).digest('hex').slice(0, 16)
+}
+
+function baseRecord(record) {
+  return {
+    id: record.id,
+    name: record.name,
+    alias: record.alias ?? '',
+    title: record.title ?? '',
+    description: record.description ?? '',
+    topics: record.topics ?? [],
+    capability: record.capability ?? [],
+    stars: record.stars ?? null,
+    verification: record.verification ?? null,
+    security: record.security ?? null,
+    repo: record.repo ?? '',
+    repoKey: record.repoKey ?? '',
+    url: record.url ?? '',
+    install: record.install ?? null,
+    dshRange: record.dshRange ?? null,
+    updatedAt: record.updatedAt ?? null,
+    kind: record.kind ?? 'plugin',
+    sources: record.sources ?? [],
+    aliases: record.aliases ?? [],
+    alternatives: [],
+  }
+}
+
+/**
+ * Build the card set.
+ * @param {unknown} dshSo parsed dsh.so index
+ * @param {unknown} catalog parsed format-1 catalog, or null when unavailable
+ */
+export function buildRecords(dshSo, catalog) {
+  if (!dshSo || typeof dshSo !== 'object' || !Array.isArray(dshSo.plugins)) {
+    throw new Error('dsh.so index is missing the plugins array')
+  }
+  const catalogPlugins = catalog && typeof catalog === 'object' && Array.isArray(catalog.plugins)
+    ? catalog.plugins
+    : []
+  const catalogByRepo = new Map()
+  const catalogByName = new Map()
+  for (const entry of catalogPlugins) {
+    if (!entry || typeof entry !== 'object' || typeof entry.name !== 'string') continue
+    const repoKey = normalizeRepo(entry.repository?.url)
+    if (repoKey && !catalogByRepo.has(repoKey)) catalogByRepo.set(repoKey, entry)
+    const name = entry.name.trim().toLowerCase()
+    if (name && !catalogByName.has(name)) catalogByName.set(name, entry)
+  }
+
+  const usedCatalog = new Set()
+  const usedNames = new Map()
+  const records = []
+  let skippedNoInstall = 0
+  let skippedBadInstall = 0
+
+  for (const plugin of dshSo.plugins) {
+    if (!plugin || typeof plugin !== 'object') continue
+    const install = parseInstallTarget(plugin.install)
+    if (plugin.install == null || String(plugin.install).trim() === '') {
+      skippedNoInstall += 1
+      continue
+    }
+    if (!install) {
+      skippedBadInstall += 1
+      continue
+    }
+    const repo = asString(plugin.source)
+    const repoKey = normalizeRepo(repo)
+    const alias = asString(plugin.alias)
+    const name = asString(plugin.name) || alias || asString(plugin.id)
+    if (name === '') continue
+    const catalogEntry = catalogByRepo.get(repoKey)
+      ?? catalogByName.get(alias.toLowerCase())
+      ?? catalogByName.get(name.toLowerCase())
+    if (catalogEntry) usedCatalog.add(catalogEntry)
+
+    const catalogVersion = catalogEntry ? latestCatalogVersion(catalogEntry) : null
+    const engines = catalogVersion?.engines && typeof catalogVersion.engines === 'object'
+      ? asString(catalogVersion.engines.dsh) : ''
+    const publishedAt = asNumber(catalogVersion?.publishTimestamp)
+    const record = baseRecord({
+      id: asString(plugin.id) || name,
+      name,
+      alias,
+      title: catalogEntry ? asString(catalogEntry.title) : '',
+      description: asString(plugin.description),
+      topics: asStringArray(plugin.topics),
+      capability: asStringArray(plugin.capability?.samples),
+      stars: asNumber(plugin.stars) ?? 0,
+      verification: fitsVerification(plugin.verification),
+      security: fitsSecurity(plugin.security),
+      repo,
+      repoKey,
+      url: asString(plugin.url) || asString(catalogEntry?.homepage) || repo,
+      install,
+      dshRange: engines || null,
+      updatedAt: publishedAt && publishedAt > 0 ? new Date(publishedAt * 1000).toISOString() : null,
+      kind: asString(plugin.kind) || 'plugin',
+      sources: catalogEntry ? ['dsh.so', 'dsh-plugin-catalog'] : ['dsh.so'],
+      aliases: [...new Set([name, alias].filter(Boolean))],
+    })
+    records.push(record)
+  }
+
+  const dshSoInstallable = records.length
+
+  for (const entry of catalogPlugins) {
+    if (usedCatalog.has(entry)) continue
+    if (!entry || typeof entry !== 'object') continue
+    const name = asString(entry.name)
+    if (!name) continue
+    const repo = asString(entry.repository?.url)
+    const repoKey = normalizeRepo(repo)
+    if (repoKey && catalogByRepo.has(repoKey) && usedCatalog.has(catalogByRepo.get(repoKey))) continue
+    if (catalogByName.has(name.toLowerCase()) && usedCatalog.has(catalogByName.get(name.toLowerCase()))) continue
+    const version = latestCatalogVersion(entry)
+    const versionText = asString(version?.version)
+    const target = versionText ? `${name}@${versionText}` : name
+    if (!isSafeInstallTarget(target)) continue
+    const engines = version?.engines && typeof version.engines === 'object' ? asString(version.engines.dsh) : ''
+    const publishedAt = asNumber(version?.publishTimestamp)
+    const record = baseRecord({
+      id: `catalog:${name}`,
+      name,
+      alias: name,
+      title: asString(entry.title),
+      description: catalogText(entry.description),
+      topics: catalogTopics(entry),
+      capability: [],
+      stars: null,
+      verification: null,
+      security: null,
+      repo,
+      repoKey,
+      url: asString(entry.homepage) || repo,
+      install: { target },
+      dshRange: engines || null,
+      updatedAt: publishedAt && publishedAt > 0 ? new Date(publishedAt * 1000).toISOString() : null,
+      kind: 'catalog',
+      sources: ['dsh-plugin-catalog'],
+      aliases: [name],
+    })
+    records.push(record)
+  }
+
+  const groups = new Map()
+  for (const record of records) {
+    const key = record.repoKey || `name:${record.name.toLowerCase()}`
+    const group = groups.get(key)
+    if (group) group.push(record)
+    else groups.set(key, [record])
+  }
+
+  const cards = []
+  let mergedCards = 0
+  for (const group of groups.values()) {
+    group.sort((a, b) => qualityPrior(b) - qualityPrior(a) || b.stars - a.stars)
+    const main = group[0]
+    if (group.length > 1) {
+      mergedCards += 1
+      main.alternatives = group.slice(1)
+        .map(item => ({ name: item.name, target: item.install?.target ?? '' }))
+        .filter(item => item.target !== '' && item.name !== main.name)
+      main.topics = [...new Set(group.flatMap(item => item.topics))]
+      main.capability = [...new Set(group.flatMap(item => item.capability))]
+      main.aliases = [...new Set(group.flatMap(item => item.aliases))]
+      const numericStars = group
+        .map(item => item.stars)
+        .filter(stars => typeof stars === 'number' && Number.isFinite(stars))
+      main.stars = numericStars.length > 0 ? Math.max(...numericStars) : null
+      const timestamps = group.map(item => item.updatedAt).filter(Boolean).sort()
+      main.updatedAt = timestamps.at(-1) ?? main.updatedAt
+      main.dshRange = main.dshRange ?? group.find(item => item.dshRange)?.dshRange ?? null
+    }
+    if (usedNames.has(main.id)) {
+      const next = usedNames.get(main.id) + 1
+      usedNames.set(main.id, next)
+      main.id = `${main.id}#${next}`
+    } else {
+      usedNames.set(main.id, 0)
+    }
+    main.contentHash = contentHash(main)
+    cards.push(main)
+  }
+  cards.sort((a, b) => a.id.localeCompare(b.id))
+
+  const generatedAt = asString(dshSo.generatedAt) || asString(dshSo.generated) || null
+  const catalogGeneratedAt = asString(catalog?.generatedAt) || asString(catalog?.metadata?.generatedAt) || null
+  return {
+    records: cards,
+    stats: {
+      dshSoTotal: dshSo.plugins.length,
+      dshSoInstallable,
+      skippedNoInstall,
+      skippedBadInstall,
+      catalogTotal: catalogPlugins.length,
+      catalogMerged: usedCatalog.size,
+      catalogOnly: cards.filter(record => record.sources.length === 1 && record.sources[0] === 'dsh-plugin-catalog').length,
+      cards: cards.length,
+      mergedCards,
+    },
+    sources: {
+      dshSo: {
+        url: DSH_SO_URL,
+        generatedAt,
+        count: dshSo.plugins.length,
+        ok: true,
+      },
+      catalog: {
+        url: CATALOG_URL,
+        generatedAt: catalogGeneratedAt,
+        count: catalogPlugins.length,
+        ok: catalogPlugins.length > 0,
+      },
+    },
+  }
+}

package/lib/paths.js

@@ -0,0 +1,49 @@
+/**
+ * Where the shared, per-user index lives. The directory name is a stable
+ * identifier: it never follows a package rename and is shared by every DSH
+ * profile of the same OS user.
+ */
+import { homedir } from 'node:os'
+import { join } from 'node:path'
+
+/** Absolute default index directory for the current platform. */
+export function defaultIndexDir(env = process.env, platform = process.platform) {
+  if (platform === 'win32') {
+    return join(env.LOCALAPPDATA || join(homedir(), 'AppData', 'Local'), 'dsh-plugin-discover')
+  }
+  if (platform === 'darwin') {
+    return join(homedir(), 'Library', 'Caches', 'dsh-plugin-discover')
+  }
+  return join(env.XDG_CACHE_HOME || join(homedir(), '.cache'), 'dsh-plugin-discover')
+}
+
+/** The pointer file naming the published index directory. */
+export function currentPath(indexDir) {
+  return join(indexDir, 'current')
+}
+
+/** The cross-process build lock. */
+export function lockPath(indexDir) {
+  return join(indexDir, 'build.lock')
+}
+
+/** Human-readable build log written by the child process. */
+export function buildLogPath(indexDir) {
+  return join(indexDir, 'build.log')
+}
+
+/** Reject any pointer value that would escape the index directory. */
+export function isIndexDirName(name) {
+  return typeof name === 'string' && /^index-[0-9A-Za-zT._-]+$/.test(name)
+}
+
+/** Resolve an index directory name below its parent. */
+export function resolveIndexDir(indexDir, name) {
+  if (!isIndexDirName(name)) return null
+  return join(indexDir, name)
+}
+
+/** New sortable directory name, e.g. `index-20261022T120000000Z`. */
+export function newIndexDirName(now = new Date()) {
+  return `index-${now.toISOString().replace(/[-:]/g, '').replace(/\.\d{3}Z$/, 'Z')}`
+}

package/lib/quality.js

@@ -0,0 +1,61 @@
+/**
+ * The quality prior. It may only lower a result by 20%: relevance stays the
+ * primary key, and stars can never silently become the sort key.
+ */
+
+const VERIFICATION_SCORE = { 5: 1, 4: 0.8, 3: 0.6, 2: 0.45, 1: 0.3 }
+
+/** Verification level as a 0..1 signal; missing is neutral. */
+export function verificationScore(record) {
+  const level = record?.verification?.level
+  return VERIFICATION_SCORE[level] ?? 0.5
+}
+
+/** Security status/risk as a 0..1 signal; missing and pending are neutral. */
+export function securityScore(record) {
+  const security = record?.security
+  if (!security || typeof security !== 'object') return 0.5
+  const status = security.status
+  const risk = security.riskLevel
+  if (status === 'failed') return 0.2
+  if (status === 'audited') {
+    if (risk === 'low') return 1
+    if (risk === 'medium') return 0.7
+    if (risk === 'high') return 0.25
+    if (risk === 'critical') return 0
+    return 0.5
+  }
+  if (status === 'pending') return 0.45
+  return 0.4
+}
+
+/** Popularity from stars, capped at 1000; unknown stars are neutral. */
+export function popularityScore(record) {
+  const stars = record?.stars
+  if (typeof stars !== 'number' || !Number.isFinite(stars)) return 0.5
+  if (stars <= 0) return 0
+  return Math.min(1, Math.log1p(stars) / Math.log1p(1000))
+}
+
+/** Recency with a 90-day half-life; missing timestamps are neutral. */
+export function recencyScore(record, now = Date.now()) {
+  const updatedAt = record?.updatedAt
+  if (!updatedAt) return 0.5
+  const time = Date.parse(updatedAt)
+  if (!Number.isFinite(time)) return 0.5
+  const ageDays = Math.max(0, (now - time) / 86_400_000)
+  return 0.5 ** (ageDays / 90)
+}
+
+/** Combined quality prior in 0..1. */
+export function qualityPrior(record, now = Date.now()) {
+  return 0.55 * verificationScore(record)
+    + 0.20 * securityScore(record)
+    + 0.15 * popularityScore(record)
+    + 0.10 * recencyScore(record, now)
+}
+
+/** The bounded multiplier applied to a relevance score. */
+export function qualityFactor(record, now = Date.now()) {
+  return 0.8 + 0.2 * qualityPrior(record, now)
+}

package/lib/readme-clean.js

@@ -0,0 +1,128 @@
+/**
+ * Strip README presentation and boilerplate that do not describe what the
+ * plugin does: badges, images, HTML, code, front matter, link-only lines, and
+ * license/install/contributing/changelog sections. The remaining text is then
+ * token-truncated by the model client.
+ */
+
+export const README_CLEAN_VERSION = 1
+
+const DROP_SECTIONS = new Set([
+  'license',
+  'licence',
+  'licensing',
+  'contributing',
+  'contribution',
+  'code of conduct',
+  'security',
+  'changelog',
+  'release notes',
+  'releases',
+  'table of contents',
+  'contents',
+  'installation',
+  'install',
+  'getting started',
+  'setup',
+  'development',
+  'developing',
+  'testing',
+  'tests',
+  'sponsors',
+  'sponsor',
+  'support',
+  'acknowledgements',
+  'acknowledgments',
+  'credits',
+  'authors',
+  'star history',
+  'changelog and versioning',
+])
+
+function normalizeHeading(value) {
+  return value
+    .toLowerCase()
+    .replace(/[`*_~]/g, '')
+    .replace(/[^\p{L}\p{N}\s/-]/gu, ' ')
+    .replace(/\s+/g, ' ')
+    .trim()
+}
+
+function isDropHeading(value) {
+  return DROP_SECTIONS.has(normalizeHeading(value))
+}
+
+function stripBoilerplateSections(text) {
+  const lines = text.split('\n')
+  const kept = []
+  let skipLevel = 0
+  for (const line of lines) {
+    const heading = line.match(/^\s{0,3}(#{1,6})\s+(.+?)\s*#*\s*$/)
+    if (heading) {
+      const level = heading[1].length
+      if (skipLevel && level <= skipLevel) skipLevel = 0
+      if (!skipLevel && isDropHeading(heading[2])) {
+        skipLevel = level
+        continue
+      }
+    }
+    if (skipLevel) continue
+    kept.push(line)
+  }
+  return kept.join('\n')
+}
+
+function stripBadges(text) {
+  return text
+    // Markdown image, including the common linked-badge form.
+    .replace(/!\[[^\]]*\]\([^)]*\)/g, '')
+    .replace(/!\[[^\]]*\]\[[^\]]*\]/g, '')
+    .replace(/\[\s*\]\([^)]*\)/g, '')
+    // Reference-style link definitions.
+    .replace(/^\s*\[[^\]]+\]:\s*\S+.*$/gm, '')
+    // Remaining markdown links keep only their label.
+    .replace(/\[([^\]]+)\]\([^)]*\)/g, '$1')
+    .replace(/\[([^\]]+)\]\[[^\]]*\]/g, '$1')
+}
+
+export function cleanReadmeText(input) {
+  let text = String(input ?? '').replace(/\r\n?/g, '\n')
+  // YAML front matter.
+  text = text.replace(/^---\n[\s\S]*?\n---\n?/, '')
+  // HTML comments.
+  text = text.replace(/<!--[\s\S]*?-->/g, '')
+  // Fenced code and indented code blocks.
+  text = text.replace(/```[\s\S]*?```/g, '\n')
+  text = text.replace(/~~~[\s\S]*?~~~/g, '\n')
+  text = text.replace(/^(?: {4}|\t).*$/gm, '')
+  // HTML tags; keep line breaks.
+  text = text.replace(/<br\s*\/?>/gi, '\n')
+  text = text.replace(/<[^>]*>/g, '')
+  // Badges, images, and link labels.
+  text = stripBadges(text)
+  // Drop badges/shields and link-only lines that survived as plain URLs.
+  text = text.split('\n').filter(line => {
+    if (/shields\.io|badgen|img\.shields|badge/i.test(line) && !/[.!?]/.test(line)) return false
+    const withoutUrls = line.replace(/https?:\/\/\S+/gi, '').replace(/[\s\-|:]/g, '')
+    return withoutUrls !== ''
+  }).join('\n')
+  // Drop boilerplate sections before removing heading markers.
+  text = stripBoilerplateSections(text)
+  // Heading/list/blockquote markers, keeping the text.
+  text = text.replace(/^\s{0,3}#{1,6}\s+/gm, '')
+  text = text.replace(/^\s{0,3}>\s?/gm, '')
+  text = text.replace(/^\s{0,3}(?:[-*+]|\d+[.)])\s+/gm, '')
+  // Table separators, pipes, emphasis, inline code markup.
+  text = text.replace(/^\s*\|?[\s:|-]+\|[\s:|-]*\|?\s*$/gm, '')
+  text = text.replace(/\|/g, ' ')
+  text = text.replace(/`([^`]+)`/g, '$1')
+  text = text.replace(/(\*\*|__)(.*?)\1/g, '$2')
+  text = text.replace(/[*_]{1,2}/g, '')
+  text = text.replace(/&(?:nbsp|amp|lt|gt|quot|#39);/gi, ' ')
+  // Horizontal rules and empty/punctuation-only lines.
+  text = text.split('\n')
+    .map(line => line.replace(/[ \t]+/g, ' ').trim())
+    .filter(line => line !== '' && !/^[-=*_]{3,}$/.test(line) && /[\p{L}\p{N}]/u.test(line))
+    .join('\n')
+  return text.replace(/\n{3,}/g, '\n\n').trim()
+}