dsh-skill-hub 0.3.6 → 0.3.8

package/src/skillfs.ts

@@ -171,10 +171,21 @@ export function parseFrontmatter(text: string): { value: FrontmatterValue } | {
   const match = /^---\s*\r?\n([\s\S]*?)\r?\n---\s*(?:\r?\n([\s\S]*))?$/.exec(text)
   if (match === null) return { error: 'missing YAML frontmatter (--- block)' }
   let data: unknown
+  let rawFrontmatter = match[1]
   try {
-    data = load(match[1])
+    data = load(rawFrontmatter)
   } catch (error) {
-    return { error: 'invalid YAML frontmatter: ' + (error instanceof Error ? error.message : String(error)) }
+    const repaired = repairFrontmatterScalarFields(rawFrontmatter)
+    if (repaired !== null) {
+      try {
+        data = load(repaired)
+        rawFrontmatter = repaired
+      } catch {
+        return { error: 'invalid YAML frontmatter: ' + (error instanceof Error ? error.message : String(error)) }
+      }
+    } else {
+      return { error: 'invalid YAML frontmatter: ' + (error instanceof Error ? error.message : String(error)) }
+    }
   }
   if (typeof data !== 'object' || data === null || Array.isArray(data)) {
     return { error: 'frontmatter must be a YAML mapping' }
@@ -200,6 +211,125 @@ export function parseFrontmatter(text: string): { value: FrontmatterValue } | {
   return { value: { name, description, ...(whenToUse !== undefined ? { whenToUse } : {}), invocation, content: (match[2] ?? '').trim() } }
 }
 
+/**
+ * Repair bare scalar fields that contain an unquoted colon (e.g. `description: Build for AWS: ECS`)
+ * or an invalid flow-like scalar (`@`, `` ` ``, `[`, `{`). Mirrors codex `repair_frontmatter_scalar_fields`:
+ * line-oriented, only touches lines where quoting makes the YAML valid, and preserves block scalars.
+ * Returns the repaired frontmatter string when at least one line was quoted, otherwise null.
+ */
+function repairFrontmatterScalarFields(frontmatter: string): string | null {
+  let changed = false
+  let blockScalarIndent: number | null = null
+  const repairedLines: string[] = []
+  for (const line of frontmatter.split('\n')) {
+    const indent = line.search(/[^ ]/)
+    const effectiveIndent = indent === -1 ? line.length : indent
+    if (blockScalarIndent !== null) {
+      if (line.trim() === '' || effectiveIndent > blockScalarIndent) {
+        repairedLines.push(line)
+        continue
+      }
+      blockScalarIndent = null
+    }
+    const colonIndex = line.indexOf(':')
+    if (colonIndex === -1) {
+      repairedLines.push(line)
+      continue
+    }
+    const key = line.slice(0, colonIndex)
+    const value = line.slice(colonIndex + 1)
+    if (key.trim() === '' || value.length === 0 || !/^\s/.test(value)) {
+      repairedLines.push(line)
+      continue
+    }
+    const trimmedStart = value.trimStart()
+    const leadingWhitespace = value.slice(0, value.length - trimmedStart.length)
+    let scalar = trimmedStart
+    let comment = ''
+    // Split trailing `# comment` only when `#` is preceded by whitespace and
+    // followed by whitespace/end (YAML comment rule). This preserves `#1` as content.
+    for (let idx = 0; idx < trimmedStart.length; idx += 1) {
+      if (trimmedStart[idx] === '#') {
+        const prev = idx === 0 ? ' ' : trimmedStart[idx - 1]
+        const next = idx + 1 < trimmedStart.length ? trimmedStart[idx + 1] : ' '
+        if (/\s/.test(prev) && /\s/.test(next)) {
+          const commentStart = trimmedStart.slice(0, idx).trimEnd().length
+          scalar = trimmedStart.slice(0, commentStart)
+          comment = trimmedStart.slice(commentStart)
+          break
+        }
+      }
+    }
+    scalar = scalar.trimEnd()
+    if (scalar === '') {
+      repairedLines.push(line)
+      continue
+    }
+    const firstChar = scalar[0]
+    if (firstChar === '|' || firstChar === '>') {
+      blockScalarIndent = effectiveIndent
+      repairedLines.push(line)
+      continue
+    }
+    if (firstChar === "'" || firstChar === '"') {
+      repairedLines.push(line)
+      continue
+    }
+    let hasColonSeparator = false
+    for (let i = 0; i < scalar.length - 1; i += 1) {
+      if (scalar[i] === ':' && /\s/.test(scalar[i + 1])) {
+        hasColonSeparator = true
+        break
+      }
+    }
+    let invalidFlowLike = false
+    if ((firstChar === '[' || firstChar === '{' || firstChar === '@' || firstChar === '`')) {
+      try {
+        load(scalar)
+      } catch {
+        invalidFlowLike = true
+      }
+    }
+    if (!hasColonSeparator && !invalidFlowLike) {
+      repairedLines.push(line)
+      continue
+    }
+    const quotedScalar = "'" + scalar.replace(/'/g, "''") + "'"
+    repairedLines.push(key + ':' + leadingWhitespace + quotedScalar + comment)
+    changed = true
+  }
+  return changed ? repairedLines.join('\n') : null
+}
+
+/** Try to repair a SKILL.md's frontmatter in place (unquoted colon etc.). Returns the new file text or null when not fixable. */
+export function repairFrontmatterFileText(text: string): string | null {
+  const match = /^---\s*\r?\n([\s\S]*?)\r?\n---\s*(?:\r?\n([\s\S]*))?$/.exec(text)
+  if (match === null) return null
+  const repaired = repairFrontmatterScalarFields(match[1])
+  if (repaired === null) return null
+  try {
+    load(repaired)
+  } catch {
+    return null
+  }
+  const body = match[2] ?? ''
+  return '---\n' + repaired + '\n---' + (body !== '' ? '\n' + body : '')
+}
+
+/** Repair one file on disk when its frontmatter is auto-fixable. Returns the new text. */
+export async function fixDiagnosticFile(path: string, home = dshHome()): Promise<string> {
+  const root = rootOfPath(path, home)
+  if (root === undefined) throw new TypeError('not a hub writable skill path: ' + path)
+  const text = await readFile(path, 'utf8')
+  const repairedText = repairFrontmatterFileText(text)
+  if (repairedText === null) throw new TypeError('diagnostic is not auto-fixable: ' + path)
+  // Validate the repaired text parses as a legal skill (prevents writing a still-broken file).
+  const parsed = parseFrontmatter(repairedText)
+  if ('error' in parsed) throw new TypeError('repaired frontmatter still invalid: ' + parsed.error)
+  await writeFile(path, repairedText, 'utf8')
+  return path
+}
+
 /** Official boolean grammar: true/false, 1/0, and the common string spellings. */
 function frontmatterBoolean(data: Record<string, unknown>, key: string): boolean | undefined {
   if (!Object.hasOwn(data, key)) return undefined
@@ -262,6 +392,55 @@ export function listSkillEntries(root: WritableRoot, home = dshHome()): Promise<
   return scanRoot(rootPath(root, home))
 }
 
+/** Read UI metadata from `agents/openai.yaml` beside a directory skill (mirrors codex SkillInterface). */
+export async function readSkillInterface(directory: string): Promise<{ displayName?: string; shortDescription?: string; brandColor?: string; iconSmall?: string; iconLarge?: string; defaultPrompt?: string } | undefined> {
+  const yamlPath = join(directory, 'agents', 'openai.yaml')
+  let text: string
+  try {
+    text = await readFile(yamlPath, 'utf8')
+  } catch {
+    return undefined
+  }
+  let data: unknown
+  try {
+    data = load(text)
+  } catch {
+    return undefined
+  }
+  if (typeof data !== 'object' || data === null || Array.isArray(data)) return undefined
+  const record = data as Record<string, unknown>
+  const iface = (record.interface ?? record) as Record<string, unknown>
+  // codex stores interface under top-level `interface` key, but some templates put display_name at top level; support both.
+  const ifaceObj = typeof record.interface === 'object' && record.interface !== null && !Array.isArray(record.interface) ? record.interface as Record<string, unknown> : iface
+  const cleanStr = (value: unknown, maxLen: number): string | undefined => {
+    if (typeof value !== 'string') return undefined
+    const cleaned = value.split(/\s+/).join(' ').trim()
+    if (cleaned === '' || cleaned.length > maxLen) return undefined
+    return cleaned
+  }
+  const hexColor = (value: unknown): string | undefined => {
+    if (typeof value !== 'string') return undefined
+    const trimmed = value.trim()
+    return /^#[0-9a-fA-F]{6}$/.test(trimmed) ? trimmed : undefined
+  }
+  const iconPath = (value: unknown): string | undefined => {
+    if (typeof value !== 'string' || value.trim() === '') return undefined
+    const trimmed = value.trim().replace(/\\/g, '/')
+    if (trimmed.startsWith('/') || trimmed.includes('..')) return undefined
+    if (!trimmed.startsWith('assets/')) return undefined
+    return trimmed
+  }
+  const displayName = cleanStr(ifaceObj.display_name, 64)
+  const shortDescription = cleanStr(ifaceObj.short_description, 1024)
+  const brandColor = hexColor(ifaceObj.brand_color)
+  const defaultPrompt = cleanStr(ifaceObj.default_prompt, 1024)
+  const iconSmall = iconPath(ifaceObj.icon_small)
+  const iconLarge = iconPath(ifaceObj.icon_large)
+  const hasAny = displayName !== undefined || shortDescription !== undefined || brandColor !== undefined || iconSmall !== undefined || iconLarge !== undefined || defaultPrompt !== undefined
+  if (!hasAny) return undefined
+  return { ...(displayName !== undefined ? { displayName } : {}), ...(shortDescription !== undefined ? { shortDescription } : {}), ...(brandColor !== undefined ? { brandColor } : {}), ...(iconSmall !== undefined ? { iconSmall } : {}), ...(iconLarge !== undefined ? { iconLarge } : {}), ...(defaultPrompt !== undefined ? { defaultPrompt } : {}) }
+}
+
 /**
  * Walk up from cwd (max 32 levels) for a project marker (.dsh or .git);
  * falls back to cwd itself. The provider roots project skills here.
@@ -298,9 +477,15 @@ export async function scanDiagnostics(root: WritableRoot, home = dshHome()): Pro
     }
     const parsed = parseFrontmatter(text)
     if ('error' in parsed) {
-      diagnostics.push({ path: entry.path, root, reason: parsed.error })
+      const fixable = repairFrontmatterFileText(text) !== null
+      diagnostics.push({ path: entry.path, root, reason: parsed.error, ...(fixable ? { fixable: true } : {}) })
       continue
     }
+    // File was auto-repaired in memory (e.g. unquoted `:`); surface a fixable hint so the user can persist it.
+    const repaired = repairFrontmatterFileText(text)
+    if (repaired !== null) {
+      diagnostics.push({ path: entry.path, root, reason: 'frontmatter contains unquoted colon/bracket (auto-repaired in memory; click Fix to persist)', fixable: true })
+    }
     const { value } = parsed
     // The provider registers a skill by its discovery path (directory name or
     // flat file name), so a frontmatter name that diverges from the path makes

package/src/stats.test.ts

@@ -199,6 +199,29 @@ async function flush(): Promise<void> {
   await new Promise((resolve) => setTimeout(resolve, 20))
 }
 
+describe('scan read concurrency', () => {
+  it('reads session logs in parallel, not one by one', async () => {
+    const ids = Array.from({ length: 8 }, (_, i) => 's' + i)
+    let inFlight = 0
+    let peak = 0
+    const query: SessionQueryLike = {
+      listSessions: async () => ids.map((id) => record(id)),
+      readSession: async (id) => {
+        inFlight += 1
+        peak = Math.max(peak, inFlight)
+        await new Promise((resolve) => setTimeout(resolve, 5))
+        inFlight -= 1
+        return { events: [invocationEvent(String(id))] }
+      },
+    }
+    // No checkpoint → full path, all 8 sessions read.
+    const stats = await readSkillStats(query)
+    expect(stats).toHaveLength(8)
+    expect(peak).toBeGreaterThan(1)
+    expect(peak).toBeLessThanOrEqual(6)
+  })
+})
+
 describe('frozen-bucket incremental scans', () => {
   it('skips frozen sessions and merges checkpoint totals with the recent window', async () => {
     const reads: string[] = []
@@ -221,15 +244,15 @@ describe('frozen-bucket incremental scans', () => {
       },
       onCheckpoint: () => { checkpoints += 1 },
     })
-    expect(await reader()).toEqual([]) // 首次调用不等待扫描
+    expect(await reader()).toEqual([]) // 首次调用不等待扫描（无上次总数可端）
     await flush()
     expect(await reader()).toEqual([
       { name: 'oldskill', count: 5, lastUsed: NOW - 20 * DAY },
       { name: 'tdd', count: 2, lastUsed: 0 },
     ])
-    // 冻结会话没有被重读；增量扫描不改检查点 → 不触发持久化回调。
+    // 冻结会话没有被重读；增量扫描也落盘（含新总数）→ 回调恰好一次。
     expect(reads).toEqual(['fresh'])
-    expect(checkpoints).toBe(0)
+    expect(checkpoints).toBe(1)
   })
 
   it('runs a full reconciliation when due: rebuilds the frozen bucket and advances the watermark', async () => {
@@ -263,9 +286,41 @@ describe('frozen-bucket incremental scans', () => {
       // 全历史模式（windowDays=0）：冻结会话进缓存且仍计入总数。
       frozenSessions: { ancient: { createdAt: NOW - 30 * DAY, counts: { a: { count: 1, lastUsed: 5 } } } },
       lastFullReconcile: NOW,
+      lastTotals: [
+        { name: 'a', count: 1, lastUsed: 5 },
+        { name: 'b', count: 2, lastUsed: 0 },
+      ],
     })
   })
 
+  it('serves persisted lastTotals instantly on cold start, then merges the rescan', async () => {
+    const reads: string[] = []
+    const query = fakeQuery(
+      [record('fresh', NOW - 1 * DAY)],
+      { fresh: [invocationEvent('tdd', 1)] },
+      reads,
+    )
+    const saved: Array<Record<string, unknown>> = []
+    const reader = createSkillStatsReader(query, 60_000, {
+      now: () => NOW,
+      checkpoint: {
+        windowDays: 0,
+        frozenBefore: 0,
+        frozenSessions: {},
+        lastFullReconcile: NOW, // 对账刚做过 → 增量路径
+        lastTotals: [{ name: 'oldskill', count: 9, lastUsed: 7 }],
+      },
+      onCheckpoint: (cp) => { saved.push(cp as unknown as Record<string, unknown>) },
+    })
+    // 冷启动直接端上次总数，同时后台重扫；重扫后以新鲜值为准（旧数不复活）。
+    expect(await reader()).toEqual([{ name: 'oldskill', count: 9, lastUsed: 7 }])
+    await flush()
+    expect(await reader()).toEqual([{ name: 'tdd', count: 1, lastUsed: 0 }])
+    expect(reads).toEqual(['fresh'])
+    expect(saved).toHaveLength(1)
+    expect((saved[0] as { lastTotals: unknown }).lastTotals).toEqual([{ name: 'tdd', count: 1, lastUsed: 0 }])
+  })
+
   it('re-reads sessions without createdAt on incremental scans (never freezes)', async () => {
     const reads: string[] = []
     const query = fakeQuery(

package/src/stats.ts

@@ -103,6 +103,24 @@ function isFrozen(record: { header: { createdAt?: number } }, watermark: number)
   return typeof created === 'number' && created > 0 && created < watermark
 }
 
+/** Bounded-concurrency map: session-log reads are independent decompressions, so wall time scales ~1/limit. Merge stays sequential in index order. */
+async function mapConcurrent<T, R>(items: readonly T[], limit: number, worker: (item: T, index: number) => Promise<R>): Promise<R[]> {
+  const results = new Array<R>(items.length)
+  let cursor = 0
+  const runners = Array.from({ length: Math.min(Math.max(limit, 1), items.length) }, async () => {
+    while (cursor < items.length) {
+      const index = cursor
+      cursor += 1
+      results[index] = await worker(items[index], index)
+    }
+  })
+  await Promise.all(runners)
+  return results
+}
+
+/** Max parallel session-log reads per scan (read-only; merge is order-independent). */
+const SCAN_READ_CONCURRENCY = 6
+
 function mergeInto(totals: Totals, counted: Map<string, InvocationStat>): void {
   for (const [name, stat] of counted) {
     const total = totals[name]
@@ -153,19 +171,22 @@ async function scan(query: SessionQueryLike, checkpoint: SkillStatsCheckpoint, n
   if (dueFullScan) {
     const cache: SkillStatsCheckpoint['frozenSessions'] = {}
     const totals: Totals = {}
-    for (const record of sessions) {
-      let counted: Map<string, InvocationStat>
+    const countedList = await mapConcurrent(sessions, SCAN_READ_CONCURRENCY, async (record) => {
       try {
-        counted = countSkillInvocations((await query.readSession(record.header.id)).events)
+        return countSkillInvocations((await query.readSession(record.header.id)).events)
       } catch {
-        continue // unreadable sessions are skipped, never fatal
+        return undefined // unreadable sessions are skipped, never fatal
       }
+    })
+    sessions.forEach((record, index) => {
+      const counted = countedList[index]
+      if (counted === undefined) return
       const created = record.header.createdAt
       if (isFrozen(record, cutoff) && counted.size > 0 && typeof created === 'number') {
         cache[record.header.id as unknown as string] = { createdAt: created, counts: Object.fromEntries(counted) }
       }
       if (inWindow(created, windowDays, nowMs)) mergeInto(totals, counted)
-    }
+    })
     checkpoint.frozenSessions = cache
     checkpoint.frozenBefore = cutoff
     checkpoint.windowDays = windowDays
@@ -174,13 +195,17 @@ async function scan(query: SessionQueryLike, checkpoint: SkillStatsCheckpoint, n
   }
 
   const recent: Totals = {}
-  for (const record of sessions) {
-    if (isFrozen(record, checkpoint.frozenBefore)) continue
+  const unfrozen = sessions.filter((record) => !isFrozen(record, checkpoint.frozenBefore))
+  const recentList = await mapConcurrent(unfrozen, SCAN_READ_CONCURRENCY, async (record) => {
     try {
-      mergeInto(recent, countSkillInvocations((await query.readSession(record.header.id)).events))
+      return countSkillInvocations((await query.readSession(record.header.id)).events)
     } catch {
-      continue // unreadable sessions are skipped, never fatal
+      return undefined // unreadable sessions are skipped, never fatal
     }
+  })
+  for (const counted of recentList) {
+    if (counted === undefined) continue
+    mergeInto(recent, counted)
   }
   const totals: Totals = {}
   for (const [id, entry] of Object.entries(checkpoint.frozenSessions)) {
@@ -216,8 +241,9 @@ export interface SkillStatsReaderOptions {
   ttlMs?: number | (() => number)
   /** Rolling window in days; a getter reads the live config each scan. 0 = all history. */
   windowDays?: () => number
-  /** Called after a full reconciliation mutated the checkpoint (never after an
-   *  incremental scan) so the host can persist it to the sidecar. */
+  /** Called after every completed scan (full or incremental) so the host can
+   *  persist the checkpoint including the fresh totals. Cadence follows the
+   *  scan TTL (minutes, not days) — the payload is tiny and writes are atomic. */
   onCheckpoint?: (checkpoint: SkillStatsCheckpoint) => void
 }
 
@@ -238,7 +264,9 @@ export interface SkillStatsReaderOptions {
 export function createSkillStatsReader(query: SessionQueryLike, ttlMs: number | (() => number) = 300_000, options: SkillStatsReaderOptions = {}): SkillStatsReader {
   const checkpoint: SkillStatsCheckpoint = options.checkpoint ?? { windowDays: 0, frozenBefore: 0, frozenSessions: {}, lastFullReconcile: 0 }
   const now = options.now ?? (() => Date.now())
-  let cached: SkillStat[] | undefined
+  // Cold start serves the last persisted totals instantly (may be a window
+  // behind); cachedAt stays 0 so the first call still kicks a background rescan.
+  let cached: SkillStat[] | undefined = checkpoint.lastTotals !== undefined ? [...checkpoint.lastTotals] : undefined
   let cachedAt = 0
   let refreshing: Promise<void> | null = null
   let lastScanDurationMs = 0
@@ -248,16 +276,17 @@ export function createSkillStatsReader(query: SessionQueryLike, ttlMs: number |
     const base = typeof ttlMs === 'function' ? ttlMs() : ttlMs
     const ttl = Math.max(base, lastScanDurationMs * STATS_TTL_SCAN_FACTOR)
     if (cached !== undefined && startedAt - cachedAt < ttl) return cached
-    // Expired (or first call): hand back the stale totals (empty on first
-    // call) and kick off one background rescan.
+    // Expired (or first call): hand back the stale totals (empty on a truly
+    // fresh start) and kick off one background rescan.
     if (refreshing === null) {
       const windowDays = options.windowDays?.() ?? 0
       refreshing = scan(query, checkpoint, startedAt, windowDays)
-        .then(({ stats, mutated }) => {
+        .then(({ stats }) => {
           cached = stats
           cachedAt = now()
           lastScanDurationMs = Math.max(0, cachedAt - startedAt)
-          if (mutated) options.onCheckpoint?.({ ...checkpoint, frozenSessions: { ...checkpoint.frozenSessions } })
+          checkpoint.lastTotals = stats
+          options.onCheckpoint?.({ ...checkpoint, frozenSessions: { ...checkpoint.frozenSessions } })
         })
         .catch(() => { /* keep the previous totals on scan failure */ })
         .finally(() => { refreshing = null })

package/src/store/errors.ts

@@ -0,0 +1,14 @@
+/**
+ * Business-rule failure the routes layer can map onto a 4xx status instead
+ * of a blanket 500: user input is invalid (validation → 400), the target
+ * does not exist (not-found → 404), or the operation conflicts with an
+ * invariant (conflict → 409).
+ */
+export class StoreError extends Error {
+  readonly kind: 'validation' | 'not-found' | 'conflict'
+  constructor(kind: StoreError['kind'], message: string) {
+    super(message)
+    this.name = 'StoreError'
+    this.kind = kind
+  }
+}