@erclx/aitk 0.8.0

package/src/tooling/manifest.ts

@@ -0,0 +1,178 @@
+import { existsSync, readFileSync } from 'node:fs'
+import { join } from 'node:path'
+
+const EXCLUDED_STACKS = ['claude']
+
+export interface GitignoreSection {
+  readonly header: string
+  readonly entries: readonly string[]
+}
+
+export interface Manifest {
+  readonly name: string
+  readonly parent?: string
+  readonly dir: string
+  readonly configsDir: string
+  readonly seedsDir: string
+  readonly referenceFile: string
+  readonly scripts: Readonly<Record<string, string>>
+  readonly scriptOverrides: Readonly<Record<string, string>>
+  readonly gitignore: readonly GitignoreSection[]
+  readonly devPackages: readonly string[]
+}
+
+export interface ChainOptions {
+  readonly skipStack?: string
+}
+
+function toolingDir(root: string): string {
+  return join(root, 'tooling')
+}
+
+function manifestPath(root: string, stack: string): string {
+  return join(toolingDir(root), stack, 'manifest.toml')
+}
+
+/**
+ * Lists every file under a directory, dotfiles included. Bun.Glob skips
+ * entries beginning with a dot unless `dot` is set, and tooling configs are
+ * almost entirely dotfiles, so omitting it silently matches nothing.
+ */
+export function listFiles(dir: string): string[] {
+  if (!existsSync(dir)) return []
+  return [
+    ...new Bun.Glob('**/*').scanSync({ cwd: dir, onlyFiles: true, dot: true }),
+  ].sort()
+}
+
+/**
+ * Reverses a chain so the furthest ancestor is applied first and nearer
+ * stacks overwrite it.
+ */
+export function ancestorsFirst(chain: readonly Manifest[]): Manifest[] {
+  return [...chain].reverse()
+}
+
+export function stackExists(root: string, stack: string): boolean {
+  return existsSync(join(toolingDir(root), stack))
+}
+
+export function isStackExcluded(stack: string): boolean {
+  return EXCLUDED_STACKS.includes(stack)
+}
+
+/**
+ * Lists installable stack names, filtering the `claude` stack the way
+ * `is_tooling_stack_excluded` in `scripts/lib/tooling.sh` did. Claude is
+ * managed by `aitk claude`, not `aitk tooling`.
+ */
+export function listStacks(root: string): string[] {
+  const dir = toolingDir(root)
+  if (!existsSync(dir)) return []
+
+  return [...new Bun.Glob('*/manifest.toml').scanSync({ cwd: dir })]
+    .map((entry) => entry.split('/')[0])
+    .filter((name) => !isStackExcluded(name))
+    .sort()
+}
+
+/**
+ * Reads one manifest into a typed shape. Returns undefined when the file is
+ * absent, matching the bash guards that silently skipped a missing manifest.
+ */
+export function loadManifest(
+  root: string,
+  stack: string,
+): Manifest | undefined {
+  const path = manifestPath(root, stack)
+  if (!existsSync(path)) return undefined
+
+  const parsed = asTable(Bun.TOML.parse(readFileSync(path, 'utf8')))
+  const stackTable = asTable(parsed.stack)
+  const parent =
+    typeof stackTable.extends === 'string' ? stackTable.extends : ''
+  const scriptsTable = asTable(parsed.scripts)
+  const dir = join(toolingDir(root), stack)
+
+  return {
+    name: stack,
+    parent: parent === '' ? undefined : parent,
+    dir,
+    configsDir: join(dir, 'configs'),
+    seedsDir: join(dir, 'seeds'),
+    referenceFile: join(dir, 'reference.md'),
+    scripts: pickStrings(scriptsTable),
+    scriptOverrides: pickStrings(asTable(scriptsTable.override)),
+    gitignore: readGitignoreSections(asTable(parsed.gitignore)),
+    devPackages: readDevPackages(parsed.dependencies),
+  }
+}
+
+/**
+ * Walks the `extends` chain once, self first. Callers that apply parent
+ * values before child values reverse the result. Replaces the seventeen
+ * copies of `grep '^extends' | cut -d'"' -f2` the bash carried.
+ *
+ * A stack matching `skipStack` truncates the chain at that point, mirroring
+ * the `[ "$stack" = "${SKIP_STACK:-}" ] && return` guard.
+ */
+export function resolveChain(
+  root: string,
+  stack: string,
+  options: ChainOptions = {},
+): Manifest[] {
+  const chain: Manifest[] = []
+  const seen = new Set<string>()
+  let current: string | undefined = stack
+
+  while (current !== undefined) {
+    if (current === options.skipStack) break
+    if (seen.has(current)) break
+    seen.add(current)
+
+    const manifest: Manifest | undefined = loadManifest(root, current)
+    if (!manifest) break
+
+    chain.push(manifest)
+    current = manifest.parent
+  }
+
+  return chain
+}
+
+function asTable(value: unknown): Record<string, unknown> {
+  if (typeof value !== 'object' || value === null || Array.isArray(value)) {
+    return {}
+  }
+  return value as Record<string, unknown>
+}
+
+function pickStrings(table: Record<string, unknown>): Record<string, string> {
+  const result: Record<string, string> = {}
+  for (const [key, value] of Object.entries(table)) {
+    if (typeof value === 'string') result[key] = value
+  }
+  return result
+}
+
+function readGitignoreSections(
+  table: Record<string, unknown>,
+): GitignoreSection[] {
+  const sections: GitignoreSection[] = []
+  for (const [header, value] of Object.entries(table)) {
+    if (!Array.isArray(value)) continue
+    const entries = value.filter(
+      (entry): entry is string => typeof entry === 'string' && entry !== '',
+    )
+    sections.push({ header, entries })
+  }
+  return sections
+}
+
+function readDevPackages(dependencies: unknown): string[] {
+  const dev = asTable(asTable(dependencies).dev)
+  if (!Array.isArray(dev.packages)) return []
+  return dev.packages.filter(
+    (entry): entry is string => typeof entry === 'string' && entry !== '',
+  )
+}

package/src/tooling/package.ts

@@ -0,0 +1,166 @@
+import { existsSync, readFileSync } from 'node:fs'
+import type { Manifest } from '@/tooling/manifest'
+
+export interface PackageJson {
+  readonly dependencies?: Record<string, string>
+  readonly devDependencies?: Record<string, string>
+  readonly scripts?: Record<string, string>
+  readonly [key: string]: unknown
+}
+
+export interface ScriptState {
+  readonly key: string
+  readonly state: 'missing' | 'drifted' | 'matching'
+}
+
+export interface DepState {
+  readonly spec: string
+  readonly name: string
+  readonly state: 'missing' | 'present'
+}
+
+export function readPackage(path: string): PackageJson | undefined {
+  if (!existsSync(path)) return undefined
+  try {
+    return JSON.parse(readFileSync(path, 'utf8')) as PackageJson
+  } catch {
+    return undefined
+  }
+}
+
+export function serializePackage(pkg: PackageJson): string {
+  return `${JSON.stringify(pkg, null, 2)}\n`
+}
+
+/**
+ * Splits a dependency spec into its package name, handling the scoped case
+ * where the leading `@` is part of the name rather than a version separator.
+ * `@scope/pkg@^1.2.3` yields `@scope/pkg`, `react@19` yields `react`.
+ */
+export function parseSpec(spec: string): { name: string; pin: string } {
+  let name = spec
+
+  if (spec.startsWith('@')) {
+    const rest = spec.slice(1)
+    if (rest.includes('@')) name = `@${rest.slice(0, rest.indexOf('@'))}`
+  } else {
+    name = spec.split('@')[0]
+  }
+
+  const pin = spec === name ? '' : spec.slice(name.length + 1)
+  return { name, pin }
+}
+
+function firstInteger(value: string): string | undefined {
+  return value.match(/[0-9]+/)?.[0]
+}
+
+/**
+ * Compares a manifest pin against what package.json records. A spec counts as
+ * missing when absent, or when the pinned major differs from the installed
+ * major. Anything looser than a major mismatch is left alone.
+ */
+function isSatisfied(installed: string | undefined, pin: string): boolean {
+  if (installed === undefined) return false
+  if (pin === '') return true
+
+  const pinMajor = firstInteger(pin)
+  const installedMajor = firstInteger(installed)
+  if (pinMajor === undefined) return true
+
+  return pinMajor === installedMajor
+}
+
+/**
+ * Resolves dev dependencies across the chain, ancestors first, deduplicated by
+ * package name so the outermost stack that names a package wins.
+ */
+export function collectDeps(
+  chain: readonly Manifest[],
+  pkg: PackageJson,
+): DepState[] {
+  const installed = { ...pkg.dependencies, ...pkg.devDependencies }
+  const seen = new Set<string>()
+  const states: DepState[] = []
+
+  for (const manifest of [...chain].reverse()) {
+    for (const spec of manifest.devPackages) {
+      const { name, pin } = parseSpec(spec)
+      if (name === '' || seen.has(name)) continue
+      seen.add(name)
+
+      states.push({
+        spec,
+        name,
+        state: isSatisfied(installed[name], pin) ? 'present' : 'missing',
+      })
+    }
+  }
+
+  return states
+}
+
+/**
+ * Compares manifest scripts against package.json, nearest stack first, so a
+ * child definition shadows the parent it extends.
+ */
+export function collectScripts(
+  chain: readonly Manifest[],
+  pkg: PackageJson,
+): ScriptState[] {
+  const scripts = pkg.scripts ?? {}
+  const seen = new Set<string>()
+  const states: ScriptState[] = []
+
+  for (const manifest of chain) {
+    for (const [key, value] of Object.entries(manifest.scripts)) {
+      if (seen.has(key)) continue
+      seen.add(key)
+
+      const current = scripts[key]
+      if (current === undefined) {
+        states.push({ key, state: 'missing' })
+      } else if (current !== value) {
+        states.push({ key, state: 'drifted' })
+      } else {
+        states.push({ key, state: 'matching' })
+      }
+    }
+  }
+
+  return states
+}
+
+/**
+ * Fills scripts the target does not define, nearest stack first, then replays
+ * `[scripts.override]` from the furthest ancestor inward so the nearest stack
+ * has the last word. This is the ordering `apply_stack_scripts` produced
+ * through its recursion.
+ */
+export function applyScripts(
+  pkg: PackageJson,
+  chain: readonly Manifest[],
+): { pkg: PackageJson; added: string[]; overridden: string[] } {
+  const scripts: Record<string, string> = { ...pkg.scripts }
+  const added: string[] = []
+  const overridden: string[] = []
+
+  for (const manifest of chain) {
+    for (const [key, value] of Object.entries(manifest.scripts)) {
+      if (scripts[key] === undefined) {
+        scripts[key] = value
+        added.push(key)
+      }
+    }
+  }
+
+  for (const manifest of [...chain].reverse()) {
+    for (const [key, value] of Object.entries(manifest.scriptOverrides)) {
+      if (scripts[key] === value) continue
+      scripts[key] = value
+      if (!overridden.includes(key)) overridden.push(key)
+    }
+  }
+
+  return { pkg: { ...pkg, scripts }, added, overridden }
+}

package/src/tooling/scan.ts

@@ -0,0 +1,186 @@
+import { existsSync, readFileSync } from 'node:fs'
+import { join } from 'node:path'
+import { mergeSections } from '@/tooling/gitignore'
+import { ancestorsFirst, listFiles, type Manifest } from '@/tooling/manifest'
+import {
+  collectDeps,
+  collectScripts,
+  type DepState,
+  readPackage,
+  type ScriptState,
+} from '@/tooling/package'
+
+export interface ConfigState {
+  readonly rel: string
+  readonly stack: string
+  readonly state: 'new' | 'drifted' | 'matching'
+}
+
+export interface SeedState {
+  readonly rel: string
+  readonly state: 'missing' | 'present'
+}
+
+export interface EntryState {
+  readonly entry: string
+  readonly state: 'missing' | 'present'
+}
+
+export interface ReferenceState {
+  readonly stack: string
+  readonly state: 'pending' | 'matching'
+}
+
+export interface ScanResult {
+  readonly configs: readonly ConfigState[]
+  readonly seeds: readonly SeedState[]
+  readonly scripts: readonly ScriptState[]
+  readonly deps: readonly DepState[]
+  readonly gitignore: readonly EntryState[]
+  readonly references: readonly ReferenceState[]
+  readonly hasPackageJson: boolean
+  readonly totalChanges: number
+}
+
+export interface ScanOptions {
+  readonly includeReferences: boolean
+}
+
+function isIdentical(a: string, b: string): boolean {
+  if (!existsSync(b)) return false
+  return readFileSync(a).equals(readFileSync(b))
+}
+
+/**
+ * Compares every stack in the chain against the target and reports what would
+ * change. Nothing is written. Which stack wins a duplicate differs per
+ * category and mirrors the bash: configs, seeds, and scripts resolve nearest
+ * stack first, while dependencies, gitignore entries, and references resolve
+ * from the furthest ancestor inward.
+ */
+export function scan(
+  chain: readonly Manifest[],
+  target: string,
+  options: ScanOptions,
+): ScanResult {
+  const configs: ConfigState[] = []
+  const seenConfigs = new Set<string>()
+
+  for (const manifest of chain) {
+    for (const rel of listFiles(manifest.configsDir)) {
+      if (seenConfigs.has(rel)) continue
+      seenConfigs.add(rel)
+
+      const source = join(manifest.configsDir, rel)
+      const dest = join(target, rel)
+      const state = !existsSync(dest)
+        ? 'new'
+        : isIdentical(source, dest)
+          ? 'matching'
+          : 'drifted'
+
+      configs.push({ rel, stack: manifest.name, state })
+    }
+  }
+
+  const seeds: SeedState[] = []
+  const seenSeeds = new Set<string>()
+
+  for (const manifest of chain) {
+    for (const rel of listFiles(manifest.seedsDir)) {
+      if (seenSeeds.has(rel)) continue
+      seenSeeds.add(rel)
+
+      seeds.push({
+        rel,
+        state: existsSync(join(target, rel)) ? 'present' : 'missing',
+      })
+    }
+  }
+
+  const packagePath = join(target, 'package.json')
+  const pkg = readPackage(packagePath)
+  const scripts = pkg ? collectScripts(chain, pkg) : []
+  const deps = pkg ? collectDeps(chain, pkg) : []
+
+  const gitignorePath = join(target, '.gitignore')
+  const gitignoreContent = existsSync(gitignorePath)
+    ? readFileSync(gitignorePath, 'utf8')
+    : ''
+  const gitignore = scanGitignore(chain, gitignoreContent)
+
+  const references = options.includeReferences
+    ? scanReferences(chain, target)
+    : []
+
+  const totalChanges =
+    configs.filter((entry) => entry.state !== 'matching').length +
+    seeds.filter((entry) => entry.state === 'missing').length +
+    scripts.filter((entry) => entry.state !== 'matching').length +
+    deps.filter((entry) => entry.state === 'missing').length +
+    gitignore.filter((entry) => entry.state === 'missing').length +
+    references.filter((entry) => entry.state === 'pending').length
+
+  return {
+    configs,
+    seeds,
+    scripts,
+    deps,
+    gitignore,
+    references,
+    hasPackageJson: pkg !== undefined,
+    totalChanges,
+  }
+}
+
+/**
+ * Replays the merge against a copy of the file so an entry that an earlier
+ * section or stack would add is not also reported as missing.
+ */
+function scanGitignore(
+  chain: readonly Manifest[],
+  content: string,
+): EntryState[] {
+  const states: EntryState[] = []
+  const seen = new Set<string>()
+  let projected = content
+
+  for (const manifest of ancestorsFirst(chain)) {
+    for (const section of manifest.gitignore) {
+      for (const entry of section.entries) {
+        if (seen.has(entry)) continue
+        seen.add(entry)
+
+        const result = mergeSections(projected, [
+          { header: section.header, entries: [entry] },
+        ])
+        states.push({
+          entry,
+          state: result.added.length > 0 ? 'missing' : 'present',
+        })
+        projected = result.content
+      }
+    }
+  }
+
+  return states
+}
+
+function scanReferences(
+  chain: readonly Manifest[],
+  target: string,
+): ReferenceState[] {
+  const states: ReferenceState[] = []
+
+  for (const manifest of ancestorsFirst(chain)) {
+    if (!existsSync(manifest.referenceFile)) continue
+
+    const dest = join(target, '.claude', 'tooling', `${manifest.name}.md`)
+    states.push({
+      stack: manifest.name,
+      state: isIdentical(manifest.referenceFile, dest) ? 'matching' : 'pending',
+    })
+  }
+
+  return states
+}

package/src/transcripts/fetch.ts

@@ -0,0 +1,156 @@
+import {
+  mkdirSync,
+  mkdtempSync,
+  readdirSync,
+  rmSync,
+  writeFileSync,
+} from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { readFileSync } from 'node:fs'
+import { execa } from 'execa'
+import {
+  type VideoMetadata,
+  formatUploadDate,
+  renderFrontmatter,
+  slugify,
+} from '@/transcripts/metadata'
+import { dedupeRollingCues, parseVtt, renderBody } from '@/transcripts/vtt'
+
+const YT_DLP_TIMEOUT_MS = 120_000
+
+export function ensureYtDlp(): void {
+  if (Bun.which('yt-dlp') === null) {
+    throw new Error(
+      'yt-dlp not found on PATH. Install it from https://github.com/yt-dlp/yt-dlp.',
+    )
+  }
+}
+
+interface FetchResult {
+  readonly metadata: VideoMetadata
+  readonly subPath: string | null
+}
+
+async function runYtDlp(url: string, workDir: string): Promise<FetchResult> {
+  const outputTemplate = join(workDir, '%(id)s.%(ext)s')
+  let stdout: string
+  try {
+    const result = await execa(
+      'yt-dlp',
+      [
+        '--dump-json',
+        '--no-simulate',
+        '--write-sub',
+        '--write-auto-sub',
+        '--sub-langs',
+        'en,en-orig',
+        '--sub-format',
+        'vtt',
+        '--skip-download',
+        '-o',
+        outputTemplate,
+        url,
+      ],
+      { timeout: YT_DLP_TIMEOUT_MS },
+    )
+    stdout = result.stdout
+  } catch (error) {
+    throw new Error(`yt-dlp failed for ${url}: ${lastLine(error)}`)
+  }
+
+  const info = JSON.parse(stdout) as Record<string, unknown>
+  const videoId = String(info['id'] ?? '').trim()
+  if (!videoId) throw new Error('yt-dlp returned no video id')
+
+  const metadata: VideoMetadata = {
+    videoId,
+    title: String(info['title'] || 'untitled'),
+    channel: String(info['channel'] || info['uploader'] || 'unknown'),
+    durationSeconds: Number(info['duration'] || 0),
+    published: formatUploadDate(asString(info['upload_date'])),
+    url: String(info['webpage_url'] || url),
+  }
+  const subPath = findSubtitle(workDir, videoId)
+  return { metadata, subPath }
+}
+
+interface WriteOptions {
+  readonly outDir: string
+  readonly keepTimestamps: boolean
+  readonly fetchedAt: string
+}
+
+export function writeTranscript(
+  metadata: VideoMetadata,
+  subPath: string | null,
+  { outDir, keepTimestamps, fetchedAt }: WriteOptions,
+): string {
+  mkdirSync(outDir, { recursive: true })
+  const slug = `${slugify(metadata.title)}--${metadata.videoId}`
+  const target = join(outDir, `${slug}.md`)
+
+  const hasTranscript = subPath !== null
+  let body = ''
+  if (subPath !== null) {
+    const cues = parseVtt(readFileSync(subPath, 'utf8'))
+    body = renderBody(dedupeRollingCues(cues), { keepTimestamps })
+  } else {
+    body =
+      'Transcript unavailable. Auto-captions are not enabled for this video.\n'
+  }
+
+  const frontmatter = renderFrontmatter(metadata, { hasTranscript, fetchedAt })
+  const heading = `# ${metadata.title}\n\n`
+  writeFileSync(target, frontmatter + heading + body)
+  return target
+}
+
+interface FetchOptions {
+  readonly outDir: string
+  readonly keepTimestamps: boolean
+}
+
+export async function fetchOne(
+  url: string,
+  { outDir, keepTimestamps }: FetchOptions,
+): Promise<string> {
+  const workDir = mkdtempSync(join(tmpdir(), 'yt-transcripts-'))
+  try {
+    const { metadata, subPath } = await runYtDlp(url, workDir)
+    return writeTranscript(metadata, subPath, {
+      outDir,
+      keepTimestamps,
+      fetchedAt: todayLocal(),
+    })
+  } finally {
+    rmSync(workDir, { recursive: true, force: true })
+  }
+}
+
+function findSubtitle(workDir: string, videoId: string): string | null {
+  const matches = readdirSync(workDir)
+    .filter((name) => name.startsWith(videoId) && name.endsWith('.vtt'))
+    .sort()
+  return matches.length ? join(workDir, matches[0]) : null
+}
+
+function todayLocal(): string {
+  const now = new Date()
+  const year = now.getFullYear()
+  const month = String(now.getMonth() + 1).padStart(2, '0')
+  const day = String(now.getDate()).padStart(2, '0')
+  return `${year}-${month}-${day}`
+}
+
+function asString(value: unknown): string | undefined {
+  return typeof value === 'string' ? value : undefined
+}
+
+function lastLine(error: unknown): string {
+  const stderr =
+    error instanceof Error && 'stderr' in error ? String(error.stderr) : ''
+  const lines = stderr.trim().split('\n').filter(Boolean)
+  if (lines.length) return lines[lines.length - 1]
+  return error instanceof Error ? error.message : String(error)
+}