@erclx/canon 4.0.0

package/src/tooling/read.ts

@@ -0,0 +1,65 @@
+import { existsSync, readFileSync } from 'node:fs'
+import { join } from 'node:path'
+import { PROJECT_ROOT } from '@/project-root'
+
+/**
+ * Spells the package root in a report where every other root spells a
+ * project-relative path. The `source` field promises a path inside the project
+ * and a package copy is the one source that promise cannot cover, so the
+ * spelling has to be one nothing will join to a project root.
+ */
+const PACKAGE_LABEL = '<canon>'
+
+export interface ReferenceRoot {
+  /** Absolute `tooling/` directory to search. */
+  readonly dir: string
+  /** How a report spells a copy found under this root. */
+  readonly label: string
+}
+
+export interface ResolvedReference {
+  readonly path: string
+  /** The label of the root that won, joined to the stack and filename. */
+  readonly source: string
+}
+
+function toolingSourceDir(root: string): string {
+  return join(root, 'tooling')
+}
+
+/**
+ * The roots a stack reference resolves against, mirroring `standardRoots` in
+ * `src/standards/read.ts`. No stack reference installs into a project any
+ * more, so the package corpus is the only root that answers there. The
+ * authoring root stays ahead of it for the toolkit's own repository, which
+ * authors the corpus this reads, and for a project authoring stacks of its
+ * own under the same layout.
+ */
+export function referenceRoots(root: string): ReferenceRoot[] {
+  return [
+    { dir: toolingSourceDir(root), label: 'tooling' },
+    {
+      dir: toolingSourceDir(PROJECT_ROOT),
+      label: join(PACKAGE_LABEL, 'tooling'),
+    },
+  ]
+}
+
+/** Resolves a stack's reference doc, working root first. */
+export function resolveReference(
+  root: string,
+  stack: string,
+): ResolvedReference | undefined {
+  for (const { dir, label } of referenceRoots(root)) {
+    const path = join(dir, stack, 'reference.md')
+    if (existsSync(path)) {
+      return { path, source: join(label, stack, 'reference.md') }
+    }
+  }
+
+  return undefined
+}
+
+export function readReference(reference: ResolvedReference): string {
+  return readFileSync(reference.path, 'utf8')
+}

package/src/tooling/scan.ts

@@ -0,0 +1,147 @@
+import { existsSync, readFileSync } from 'node:fs'
+import { join } from 'node:path'
+import { mergeSections } from '@/tooling/gitignore'
+import { ancestorsFirst, listFiles, type Manifest } from '@/tooling/manifest'
+import {
+  collectDeps,
+  collectScripts,
+  type DepState,
+  readPackage,
+  type ScriptState,
+} from '@/tooling/package'
+
+export interface ConfigState {
+  readonly rel: string
+  readonly stack: string
+  readonly state: 'new' | 'drifted' | 'matching'
+}
+
+export interface SeedState {
+  readonly rel: string
+  readonly state: 'missing' | 'present'
+}
+
+export interface EntryState {
+  readonly entry: string
+  readonly state: 'missing' | 'present'
+}
+
+export interface ScanResult {
+  readonly configs: readonly ConfigState[]
+  readonly seeds: readonly SeedState[]
+  readonly scripts: readonly ScriptState[]
+  readonly deps: readonly DepState[]
+  readonly gitignore: readonly EntryState[]
+  readonly hasPackageJson: boolean
+  readonly totalChanges: number
+}
+
+function isIdentical(a: string, b: string): boolean {
+  if (!existsSync(b)) return false
+  return readFileSync(a).equals(readFileSync(b))
+}
+
+/**
+ * Compares every stack in the chain against the target and reports what would
+ * change. Nothing is written. Which stack wins a duplicate differs per
+ * category and mirrors the bash: configs, seeds, and scripts resolve nearest
+ * stack first, while dependencies and gitignore entries resolve from the
+ * furthest ancestor inward.
+ */
+export function scan(chain: readonly Manifest[], target: string): ScanResult {
+  const configs: ConfigState[] = []
+  const seenConfigs = new Set<string>()
+
+  for (const manifest of chain) {
+    for (const rel of listFiles(manifest.configsDir)) {
+      if (seenConfigs.has(rel)) continue
+      seenConfigs.add(rel)
+
+      const source = join(manifest.configsDir, rel)
+      const dest = join(target, rel)
+      const state = !existsSync(dest)
+        ? 'new'
+        : isIdentical(source, dest)
+          ? 'matching'
+          : 'drifted'
+
+      configs.push({ rel, stack: manifest.name, state })
+    }
+  }
+
+  const seeds: SeedState[] = []
+  const seenSeeds = new Set<string>()
+
+  for (const manifest of chain) {
+    for (const rel of listFiles(manifest.seedsDir)) {
+      if (seenSeeds.has(rel)) continue
+      seenSeeds.add(rel)
+
+      seeds.push({
+        rel,
+        state: existsSync(join(target, rel)) ? 'present' : 'missing',
+      })
+    }
+  }
+
+  const packagePath = join(target, 'package.json')
+  const pkg = readPackage(packagePath)
+  const scripts = pkg ? collectScripts(chain, pkg) : []
+  const deps = pkg ? collectDeps(chain, pkg) : []
+
+  const gitignorePath = join(target, '.gitignore')
+  const gitignoreContent = existsSync(gitignorePath)
+    ? readFileSync(gitignorePath, 'utf8')
+    : ''
+  const gitignore = scanGitignore(chain, gitignoreContent)
+
+  const totalChanges =
+    configs.filter((entry) => entry.state !== 'matching').length +
+    seeds.filter((entry) => entry.state === 'missing').length +
+    scripts.filter((entry) => entry.state !== 'matching').length +
+    deps.filter((entry) => entry.state === 'missing').length +
+    gitignore.filter((entry) => entry.state === 'missing').length
+
+  return {
+    configs,
+    seeds,
+    scripts,
+    deps,
+    gitignore,
+    hasPackageJson: pkg !== undefined,
+    totalChanges,
+  }
+}
+
+/**
+ * Replays the merge against a copy of the file so an entry that an earlier
+ * section or stack would add is not also reported as missing.
+ */
+function scanGitignore(
+  chain: readonly Manifest[],
+  content: string,
+): EntryState[] {
+  const states: EntryState[] = []
+  const seen = new Set<string>()
+  let projected = content
+
+  for (const manifest of ancestorsFirst(chain)) {
+    for (const section of manifest.gitignore) {
+      for (const entry of section.entries) {
+        if (seen.has(entry)) continue
+        seen.add(entry)
+
+        const result = mergeSections(projected, [
+          { header: section.header, entries: [entry] },
+        ])
+        states.push({
+          entry,
+          state: result.added.length > 0 ? 'missing' : 'present',
+        })
+        projected = result.content
+      }
+    }
+  }
+
+  return states
+}

package/src/tooling/stamp.ts

@@ -0,0 +1,44 @@
+import { existsSync } from 'node:fs'
+import { join } from 'node:path'
+import { writeChainStamp } from '@/sync/stamp'
+import type { Manifest } from '@/tooling/manifest'
+import { readPackage } from '@/tooling/package'
+
+/**
+ * Whether the target is a workspace root, where every package resolves its own
+ * stack. `package.json` covers bun, npm, and yarn, and pnpm declares the same
+ * fact in its own file instead.
+ */
+export function isWorkspaceRoot(target: string): boolean {
+  if (existsSync(join(target, 'pnpm-workspace.yaml'))) return true
+
+  const workspaces = readPackage(join(target, 'package.json'))?.workspaces
+  return Array.isArray(workspaces) || isRecord(workspaces)
+}
+
+/**
+ * Records the chain an install resolved, and returns whether it wrote one. A
+ * workspace root is the single refusal: one chain recorded there would be a
+ * guess at what its packages hold, and the report reading tooling as unmeasured
+ * is the true answer rather than a clean one.
+ */
+export async function recordToolingChain(
+  toolkitRoot: string,
+  target: string,
+  chain: readonly Manifest[],
+  now: Date,
+): Promise<boolean> {
+  if (isWorkspaceRoot(target)) return false
+
+  await writeChainStamp(
+    target,
+    { domain: 'tooling', toolkitRoot },
+    chain.map((manifest) => manifest.name),
+    now,
+  )
+  return true
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === 'object' && value !== null && !Array.isArray(value)
+}

package/src/transcripts/fetch.ts

@@ -0,0 +1,156 @@
+import {
+  mkdirSync,
+  mkdtempSync,
+  readdirSync,
+  rmSync,
+  writeFileSync,
+} from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { readFileSync } from 'node:fs'
+import { execa } from 'execa'
+import {
+  type VideoMetadata,
+  formatUploadDate,
+  renderFrontmatter,
+  slugify,
+} from '@/transcripts/metadata'
+import { dedupeRollingCues, parseVtt, renderBody } from '@/transcripts/vtt'
+
+const YT_DLP_TIMEOUT_MS = 120_000
+
+export function ensureYtDlp(): void {
+  if (Bun.which('yt-dlp') === null) {
+    throw new Error(
+      'yt-dlp not found on PATH. Install it from https://github.com/yt-dlp/yt-dlp.',
+    )
+  }
+}
+
+interface FetchResult {
+  readonly metadata: VideoMetadata
+  readonly subPath: string | null
+}
+
+async function runYtDlp(url: string, workDir: string): Promise<FetchResult> {
+  const outputTemplate = join(workDir, '%(id)s.%(ext)s')
+  let stdout: string
+  try {
+    const result = await execa(
+      'yt-dlp',
+      [
+        '--dump-json',
+        '--no-simulate',
+        '--write-sub',
+        '--write-auto-sub',
+        '--sub-langs',
+        'en,en-orig',
+        '--sub-format',
+        'vtt',
+        '--skip-download',
+        '-o',
+        outputTemplate,
+        url,
+      ],
+      { timeout: YT_DLP_TIMEOUT_MS },
+    )
+    stdout = result.stdout
+  } catch (error) {
+    throw new Error(`yt-dlp failed for ${url}: ${lastLine(error)}`)
+  }
+
+  const info = JSON.parse(stdout) as Record<string, unknown>
+  const videoId = String(info['id'] ?? '').trim()
+  if (!videoId) throw new Error('yt-dlp returned no video id')
+
+  const metadata: VideoMetadata = {
+    videoId,
+    title: String(info['title'] || 'untitled'),
+    channel: String(info['channel'] || info['uploader'] || 'unknown'),
+    durationSeconds: Number(info['duration'] || 0),
+    published: formatUploadDate(asString(info['upload_date'])),
+    url: String(info['webpage_url'] || url),
+  }
+  const subPath = findSubtitle(workDir, videoId)
+  return { metadata, subPath }
+}
+
+interface WriteOptions {
+  readonly outDir: string
+  readonly keepTimestamps: boolean
+  readonly fetchedAt: string
+}
+
+export function writeTranscript(
+  metadata: VideoMetadata,
+  subPath: string | null,
+  { outDir, keepTimestamps, fetchedAt }: WriteOptions,
+): string {
+  mkdirSync(outDir, { recursive: true })
+  const slug = `${slugify(metadata.title)}--${metadata.videoId}`
+  const target = join(outDir, `${slug}.md`)
+
+  const hasTranscript = subPath !== null
+  let body = ''
+  if (subPath !== null) {
+    const cues = parseVtt(readFileSync(subPath, 'utf8'))
+    body = renderBody(dedupeRollingCues(cues), { keepTimestamps })
+  } else {
+    body =
+      'Transcript unavailable. Auto-captions are not enabled for this video.\n'
+  }
+
+  const frontmatter = renderFrontmatter(metadata, { hasTranscript, fetchedAt })
+  const heading = `# ${metadata.title}\n\n`
+  writeFileSync(target, frontmatter + heading + body)
+  return target
+}
+
+interface FetchOptions {
+  readonly outDir: string
+  readonly keepTimestamps: boolean
+}
+
+export async function fetchOne(
+  url: string,
+  { outDir, keepTimestamps }: FetchOptions,
+): Promise<string> {
+  const workDir = mkdtempSync(join(tmpdir(), 'yt-transcripts-'))
+  try {
+    const { metadata, subPath } = await runYtDlp(url, workDir)
+    return writeTranscript(metadata, subPath, {
+      outDir,
+      keepTimestamps,
+      fetchedAt: todayLocal(),
+    })
+  } finally {
+    rmSync(workDir, { recursive: true, force: true })
+  }
+}
+
+function findSubtitle(workDir: string, videoId: string): string | null {
+  const matches = readdirSync(workDir)
+    .filter((name) => name.startsWith(videoId) && name.endsWith('.vtt'))
+    .sort()
+  return matches.length ? join(workDir, matches[0]) : null
+}
+
+function todayLocal(): string {
+  const now = new Date()
+  const year = now.getFullYear()
+  const month = String(now.getMonth() + 1).padStart(2, '0')
+  const day = String(now.getDate()).padStart(2, '0')
+  return `${year}-${month}-${day}`
+}
+
+function asString(value: unknown): string | undefined {
+  return typeof value === 'string' ? value : undefined
+}
+
+function lastLine(error: unknown): string {
+  const stderr =
+    error instanceof Error && 'stderr' in error ? String(error.stderr) : ''
+  const lines = stderr.trim().split('\n').filter(Boolean)
+  if (lines.length) return lines[lines.length - 1]
+  return error instanceof Error ? error.message : String(error)
+}

package/src/transcripts/metadata.ts

@@ -0,0 +1,54 @@
+export interface VideoMetadata {
+  readonly videoId: string
+  readonly title: string
+  readonly channel: string
+  readonly durationSeconds: number
+  readonly published: string
+  readonly url: string
+}
+
+const SLUG_PATTERN = /[^a-z0-9]+/g
+
+export function slugify(text: string, maxLength = 50): string {
+  const cleaned = text
+    .toLowerCase()
+    .replace(SLUG_PATTERN, '-')
+    .replace(/(^-|-$)/g, '')
+  if (!cleaned) return 'untitled'
+  return cleaned.slice(0, maxLength).replace(/-$/, '') || 'untitled'
+}
+
+export function formatUploadDate(yyyymmdd: string | undefined): string {
+  if (!yyyymmdd || yyyymmdd.length !== 8 || !/^\d{8}$/.test(yyyymmdd)) return ''
+  return `${yyyymmdd.slice(0, 4)}-${yyyymmdd.slice(4, 6)}-${yyyymmdd.slice(6)}`
+}
+
+function yamlEscape(value: string): string {
+  return `"${value.replace(/\\/g, '\\\\').replace(/"/g, '\\"')}"`
+}
+
+interface FrontmatterOptions {
+  readonly hasTranscript: boolean
+  readonly fetchedAt: string
+}
+
+export function renderFrontmatter(
+  video: VideoMetadata,
+  { hasTranscript, fetchedAt }: FrontmatterOptions,
+): string {
+  const lines = [
+    '---',
+    `title: ${yamlEscape(video.title)}`,
+    `channel: ${yamlEscape(video.channel)}`,
+    `video_id: ${video.videoId}`,
+    `url: ${video.url}`,
+    `duration: ${video.durationSeconds}`,
+    video.published ? `published: ${video.published}` : 'published:',
+    `fetched_at: ${fetchedAt}`,
+    `has_transcript: ${hasTranscript ? 'true' : 'false'}`,
+    '---',
+    '',
+    '',
+  ]
+  return lines.join('\n')
+}

package/src/transcripts/vtt.ts

@@ -0,0 +1,114 @@
+export interface Cue {
+  readonly timestamp: string
+  readonly text: string
+}
+
+const VTT_TIMESTAMP = /^(\d\d:)?\d\d:\d\d\.\d{3} --> .*$/
+const VTT_INLINE_TAG = /<[^>]+>/g
+const VTT_NUMERIC_LINE = /^\d+$/
+
+export function parseVtt(raw: string): Cue[] {
+  const cues: Cue[] = []
+  let currentTimestamp: string | null = null
+  let currentLines: string[] = []
+  let inHeader = true
+
+  const flush = (): void => {
+    if (currentTimestamp !== null && currentLines.length) {
+      const text = currentLines.join(' ').trim()
+      if (text) cues.push({ timestamp: currentTimestamp, text })
+    }
+    currentTimestamp = null
+    currentLines = []
+  }
+
+  for (const rawLine of raw.split(/\r?\n/)) {
+    const line = rawLine.replace(/\s+$/, '')
+    if (inHeader) {
+      if (VTT_TIMESTAMP.test(line)) inHeader = false
+      else continue
+    }
+    if (VTT_TIMESTAMP.test(line)) {
+      flush()
+      currentTimestamp = line.split(' --> ', 1)[0].trim()
+      continue
+    }
+    if (line.trim() === '') {
+      flush()
+      continue
+    }
+    if (VTT_NUMERIC_LINE.test(line.trim())) continue
+    const cleaned = line.replace(VTT_INLINE_TAG, '').trim()
+    if (cleaned) currentLines.push(cleaned)
+  }
+  flush()
+  return cues
+}
+
+export function dedupeRollingCues(cues: Cue[]): Cue[] {
+  const deduped: Cue[] = []
+  let running = ''
+  for (const { timestamp, text } of cues) {
+    const maxOverlap = Math.min(running.length, text.length)
+    let overlap = 0
+    for (let size = maxOverlap; size > 0; size -= 1) {
+      if (running.endsWith(text.slice(0, size))) {
+        overlap = size
+        break
+      }
+    }
+    const addition = text.slice(overlap).trim()
+    if (!addition) continue
+    running = `${running} ${addition}`.trim()
+    deduped.push({ timestamp, text: addition })
+  }
+  return deduped
+}
+
+function vttTimestampToSeconds(stamp: string): number {
+  const parts = stamp.split(':')
+  let hours = '0'
+  let minutes: string
+  let rest: string
+  if (parts.length === 3) [hours, minutes, rest] = parts
+  else if (parts.length === 2) [minutes, rest] = parts
+  else return 0
+  const seconds = rest.split('.', 1)[0]
+  return Number(hours) * 3600 + Number(minutes) * 60 + Number(seconds)
+}
+
+function formatSeconds(seconds: number): string {
+  const minutes = Math.floor(seconds / 60)
+  const secs = seconds % 60
+  return `${String(minutes).padStart(2, '0')}:${String(secs).padStart(2, '0')}`
+}
+
+interface RenderOptions {
+  readonly keepTimestamps: boolean
+}
+
+export function renderBody(
+  cues: Cue[],
+  { keepTimestamps }: RenderOptions,
+): string {
+  if (!cues.length) return ''
+  if (keepTimestamps) {
+    const lines = cues.map(
+      ({ timestamp, text }) =>
+        `[${formatSeconds(vttTimestampToSeconds(timestamp))}] ${text}`,
+    )
+    return `${lines.join('\n')}\n`
+  }
+  const joined = cues.map(({ text }) => text).join(' ')
+  return `${paragraphWrap(joined)}\n`
+}
+
+function paragraphWrap(text: string, wordsPerParagraph = 60): string {
+  const words = text.split(/\s+/).filter(Boolean)
+  if (!words.length) return ''
+  const paragraphs: string[] = []
+  for (let start = 0; start < words.length; start += wordsPerParagraph) {
+    paragraphs.push(words.slice(start, start + wordsPerParagraph).join(' '))
+  }
+  return paragraphs.join('\n\n')
+}