@monoes/monograph 1.1.0 → 1.2.0

package/src/ingest/transcribe.ts

@@ -0,0 +1,254 @@
+/**
+ * Video / audio transcription support.
+ *
+ * Mirrors graphify's `transcribe.py`: converts video/audio files or URLs to
+ * plain-text transcript files that the pipeline can then extract as document
+ * nodes.
+ *
+ * Implementation strategy
+ * -----------------------
+ * TypeScript cannot import Python's faster-whisper directly.  Instead this
+ * module shells out to the faster-whisper CLI (`faster-whisper`) or the
+ * `whisper` CLI (OpenAI's original).  yt-dlp is used for URL audio downloads.
+ * If neither tool is installed, functions throw descriptive errors.
+ *
+ * Environment variables
+ * ---------------------
+ * MONOGRAPH_WHISPER_MODEL   - Whisper model name (default: "base")
+ * MONOGRAPH_WHISPER_PROMPT  - Override initial prompt for Whisper
+ */
+
+import { execFile } from 'child_process';
+import { promisify } from 'util';
+import { createHash } from 'crypto';
+import { mkdirSync, existsSync, writeFileSync } from 'fs';
+import { readFile, writeFile } from 'fs/promises';
+import path from 'path';
+
+const execFileAsync = promisify(execFile);
+
+// ── Constants ──────────────────────────────────────────────────────────────────
+
+export const VIDEO_EXTENSIONS = new Set([
+  '.mp4', '.mov', '.webm', '.mkv', '.avi', '.m4v',
+  '.mp3', '.wav', '.m4a', '.ogg',
+]);
+
+const URL_PREFIXES = ['http://', 'https://', 'www.'];
+const DEFAULT_MODEL = 'base';
+const FALLBACK_PROMPT = 'Use proper punctuation and paragraph breaks.';
+const DEFAULT_TRANSCRIPTS_DIR = 'monograph-out/transcripts';
+
+// ── Helpers ────────────────────────────────────────────────────────────────────
+
+export function isUrl(input: string): boolean {
+  return URL_PREFIXES.some(p => input.startsWith(p));
+}
+
+function getModel(): string {
+  return process.env['MONOGRAPH_WHISPER_MODEL'] ?? DEFAULT_MODEL;
+}
+
+async function whisperCliAvailable(): Promise<string | null> {
+  for (const cmd of ['faster-whisper', 'whisper']) {
+    try {
+      await execFileAsync(cmd, ['--help']);
+      return cmd;
+    } catch {
+      // not found — try next
+    }
+  }
+  return null;
+}
+
+async function ytdlpAvailable(): Promise<boolean> {
+  try {
+    await execFileAsync('yt-dlp', ['--version']);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+// ── Download audio ─────────────────────────────────────────────────────────────
+
+/**
+ * Download audio from a URL using yt-dlp.
+ * Returns the path to the downloaded audio file.
+ * Cached: if a file with the same URL hash already exists it is returned immediately.
+ *
+ * @throws {Error} if yt-dlp is not installed.
+ */
+export async function downloadAudio(url: string, outputDir: string): Promise<string> {
+  if (!await ytdlpAvailable()) {
+    throw new Error(
+      'Video URL download requires yt-dlp. Install it with: pip install yt-dlp',
+    );
+  }
+
+  mkdirSync(outputDir, { recursive: true });
+
+  const urlHash = createHash('sha1').update(url).digest('hex').slice(0, 12);
+  const audioExts = ['.m4a', '.opus', '.mp3', '.ogg', '.wav', '.webm'];
+
+  // Return cached file if present
+  for (const ext of audioExts) {
+    const candidate = path.join(outputDir, `yt_${urlHash}${ext}`);
+    if (existsSync(candidate)) {
+      return candidate;
+    }
+  }
+
+  const outTemplate = path.join(outputDir, `yt_${urlHash}.%(ext)s`);
+  await execFileAsync('yt-dlp', [
+    '--format', 'bestaudio[ext=m4a]/bestaudio/best',
+    '--output', outTemplate,
+    '--quiet',
+    '--no-warnings',
+    '--no-playlist',
+    url,
+  ]);
+
+  // Find the downloaded file
+  for (const ext of audioExts) {
+    const candidate = path.join(outputDir, `yt_${urlHash}${ext}`);
+    if (existsSync(candidate)) return candidate;
+  }
+
+  throw new Error(`yt-dlp download completed but output file not found for URL: ${url}`);
+}
+
+// ── Whisper prompt ─────────────────────────────────────────────────────────────
+
+/**
+ * Build a domain hint for Whisper from a set of god-node labels.
+ * Mirrors graphify's `build_whisper_prompt`.
+ */
+export function buildWhisperPrompt(godNodeLabels: string[]): string {
+  const override = process.env['MONOGRAPH_WHISPER_PROMPT'];
+  if (override) return override;
+
+  if (godNodeLabels.length === 0) return FALLBACK_PROMPT;
+
+  const topics = godNodeLabels.slice(0, 5).join(', ');
+  return `Technical discussion about ${topics}. Use proper punctuation and paragraph breaks.`;
+}
+
+// ── Transcribe one file ────────────────────────────────────────────────────────
+
+export interface TranscribeOptions {
+  /** Directory to write transcript files. Default: monograph-out/transcripts */
+  outputDir?: string;
+  /** Initial prompt / domain hint passed to Whisper. */
+  initialPrompt?: string;
+  /** Re-transcribe even if transcript already exists. Default: false */
+  force?: boolean;
+}
+
+export interface TranscribeResult {
+  transcriptPath: string;
+  /** Number of non-empty lines in the transcript. */
+  lineCount: number;
+  /** Whether the transcript was loaded from cache (not freshly generated). */
+  fromCache: boolean;
+}
+
+/**
+ * Transcribe a video/audio file or URL to a plain-text transcript.
+ *
+ * Requires either `faster-whisper` or `whisper` CLI to be on the PATH.
+ * For URLs, also requires `yt-dlp`.
+ *
+ * @param input   - A local file path or a URL (http/https/www).
+ * @param options - Transcription options.
+ * @returns Resolved transcript path and metadata.
+ */
+export async function transcribe(
+  input: string,
+  options: TranscribeOptions = {},
+): Promise<TranscribeResult> {
+  const outDir = options.outputDir ?? DEFAULT_TRANSCRIPTS_DIR;
+  mkdirSync(outDir, { recursive: true });
+
+  // Resolve audio path
+  let audioPath: string;
+  if (isUrl(input)) {
+    const dlDir = path.join(outDir, 'downloads');
+    audioPath = await downloadAudio(input, dlDir);
+  } else {
+    audioPath = input;
+  }
+
+  const stem = path.basename(audioPath, path.extname(audioPath));
+  const transcriptPath = path.join(outDir, `${stem}.txt`);
+
+  // Return cached transcript if it exists
+  if (!options.force && existsSync(transcriptPath)) {
+    const content = await readFile(transcriptPath, 'utf-8');
+    const lines = content.split('\n').filter(l => l.trim().length > 0);
+    return { transcriptPath, lineCount: lines.length, fromCache: true };
+  }
+
+  // Find available Whisper CLI
+  const whisperCmd = await whisperCliAvailable();
+  if (!whisperCmd) {
+    throw new Error(
+      'Video transcription requires faster-whisper or whisper CLI. ' +
+      'Install with: pip install faster-whisper  or  pip install openai-whisper',
+    );
+  }
+
+  const model = getModel();
+  const prompt = options.initialPrompt ?? FALLBACK_PROMPT;
+
+  const args = [audioPath, '--model', model, '--output_format', 'txt', '--output_dir', outDir];
+  if (prompt) args.push('--initial_prompt', prompt);
+
+  await execFileAsync(whisperCmd, args);
+
+  // Whisper may write <stem>.txt directly or <stem>.<ext>.txt — normalise
+  if (!existsSync(transcriptPath)) {
+    // Try with original extension included in stem
+    const altPath = path.join(outDir, `${path.basename(audioPath)}.txt`);
+    if (existsSync(altPath)) {
+      // Rename to canonical path
+      const content = await readFile(altPath, 'utf-8');
+      writeFileSync(transcriptPath, content, 'utf-8');
+    }
+  }
+
+  const content = existsSync(transcriptPath)
+    ? await readFile(transcriptPath, 'utf-8')
+    : '';
+  const lines = content.split('\n').filter(l => l.trim().length > 0);
+
+  return { transcriptPath, lineCount: lines.length, fromCache: false };
+}
+
+// ── Transcribe many ────────────────────────────────────────────────────────────
+
+/**
+ * Transcribe a list of video/audio files or URLs.
+ * Already-transcribed files are returned from cache instantly.
+ * The `initialPrompt` is shared across all files.
+ *
+ * @param inputs  - Array of file paths or URLs.
+ * @param options - Shared transcription options.
+ * @returns Array of TranscribeResult (one per input, in order).
+ */
+export async function transcribeAll(
+  inputs: string[],
+  options: TranscribeOptions = {},
+): Promise<TranscribeResult[]> {
+  if (inputs.length === 0) return [];
+
+  const results: TranscribeResult[] = [];
+  for (const input of inputs) {
+    try {
+      results.push(await transcribe(input, options));
+    } catch (err) {
+      console.error(`[monograph transcribe] Failed for ${input}: ${(err as Error).message}`);
+    }
+  }
+  return results;
+}

package/src/ingest/url-ingest.ts

@@ -0,0 +1,87 @@
+import { createHash } from 'crypto';
+import { validateUrl } from '../security/safe-fetch.js';
+import { classifyFile } from '../analysis/file-classifier.js';
+
+export type UrlType = 'paper' | 'github' | 'video' | 'webpage' | 'pdf' | 'image';
+
+export interface IngestResult {
+  id: string;
+  url: string;
+  type: UrlType;
+  title: string;
+  snippet: string;
+  fetchedAt: string;
+}
+
+export interface IngestOptions {
+  fetch?: typeof globalThis.fetch;
+  maxBytes?: number;
+}
+
+const URL_TYPE_PATTERNS: Array<[RegExp, UrlType]> = [
+  [/arxiv\.org|semanticscholar\.org|openreview\.net/, 'paper'],
+  [/github\.com/, 'github'],
+  [/youtube\.com|youtu\.be/, 'video'],
+  [/\.pdf($|\?)/, 'pdf'],
+];
+
+export function classifyUrlType(url: string): UrlType {
+  for (const [pattern, type] of URL_TYPE_PATTERNS) {
+    if (pattern.test(url)) return type;
+  }
+  const fileType = classifyFile(url);
+  if (fileType === 'IMAGE') return 'image';
+  if (fileType === 'PAPER') return 'paper';
+  return 'webpage';
+}
+
+function extractTitle(html: string): string {
+  const m = /<title[^>]*>([^<]+)<\/title>/i.exec(html);
+  return m ? m[1].trim() : 'Untitled';
+}
+
+function extractSnippet(html: string, maxLen = 500): string {
+  const text = html
+    .replace(/<script[^>]*>[\s\S]*?<\/script>/gi, '')
+    .replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '')
+    .replace(/<[^>]+>/g, ' ')
+    .replace(/\s+/g, ' ')
+    .trim();
+  return text.slice(0, maxLen);
+}
+
+export async function ingestUrl(url: string, options: IngestOptions = {}): Promise<IngestResult> {
+  validateUrl(url);
+
+  const { fetch: fetchFn = globalThis.fetch, maxBytes = 10 * 1024 * 1024 } = options;
+
+  const response = await fetchFn(url, { redirect: 'follow' } as RequestInit);
+  if (!response.ok) throw new Error(`HTTP ${response.status} fetching ${url}`);
+  // Re-validate the final URL after redirects to prevent SSRF via open redirects.
+  // Only validate when response.url is truthy (some fetch impls omit it on same-URL responses).
+  if (response.url) validateUrl(response.url);
+
+  const contentLength = response.headers.get('content-length');
+  if (contentLength && parseInt(contentLength, 10) > maxBytes) {
+    throw new Error(`Response too large: ${contentLength} bytes exceeds ${maxBytes}`);
+  }
+
+  const text = await response.text();
+  if (text.length > maxBytes) throw new Error(`Response too large: body exceeds ${maxBytes} bytes`);
+
+  const type = classifyUrlType(url);
+  const contentType = response.headers.get('content-type') ?? '';
+  const isHtml = contentType.includes('text/html') || text.trimStart().startsWith('<');
+
+  const title = isHtml ? extractTitle(text) : (url.split('/').pop() ?? url);
+  const snippet = isHtml ? extractSnippet(text) : text.slice(0, 500);
+
+  return {
+    id: createHash('sha256').update(url).digest('hex').slice(0, 16),
+    url,
+    type,
+    title,
+    snippet,
+    fetchedAt: new Date().toISOString(),
+  };
+}

package/src/init/agent-hooks.ts

@@ -0,0 +1,86 @@
+// Install/uninstall monograph quality gate into AI coding agents (Claude Code, Codex).
+
+export type AgentHooksTarget = 'claude-code' | 'codex' | 'auto';
+
+export interface SetupHooksOptions {
+  root: string;
+  target?: AgentHooksTarget;
+  command?: string;
+  uninstall?: boolean;
+}
+
+export interface SetupHooksResult {
+  target: AgentHooksTarget;
+  installed: boolean;
+  modifiedFiles: string[];
+  message: string;
+}
+
+export const AGENTS_BLOCK_START = '<!-- monograph-gate-start -->';
+export const AGENTS_BLOCK_END   = '<!-- monograph-gate-end -->';
+export const MONOGRAPH_GATE_SCRIPT = 'npx monograph check --since $(git merge-base HEAD main)';
+
+/** Build the AGENTS.md block content. */
+export function buildAgentsMdBlock(command: string): string {
+  return [
+    AGENTS_BLOCK_START,
+    '',
+    '## Pre-commit Quality Gate (monograph)',
+    '',
+    'Before completing any coding task, run:',
+    '',
+    '```sh',
+    command,
+    '```',
+    '',
+    'Fix any new issues before marking the task complete.',
+    '',
+    AGENTS_BLOCK_END,
+  ].join('\n');
+}
+
+/** Merge the monograph gate block into AGENTS.md content (idempotent). */
+export function mergeAgentsMdBlock(existing: string, block: string): string {
+  const startIdx = existing.indexOf(AGENTS_BLOCK_START);
+  const endIdx = existing.indexOf(AGENTS_BLOCK_END);
+  if (startIdx !== -1 && endIdx !== -1) {
+    return existing.slice(0, startIdx) + block + existing.slice(endIdx + AGENTS_BLOCK_END.length);
+  }
+  return existing ? existing + '\n\n' + block : block;
+}
+
+/** Remove the managed block from AGENTS.md. */
+export function removeAgentsMdBlock(content: string): string {
+  const startIdx = content.indexOf(AGENTS_BLOCK_START);
+  const endIdx = content.indexOf(AGENTS_BLOCK_END);
+  if (startIdx === -1 || endIdx === -1) return content;
+  return content.slice(0, startIdx).trimEnd() + '\n' + content.slice(endIdx + AGENTS_BLOCK_END.length).trimStart();
+}
+
+/** Build a Claude Code settings.json PreToolUse hook entry. */
+export function buildClaudeCodeHookEntry(command: string): Record<string, unknown> {
+  return {
+    hooks: {
+      PreToolUse: [
+        {
+          matcher: 'Edit|Write|MultiEdit',
+          hooks: [{ type: 'command', command: `sh -c '${command} --quiet || true'` }],
+        },
+      ],
+    },
+  };
+}
+
+/** Merge a hook entry into an existing Claude Code settings object (idempotent). */
+export function mergeClaudeCodeSettings(
+  existing: Record<string, unknown>,
+  entry: Record<string, unknown>,
+): Record<string, unknown> {
+  const result = { $schema: '', ...existing, ...entry };
+  if (!result.$schema) { (result as Record<string, unknown>).$schema = undefined; }
+  return result;
+}
+
+export const DEFAULT_SETUP_RESULT: SetupHooksResult = {
+  target: 'auto', installed: false, modifiedFiles: [], message: 'No agent detected',
+};

package/src/init/git-hooks.ts

@@ -0,0 +1,71 @@
+// Detect git hooks manager and install/uninstall a pre-commit monograph gate.
+
+export type GitHooksManager = 'husky' | 'lefthook' | 'raw' | 'none';
+
+export interface GitHooksInstallOptions {
+  root: string;
+  branch?: string;
+  command?: string;
+}
+
+export interface GitHooksInstallResult {
+  manager: GitHooksManager;
+  installed: boolean;
+  hookPath: string;
+  message: string;
+}
+
+const MANAGED_BLOCK_START = '# monograph-gate-start';
+const MANAGED_BLOCK_END   = '# monograph-gate-end';
+
+/** Detect which hooks manager is active in the project. */
+export function detectHooksManager(rootFiles: string[]): GitHooksManager {
+  if (rootFiles.some(f => f.includes('.husky/'))) return 'husky';
+  if (rootFiles.some(f => f.includes('lefthook.yml') || f.includes('lefthook.yaml'))) return 'lefthook';
+  if (rootFiles.some(f => f.includes('.git/hooks/'))) return 'raw';
+  return 'none';
+}
+
+/** Validate a branch name against shell injection. */
+export function validateBranchName(branch: string): boolean {
+  return /^[a-zA-Z0-9/_.\-]+$/.test(branch) && !branch.includes('..') && branch.length < 256;
+}
+
+/** Generate the pre-commit hook script content. */
+export function renderedHookScript(opts: GitHooksInstallOptions): string {
+  const branch = opts.branch ?? 'main';
+  const cmd = opts.command ?? 'npx monograph check';
+  return [
+    '#!/bin/sh',
+    MANAGED_BLOCK_START,
+    `BASE=$(git merge-base HEAD ${branch} 2>/dev/null || echo "")`,
+    `if [ -n "$BASE" ]; then`,
+    `  ${cmd} --since "$BASE"`,
+    `else`,
+    `  ${cmd}`,
+    `fi`,
+    MANAGED_BLOCK_END,
+  ].join('\n');
+}
+
+/** Build the hook content by merging with existing content (idempotent). */
+export function mergeHookContent(existing: string, script: string): string {
+  const startIdx = existing.indexOf(MANAGED_BLOCK_START);
+  const endIdx = existing.indexOf(MANAGED_BLOCK_END);
+  if (startIdx !== -1 && endIdx !== -1) {
+    return existing.slice(0, startIdx) + script + existing.slice(endIdx + MANAGED_BLOCK_END.length);
+  }
+  return existing ? existing + '\n' + script : script;
+}
+
+/** Remove the managed block from hook content. */
+export function removeHookBlock(content: string): string {
+  const startIdx = content.indexOf(MANAGED_BLOCK_START);
+  const endIdx = content.indexOf(MANAGED_BLOCK_END);
+  if (startIdx === -1 || endIdx === -1) return content;
+  return content.slice(0, startIdx).trimEnd() + content.slice(endIdx + MANAGED_BLOCK_END.length);
+}
+
+export const GIT_HOOK_INSTALL_RESULT_NONE: GitHooksInstallResult = {
+  manager: 'none', installed: false, hookPath: '', message: 'No hooks manager detected',
+};

package/src/init/project-detection.ts

@@ -0,0 +1,105 @@
+// Inspects a project root to scaffold a tailored monograph config.
+
+export type DetectedFramework = 'react' | 'vue' | 'svelte' | 'angular' | 'none';
+export type DetectedTestRunner = 'vitest' | 'jest' | 'playwright' | 'none';
+export type DetectedPackageManager = 'pnpm' | 'yarn' | 'npm' | 'bun' | 'unknown';
+export type DetectedMonorepoTool = 'pnpm-workspaces' | 'npm-workspaces' | 'yarn-workspaces' | 'nx' | 'turborepo' | 'none';
+
+export interface ProjectInfo {
+  root: string;
+  hasTypeScript: boolean;
+  framework: DetectedFramework;
+  testRunner: DetectedTestRunner;
+  packageManager: DetectedPackageManager;
+  monorepoTool: DetectedMonorepoTool;
+  workspaceGlobs: string[];
+  hasStorybook: boolean;
+  entryPoints: string[];
+  testPatterns: string[];
+}
+
+export function detectFramework(deps: Record<string, string>): DetectedFramework {
+  if ('react' in deps || 'react-dom' in deps) return 'react';
+  if ('vue' in deps) return 'vue';
+  if ('svelte' in deps) return 'svelte';
+  if ('@angular/core' in deps) return 'angular';
+  return 'none';
+}
+
+export function detectTestRunner(deps: Record<string, string>): DetectedTestRunner {
+  if ('vitest' in deps) return 'vitest';
+  if ('jest' in deps || '@jest/core' in deps) return 'jest';
+  if ('@playwright/test' in deps) return 'playwright';
+  return 'none';
+}
+
+export function detectPackageManager(files: string[]): DetectedPackageManager {
+  if (files.includes('pnpm-lock.yaml')) return 'pnpm';
+  if (files.includes('bun.lockb')) return 'bun';
+  if (files.includes('yarn.lock')) return 'yarn';
+  if (files.includes('package-lock.json')) return 'npm';
+  return 'unknown';
+}
+
+export function detectProject(
+  rootFiles: string[],
+  packageJson: { dependencies?: Record<string, string>; devDependencies?: Record<string, string>; workspaces?: string[] | { packages: string[] } } | null,
+  hasPnpmWorkspace: boolean,
+): ProjectInfo {
+  const allDeps = { ...(packageJson?.dependencies ?? {}), ...(packageJson?.devDependencies ?? {}) };
+
+  const workspaceGlobs = hasPnpmWorkspace
+    ? ['packages/*']
+    : Array.isArray(packageJson?.workspaces)
+      ? packageJson.workspaces
+      : (packageJson?.workspaces as { packages?: string[] })?.packages ?? [];
+
+  const monorepoTool: DetectedMonorepoTool = hasPnpmWorkspace
+    ? 'pnpm-workspaces'
+    : workspaceGlobs.length > 0
+      ? 'npm-workspaces'
+      : 'none';
+
+  const framework = detectFramework(allDeps);
+  const testRunner = detectTestRunner(allDeps);
+  const packageManager = detectPackageManager(rootFiles);
+
+  const testPatterns = testRunner !== 'none'
+    ? ['**/*.test.{ts,tsx,js,jsx}', '**/*.spec.{ts,tsx,js,jsx}']
+    : [];
+
+  const entryPoints = ['src/index.ts', 'src/index.tsx', 'src/main.ts', 'index.ts']
+    .filter(ep => rootFiles.some(f => f.endsWith(ep.replace('src/', ''))));
+
+  return {
+    root: '',
+    hasTypeScript: rootFiles.includes('tsconfig.json'),
+    framework,
+    testRunner,
+    packageManager,
+    monorepoTool,
+    workspaceGlobs,
+    hasStorybook: rootFiles.some(f => f.includes('.storybook')),
+    entryPoints: entryPoints.length > 0 ? entryPoints : ['src/index.ts'],
+    testPatterns,
+  };
+}
+
+export function buildJsonConfig(info: ProjectInfo): string {
+  const config = {
+    $schema: 'https://monograph.dev/schema.json',
+    entryPoints: info.entryPoints,
+    ignore: ['**/*.d.ts', '**/*.test.*', '**/*.spec.*', ...(info.testPatterns.length > 0 ? info.testPatterns : [])],
+    ...(info.workspaceGlobs.length > 0 ? { workspaces: info.workspaceGlobs } : {}),
+  };
+  return JSON.stringify(config, null, 2);
+}
+
+export function buildTomlConfig(info: ProjectInfo): string {
+  const lines = ['[monograph]'];
+  lines.push(`entry_points = [${info.entryPoints.map(e => `"${e}"`).join(', ')}]`);
+  if (info.workspaceGlobs.length > 0) {
+    lines.push(`workspaces = [${info.workspaceGlobs.map(g => `"${g}"`).join(', ')}]`);
+  }
+  return lines.join('\n');
+}