@lincoln504/pi-research 0.1.12 → 1.0.0

package/src/youtube/video-id.ts

@@ -0,0 +1,118 @@
+/**
+ * YouTube video-ID extraction
+ *
+ * Pure helpers (no I/O) for turning the URLs a researcher already has in its
+ * link pool into canonical 11-character YouTube video IDs. The researcher
+ * selects which YouTube links to transcribe — exactly like it selects which
+ * pages to scrape — and this module validates and normalizes that selection.
+ */
+
+/** A YouTube video ID is always 11 chars from the URL-safe base64 alphabet. */
+const VIDEO_ID_RE = /^[A-Za-z0-9_-]{11}$/;
+
+const YOUTUBE_HOSTS = new Set([
+  'youtube.com',
+  'www.youtube.com',
+  'm.youtube.com',
+  'music.youtube.com',
+  'youtube-nocookie.com',
+  'www.youtube-nocookie.com',
+  'youtu.be',
+]);
+
+/** Path prefixes that carry the video ID as the next path segment. */
+const PATH_PREFIXES = ['embed', 'shorts', 'live', 'v'];
+
+/**
+ * Returns true when the input looks like a YouTube watch/share URL (or a bare
+ * 11-char video ID). Used to filter a mixed link pool down to YouTube links.
+ */
+export function isYoutubeUrl(input: string): boolean {
+  const trimmed = input.trim();
+  if (VIDEO_ID_RE.test(trimmed)) return true;
+  try {
+    const url = new URL(trimmed);
+    return YOUTUBE_HOSTS.has(url.hostname.toLowerCase());
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Extract the canonical 11-character video ID from any common YouTube URL form,
+ * or from a bare ID. Returns null when no valid ID can be found — callers must
+ * treat null as "not a usable YouTube video" and skip it.
+ *
+ * Handles:
+ *   - bare ID:                         dQw4w9WgXcQ
+ *   - youtu.be/<id>:                   https://youtu.be/dQw4w9WgXcQ?si=...
+ *   - watch?v=<id>:                    https://www.youtube.com/watch?v=dQw4w9WgXcQ&t=42
+ *   - /embed/<id>, /shorts/<id>,
+ *     /live/<id>, /v/<id>:             https://www.youtube.com/shorts/dQw4w9WgXcQ
+ */
+export function extractVideoId(input: string): string | null {
+  const trimmed = input.trim();
+  if (!trimmed) return null;
+  if (VIDEO_ID_RE.test(trimmed)) return trimmed;
+
+  let url: URL;
+  try {
+    url = new URL(trimmed);
+  } catch {
+    return null;
+  }
+
+  const host = url.hostname.toLowerCase();
+  if (!YOUTUBE_HOSTS.has(host)) return null;
+
+  // youtu.be/<id>
+  if (host === 'youtu.be') {
+    const id = url.pathname.split('/').filter(Boolean)[0];
+    return id && VIDEO_ID_RE.test(id) ? id : null;
+  }
+
+  // watch?v=<id> (also covers /watch and music.youtube.com/watch)
+  const vParam = url.searchParams.get('v');
+  if (vParam && VIDEO_ID_RE.test(vParam)) return vParam;
+
+  // /embed/<id>, /shorts/<id>, /live/<id>, /v/<id>
+  const segments = url.pathname.split('/').filter(Boolean);
+  if (segments.length >= 2 && PATH_PREFIXES.includes(segments[0]!.toLowerCase())) {
+    const id = segments[1]!;
+    if (VIDEO_ID_RE.test(id)) return id;
+  }
+
+  return null;
+}
+
+/**
+ * Map a list of raw link-pool entries to a deduplicated list of valid video IDs
+ * (preserving order, capped to `max`), while reporting which inputs were not
+ * usable YouTube links so the caller can surface that to the agent.
+ */
+export function selectVideoIds(
+  urls: string[],
+  max: number,
+): { videoIds: string[]; rejected: string[] } {
+  const seen = new Set<string>();
+  const videoIds: string[] = [];
+  const rejected: string[] = [];
+
+  for (const raw of urls) {
+    const id = extractVideoId(raw);
+    if (!id) {
+      rejected.push(raw);
+      continue;
+    }
+    if (seen.has(id)) continue;
+    seen.add(id);
+    if (videoIds.length < max) videoIds.push(id);
+  }
+
+  return { videoIds, rejected };
+}
+
+/** Canonical watch URL for a video ID (used in result formatting). */
+export function watchUrl(videoId: string): string {
+  return `https://www.youtube.com/watch?v=${videoId}`;
+}

package/config/searxng/default-settings.yml

@@ -1,189 +0,0 @@
-# SearXNG Default Configuration for pi-search-scrape Extension
-# IMPROVED: Better timeouts, removed problematic engines, added real user agent
-
-general:
-  instance_name: "pi SearXNG"
-  debug: false
-  privacypolicy_url: false
-  contact_url: false
-  enable_metrics: true  # ENABLED for debugging engine issues
-  open_metrics: ''
-
-brand:
-  new_issue_url: https://github.com/searxng/searxng/issues/new
-  docs_url: https://docs.searxng.org/
-  public_instances: https://searx.space
-  wiki_url: https://github.com/searxng/searxng/wiki
-  issue_url: https://github.com/searxng/searxng/issues
-
-server:
-  port: 8080
-  bind_address: "0.0.0.0"
-  secret_key: ""
-  base_url: false
-  limiter: false
-  public_instance: false
-  image_proxy: false
-  http_protocol_version: "1.1"
-  method: "GET"
-  default_http_headers:
-    # REAL BROWSER USER AGENT - prevents bot detection
-    User-Agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
-    Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8"
-    Accept-Language: "en-US,en;q=0.9"
-    Accept-Encoding: "gzip, deflate, br"
-    X-Content-Type-Options: nosniff
-    X-Download-Options: noopen
-    X-Robots-Tag: noindex, nofollow
-    Referrer-Policy: no-referrer
-
-search:
-  safe_search: 1
-  autocomplete: ""
-  autocomplete_min: 4
-  favicon_resolver: ""
-  default_lang: "en-US"
-  ban_time_on_fail: 5
-  max_ban_time_on_fail: 120
-  formats:
-    - html
-    - json
-    - csv
-    - rss
-  suspended_times:
-    # Long backoff prevents hammering blocked engines — re-check after 24h
-    SearxEngineAccessDenied: 86400
-    SearxEngineCaptcha: 86400
-    SearxEngineTooManyRequests: 3600
-    cf_SearxEngineCaptcha: 1296000
-    cf_SearxEngineAccessDenied: 86400
-    recaptcha_SearxEngineCaptcha: 604800
-
-ui:
-  static_path: ""
-  templates_path: ""
-  query_in_title: false
-  default_theme: simple
-  center_alignment: false
-  default_locale: ""
-  results_on_new_tab: false
-  theme_args:
-    simple_style: auto
-  search_on_category_select: true
-  hotkeys: default
-  url_formatting: pretty
-
-outgoing:
-  request_timeout: 20.0        # INCREASED from 10.0
-  max_request_timeout: 60.0      # INCREASED from 30.0
-  useragent_suffix: ""
-  pool_connections: 200            # INCREASED from 150
-  pool_maxsize: 30                # INCREASED from 20
-  enable_http2: true              # ENABLED (was false)
-  retries: 3                      # INCREASED from 2
-  using_tor_proxy: false
-  extra_proxy_timeout: 15
-
-doi_resolvers:
-  oadoi.org: 'https://oadoi.org/'
-  doi.org: 'https://doi.org/'
-default_doi_resolver: 'oadoi.org'
-
-# Search engines configuration
-#
-# Engine reliability for a private low-volume self-hosted instance (2026):
-#   Bing       — most consistently available; scrapes bing.com directly; weak on ambiguous queries
-#   Brave      — independent index; moderate bot detection; occasional suspension at burst rates
-#   Yahoo      — different backend from Bing; low blocking rate; lower result quality
-#   Qwant      — EU-based independent index; low-volume private use is generally fine
-#   DuckDuckGo — ddg-lite endpoint (no vqd); intermittent IP blocks (issue #4824, open)
-#   StartPage  — DISABLED: broken in current SearXNG due to missing sgt param (issue #4549, open)
-#   Google     — DISABLED: IP-blocks any nontrivial automated use
-#
-# If BRAVE_SEARCH_API_KEY is set, searxng-lifecycle.ts injects a braveapi engine at weight 1.2
-# which uses the official REST API and is not subject to bot detection.
-
-engines:
-  # Bing: most reliable free general-web engine for private instances at low volume
-  - name: bing
-    engine: bing
-    shortcut: bi
-    weight: 1.5
-
-  # Brave (scraping): independent index, different coverage from Bing
-  # Can enter suspension after bursts; max_page 10 respected by SearXNG
-  - name: brave
-    engine: brave
-    shortcut: bv
-    weight: 0.8
-
-  # Yahoo: distinct backend, low blocking rate, fills gaps when others are suspended
-  - name: yahoo
-    engine: yahoo
-    shortcut: yh
-    weight: 0.8
-
-  # Qwant: EU-based independent index, low-volume private use stable
-  - name: qwant
-    engine: qwant
-    shortcut: qw
-    weight: 0.7
-
-  # DuckDuckGo: ddg-lite endpoint (no vqd); intermittent IP blocks (issue #4824 open)
-  - name: duckduckgo
-    engine: duckduckgo
-    shortcut: ddg
-    weight: 0.6
-
-  # DISABLED: startpage — sgt parameter missing in SearXNG parser (issue #4549, open)
-  # DISABLED: google — IP-blocks automated access within minutes
-
-  # Wikipedia: Encyclopedic reference
-  # - Reliability: HIGH (uses REST API)
-  # - Note: Keep weight low to prioritize search engines
-  - name: wikipedia
-    engine: wikipedia
-    shortcut: wp
-    categories: [general]
-    weight: 0.3  # LOWER - reference only, not primary search
-
-  # Images (only when category=images is requested)
-  - name: bing images
-    engine: bing_images
-    categories: [images]
-    shortcut: bii
-
-  # News (only when category=news is requested)
-  - name: bing news
-    engine: bing_news
-    categories: [news]
-    shortcut: bin
-
-  # REMOVED: brave_news (engine not available in SearXNG container)
-
-  # Videos (only when category=videos is requested)
-  - name: bing videos
-    engine: bing_videos
-    categories: [videos]
-    shortcut: biv
-
-  # Technical/IT (only when category=it is requested)
-  # REMOVED: stackoverflow (engine not available in SearXNG container)
-  - name: github
-    engine: github
-    categories: [it]
-    shortcut: gh
-    weight: 1.0
-
-  # Science/Academic (only when category=science is requested)
-  - name: arxiv
-    engine: arxiv
-    categories: [science]
-    weight: 1.5  # INCREASED
-    shortcut: arx
-
-  - name: semantic scholar
-    engine: semantic_scholar
-    categories: [science]
-    weight: 1.5  # INCREASED
-    shortcut: sem

package/config/searxng/limiter.toml

@@ -1,34 +0,0 @@
-# SearXNG Limiter Configuration for pi-search-scrape
-# This configuration disables all bot detection and rate limiting for local use
-
-[botdetection]
-# The prefix defines number of leading bits in an address that are compared
-# to determine whether or not an address is part of a (client) network.
-ipv4_prefix = 32
-ipv6_prefix = 48
-
-# Trust localhost requests completely (no bot detection for local connections)
-trusted_proxies = [
-  '127.0.0.0/8',
-  '::1',
-]
-
-[botdetection.ip_limit]
-# Disable rate limiting for link-local addresses (localhost)
-filter_link_local = false
-# Disable link token method (no captcha required)
-link_token = false
-
-[botdetection.ip_lists]
-# No IP blocking for local use
-block_ip = []
-
-# Allow all localhost requests without restrictions
-pass_ip = [
-  '127.0.0.0/8',      # IPv4 localhost
-  'fe80::/10',         # IPv6 link-local
-  '0.0.0.0/0',        # Allow all IPv4 (for local container use)
-]
-
-# Allow SearXNG org IPs for checks
-pass_searxng_org = true

package/config/tooling/eslint.config.js

@@ -1,73 +0,0 @@
-import js from '@eslint/js';
-import tseslint from '@typescript-eslint/eslint-plugin';
-import tsparser from '@typescript-eslint/parser';
-
-export default [
-  // Global ignores
-  {
-    ignores: [
-      'node_modules/**',
-      'dist/**',
-      '*.d.ts',
-      'examples/**',
-    ],
-  },
-  // JavaScript recommended rules
-  js.configs.recommended,
-  // TypeScript recommended rules
-  {
-    files: ['**/*.ts'],
-    languageOptions: {
-      parser: tsparser,
-      parserOptions: {
-        ecmaVersion: 2022,
-        sourceType: 'module',
-        project: './tsconfig.json',
-      },
-      globals: {
-        process: 'readonly',
-        console: 'readonly',
-        Buffer: 'readonly',
-        __dirname: 'readonly',
-        __filename: 'readonly',
-        require: 'readonly',
-        module: 'readonly',
-        exports: 'readonly',
-        NodeJS: 'readonly',
-        setTimeout: 'readonly',
-        setInterval: 'readonly',
-        clearTimeout: 'readonly',
-        clearInterval: 'readonly',
-        crypto: 'readonly',
-        AbortSignal: 'readonly',
-        AbortController: 'readonly',
-        Request: 'readonly',
-        RequestInit: 'readonly',
-        Response: 'readonly',
-        FormData: 'readonly',
-        fetch: 'readonly',
-        URL: 'readonly',
-        URLSearchParams: 'readonly',
-        ArrayBuffer: 'readonly',
-        ReadableStream: 'readonly',
-        Blob: 'readonly',
-      },
-    },
-    plugins: {
-      '@typescript-eslint': tseslint,
-    },
-    rules: {
-      // Override specific rules
-      'no-console': 'off', // Allow console.log for debugging
-      'no-unused-vars': 'off', // Disable in favor of TypeScript version
-      '@typescript-eslint/no-unused-vars': [
-        'error',
-        {
-          argsIgnorePattern: '^_',
-          varsIgnorePattern: '^_',
-          caughtErrorsIgnorePattern: '^_',
-        },
-      ],
-    },
-  },
-];

package/config/tooling/vitest.config.integration.ts

@@ -1,17 +0,0 @@
-import { defineConfig } from 'vitest/config';
-import baseConfig from './vitest.config';
-
-export default defineConfig({
-  ...baseConfig,
-  test: {
-    ...baseConfig.test,
-    // Integration tests using testcontainers
-    include: ['test/integration/**/*.test.ts'],
-    setupFiles: [],
-    testTimeout: 180000, // 180 second timeout for integration tests
-    maxConcurrency: 1, // Only one integration test at a time due to containers
-    fileParallelism: false,
-    pool: 'forks',
-    hookTimeout: 60000,
-  },
-});

package/config/tooling/vitest.config.ts

@@ -1,40 +0,0 @@
-import { defineConfig } from 'vitest/config';
-import { fileURLToPath } from 'node:url';
-import path from 'node:path';
-
-const __dirname = path.dirname(fileURLToPath(import.meta.url));
-
-export default defineConfig({
-  test: {
-    globals: true,
-    environment: 'node',
-    include: ['**/*.test.ts', '**/*.spec.ts'],
-    exclude: ['node_modules', 'dist', '.tmp'],
-    coverage: {
-      provider: 'v8',
-      reporter: ['text', 'json', 'html', 'lcov'],
-      exclude: [
-        'node_modules/',
-        'dist/',
-        '**/*.test.ts',
-        '**/*.spec.ts',
-        '**/types.ts',
-        '**/index.ts',
-        'test/',
-      ],
-      reportOnFailure: true,
-    },
-    testTimeout: 60000,
-    hookTimeout: 30000,
-    teardownTimeout: 10000,
-    pool: 'threads',
-    includeSource: ['src/**/*.ts'],
-    setupFiles: ['./test/setup/unit.ts'],
-  },
-  resolve: {
-    alias: {
-      '@': path.resolve(__dirname, './src'),
-      '@test': path.resolve(__dirname, './test'),
-    },
-  },
-});

package/config/tooling/vitest.config.unit.ts

@@ -1,12 +0,0 @@
-import { defineConfig } from 'vitest/config';
-import baseConfig from './vitest.config';
-
-export default defineConfig({
-  ...baseConfig,
-  test: {
-    ...baseConfig.test,
-    include: ['test/unit/**/*.test.ts'],
-    setupFiles: ['./test/setup/unit.ts'],
-    testTimeout: 10000,
-  },
-});

package/scripts/setup.js

@@ -1,174 +0,0 @@
-#!/usr/bin/env node
-
-/**
- * pi-research Postinstall Setup Script
- *
- * This script runs automatically after `npm install` and ensures:
- * 1. Playwright browser binaries are installed (Chromium, Firefox, WebKit)
- * 2. System dependencies are available (if run with --system-deps flag)
- *
- * Usage:
- *   npm run setup          # Full setup: install deps + browsers
- *   npm run install:browsers # Install only browser binaries
- *   npm run install:deps    # Install system dependencies only
- *
- * Environment Variables:
- *   PLAYWRIGHT_BROWSERS_PATH    - Custom path for browser cache
- *   PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1  - Skip browser download
- */
-
-import { execSync } from 'child_process';
-import { existsSync, readdirSync, statSync } from 'fs';
-import { join, dirname } from 'path';
-import { fileURLToPath } from 'url';
-import { homedir, platform } from 'os';
-
-const __filename = fileURLToPath(import.meta.url);
-const __dirname = dirname(__filename);
-
-console.log('='.repeat(60));
-console.log('🔧 pi-research Setup');
-console.log('='.repeat(60));
-
-// Check Node.js version
-const nodeVersion = process.version.replace('v', '');
-const [major] = nodeVersion.split('.').map(Number);
-
-console.log(`\nℹ️  Node.js version: v${nodeVersion}`);
-
-const requiredVersions = {
-  min: 22,
-  preferred: [22, 24]
-};
-
-if (major < requiredVersions.min) {
-  console.warn(
-    `⚠️  WARNING: Node.js ${nodeVersion} is below the minimum required version ${requiredVersions.min}.`
-  );
-  console.warn('   Please upgrade to Node.js 22.13.0+ or 24.x+');
-}
-
-console.log(`   Required: >= ${requiredVersions.min}.0.0`);
-
-// Check if playwright is installed
-const playwrightPath = join(process.cwd(), 'node_modules', 'playwright');
-
-if (!existsSync(playwrightPath)) {
-  console.error('\n❌ ERROR: playwright is not installed.');
-  console.error('   Run `npm install` first, then `npm run setup`.');
-  process.exit(1);
-}
-
-console.log('✅ Playwright package installed');
-
-// Detect OS for browser installation
-const isLinux = platform() === 'linux';
-const isDarwin = platform() === 'darwin';
-const isWindows = platform() === 'win32';
-
-console.log(`\n📦 Detected platform: ${platform()}`);
-
-// Install browsers
-console.log('\n🌐 Installing Playwright browsers...');
-
-try {
-  // Check if user wants to install system dependencies
-  const installDeps = process.argv.includes('--system-deps') ||
-                      process.env.PLAYWRIGHT_INSTALL_DEPS === 'true';
-
-  if (installDeps && isLinux) {
-    console.log('⚙️  Installing Linux system dependencies for Playwright...');
-    try {
-      execSync('npx playwright install-deps', {
-        stdio: 'inherit',
-        env: { ...process.env, PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: '0' }
-      });
-      console.log('✅ System dependencies installed');
-    } catch (error) {
-      console.warn('⚠️  Could not install system dependencies automatically.');
-      console.warn('   Try running: sudo apt-get install $(npx playwright install-deps --print-deps)');
-    }
-  }
-
-  // Install browser binaries
-  // Only Chromium is needed for scraping (as per scrapers.ts implementation)
-  console.log('   Installing Chromium browser...');
-  execSync('npx playwright install chromium', {
-    stdio: 'inherit',
-    env: {
-      ...process.env,
-      PLAYWRIGHT_BROWSERS_PATH: process.env.PLAYWRIGHT_BROWSERS_PATH || ''
-    }
-  });
-
-  console.log('✅ Browsers installed successfully');
-
-  // Verify installation
-  console.log('\n📋 Verifying browser installation...');
-  const browsers = ['chromium']; // Only Chromium is required for scraping
-  const cacheDir = process.env['PLAYWRIGHT_BROWSERS_PATH'] ||
-    (process.platform === 'win32'
-      ? join(process.env['LOCALAPPDATA'] || join(homedir(), 'AppData', 'Local'), 'ms-playwright')
-      : join(homedir(), '.cache', 'ms-playwright'));
-
-  browsers.forEach(browser => {
-    let browserDirs = [];
-    try {
-      browserDirs = readdirSync(cacheDir).filter(f =>
-        f.startsWith(browser) && statSync(join(cacheDir, f)).isDirectory()
-      );
-    } catch {
-      // cache dir may not exist yet or may be at a different path
-    }
-
-    if (browserDirs.length > 0) {
-      console.log(`   ✅ ${browser}: ${browserDirs.sort().join(', ')}`);
-    } else {
-      console.warn(`   ⚠️  ${browser}: Not found in cache`);
-    }
-  });
-
-  // Show browser versions
-  console.log('\n🔍 Browser versions:');
-  try {
-    const playwrightVersion = execSync('npx playwright --version', { encoding: 'utf8' }).trim();
-    console.log(`   Playwright: ${playwrightVersion}`);
-  } catch (error) {
-    console.warn('   Could not determine Playwright version');
-  }
-
-} catch (error) {
-  console.error('\n❌ ERROR: Failed to install browsers');
-  console.error('   Error:', error.message);
-  console.error('\n💡 Troubleshooting:');
-  console.error('   1. Ensure you have internet connectivity');
-  console.error('   2. Try running: npm run install:browsers');
-  console.error('   3. Check Node.js version: node --version');
-  console.error('   4. For Linux: sudo apt-get install -y $(npx playwright install-deps --print-deps)');
-  process.exit(1);
-}
-
-console.log('\n' + '='.repeat(60));
-console.log('🎉 Setup complete! Chromium browser installed. pi-research is ready to use.');
-console.log('='.repeat(60));
-
-// Additional info
-console.log('\n📚 Quick start:');
-console.log('   • npm test              - Run unit tests');
-console.log('   • npm run type-check    - Type check TypeScript');
-console.log('   • npm run lint          - Run ESLint');
-console.log('   • npm run setup         - Full setup with system deps');
-
-// Platform-specific tips
-if (isLinux) {
-  console.log('\n💡 Linux tip: If you encounter issues, install system deps:');
-  console.log('   sudo apt-get install -y libgbm1 libnss3 libatk1.0-0');
-  console.log('   libatk-bridge2.0-0 libcups2 libxkbcommon0 libxcomposite1');
-} else if (isDarwin) {
-  console.log('\n💡 macOS: Ensure Xcode Command Line Tools are installed:');
-  console.log('   xcode-select --install');
-} else if (isWindows) {
-  console.log('\n💡 Windows: Ensure you have the latest Visual C++ redistributables');
-}
-
-console.log('\n');