@crawlith/core 0.1.0

package/src/crawler/sitemap.ts

@@ -0,0 +1,73 @@
+import { request } from 'undici';
+import * as cheerio from 'cheerio';
+import { normalizeUrl } from './normalize.js';
+
+export class Sitemap {
+  /**
+   * Fetches and parses a sitemap (or sitemap index) to extract URLs.
+   * Recursively handles sitemap indexes with loop detection and depth limits.
+   */
+  async fetch(url: string): Promise<string[]> {
+    const visited = new Set<string>();
+    const urls = new Set<string>();
+
+    await this.processSitemap(url, visited, urls);
+
+    return Array.from(urls);
+  }
+
+  private async processSitemap(url: string, visited: Set<string>, urls: Set<string>) {
+    if (visited.has(url)) return;
+    visited.add(url);
+
+    // Hard limit on number of sitemaps to fetch to prevent abuse
+    if (visited.size > 50) return;
+
+    try {
+      const res = await request(url, {
+        maxRedirections: 3,
+        headers: { 'User-Agent': 'crawlith/1.0' },
+        headersTimeout: 10000,
+        bodyTimeout: 10000
+      });
+
+      if (res.statusCode >= 200 && res.statusCode < 300) {
+        const xml = await res.body.text();
+        // Basic validation: must verify it looks like XML
+        if (!xml.trim().startsWith('<')) return;
+
+        const $ = cheerio.load(xml, { xmlMode: true });
+
+        // Check if it's a sitemap index
+        const sitemaps = $('sitemap > loc');
+        if (sitemaps.length > 0) {
+          const childSitemaps: string[] = [];
+          sitemaps.each((_, el) => {
+            const loc = $(el).text().trim();
+            if (loc) childSitemaps.push(loc);
+          });
+
+          // Process children sequentially to avoid massive concurrency spike
+          for (const childUrl of childSitemaps) {
+            await this.processSitemap(childUrl, visited, urls);
+          }
+        } else {
+          // It's a URL Set
+          $('url > loc').each((_, el) => {
+            const loc = $(el).text().trim();
+            if (loc) {
+              const normalized = normalizeUrl(loc, '');
+              if (normalized) {
+                urls.add(normalized);
+              }
+            }
+          });
+        }
+      } else {
+        await res.body.dump();
+      }
+    } catch (e) {
+      console.warn(`Failed to fetch sitemap ${url}:`, e);
+    }
+  }
+}

package/src/crawler/trap.ts

@@ -0,0 +1,96 @@
+
+export type TrapType = 'faceted_navigation' | 'calendar_trap' | 'pagination_loop' | 'session_trap';
+
+export interface TrapResult {
+    risk: number;
+    type: TrapType | null;
+}
+
+export class TrapDetector {
+    private pathCounters = new Map<string, Set<string>>();
+    private paginationCounters = new Map<string, number>();
+    private sessionParams = new Set(['sid', 'session', 'phpsessid', 'sessid', 'token']);
+
+    // Configurable thresholds
+    private PARAM_EXPLOSION_THRESHOLD = 30;
+    private PAGINATION_THRESHOLD = 50;
+
+    constructor(options: { paramThreshold?: number, paginationThreshold?: number } = {}) {
+        if (options.paramThreshold) this.PARAM_EXPLOSION_THRESHOLD = options.paramThreshold;
+        if (options.paginationThreshold) this.PAGINATION_THRESHOLD = options.paginationThreshold;
+    }
+
+    /**
+     * Checks if a URL represents a potential crawl trap.
+     */
+    checkTrap(rawUrl: string, _depth: number): TrapResult {
+        let risk = 0;
+        let type: TrapType | null = null;
+
+        try {
+            const u = new URL(rawUrl);
+            const params = new URLSearchParams(u.search);
+            const pathname = u.pathname;
+            const pathKey = `${u.origin}${pathname}`;
+
+            // 1. Session IDs / Tracking Parameters
+            for (const [key] of params) {
+                if (this.sessionParams.has(key.toLowerCase()) || key.toLowerCase().includes('session')) {
+                    risk = Math.max(risk, 0.9);
+                    type = 'session_trap';
+                }
+            }
+
+            // 2. Calendar Pattern Detection
+            // Matches /2023/12/01, /2023-12-01, /12-2023 etc
+            const calendarRegex = /\/\d{4}[-/]\d{2}[-/]\d{2}\/|\/\d{2}[-/]\d{2}[-/]\d{4}\//;
+            if (calendarRegex.test(pathname)) {
+                risk = Math.max(risk, 0.7);
+                type = 'calendar_trap';
+            }
+
+            // 3. Pagination Loop
+            const pageParam = params.get('page') || params.get('p') || params.get('pg');
+            if (pageParam && /^\d+$/.test(pageParam)) {
+                const pageNum = parseInt(pageParam, 10);
+                const currentMaxPage = this.paginationCounters.get(pathKey) || 0;
+
+                if (pageNum > currentMaxPage) {
+                    this.paginationCounters.set(pathKey, pageNum);
+                }
+
+                if (pageNum > this.PAGINATION_THRESHOLD) {
+                    risk = Math.max(risk, 0.85);
+                    type = 'pagination_loop';
+                }
+            }
+
+            // 4. Infinite Parameter Explosion (Faceted Navigation)
+            if (params.size > 0) {
+                const paramSet = this.pathCounters.get(pathKey) || new Set<string>();
+                params.sort();
+                const paramKey = params.toString();
+                paramSet.add(paramKey);
+                this.pathCounters.set(pathKey, paramSet);
+
+                if (paramSet.size > this.PARAM_EXPLOSION_THRESHOLD) {
+                    risk = Math.max(risk, 0.95);
+                    if (!type) type = 'faceted_navigation';
+                }
+            }
+
+        } catch (_e) {
+            // Invalid URL
+        }
+
+        return { risk, type };
+    }
+
+    /**
+     * Resets internal state (useful for multi-crawl sessions if needed)
+     */
+    reset() {
+        this.pathCounters.clear();
+        this.paginationCounters.clear();
+    }
+}

package/src/db/graphLoader.ts

@@ -0,0 +1,105 @@
+import { getDb } from './index.js';
+import { PageRepository } from './repositories/PageRepository.js';
+import { EdgeRepository } from './repositories/EdgeRepository.js';
+import { MetricsRepository, DbMetrics } from './repositories/MetricsRepository.js';
+import { SnapshotRepository } from './repositories/SnapshotRepository.js';
+import { Graph } from '../graph/graph.js';
+
+export function loadGraphFromSnapshot(snapshotId: number): Graph {
+    const db = getDb();
+    const pageRepo = new PageRepository(db);
+    const edgeRepo = new EdgeRepository(db);
+    const metricsRepo = new MetricsRepository(db);
+    const snapshotRepo = new SnapshotRepository(db);
+
+    const pages = pageRepo.getPagesBySnapshot(snapshotId);
+    const metrics = metricsRepo.getMetrics(snapshotId);
+    const snapshot = snapshotRepo.getSnapshot(snapshotId);
+    const metricsMap = new Map<number, DbMetrics>();
+    for (const m of metrics) {
+        metricsMap.set(m.page_id, m);
+    }
+
+    const graph = new Graph();
+    if (snapshot) {
+        graph.limitReached = !!snapshot.limit_reached;
+    }
+    const idMap = new Map<number, string>();
+
+    for (const p of pages) {
+        idMap.set(p.id, p.normalized_url);
+        graph.addNode(p.normalized_url, p.depth, p.http_status || 0);
+
+        const m = metricsMap.get(p.id);
+        let incrementalStatus: 'new' | 'changed' | 'unchanged' | undefined;
+        if (p.first_seen_snapshot_id === snapshotId) {
+            incrementalStatus = 'new';
+        } else if (m?.crawl_status === 'cached') {
+            incrementalStatus = 'unchanged';
+        } else if (m?.crawl_status === 'fetched') {
+            incrementalStatus = 'changed';
+        }
+
+        graph.updateNodeData(p.normalized_url, {
+            canonical: p.canonical_url || undefined,
+            contentHash: p.content_hash || undefined,
+            simhash: p.simhash || undefined,
+            etag: p.etag || undefined,
+            lastModified: p.last_modified || undefined,
+            html: p.html || undefined,
+            soft404Score: p.soft404_score || undefined,
+            noindex: !!p.noindex,
+            nofollow: !!p.nofollow,
+            incrementalStatus,
+            securityError: p.security_error || undefined,
+            retries: p.retries || undefined,
+            bytesReceived: p.bytes_received || undefined,
+            redirectChain: p.redirect_chain ? JSON.parse(p.redirect_chain) : undefined,
+            crawlTrapFlag: !!p.crawl_trap_flag,
+            crawlTrapRisk: p.crawl_trap_risk || undefined,
+            trapType: p.trap_type || undefined,
+            // Metrics
+            pageRank: m?.pagerank ?? undefined,
+            pageRankScore: m?.pagerank_score ?? m?.pagerank ?? undefined,
+            authorityScore: m?.authority_score ?? undefined,
+            hubScore: m?.hub_score ?? undefined,
+            linkRole: m?.link_role ?? undefined,
+            // Duplicate info
+            duplicateClusterId: m?.duplicate_cluster_id ?? undefined,
+            duplicateType: m?.duplicate_type ?? undefined,
+            isClusterPrimary: m?.is_cluster_primary ? true : undefined,
+        });
+    }
+
+    const edges = edgeRepo.getEdgesBySnapshot(snapshotId);
+
+    for (const e of edges) {
+        const source = idMap.get(e.source_page_id);
+        const target = idMap.get(e.target_page_id);
+        if (source && target) {
+            graph.addEdge(source, target, e.weight || 1.0);
+        }
+    }
+
+    // Load duplicate clusters
+    const dupClusters = db.prepare('SELECT * FROM duplicate_clusters WHERE snapshot_id = ?').all(snapshotId) as any[];
+    graph.duplicateClusters = dupClusters.map(c => ({
+        id: c.id,
+        type: c.type,
+        size: c.size,
+        representative: c.representative,
+        severity: c.severity
+    }));
+
+    // Load content clusters
+    const contentClusters = db.prepare('SELECT * FROM content_clusters WHERE snapshot_id = ?').all(snapshotId) as any[];
+    graph.contentClusters = contentClusters.map(c => ({
+        id: c.id,
+        count: c.count,
+        primaryUrl: c.primary_url,
+        risk: c.risk,
+        sharedPathPrefix: c.shared_path_prefix || undefined
+    }));
+
+    return graph;
+}

package/src/db/index.ts

@@ -0,0 +1,70 @@
+import Database from 'better-sqlite3';
+import path from 'node:path';
+import fs from 'node:fs';
+import os from 'node:os';
+import { initSchema } from './schema.js';
+
+let dbInstance: Database.Database | null = null;
+
+export function getDbPath(): string {
+  if (process.env.NODE_ENV === 'test') {
+    return ':memory:';
+  }
+  if (process.env.CRAWLITH_DB_PATH) {
+    return process.env.CRAWLITH_DB_PATH;
+  }
+  const homeDir = os.homedir();
+  const crawlithDir = path.join(homeDir, '.crawlith');
+  if (!fs.existsSync(crawlithDir)) {
+    fs.mkdirSync(crawlithDir, { recursive: true });
+    // Set permissions to 700 (user only)
+    fs.chmodSync(crawlithDir, 0o700);
+  }
+  return path.join(crawlithDir, 'crawlith.db');
+}
+
+export function getDb(): Database.Database {
+  if (dbInstance) {
+    return dbInstance;
+  }
+
+  const dbPath = getDbPath();
+  const db = new Database(dbPath);
+
+  // Hardening & Performance Configuration
+  db.pragma('journal_mode = WAL');
+  db.pragma('synchronous = NORMAL');
+  db.pragma('foreign_keys = ON');
+  db.pragma('temp_store = MEMORY');
+  db.pragma('mmap_size = 30000000000');
+  db.pragma('cache_size = -20000');
+  db.pragma('busy_timeout = 5000');
+
+  // Security controls
+  // Ensure file permissions are 600 (user read/write only)
+  try {
+    fs.chmodSync(dbPath, 0o600);
+  } catch (_e) {
+    // might fail on first creation if file doesn't exist yet, but better-sqlite3 creates it
+    // so we can try again or ignore if it's new
+  }
+
+  // Integrity check on startup
+  const integrity = db.pragma('integrity_check', { simple: true });
+  if (integrity !== 'ok') {
+    console.warn('Database integrity check failed:', integrity);
+  }
+
+  // Initialize schema
+  initSchema(db);
+
+  dbInstance = db;
+  return db;
+}
+
+export function closeDb() {
+  if (dbInstance) {
+    dbInstance.close();
+    dbInstance = null;
+  }
+}

package/src/db/repositories/EdgeRepository.ts

@@ -0,0 +1,29 @@
+import { Database } from 'better-sqlite3';
+
+export interface Edge {
+  id: number;
+  snapshot_id: number;
+  source_page_id: number;
+  target_page_id: number;
+  weight: number;
+  rel: 'nofollow' | 'sponsored' | 'ugc' | 'internal' | 'external' | 'unknown';
+}
+
+export class EdgeRepository {
+  private insertStmt;
+
+  constructor(private db: Database) {
+    this.insertStmt = this.db.prepare(`
+      INSERT INTO edges (snapshot_id, source_page_id, target_page_id, weight, rel)
+      VALUES (?, ?, ?, ?, ?)
+    `);
+  }
+
+  insertEdge(snapshotId: number, sourcePageId: number, targetPageId: number, weight: number = 1.0, rel: string = 'internal') {
+    this.insertStmt.run(snapshotId, sourcePageId, targetPageId, weight, rel);
+  }
+
+  getEdgesBySnapshot(snapshotId: number): Edge[] {
+    return this.db.prepare('SELECT * FROM edges WHERE snapshot_id = ?').all(snapshotId) as Edge[];
+  }
+}

package/src/db/repositories/MetricsRepository.ts

@@ -0,0 +1,49 @@
+import { Database } from 'better-sqlite3';
+
+export interface DbMetrics {
+  snapshot_id: number;
+  page_id: number;
+  authority_score: number | null;
+  hub_score: number | null;
+  pagerank: number | null;
+  pagerank_score: number | null;
+  link_role: 'hub' | 'authority' | 'power' | 'balanced' | 'peripheral' | null;
+  crawl_status: string | null;
+  word_count: number | null;
+  thin_content_score: number | null;
+  external_link_ratio: number | null;
+  orphan_score: number | null;
+  duplicate_cluster_id: string | null;
+  duplicate_type: 'exact' | 'near' | 'template_heavy' | 'none' | null;
+  is_cluster_primary: number;
+}
+
+export class MetricsRepository {
+  private insertStmt;
+
+  constructor(private db: Database) {
+    this.insertStmt = this.db.prepare(`
+      INSERT OR REPLACE INTO metrics (
+        snapshot_id, page_id, authority_score, hub_score, pagerank, pagerank_score,
+        link_role, crawl_status, word_count, thin_content_score, external_link_ratio,
+        orphan_score, duplicate_cluster_id, duplicate_type, is_cluster_primary
+      ) VALUES (
+        @snapshot_id, @page_id, @authority_score, @hub_score, @pagerank, @pagerank_score,
+        @link_role, @crawl_status, @word_count, @thin_content_score, @external_link_ratio,
+        @orphan_score, @duplicate_cluster_id, @duplicate_type, @is_cluster_primary
+      )
+    `);
+  }
+
+  insertMetrics(metrics: DbMetrics) {
+    this.insertStmt.run(metrics);
+  }
+
+  getMetrics(snapshotId: number): DbMetrics[] {
+    return this.db.prepare('SELECT * FROM metrics WHERE snapshot_id = ?').all(snapshotId) as DbMetrics[];
+  }
+
+  getMetricsForPage(snapshotId: number, pageId: number): DbMetrics | undefined {
+    return this.db.prepare('SELECT * FROM metrics WHERE snapshot_id = ? AND page_id = ?').get(snapshotId, pageId) as DbMetrics | undefined;
+  }
+}

package/src/db/repositories/PageRepository.ts

@@ -0,0 +1,128 @@
+import { Database } from 'better-sqlite3';
+
+export interface Page {
+  id: number;
+  site_id: number;
+  normalized_url: string;
+  first_seen_snapshot_id: number | null;
+  last_seen_snapshot_id: number | null;
+  http_status: number | null;
+  canonical_url: string | null;
+  content_hash: string | null;
+  simhash: string | null;
+  etag: string | null;
+  last_modified: string | null;
+  html: string | null;
+  soft404_score: number | null;
+  noindex: number;
+  nofollow: number;
+  security_error: string | null;
+  retries: number;
+  depth: number;
+  redirect_chain: string | null;
+  bytes_received: number | null;
+  crawl_trap_flag: number;
+  crawl_trap_risk: number | null;
+  trap_type: string | null;
+  created_at: string;
+  updated_at: string;
+}
+
+export class PageRepository {
+  private upsertStmt;
+  private getIdStmt;
+
+  constructor(private db: Database) {
+    this.upsertStmt = this.db.prepare(`
+      INSERT INTO pages (
+        site_id, normalized_url, first_seen_snapshot_id, last_seen_snapshot_id,
+        http_status, canonical_url, content_hash, simhash, etag, last_modified, html,
+        soft404_score, noindex, nofollow, security_error, retries, depth,
+        redirect_chain, bytes_received, crawl_trap_flag, crawl_trap_risk, trap_type,
+        updated_at
+      ) VALUES (
+        @site_id, @normalized_url, @first_seen_snapshot_id, @last_seen_snapshot_id,
+        @http_status, @canonical_url, @content_hash, @simhash, @etag, @last_modified, @html,
+        @soft404_score, @noindex, @nofollow, @security_error, @retries, @depth,
+        @redirect_chain, @bytes_received, @crawl_trap_flag, @crawl_trap_risk, @trap_type,
+        datetime('now')
+      )
+      ON CONFLICT(site_id, normalized_url) DO UPDATE SET
+        last_seen_snapshot_id = excluded.last_seen_snapshot_id,
+        http_status = excluded.http_status,
+        canonical_url = excluded.canonical_url,
+        content_hash = excluded.content_hash,
+        simhash = excluded.simhash,
+        etag = excluded.etag,
+        last_modified = excluded.last_modified,
+        html = excluded.html,
+        soft404_score = excluded.soft404_score,
+        noindex = excluded.noindex,
+        nofollow = excluded.nofollow,
+        security_error = excluded.security_error,
+        retries = excluded.retries,
+        depth = excluded.depth,
+        redirect_chain = excluded.redirect_chain,
+        bytes_received = excluded.bytes_received,
+        crawl_trap_flag = excluded.crawl_trap_flag,
+        crawl_trap_risk = excluded.crawl_trap_risk,
+        trap_type = excluded.trap_type,
+        updated_at = datetime('now')
+    `);
+
+    this.getIdStmt = this.db.prepare('SELECT id FROM pages WHERE site_id = ? AND normalized_url = ?');
+  }
+
+  upsertPage(page: Partial<Page> & { site_id: number; normalized_url: string; last_seen_snapshot_id: number }) {
+    const params = {
+      site_id: page.site_id,
+      normalized_url: page.normalized_url,
+      first_seen_snapshot_id: page.first_seen_snapshot_id ?? page.last_seen_snapshot_id,
+      last_seen_snapshot_id: page.last_seen_snapshot_id,
+      http_status: page.http_status ?? null,
+      canonical_url: page.canonical_url ?? null,
+      content_hash: page.content_hash ?? null,
+      simhash: page.simhash ?? null,
+      etag: page.etag ?? null,
+      last_modified: page.last_modified ?? null,
+      html: page.html ?? null,
+      soft404_score: page.soft404_score ?? null,
+      noindex: page.noindex ?? 0,
+      nofollow: page.nofollow ?? 0,
+      security_error: page.security_error ?? null,
+      retries: page.retries ?? 0,
+      depth: page.depth ?? 0,
+      redirect_chain: page.redirect_chain ?? null,
+      bytes_received: page.bytes_received ?? null,
+      crawl_trap_flag: page.crawl_trap_flag ?? 0,
+      crawl_trap_risk: page.crawl_trap_risk ?? null,
+      trap_type: page.trap_type ?? null,
+    };
+
+    const info = this.upsertStmt.run(params);
+    return info;
+  }
+
+  upsertAndGetId(page: Partial<Page> & { site_id: number; normalized_url: string; last_seen_snapshot_id: number }): number {
+    const tx = this.db.transaction(() => {
+      this.upsertPage(page);
+      const row = this.getIdStmt.get(page.site_id, page.normalized_url) as { id: number } | undefined;
+      if (!row) throw new Error(`Failed to retrieve ID for upserted page: ${page.normalized_url}`);
+      return row.id;
+    });
+    return tx();
+  }
+
+  getPage(siteId: number, url: string): Page | undefined {
+    return this.db.prepare('SELECT * FROM pages WHERE site_id = ? AND normalized_url = ?').get(siteId, url) as Page | undefined;
+  }
+
+  getPagesBySnapshot(snapshotId: number): Page[] {
+    return this.db.prepare('SELECT * FROM pages WHERE last_seen_snapshot_id = ?').all(snapshotId) as Page[];
+  }
+
+  getIdByUrl(siteId: number, url: string): number | undefined {
+    const row = this.getIdStmt.get(siteId, url) as { id: number } | undefined;
+    return row?.id;
+  }
+}

package/src/db/repositories/SiteRepository.ts

@@ -0,0 +1,32 @@
+import { Database } from 'better-sqlite3';
+
+export interface Site {
+  id: number;
+  domain: string;
+  created_at: string;
+  settings_json: string | null;
+  is_active: number;
+}
+
+export class SiteRepository {
+  constructor(private db: Database) { }
+
+  getSite(domain: string): Site | undefined {
+    return this.db.prepare('SELECT * FROM sites WHERE domain = ?').get(domain) as Site | undefined;
+  }
+
+  createSite(domain: string): number {
+    const stmt = this.db.prepare('INSERT INTO sites (domain) VALUES (?)');
+    const info = stmt.run(domain);
+    return info.lastInsertRowid as number;
+  }
+
+  firstOrCreateSite(domain: string): Site {
+    let site = this.getSite(domain);
+    if (!site) {
+      this.createSite(domain);
+      site = this.getSite(domain);
+    }
+    return site!;
+  }
+}

package/src/db/repositories/SnapshotRepository.ts

@@ -0,0 +1,74 @@
+import { Database } from 'better-sqlite3';
+
+export interface Snapshot {
+  id: number;
+  site_id: number;
+  type: 'full' | 'partial' | 'incremental';
+  created_at: string;
+  node_count: number;
+  edge_count: number;
+  status: 'running' | 'completed' | 'failed';
+  limit_reached: number;
+  health_score: number | null;
+  orphan_count: number | null;
+  thin_content_count: number | null;
+}
+
+export class SnapshotRepository {
+  constructor(private db: Database) {}
+
+  createSnapshot(siteId: number, type: 'full' | 'partial' | 'incremental', status: 'running' | 'completed' | 'failed' = 'running'): number {
+    const stmt = this.db.prepare('INSERT INTO snapshots (site_id, type, status) VALUES (?, ?, ?)');
+    const info = stmt.run(siteId, type, status);
+    return info.lastInsertRowid as number;
+  }
+
+  getLatestSnapshot(siteId: number, status?: 'completed' | 'running' | 'failed'): Snapshot | undefined {
+    let sql = 'SELECT * FROM snapshots WHERE site_id = ?';
+    const params: any[] = [siteId];
+    if (status) {
+      sql += ' AND status = ?';
+      params.push(status);
+    }
+    sql += ' ORDER BY created_at DESC LIMIT 1';
+    return this.db.prepare(sql).get(...params) as Snapshot | undefined;
+  }
+
+  updateSnapshotStatus(id: number, status: 'completed' | 'failed', stats: Partial<Snapshot> = {}) {
+    const sets: string[] = ['status = ?'];
+    const params: any[] = [status];
+
+    if (stats.node_count !== undefined) {
+      sets.push('node_count = ?');
+      params.push(stats.node_count);
+    }
+    if (stats.edge_count !== undefined) {
+      sets.push('edge_count = ?');
+      params.push(stats.edge_count);
+    }
+    if (stats.limit_reached !== undefined) {
+      sets.push('limit_reached = ?');
+      params.push(stats.limit_reached);
+    }
+    if (stats.health_score !== undefined) {
+      sets.push('health_score = ?');
+      params.push(stats.health_score);
+    }
+    if (stats.orphan_count !== undefined) {
+      sets.push('orphan_count = ?');
+      params.push(stats.orphan_count);
+    }
+    if (stats.thin_content_count !== undefined) {
+      sets.push('thin_content_count = ?');
+      params.push(stats.thin_content_count);
+    }
+
+    params.push(id);
+    const sql = `UPDATE snapshots SET ${sets.join(', ')} WHERE id = ?`;
+    this.db.prepare(sql).run(...params);
+  }
+
+  getSnapshot(id: number): Snapshot | undefined {
+    return this.db.prepare('SELECT * FROM snapshots WHERE id = ?').get(id) as Snapshot | undefined;
+  }
+}