@dependabit/monitor 0.1.1

package/src/checkers/url-content.ts

@@ -0,0 +1,78 @@
+/**
+ * URL Content Checker
+ * Monitors documentation URLs for content changes using SHA256 hashing
+ */
+
+import type { Checker, DependencySnapshot, ChangeDetection, AccessConfig } from '../types.js';
+import { normalizeHTML } from '../normalizer.js';
+import crypto from 'node:crypto';
+
+export class URLContentChecker implements Checker {
+  /**
+   * Fetches and hashes URL content
+   */
+  async fetch(config: AccessConfig): Promise<DependencySnapshot> {
+    const { url } = config;
+
+    try {
+      const response = await fetch(url, {
+        headers: {
+          'User-Agent': 'dependabit-monitor/1.0'
+        }
+      });
+
+      if (!response.ok) {
+        throw new Error(`HTTP error: ${response.status} ${response.statusText}`);
+      }
+
+      const contentType = response.headers.get('content-type') || '';
+      const content = await response.text();
+
+      let normalizedContent: string;
+
+      // Apply HTML normalization if content is HTML
+      if (
+        contentType.includes('text/html') ||
+        content.trim().startsWith('<!DOCTYPE') ||
+        content.trim().startsWith('<html')
+      ) {
+        normalizedContent = normalizeHTML(content);
+      } else {
+        // For non-HTML content (markdown, plain text, etc.), just normalize whitespace
+        normalizedContent = content.replace(/\s+/g, ' ').trim();
+      }
+
+      // Generate SHA256 hash of normalized content
+      const stateHash = crypto.createHash('sha256').update(normalizedContent).digest('hex');
+
+      return {
+        stateHash,
+        fetchedAt: new Date(),
+        metadata: {
+          contentType,
+          contentLength: content.length,
+          normalizedLength: normalizedContent.length
+        }
+      };
+    } catch (error) {
+      throw new Error(`Failed to fetch URL content: ${(error as Error).message}`);
+    }
+  }
+
+  /**
+   * Compares two snapshots to detect content changes
+   */
+  async compare(prev: DependencySnapshot, curr: DependencySnapshot): Promise<ChangeDetection> {
+    const changes: string[] = [];
+
+    // Content changed if hashes differ
+    if (prev.stateHash !== curr.stateHash) {
+      changes.push('content');
+    }
+
+    return {
+      hasChanged: changes.length > 0,
+      changes
+    };
+  }
+}

package/src/comparator.ts

@@ -0,0 +1,68 @@
+/**
+ * State Comparator
+ * Generic comparison logic for dependency state snapshots
+ */
+
+import type { DependencySnapshot, ChangeDetection } from './types.js';
+
+export class StateComparator {
+  /**
+   * Compares two dependency snapshots to detect changes
+   */
+  compare(oldState: DependencySnapshot, newState: DependencySnapshot): ChangeDetection {
+    const changes: string[] = [];
+
+    // Check state hash
+    if (oldState.stateHash !== newState.stateHash) {
+      changes.push('stateHash');
+    }
+
+    // Check version
+    if (oldState.version !== newState.version) {
+      changes.push('version');
+    }
+
+    // Check metadata changes (shallow comparison)
+    if (this.hasMetadataChanges(oldState.metadata, newState.metadata)) {
+      changes.push('metadata');
+    }
+
+    return {
+      hasChanged: changes.length > 0,
+      changes,
+      oldVersion: oldState.version,
+      newVersion: newState.version
+    };
+  }
+
+  /**
+   * Checks if metadata has changed (shallow comparison)
+   */
+  private hasMetadataChanges(
+    oldMeta?: Record<string, unknown>,
+    newMeta?: Record<string, unknown>
+  ): boolean {
+    if (!oldMeta && !newMeta) {
+      return false;
+    }
+
+    if (!oldMeta || !newMeta) {
+      return true;
+    }
+
+    const oldKeys = Object.keys(oldMeta);
+    const newKeys = Object.keys(newMeta);
+
+    if (oldKeys.length !== newKeys.length) {
+      return true;
+    }
+
+    for (const key of oldKeys) {
+      if (oldMeta[key] !== newMeta[key]) {
+        return true;
+      }
+    }
+
+    return false;
+  }
+}

package/src/index.ts

@@ -0,0 +1,20 @@
+/**
+ * @dependabit/monitor - Dependency change detection and monitoring
+ */
+
+export { Monitor } from './monitor.js';
+export type { DependencyConfig, CheckResult } from './monitor.js';
+
+export { GitHubRepoChecker } from './checkers/github-repo.js';
+export { URLContentChecker } from './checkers/url-content.js';
+export { OpenAPIChecker } from './checkers/openapi.js';
+
+export { StateComparator } from './comparator.js';
+export { SeverityClassifier } from './severity.js';
+export type { Severity } from './severity.js';
+
+export { normalizeHTML, normalizeURL } from './normalizer.js';
+
+export { Scheduler } from './scheduler.js';
+
+export type { Checker, DependencySnapshot, ChangeDetection, AccessConfig } from './types.js';

package/src/monitor.ts

@@ -0,0 +1,120 @@
+/**
+ * Monitor Orchestrator
+ * Coordinates dependency checking across multiple access methods
+ */
+
+import { GitHubRepoChecker } from './checkers/github-repo.js';
+import { URLContentChecker } from './checkers/url-content.js';
+import { OpenAPIChecker } from './checkers/openapi.js';
+import { SeverityClassifier } from './severity.js';
+import type { Checker, AccessConfig, DependencySnapshot, ChangeDetection } from './types.js';
+
+export interface DependencyConfig extends AccessConfig {
+  id: string;
+  name?: string;
+  type?: string;
+  currentStateHash: string;
+  currentVersion?: string;
+  lastChecked?: string;
+  monitoring?: {
+    enabled?: boolean;
+    ignoreChanges?: boolean;
+  };
+}
+
+export interface CheckResult {
+  dependency: DependencyConfig;
+  hasChanged: boolean;
+  changes?: ChangeDetection;
+  severity?: 'breaking' | 'major' | 'minor' | undefined;
+  newSnapshot?: DependencySnapshot;
+  error?: string;
+}
+
+export class Monitor {
+  private checkers: Map<string, Checker>;
+  private classifier: SeverityClassifier;
+
+  constructor() {
+    this.checkers = new Map();
+    this.checkers.set('github-api', new GitHubRepoChecker());
+    this.checkers.set('http', new URLContentChecker());
+    this.checkers.set('openapi', new OpenAPIChecker());
+
+    this.classifier = new SeverityClassifier();
+  }
+
+  /**
+   * Checks a single dependency for changes
+   */
+  async checkDependency(dependency: DependencyConfig): Promise<CheckResult> {
+    try {
+      // Get appropriate checker for access method
+      const checker = this.checkers.get(dependency.accessMethod);
+      if (!checker) {
+        throw new Error(`Unsupported access method: ${dependency.accessMethod}`);
+      }
+
+      // Fetch current state
+      const newSnapshot = await checker.fetch(dependency);
+
+      // Create old snapshot from stored data
+      const oldSnapshot: DependencySnapshot = {
+        stateHash: dependency.currentStateHash,
+        version: dependency.currentVersion,
+        fetchedAt: dependency.lastChecked ? new Date(dependency.lastChecked) : new Date()
+      };
+
+      // Compare states
+      const changes = await checker.compare(oldSnapshot, newSnapshot);
+
+      // Classify severity if changes detected
+      let severity: 'breaking' | 'major' | 'minor' | undefined;
+      if (changes.hasChanged) {
+        severity = this.classifier.classify(changes);
+      }
+
+      return {
+        dependency,
+        hasChanged: changes.hasChanged,
+        changes,
+        severity,
+        newSnapshot
+      };
+    } catch (error) {
+      return {
+        dependency,
+        hasChanged: false,
+        error: (error as Error).message
+      };
+    }
+  }
+
+  /**
+   * Checks multiple dependencies, respecting monitoring rules
+   */
+  async checkAll(dependencies: DependencyConfig[]): Promise<CheckResult[]> {
+    // Filter out disabled dependencies
+    const enabledDeps = dependencies.filter((dep) => {
+      if (dep.monitoring?.enabled === false) {
+        return false;
+      }
+      if (dep.monitoring?.ignoreChanges === true) {
+        return false;
+      }
+      return true;
+    });
+
+    // Check all dependencies in parallel
+    const results = await Promise.all(enabledDeps.map((dep) => this.checkDependency(dep)));
+
+    return results;
+  }
+
+  /**
+   * Registers a custom checker for an access method
+   */
+  registerChecker(accessMethod: string, checker: Checker): void {
+    this.checkers.set(accessMethod, checker);
+  }
+}

package/src/normalizer.ts

@@ -0,0 +1,122 @@
+/**
+ * HTML Normalizer for content-based change detection
+ *
+ * Implements 6-step normalization to reduce false positives:
+ * 1. Remove <script> and <style> tags
+ * 2. Strip HTML comments
+ * 3. Normalize whitespace
+ * 4. Remove timestamp patterns
+ * 5. Remove analytics/tracking parameters
+ * 6. Preserve meaningful content
+ */
+
+/**
+ * Normalizes HTML content for consistent comparison
+ * @param html Raw HTML content
+ * @returns Normalized HTML string
+ */
+export function normalizeHTML(html: string): string {
+  if (!html || html.trim() === '') {
+    return '';
+  }
+
+  let normalized = html;
+
+  // Step 1: Remove script and style tags (with content)
+  // Use a simpler, more secure approach - remove everything between opening and closing tags
+  // This handles variations like </script >, </script\n>, etc.
+  let prevLength = 0;
+  while (normalized.length !== prevLength) {
+    prevLength = normalized.length;
+    // Match <script...> then any content, then </script with any whitespace/attributes before >
+    normalized = normalized.replace(/<script\b[^>]*>[\s\S]*?<\/script\s*[^>]*>/gi, '');
+  }
+  prevLength = 0;
+  while (normalized.length !== prevLength) {
+    prevLength = normalized.length;
+    normalized = normalized.replace(/<style\b[^>]*>[\s\S]*?<\/style\s*[^>]*>/gi, '');
+  }
+
+  // Step 2: Strip HTML comments - iterative to handle nested comments
+  prevLength = 0;
+  while (normalized.length !== prevLength) {
+    prevLength = normalized.length;
+    normalized = normalized.replace(/<!--[\s\S]*?-->/g, '');
+  }
+
+  // Step 3: Normalize whitespace (collapse multiple spaces/newlines)
+  normalized = normalized.replace(/\s+/g, ' ');
+  normalized = normalized.trim();
+
+  // Step 4: Remove common timestamp patterns
+  // Patterns like "Updated: 2024-01-01" or "Last modified: Jan 1, 2024"
+  normalized = normalized.replace(
+    /\b(Updated|Last\s+modified|Modified|Created|Published):\s*[^<\n]+/gi,
+    ''
+  );
+
+  // Remove ISO timestamps
+  normalized = normalized.replace(
+    /\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?(Z|[+-]\d{2}:\d{2})?/g,
+    ''
+  );
+
+  // Remove common date formats
+  normalized = normalized.replace(
+    /\b(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+\d{1,2},?\s+\d{4}\b/gi,
+    ''
+  );
+
+  // Step 5: Remove analytics/tracking parameters from URLs
+  // UTM parameters, tracking IDs, session IDs, etc.
+  normalized = normalized.replace(
+    /[?&](utm_[^&\s"']+|fbclid|gclid|msclkid|mc_[^&\s"']+|_ga|sessionid)[^&\s"']*/gi,
+    ''
+  );
+
+  // Step 6: Final cleanup - preserve meaningful content structure
+  // Remove extra spaces that may have been introduced
+  normalized = normalized.replace(/\s+/g, ' ');
+  normalized = normalized.trim();
+
+  return normalized;
+}
+
+/**
+ * Normalizes a URL by removing tracking parameters
+ * @param url URL string
+ * @returns Normalized URL
+ */
+export function normalizeURL(url: string): string {
+  try {
+    const urlObj = new URL(url);
+
+    // Remove common tracking parameters
+    const trackingParams = [
+      'utm_source',
+      'utm_medium',
+      'utm_campaign',
+      'utm_term',
+      'utm_content',
+      'fbclid',
+      'gclid',
+      'msclkid',
+      'mc_cid',
+      'mc_eid',
+      '_ga',
+      '_gac',
+      'sessionid',
+      'sid',
+      'SSID'
+    ];
+
+    trackingParams.forEach((param) => {
+      urlObj.searchParams.delete(param);
+    });
+
+    return urlObj.toString();
+  } catch  {
+    // If URL parsing fails, return original
+    return url;
+  }
+}

package/src/scheduler.ts

@@ -0,0 +1,175 @@
+import {
+  type DependencyEntry,
+  type DependabitConfig,
+  getEffectiveMonitoringRules
+} from '@dependabit/manifest';
+
+export interface SchedulerOptions {
+  currentTime?: Date;
+}
+
+/**
+ * Scheduler for per-dependency monitoring
+ *
+ * Determines which dependencies should be checked based on:
+ * - Check frequency (hourly, daily, weekly, monthly)
+ * - Last checked timestamp
+ * - Enabled/disabled status
+ * - IgnoreChanges flag
+ * - Config overrides
+ */
+export class Scheduler {
+  /**
+   * Check if a dependency should be checked now
+   *
+   * @param dependency Dependency entry
+   * @param config Configuration
+   * @param currentTime Current time (defaults to now)
+   * @returns true if dependency should be checked
+   */
+  shouldCheckDependency(
+    dependency: DependencyEntry,
+    config: DependabitConfig,
+    currentTime: Date = new Date()
+  ): boolean {
+    // Check dependency's own monitoring rules first
+    if (dependency.monitoring) {
+      if (!dependency.monitoring.enabled) {
+        return false;
+      }
+
+      if (dependency.monitoring.ignoreChanges) {
+        return false;
+      }
+    }
+
+    // Get effective monitoring rules (applies config overrides)
+    const rules = getEffectiveMonitoringRules(config, dependency.url);
+
+    // Check if monitoring is enabled at config level
+    if (!rules.enabled) {
+      return false;
+    }
+
+    // Check if changes should be ignored at config level
+    if (rules.ignoreChanges) {
+      return false;
+    }
+
+    // Get last checked time
+    const lastChecked = new Date(dependency.lastChecked);
+    const timeSinceCheck = currentTime.getTime() - lastChecked.getTime();
+
+    // Determine frequency to use (dependency's own monitoring rules take precedence)
+    const checkFrequency = dependency.monitoring?.checkFrequency || rules.checkFrequency;
+
+    // Determine if enough time has passed based on frequency
+    const intervalMs = this.getIntervalMs(checkFrequency);
+
+    return timeSinceCheck >= intervalMs;
+  }
+
+  /**
+   * Filter dependencies that should be checked
+   *
+   * @param dependencies Array of dependencies
+   * @param config Configuration
+   * @param currentTime Current time (defaults to now)
+   * @returns Filtered array of dependencies to check
+   */
+  filterDependenciesToCheck(
+    dependencies: DependencyEntry[],
+    config: DependabitConfig,
+    currentTime: Date = new Date()
+  ): DependencyEntry[] {
+    return dependencies.filter((dep) => this.shouldCheckDependency(dep, config, currentTime));
+  }
+
+  /**
+   * Get interval in milliseconds for a check frequency
+   *
+   * @param frequency Check frequency
+   * @returns Interval in milliseconds
+   */
+  private getIntervalMs(frequency: 'hourly' | 'daily' | 'weekly' | 'monthly'): number {
+    switch (frequency) {
+      case 'hourly':
+        return 60 * 60 * 1000; // 1 hour
+      case 'daily':
+        return 24 * 60 * 60 * 1000; // 24 hours
+      case 'weekly':
+        return 7 * 24 * 60 * 60 * 1000; // 7 days
+      case 'monthly':
+        return 30 * 24 * 60 * 60 * 1000; // 30 days (approximate)
+      default:
+        return 24 * 60 * 60 * 1000; // Default to daily
+    }
+  }
+
+  /**
+   * Get next check time for a dependency
+   *
+   * @param dependency Dependency entry
+   * @param config Configuration
+   * @returns Next check time
+   */
+  getNextCheckTime(dependency: DependencyEntry, config: DependabitConfig): Date {
+    const rules = getEffectiveMonitoringRules(config, dependency.url);
+    const lastChecked = new Date(dependency.lastChecked);
+
+    // Determine frequency to use (dependency's own monitoring rules take precedence)
+    const checkFrequency = dependency.monitoring?.checkFrequency || rules.checkFrequency;
+    const intervalMs = this.getIntervalMs(checkFrequency);
+
+    return new Date(lastChecked.getTime() + intervalMs);
+  }
+
+  /**
+   * Get schedule summary for all dependencies
+   *
+   * @param dependencies Array of dependencies
+   * @param config Configuration
+   * @returns Schedule summary grouped by frequency
+   */
+  getScheduleSummary(
+    dependencies: DependencyEntry[],
+    config: DependabitConfig
+  ): {
+    hourly: number;
+    daily: number;
+    weekly: number;
+    monthly: number;
+    disabled: number;
+  } {
+    const summary = {
+      hourly: 0,
+      daily: 0,
+      weekly: 0,
+      monthly: 0,
+      disabled: 0
+    };
+
+    for (const dep of dependencies) {
+      // Check dependency's own monitoring rules first
+      if (dep.monitoring) {
+        if (!dep.monitoring.enabled || dep.monitoring.ignoreChanges) {
+          summary.disabled++;
+          continue;
+        }
+      }
+
+      const rules = getEffectiveMonitoringRules(config, dep.url);
+
+      if (!rules.enabled || rules.ignoreChanges) {
+        summary.disabled++;
+      } else {
+        // Use dependency-level frequency if available, otherwise use rules
+        const checkFrequency: 'hourly' | 'daily' | 'weekly' | 'monthly' =
+          dep.monitoring?.checkFrequency ?? rules.checkFrequency;
+        summary[checkFrequency]++;
+      }
+    }
+
+    return summary;
+  }
+}

package/src/severity.ts

@@ -0,0 +1,112 @@
+/**
+ * Severity Classifier
+ * Classifies dependency changes into breaking, major, or minor severity levels
+ */
+
+import type { ChangeDetection } from './types.js';
+
+export type Severity = 'breaking' | 'major' | 'minor';
+
+export class SeverityClassifier {
+  /**
+   * Classifies the severity of a change based on version changes and change types
+   */
+  classify(changes: ChangeDetection): Severity {
+    if (!changes.hasChanged) {
+      return 'minor';
+    }
+
+    // Check for version-based classification
+    if (changes.oldVersion && changes.newVersion) {
+      const severity = this.classifyVersionChange(changes.oldVersion, changes.newVersion);
+      if (severity) {
+        return severity;
+      }
+    }
+
+    // Check for OpenAPI/schema changes
+    if (changes.diff && typeof changes.diff === 'object') {
+      const diff = changes.diff as {
+        removedEndpoints?: string[];
+        schemaChanges?: string[];
+        addedEndpoints?: string[];
+      };
+
+      // Breaking: Removed endpoints or incompatible schema changes
+      if (diff.removedEndpoints && diff.removedEndpoints.length > 0) {
+        return 'breaking';
+      }
+
+      if (diff.schemaChanges && diff.schemaChanges.length > 0) {
+        return 'breaking';
+      }
+
+      // Major: New endpoints or features
+      if (diff.addedEndpoints && diff.addedEndpoints.length > 0) {
+        return 'major';
+      }
+    }
+
+    // Default to minor for content changes
+    return 'minor';
+  }
+
+  /**
+   * Classifies severity based on semantic versioning
+   * @returns Severity level or undefined if version format not recognized
+   */
+  private classifyVersionChange(oldVersion: string, newVersion: string): Severity | undefined {
+    // Parse semver-like versions
+    const oldParts = this.parseVersion(oldVersion);
+    const newParts = this.parseVersion(newVersion);
+
+    if (!oldParts || !newParts) {
+      return undefined;
+    }
+
+    const [oldMajor, oldMinor, oldPatch] = oldParts;
+    const [newMajor, newMinor, newPatch] = newParts;
+
+    // Breaking: Major version increase
+    if (newMajor > oldMajor) {
+      return 'breaking';
+    }
+
+    // Major: Minor version increase
+    if (newMajor === oldMajor && newMinor > oldMinor) {
+      return 'major';
+    }
+
+    // Minor: Patch version increase or same version
+    if (newMajor === oldMajor && newMinor === oldMinor && newPatch >= oldPatch) {
+      return 'minor';
+    }
+
+    // Default for other cases
+    return 'major';
+  }
+
+  /**
+   * Parses a version string into [major, minor, patch]
+   */
+  private parseVersion(version: string): [number, number, number] | null {
+    if (!version) {
+      return null;
+    }
+
+    // Remove 'v' prefix if present
+    const cleaned = version.replace(/^v/, '');
+
+    // Match semver pattern
+    const match = cleaned.match(/^(\d+)\.(\d+)\.(\d+)/);
+    if (!match) {
+      return null;
+    }
+
+    return [
+      parseInt(match[1] || '0', 10),
+      parseInt(match[2] || '0', 10),
+      parseInt(match[3] || '0', 10)
+    ];
+  }
+}