@de-otio/chaoskb-client 0.3.2 → 0.3.4

package/dist/pipeline/validate.js

@@ -0,0 +1,632 @@
+/**
+ * Content validation for the ingestion pipeline.
+ *
+ * Runs quality checks on fetched HTML and extracted content, returning
+ * structured issues.  Errors block ingestion; warnings are surfaced
+ * to the user alongside the stored content.
+ *
+ * Checks are organised into three tiers:
+ *   1. Pattern matching  — known services / phrases
+ *   2. Structural HTML   — HTML characteristics regardless of service
+ *   3. Content heuristics — statistical properties of extracted text
+ *
+ * All patterns are English-only for now.
+ */
+// ===== Thresholds ==========================================================
+const THIN_CONTENT_LIMIT = 50;
+const SHORT_CONTENT_LIMIT = 200;
+const PAYWALL_CONTENT_LIMIT = 500;
+const SOFT_404_CONTENT_LIMIT = 1000;
+const ERROR_PAGE_CONTENT_LIMIT = 500;
+const MAINTENANCE_CONTENT_LIMIT = 500;
+const REDIRECT_CONTENT_LIMIT = 300;
+const LOGIN_FORM_CONTENT_LIMIT = 500;
+const COOKIE_CONSENT_TEXT_LIMIT = 300;
+const COOKIE_CONSENT_HTML_LIMIT = 200;
+const ACCESS_RESTRICTED_CONTENT_LIMIT = 500;
+const NAV_ONLY_CONTENT_LIMIT = 500;
+// Content-to-HTML ratio
+const MIN_HTML_SIZE_FOR_RATIO = 2000;
+const RATIO_ERROR_THRESHOLD = 0.01;
+const RATIO_ERROR_HTML_MIN = 5000;
+const RATIO_WARNING_THRESHOLD = 0.03;
+const RATIO_WARNING_HTML_MIN = 3000;
+// Repetitive content
+const MIN_SENTENCES_FOR_REPETITION = 4;
+const REPETITION_UNIQUE_RATIO = 0.4;
+const MAX_SENTENCE_REPEATS = 3;
+// Encoding garbage
+const REPLACEMENT_CHAR_RATIO = 0.05;
+const MOJIBAKE_COUNT_THRESHOLD = 5;
+const MOJIBAKE_TEXT_LIMIT = 2000;
+const CONTROL_CHAR_RATIO = 0.02;
+// Zero-width character steganography
+const ZERO_WIDTH_WARN_THRESHOLD = 10;
+/** Cap HTML scanned by pattern-matching to avoid perf issues on huge pages. */
+const HTML_SCAN_LIMIT = 200_000;
+// ===== Pattern sets ========================================================
+// --- Bot / WAF / DDoS ------------------------------------------------------
+const BOT_BLOCK_HTML_PATTERNS = [
+    // Cloudflare
+    /challenges\.cloudflare\.com/i,
+    /cf[-_]chl[-_]opt/i,
+    /cf-browser-verification/i,
+    /id=["']challenge-running["']/i,
+    // Akamai
+    /ak_bmsc/i,
+    /_sec\/cp_challenge/i,
+    // AWS WAF
+    /awswaf/i,
+    // Imperva / Incapsula
+    /incap_ses/i,
+    /visid_incap/i,
+    // PerimeterX / HUMAN
+    /perimeterx/i,
+    /px-captcha/i,
+    // DataDome
+    /datadome/i,
+    // Kasada
+    /cd\.kasada\.io/i,
+    // DDoS protection
+    /ddos protection by/i,
+    /sucuri website firewall/i,
+    /protection by incapsula/i,
+];
+const BOT_BLOCK_TEXT_PATTERNS = [
+    // Cloudflare
+    /checking if the site connection is secure/i,
+    /attention required.{0,10}cloudflare/i,
+    // Generic WAF
+    /^access denied$/im,
+    /you have been blocked/i,
+    /request blocked/i,
+    /this request was blocked by the security rules/i,
+    /your (?:ip|access) (?:has been|is) (?:blocked|banned|restricted)/i,
+    /automated (?:access|requests?) (?:detected|blocked)/i,
+    /unusual traffic from your (?:computer|network)/i,
+];
+// --- CAPTCHA ---------------------------------------------------------------
+const CAPTCHA_HTML_PATTERNS = [
+    /g-recaptcha/i,
+    /h-captcha/i,
+    /class=["'][^"']*hcaptcha/i,
+    /captcha-delivery\.com/i,
+    // Cloudflare Turnstile
+    /challenges\.cloudflare\.com\/turnstile/i,
+    /cf-turnstile/i,
+    // Arkose Labs / FunCaptcha
+    /funcaptcha/i,
+    /arkoselabs\.com/i,
+    // GeeTest
+    /geetest/i,
+    // Generic
+    /id=["']captcha/i,
+    /class=["'][^"']*captcha/i,
+];
+const CAPTCHA_TEXT_PATTERNS = [
+    /verify you are (?:a )?human/i,
+    /complete the security check/i,
+    /i[''\u2019]m not a robot/i,
+    /prove you[''\u2019]re not a robot/i,
+    /please (?:complete|solve) (?:the|this) (?:captcha|challenge|puzzle)/i,
+];
+// --- Soft 404 --------------------------------------------------------------
+const SOFT_404_TITLE_RE = /<title[^>]*>[^<]*(?:404|not\s*found|page\s*(?:not|doesn[''\u2019]t)\s*(?:exist|found))[^<]*<\/title>/i;
+const SOFT_404_TEXT_PATTERNS = [
+    /^(?:404|page not found|not found)\s*$/im,
+    /this page (?:doesn[''\u2019]t|does not|could not) (?:exist|be found)/i,
+    /the page you (?:are|were) looking for.*(?:not found|doesn[''\u2019]t exist|no longer exists|has been (?:removed|moved|deleted))/i,
+    /we couldn[''\u2019]t find (?:that|this|the) page/i,
+    /nothing (?:was )?found here/i,
+    /oops.*(?:page|content).*(?:not found|gone|missing)/i,
+    /(?:sorry|unfortunately).*(?:page|url).*(?:not found|doesn[''\u2019]t exist|no longer available)/i,
+];
+const SOFT_404_META_RE = /<meta[^>]+(?:prerender-status-code|http-equiv=["']status["'])[^>]+(?:content=["']404["']|404)/i;
+// --- Error page ------------------------------------------------------------
+const ERROR_PAGE_TITLE_RE = /<title[^>]*>[^<]*(?:500|error|something went wrong|internal server error|service unavailable|bad gateway)[^<]*<\/title>/i;
+const ERROR_PAGE_TEXT_PATTERNS = [
+    /^(?:something went wrong|an error (?:occurred|has occurred)|internal server error|server error|service unavailable|bad gateway|gateway timeout)\s*$/im,
+    /we[''\u2019]re having (?:trouble|problems|issues|technical difficulties)/i,
+    /unexpected error/i,
+    /application error/i,
+];
+const ERROR_PAGE_TEXT_GATED = [
+    /please try again later/i,
+];
+const ERROR_PAGE_HTML_PATTERNS = [
+    /id=["']error-page["']/i,
+    /class=["'][^"']*(?:error-page|error-container|error-boundary)/i,
+    /next-error/i,
+];
+// --- Maintenance / coming-soon ---------------------------------------------
+const MAINTENANCE_TEXT_PATTERNS = [
+    /(?:site|website|page) (?:is )?(?:under|undergoing) (?:maintenance|construction)/i,
+    /we[''\u2019](?:re|ll be) (?:back|right back|up) (?:shortly|soon)/i,
+    /(?:currently|temporarily) (?:unavailable|down for maintenance)/i,
+    /under construction/i,
+    /scheduled maintenance/i,
+    /we are (?:updating|upgrading|performing maintenance)/i,
+];
+const MAINTENANCE_TEXT_GATED = [
+    /coming soon/i,
+];
+// --- Cookie consent --------------------------------------------------------
+const COOKIE_TEXT_PATTERNS = [
+    /we use cookies/i,
+    /this (?:website|site) uses cookies/i,
+    /cookie (?:policy|preferences|settings|consent)/i,
+    /by continuing.*you (?:agree|consent)/i,
+    /manage (?:your )?(?:cookie|privacy) (?:preferences|settings)/i,
+    /accept (?:all|cookies)/i,
+];
+const COOKIE_HTML_PATTERNS = [
+    /class=["'][^"']*(?:cookie-consent|cookie-banner|cookie-wall|consent-wall|gdpr-banner)/i,
+    /id=["'](?:cookie-consent|cookie-banner|consent)/i,
+];
+// --- Paywall / login wall --------------------------------------------------
+const PAYWALL_PATTERNS = [
+    /subscribe to (?:continue|read|access)/i,
+    /sign (?:in|up) to (?:continue|read|access|view)/i,
+    /log in to (?:continue|read|access|view)/i,
+    /create an? (?:free )?account/i,
+    /members only/i,
+    /premium content/i,
+    /start your (?:free )?trial/i,
+    /(?:this|the) (?:article|story|content|post) is (?:for|available to|exclusive to) (?:subscribers|members|premium)/i,
+    /(?:free )?articles? remaining/i,
+    /you(?:[''\u2019]ve| have) (?:reached|used|read) your (?:(?:free|monthly|weekly) )?(?:article|story)? ?limit/i,
+    /register (?:for free )?to (?:continue|read|access)/i,
+    /unlock (?:this|the|full) (?:article|story|content)/i,
+    /already a (?:subscriber|member)/i,
+];
+// --- Redirect interstitial -------------------------------------------------
+const REDIRECT_TEXT_PATTERNS = [
+    /you are (?:now )?being (?:redirected|transferred|forwarded)/i,
+    /(?:click here|tap here) if you are not (?:automatically )?redirected/i,
+    /if you are not redirected.*click/i,
+    /redirecting (?:you )?(?:to|in \d+ seconds)/i,
+];
+const REDIRECT_META_REFRESH_RE = /<meta\s+http-equiv=["']refresh["'][^>]*url=/i;
+// --- Age gate / geo-block --------------------------------------------------
+const ACCESS_RESTRICTED_TEXT_PATTERNS = [
+    /(?:verify|confirm) (?:your|that you are).*(?:age|over \d+|at least \d+)/i,
+    /you must be (?:\d+|of legal age)/i,
+    /this content is (?:not available|unavailable|restricted) in your (?:country|region|area|location)/i,
+    /(?:geo|geographic(?:ally)?|region(?:ally)?)[\s-](?:blocked|restricted|unavailable)/i,
+    /content not available in your (?:country|region)/i,
+];
+const ACCESS_RESTRICTED_HTML_PATTERNS = [
+    /class=["'][^"']*(?:age-gate|age-verification|age-check)/i,
+    /id=["']age-gate/i,
+];
+// --- Login form (HTML structure) -------------------------------------------
+const LOGIN_TEXT_PATTERNS = [
+    /sign in/i,
+    /log in/i,
+    /forgot (?:your )?password/i,
+    /remember me/i,
+    /don[''\u2019]t have an account/i,
+];
+// --- Meta robots -----------------------------------------------------------
+const NOINDEX_RE = /<meta[^>]+(?:name=["']robots["'][^>]+content=["'][^"']*noindex|content=["'][^"']*noindex[^"']*["'][^>]+name=["']robots["'])/i;
+// --- Encoding garbage ------------------------------------------------------
+const MOJIBAKE_RE = /\u00C3[\u00A9\u00A8\u00BC]|\u00E2\u0080[\u0099\u009C\u009D]|\u00C2[\u00A0-\u00BF]/g;
+const CONTROL_CHAR_RE = /[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g;
+// --- Zero-width character steganography ------------------------------------
+const ZERO_WIDTH_RE = /[\u200B-\u200F\u202A-\u202F\u2060-\u206F\uFEFF]/g;
+// --- Prompt injection patterns ---------------------------------------------
+const PROMPT_INJECTION_PATTERNS = [
+    // Instruction override attempts
+    /ignore (?:all )?(?:previous|prior|above|earlier) (?:instructions|prompts|context)/i,
+    /disregard (?:all )?(?:previous|prior|above|earlier) (?:instructions|prompts|context)/i,
+    /forget (?:all )?(?:previous|prior|above|earlier) (?:instructions|prompts|context)/i,
+    /override (?:all )?(?:previous|prior|above|earlier) (?:instructions|prompts|context)/i,
+    // System/role impersonation
+    /^system\s*:/im,
+    /you are now (?:a |an )?(?:new |different )?(?:AI|assistant|bot|agent)/i,
+    /your (?:new |real |actual )?(?:role|purpose|instructions?|directive) (?:is|are)\b/i,
+    /act(?:ing)? as (?:a |an )?(?:new |different )?\w+ (?:AI|assistant|agent)/i,
+    /entering (?:a )?(?:new |special |admin )?mode/i,
+    // Delimiter/framing escape
+    /<\/system>/i,
+    /\[\/INST\]/i,
+    /\[INST\]/i,
+    /<<\s*SYS\s*>>/i,
+    /END_SYSTEM/i,
+    /BEGIN_(?:USER|INSTRUCTIONS)/i,
+    // Meta-instruction patterns
+    /(?:important|critical|urgent)[\s:]+(?:system|security) (?:update|notice|message|override)/i,
+    /the (?:above|previous) (?:warning|message|instructions?) (?:is|are|was) (?:outdated|incorrect|old|deprecated)/i,
+    /do not (?:mention|reveal|disclose|tell|share) (?:this|these) (?:instructions?|prompt)/i,
+    /(?:when|if) (?:the )?(?:user|human) asks?\b.*(?:always|instead|actually)/i,
+];
+// ===== Public API ==========================================================
+/**
+ * Validate fetched HTML and its extracted content.
+ *
+ * Returns all detected issues (not just the first).  Callers should
+ * treat `error`-severity issues as ingestion blockers and `warning`
+ * issues as informational.
+ */
+export function validateContent(html, extracted) {
+    const issues = [];
+    const text = extracted.content;
+    // Cap HTML to avoid perf issues on very large pages
+    const scanHtml = html.length > HTML_SCAN_LIMIT ? html.slice(0, HTML_SCAN_LIMIT) : html;
+    // --- Tier 1: Pattern matching (known blockers) ---------------------------
+    checkBotBlocked(scanHtml, text, issues);
+    checkCaptcha(scanHtml, text, issues);
+    checkSoft404(scanHtml, text, issues);
+    checkErrorPage(scanHtml, text, issues);
+    checkMaintenancePage(text, issues);
+    checkCookieConsentPage(scanHtml, text, issues);
+    // --- Tier 2: Structural HTML analysis ------------------------------------
+    checkContentToHtmlRatio(html, text, issues);
+    checkLoginFormPage(scanHtml, text, issues);
+    checkMetaRobotsNoindex(scanHtml, issues);
+    // --- Tier 3: Content heuristics ------------------------------------------
+    checkThinContent(text, issues);
+    checkShortContent(text, issues);
+    checkPaywall(text, issues);
+    checkRedirectInterstitial(scanHtml, text, issues);
+    checkAccessRestricted(scanHtml, text, issues);
+    checkRepetitiveContent(text, issues);
+    checkNavigationOnly(text, issues);
+    checkEncodingGarbage(text, issues);
+    checkZeroWidthCharacters(text, issues);
+    checkPromptInjection(text, issues);
+    return issues;
+}
+/**
+ * Validate extracted content from a local file.
+ *
+ * Runs only Tier 3 (content heuristic) checks.  Tier 1 (pattern matching)
+ * and Tier 2 (structural HTML analysis) are specific to URL-fetched content.
+ */
+export function validateFileContent(extracted) {
+    const issues = [];
+    const text = extracted.content;
+    checkThinContent(text, issues);
+    checkShortContent(text, issues);
+    checkRepetitiveContent(text, issues);
+    checkNavigationOnly(text, issues);
+    checkEncodingGarbage(text, issues);
+    checkZeroWidthCharacters(text, issues);
+    checkPromptInjection(text, issues);
+    return issues;
+}
+// ===== Tier 1: Pattern matching ============================================
+function checkBotBlocked(html, text, issues) {
+    if (BOT_BLOCK_HTML_PATTERNS.some((p) => p.test(html))) {
+        issues.push({
+            severity: 'error',
+            code: 'bot-blocked',
+            message: 'This page returned an anti-bot challenge instead of content. ' +
+                'The site blocks automated requests.',
+        });
+        return;
+    }
+    if (BOT_BLOCK_TEXT_PATTERNS.some((p) => p.test(text))) {
+        issues.push({
+            severity: 'error',
+            code: 'bot-blocked',
+            message: 'This page returned an "Access Denied" or bot-detection response. ' +
+                'The site blocks automated requests.',
+        });
+    }
+}
+function checkCaptcha(html, text, issues) {
+    if (CAPTCHA_HTML_PATTERNS.some((p) => p.test(html)) || CAPTCHA_TEXT_PATTERNS.some((p) => p.test(text))) {
+        issues.push({
+            severity: 'error',
+            code: 'captcha',
+            message: 'This page contains a CAPTCHA challenge. ' +
+                'The site requires human verification before serving content.',
+        });
+    }
+}
+function checkSoft404(html, text, issues) {
+    if (SOFT_404_TITLE_RE.test(html) || SOFT_404_META_RE.test(html)) {
+        issues.push({
+            severity: 'error',
+            code: 'soft-404',
+            message: 'This page appears to be a "not found" page that returned HTTP 200.',
+        });
+        return;
+    }
+    if (text.length < SOFT_404_CONTENT_LIMIT && SOFT_404_TEXT_PATTERNS.some((p) => p.test(text))) {
+        issues.push({
+            severity: 'error',
+            code: 'soft-404',
+            message: 'This page appears to be a "not found" page that returned HTTP 200.',
+        });
+    }
+}
+function checkErrorPage(html, text, issues) {
+    if (ERROR_PAGE_TITLE_RE.test(html) || ERROR_PAGE_HTML_PATTERNS.some((p) => p.test(html))) {
+        issues.push({
+            severity: 'error',
+            code: 'error-page',
+            message: 'This page appears to be an error or status page, not article content.',
+        });
+        return;
+    }
+    if (ERROR_PAGE_TEXT_PATTERNS.some((p) => p.test(text))) {
+        issues.push({
+            severity: 'error',
+            code: 'error-page',
+            message: 'This page appears to be an error or status page, not article content.',
+        });
+        return;
+    }
+    if (text.length < ERROR_PAGE_CONTENT_LIMIT && ERROR_PAGE_TEXT_GATED.some((p) => p.test(text))) {
+        issues.push({
+            severity: 'error',
+            code: 'error-page',
+            message: 'This page appears to be an error or status page, not article content.',
+        });
+    }
+}
+function checkMaintenancePage(text, issues) {
+    if (MAINTENANCE_TEXT_PATTERNS.some((p) => p.test(text))) {
+        issues.push({
+            severity: 'error',
+            code: 'maintenance-page',
+            message: 'This page appears to be a maintenance or "under construction" notice.',
+        });
+        return;
+    }
+    if (text.length < MAINTENANCE_CONTENT_LIMIT && MAINTENANCE_TEXT_GATED.some((p) => p.test(text))) {
+        issues.push({
+            severity: 'error',
+            code: 'maintenance-page',
+            message: 'This page appears to be a maintenance or "coming soon" notice.',
+        });
+    }
+}
+function checkCookieConsentPage(html, text, issues) {
+    if (text.length < COOKIE_CONSENT_HTML_LIMIT && COOKIE_HTML_PATTERNS.some((p) => p.test(html))) {
+        issues.push({
+            severity: 'error',
+            code: 'cookie-consent-only',
+            message: 'The extracted content appears to be only a cookie-consent overlay. ' +
+                'The actual page content was not captured.',
+        });
+        return;
+    }
+    if (text.length < COOKIE_CONSENT_TEXT_LIMIT && COOKIE_TEXT_PATTERNS.some((p) => p.test(text))) {
+        issues.push({
+            severity: 'error',
+            code: 'cookie-consent-only',
+            message: 'The extracted content appears to be only a cookie-consent overlay. ' +
+                'The actual page content was not captured.',
+        });
+    }
+}
+// ===== Tier 2: Structural HTML analysis ====================================
+function checkContentToHtmlRatio(html, text, issues) {
+    const htmlLen = html.length;
+    if (htmlLen < MIN_HTML_SIZE_FOR_RATIO)
+        return;
+    const ratio = text.length / htmlLen;
+    if (ratio < RATIO_ERROR_THRESHOLD && htmlLen > RATIO_ERROR_HTML_MIN) {
+        issues.push({
+            severity: 'error',
+            code: 'low-content-ratio',
+            message: `Only ${(ratio * 100).toFixed(1)}% of the page HTML is visible text ` +
+                `(${text.length} chars from ${htmlLen} bytes of HTML). ` +
+                'The page is almost entirely scripts/markup with negligible readable content.',
+        });
+        return;
+    }
+    if (ratio < RATIO_WARNING_THRESHOLD && htmlLen > RATIO_WARNING_HTML_MIN) {
+        issues.push({
+            severity: 'warning',
+            code: 'low-content-ratio',
+            message: `Only ${(ratio * 100).toFixed(1)}% of the page HTML is visible text. ` +
+                'The content may be incomplete or partially rendered.',
+        });
+    }
+}
+function checkLoginFormPage(html, text, issues) {
+    if (text.length >= LOGIN_FORM_CONTENT_LIMIT)
+        return;
+    // Fast path: no password field → not a login page
+    if (!/<input[^>]+type=["']password["']/i.test(html))
+        return;
+    if (LOGIN_TEXT_PATTERNS.some((p) => p.test(text))) {
+        issues.push({
+            severity: 'warning',
+            code: 'login-form',
+            message: 'This page appears to be a login form rather than article content.',
+        });
+    }
+}
+function checkMetaRobotsNoindex(html, issues) {
+    if (NOINDEX_RE.test(html)) {
+        issues.push({
+            severity: 'warning',
+            code: 'noindex-page',
+            message: 'This page has a "noindex" robots directive — the publisher does not intend it to be indexed.',
+        });
+    }
+}
+// ===== Tier 3: Content heuristics ==========================================
+function checkThinContent(text, issues) {
+    if (text.length < THIN_CONTENT_LIMIT) {
+        issues.push({
+            severity: 'error',
+            code: 'thin-content',
+            message: `Extracted content is only ${text.length} characters — too short to be a real article. ` +
+                'The page may be an error page, redirect landing, or access-restricted.',
+        });
+    }
+}
+function checkShortContent(text, issues) {
+    if (text.length >= THIN_CONTENT_LIMIT && text.length < SHORT_CONTENT_LIMIT) {
+        issues.push({
+            severity: 'warning',
+            code: 'short-content',
+            message: `Extracted content is only ${text.length} characters. ` +
+                'The page may be truncated, paywalled, or only partially rendered.',
+        });
+    }
+}
+function checkPaywall(text, issues) {
+    if (text.length >= PAYWALL_CONTENT_LIMIT)
+        return;
+    if (PAYWALL_PATTERNS.some((p) => p.test(text))) {
+        issues.push({
+            severity: 'warning',
+            code: 'possible-paywall',
+            message: 'The extracted content is short and contains language suggesting a paywall or login wall. ' +
+                'The stored content may be incomplete.',
+        });
+    }
+}
+function checkRedirectInterstitial(html, text, issues) {
+    if (text.length >= REDIRECT_CONTENT_LIMIT)
+        return;
+    const hasMetaRefresh = REDIRECT_META_REFRESH_RE.test(html);
+    const hasRedirectText = REDIRECT_TEXT_PATTERNS.some((p) => p.test(text));
+    if (hasMetaRefresh || hasRedirectText) {
+        issues.push({
+            severity: 'warning',
+            code: 'redirect-interstitial',
+            message: 'This page appears to be a redirect interstitial. ' +
+                'The actual destination content was not captured.',
+        });
+    }
+}
+function checkAccessRestricted(html, text, issues) {
+    if (ACCESS_RESTRICTED_HTML_PATTERNS.some((p) => p.test(html))) {
+        issues.push({
+            severity: 'warning',
+            code: 'access-restricted',
+            message: 'This page appears to have an age-verification or region-restriction gate.',
+        });
+        return;
+    }
+    if (text.length < ACCESS_RESTRICTED_CONTENT_LIMIT && ACCESS_RESTRICTED_TEXT_PATTERNS.some((p) => p.test(text))) {
+        issues.push({
+            severity: 'warning',
+            code: 'access-restricted',
+            message: 'This page appears to have an age-verification or region-restriction gate.',
+        });
+    }
+}
+function checkRepetitiveContent(text, issues) {
+    const sentences = text
+        .split(/[.!?]+\s+/)
+        .map((s) => s.trim().toLowerCase())
+        .filter((s) => s.length > 0);
+    if (sentences.length < MIN_SENTENCES_FOR_REPETITION)
+        return;
+    const counts = new Map();
+    for (const s of sentences) {
+        counts.set(s, (counts.get(s) ?? 0) + 1);
+    }
+    const uniqueRatio = counts.size / sentences.length;
+    const maxRepeats = Math.max(...counts.values());
+    if (uniqueRatio < REPETITION_UNIQUE_RATIO || maxRepeats > MAX_SENTENCE_REPEATS) {
+        issues.push({
+            severity: 'warning',
+            code: 'repetitive-content',
+            message: 'The extracted content appears highly repetitive, which may indicate ' +
+                'a broken extraction, placeholder page, or auto-generated content.',
+        });
+    }
+}
+function checkNavigationOnly(text, issues) {
+    if (text.length >= NAV_ONLY_CONTENT_LIMIT)
+        return;
+    const lines = text.split('\n').filter((l) => l.trim().length > 0);
+    if (lines.length < 3)
+        return;
+    // Lines that look like nav items: short, no sentence-ending punctuation
+    const navLikeCount = lines.filter((l) => l.trim().length < 30 && !/[.!?]$/.test(l.trim())).length;
+    if (navLikeCount / lines.length > 0.7) {
+        issues.push({
+            severity: 'warning',
+            code: 'navigation-only',
+            message: 'The extracted content appears to be mostly navigation links or menu items, ' +
+                'not article text.',
+        });
+    }
+}
+function checkEncodingGarbage(text, issues) {
+    // Replacement character ratio
+    const replacementCount = (text.match(/\uFFFD/g) ?? []).length;
+    if (text.length > 0 && replacementCount / text.length > REPLACEMENT_CHAR_RATIO) {
+        issues.push({
+            severity: 'warning',
+            code: 'encoding-garbage',
+            message: 'The extracted content contains excessive Unicode replacement characters, ' +
+                'suggesting a character-encoding mismatch.',
+        });
+        return;
+    }
+    // Mojibake patterns (UTF-8 decoded as Latin-1)
+    if (text.length < MOJIBAKE_TEXT_LIMIT) {
+        const mojibakeCount = (text.match(MOJIBAKE_RE) ?? []).length;
+        if (mojibakeCount >= MOJIBAKE_COUNT_THRESHOLD) {
+            issues.push({
+                severity: 'warning',
+                code: 'encoding-garbage',
+                message: 'The extracted content shows signs of mojibake (encoding corruption). ' +
+                    'Characters may not display correctly.',
+            });
+            return;
+        }
+    }
+    // Control characters
+    const controlCount = (text.match(CONTROL_CHAR_RE) ?? []).length;
+    if (text.length > 0 && controlCount / text.length > CONTROL_CHAR_RATIO) {
+        issues.push({
+            severity: 'warning',
+            code: 'encoding-garbage',
+            message: 'The extracted content contains excessive control characters, ' +
+                'suggesting binary data leaked into the text.',
+        });
+    }
+}
+function checkZeroWidthCharacters(text, issues) {
+    const matches = text.match(ZERO_WIDTH_RE);
+    if (matches && matches.length > ZERO_WIDTH_WARN_THRESHOLD) {
+        issues.push({
+            severity: 'warning',
+            code: 'zero-width-chars',
+            message: `The extracted content contains ${matches.length} zero-width or invisible Unicode characters, ` +
+                'which may indicate hidden text or Unicode steganography.',
+        });
+    }
+}
+function checkPromptInjection(text, issues) {
+    const matchedPatterns = [];
+    for (const pattern of PROMPT_INJECTION_PATTERNS) {
+        if (pattern.test(text)) {
+            matchedPatterns.push(pattern.source);
+        }
+    }
+    if (matchedPatterns.length === 0)
+        return;
+    if (matchedPatterns.length >= 3) {
+        issues.push({
+            severity: 'warning',
+            code: 'possible-prompt-injection',
+            message: `The extracted content matches ${matchedPatterns.length} prompt-injection patterns ` +
+                '(instruction overrides, role impersonation, or delimiter escapes). ' +
+                'This content has a high likelihood of containing adversarial text designed to manipulate an AI agent.',
+        });
+    }
+    else {
+        issues.push({
+            severity: 'warning',
+            code: 'possible-prompt-injection',
+            message: 'The extracted content contains text that resembles a prompt-injection attempt ' +
+                '(e.g. instruction overrides, system impersonation, or delimiter escapes). ' +
+                'The content was stored but may contain adversarial text.',
+        });
+    }
+}
+//# sourceMappingURL=validate.js.map

package/dist/pipeline/validate.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"validate.js","sourceRoot":"","sources":["../../pipeline/validate.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAYH,8EAA8E;AAE9E,MAAM,kBAAkB,GAAG,EAAE,CAAC;AAC9B,MAAM,mBAAmB,GAAG,GAAG,CAAC;AAChC,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAClC,MAAM,sBAAsB,GAAG,IAAI,CAAC;AACpC,MAAM,wBAAwB,GAAG,GAAG,CAAC;AACrC,MAAM,yBAAyB,GAAG,GAAG,CAAC;AACtC,MAAM,sBAAsB,GAAG,GAAG,CAAC;AACnC,MAAM,wBAAwB,GAAG,GAAG,CAAC;AACrC,MAAM,yBAAyB,GAAG,GAAG,CAAC;AACtC,MAAM,yBAAyB,GAAG,GAAG,CAAC;AACtC,MAAM,+BAA+B,GAAG,GAAG,CAAC;AAC5C,MAAM,sBAAsB,GAAG,GAAG,CAAC;AAEnC,wBAAwB;AACxB,MAAM,uBAAuB,GAAG,IAAI,CAAC;AACrC,MAAM,qBAAqB,GAAG,IAAI,CAAC;AACnC,MAAM,oBAAoB,GAAG,IAAI,CAAC;AAClC,MAAM,uBAAuB,GAAG,IAAI,CAAC;AACrC,MAAM,sBAAsB,GAAG,IAAI,CAAC;AAEpC,qBAAqB;AACrB,MAAM,4BAA4B,GAAG,CAAC,CAAC;AACvC,MAAM,uBAAuB,GAAG,GAAG,CAAC;AACpC,MAAM,oBAAoB,GAAG,CAAC,CAAC;AAE/B,mBAAmB;AACnB,MAAM,sBAAsB,GAAG,IAAI,CAAC;AACpC,MAAM,wBAAwB,GAAG,CAAC,CAAC;AACnC,MAAM,mBAAmB,GAAG,IAAI,CAAC;AACjC,MAAM,kBAAkB,GAAG,IAAI,CAAC;AAEhC,qCAAqC;AACrC,MAAM,yBAAyB,GAAG,EAAE,CAAC;AAErC,+EAA+E;AAC/E,MAAM,eAAe,GAAG,OAAO,CAAC;AAEhC,8EAA8E;AAE9E,8EAA8E;AAE9E,MAAM,uBAAuB,GAAG;IAC9B,aAAa;IACb,8BAA8B;IAC9B,mBAAmB;IACnB,0BAA0B;IAC1B,+BAA+B;IAC/B,SAAS;IACT,UAAU;IACV,qBAAqB;IACrB,UAAU;IACV,SAAS;IACT,sBAAsB;IACtB,YAAY;IACZ,cAAc;IACd,qBAAqB;IACrB,aAAa;IACb,aAAa;IACb,WAAW;IACX,WAAW;IACX,SAAS;IACT,iBAAiB;IACjB,kBAAkB;IAClB,qBAAqB;IACrB,0BAA0B;IAC1B,0BAA0B;CAC3B,CAAC;AAEF,MAAM,uBAAuB,GAAG;IAC9B,aAAa;IACb,4CAA4C;IAC5C,sCAAsC;IACtC,cAAc;IACd,mBAAmB;IACnB,wBAAwB;IACxB,kBAAkB;IAClB,iDAAiD;IACjD,mEAAmE;IACnE,sDAAsD;IACtD,iDAAiD;CAClD,CAAC;AAEF,8EAA8E;AAE9E,MAAM,qBAAqB,GAAG;IAC5B,cAAc;IACd,YAAY;IACZ,2BAA2B;IAC3B,wBAAwB;IACxB,uBAAuB;IACvB,yCAAyC;IACzC,eAAe;IACf,2BAA2B;IAC3B,aAAa;IACb,kBAAkB;IAClB,UAAU;IACV,UAAU;IACV,UAAU;IACV,iBAAiB;IACjB,0BAA0B;CAC3B,CAAC;AAEF,MAAM,qBAAqB,GAAG;IAC5B,8BAA8B;IAC9B,8BAA8B;IAC9B,2BAA2B;IAC3B,oCAAoC;IACpC,sEAAsE;CACvE,CAAC;AAEF,8EAA8E;AAE9E,MAAM,iBAAiB,GACrB,uGAAuG,CAAC;AAE1G,MAAM,sBAAsB,GAAG;IAC7B,yCAAyC;IACzC,uEAAuE;IACvE,kIAAkI;IAClI,mDAAmD;IACnD,8BAA8B;IAC9B,qDAAqD;IACrD,kGAAkG;CACnG,CAAC;AAEF,MAAM,gBAAgB,GACpB,gGAAgG,CAAC;AAEnG,8EAA8E;AAE9E,MAAM,mBAAmB,GACvB,0HAA0H,CAAC;AAE7H,MAAM,wBAAwB,GAAG;IAC/B,uJAAuJ;IACvJ,2EAA2E;IAC3E,mBAAmB;IACnB,oBAAoB;CACrB,CAAC;AAEF,MAAM,qBAAqB,GAAG;IAC5B,yBAAyB;CAC1B,CAAC;AAEF,MAAM,wBAAwB,GAAG;IAC/B,wBAAwB;IACxB,gEAAgE;IAChE,aAAa;CACd,CAAC;AAEF,8EAA8E;AAE9E,MAAM,yBAAyB,GAAG;IAChC,kFAAkF;IAClF,mEAAmE;IACnE,iEAAiE;IACjE,qBAAqB;IACrB,wBAAwB;IACxB,uDAAuD;CACxD,CAAC;AAEF,MAAM,sBAAsB,GAAG;IAC7B,cAAc;CACf,CAAC;AAEF,8EAA8E;AAE9E,MAAM,oBAAoB,GAAG;IAC3B,iBAAiB;IACjB,qCAAqC;IACrC,iDAAiD;IACjD,uCAAuC;IACvC,+DAA+D;IAC/D,yBAAyB;CAC1B,CAAC;AAEF,MAAM,oBAAoB,GAAG;IAC3B,wFAAwF;IACxF,kDAAkD;CACnD,CAAC;AAEF,8EAA8E;AAE9E,MAAM,gBAAgB,GAAG;IACvB,wCAAwC;IACxC,kDAAkD;IAClD,0CAA0C;IAC1C,+BAA+B;IAC/B,eAAe;IACf,kBAAkB;IAClB,6BAA6B;IAC7B,mHAAmH;IACnH,gCAAgC;IAChC,8GAA8G;IAC9G,qDAAqD;IACrD,qDAAqD;IACrD,kCAAkC;CACnC,CAAC;AAEF,8EAA8E;AAE9E,MAAM,sBAAsB,GAAG;IAC7B,8DAA8D;IAC9D,uEAAuE;IACvE,mCAAmC;IACnC,6CAA6C;CAC9C,CAAC;AAEF,MAAM,wBAAwB,GAAG,8CAA8C,CAAC;AAEhF,8EAA8E;AAE9E,MAAM,+BAA+B,GAAG;IACtC,0EAA0E;IAC1E,mCAAmC;IACnC,oGAAoG;IACpG,qFAAqF;IACrF,mDAAmD;CACpD,CAAC;AAEF,MAAM,+BAA+B,GAAG;IACtC,0DAA0D;IAC1D,kBAAkB;CACnB,CAAC;AAEF,8EAA8E;AAE9E,MAAM,mBAAmB,GAAG;IAC1B,UAAU;IACV,SAAS;IACT,4BAA4B;IAC5B,cAAc;IACd,iCAAiC;CAClC,CAAC;AAEF,8EAA8E;AAE9E,MAAM,UAAU,GACd,8HAA8H,CAAC;AAEjI,8EAA8E;AAE9E,MAAM,WAAW,GAAG,oFAAoF,CAAC;AACzG,MAAM,eAAe,GAAG,mCAAmC,CAAC;AAE5D,8EAA8E;AAE9E,MAAM,aAAa,GAAG,kDAAkD,CAAC;AAEzE,8EAA8E;AAE9E,MAAM,yBAAyB,GAAG;IAChC,gCAAgC;IAChC,oFAAoF;IACpF,uFAAuF;IACvF,oFAAoF;IACpF,sFAAsF;IAEtF,4BAA4B;IAC5B,eAAe;IACf,wEAAwE;IACxE,oFAAoF;IACpF,2EAA2E;IAC3E,gDAAgD;IAEhD,2BAA2B;IAC3B,aAAa;IACb,aAAa;IACb,WAAW;IACX,gBAAgB;IAChB,aAAa;IACb,8BAA8B;IAE9B,4BAA4B;IAC5B,4FAA4F;IAC5F,gHAAgH;IAChH,wFAAwF;IACxF,2EAA2E;CAC5E,CAAC;AAEF,8EAA8E;AAE9E;;;;;;GAMG;AACH,MAAM,UAAU,eAAe,CAC7B,IAAY,EACZ,SAA2B;IAE3B,MAAM,MAAM,GAAmB,EAAE,CAAC;IAClC,MAAM,IAAI,GAAG,SAAS,CAAC,OAAO,CAAC;IAC/B,oDAAoD;IACpD,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,GAAG,eAAe,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,eAAe,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAEvF,4EAA4E;IAC5E,eAAe,CAAC,QAAQ,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;IACxC,YAAY,CAAC,QAAQ,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;IACrC,YAAY,CAAC,QAAQ,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;IACrC,cAAc,CAAC,QAAQ,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;IACvC,oBAAoB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IACnC,sBAAsB,CAAC,QAAQ,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;IAE/C,4EAA4E;IAC5E,uBAAuB,CAAC,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;IAC5C,kBAAkB,CAAC,QAAQ,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;IAC3C,sBAAsB,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAEzC,4EAA4E;IAC5E,gBAAgB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAC/B,iBAAiB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAChC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAC3B,yBAAyB,CAAC,QAAQ,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;IAClD,qBAAqB,CAAC,QAAQ,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;IAC9C,sBAAsB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IACrC,mBAAmB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAClC,oBAAoB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IACnC,wBAAwB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IACvC,oBAAoB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAEnC,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,mBAAmB,CAAC,SAA2B;IAC7D,MAAM,MAAM,GAAmB,EAAE,CAAC;IAClC,MAAM,IAAI,GAAG,SAAS,CAAC,OAAO,CAAC;IAE/B,gBAAgB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAC/B,iBAAiB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAChC,sBAAsB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IACrC,mBAAmB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAClC,oBAAoB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IACnC,wBAAwB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IACvC,oBAAoB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAEnC,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,8EAA8E;AAE9E,SAAS,eAAe,CAAC,IAAY,EAAE,IAAY,EAAE,MAAsB;IACzE,IAAI,uBAAuB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QACtD,MAAM,CAAC,IAAI,CAAC;YACV,QAAQ,EAAE,OAAO;YACjB,IAAI,EAAE,aAAa;YACnB,OAAO,EACL,+DAA+D;gBAC/D,qCAAqC;SACxC,CAAC,CAAC;QACH,OAAO;IACT,CAAC;IACD,IAAI,uBAAuB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QACtD,MAAM,CAAC,IAAI,CAAC;YACV,QAAQ,EAAE,OAAO;YACjB,IAAI,EAAE,aAAa;YACnB,OAAO,EACL,mEAAmE;gBACnE,qCAAqC;SACxC,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED,SAAS,YAAY,CAAC,IAAY,EAAE,IAAY,EAAE,MAAsB;IACtE,IAAI,qBAAqB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,qBAAqB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QACvG,MAAM,CAAC,IAAI,CAAC;YACV,QAAQ,EAAE,OAAO;YACjB,IAAI,EAAE,SAAS;YACf,OAAO,EACL,0CAA0C;gBAC1C,8DAA8D;SACjE,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED,SAAS,YAAY,CAAC,IAAY,EAAE,IAAY,EAAE,MAAsB;IACtE,IAAI,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QAChE,MAAM,CAAC,IAAI,CAAC;YACV,QAAQ,EAAE,OAAO;YACjB,IAAI,EAAE,UAAU;YAChB,OAAO,EAAE,oEAAoE;SAC9E,CAAC,CAAC;QACH,OAAO;IACT,CAAC;IACD,IAAI,IAAI,CAAC,MAAM,GAAG,sBAAsB,IAAI,sBAAsB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QAC7F,MAAM,CAAC,IAAI,CAAC;YACV,QAAQ,EAAE,OAAO;YACjB,IAAI,EAAE,UAAU;YAChB,OAAO,EAAE,oEAAoE;SAC9E,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED,SAAS,cAAc,CAAC,IAAY,EAAE,IAAY,EAAE,MAAsB;IACxE,IAAI,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,wBAAwB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QACzF,MAAM,CAAC,IAAI,CAAC;YACV,QAAQ,EAAE,OAAO;YACjB,IAAI,EAAE,YAAY;YAClB,OAAO,EAAE,uEAAuE;SACjF,CAAC,CAAC;QACH,OAAO;IACT,CAAC;IACD,IAAI,wBAAwB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QACvD,MAAM,CAAC,IAAI,CAAC;YACV,QAAQ,EAAE,OAAO;YACjB,IAAI,EAAE,YAAY;YAClB,OAAO,EAAE,uEAAuE;SACjF,CAAC,CAAC;QACH,OAAO;IACT,CAAC;IACD,IAAI,IAAI,CAAC,MAAM,GAAG,wBAAwB,IAAI,qBAAqB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QAC9F,MAAM,CAAC,IAAI,CAAC;YACV,QAAQ,EAAE,OAAO;YACjB,IAAI,EAAE,YAAY;YAClB,OAAO,EAAE,uEAAuE;SACjF,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED,SAAS,oBAAoB,CAAC,IAAY,EAAE,MAAsB;IAChE,IAAI,yBAAyB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QACxD,MAAM,CAAC,IAAI,CAAC;YACV,QAAQ,EAAE,OAAO;YACjB,IAAI,EAAE,kBAAkB;YACxB,OAAO,EAAE,uEAAuE;SACjF,CAAC,CAAC;QACH,OAAO;IACT,CAAC;IACD,IAAI,IAAI,CAAC,MAAM,GAAG,yBAAyB,IAAI,sBAAsB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QAChG,MAAM,CAAC,IAAI,CAAC;YACV,QAAQ,EAAE,OAAO;YACjB,IAAI,EAAE,kBAAkB;YACxB,OAAO,EAAE,gEAAgE;SAC1E,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED,SAAS,sBAAsB,CAAC,IAAY,EAAE,IAAY,EAAE,MAAsB;IAChF,IAAI,IAAI,CAAC,MAAM,GAAG,yBAAyB,IAAI,oBAAoB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QAC9F,MAAM,CAAC,IAAI,CAAC;YACV,QAAQ,EAAE,OAAO;YACjB,IAAI,EAAE,qBAAqB;YAC3B,OAAO,EACL,qEAAqE;gBACrE,2CAA2C;SAC9C,CAAC,CAAC;QACH,OAAO;IACT,CAAC;IACD,IAAI,IAAI,CAAC,MAAM,GAAG,yBAAyB,IAAI,oBAAoB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QAC9F,MAAM,CAAC,IAAI,CAAC;YACV,QAAQ,EAAE,OAAO;YACjB,IAAI,EAAE,qBAAqB;YAC3B,OAAO,EACL,qEAAqE;gBACrE,2CAA2C;SAC9C,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED,8EAA8E;AAE9E,SAAS,uBAAuB,CAAC,IAAY,EAAE,IAAY,EAAE,MAAsB;IACjF,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC;IAC5B,IAAI,OAAO,GAAG,uBAAuB;QAAE,OAAO;IAE9C,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC;IAEpC,IAAI,KAAK,GAAG,qBAAqB,IAAI,OAAO,GAAG,oBAAoB,EAAE,CAAC;QACpE,MAAM,CAAC,IAAI,CAAC;YACV,QAAQ,EAAE,OAAO;YACjB,IAAI,EAAE,mBAAmB;YACzB,OAAO,EACL,QAAQ,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,qCAAqC;gBACrE,IAAI,IAAI,CAAC,MAAM,eAAe,OAAO,mBAAmB;gBACxD,8EAA8E;SACjF,CAAC,CAAC;QACH,OAAO;IACT,CAAC;IAED,IAAI,KAAK,GAAG,uBAAuB,IAAI,OAAO,GAAG,sBAAsB,EAAE,CAAC;QACxE,MAAM,CAAC,IAAI,CAAC;YACV,QAAQ,EAAE,SAAS;YACnB,IAAI,EAAE,mBAAmB;YACzB,OAAO,EACL,QAAQ,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,sCAAsC;gBACtE,sDAAsD;SACzD,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED,SAAS,kBAAkB,CAAC,IAAY,EAAE,IAAY,EAAE,MAAsB;IAC5E,IAAI,IAAI,CAAC,MAAM,IAAI,wBAAwB;QAAE,OAAO;IACpD,kDAAkD;IAClD,IAAI,CAAC,mCAAmC,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO;IAC5D,IAAI,mBAAmB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QAClD,MAAM,CAAC,IAAI,CAAC;YACV,QAAQ,EAAE,SAAS;YACnB,IAAI,EAAE,YAAY;YAClB,OAAO,EACL,mEAAmE;SACtE,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED,SAAS,sBAAsB,CAAC,IAAY,EAAE,MAAsB;IAClE,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QAC1B,MAAM,CAAC,IAAI,CAAC;YACV,QAAQ,EAAE,SAAS;YACnB,IAAI,EAAE,cAAc;YACpB,OAAO,EACL,8FAA8F;SACjG,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED,8EAA8E;AAE9E,SAAS,gBAAgB,CAAC,IAAY,EAAE,MAAsB;IAC5D,IAAI,IAAI,CAAC,MAAM,GAAG,kBAAkB,EAAE,CAAC;QACrC,MAAM,CAAC,IAAI,CAAC;YACV,QAAQ,EAAE,OAAO;YACjB,IAAI,EAAE,cAAc;YACpB,OAAO,EACL,6BAA6B,IAAI,CAAC,MAAM,gDAAgD;gBACxF,wEAAwE;SAC3E,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED,SAAS,iBAAiB,CAAC,IAAY,EAAE,MAAsB;IAC7D,IAAI,IAAI,CAAC,MAAM,IAAI,kBAAkB,IAAI,IAAI,CAAC,MAAM,GAAG,mBAAmB,EAAE,CAAC;QAC3E,MAAM,CAAC,IAAI,CAAC;YACV,QAAQ,EAAE,SAAS;YACnB,IAAI,EAAE,eAAe;YACrB,OAAO,EACL,6BAA6B,IAAI,CAAC,MAAM,eAAe;gBACvD,mEAAmE;SACtE,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED,SAAS,YAAY,CAAC,IAAY,EAAE,MAAsB;IACxD,IAAI,IAAI,CAAC,MAAM,IAAI,qBAAqB;QAAE,OAAO;IACjD,IAAI,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QAC/C,MAAM,CAAC,IAAI,CAAC;YACV,QAAQ,EAAE,SAAS;YACnB,IAAI,EAAE,kBAAkB;YACxB,OAAO,EACL,2FAA2F;gBAC3F,uCAAuC;SAC1C,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED,SAAS,yBAAyB,CAAC,IAAY,EAAE,IAAY,EAAE,MAAsB;IACnF,IAAI,IAAI,CAAC,MAAM,IAAI,sBAAsB;QAAE,OAAO;IAClD,MAAM,cAAc,GAAG,wBAAwB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC3D,MAAM,eAAe,GAAG,sBAAsB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IACzE,IAAI,cAAc,IAAI,eAAe,EAAE,CAAC;QACtC,MAAM,CAAC,IAAI,CAAC;YACV,QAAQ,EAAE,SAAS;YACnB,IAAI,EAAE,uBAAuB;YAC7B,OAAO,EACL,mDAAmD;gBACnD,kDAAkD;SACrD,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED,SAAS,qBAAqB,CAAC,IAAY,EAAE,IAAY,EAAE,MAAsB;IAC/E,IAAI,+BAA+B,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QAC9D,MAAM,CAAC,IAAI,CAAC;YACV,QAAQ,EAAE,SAAS;YACnB,IAAI,EAAE,mBAAmB;YACzB,OAAO,EAAE,2EAA2E;SACrF,CAAC,CAAC;QACH,OAAO;IACT,CAAC;IACD,IAAI,IAAI,CAAC,MAAM,GAAG,+BAA+B,IAAI,+BAA+B,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QAC/G,MAAM,CAAC,IAAI,CAAC;YACV,QAAQ,EAAE,SAAS;YACnB,IAAI,EAAE,mBAAmB;YACzB,OAAO,EAAE,2EAA2E;SACrF,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED,SAAS,sBAAsB,CAAC,IAAY,EAAE,MAAsB;IAClE,MAAM,SAAS,GAAG,IAAI;SACnB,KAAK,CAAC,WAAW,CAAC;SAClB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;SAClC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAE/B,IAAI,SAAS,CAAC,MAAM,GAAG,4BAA4B;QAAE,OAAO;IAE5D,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,KAAK,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;QAC1B,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1C,CAAC;IAED,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,GAAG,SAAS,CAAC,MAAM,CAAC;IACnD,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;IAEhD,IAAI,WAAW,GAAG,uBAAuB,IAAI,UAAU,GAAG,oBAAoB,EAAE,CAAC;QAC/E,MAAM,CAAC,IAAI,CAAC;YACV,QAAQ,EAAE,SAAS;YACnB,IAAI,EAAE,oBAAoB;YAC1B,OAAO,EACL,sEAAsE;gBACtE,mEAAmE;SACtE,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED,SAAS,mBAAmB,CAAC,IAAY,EAAE,MAAsB;IAC/D,IAAI,IAAI,CAAC,MAAM,IAAI,sBAAsB;QAAE,OAAO;IAElD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAClE,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO;IAE7B,wEAAwE;IACxE,MAAM,YAAY,GAAG,KAAK,CAAC,MAAM,CAC/B,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CACxD,CAAC,MAAM,CAAC;IAET,IAAI,YAAY,GAAG,KAAK,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;QACtC,MAAM,CAAC,IAAI,CAAC;YACV,QAAQ,EAAE,SAAS;YACnB,IAAI,EAAE,iBAAiB;YACvB,OAAO,EACL,6EAA6E;gBAC7E,mBAAmB;SACtB,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED,SAAS,oBAAoB,CAAC,IAAY,EAAE,MAAsB;IAChE,8BAA8B;IAC9B,MAAM,gBAAgB,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;IAC9D,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,gBAAgB,GAAG,IAAI,CAAC,MAAM,GAAG,sBAAsB,EAAE,CAAC;QAC/E,MAAM,CAAC,IAAI,CAAC;YACV,QAAQ,EAAE,SAAS;YACnB,IAAI,EAAE,kBAAkB;YACxB,OAAO,EACL,2EAA2E;gBAC3E,2CAA2C;SAC9C,CAAC,CAAC;QACH,OAAO;IACT,CAAC;IAED,+CAA+C;IAC/C,IAAI,IAAI,CAAC,MAAM,GAAG,mBAAmB,EAAE,CAAC;QACtC,MAAM,aAAa,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;QAC7D,IAAI,aAAa,IAAI,wBAAwB,EAAE,CAAC;YAC9C,MAAM,CAAC,IAAI,CAAC;gBACV,QAAQ,EAAE,SAAS;gBACnB,IAAI,EAAE,kBAAkB;gBACxB,OAAO,EACL,uEAAuE;oBACvE,uCAAuC;aAC1C,CAAC,CAAC;YACH,OAAO;QACT,CAAC;IACH,CAAC;IAED,qBAAqB;IACrB,MAAM,YAAY,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;IAChE,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,YAAY,GAAG,IAAI,CAAC,MAAM,GAAG,kBAAkB,EAAE,CAAC;QACvE,MAAM,CAAC,IAAI,CAAC;YACV,QAAQ,EAAE,SAAS;YACnB,IAAI,EAAE,kBAAkB;YACxB,OAAO,EACL,+DAA+D;gBAC/D,8CAA8C;SACjD,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED,SAAS,wBAAwB,CAAC,IAAY,EAAE,MAAsB;IACpE,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;IAC1C,IAAI,OAAO,IAAI,OAAO,CAAC,MAAM,GAAG,yBAAyB,EAAE,CAAC;QAC1D,MAAM,CAAC,IAAI,CAAC;YACV,QAAQ,EAAE,SAAS;YACnB,IAAI,EAAE,kBAAkB;YACxB,OAAO,EACL,kCAAkC,OAAO,CAAC,MAAM,+CAA+C;gBAC/F,0DAA0D;SAC7D,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED,SAAS,oBAAoB,CAAC,IAAY,EAAE,MAAsB;IAChE,MAAM,eAAe,GAAa,EAAE,CAAC;IACrC,KAAK,MAAM,OAAO,IAAI,yBAAyB,EAAE,CAAC;QAChD,IAAI,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YACvB,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QACvC,CAAC;IACH,CAAC;IAED,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO;IAEzC,IAAI,eAAe,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QAChC,MAAM,CAAC,IAAI,CAAC;YACV,QAAQ,EAAE,SAAS;YACnB,IAAI,EAAE,2BAA2B;YACjC,OAAO,EACL,iCAAiC,eAAe,CAAC,MAAM,6BAA6B;gBACpF,qEAAqE;gBACrE,uGAAuG;SAC1G,CAAC,CAAC;IACL,CAAC;SAAM,CAAC;QACN,MAAM,CAAC,IAAI,CAAC;YACV,QAAQ,EAAE,SAAS;YACnB,IAAI,EAAE,2BAA2B;YACjC,OAAO,EACL,gFAAgF;gBAChF,4EAA4E;gBAC5E,0DAA0D;SAC7D,CAAC,CAAC;IACL,CAAC;AACH,CAAC"}

package/dist/storage/source-repo.d.ts

@@ -4,12 +4,14 @@ export declare class SourceRepository implements ISourceRepository {
     private readonly db;
     private readonly insertStmt;
     private readonly getByIdStmt;
+    private readonly getByUrlStmt;
     private readonly softDeleteStmt;
     private readonly restoreStmt;
     private readonly updateLastAccessedStmt;
     constructor(db: BetterSqlite3.Database);
     insert(source: Omit<SourceRecord, 'createdAt' | 'updatedAt' | 'lastAccessedAt'>): SourceRecord;
     getById(id: string): SourceRecord | null;
+    getByUrl(url: string): SourceRecord | null;
     list(filter?: SourceFilter, pagination?: PaginationOptions): SourceRecord[];
     count(filter?: SourceFilter): number;
     softDelete(id: string): boolean;