arcfetch 1.1.0 → 1.2.0

package/src/config/schema.ts

@@ -6,7 +6,7 @@ export const QualityConfigSchema = z.object({
 });
 
 export const PathsConfigSchema = z.object({
-  tempDir: z.string().default('.tmp'),
+  tempDir: z.string().default('.tmp/arcfetch'),
   docsDir: z.string().default('docs/ai/references'),
 });
 
@@ -15,20 +15,13 @@ export const PlaywrightConfigSchema = z.object({
   waitStrategy: z.enum(['networkidle', 'domcontentloaded', 'load']).default('networkidle'),
 });
 
-export const RetryConfigSchema = z.object({
-  maxAttempts: z.number().default(2),
-  backoffMs: z.number().default(1000),
-});
-
 export const FetchiConfigSchema = z.object({
   quality: QualityConfigSchema.default({}),
   paths: PathsConfigSchema.default({}),
   playwright: PlaywrightConfigSchema.default({}),
-  retry: RetryConfigSchema.default({}),
 });
 
 export type FetchiConfig = z.infer<typeof FetchiConfigSchema>;
 export type QualityConfig = z.infer<typeof QualityConfigSchema>;
 export type PathsConfig = z.infer<typeof PathsConfigSchema>;
 export type PlaywrightConfig = z.infer<typeof PlaywrightConfigSchema>;
-export type RetryConfig = z.infer<typeof RetryConfigSchema>;

package/src/core/cache.ts

@@ -1,6 +1,6 @@
 import { existsSync, mkdirSync, readdirSync, readFileSync, unlinkSync, writeFileSync } from 'node:fs';
 import { writeFile } from 'node:fs/promises';
-import { join, basename } from 'node:path';
+import { basename, join } from 'node:path';
 import type { FetchiConfig } from '../config/schema';
 
 export interface CachedReference {
@@ -43,7 +43,7 @@ export interface DeleteResult {
  */
 export function findByUrl(config: FetchiConfig, url: string): CachedReference | null {
   const { references } = listCached(config);
-  return references.find(r => r.url === url) || null;
+  return references.find((r) => r.url === url) || null;
 }
 
 /**
@@ -89,9 +89,10 @@ export async function saveToTemp(
     const filepath = existing && refetch ? existing.filepath : join(tempDir, filename);
 
     const today = new Date().toISOString().split('T')[0];
+    const sanitizedUrl = url.replace(/[\r\n]/g, '');
     let fileContent = `---\n`;
     fileContent += `title: "${title.replace(/"/g, '\\"')}"\n`;
-    fileContent += `source_url: ${url}\n`;
+    fileContent += `source_url: ${sanitizedUrl}\n`;
     fileContent += `fetched_date: ${today}\n`;
     fileContent += `type: web\n`;
     fileContent += `status: temporary\n`;
@@ -121,7 +122,7 @@ export function listCached(config: FetchiConfig): ListResult {
       return { references: [] };
     }
 
-    const files = readdirSync(tempDir).filter(f => f.endsWith('.md'));
+    const files = readdirSync(tempDir).filter((f) => f.endsWith('.md'));
     const references: CachedReference[] = [];
 
     for (const file of files) {
@@ -135,7 +136,12 @@ export function listCached(config: FetchiConfig): ListResult {
 
       const getValue = (key: string): string => {
         const match = frontmatter.match(new RegExp(`^${key}:\\s*(.+)$`, 'm'));
-        return match ? match[1].trim().replace(/^["']|["']$/g, '').trim() : '';
+        return match
+          ? match[1]
+              .trim()
+              .replace(/^["']|["']$/g, '')
+              .trim()
+          : '';
       };
 
       // Use filename (without .md) as refId
@@ -168,7 +174,7 @@ export function listCached(config: FetchiConfig): ListResult {
  */
 export function findCached(config: FetchiConfig, refId: string): CachedReference | null {
   const { references } = listCached(config);
-  return references.find(r => r.refId === refId) || null;
+  return references.find((r) => r.refId === refId) || null;
 }
 
 /**
@@ -249,13 +255,6 @@ export function deleteCached(config: FetchiConfig, refId: string): DeleteResult
   }
 }
 
-/**
- * Get cache root (for backwards compatibility)
- */
-export function findCacheRoot(): string {
-  return process.cwd();
-}
-
 // ============================================================================
 // LINK EXTRACTION
 // ============================================================================

package/src/core/extractor.ts

@@ -1,7 +1,7 @@
-import TurndownService from 'turndown';
-import { gfm } from 'turndown-plugin-gfm';
 import { Readability } from '@mozilla/readability';
 import { parseHTML } from 'linkedom';
+import TurndownService from 'turndown';
+import { gfm } from 'turndown-plugin-gfm';
 import { cleanMarkdownComplete } from '../utils/markdown-cleaner';
 
 export interface ExtractionResult {
@@ -29,11 +29,7 @@ turndown.addRule('removeComments', {
   replacement: () => '',
 });
 
-export async function processHtmlToMarkdown(
-  html: string,
-  url: string,
-  verbose = false
-): Promise<ExtractionResult> {
+export async function processHtmlToMarkdown(html: string, url: string, verbose = false): Promise<ExtractionResult> {
   try {
     if (verbose) {
       console.error(`📝 Processing HTML (${html.length} chars)`);

package/src/core/fetch-links.ts

@@ -0,0 +1,85 @@
+import type { FetchiConfig } from '../config/schema';
+import { extractLinksFromCached, saveToTemp } from './cache';
+import { closeBrowser, fetchUrl } from './pipeline';
+
+export interface FetchLinkResult {
+  url: string;
+  status: 'new' | 'cached' | 'failed';
+  refId?: string;
+  error?: string;
+}
+
+export interface FetchLinksFromRefResult {
+  results: FetchLinkResult[];
+  summary: { new: number; cached: number; failed: number };
+  error?: string;
+}
+
+export async function fetchLinksFromRef(
+  config: FetchiConfig,
+  refId: string,
+  options?: { refetch?: boolean; verbose?: boolean; onProgress?: (result: FetchLinkResult) => void }
+): Promise<FetchLinksFromRefResult> {
+  const linksResult = extractLinksFromCached(config, refId);
+
+  if (linksResult.error) {
+    return { results: [], summary: { new: 0, cached: 0, failed: 0 }, error: linksResult.error };
+  }
+
+  if (linksResult.count === 0) {
+    return { results: [], summary: { new: 0, cached: 0, failed: 0 } };
+  }
+
+  const results: FetchLinkResult[] = [];
+  const concurrency = 5;
+  const urls = linksResult.links.map((l) => l.href);
+  const verbose = options?.verbose ?? false;
+  const refetch = options?.refetch ?? false;
+
+  for (let i = 0; i < urls.length; i += concurrency) {
+    const batch = urls.slice(i, i + concurrency);
+    const batchPromises = batch.map(async (url): Promise<FetchLinkResult> => {
+      try {
+        const fetchResult = await fetchUrl(url, config, verbose);
+
+        if (!fetchResult.success) {
+          return { url, status: 'failed', error: fetchResult.error };
+        }
+
+        const saveResult = await saveToTemp(config, fetchResult.title!, url, fetchResult.markdown!, undefined, refetch);
+
+        if (saveResult.error) {
+          return { url, status: 'failed', error: saveResult.error };
+        }
+
+        if (saveResult.alreadyExists) {
+          return { url, status: 'cached', refId: saveResult.refId };
+        }
+
+        return { url, status: 'new', refId: saveResult.refId };
+      } catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        return { url, status: 'failed', error: message };
+      }
+    });
+
+    const batchResults = await Promise.all(batchPromises);
+    results.push(...batchResults);
+
+    if (options?.onProgress) {
+      for (const r of batchResults) {
+        options.onProgress(r);
+      }
+    }
+  }
+
+  await closeBrowser();
+
+  const summary = {
+    new: results.filter((r) => r.status === 'new').length,
+    cached: results.filter((r) => r.status === 'cached').length,
+    failed: results.filter((r) => r.status === 'failed').length,
+  };
+
+  return { results, summary };
+}

package/src/core/index.ts

@@ -1,4 +1,5 @@
-export * from './pipeline';
-export * from './extractor';
 export * from './cache';
+export * from './extractor';
+export * from './fetch-links';
+export * from './pipeline';
 export * from './playwright/index';

package/src/core/pipeline.ts

@@ -1,7 +1,7 @@
 import type { FetchiConfig } from '../config/schema';
-import { validateMarkdown, type ValidationResult } from '../utils/markdown-validator';
-import { fetchWithBrowser, closeBrowser } from './playwright/manager';
+import { type ValidationResult, validateMarkdown } from '../utils/markdown-validator';
 import { processHtmlToMarkdown } from './extractor';
+import { closeBrowser, fetchWithBrowser } from './playwright/manager';
 
 export interface FetchResult {
   success: boolean;
@@ -31,8 +31,9 @@ async function simpleFetch(url: string, verbose = false): Promise<SimpleFetchRes
     const response = await fetch(url, {
       redirect: 'follow',
       headers: {
-        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
-        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+        'User-Agent':
+          'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+        Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
         'Accept-Language': 'en-US,en;q=0.5',
       },
     });
@@ -54,12 +55,7 @@ async function simpleFetch(url: string, verbose = false): Promise<SimpleFetchRes
   }
 }
 
-async function tryPlaywright(
-  url: string,
-  config: FetchiConfig,
-  reason: string,
-  verbose = false
-): Promise<FetchResult> {
+async function tryPlaywright(url: string, config: FetchiConfig, reason: string, verbose = false): Promise<FetchResult> {
   if (verbose) {
     console.error(`🎭 Trying Playwright (reason: ${reason})`);
   }
@@ -82,7 +78,7 @@ async function tryPlaywright(
     };
   }
 
-  const quality = validateMarkdown(extracted.markdown!);
+  const quality = validateMarkdown(extracted.markdown!, { sourceHtmlLength: browserResult.html.length });
 
   if (quality.score < config.quality.minScore) {
     return {
@@ -114,9 +110,24 @@ export async function fetchUrl(
   verbose = false,
   forcePlaywright = false
 ): Promise<FetchResult> {
+  try {
+    const parsed = new URL(url);
+    if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
+      return {
+        success: false,
+        error: `Invalid URL protocol: ${parsed.protocol} — only http and https are supported`,
+      };
+    }
+  } catch {
+    return {
+      success: false,
+      error: `Invalid URL: ${url}`,
+    };
+  }
+
   if (forcePlaywright) {
     if (verbose) {
-      console.error(`⚡ Force Playwright mode enabled`);
+      console.error('⚡ Force Playwright mode enabled');
     }
     return tryPlaywright(url, config, 'forced', verbose);
   }
@@ -139,10 +150,15 @@ export async function fetchUrl(
     return tryPlaywright(url, config, 'extraction_failed', verbose);
   }
 
-  const quality = validateMarkdown(extracted.markdown!);
+  const quality = validateMarkdown(extracted.markdown!, { sourceHtmlLength: simpleResult.html.length });
 
   if (verbose) {
     console.error(`📊 Quality score: ${quality.score}/100`);
+    if (quality.issues.length > 0) {
+      for (const issue of quality.issues) {
+        console.error(`   ⚠ ${issue}`);
+      }
+    }
   }
 
   if (quality.score >= config.quality.jsRetryThreshold) {

package/src/core/playwright/index.ts

@@ -1,2 +1,2 @@
-export * from './types';
 export * from './manager';
+export * from './types';

package/src/core/playwright/local.ts

@@ -1,5 +1,5 @@
-import { chromium } from 'playwright-extra';
 import type { Browser } from 'playwright';
+import { chromium } from 'playwright-extra';
 import stealth from 'puppeteer-extra-plugin-stealth';
 import type { PlaywrightConfig } from '../../config/schema';
 import type { BrowserManager } from './types';
@@ -10,28 +10,37 @@ let browserInstance: Browser | null = null;
 
 export class LocalBrowserManager implements BrowserManager {
   private config: PlaywrightConfig;
-  
+
   constructor(config: PlaywrightConfig) {
     this.config = config;
   }
-  
+
   async getBrowser(): Promise<Browser> {
     if (!browserInstance) {
-      browserInstance = await chromium.launch({ 
+      browserInstance = await chromium.launch({
         headless: true,
         timeout: this.config.timeout,
+        args: [
+          '--disable-blink-features=AutomationControlled',
+          '--disable-features=IsolateOrigins,site-per-process',
+          '--disable-infobars',
+          '--no-first-run',
+          '--no-default-browser-check',
+          '--disable-background-networking',
+          '--disable-dev-shm-usage',
+        ],
       });
     }
     return browserInstance;
   }
-  
+
   async closeBrowser(): Promise<void> {
     if (browserInstance) {
       await browserInstance.close();
       browserInstance = null;
     }
   }
-  
+
   isDocker(): boolean {
     return false;
   }

package/src/core/playwright/manager.ts

@@ -1,6 +1,6 @@
 import type { PlaywrightConfig } from '../../config/schema';
-import type { BrowserManager, FetchWithBrowserResult } from './types';
 import { LocalBrowserManager } from './local';
+import type { BrowserManager, FetchWithBrowserResult } from './types';
 
 let currentManager: BrowserManager | null = null;
 
@@ -14,6 +14,23 @@ export async function getBrowserManager(config: PlaywrightConfig): Promise<Brows
   return currentManager;
 }
 
+/** Common desktop viewport sizes to rotate through for fingerprint diversity */
+const VIEWPORTS = [
+  { width: 1920, height: 1080 },
+  { width: 1536, height: 864 },
+  { width: 1440, height: 900 },
+  { width: 1366, height: 768 },
+  { width: 1280, height: 720 },
+];
+
+const TIMEZONES = ['America/New_York', 'America/Chicago', 'America/Denver', 'America/Los_Angeles'];
+
+const LOCALES = ['en-US', 'en-US', 'en-US', 'en-GB'];
+
+function pick<T>(arr: T[]): T {
+  return arr[Math.floor(Math.random() * arr.length)];
+}
+
 export async function fetchWithBrowser(
   url: string,
   config: PlaywrightConfig,
@@ -21,35 +38,90 @@ export async function fetchWithBrowser(
 ): Promise<FetchWithBrowserResult> {
   const manager = await getBrowserManager(config);
   const browser = await manager.getBrowser();
-  
+
+  const viewport = pick(VIEWPORTS);
+  const locale = pick(LOCALES);
+  const timezone = pick(TIMEZONES);
+
+  const context = await browser.newContext({
+    viewport,
+    locale,
+    timezoneId: timezone,
+    userAgent:
+      'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36',
+    // Realistic browser headers
+    extraHTTPHeaders: {
+      'Accept-Language': `${locale},en;q=0.9`,
+      'Accept-Encoding': 'gzip, deflate, br',
+      'Sec-CH-UA': '"Chromium";v="122", "Not(A:Brand";v="24", "Google Chrome";v="122"',
+      'Sec-CH-UA-Mobile': '?0',
+      'Sec-CH-UA-Platform': '"macOS"',
+      'Sec-Fetch-Dest': 'document',
+      'Sec-Fetch-Mode': 'navigate',
+      'Sec-Fetch-Site': 'none',
+      'Sec-Fetch-User': '?1',
+      'Upgrade-Insecure-Requests': '1',
+    },
+    // Pretend we have granted permissions a real user would have
+    permissions: ['geolocation'],
+    deviceScaleFactor: 2,
+    isMobile: false,
+    hasTouch: false,
+    javaScriptEnabled: true,
+  });
+
+  const page = await context.newPage();
+
   try {
-    const context = await browser.newContext({
-      userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
-    });
-    const page = await context.newPage();
-    
+    // Override navigator properties that leak headless signals
+    await page.addInitScript(`
+      Object.defineProperty(navigator, 'webdriver', { get: () => false });
+      Object.defineProperty(navigator, 'plugins', {
+        get: () => [
+          { name: 'Chrome PDF Plugin', filename: 'internal-pdf-viewer' },
+          { name: 'Chrome PDF Viewer', filename: 'mhjfbmdgcfjbbpaeojofohoefgiehjai' },
+          { name: 'Native Client', filename: 'internal-nacl-plugin' },
+        ],
+      });
+      Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] });
+      Object.defineProperty(navigator, 'maxTouchPoints', { get: () => 0 });
+      if (typeof Notification !== 'undefined') {
+        Object.defineProperty(Notification, 'permission', { get: () => 'default' });
+      }
+      window.chrome = window.chrome || {};
+      window.chrome.runtime = window.chrome.runtime || {};
+    `);
+
     if (verbose) {
-      console.error(`🎭 Playwright: Navigating to ${url}`);
+      console.error(
+        `🎭 Playwright: Navigating to ${url} (${viewport.width}x${viewport.height}, ${locale}, ${timezone})`
+      );
     }
-    
+
+    // Small random delay to avoid machine-like timing patterns
+    await page.waitForTimeout(200 + Math.floor(Math.random() * 300));
+
     await page.goto(url, {
       waitUntil: config.waitStrategy,
       timeout: config.timeout,
     });
-    
+
+    // Wait a bit after load for lazy-loaded content / hydration
+    await page.waitForTimeout(500 + Math.floor(Math.random() * 500));
+
     const html = await page.content();
-    
+
     if (verbose) {
       console.error(`🎭 Playwright: Got ${html.length} chars of HTML`);
     }
-    
-    await page.close();
-    await context.close();
 
     return { html };
   } catch (error) {
     const message = error instanceof Error ? error.message : String(error);
     return { html: '', error: message };
+  } finally {
+    await page.close();
+    await context.close();
   }
 }
 

package/src/utils/markdown-cleaner.ts

@@ -9,44 +9,44 @@ export function cleanMarkdown(markdown: string): string {
     return markdown;
   }
 
-  let cleaned = markdown
+  let cleaned = markdown;
 
-  cleaned = cleaned.replace(/\r\n/g, '\n')
-  cleaned = cleaned.replace(/\n{3,}/g, '\n\n')
-  cleaned = cleaned.replace(/[ \t]+$/gm, '')
-  cleaned = cleaned.trim()
-  cleaned = cleaned.replace(/([^\n])\n(#{1,6} )/g, '$1\n\n$2')
-  cleaned = cleaned.replace(/(#{1,6} .+)\n([^#\n])/g, '$1\n\n$2')
-  cleaned = cleaned.replace(/([^\n])\n([-*+] |\d+\. )/g, '$1\n\n$2')
-  cleaned = cleaned.replace(/(\*|_) +/g, '$1')
-  cleaned = cleaned.replace(/ +(\*|_)/g, '$1')
-  cleaned = cleaned.replace(/([^\n])\n```/g, '$1\n\n```')
-  cleaned = cleaned.replace(/```\n([^`])/g, '```\n\n$1')
-  cleaned = cleaned.replace(/<!--[\s\S]*?-->/g, ' ')
-  cleaned = cleaned.replace(/ {2,}/g, ' ')
+  cleaned = cleaned.replace(/\r\n/g, '\n');
+  cleaned = cleaned.replace(/\n{3,}/g, '\n\n');
+  cleaned = cleaned.replace(/[ \t]+$/gm, '');
+  cleaned = cleaned.trim();
+  cleaned = cleaned.replace(/([^\n])\n(#{1,6} )/g, '$1\n\n$2');
+  cleaned = cleaned.replace(/(#{1,6} .+)\n([^#\n])/g, '$1\n\n$2');
+  cleaned = cleaned.replace(/([^\n])\n([-*+] |\d+\. )/g, '$1\n\n$2');
+  cleaned = cleaned.replace(/(\*|_) +/g, '$1');
+  cleaned = cleaned.replace(/ +(\*|_)/g, '$1');
+  cleaned = cleaned.replace(/([^\n])\n```/g, '$1\n\n```');
+  cleaned = cleaned.replace(/```\n([^`])/g, '```\n\n$1');
+  cleaned = cleaned.replace(/<!--[\s\S]*?-->/g, ' ');
+  cleaned = cleaned.replace(/ {2,}/g, ' ');
 
-  return cleaned
+  return cleaned;
 }
 
 export function advancedClean(markdown: string): string {
-  let cleaned = markdown
+  let cleaned = markdown;
 
-  cleaned = cleaned.replace(/\[([^\]]+)\]\(\)/g, '$1')
-  cleaned = cleaned.replace(/<[^>]+>/g, '')
-  cleaned = cleaned.replace(/\*\*\*\*/g, '')
-  cleaned = cleaned.replace(/(?<!\*)\*\*(?!\*)/g, '')
-  cleaned = cleaned.replace(/__/g, '')
-  cleaned = cleaned.replace(/!\[\]\(([^)]+)\)/g, '![]($1)')
-  cleaned = cleaned.replace(/[\u200B-\u200D\uFEFF]/g, '')
-  cleaned = cleaned.replace(/[\u201C\u201D]/g, '"')
-  cleaned = cleaned.replace(/[\u2018\u2019]/g, "'")
-  cleaned = cleaned.replace(/[\u2013\u2014]/g, '-')
+  cleaned = cleaned.replace(/\[([^\]]+)\]\(\)/g, '$1');
+  cleaned = cleaned.replace(/<[^>]+>/g, '');
+  cleaned = cleaned.replace(/\*\*\*\*/g, '');
+  cleaned = cleaned.replace(/(?<!\*)\*\*(?!\*)/g, '');
+  cleaned = cleaned.replace(/__/g, '');
+  cleaned = cleaned.replace(/!\[\]\(([^)]+)\)/g, '![]($1)');
+  cleaned = cleaned.replace(/[\u200B-\u200D\uFEFF]/g, '');
+  cleaned = cleaned.replace(/[\u201C\u201D]/g, '"');
+  cleaned = cleaned.replace(/[\u2018\u2019]/g, "'");
+  cleaned = cleaned.replace(/[\u2013\u2014]/g, '-');
 
   cleaned = cleaned.replace(/^(?!```)[^\n]*$/gm, (line) => {
-    return line.replace(/ {2,}/g, ' ')
-  })
+    return line.replace(/ {2,}/g, ' ');
+  });
 
-  return cleaned
+  return cleaned;
 }
 
 export function finalCleanup(markdown: string): string {
@@ -54,16 +54,16 @@ export function finalCleanup(markdown: string): string {
     return markdown;
   }
 
-  let cleaned = markdown
+  let cleaned = markdown;
 
-  cleaned = cleaned.replace(/^(\s*)[*+] /gm, '$1- ')
-  cleaned = cleaned.replace(/_([^_]+)_/g, '*$1*')
-  cleaned = cleaned.replace(/^~~~(\w*)\n/gm, '```$1\n')
-  cleaned = cleaned.replace(/^~~~$/gm, '```')
-  cleaned = cleaned.replace(/\n{3,}/g, '\n\n')
-  cleaned = `${cleaned.trim()}\n`
+  cleaned = cleaned.replace(/^(\s*)[*+] /gm, '$1- ');
+  cleaned = cleaned.replace(/_([^_]+)_/g, '*$1*');
+  cleaned = cleaned.replace(/^~~~(\w*)\n/gm, '```$1\n');
+  cleaned = cleaned.replace(/^~~~$/gm, '```');
+  cleaned = cleaned.replace(/\n{3,}/g, '\n\n');
+  cleaned = `${cleaned.trim()}\n`;
 
-  return cleaned
+  return cleaned;
 }
 
 export function cleanMarkdownComplete(markdown: string): string {
@@ -71,9 +71,9 @@ export function cleanMarkdownComplete(markdown: string): string {
     return markdown;
   }
 
-  let cleaned = cleanMarkdown(markdown)
-  cleaned = advancedClean(cleaned)
-  cleaned = finalCleanup(cleaned)
+  let cleaned = cleanMarkdown(markdown);
+  cleaned = advancedClean(cleaned);
+  cleaned = finalCleanup(cleaned);
 
-  return cleaned
+  return cleaned;
 }