@yusufffararatt/dombridge-mcp 2.7.5

package/src/profiles/profile-manager.js

@@ -0,0 +1,563 @@
+/**
+ * Site Profile Manager
+ * Stores domain-scoped scraper profiles under mcp-server/profiles/.
+ *
+ * Profiles are both:
+ * - a machine-usable cache for future tool runs
+ * - a human-readable dossier for scraper maintenance
+ */
+
+import { readFileSync, writeFileSync, mkdirSync, readdirSync, existsSync } from 'fs';
+import { join, dirname } from 'path';
+import { fileURLToPath } from 'url';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const PROFILES_DIR = join(__dirname, '../../profiles');
+
+const MAX_ENDPOINTS = 100;
+const MAX_AUTO_NOTES = 50;
+const MAX_KNOWN_PATHS = 250;
+const MAX_SAMPLE_PATHS = 20;
+const PROFILE_VERSION = 2;
+const MAX_DISCOVER_APIS_ENDPOINTS = 10;
+const MAX_EXTRACT_DATA_KNOWN_PATHS = 10;
+
+export const PROFILE_FIELD_ALIASES = {
+    framework: ['framework'],
+    pageCharacteristics: ['pageCharacteristics'],
+    authInfo: ['authInfo'],
+    apiEndpoints: ['apiEndpoints'],
+    knownPaths: ['knownPaths'],
+    dataSchema: ['dataSchema'],
+    stableSelectors: ['stableSelectors'],
+    paginationPatterns: ['paginationPatterns'],
+    cspStatus: ['cspStatus'],
+    rateLimit: ['rateLimit'],
+    botProtection: ['botProtection'],
+    responseHeaderPatterns: ['responseHeaderPatterns'],
+    autoNotes: ['autoNotes']
+};
+
+function ensureProfilesDir() {
+    if (!existsSync(PROFILES_DIR)) {
+        mkdirSync(PROFILES_DIR, { recursive: true });
+    }
+}
+
+function domainToFilename(domain) {
+    return domain.replace(/[^a-zA-Z0-9.-]/g, '_') + '.json';
+}
+
+function profilePath(domain) {
+    return join(PROFILES_DIR, domainToFilename(domain));
+}
+
+function isoNow() {
+    return new Date().toISOString();
+}
+
+function asArray(value) {
+    return Array.isArray(value) ? value : [];
+}
+
+function asObject(value) {
+    return value && typeof value === 'object' && !Array.isArray(value) ? value : {};
+}
+
+function stripUndefined(obj) {
+    return Object.fromEntries(Object.entries(obj).filter(([, value]) => value !== undefined));
+}
+
+export function extractDomain(urlOrDomain) {
+    try {
+        if (urlOrDomain.startsWith('http')) {
+            return new URL(urlOrDomain).hostname;
+        }
+        return urlOrDomain.toLowerCase().trim();
+    } catch {
+        return urlOrDomain.toLowerCase().trim();
+    }
+}
+
+export function normalizeEndpointKey(url, method) {
+    try {
+        const u = new URL(url);
+        return `${(method || 'GET').toUpperCase()}:${u.hostname}${u.pathname}`;
+    } catch {
+        const path = String(url || '').split('?')[0].split('#')[0];
+        return `${(method || 'GET').toUpperCase()}:${path}`;
+    }
+}
+
+function normalizeEndpoint(endpoint, now, fallback = {}) {
+    if (!endpoint) return null;
+    const rawUrl = typeof endpoint === 'string' ? endpoint : endpoint.url;
+    if (!rawUrl) return null;
+
+    const method = (endpoint.method || fallback.method || 'GET').toUpperCase();
+    const firstSeenAt = endpoint.firstSeenAt || fallback.firstSeenAt || now;
+    const lastSeenAt = endpoint.lastSeenAt || fallback.lastSeenAt || now;
+
+    return stripUndefined({
+        url: rawUrl,
+        method,
+        status: endpoint.status ?? fallback.status ?? null,
+        contentType: endpoint.contentType ?? fallback.contentType ?? null,
+        confidence: endpoint.confidence ?? fallback.confidence ?? null,
+        dataPath: endpoint.dataPath ?? fallback.dataPath ?? null,
+        sourceKey: endpoint.sourceKey ?? fallback.sourceKey ?? null,
+        operationName: endpoint.operationName ?? fallback.operationName ?? null,
+        operationType: endpoint.operationType ?? fallback.operationType ?? null,
+        originTool: endpoint.originTool ?? fallback.originTool ?? null,
+        originDescription: endpoint.originDescription ?? fallback.originDescription ?? null,
+        firstSeenAt,
+        lastSeenAt
+    });
+}
+
+export function mergeEndpoints(existing, incoming, now = isoNow()) {
+    const merged = [];
+    const seen = new Map();
+
+    for (const raw of asArray(existing)) {
+        const normalized = normalizeEndpoint(raw, now);
+        if (!normalized) continue;
+        const key = normalizeEndpointKey(normalized.url, normalized.method);
+        seen.set(key, merged.length);
+        merged.push(normalized);
+    }
+
+    for (const raw of asArray(incoming)) {
+        const key = normalizeEndpointKey(raw?.url || raw, raw?.method);
+        const idx = seen.get(key);
+        const base = idx !== undefined ? merged[idx] : {};
+        const normalized = normalizeEndpoint(raw, now, base);
+        if (!normalized) continue;
+
+        if (idx !== undefined) {
+            merged[idx] = {
+                ...base,
+                ...normalized,
+                firstSeenAt: base.firstSeenAt || normalized.firstSeenAt || now,
+                lastSeenAt: normalized.lastSeenAt || now,
+                confidence: normalized.confidence ?? base.confidence ?? null
+            };
+        } else {
+            seen.set(key, merged.length);
+            merged.push(normalized);
+        }
+    }
+
+    const sorted = merged
+        .sort((a, b) => {
+            const aTime = new Date(a.lastSeenAt || a.firstSeenAt || 0).getTime();
+            const bTime = new Date(b.lastSeenAt || b.firstSeenAt || 0).getTime();
+            return bTime - aTime;
+        })
+        .slice(0, MAX_ENDPOINTS);
+
+    const discoverApis = sorted
+        .filter((entry) => entry.originTool === 'discover_apis')
+        .slice(0, MAX_DISCOVER_APIS_ENDPOINTS);
+    const others = sorted.filter((entry) => entry.originTool !== 'discover_apis');
+
+    return [...others, ...discoverApis]
+        .sort((a, b) => {
+            const aTime = new Date(a.lastSeenAt || a.firstSeenAt || 0).getTime();
+            const bTime = new Date(b.lastSeenAt || b.firstSeenAt || 0).getTime();
+            return bTime - aTime;
+        })
+        .slice(0, MAX_ENDPOINTS);
+}
+
+function normalizeKnownPath(pathEntry, now, fallback = {}) {
+    if (!pathEntry) return null;
+    const rawPath = typeof pathEntry === 'string' ? pathEntry : pathEntry.path;
+    if (!rawPath) return null;
+
+    const sourceKey = pathEntry.sourceKey ?? fallback.sourceKey ?? null;
+    const originTool = pathEntry.originTool ?? fallback.originTool ?? null;
+    const firstSeenAt = pathEntry.firstSeenAt || fallback.firstSeenAt || now;
+    const lastSeenAt = pathEntry.lastSeenAt || fallback.lastSeenAt || now;
+
+    return stripUndefined({
+        path: rawPath,
+        type: pathEntry.type ?? fallback.type ?? 'unknown',
+        example: pathEntry.example ?? fallback.example ?? null,
+        sourceKey,
+        originTool,
+        confidence: pathEntry.confidence ?? fallback.confidence ?? null,
+        firstSeenAt,
+        lastSeenAt
+    });
+}
+
+function knownPathKey(pathEntry) {
+    return `${pathEntry.path}::${pathEntry.sourceKey || ''}`;
+}
+
+function mergeStableSelectors(existing, incoming) {
+    const seen = new Map();
+    const merged = [];
+
+    for (const s of asArray(existing)) {
+        if (!s || !s.selector) continue;
+        seen.set(s.selector, merged.length);
+        merged.push(s);
+    }
+
+    for (const s of asArray(incoming)) {
+        if (!s || !s.selector) continue;
+        const idx = seen.get(s.selector);
+        if (idx !== undefined) {
+            // Gelen entry daha güncel — güncelle
+            merged[idx] = { ...merged[idx], ...s };
+        } else {
+            seen.set(s.selector, merged.length);
+            merged.push(s);
+        }
+    }
+
+    return merged;
+}
+
+function mergeKnownPaths(existing, incoming, now = isoNow()) {
+    const merged = [];
+    const seen = new Map();
+
+    for (const raw of asArray(existing)) {
+        const normalized = normalizeKnownPath(raw, now);
+        if (!normalized) continue;
+        const key = knownPathKey(normalized);
+        seen.set(key, merged.length);
+        merged.push(normalized);
+    }
+
+    for (const raw of asArray(incoming)) {
+        const normalized = normalizeKnownPath(raw, now);
+        if (!normalized) continue;
+        const key = knownPathKey(normalized);
+        const idx = seen.get(key);
+        if (idx !== undefined) {
+            const base = merged[idx];
+            merged[idx] = {
+                ...base,
+                ...normalized,
+                firstSeenAt: base.firstSeenAt || normalized.firstSeenAt || now,
+                lastSeenAt: normalized.lastSeenAt || now
+            };
+        } else {
+            seen.set(key, merged.length);
+            merged.push(normalized);
+        }
+    }
+
+    const sorted = merged
+        .sort((a, b) => {
+            const scoreA = Number(Boolean(a.confidence)) * 100 + (a.path?.length || 0);
+            const scoreB = Number(Boolean(b.confidence)) * 100 + (b.path?.length || 0);
+            return scoreB - scoreA;
+        })
+        .slice(0, MAX_KNOWN_PATHS);
+
+    const extractDataPaths = sorted
+        .filter((entry) => entry.originTool === 'extract_data')
+        .slice(0, MAX_EXTRACT_DATA_KNOWN_PATHS);
+    const others = sorted.filter((entry) => entry.originTool !== 'extract_data');
+
+    return [...others, ...extractDataPaths]
+        .sort((a, b) => {
+            const scoreA = Number(Boolean(a.confidence)) * 100 + (a.path?.length || 0);
+            const scoreB = Number(Boolean(b.confidence)) * 100 + (b.path?.length || 0);
+            return scoreB - scoreA;
+        })
+        .slice(0, MAX_KNOWN_PATHS);
+}
+
+function normalizeDataSource(source, now, fallback = {}) {
+    if (!source?.key) return null;
+
+    const samplePaths = asArray(source.samplePaths || source.leafPaths || fallback.samplePaths)
+        .map((entry) => {
+            if (!entry) return null;
+            if (typeof entry === 'string') return { path: entry };
+            if (!entry.path) return null;
+            return stripUndefined({
+                path: entry.path,
+                type: entry.type ?? null,
+                example: entry.example ?? null,
+                confidence: entry.confidence ?? null
+            });
+        })
+        .filter(Boolean)
+        .slice(0, MAX_SAMPLE_PATHS);
+
+    const topLevelKeys = asArray(source.topLevelKeys || fallback.topLevelKeys).slice(0, 30);
+
+    return stripUndefined({
+        key: source.key,
+        type: source.type ?? fallback.type ?? 'unknown',
+        isArray: source.isArray ?? fallback.isArray ?? false,
+        arrayLength: source.arrayLength ?? fallback.arrayLength ?? null,
+        topLevelKeyCount: source.topLevelKeyCount ?? fallback.topLevelKeyCount ?? null,
+        topLevelKeys,
+        leafPathCount: source.leafPathCount ?? source.leafPaths?.length ?? fallback.leafPathCount ?? null,
+        samplePaths,
+        capturedAt: source.capturedAt ?? fallback.capturedAt ?? now
+    });
+}
+
+function mergeDataSchema(existing, incoming, now = isoNow()) {
+    const existingSources = asArray(existing?.sources);
+    const incomingSources = asArray(incoming?.sources);
+    if (existingSources.length === 0 && incomingSources.length === 0) return null;
+
+    const merged = [];
+    const seen = new Map();
+
+    for (const raw of existingSources) {
+        const normalized = normalizeDataSource(raw, now);
+        if (!normalized) continue;
+        seen.set(normalized.key, merged.length);
+        merged.push(normalized);
+    }
+
+    for (const raw of incomingSources) {
+        const idx = seen.get(raw?.key);
+        const base = idx !== undefined ? merged[idx] : {};
+        const normalized = normalizeDataSource(raw, now, base);
+        if (!normalized) continue;
+
+        if (idx !== undefined) {
+            merged[idx] = {
+                ...base,
+                ...normalized,
+                capturedAt: normalized.capturedAt || base.capturedAt || now
+            };
+        } else {
+            seen.set(normalized.key, merged.length);
+            merged.push(normalized);
+        }
+    }
+
+    return {
+        sources: merged.sort((a, b) => (b.leafPathCount || 0) - (a.leafPathCount || 0))
+    };
+}
+
+function mergeAutoNotes(existing, incoming) {
+    const merged = [...asArray(existing)];
+    const keyFor = (note) => `${note.source || note.tool || 'unknown'}::${note.kind || 'summary'}`;
+    const seen = new Map(merged.map((note, index) => [keyFor(note), index]));
+
+    for (const note of asArray(incoming)) {
+        const normalized = {
+            source: note.source || note.tool || 'unknown',
+            kind: note.kind || 'summary',
+            timestamp: note.timestamp || isoNow(),
+            text: note.text || ''
+        };
+        const key = keyFor(normalized);
+        if (seen.has(key)) {
+            merged[seen.get(key)] = normalized;
+        } else {
+            seen.set(key, merged.length);
+            merged.push(normalized);
+        }
+    }
+
+    return merged.slice(-MAX_AUTO_NOTES);
+}
+
+function mergeFieldTimestamps(existing, profileData, now) {
+    const merged = {
+        ...asObject(existing)
+    };
+
+    if (profileData.fieldTimestamps) {
+        Object.assign(merged, profileData.fieldTimestamps);
+    }
+
+    for (const [field, aliases] of Object.entries(PROFILE_FIELD_ALIASES)) {
+        if (aliases.some((alias) => profileData[alias] !== undefined)) {
+            merged[field] = now;
+        }
+    }
+
+    return merged;
+}
+
+function mergeProfileMeta(existing, profileData, now) {
+    const previous = asObject(existing);
+    const current = asObject(profileData.profileMeta);
+
+    return {
+        createdAt: previous.createdAt || current.createdAt || now,
+        lastUpdated: now,
+        lastAnalyzedAt: current.lastAnalyzedAt ?? previous.lastAnalyzedAt ?? null,
+        lastDiscoveryAt: current.lastDiscoveryAt ?? previous.lastDiscoveryAt ?? null,
+        lastDriftCheckAt: current.lastDriftCheckAt ?? previous.lastDriftCheckAt ?? null
+    };
+}
+
+export function getProfileFieldTimestamp(profile, field) {
+    if (!profile) return null;
+    return profile.fieldTimestamps?.[field] || null;
+}
+
+export function isProfileFieldFresh(profile, field, maxAgeMs) {
+    const ts = getProfileFieldTimestamp(profile, field);
+    if (!ts) return false;
+    return Date.now() - new Date(ts).getTime() < maxAgeMs;
+}
+
+function normalizeProfile(raw) {
+    if (!raw || typeof raw !== 'object') return null;
+
+    const now = raw.lastUpdated || raw.createdAt || isoNow();
+    const version = raw.version || 1;
+    const createdAt = raw.createdAt || now;
+    const lastUpdated = raw.lastUpdated || now;
+
+    const normalized = {
+        domain: raw.domain,
+        createdAt,
+        lastUpdated,
+        version,
+        notes: typeof raw.notes === 'string' ? raw.notes : '',
+        framework: asArray(raw.framework),
+        pageCharacteristics: raw.pageCharacteristics ?? null,
+        authInfo: raw.authInfo ?? null,
+        paginationPatterns: raw.paginationPatterns ?? null,
+        cspStatus: raw.cspStatus ?? null,
+        rateLimit: raw.rateLimit ?? null,
+        botProtection: asArray(raw.botProtection),
+        responseHeaderPatterns: asObject(raw.responseHeaderPatterns),
+        stableSelectors: asArray(raw.stableSelectors),
+        apiEndpoints: mergeEndpoints([], raw.apiEndpoints, lastUpdated),
+        knownPaths: mergeKnownPaths([], raw.knownPaths, lastUpdated),
+        dataSchema: mergeDataSchema(null, raw.dataSchema, lastUpdated),
+        autoNotes: mergeAutoNotes([], raw.autoNotes),
+        fieldTimestamps: mergeFieldTimestamps(raw.fieldTimestamps, {}, lastUpdated),
+        profileMeta: mergeProfileMeta(raw.profileMeta, { profileMeta: raw.profileMeta }, lastUpdated)
+    };
+
+    normalized.profileMeta.createdAt = createdAt;
+    normalized.profileMeta.lastUpdated = lastUpdated;
+
+    if (!normalized.fieldTimestamps.framework && normalized.framework.length > 0) {
+        normalized.fieldTimestamps.framework = lastUpdated;
+    }
+    if (!normalized.fieldTimestamps.apiEndpoints && normalized.apiEndpoints.length > 0) {
+        normalized.fieldTimestamps.apiEndpoints = lastUpdated;
+    }
+    if (!normalized.fieldTimestamps.knownPaths && normalized.knownPaths.length > 0) {
+        normalized.fieldTimestamps.knownPaths = lastUpdated;
+    }
+    if (!normalized.fieldTimestamps.dataSchema && normalized.dataSchema?.sources?.length) {
+        normalized.fieldTimestamps.dataSchema = lastUpdated;
+    }
+
+    return normalized;
+}
+
+export function saveProfile(domain, profileData) {
+    ensureProfilesDir();
+
+    const existing = loadProfile(domain) || {};
+    const now = isoNow();
+
+    const profile = {
+        domain,
+        createdAt: existing.createdAt || now,
+        lastUpdated: now,
+        version: Math.max(existing.version || 0, PROFILE_VERSION - 1) + 1,
+        notes: profileData.notes !== undefined && profileData.notes !== null
+            ? profileData.notes
+            : (existing.notes ?? ''),
+        framework: profileData.framework ?? existing.framework ?? [],
+        pageCharacteristics: profileData.pageCharacteristics ?? existing.pageCharacteristics ?? null,
+        authInfo: profileData.authInfo ?? existing.authInfo ?? null,
+        paginationPatterns: profileData.paginationPatterns ?? existing.paginationPatterns ?? null,
+        cspStatus: profileData.cspStatus ?? existing.cspStatus ?? null,
+        rateLimit: profileData.rateLimit ?? existing.rateLimit ?? null,
+        botProtection: profileData.botProtection ?? existing.botProtection ?? [],
+        responseHeaderPatterns: profileData.responseHeaderPatterns ?? existing.responseHeaderPatterns ?? {},
+        stableSelectors: profileData.stableSelectors !== undefined
+            ? mergeStableSelectors(existing.stableSelectors, profileData.stableSelectors)
+            : (existing.stableSelectors ?? []),
+        apiEndpoints: profileData.apiEndpoints !== undefined
+            ? mergeEndpoints(existing.apiEndpoints, profileData.apiEndpoints, now)
+            : (existing.apiEndpoints ?? []),
+        knownPaths: profileData.knownPaths !== undefined
+            ? mergeKnownPaths(existing.knownPaths, profileData.knownPaths, now)
+            : (existing.knownPaths ?? []),
+        dataSchema: profileData.dataSchema !== undefined
+            ? mergeDataSchema(existing.dataSchema, profileData.dataSchema, now)
+            : (existing.dataSchema ?? null),
+        autoNotes: mergeAutoNotes(existing.autoNotes, profileData.autoNotes),
+        fieldTimestamps: mergeFieldTimestamps(existing.fieldTimestamps, profileData, now),
+        profileMeta: mergeProfileMeta(existing.profileMeta, profileData, now)
+    };
+
+    profile.profileMeta.createdAt = existing.profileMeta?.createdAt || existing.createdAt || now;
+    profile.profileMeta.lastUpdated = now;
+    profile.fieldTimestamps = {
+        ...profile.fieldTimestamps,
+        notes: profileData.notes !== undefined && profileData.notes !== null
+            ? now
+            : profile.fieldTimestamps.notes
+    };
+
+    writeFileSync(profilePath(domain), JSON.stringify(profile, null, 2), 'utf-8');
+    return profile;
+}
+
+export function loadProfile(domain) {
+    const p = profilePath(domain);
+    if (!existsSync(p)) return null;
+
+    try {
+        return normalizeProfile(JSON.parse(readFileSync(p, 'utf-8')));
+    } catch {
+        return null;
+    }
+}
+
+/**
+ * Get SSR-related window.__*__PROPS paths from saved profile.
+ * Used by get_network_trace as fallback when no API match found.
+ */
+export function getSavedSsrPaths(domain) {
+    const profile = loadProfile(domain);
+    if (!profile?.dataSchema?.sources) return [];
+    return profile.dataSchema.sources.filter(s => s.key && s.key.startsWith('__'));
+}
+
+export function listProfiles() {
+    ensureProfilesDir();
+
+    try {
+        return readdirSync(PROFILES_DIR)
+            .filter((file) => file.endsWith('.json'))
+            .map((file) => {
+                try {
+                    const raw = normalizeProfile(JSON.parse(readFileSync(join(PROFILES_DIR, file), 'utf-8')));
+                    return {
+                        domain: raw.domain,
+                        lastUpdated: raw.lastUpdated,
+                        version: raw.version,
+                        knownPathCount: asArray(raw.knownPaths).length,
+                        apiEndpointCount: asArray(raw.apiEndpoints).length,
+                        dataSourceCount: asArray(raw.dataSchema?.sources).length,
+                        hasNotes: Boolean(raw.notes && raw.notes.trim())
+                    };
+                } catch {
+                    return null;
+                }
+            })
+            .filter(Boolean);
+    } catch {
+        return [];
+    }
+}