@vespermcp/mcp-server 1.0.0

package/build/metadata/scraper.js

@@ -0,0 +1,353 @@
+import { listDatasets, datasetInfo } from "@huggingface/hub";
+import { categorizeLicense } from "./license.js";
+import { calculateQualityScore } from "./quality.js";
+import { classifyDomain } from "./domain.js";
+import { retryWithBackoff, delayBetweenRequests } from "./rate-limiter.js";
+export class HuggingFaceScraper {
+    /**
+     * Bulk discovery: Fetch many datasets quickly without deep details.
+     * Hits the 25k target in minutes.
+     */
+    async scrapeBulk(limit = 1000, domainFilter) {
+        const filterMsg = domainFilter ? `, domain: ${domainFilter}` : "";
+        console.error(`[Bulk Scraper] Fetching datasets (target limit: ${limit}${filterMsg})...`);
+        const results = [];
+        let processed = 0;
+        try {
+            const hfToken = process.env.HF_TOKEN || process.env.HUGGINGFACE_TOKEN;
+            for await (const ds of listDatasets({
+                limit: limit,
+                additionalFields: ["description", "tags", "downloadsAllTime", "createdAt"],
+                search: { query: domainFilter },
+                ...(hfToken ? { accessToken: hfToken } : {})
+            })) {
+                if (results.length >= limit)
+                    break;
+                processed++;
+                if (processed % 1000 === 0) {
+                    console.error(`[Bulk Scraper] Found ${processed} datasets...`);
+                }
+                const repoId = ds.name || ds.id;
+                const description = ds.description || "";
+                const tags = ds.tags || [];
+                // Filter out non-repo IDs (hex IDs) if they don't have a namespace
+                if (!repoId.includes("/") && /^[a-f0-9]{24}$/.test(repoId))
+                    continue;
+                // Hard skip: Empty or very short description (cannot search semantically)
+                if (!description || description.length < 20)
+                    continue;
+                const createdAt = ds.createdAt;
+                const downloads = ds.downloadsAllTime || ds.downloads || 0;
+                const task = this.extractTask(tags);
+                // Create "skeleton" metadata
+                const metadata = {
+                    id: repoId,
+                    source: "huggingface",
+                    name: repoId.split("/").pop() || repoId,
+                    description: description,
+                    quality_warnings: ["Incomplete metadata: run hydration to get full details"],
+                    downloads: downloads,
+                    likes: ds.likes || 0,
+                    stars: 0,
+                    tags: tags,
+                    last_updated: createdAt instanceof Date ? createdAt.toISOString() : new Date().toISOString(),
+                    task: task,
+                    domain: classifyDomain(description, tags, repoId, task),
+                    languages: this.extractLanguages(tags),
+                    splits: [],
+                    license: {
+                        id: "unknown",
+                        category: "unknown",
+                        usage_restrictions: [],
+                        warnings: ["License not verified yet"]
+                    },
+                    quality_score: 10, // Default low score for skeleton
+                    download_url: `https://huggingface.co/datasets/${repoId}`,
+                    total_examples: 0,
+                    is_structured: false,
+                    has_target_column: false,
+                    is_safe_source: true, // Default to true, will be verified during hydration
+                    has_personal_data: false,
+                    is_paywalled: false,
+                    is_scraped_web_data: false,
+                    uses_https: true,
+                    has_train_split: false,
+                    has_test_split: false,
+                    has_validation_split: false,
+                    description_length: description.length,
+                    has_readme: false,
+                    is_incomplete: true // Flag for Phase 2
+                };
+                results.push(metadata);
+            }
+        }
+        catch (e) {
+            console.error("[Bulk Scraper] Error:", e.message);
+        }
+        return results;
+    }
+    async scrape(limit = 100, applyMVPFilters = true, domainFilter // Optional: filter by domain (medicine, healthcare, security, etc.)
+    ) {
+        const filterMsg = domainFilter ? `, domain: ${domainFilter}` : "";
+        console.error(`Fetching datasets (target limit: ${limit}, MVP filters: ${applyMVPFilters}${filterMsg})...`);
+        const results = [];
+        let processed = 0;
+        let skippedMVP = 0;
+        try {
+            // Fetch more datasets to account for filtering
+            const fetchLimit = applyMVPFilters ? limit * 30 : limit * 10;
+            const CONCURRENCY = 25; // Increased for high-volume indexing
+            const queue = [];
+            // Support HuggingFace token from environment variable
+            const hfToken = process.env.HF_TOKEN || process.env.HUGGINGFACE_TOKEN;
+            // Add delay between batches to avoid rate limits
+            const BATCH_DELAY = 1000; // 1 second delay between batches
+            for await (const ds of listDatasets({
+                limit: fetchLimit,
+                additionalFields: ["description", "tags"],
+                search: { query: domainFilter },
+                ...(hfToken ? { accessToken: hfToken } : {})
+            })) {
+                if (results.length >= limit)
+                    break;
+                processed++;
+                // Add delay every 100 datasets to avoid aggressive rate limits
+                if (processed % 100 === 0 && processed > 0) {
+                    await delayBetweenRequests(hfToken ? 500 : 2000);
+                }
+                const repoId = ds.name || ds.id;
+                // Filter out non-repo IDs (hex IDs) if they don't have a namespace
+                if (!repoId.includes("/")) {
+                    if (/^[a-f0-9]{24}$/.test(repoId))
+                        continue;
+                }
+                const downloads = ds.downloads || 0;
+                const tags = ds.tags || [];
+                const description = ds.description || "";
+                if (processed % 100 === 0) {
+                    console.error(`[HF Scraper] Processed ${processed}, found ${results.length}, workers: ${queue.length}...`);
+                }
+                // Hard skip: Empty or very short description (cannot search semantically)
+                if (!description || description.length < 20)
+                    continue;
+                // Process dataset with concurrency and retry logic
+                const processTask = (async () => {
+                    try {
+                        const fullInfo = await retryWithBackoff(() => datasetInfo({
+                            name: repoId,
+                            additionalFields: ["cardData"],
+                            ...(hfToken ? { accessToken: hfToken } : {})
+                        }), {
+                            maxRetries: 3,
+                            initialDelay: 2000, // Start with 2 seconds for HF API
+                            maxDelay: 30000 // Max 30 seconds
+                        });
+                        const splits = fullInfo.splits?.map((s) => ({
+                            name: s.name,
+                            num_examples: s.numExamples || 0,
+                            size_bytes: s.sizeBytes
+                        })) || [];
+                        const totalExamples = splits.reduce((sum, s) => sum + (s.num_examples || 0), 0);
+                        const totalSizeBytes = splits.reduce((sum, s) => sum + (s.size_bytes || 0), 0);
+                        const totalSizeMB = totalSizeBytes ? Math.round(totalSizeBytes / (1024 * 1024) * 100) / 100 : undefined;
+                        const hasValidationSplit = splits.some((s) => s.name === "validation" || s.name === "val");
+                        const licenseTag = tags.find(t => t.startsWith("license:"));
+                        const licenseId = licenseTag ? licenseTag.replace("license:", "") : fullInfo.license;
+                        const cardData = fullInfo.cardData || {};
+                        const licenseUrl = cardData.license?.[0]?.link || cardData.license_link;
+                        const license = categorizeLicense(licenseId, licenseUrl);
+                        if (license.category === "restricted") {
+                            skippedMVP++;
+                            return;
+                        }
+                        const warnings = [];
+                        if (totalExamples < 50)
+                            warnings.push("Dataset has very few examples (< 50)");
+                        if (description.length < 100)
+                            warnings.push("Short description; results may be less relevant");
+                        const lastUpdated = ds.updatedAt || fullInfo.updatedAt;
+                        if (lastUpdated) {
+                            const updateDate = new Date(lastUpdated);
+                            const fourYearsAgo = new Date();
+                            fourYearsAgo.setFullYear(fourYearsAgo.getFullYear() - 4);
+                            if (updateDate < fourYearsAgo) {
+                                warnings.push(`Stale data: Last updated ${updateDate.getFullYear()}`);
+                            }
+                        }
+                        if (splits.length === 0)
+                            warnings.push("No data splits found; could be a non-standard format");
+                        const descriptionLower = description.toLowerCase();
+                        const tagsLower = tags.map(t => t.toLowerCase()).join(" ");
+                        const hasPersonalData = descriptionLower.includes("personal data") ||
+                            descriptionLower.includes("gdpr") ||
+                            tagsLower.includes("personal-data");
+                        const isPaywalled = descriptionLower.includes("paywall") ||
+                            descriptionLower.includes("paid");
+                        const isScrapedWebData = descriptionLower.includes("scraped") ||
+                            tagsLower.includes("scraped");
+                        const isSafeSource = !isScrapedWebData && !hasPersonalData && !isPaywalled;
+                        if (!isSafeSource)
+                            warnings.push("Contains potentially sensitive or paywalled data sources");
+                        const format = this.extractFormat(tags, cardData);
+                        const columns = this.extractColumns(cardData, splits);
+                        const task = this.extractTask(tags);
+                        const domain = classifyDomain(description, tags, repoId, task);
+                        if (domainFilter && domain !== domainFilter)
+                            return;
+                        const metadata = {
+                            id: repoId,
+                            source: "huggingface",
+                            name: repoId.split("/").pop() || repoId,
+                            description: description,
+                            quality_warnings: warnings,
+                            downloads: downloads,
+                            likes: ds.likes || 0,
+                            stars: fullInfo.stars || 0,
+                            tags: tags,
+                            last_updated: lastUpdated?.toISOString() || new Date().toISOString(),
+                            task: task,
+                            domain: domain,
+                            languages: this.extractLanguages(tags),
+                            splits,
+                            license,
+                            quality_score: calculateQualityScore({
+                                downloads,
+                                likes: ds.likes || 0,
+                                hasDescription: true,
+                                descriptionLength: description.length,
+                                hasTrainSplit: splits.some((s) => s.name === "train"),
+                                hasTestSplit: splits.some((s) => s.name === "test"),
+                                lastUpdated: lastUpdated?.toISOString() || new Date().toISOString(),
+                                licenseCategory: license.category
+                            }),
+                            download_url: `https://huggingface.co/datasets/${repoId}`,
+                            format,
+                            total_examples: totalExamples,
+                            total_size_bytes: totalSizeBytes,
+                            total_size_mb: totalSizeMB,
+                            columns,
+                            is_structured: columns.length > 0 && splits.some((s) => s.name === "train"),
+                            has_target_column: columns.some(c => c.is_target === true),
+                            is_safe_source: isSafeSource,
+                            has_personal_data: hasPersonalData,
+                            is_paywalled: isPaywalled,
+                            is_scraped_web_data: isScrapedWebData,
+                            uses_https: true,
+                            has_train_split: splits.some((s) => s.name === "train"),
+                            has_test_split: splits.some((s) => s.name === "test"),
+                            has_validation_split: hasValidationSplit,
+                            description_length: description.length,
+                            has_readme: !!(cardData.readme || cardData.readme_content)
+                        };
+                        results.push(metadata);
+                    }
+                    catch (e) {
+                        // Log rate limit errors, silently skip others
+                        if (e?.status === 429 || e?.message?.includes('rate limit')) {
+                            console.error(`[HF Scraper] Rate limit error for ${repoId}: ${e.message}`);
+                        }
+                        // Silently skip other errors
+                    }
+                })();
+                queue.push(processTask);
+                processTask.then(() => {
+                    const idx = queue.indexOf(processTask);
+                    if (idx !== -1)
+                        queue.splice(idx, 1);
+                });
+                if (queue.length >= CONCURRENCY) {
+                    await Promise.race(queue);
+                }
+            }
+            // Wait for remaining tasks
+            await Promise.all(queue);
+        }
+        catch (e) {
+            // Handle rate limit errors with better messaging
+            if (e?.status === 429 || e?.message?.includes('rate limit')) {
+                console.error("Scraping failed due to rate limit:", e.message);
+                console.error("Consider setting HF_TOKEN environment variable to increase rate limits");
+            }
+            else {
+                console.error("Scraping failed overall:", e.message);
+            }
+        }
+        console.error(`[HF Scraper] Complete: ${results.length} datasets scraped, ${skippedMVP} skipped`);
+        // Sort by downloads descending
+        return results.sort((a, b) => b.downloads - a.downloads);
+    }
+    extractTask(tags) {
+        const taskTags = [
+            "text-classification",
+            "token-classification",
+            "question-answering",
+            "translation",
+            "summarization",
+            "text-generation",
+            "image-classification",
+            "object-detection",
+            "named-entity-recognition",
+            "sentiment-analysis",
+            "machine-translation"
+        ];
+        return tags.find(t => taskTags.includes(t)) || "unknown";
+    }
+    extractLanguages(tags) {
+        const langs = tags
+            .filter(t => t.startsWith("language:"))
+            .map(t => t.replace("language:", ""));
+        const isoLangs = tags.filter(t => t.length === 2 && /^[a-z]{2}$/.test(t));
+        return Array.from(new Set([...langs, ...isoLangs]));
+    }
+    extractFormat(tags, cardData) {
+        // Check tags first
+        const formatTag = tags.find(t => t.startsWith("format:"));
+        if (formatTag) {
+            return formatTag.replace("format:", "").toUpperCase();
+        }
+        // Check cardData
+        if (cardData.format) {
+            return String(cardData.format).toUpperCase();
+        }
+        // Infer from other tags
+        if (tags.includes("parquet"))
+            return "PARQUET";
+        if (tags.includes("csv"))
+            return "CSV";
+        if (tags.includes("json"))
+            return "JSON";
+        if (tags.includes("arrow"))
+            return "ARROW";
+        return undefined;
+    }
+    extractColumns(cardData, splits) {
+        const columns = [];
+        // Try to get columns from cardData
+        if (cardData.columns) {
+            if (Array.isArray(cardData.columns)) {
+                return cardData.columns.map((col) => ({
+                    name: col.name || String(col),
+                    type: col.type,
+                    is_target: col.is_target || false
+                }));
+            }
+        }
+        // Try to infer from features if available
+        if (cardData.features) {
+            const features = cardData.features;
+            if (typeof features === 'object') {
+                for (const [key, value] of Object.entries(features)) {
+                    const feature = value;
+                    columns.push({
+                        name: key,
+                        type: feature?.dtype || feature?.type,
+                        is_target: key.toLowerCase().includes("label") ||
+                            key.toLowerCase().includes("target") ||
+                            key.toLowerCase().includes("y")
+                    });
+                }
+            }
+        }
+        return columns;
+    }
+}

package/build/metadata/store.js

@@ -0,0 +1,325 @@
+import Database from "better-sqlite3";
+import path from "path";
+import fs from "fs";
+export class MetadataStore {
+    db;
+    constructor(dbPath) {
+        // Ensure data directory exists
+        const dir = path.dirname(dbPath);
+        if (!fs.existsSync(dir)) {
+            fs.mkdirSync(dir, { recursive: true });
+        }
+        this.db = new Database(dbPath);
+        this.init();
+        this.migrate();
+    }
+    migrate() {
+        // Add install_path if missing
+        try {
+            this.db.exec("ALTER TABLE datasets ADD COLUMN install_path TEXT");
+            console.log("[MetadataStore] Migrated: Added install_path column");
+        }
+        catch (e) {
+            // Probably already exists
+        }
+    }
+    init() {
+        // Create table with all columns if it doesn't exist
+        this.db.exec(`
+      CREATE TABLE IF NOT EXISTS datasets (
+        id TEXT PRIMARY KEY,
+        source TEXT, -- 'huggingface', 'kaggle', 'uci', etc.
+        name TEXT,
+        description TEXT,
+        downloads INTEGER,
+        likes INTEGER,
+        stars INTEGER,
+        tags TEXT, -- JSON string
+        license_id TEXT,
+        license_category TEXT,
+        quality_score INTEGER,
+        has_train_split BOOLEAN,
+        total_examples INTEGER,
+        total_size_mb REAL,
+        is_safe_source BOOLEAN,
+        is_structured BOOLEAN,
+        last_updated TEXT,
+        quality_warnings TEXT, -- JSON array string
+        metadata_json TEXT, -- Full metadata as JSON
+        install_path TEXT -- Path to locally installed data
+      );
+
+      CREATE TABLE IF NOT EXISTS jobs (
+        id TEXT PRIMARY KEY,
+        type TEXT,
+        status TEXT,
+        priority INTEGER DEFAULT 0,
+        progress INTEGER DEFAULT 0,
+        status_text TEXT,
+        result_url TEXT,
+        error TEXT,
+        attempts INTEGER DEFAULT 0,
+        max_attempts INTEGER DEFAULT 3,
+        created_at TEXT,
+        updated_at TEXT,
+        metadata TEXT
+      );
+
+      CREATE TABLE IF NOT EXISTS local_files (
+        id TEXT PRIMARY KEY,
+        local_path TEXT,
+        status TEXT, -- 'downloading', 'completed', 'failed'
+        size_bytes INTEGER,
+        last_checked TEXT,
+        error TEXT
+      );
+
+      CREATE TABLE IF NOT EXISTS jobs_archive (
+        id TEXT PRIMARY KEY,
+        type TEXT,
+        status TEXT,
+        priority INTEGER DEFAULT 0,
+        progress INTEGER DEFAULT 0,
+        status_text TEXT,
+        result_url TEXT,
+        error TEXT,
+        attempts INTEGER DEFAULT 0,
+        max_attempts INTEGER DEFAULT 3,
+        created_at TEXT,
+        updated_at TEXT,
+        metadata TEXT
+      );
+    `);
+        // Migrate existing tables: add new columns if they don't exist
+        const tableInfo = this.db.prepare("PRAGMA table_info(datasets)").all();
+        const existingColumns = new Set(tableInfo.map(col => col.name));
+        const migrations = [
+            { column: "stars", type: "INTEGER DEFAULT 0" },
+            { column: "total_examples", type: "INTEGER" },
+            { column: "total_size_mb", type: "REAL" },
+            { column: "is_safe_source", type: "BOOLEAN DEFAULT 1" },
+            { column: "is_structured", type: "BOOLEAN DEFAULT 0" },
+            { column: "domain", type: "TEXT" },
+            { column: "source", type: "TEXT DEFAULT 'huggingface'" },
+            { column: "quality_warnings", type: "TEXT" }
+        ];
+        for (const migration of migrations) {
+            if (!existingColumns.has(migration.column)) {
+                console.error(`[MetadataStore] Migrating: adding column ${migration.column}`);
+                this.db.exec(`ALTER TABLE datasets ADD COLUMN ${migration.column} ${migration.type}`);
+            }
+        }
+        // Job migrations
+        const jobTableInfo = this.db.prepare("PRAGMA table_info(jobs)").all();
+        const existingJobColumns = new Set(jobTableInfo.map(col => col.name));
+        const jobMigrations = [
+            { column: "priority", type: "INTEGER DEFAULT 0" },
+            { column: "attempts", type: "INTEGER DEFAULT 0" },
+            { column: "max_attempts", type: "INTEGER DEFAULT 3" }
+        ];
+        for (const migration of jobMigrations) {
+            if (!existingJobColumns.has(migration.column)) {
+                console.error(`[MetadataStore] Migrating Job: adding column ${migration.column}`);
+                this.db.exec(`ALTER TABLE jobs ADD COLUMN ${migration.column} ${migration.type}`);
+            }
+        }
+        // Get updated column list after migrations
+        const updatedTableInfo = this.db.prepare("PRAGMA table_info(datasets)").all();
+        const allColumns = new Set(updatedTableInfo.map(col => col.name));
+        // Create indexes (only if columns exist)
+        const indexes = [
+            { name: "idx_license_category", table: "datasets", column: "license_category" },
+            { name: "idx_quality_score", table: "datasets", column: "quality_score" },
+            { name: "idx_downloads", table: "datasets", column: "downloads" },
+            { name: "idx_has_train_split", table: "datasets", column: "has_train_split" },
+            { name: "idx_is_safe_source", table: "datasets", column: "is_safe_source" },
+            { name: "idx_is_structured", table: "datasets", column: "is_structured" },
+            { name: "idx_domain", table: "datasets", column: "domain" },
+            { name: "idx_jobs_status", table: "jobs", column: "status" },
+            { name: "idx_jobs_created", table: "jobs", column: "created_at" },
+            { name: "idx_jobs_archive_created", table: "jobs_archive", column: "created_at" }
+        ];
+        for (const idx of indexes) {
+            // For datasets table, check if column exists first
+            if (idx.table === "datasets" && !allColumns.has(idx.column))
+                continue;
+            try {
+                this.db.exec(`CREATE INDEX IF NOT EXISTS ${idx.name} ON ${idx.table}(${idx.column})`);
+            }
+            catch (e) {
+                // Ignore index errors
+            }
+        }
+    }
+    saveDataset(dataset) {
+        // If the incoming dataset is incomplete, check if we already have a complete one
+        if (dataset.is_incomplete) {
+            const existing = this.getDataset(dataset.id);
+            if (existing && !existing.is_incomplete) {
+                // Already have better data, only update stats
+                const updateStats = this.db.prepare(`
+                    UPDATE datasets SET
+                        downloads = ?,
+                        likes = ?,
+                        stars = ?,
+                        last_updated = ?
+                    WHERE id = ?
+                `);
+                updateStats.run(dataset.downloads, dataset.likes, dataset.stars || 0, dataset.last_updated, dataset.id);
+                return;
+            }
+        }
+        const stmt = this.db.prepare(`
+      INSERT INTO datasets (
+        id, source, name, description, downloads, likes, stars, tags, 
+        license_id, license_category, quality_score, 
+        has_train_split, total_examples, total_size_mb,
+        is_safe_source, is_structured, last_updated, 
+        quality_warnings, metadata_json, install_path
+      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+      ON CONFLICT(id) DO UPDATE SET
+        source=excluded.source,
+        name=excluded.name,
+        description=excluded.description,
+        downloads=excluded.downloads,
+        likes=excluded.likes,
+        stars=excluded.stars,
+        tags=excluded.tags,
+        license_id=excluded.license_id,
+        license_category=excluded.license_category,
+        quality_score=excluded.quality_score,
+        has_train_split=excluded.has_train_split,
+        total_examples=excluded.total_examples,
+        total_size_mb=excluded.total_size_mb,
+        is_safe_source=excluded.is_safe_source,
+        is_structured=excluded.is_structured,
+        last_updated=excluded.last_updated,
+        quality_warnings=excluded.quality_warnings,
+        metadata_json=excluded.metadata_json,
+        install_path=excluded.install_path
+    `);
+        stmt.run(dataset.id, dataset.source, dataset.name, dataset.description, dataset.downloads, dataset.likes, dataset.stars || 0, JSON.stringify(dataset.tags), dataset.license.id, dataset.license.category, dataset.quality_score, dataset.has_train_split ? 1 : 0, dataset.total_examples, dataset.total_size_mb || null, dataset.is_safe_source ? 1 : 0, dataset.is_structured ? 1 : 0, dataset.last_updated, JSON.stringify(dataset.quality_warnings || []), JSON.stringify(dataset), dataset.install_path || null);
+    }
+    updateInstallPath(id, path) {
+        this.db.prepare("UPDATE datasets SET install_path = ? WHERE id = ?").run(path, id);
+    }
+    getDataset(id) {
+        const row = this.db.prepare("SELECT metadata_json, install_path FROM datasets WHERE id = ?").get(id);
+        if (!row)
+            return null;
+        const metadata = JSON.parse(row.metadata_json);
+        metadata.install_path = row.install_path || undefined;
+        return metadata;
+    }
+    getAllDatasets() {
+        const rows = this.db.prepare("SELECT metadata_json FROM datasets").all();
+        return rows.map(r => JSON.parse(r.metadata_json));
+    }
+    beginTransaction() {
+        this.db.exec("BEGIN");
+    }
+    commit() {
+        this.db.exec("COMMIT");
+    }
+    rollback() {
+        this.db.exec("ROLLBACK");
+    }
+    search(options) {
+        let query = "SELECT metadata_json FROM datasets WHERE 1=1";
+        const params = [];
+        if (options.licenseCategories && options.licenseCategories.length > 0) {
+            const placeholders = options.licenseCategories.map(() => "?").join(",");
+            query += ` AND license_category IN (${placeholders})`;
+            params.push(...options.licenseCategories);
+        }
+        if (options.minQualityScore !== undefined) {
+            query += " AND quality_score >= ?";
+            params.push(options.minQualityScore);
+        }
+        query += " ORDER BY quality_score DESC";
+        if (options.limit) {
+            query += " LIMIT ?";
+            params.push(options.limit);
+        }
+        const rows = this.db.prepare(query).all(...params);
+        return rows.map(r => JSON.parse(r.metadata_json));
+    }
+    saveJob(job) {
+        const upsert = this.db.prepare(`
+            INSERT INTO jobs (id, type, status, priority, progress, status_text, result_url, error, attempts, max_attempts, created_at, updated_at, metadata)
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+            ON CONFLICT(id) DO UPDATE SET
+                status=excluded.status,
+                priority=excluded.priority,
+                progress=excluded.progress,
+                status_text=excluded.status_text,
+                result_url=excluded.result_url,
+                error=excluded.error,
+                attempts=excluded.attempts,
+                max_attempts=excluded.max_attempts,
+                updated_at=excluded.updated_at,
+                metadata=excluded.metadata
+        `);
+        upsert.run(job.id, job.type, job.status, job.priority, job.progress, job.status_text, job.result_url || null, job.error || null, job.attempts, job.max_attempts, job.created_at, job.updated_at, job.metadata || null);
+    }
+    getJob(id) {
+        const row = this.db.prepare("SELECT * FROM jobs WHERE id = ?").get(id);
+        return row || null;
+    }
+    /**
+     * Register or update a local file record
+     */
+    registerDownload(id, localPath, status, sizeBytes, error) {
+        const upsert = this.db.prepare(`
+            INSERT INTO local_files (id, local_path, status, size_bytes, last_checked, error)
+            VALUES (?, ?, ?, ?, ?, ?)
+            ON CONFLICT(id) DO UPDATE SET
+                local_path=excluded.local_path,
+                status=excluded.status,
+                size_bytes=excluded.size_bytes,
+                last_checked=excluded.last_checked,
+                error=excluded.error
+        `);
+        upsert.run(id, localPath, status, sizeBytes || 0, new Date().toISOString(), error || null);
+    }
+    /**
+     * Get download status and path for a dataset
+     */
+    getDownloadStatus(id) {
+        return this.db.prepare("SELECT * FROM local_files WHERE id = ?").get(id);
+    }
+    /**
+     * Archive old jobs to the cold storage table.
+     * @param days Age in days
+     */
+    archiveOldJobs(days) {
+        const cutoff = new Date();
+        cutoff.setDate(cutoff.getDate() - days);
+        const cutoffStr = cutoff.toISOString();
+        return this.db.transaction(() => {
+            // Copy to archive
+            this.db.prepare(`
+                INSERT OR IGNORE INTO jobs_archive 
+                SELECT * FROM jobs 
+                WHERE created_at < ?
+            `).run(cutoffStr);
+            // Delete from active jobs
+            const info = this.db.prepare(`
+                DELETE FROM jobs 
+                WHERE created_at < ?
+            `).run(cutoffStr);
+            return info.changes;
+        })();
+    }
+    /**
+     * Perform database maintenance (VACUUM, ANALYZE).
+     */
+    optimize() {
+        console.log("[MetadataStore] Running database optimization (VACUUM, ANALYZE)...");
+        this.db.exec("VACUUM");
+        this.db.exec("ANALYZE");
+    }
+    close() {
+        this.db.close();
+    }
+}

package/build/metadata/types.js

@@ -0,0 +1 @@
+export {};