@ecubelabs/mysql-mcp 0.0.0-stage → 0.0.1

package/dist/src/catalog/store.js

@@ -0,0 +1,617 @@
+import * as fs from 'fs';
+import * as path from 'path';
+import { CATALOG_VERSION, emptyCatalog, emptyTable, normalizeName, pruneJoins } from './types.js';
+const FLUSH_DELAY_MS = 2_000;
+const LOCK_RETRY_MS = 25;
+const LOCK_WAIT_MS = 15_000;
+const LOCK_CLOSE_WAIT_MS = 2_000;
+const LOCK_STALE_MS = 30_000;
+const CLOSE_FLUSH_ATTEMPTS = 2;
+class CatalogLockTimeoutError extends Error {
+}
+function errorCode(error) {
+    return error && typeof error === 'object' && 'code' in error ? String(error.code) : null;
+}
+function processIsAlive(pid) {
+    try {
+        process.kill(pid, 0);
+        return true;
+    }
+    catch (error) {
+        return errorCode(error) !== 'ESRCH';
+    }
+}
+function delay(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+async function acquireCatalogLock(filePath, waitMs) {
+    const lockPath = `${filePath}.lock`;
+    const deadline = Date.now() + waitMs;
+    const record = {
+        pid: process.pid,
+        token: `${process.pid}-${Date.now()}-${Math.random().toString(16).slice(2)}`,
+        createdAt: Date.now(),
+    };
+    while (true) {
+        try {
+            const handle = await fs.promises.open(lockPath, 'wx', 0o600);
+            try {
+                await handle.writeFile(JSON.stringify(record), 'utf8');
+                await handle.sync();
+            }
+            catch (error) {
+                await handle.close().catch(() => undefined);
+                await fs.promises.unlink(lockPath).catch(() => undefined);
+                throw error;
+            }
+            let released = false;
+            return async () => {
+                if (released) {
+                    return;
+                }
+                released = true;
+                await handle.close();
+                try {
+                    const current = JSON.parse(await fs.promises.readFile(lockPath, 'utf8'));
+                    if (current.token === record.token) {
+                        await fs.promises.unlink(lockPath);
+                    }
+                }
+                catch (error) {
+                    if (errorCode(error) !== 'ENOENT') {
+                        throw error;
+                    }
+                }
+            };
+        }
+        catch (error) {
+            if (errorCode(error) !== 'EEXIST') {
+                throw error;
+            }
+            let lockText = null;
+            let stale = false;
+            try {
+                const [text, stat] = await Promise.all([
+                    fs.promises.readFile(lockPath, 'utf8'),
+                    fs.promises.stat(lockPath),
+                ]);
+                lockText = text;
+                let owner = {};
+                try {
+                    owner = JSON.parse(text);
+                }
+                catch {
+                }
+                stale =
+                    typeof owner.pid === 'number'
+                        ? !processIsAlive(owner.pid)
+                        : Date.now() - stat.mtimeMs >= LOCK_STALE_MS;
+            }
+            catch (readError) {
+                if (errorCode(readError) === 'ENOENT') {
+                    continue;
+                }
+                throw readError;
+            }
+            if (stale && lockText !== null) {
+                try {
+                    if ((await fs.promises.readFile(lockPath, 'utf8')) === lockText) {
+                        await fs.promises.unlink(lockPath);
+                        console.error(`[catalog] removed stale lock ${lockPath}`);
+                        continue;
+                    }
+                }
+                catch (cleanupError) {
+                    if (errorCode(cleanupError) === 'ENOENT') {
+                        continue;
+                    }
+                    throw cleanupError;
+                }
+            }
+            if (Date.now() >= deadline) {
+                throw new CatalogLockTimeoutError(`timed out after ${waitMs}ms waiting for catalog lock ${lockPath}`);
+            }
+            await delay(LOCK_RETRY_MS + Math.floor(Math.random() * LOCK_RETRY_MS));
+        }
+    }
+}
+const EPOCH = new Date(0).toISOString();
+function timestamp(value) {
+    if (!value) {
+        return 0;
+    }
+    const parsed = Date.parse(value);
+    return Number.isFinite(parsed) ? parsed : 0;
+}
+function mergeEditedSet(external, local, base) {
+    const merged = new Set(external);
+    for (const previous of base) {
+        if (!local.includes(previous)) {
+            merged.delete(previous);
+        }
+    }
+    for (const current of local) {
+        if (!base.includes(current)) {
+            merged.add(current);
+        }
+    }
+    return [...merged];
+}
+function mergeCurated(external, local, base) {
+    const baseDoc = Object.prototype.hasOwnProperty.call(base ?? {}, 'doc') ? JSON.stringify(base?.doc) : '__missing__';
+    const localDoc = Object.prototype.hasOwnProperty.call(local, 'doc') ? JSON.stringify(local.doc) : '__missing__';
+    const chosen = localDoc !== baseDoc ? local : external;
+    return {
+        notes: mergeEditedSet(external.notes ?? [], local.notes ?? [], base?.notes ?? []),
+        aliases: mergeEditedSet(external.aliases ?? [], local.aliases ?? [], base?.aliases ?? []),
+        ...(Object.prototype.hasOwnProperty.call(chosen, 'doc') ? { doc: chosen.doc } : {}),
+    };
+}
+function mergedCounter(external, local, base) {
+    if (local < base) {
+        return local + Math.max(0, external - base);
+    }
+    return external + Math.max(0, local - base);
+}
+function newerUsageTimestamp(externalValue, localValue, externalCount, localCount, baseCount) {
+    if (localCount < baseCount && externalCount <= baseCount) {
+        return localValue ?? null;
+    }
+    if (externalCount < baseCount && localCount <= baseCount) {
+        return externalValue ?? null;
+    }
+    return timestamp(localValue) >= timestamp(externalValue) ? (localValue ?? null) : (externalValue ?? null);
+}
+function mergeTable(external, local, base) {
+    const baseDetailAt = timestamp(base?.detailScannedAt);
+    const localInvalidated = local.detailStale === true && (base?.detailStale !== true || local.detailScannedAt === null);
+    const externalInvalidated = external.detailStale === true && (base?.detailStale !== true || external.detailScannedAt === null);
+    const externalRefreshed = timestamp(external.detailScannedAt) > baseDetailAt;
+    const localRefreshed = timestamp(local.detailScannedAt) > baseDetailAt;
+    const newerDetail = localInvalidated && !externalRefreshed
+        ? local
+        : externalInvalidated && !localRefreshed
+            ? external
+            : timestamp(local.detailScannedAt) >= timestamp(external.detailScannedAt)
+                ? local
+                : external;
+    const externalUsageCount = external.usage?.count ?? 0;
+    const localUsageCount = local.usage?.count ?? 0;
+    const baseUsageCount = base?.usage?.count ?? 0;
+    const columns = [
+        ...new Set([...Object.keys(external.usage?.columns ?? {}), ...Object.keys(local.usage?.columns ?? {})]),
+    ]
+        .map((column) => [
+        column,
+        mergedCounter(external.usage?.columns?.[column] ?? 0, local.usage?.columns?.[column] ?? 0, base?.usage?.columns?.[column] ?? 0),
+    ])
+        .filter(([, count]) => count > 0);
+    return {
+        ...newerDetail,
+        usage: {
+            count: mergedCounter(externalUsageCount, localUsageCount, baseUsageCount),
+            successCount: mergedCounter(external.usage?.successCount ?? 0, local.usage?.successCount ?? 0, base?.usage?.successCount ?? 0),
+            failureCount: mergedCounter(external.usage?.failureCount ?? 0, local.usage?.failureCount ?? 0, base?.usage?.failureCount ?? 0),
+            lastUsedAt: newerUsageTimestamp(external.usage?.lastUsedAt, local.usage?.lastUsedAt, externalUsageCount, localUsageCount, baseUsageCount),
+            columns: Object.fromEntries(columns),
+        },
+        curated: mergeCurated(external.curated ?? { notes: [], aliases: [] }, local.curated ?? { notes: [], aliases: [] }, base?.curated),
+    };
+}
+function mergeSchema(external, local, base) {
+    const newerInventory = timestamp(local.scannedAt) >= timestamp(external.scannedAt) ? local : external;
+    const olderInventory = newerInventory === local ? external : local;
+    const tables = {};
+    for (const [name, table] of Object.entries(newerInventory.tables)) {
+        tables[name] = olderInventory.tables[name]
+            ? mergeTable(newerInventory === local ? olderInventory.tables[name] : table, newerInventory === local ? table : olderInventory.tables[name], base?.tables[name])
+            : table;
+    }
+    return { ...newerInventory, tables };
+}
+function mergeCatalog(external, local, base) {
+    const schemas = {};
+    for (const name of new Set([...Object.keys(external.schemas), ...Object.keys(local.schemas)])) {
+        if (external.schemas[name] && local.schemas[name]) {
+            schemas[name] = mergeSchema(external.schemas[name], local.schemas[name], base.schemas[name]);
+        }
+        else {
+            schemas[name] = external.schemas[name] ?? local.schemas[name];
+        }
+    }
+    const joins = new Map();
+    const externalJoins = new Map(external.joins.map((edge) => [`${edge.a}\u0000${edge.b}`, edge]));
+    const localJoins = new Map(local.joins.map((edge) => [`${edge.a}\u0000${edge.b}`, edge]));
+    const baseJoins = new Map(base.joins.map((edge) => [`${edge.a}\u0000${edge.b}`, edge]));
+    for (const key of new Set([...externalJoins.keys(), ...localJoins.keys()])) {
+        const outside = externalJoins.get(key);
+        const inside = localJoins.get(key);
+        const ancestor = baseJoins.get(key);
+        const edge = outside ?? inside;
+        if (!edge) {
+            continue;
+        }
+        const count = mergedCounter(outside?.count ?? 0, inside?.count ?? 0, ancestor?.count ?? 0);
+        if (count <= 0) {
+            continue;
+        }
+        joins.set(key, {
+            a: edge.a,
+            b: edge.b,
+            count,
+            lastUsedAt: newerUsageTimestamp(outside?.lastUsedAt, inside?.lastUsedAt, outside?.count ?? 0, inside?.count ?? 0, ancestor?.count ?? 0) ?? EPOCH,
+        });
+    }
+    const localDocsChanged = local.docs.refCommit !== base.docs.refCommit;
+    const externalDocsChanged = external.docs.refCommit !== base.docs.refCommit;
+    const chosenDocs = localDocsChanged && externalDocsChanged
+        ? timestamp(local.docs.refUpdatedAt) >= timestamp(external.docs.refUpdatedAt)
+            ? local.docs
+            : external.docs
+        : localDocsChanged
+            ? local.docs
+            : external.docs;
+    const merged = {
+        ...local,
+        docs: {
+            ...chosenDocs,
+            repo: local.docs.repo,
+            ref: local.docs.ref,
+            paths: chosenDocs.paths ?? [],
+            unlinked: [],
+        },
+        schemas,
+        joins: pruneJoins([...joins.values()]),
+    };
+    const linked = new Set();
+    for (const schema of Object.values(merged.schemas)) {
+        for (const table of Object.values(schema.tables)) {
+            if (table.curated.doc && !merged.docs.paths.includes(table.curated.doc.path)) {
+                delete table.curated.doc;
+            }
+            if (table.curated.doc) {
+                linked.add(table.curated.doc.path);
+            }
+        }
+    }
+    merged.docs.unlinked = merged.docs.paths.filter((path) => !linked.has(path));
+    return merged;
+}
+export class CatalogStore {
+    options;
+    enabled;
+    data;
+    baseData;
+    flushTimer = null;
+    flushPromise = null;
+    revision = 0;
+    flushedRevision = 0;
+    closing = false;
+    nameIndex = null;
+    constructor(options) {
+        this.options = options;
+        this.enabled = options.enabled;
+        this.data = emptyCatalog(options);
+        this.baseData = structuredClone(this.data);
+        if (this.enabled) {
+            this.initialize();
+        }
+    }
+    initialize() {
+        try {
+            fs.mkdirSync(path.dirname(this.options.filePath), {
+                recursive: true,
+                mode: 0o700,
+            });
+            this.loadFromDisk();
+        }
+        catch (error) {
+            this.disable(`cannot initialize ${this.options.filePath}: ${error instanceof Error ? error.message : String(error)}`);
+        }
+    }
+    loadFromDisk() {
+        if (!fs.existsSync(this.options.filePath)) {
+            return;
+        }
+        let parsed;
+        try {
+            parsed = JSON.parse(fs.readFileSync(this.options.filePath, 'utf8'));
+        }
+        catch (error) {
+            console.error(`[catalog] ignoring unreadable ${this.options.filePath}; it will be replaced: ${error instanceof Error ? error.message : String(error)}`);
+            return;
+        }
+        if (parsed.version !== CATALOG_VERSION ||
+            parsed.profile !== this.options.profile ||
+            parsed.fingerprint !== this.options.fingerprint) {
+            console.error(`[catalog] ignoring incompatible cache at ${this.options.filePath}; it will be replaced.`);
+            return;
+        }
+        this.data = this.sanitizeLoaded(parsed);
+        this.invalidateNameIndex();
+        this.baseData = structuredClone(this.data);
+        this.revision += 1;
+        this.scheduleFlush();
+        console.error(`[catalog] loaded ${this.options.filePath}`);
+    }
+    sanitizeLoaded(loaded) {
+        const fresh = emptyCatalog(this.options);
+        const docsCompatible = loaded.docs?.repo === this.options.docsRepo && loaded.docs?.ref === this.options.docsRef;
+        for (const [schemaName, schema] of Object.entries(fresh.schemas)) {
+            const cached = loaded.schemas?.[schemaName];
+            if (!cached) {
+                continue;
+            }
+            schema.scannedAt = cached.scannedAt ?? null;
+            for (const [tableName, raw] of Object.entries(cached.tables ?? {})) {
+                const base = emptyTable();
+                const table = {
+                    ...base,
+                    ...raw,
+                    columns: raw.columns ?? [],
+                    pk: raw.pk ?? [],
+                    indexes: raw.indexes ?? [],
+                    fks: raw.fks ?? [],
+                    usage: {
+                        count: raw.usage?.count ?? 0,
+                        successCount: raw.usage?.successCount ?? raw.usage?.count ?? 0,
+                        failureCount: raw.usage?.failureCount ?? 0,
+                        lastUsedAt: raw.usage?.lastUsedAt ?? null,
+                        columns: raw.usage?.columns ?? {},
+                    },
+                    curated: {
+                        notes: raw.curated?.notes ?? [],
+                        aliases: raw.curated?.aliases ?? [],
+                        ...(docsCompatible && Object.prototype.hasOwnProperty.call(raw.curated ?? {}, 'doc')
+                            ? { doc: raw.curated.doc }
+                            : {}),
+                    },
+                };
+                schema.tables[tableName] = table;
+            }
+        }
+        if (docsCompatible) {
+            fresh.docs = {
+                ...fresh.docs,
+                refCommit: loaded.docs.refCommit ?? null,
+                refUpdatedAt: loaded.docs.refUpdatedAt ?? null,
+                paths: loaded.docs.paths ?? [],
+                unlinked: loaded.docs.unlinked ?? [],
+            };
+        }
+        fresh.joins = pruneJoins((loaded.joins ?? [])
+            .map((edge) => ({ ...edge, lastUsedAt: edge.lastUsedAt ?? EPOCH })));
+        return fresh;
+    }
+    disable(reason) {
+        this.enabled = false;
+        if (this.flushTimer) {
+            clearTimeout(this.flushTimer);
+        }
+        this.flushTimer = null;
+        console.error(`[catalog] disabled: ${reason}`);
+    }
+    isEnabled() {
+        return this.enabled;
+    }
+    invalidateNameIndex() {
+        this.nameIndex = null;
+    }
+    ensureNameIndex() {
+        if (this.nameIndex) {
+            return this.nameIndex;
+        }
+        const index = new Map();
+        for (const [schemaName, schema] of Object.entries(this.data.schemas)) {
+            const tables = new Map();
+            for (const tableName of Object.keys(schema.tables)) {
+                tables.set(normalizeName(tableName), tableName);
+            }
+            index.set(normalizeName(schemaName), { schema: schemaName, tables });
+        }
+        this.nameIndex = index;
+        return index;
+    }
+    snapshot() {
+        return structuredClone(this.data);
+    }
+    docsView() {
+        return structuredClone(this.data.docs);
+    }
+    lookup(schemaHint, table) {
+        const index = this.ensureNameIndex();
+        const wanted = normalizeName(table);
+        const scopes = schemaHint ? [index.get(normalizeName(schemaHint))] : [...index.values()];
+        const matches = [];
+        for (const scope of scopes) {
+            if (!scope) {
+                continue;
+            }
+            const canonical = scope.tables.get(wanted);
+            if (canonical) {
+                matches.push({ schema: scope.schema, table: canonical });
+            }
+        }
+        return matches;
+    }
+    schemaState(schema) {
+        const entry = this.data.schemas[schema];
+        return entry ? { scannedAt: entry.scannedAt } : null;
+    }
+    tableMeta(schema, table) {
+        const entry = this.data.schemas[schema]?.tables[table];
+        if (!entry) {
+            return null;
+        }
+        return {
+            detailScannedAt: entry.detailScannedAt,
+            detailStale: entry.detailStale === true,
+            docReviewed: Object.prototype.hasOwnProperty.call(entry.curated, 'doc'),
+        };
+    }
+    tableIndexes(schema, table) {
+        const entry = this.data.schemas[schema]?.tables[table];
+        if (!entry) {
+            return null;
+        }
+        return {
+            schema,
+            table,
+            indexes: entry.indexes.map((index) => ({
+                ...index,
+                columns: [...index.columns],
+            })),
+            pk: [...entry.pk],
+            rowsEstimate: entry.rowsEstimate,
+            detailScannedAt: entry.detailScannedAt,
+            detailStale: entry.detailStale === true,
+        };
+    }
+    readTable(schema, table) {
+        const entry = this.data.schemas[schema]?.tables[table];
+        return entry ? structuredClone(entry) : null;
+    }
+    hasSuccessfulUse() {
+        for (const schema of Object.values(this.data.schemas)) {
+            for (const table of Object.values(schema.tables)) {
+                if (table.usage.successCount > 0) {
+                    return true;
+                }
+            }
+        }
+        return false;
+    }
+    readJoins(schema, table) {
+        const prefix = `${schema}.${table}.`.toLowerCase();
+        return this.data.joins
+            .filter((edge) => edge.a.toLowerCase().startsWith(prefix) || edge.b.toLowerCase().startsWith(prefix))
+            .sort((a, b) => b.count - a.count || a.a.localeCompare(b.a))
+            .slice(0, 20)
+            .map((edge) => ({ ...edge }));
+    }
+    update(mutator) {
+        if (!this.enabled) {
+            return;
+        }
+        mutator(this.data);
+        this.invalidateNameIndex();
+        this.revision += 1;
+        this.scheduleFlush();
+    }
+    scheduleFlush() {
+        if (!this.enabled || this.closing || this.flushTimer) {
+            return;
+        }
+        this.flushTimer = setTimeout(() => {
+            this.flushTimer = null;
+            void this.flush();
+        }, FLUSH_DELAY_MS);
+        this.flushTimer.unref();
+    }
+    async flush() {
+        if (!this.enabled) {
+            return;
+        }
+        if (this.flushPromise) {
+            return this.flushPromise;
+        }
+        if (this.revision === this.flushedRevision) {
+            return;
+        }
+        this.flushPromise = this.writeAtomically().finally(() => {
+            this.flushPromise = null;
+        });
+        return this.flushPromise;
+    }
+    async writeAtomically() {
+        let tempPath = null;
+        let releaseLock = null;
+        try {
+            releaseLock = await acquireCatalogLock(this.options.filePath, this.closing ? LOCK_CLOSE_WAIT_MS : LOCK_WAIT_MS);
+            tempPath = `${this.options.filePath}.${process.pid}.${Date.now()}.tmp`;
+            const writingRevision = this.revision;
+            let toWrite = this.sanitizeLoaded(structuredClone(this.data));
+            const external = this.readExternal();
+            if (external &&
+                external.version === CATALOG_VERSION &&
+                external.profile === this.options.profile &&
+                external.fingerprint === this.options.fingerprint) {
+                toWrite = mergeCatalog(this.sanitizeLoaded(external), toWrite, this.baseData);
+                this.data = toWrite;
+                this.invalidateNameIndex();
+            }
+            const payload = `${JSON.stringify(toWrite, null, 2)}\n`;
+            const persisted = structuredClone(toWrite);
+            await fs.promises.writeFile(tempPath, payload, {
+                encoding: 'utf8',
+                mode: 0o600,
+                flag: 'wx',
+            });
+            await fs.promises.rename(tempPath, this.options.filePath);
+            await fs.promises.chmod(this.options.filePath, 0o600);
+            this.baseData = persisted;
+            this.flushedRevision = writingRevision;
+            if (this.revision !== this.flushedRevision) {
+                this.scheduleFlush();
+            }
+        }
+        catch (error) {
+            if (tempPath) {
+                try {
+                    await fs.promises.unlink(tempPath);
+                }
+                catch {
+                }
+            }
+            if (error instanceof CatalogLockTimeoutError) {
+                console.error(`[catalog] ${error.message}; will retry`);
+                this.scheduleFlush();
+                return;
+            }
+            this.disable(`cannot write ${this.options.filePath}: ${error instanceof Error ? error.message : String(error)}`);
+        }
+        finally {
+            if (releaseLock) {
+                try {
+                    await releaseLock();
+                }
+                catch (error) {
+                    this.disable(`cannot release catalog lock: ${error instanceof Error ? error.message : String(error)}`);
+                }
+            }
+        }
+    }
+    readExternal() {
+        if (!fs.existsSync(this.options.filePath)) {
+            return null;
+        }
+        try {
+            return JSON.parse(fs.readFileSync(this.options.filePath, 'utf8'));
+        }
+        catch (error) {
+            console.error(`[catalog] ignoring unreadable ${this.options.filePath}; it will be replaced: ${error instanceof Error ? error.message : String(error)}`);
+            return null;
+        }
+    }
+    async close() {
+        this.closing = true;
+        if (this.flushTimer) {
+            clearTimeout(this.flushTimer);
+            this.flushTimer = null;
+        }
+        for (let attempt = 0; attempt < CLOSE_FLUSH_ATTEMPTS; attempt += 1) {
+            if (!this.enabled || this.revision === this.flushedRevision) {
+                return;
+            }
+            if (this.flushPromise) {
+                await this.flushPromise;
+            }
+            else {
+                await this.flush();
+            }
+        }
+        if (this.enabled && this.revision !== this.flushedRevision) {
+            console.error(`[catalog] gave up persisting the last updates to ${this.options.filePath} after ${CLOSE_FLUSH_ATTEMPTS} attempts`);
+        }
+    }
+}

package/dist/src/catalog/types.js

@@ -0,0 +1,59 @@
+export const CATALOG_VERSION = 2;
+export const JOIN_EDGE_LIMIT = 500;
+export function pruneJoins(joins) {
+    if (joins.length <= JOIN_EDGE_LIMIT) {
+        return joins;
+    }
+    return [...joins].sort((a, b) => b.lastUsedAt.localeCompare(a.lastUsedAt)).slice(0, JOIN_EDGE_LIMIT);
+}
+export function normalizeName(value) {
+    return value.trim().toLowerCase();
+}
+export function emptyTable() {
+    return {
+        comment: '',
+        rowsEstimate: null,
+        detailScannedAt: null,
+        columns: [],
+        pk: [],
+        indexes: [],
+        fks: [],
+        usage: emptyUsage(),
+        curated: { notes: [], aliases: [] },
+    };
+}
+export function emptyUsage() {
+    return {
+        count: 0,
+        successCount: 0,
+        failureCount: 0,
+        lastUsedAt: null,
+        columns: {},
+    };
+}
+export function emptyCatalog(options) {
+    const schemas = {};
+    for (const entry of options.appSchemas) {
+        schemas[entry.schema] = {
+            app: entry.app,
+            ...(entry.description ? { description: entry.description } : {}),
+            scannedAt: null,
+            tables: {},
+        };
+    }
+    return {
+        version: CATALOG_VERSION,
+        profile: options.profile,
+        fingerprint: options.fingerprint,
+        docs: {
+            repo: options.docsRepo,
+            ref: options.docsRef,
+            refCommit: null,
+            refUpdatedAt: null,
+            paths: [],
+            unlinked: [],
+        },
+        schemas,
+        joins: [],
+    };
+}