daeda-mcp 1.0.0

package/dist/sync/csv-loader.js

@@ -0,0 +1,450 @@
+import { clearTable, setMetadata, batchInsertContacts, batchInsertCompanies, batchInsertDeals, batchInsertAssociations, } from "../db/sqlite.js";
+function findIdColumn(headers) {
+    const idCandidates = [
+        "Record ID",
+        "hs_object_id",
+        "id",
+        "ID",
+        "Record Id",
+        "record_id",
+        "recordId",
+        "Object ID",
+        "object_id",
+        "Contact ID",
+        "Company ID",
+        "Deal ID",
+    ];
+    for (const candidate of idCandidates) {
+        if (headers.includes(candidate)) {
+            return candidate;
+        }
+    }
+    const lowerHeaders = headers.map((h) => h.toLowerCase());
+    for (const candidate of idCandidates) {
+        const idx = lowerHeaders.indexOf(candidate.toLowerCase());
+        if (idx !== -1) {
+            return headers[idx];
+        }
+    }
+    const recordIdCol = headers.find((h) => h.toLowerCase().includes("record") && h.toLowerCase().includes("id"));
+    if (recordIdCol)
+        return recordIdCol;
+    const anyIdCol = headers.find((h) => h.toLowerCase() === "id" || h.toLowerCase().endsWith("_id"));
+    if (anyIdCol)
+        return anyIdCol;
+    return null;
+}
+async function loadCsvWithWorker(filePath, tableName, mapRow, insertBatch, onProgress) {
+    await clearTable(tableName);
+    return new Promise((resolve, reject) => {
+        const workerUrl = new URL("./csv-worker.ts", import.meta.url);
+        const worker = new Worker(workerUrl);
+        let idColumn = null;
+        let totalInserted = 0;
+        let pendingInserts = [];
+        worker.onmessage = async (event) => {
+            const msg = event.data;
+            try {
+                switch (msg.type) {
+                    case "headers": {
+                        idColumn = findIdColumn(msg.headers);
+                        if (!idColumn) {
+                            worker.postMessage({ type: "cancel" });
+                            worker.terminate();
+                            reject(new Error(`Could not find ID column. Available columns: ${msg.headers.slice(0, 20).join(", ")}${msg.headers.length > 20 ? "..." : ""}`));
+                        }
+                        break;
+                    }
+                    case "batch": {
+                        if (!idColumn)
+                            break;
+                        const mappedRows = [];
+                        for (const row of msg.rows) {
+                            const mapped = mapRow(row, idColumn);
+                            if (mapped) {
+                                mappedRows.push(mapped);
+                            }
+                        }
+                        if (mappedRows.length > 0) {
+                            const insertPromise = insertBatch(mappedRows).then(() => {
+                                totalInserted += mappedRows.length;
+                                onProgress?.(totalInserted);
+                            });
+                            pendingInserts.push(insertPromise);
+                            if (pendingInserts.length >= 3) {
+                                await Promise.all(pendingInserts);
+                                pendingInserts = [];
+                            }
+                        }
+                        break;
+                    }
+                    case "done": {
+                        await Promise.all(pendingInserts);
+                        worker.terminate();
+                        resolve(totalInserted);
+                        break;
+                    }
+                    case "error": {
+                        worker.terminate();
+                        reject(new Error(msg.error));
+                        break;
+                    }
+                }
+            }
+            catch (err) {
+                worker.terminate();
+                reject(err);
+            }
+        };
+        worker.onerror = (err) => {
+            worker.terminate();
+            reject(new Error(`Worker error: ${err.message}`));
+        };
+        worker.postMessage({
+            type: "start",
+            filePath,
+            batchSize: 5000,
+        });
+    });
+}
+function mapContact(row, idColumn) {
+    const id = row[idColumn];
+    if (!id)
+        return null;
+    const email = row.email || row.Email || null;
+    const properties = {};
+    for (const [key, value] of Object.entries(row)) {
+        if (key !== idColumn && value) {
+            properties[key] = value;
+        }
+    }
+    return { id, email, properties };
+}
+function mapCompany(row, idColumn) {
+    const id = row[idColumn];
+    if (!id)
+        return null;
+    const domain = row.domain || row.Domain || null;
+    const properties = {};
+    for (const [key, value] of Object.entries(row)) {
+        if (key !== idColumn && value) {
+            properties[key] = value;
+        }
+    }
+    return { id, domain, properties };
+}
+function mapDeal(row, idColumn) {
+    const id = row[idColumn];
+    if (!id)
+        return null;
+    const dealname = row.dealname || row["Deal Name"] || row.Dealname || null;
+    const properties = {};
+    for (const [key, value] of Object.entries(row)) {
+        if (key !== idColumn && value) {
+            properties[key] = value;
+        }
+    }
+    return { id, dealname, properties };
+}
+export async function loadContactsCsvFromFile(filePath, onProgress) {
+    return loadCsvWithWorker(filePath, "contacts", mapContact, batchInsertContacts, onProgress);
+}
+export async function loadCompaniesCsvFromFile(filePath, onProgress) {
+    return loadCsvWithWorker(filePath, "companies", mapCompany, batchInsertCompanies, onProgress);
+}
+export async function loadDealsCsvFromFile(filePath, onProgress) {
+    return loadCsvWithWorker(filePath, "deals", mapDeal, batchInsertDeals, onProgress);
+}
+export async function loadAssociationsCsvFromFile(filePath, associationType, onProgress) {
+    await clearTable(associationType);
+    return new Promise((resolve, reject) => {
+        const workerUrl = new URL("./csv-worker.ts", import.meta.url);
+        const worker = new Worker(workerUrl);
+        let idColumn = null;
+        let associatedIdColumn = null;
+        let totalInserted = 0;
+        let pendingInserts = [];
+        worker.onmessage = async (event) => {
+            const msg = event.data;
+            try {
+                switch (msg.type) {
+                    case "headers": {
+                        idColumn = findIdColumn(msg.headers);
+                        if (!idColumn) {
+                            worker.postMessage({ type: "cancel" });
+                            worker.terminate();
+                            reject(new Error(`Could not find ID column in ${associationType} association CSV`));
+                            return;
+                        }
+                        associatedIdColumn =
+                            msg.headers.find((h) => h.toLowerCase().includes("associated") &&
+                                (h.toLowerCase().includes("id") || h.toLowerCase().includes("record"))) || null;
+                        if (!associatedIdColumn) {
+                            worker.postMessage({ type: "cancel" });
+                            worker.terminate();
+                            reject(new Error(`Could not find associated IDs column in ${associationType} CSV`));
+                            return;
+                        }
+                        break;
+                    }
+                    case "batch": {
+                        if (!idColumn || !associatedIdColumn)
+                            break;
+                        const associationRows = [];
+                        for (const row of msg.rows) {
+                            const fromId = row[idColumn];
+                            if (!fromId)
+                                continue;
+                            const associatedIdsRaw = row[associatedIdColumn];
+                            if (!associatedIdsRaw)
+                                continue;
+                            const associatedIds = associatedIdsRaw
+                                .split(/[;,]/)
+                                .map((id) => id.trim())
+                                .filter(Boolean);
+                            for (const toId of associatedIds) {
+                                associationRows.push({ fromId, toId });
+                            }
+                        }
+                        if (associationRows.length > 0) {
+                            const insertPromise = batchInsertAssociations(associationType, associationRows).then(() => {
+                                totalInserted += associationRows.length;
+                                onProgress?.(totalInserted);
+                            });
+                            pendingInserts.push(insertPromise);
+                            if (pendingInserts.length >= 3) {
+                                await Promise.all(pendingInserts);
+                                pendingInserts = [];
+                            }
+                        }
+                        break;
+                    }
+                    case "done": {
+                        await Promise.all(pendingInserts);
+                        worker.terminate();
+                        resolve(totalInserted);
+                        break;
+                    }
+                    case "error": {
+                        worker.terminate();
+                        reject(new Error(msg.error));
+                        break;
+                    }
+                }
+            }
+            catch (err) {
+                worker.terminate();
+                reject(err);
+            }
+        };
+        worker.onerror = (err) => {
+            worker.terminate();
+            reject(new Error(`Worker error: ${err.message}`));
+        };
+        worker.postMessage({
+            type: "start",
+            filePath,
+            batchSize: 5000,
+        });
+    });
+}
+// Legacy functions for backwards compatibility (loads from string, not recommended for large files)
+function parseCSV(csvData) {
+    const lines = csvData.split("\n");
+    if (lines.length < 2)
+        return [];
+    const headers = parseCSVLine(lines[0]);
+    const rows = [];
+    for (let i = 1; i < lines.length; i++) {
+        const line = lines[i].trim();
+        if (!line)
+            continue;
+        const values = parseCSVLine(line);
+        const row = {};
+        headers.forEach((header, idx) => {
+            row[header] = values[idx] || "";
+        });
+        rows.push(row);
+    }
+    return rows;
+}
+function parseCSVLine(line) {
+    const result = [];
+    let current = "";
+    let inQuotes = false;
+    for (let i = 0; i < line.length; i++) {
+        const char = line[i];
+        if (char === '"') {
+            if (inQuotes && line[i + 1] === '"') {
+                current += '"';
+                i++;
+            }
+            else {
+                inQuotes = !inQuotes;
+            }
+        }
+        else if (char === "," && !inQuotes) {
+            result.push(current);
+            current = "";
+        }
+        else {
+            current += char;
+        }
+    }
+    result.push(current);
+    return result;
+}
+export async function loadContactsCsv(csvData, onProgress) {
+    await clearTable("contacts");
+    const rows = parseCSV(csvData);
+    if (rows.length === 0)
+        return 0;
+    const headers = Object.keys(rows[0]);
+    const idColumn = findIdColumn(headers);
+    if (!idColumn) {
+        throw new Error(`Could not find ID column in contacts CSV. Available columns: ${headers.slice(0, 20).join(", ")}${headers.length > 20 ? "..." : ""}`);
+    }
+    const contactRows = [];
+    for (const row of rows) {
+        const id = row[idColumn];
+        if (!id)
+            continue;
+        const email = row.email || row.Email || null;
+        const properties = {};
+        for (const [key, value] of Object.entries(row)) {
+            if (key !== idColumn && value) {
+                properties[key] = value;
+            }
+        }
+        contactRows.push({ id, email, properties });
+    }
+    onProgress?.(0);
+    await batchInsertContacts(contactRows);
+    onProgress?.(contactRows.length);
+    return contactRows.length;
+}
+export async function loadCompaniesCsv(csvData, onProgress) {
+    await clearTable("companies");
+    const rows = parseCSV(csvData);
+    if (rows.length === 0)
+        return 0;
+    const headers = Object.keys(rows[0]);
+    const idColumn = findIdColumn(headers);
+    if (!idColumn) {
+        throw new Error(`Could not find ID column in companies CSV. Available columns: ${headers.slice(0, 20).join(", ")}${headers.length > 20 ? "..." : ""}`);
+    }
+    const companyRows = [];
+    for (const row of rows) {
+        const id = row[idColumn];
+        if (!id)
+            continue;
+        const domain = row.domain || row.Domain || null;
+        const properties = {};
+        for (const [key, value] of Object.entries(row)) {
+            if (key !== idColumn && value) {
+                properties[key] = value;
+            }
+        }
+        companyRows.push({ id, domain, properties });
+    }
+    onProgress?.(0);
+    await batchInsertCompanies(companyRows);
+    onProgress?.(companyRows.length);
+    return companyRows.length;
+}
+export async function loadDealsCsv(csvData, onProgress) {
+    await clearTable("deals");
+    const rows = parseCSV(csvData);
+    if (rows.length === 0)
+        return 0;
+    const headers = Object.keys(rows[0]);
+    const idColumn = findIdColumn(headers);
+    if (!idColumn) {
+        throw new Error(`Could not find ID column in deals CSV. Available columns: ${headers.slice(0, 20).join(", ")}${headers.length > 20 ? "..." : ""}`);
+    }
+    const dealRows = [];
+    for (const row of rows) {
+        const id = row[idColumn];
+        if (!id)
+            continue;
+        const dealname = row.dealname || row["Deal Name"] || row.Dealname || null;
+        const properties = {};
+        for (const [key, value] of Object.entries(row)) {
+            if (key !== idColumn && value) {
+                properties[key] = value;
+            }
+        }
+        dealRows.push({ id, dealname, properties });
+    }
+    onProgress?.(0);
+    await batchInsertDeals(dealRows);
+    onProgress?.(dealRows.length);
+    return dealRows.length;
+}
+export async function loadAssociationsCsv(csvData, associationType, onProgress) {
+    await clearTable(associationType);
+    const rows = parseCSV(csvData);
+    if (rows.length === 0)
+        return 0;
+    const headers = Object.keys(rows[0]);
+    const idColumn = findIdColumn(headers);
+    if (!idColumn) {
+        throw new Error(`Could not find ID column in ${associationType} association CSV`);
+    }
+    const associatedIdColumn = headers.find((h) => h.toLowerCase().includes("associated") &&
+        (h.toLowerCase().includes("id") || h.toLowerCase().includes("record")));
+    if (!associatedIdColumn) {
+        throw new Error(`Could not find associated IDs column in ${associationType} CSV`);
+    }
+    const associationRows = [];
+    for (const row of rows) {
+        const fromId = row[idColumn];
+        if (!fromId)
+            continue;
+        const associatedIdsRaw = row[associatedIdColumn];
+        if (!associatedIdsRaw)
+            continue;
+        const associatedIds = associatedIdsRaw
+            .split(/[;,]/)
+            .map((id) => id.trim())
+            .filter(Boolean);
+        for (const toId of associatedIds) {
+            associationRows.push({ fromId, toId });
+        }
+    }
+    onProgress?.(0);
+    await batchInsertAssociations(associationType, associationRows);
+    onProgress?.(associationRows.length);
+    return associationRows.length;
+}
+export async function loadAllData(objects, associations, onProgress) {
+    const now = new Date().toISOString();
+    onProgress?.("Loading contacts...");
+    const contactsCount = await loadContactsCsv(objects.contacts, (n) => onProgress?.(`Contacts: ${n} loaded`));
+    await setMetadata("contacts_count", String(contactsCount));
+    onProgress?.("Loading companies...");
+    const companiesCount = await loadCompaniesCsv(objects.companies, (n) => onProgress?.(`Companies: ${n} loaded`));
+    await setMetadata("companies_count", String(companiesCount));
+    onProgress?.("Loading deals...");
+    const dealsCount = await loadDealsCsv(objects.deals, (n) => onProgress?.(`Deals: ${n} loaded`));
+    await setMetadata("deals_count", String(dealsCount));
+    const associationCounts = {
+        contact_company: 0,
+        deal_contact: 0,
+        deal_company: 0,
+    };
+    for (const [assocType, csvData] of Object.entries(associations)) {
+        const typedAssocType = assocType;
+        onProgress?.(`Loading ${assocType} associations...`);
+        associationCounts[typedAssocType] = await loadAssociationsCsv(csvData, typedAssocType, (n) => onProgress?.(`${assocType}: ${n} loaded`));
+        await setMetadata(`${assocType}_count`, String(associationCounts[typedAssocType]));
+    }
+    await setMetadata("last_synced", now);
+    await setMetadata("initialized_at", now);
+    return {
+        contacts: contactsCount,
+        companies: companiesCount,
+        deals: dealsCount,
+        associations: associationCounts,
+    };
+}

package/dist/sync/csv-worker.d.ts

@@ -0,0 +1,21 @@
+export type CsvWorkerMessage = {
+    type: "start";
+    filePath: string;
+    batchSize?: number;
+} | {
+    type: "cancel";
+};
+export type CsvWorkerResponse = {
+    type: "batch";
+    rows: Record<string, string>[];
+    processedCount: number;
+} | {
+    type: "headers";
+    headers: string[];
+} | {
+    type: "done";
+    totalCount: number;
+} | {
+    type: "error";
+    error: string;
+};

package/dist/sync/csv-worker.js

@@ -0,0 +1,60 @@
+import { createReadStream } from "node:fs";
+import { parse } from "csv-parse";
+let cancelled = false;
+self.onmessage = async (event) => {
+    const msg = event.data;
+    if (msg.type === "cancel") {
+        cancelled = true;
+        return;
+    }
+    if (msg.type === "start") {
+        cancelled = false;
+        await processFile(msg.filePath, msg.batchSize ?? 5000);
+    }
+};
+async function processFile(filePath, batchSize) {
+    try {
+        const parser = createReadStream(filePath).pipe(parse({
+            columns: true,
+            skip_empty_lines: true,
+            relax_column_count: true,
+            trim: true,
+        }));
+        let batch = [];
+        let totalProcessed = 0;
+        let headersSent = false;
+        for await (const record of parser) {
+            if (cancelled) {
+                self.postMessage({ type: "done", totalCount: totalProcessed });
+                return;
+            }
+            if (!headersSent) {
+                self.postMessage({ type: "headers", headers: Object.keys(record) });
+                headersSent = true;
+            }
+            batch.push(record);
+            if (batch.length >= batchSize) {
+                totalProcessed += batch.length;
+                self.postMessage({
+                    type: "batch",
+                    rows: batch,
+                    processedCount: totalProcessed,
+                });
+                batch = [];
+            }
+        }
+        if (batch.length > 0) {
+            totalProcessed += batch.length;
+            self.postMessage({
+                type: "batch",
+                rows: batch,
+                processedCount: totalProcessed,
+            });
+        }
+        self.postMessage({ type: "done", totalCount: totalProcessed });
+    }
+    catch (err) {
+        const errorMessage = err instanceof Error ? err.message : String(err);
+        self.postMessage({ type: "error", error: errorMessage });
+    }
+}

package/dist/sync/export-api.d.ts

@@ -0,0 +1,57 @@
+import type { AssociationType } from "../db/schema.js";
+export declare function getCachedCsv(name: string): string | null;
+export declare function saveCachedCsv(name: string, data: string): void;
+export declare function hasCachedExports(): boolean;
+export type ExportObjectType = "contacts" | "companies" | "deals";
+export interface ExportStatus {
+    id: string;
+    status: "PENDING" | "PROCESSING" | "COMPLETE" | "FAILED";
+    result?: string;
+    message?: string;
+}
+export interface AssociationExportConfig {
+    fromObjectType: string;
+    toObjectType: string;
+    associationType: AssociationType;
+}
+declare const ASSOCIATION_CONFIGS: AssociationExportConfig[];
+export declare function fetchObjectProperties(token: string, objectType: ExportObjectType): Promise<string[]>;
+export declare function startObjectExport(token: string, objectType: ExportObjectType, properties: string[]): Promise<string>;
+export declare function startAssociationExport(token: string, fromObjectType: string, toObjectType: string): Promise<string>;
+export declare function getExportStatus(token: string, exportId: string): Promise<ExportStatus>;
+export interface ExportTask {
+    id: string;
+    status: "PENDING" | "PROCESSING" | "COMPLETE" | "FAILED" | "CANCELED";
+    objectType: string;
+    exportName: string;
+    createdAt: string;
+}
+export declare function listExportTasks(token: string): Promise<ExportTask[]>;
+export interface ReusableExport {
+    exportName: string;
+    exportId: string;
+    createdAt: Date;
+}
+export declare function findReusableExports(token: string, maxAgeDays?: number): Promise<Map<string, ReusableExport>>;
+export declare function pollExportUntilComplete(token: string, exportId: string, onProgress?: (status: string) => void, maxWaitMs?: number, // 5 minutes
+pollIntervalMs?: number): Promise<string>;
+export declare function downloadExportCsv(token: string, downloadUrl: string): Promise<string>;
+export declare function downloadExportCsvToFile(token: string, downloadUrl: string, exportName: string): Promise<string>;
+export interface ExportJob {
+    type: "object" | "association";
+    objectType?: ExportObjectType;
+    associationConfig?: AssociationExportConfig;
+    exportId?: string;
+    status: "pending" | "exporting" | "downloading" | "complete" | "failed";
+    csvData?: string;
+    error?: string;
+}
+export declare function runAllExports(token: string, onProgress?: (message: string) => void, useCache?: boolean): Promise<{
+    objects: Record<ExportObjectType, string>;
+    associations: Record<AssociationType, string>;
+}>;
+export declare function validateToken(token: string): Promise<{
+    valid: boolean;
+    error?: string;
+}>;
+export { ASSOCIATION_CONFIGS };