n8n 2.23.2 → 2.24.0

package/dist/modules/agents/tools/knowledge/csv-helpers.js

@@ -0,0 +1,341 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CSV_MAX_AGGREGATE_GROUPS = exports.CSV_DISTINCT_TRACK_LIMIT = exports.CSV_PROFILE_DISTINCT_LIMIT = exports.CSV_SAMPLE_VALUE_LIMIT = void 0;
+exports.resolveCsvFile = resolveCsvFile;
+exports.streamCsvRecords = streamCsvRecords;
+exports.validateCsvColumns = validateCsvColumns;
+exports.matchesFilters = matchesFilters;
+exports.normaliseCsvValue = normaliseCsvValue;
+exports.toCsvRecordValues = toCsvRecordValues;
+exports.createCsvDistinctTracker = createCsvDistinctTracker;
+exports.buildCsvAmbiguity = buildCsvAmbiguity;
+exports.getSuggestedDisambiguatingColumns = getSuggestedDisambiguatingColumns;
+exports.createCsvColumnProfileState = createCsvColumnProfileState;
+exports.getLikelyDisambiguatingColumns = getLikelyDisambiguatingColumns;
+exports.createCsvAggregateGroup = createCsvAggregateGroup;
+exports.formatCsvAggregateGroup = formatCsvAggregateGroup;
+exports.sortCsvAggregateResults = sortCsvAggregateResults;
+const node_fs_1 = require("node:fs");
+const node_path_1 = __importDefault(require("node:path"));
+const fastest_levenshtein_1 = require("fastest-levenshtein");
+const file_references_1 = require("./file-references");
+exports.CSV_SAMPLE_VALUE_LIMIT = 5;
+exports.CSV_PROFILE_DISTINCT_LIMIT = 100;
+exports.CSV_DISTINCT_TRACK_LIMIT = 10_000;
+exports.CSV_MAX_AGGREGATE_GROUPS = 50_000;
+const CSV_OPERATION_TIMEOUT_MS = 15_000;
+function isCsvFile(file) {
+    return file.mimeType === 'text/csv' || file.relativePath.toLowerCase().endsWith('.csv');
+}
+function resolveCsvFile(files, reference) {
+    const resolvedFile = (0, file_references_1.resolveFileReference)(files, reference);
+    if (resolvedFile.status !== 'found') {
+        throw new Error(resolvedFile.error);
+    }
+    const { file } = resolvedFile;
+    if (!isCsvFile(file)) {
+        throw new Error(`File "${file.fileName}" is not queryable as CSV.`);
+    }
+    return file;
+}
+async function streamCsvRecords(workspaceRoot, file, handlers) {
+    const filePath = node_path_1.default.join(workspaceRoot, file.relativePath);
+    const { parse } = await Promise.resolve().then(() => __importStar(require('csv-parse')));
+    const readStream = (0, node_fs_1.createReadStream)(filePath);
+    const parser = readStream.pipe(parse({
+        columns: (parsedHeaders) => {
+            handlers.onHeaders?.(parsedHeaders);
+            return parsedHeaders;
+        },
+        skip_empty_lines: true,
+        bom: true,
+        info: true,
+        relax_column_count: true,
+    }));
+    const timeout = setTimeout(() => {
+        parser.destroy(new Error('CSV operation exceeded the time limit'));
+        readStream.destroy();
+    }, CSV_OPERATION_TIMEOUT_MS);
+    try {
+        for await (const { record, info } of parser) {
+            handlers.onRecord({ record, fileLineNumber: info.lines });
+        }
+    }
+    finally {
+        clearTimeout(timeout);
+        readStream.destroy();
+        parser.destroy();
+    }
+}
+function validateCsvColumns(headers, fileName, columns) {
+    for (const column of columns) {
+        if (!headers.includes(column)) {
+            throw new Error(formatMissingCsvColumnError(fileName, column, headers));
+        }
+    }
+}
+function matchesFilters(record, filters) {
+    return filters.every((filter) => {
+        const value = normaliseCsvValue(record[filter.column]);
+        if (filter.op === 'eq')
+            return value === filter.value;
+        if (filter.op === 'contains')
+            return value.includes(filter.value);
+        return filter.value.includes(value);
+    });
+}
+function normaliseCsvValue(value) {
+    if (value === null || value === undefined)
+        return '';
+    return String(value);
+}
+function toCsvRecordValues(record, columns) {
+    return Object.fromEntries(columns.map((column) => [column, normaliseCsvValue(record[column])]));
+}
+function formatMissingCsvColumnError(fileName, requestedColumn, headers) {
+    const suggestions = getClosestColumnMatches(requestedColumn, headers);
+    const didYouMean = suggestions.length > 0
+        ? ` Did you mean ${suggestions.map((value) => `"${value}"`).join(', ')}?`
+        : '';
+    return `CSV column "${requestedColumn}" not found in "${fileName}". Available columns: ${headers.join(', ')}.${didYouMean} Run csv_profile if you are uncertain about the schema.`;
+}
+function getClosestColumnMatches(requestedColumn, headers) {
+    const requested = requestedColumn.toLowerCase();
+    return headers
+        .map((header) => ({ header, distance: (0, fastest_levenshtein_1.distance)(requested, header.toLowerCase()) }))
+        .filter(({ header, distance: editDistance }) => {
+        return header.toLowerCase().includes(requested) || editDistance <= 3;
+    })
+        .sort((left, right) => left.distance - right.distance || left.header.localeCompare(right.header))
+        .slice(0, 3)
+        .map(({ header }) => header);
+}
+function createCsvDistinctTracker(columns, limit) {
+    const values = new Map(columns.map((column) => [column, new Set()]));
+    return {
+        add(record) {
+            for (const [column, distinctValues] of values) {
+                if (distinctValues.size < limit)
+                    distinctValues.add(normaliseCsvValue(record[column]));
+            }
+        },
+        toOutput() {
+            return Object.fromEntries(Array.from(values.entries()).flatMap(([column, distinctValues]) => distinctValues.size > 0 ? [[column, Array.from(distinctValues)]] : []));
+        },
+        columns: Array.from(values.keys()),
+    };
+}
+function buildCsvAmbiguity(matchedRows, limit, tracker) {
+    return {
+        matchedRows,
+        message: matchedRows > limit
+            ? `Matched ${matchedRows} rows and returned only the first ${limit}. This is not a unique result. Refine filters before answering.`
+            : `Matched ${matchedRows} rows. This is not a unique result. Refine filters before answering.`,
+        suggestedColumns: tracker?.columns ?? [],
+        sampleDistinctValues: tracker?.toOutput(),
+    };
+}
+function getSuggestedDisambiguatingColumns(headers, filters, selectedColumns) {
+    const alreadyUsed = new Set([...filters.map((filter) => filter.column), ...selectedColumns]);
+    return headers
+        .filter((header) => !alreadyUsed.has(header))
+        .sort((left, right) => preferenceScore(left) - preferenceScore(right))
+        .slice(0, 5);
+}
+const PREFERRED_DISAMBIGUATING_COLUMNS = [
+    'Year',
+    'Date',
+    'Month',
+    'Country',
+    'Country Name',
+    'Source',
+    'Category',
+    'Name',
+];
+function preferenceScore(column) {
+    const exactIndex = PREFERRED_DISAMBIGUATING_COLUMNS.findIndex((candidate) => candidate.toLowerCase() === column.toLowerCase());
+    if (exactIndex !== -1)
+        return exactIndex;
+    const partialIndex = PREFERRED_DISAMBIGUATING_COLUMNS.findIndex((candidate) => column.toLowerCase().includes(candidate.toLowerCase()));
+    return partialIndex === -1 ? PREFERRED_DISAMBIGUATING_COLUMNS.length + 1 : partialIndex + 0.5;
+}
+function createCsvColumnProfileState(distinctLimit) {
+    const distinctValues = new Set();
+    const sampleValues = [];
+    let distinctCountTruncated = false;
+    let emptyCount = 0;
+    let nonEmptyCount = 0;
+    let allInteger = true;
+    let allNumber = true;
+    let allBoolean = true;
+    let allDate = true;
+    return {
+        add(value) {
+            if (value === '') {
+                emptyCount++;
+                return;
+            }
+            nonEmptyCount++;
+            if (distinctValues.size < distinctLimit) {
+                distinctValues.add(value);
+            }
+            else if (!distinctValues.has(value)) {
+                distinctCountTruncated = true;
+            }
+            if (!sampleValues.includes(value) && sampleValues.length < exports.CSV_SAMPLE_VALUE_LIMIT) {
+                sampleValues.push(value);
+            }
+            allInteger &&= /^-?\d+$/.test(value);
+            allNumber &&= Number.isFinite(Number(value));
+            allBoolean &&= /^(true|false|yes|no|0|1)$/i.test(value);
+            allDate &&= isLikelyDate(value);
+        },
+        toOutput(name) {
+            return {
+                name,
+                inferredType: inferCsvColumnType({
+                    nonEmptyCount,
+                    allInteger,
+                    allNumber,
+                    allBoolean,
+                    allDate,
+                }),
+                emptyCount,
+                distinctCount: distinctValues.size,
+                distinctCountTruncated,
+                sampleValues,
+            };
+        },
+    };
+}
+function inferCsvColumnType({ nonEmptyCount, allInteger, allNumber, allBoolean, allDate, }) {
+    if (nonEmptyCount === 0)
+        return 'empty';
+    if (allBoolean)
+        return 'boolean';
+    if (allInteger)
+        return 'integer';
+    if (allNumber)
+        return 'number';
+    if (allDate)
+        return 'date';
+    return 'string';
+}
+function isLikelyDate(value) {
+    if (!/^\d{4}[-/]\d{1,2}([-/]\d{1,2})?$/.test(value))
+        return false;
+    return Number.isFinite(Date.parse(value));
+}
+function getLikelyDisambiguatingColumns(columnProfiles, rowCount) {
+    return columnProfiles
+        .filter((column) => {
+        const distinctCount = column.distinctCount ?? 0;
+        return distinctCount > 1 && distinctCount < rowCount && !column.distinctCountTruncated;
+    })
+        .sort((left, right) => preferenceScore(left.name) - preferenceScore(right.name))
+        .slice(0, 5)
+        .map((column) => column.name);
+}
+function createCsvAggregateGroup(groupValues, metrics) {
+    return {
+        groupValues,
+        count: 0,
+        metrics: Object.fromEntries(metrics.map((metric) => [metric, createNumericAggregateState()])),
+    };
+}
+function createNumericAggregateState() {
+    return {
+        count: 0,
+        sum: 0,
+        min: undefined,
+        max: undefined,
+        skipped: 0,
+        add(value) {
+            const trimmedValue = value.trim();
+            const numericValue = Number(trimmedValue);
+            if (trimmedValue === '' || !Number.isFinite(numericValue)) {
+                this.skipped++;
+                return;
+            }
+            this.count++;
+            this.sum += numericValue;
+            this.min = this.min === undefined ? numericValue : Math.min(this.min, numericValue);
+            this.max = this.max === undefined ? numericValue : Math.max(this.max, numericValue);
+        },
+    };
+}
+function formatCsvAggregateGroup(group, functions, metrics) {
+    const output = { ...group.groupValues };
+    for (const fn of functions) {
+        if (fn === 'count')
+            output.count = group.count;
+    }
+    for (const metric of metrics) {
+        const state = group.metrics[metric];
+        for (const fn of functions) {
+            if (fn === 'min')
+                output[`min_${metric}`] = state.min ?? null;
+            if (fn === 'max')
+                output[`max_${metric}`] = state.max ?? null;
+            if (fn === 'sum')
+                output[`sum_${metric}`] = state.count > 0 ? state.sum : null;
+            if (fn === 'avg')
+                output[`avg_${metric}`] = state.count > 0 ? state.sum / state.count : null;
+        }
+    }
+    return output;
+}
+function sortCsvAggregateResults(results, orderBy) {
+    if (!orderBy)
+        return;
+    const direction = orderBy.direction === 'desc' ? -1 : 1;
+    results.sort((left, right) => compareCsvAggregateValues(left[orderBy.column], right[orderBy.column]) * direction);
+}
+function compareCsvAggregateValues(left, right) {
+    if (left === right)
+        return 0;
+    if (left === null || left === undefined)
+        return 1;
+    if (right === null || right === undefined)
+        return -1;
+    if (typeof left === 'number' && typeof right === 'number')
+        return left - right;
+    return String(left).localeCompare(String(right));
+}
+//# sourceMappingURL=csv-helpers.js.map

package/dist/modules/agents/tools/knowledge/csv-helpers.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"csv-helpers.js","sourceRoot":"","sources":["../../../../../src/modules/agents/tools/knowledge/csv-helpers.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAoBA,wCAUC;AAED,4CAyCC;AAED,gDAMC;AAED,wCAOC;AAED,8CAGC;AAED,8CAEC;AA2BD,4DAiBC;AAED,8CAcC;AAED,8EAUC;AA+BD,kEA+CC;AA8BD,wEAgBC;AAED,0DAMC;AA0BD,0DAmBC;AAED,0DAUC;AAxWD,qCAA2C;AAC3C,0DAA6B;AAE7B,6DAA+C;AAE/C,uDAA8E;AAGjE,QAAA,sBAAsB,GAAG,CAAC,CAAC;AAC3B,QAAA,0BAA0B,GAAG,GAAG,CAAC;AACjC,QAAA,wBAAwB,GAAG,MAAM,CAAC;AAElC,QAAA,wBAAwB,GAAG,MAAM,CAAC;AAE/C,MAAM,wBAAwB,GAAG,MAAM,CAAC;AAExC,SAAS,SAAS,CAAC,IAA4B;IAC9C,OAAO,IAAI,CAAC,QAAQ,KAAK,UAAU,IAAI,IAAI,CAAC,YAAY,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;AACzF,CAAC;AAED,SAAgB,cAAc,CAAC,KAAqB,EAAE,SAAiB;IACtE,MAAM,YAAY,GAAG,IAAA,sCAAoB,EAAC,KAAK,EAAE,SAAS,CAAC,CAAC;IAC5D,IAAI,YAAY,CAAC,MAAM,KAAK,OAAO,EAAE,CAAC;QACrC,MAAM,IAAI,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC;IACrC,CAAC;IACD,MAAM,EAAE,IAAI,EAAE,GAAG,YAAY,CAAC;IAC9B,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC;QACtB,MAAM,IAAI,KAAK,CAAC,SAAS,IAAI,CAAC,QAAQ,4BAA4B,CAAC,CAAC;IACrE,CAAC;IACD,OAAO,IAAI,CAAC;AACb,CAAC;AAEM,KAAK,UAAU,gBAAgB,CACrC,aAAqB,EACrB,IAA4B,EAC5B,QAGC;IAED,MAAM,QAAQ,GAAG,mBAAI,CAAC,IAAI,CAAC,aAAa,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;IAC7D,MAAM,EAAE,KAAK,EAAE,GAAG,wDAAa,WAAW,GAAC,CAAC;IAC5C,MAAM,UAAU,GAAG,IAAA,0BAAgB,EAAC,QAAQ,CAAC,CAAC;IAC9C,MAAM,MAAM,GAAG,UAAU,CAAC,IAAI,CAC7B,KAAK,CAAC;QACL,OAAO,EAAE,CAAC,aAAuB,EAAE,EAAE;YACpC,QAAQ,CAAC,SAAS,EAAE,CAAC,aAAa,CAAC,CAAC;YACpC,OAAO,aAAa,CAAC;QACtB,CAAC;QACD,gBAAgB,EAAE,IAAI;QACtB,GAAG,EAAE,IAAI;QACT,IAAI,EAAE,IAAI;QACV,kBAAkB,EAAE,IAAI;KACxB,CAAC,CACF,CAAC;IAGF,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE;QAC/B,MAAM,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC,CAAC;QACnE,UAAU,CAAC,OAAO,EAAE,CAAC;IACtB,CAAC,EAAE,wBAAwB,CAAC,CAAC;IAC7B,IAAI,CAAC;QACJ,IAAI,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,MAGnC,EAAE,CAAC;YACJ,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,cAAc,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;QAC3D,CAAC;IACF,CAAC;YAAS,CAAC;QACV,YAAY,CAAC,OAAO,CAAC,CAAC;QACtB,UAAU,CAAC,OAAO,EAAE,CAAC;QACrB,MAAM,CAAC,OAAO,EAAE,CAAC;IAClB,CAAC;AACF,CAAC;AAED,SAAgB,kBAAkB,CAAC,OAAiB,EAAE,QAAgB,EAAE,OAAiB;IACxF,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC9B,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YAC/B,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;QACzE,CAAC;IACF,CAAC;AACF,CAAC;AAED,SAAgB,cAAc,CAAC,MAA+B,EAAE,OAAoB;IACnF,OAAO,OAAO,CAAC,KAAK,CAAC,CAAC,MAAM,EAAE,EAAE;QAC/B,MAAM,KAAK,GAAG,iBAAiB,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;QACvD,IAAI,MAAM,CAAC,EAAE,KAAK,IAAI;YAAE,OAAO,KAAK,KAAK,MAAM,CAAC,KAAK,CAAC;QACtD,IAAI,MAAM,CAAC,EAAE,KAAK,UAAU;YAAE,OAAO,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAClE,OAAO,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IACrC,CAAC,CAAC,CAAC;AACJ,CAAC;AAED,SAAgB,iBAAiB,CAAC,KAAc;IAC/C,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,EAAE,CAAC;IACrD,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;AACtB,CAAC;AAED,SAAgB,iBAAiB,CAAC,MAA+B,EAAE,OAAiB;IACnF,OAAO,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,iBAAiB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACjG,CAAC;AAED,SAAS,2BAA2B,CAAC,QAAgB,EAAE,eAAuB,EAAE,OAAiB;IAChG,MAAM,WAAW,GAAG,uBAAuB,CAAC,eAAe,EAAE,OAAO,CAAC,CAAC;IACtE,MAAM,UAAU,GACf,WAAW,CAAC,MAAM,GAAG,CAAC;QACrB,CAAC,CAAC,iBAAiB,WAAW,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG;QACzE,CAAC,CAAC,EAAE,CAAC;IACP,OAAO,eAAe,eAAe,mBAAmB,QAAQ,yBAAyB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,UAAU,yDAAyD,CAAC;AACpL,CAAC;AAED,SAAS,uBAAuB,CAAC,eAAuB,EAAE,OAAiB;IAC1E,MAAM,SAAS,GAAG,eAAe,CAAC,WAAW,EAAE,CAAC;IAChD,OAAO,OAAO;SACZ,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAA,8BAAQ,EAAC,SAAS,EAAE,MAAM,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC,CAAC;SAClF,MAAM,CAAC,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,YAAY,EAAE,EAAE,EAAE;QAC9C,OAAO,MAAM,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,YAAY,IAAI,CAAC,CAAC;IACtE,CAAC,CAAC;SACD,IAAI,CACJ,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC,QAAQ,IAAI,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,MAAM,CAAC,CAC1F;SACA,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;SACX,GAAG,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;AAC/B,CAAC;AAID,SAAgB,wBAAwB,CAAC,OAAiB,EAAE,KAAa;IACxE,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,IAAI,GAAG,EAAU,CAAC,CAAC,CAAC,CAAC;IAC7E,OAAO;QACN,GAAG,CAAC,MAA+B;YAClC,KAAK,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,IAAI,MAAM,EAAE,CAAC;gBAC/C,IAAI,cAAc,CAAC,IAAI,GAAG,KAAK;oBAAE,cAAc,CAAC,GAAG,CAAC,iBAAiB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YACxF,CAAC;QACF,CAAC;QACD,QAAQ;YACP,OAAO,MAAM,CAAC,WAAW,CACxB,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,EAAE,cAAc,CAAC,EAAE,EAAE,CACjE,cAAc,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CACrE,CACD,CAAC;QACH,CAAC;QACD,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;KAClC,CAAC;AACH,CAAC;AAED,SAAgB,iBAAiB,CAChC,WAAmB,EACnB,KAAa,EACb,OAAuC;IAEvC,OAAO;QACN,WAAW;QACX,OAAO,EACN,WAAW,GAAG,KAAK;YAClB,CAAC,CAAC,WAAW,WAAW,qCAAqC,KAAK,iEAAiE;YACnI,CAAC,CAAC,WAAW,WAAW,sEAAsE;QAChG,gBAAgB,EAAE,OAAO,EAAE,OAAO,IAAI,EAAE;QACxC,oBAAoB,EAAE,OAAO,EAAE,QAAQ,EAAE;KACzC,CAAC;AACH,CAAC;AAED,SAAgB,iCAAiC,CAChD,OAAiB,EACjB,OAAoB,EACpB,eAAyB;IAEzB,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,GAAG,eAAe,CAAC,CAAC,CAAC;IAC7F,OAAO,OAAO;SACZ,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;SAC5C,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,eAAe,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;SACrE,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AACf,CAAC;AAMD,MAAM,gCAAgC,GAAG;IACxC,MAAM;IACN,MAAM;IACN,OAAO;IACP,SAAS;IACT,cAAc;IACd,QAAQ;IACR,UAAU;IACV,MAAM;CACN,CAAC;AAEF,SAAS,eAAe,CAAC,MAAc;IACtC,MAAM,UAAU,GAAG,gCAAgC,CAAC,SAAS,CAC5D,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,WAAW,EAAE,KAAK,MAAM,CAAC,WAAW,EAAE,CAC/D,CAAC;IACF,IAAI,UAAU,KAAK,CAAC,CAAC;QAAE,OAAO,UAAU,CAAC;IACzC,MAAM,YAAY,GAAG,gCAAgC,CAAC,SAAS,CAAC,CAAC,SAAS,EAAE,EAAE,CAC7E,MAAM,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,WAAW,EAAE,CAAC,CACtD,CAAC;IACF,OAAO,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,gCAAgC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,GAAG,CAAC;AAC/F,CAAC;AAKD,SAAgB,2BAA2B,CAAC,aAAqB;IAChE,MAAM,cAAc,GAAG,IAAI,GAAG,EAAU,CAAC;IACzC,MAAM,YAAY,GAAa,EAAE,CAAC;IAClC,IAAI,sBAAsB,GAAG,KAAK,CAAC;IACnC,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,UAAU,GAAG,IAAI,CAAC;IACtB,IAAI,SAAS,GAAG,IAAI,CAAC;IACrB,IAAI,UAAU,GAAG,IAAI,CAAC;IACtB,IAAI,OAAO,GAAG,IAAI,CAAC;IACnB,OAAO;QACN,GAAG,CAAC,KAAa;YAChB,IAAI,KAAK,KAAK,EAAE,EAAE,CAAC;gBAClB,UAAU,EAAE,CAAC;gBACb,OAAO;YACR,CAAC;YACD,aAAa,EAAE,CAAC;YAChB,IAAI,cAAc,CAAC,IAAI,GAAG,aAAa,EAAE,CAAC;gBACzC,cAAc,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YAC3B,CAAC;iBAAM,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;gBACvC,sBAAsB,GAAG,IAAI,CAAC;YAC/B,CAAC;YACD,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,YAAY,CAAC,MAAM,GAAG,8BAAsB,EAAE,CAAC;gBACnF,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC1B,CAAC;YACD,UAAU,KAAK,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACrC,SAAS,KAAK,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YAC7C,UAAU,KAAK,4BAA4B,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACxD,OAAO,KAAK,YAAY,CAAC,KAAK,CAAC,CAAC;QACjC,CAAC;QACD,QAAQ,CAAC,IAAY;YACpB,OAAO;gBACN,IAAI;gBACJ,YAAY,EAAE,kBAAkB,CAAC;oBAChC,aAAa;oBACb,UAAU;oBACV,SAAS;oBACT,UAAU;oBACV,OAAO;iBACP,CAAC;gBACF,UAAU;gBACV,aAAa,EAAE,cAAc,CAAC,IAAI;gBAClC,sBAAsB;gBACtB,YAAY;aACZ,CAAC;QACH,CAAC;KACD,CAAC;AACH,CAAC;AAID,SAAS,kBAAkB,CAAC,EAC3B,aAAa,EACb,UAAU,EACV,SAAS,EACT,UAAU,EACV,OAAO,GAOP;IACA,IAAI,aAAa,KAAK,CAAC;QAAE,OAAO,OAAO,CAAC;IACxC,IAAI,UAAU;QAAE,OAAO,SAAS,CAAC;IACjC,IAAI,UAAU;QAAE,OAAO,SAAS,CAAC;IACjC,IAAI,SAAS;QAAE,OAAO,QAAQ,CAAC;IAC/B,IAAI,OAAO;QAAE,OAAO,MAAM,CAAC;IAC3B,OAAO,QAAQ,CAAC;AACjB,CAAC;AAED,SAAS,YAAY,CAAC,KAAa;IAClC,IAAI,CAAC,kCAAkC,CAAC,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAClE,OAAO,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;AAC3C,CAAC;AAED,SAAgB,8BAA8B,CAC7C,cAIE,EACF,QAAgB;IAEhB,OAAO,cAAc;SACnB,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE;QAClB,MAAM,aAAa,GAAG,MAAM,CAAC,aAAa,IAAI,CAAC,CAAC;QAChD,OAAO,aAAa,GAAG,CAAC,IAAI,aAAa,GAAG,QAAQ,IAAI,CAAC,MAAM,CAAC,sBAAsB,CAAC;IACxF,CAAC,CAAC;SACD,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;SAC/E,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;SACX,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;AAChC,CAAC;AAED,SAAgB,uBAAuB,CAAC,WAAmC,EAAE,OAAiB;IAC7F,OAAO;QACN,WAAW;QACX,KAAK,EAAE,CAAC;QACR,OAAO,EAAE,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,2BAA2B,EAAE,CAAC,CAAC,CAAC;KAC7F,CAAC;AACH,CAAC;AAID,SAAS,2BAA2B;IACnC,OAAO;QACN,KAAK,EAAE,CAAC;QACR,GAAG,EAAE,CAAC;QACN,GAAG,EAAE,SAA+B;QACpC,GAAG,EAAE,SAA+B;QACpC,OAAO,EAAE,CAAC;QACV,GAAG,CAAC,KAAa;YAChB,MAAM,YAAY,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;YAClC,MAAM,YAAY,GAAG,MAAM,CAAC,YAAY,CAAC,CAAC;YAC1C,IAAI,YAAY,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;gBAC3D,IAAI,CAAC,OAAO,EAAE,CAAC;gBACf,OAAO;YACR,CAAC;YACD,IAAI,CAAC,KAAK,EAAE,CAAC;YACb,IAAI,CAAC,GAAG,IAAI,YAAY,CAAC;YACzB,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,KAAK,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;YACpF,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,KAAK,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;QACrF,CAAC;KACD,CAAC;AACH,CAAC;AAED,SAAgB,uBAAuB,CACtC,KAAwB,EACxB,SAAyD,EACzD,OAAiB;IAEjB,MAAM,MAAM,GAA2C,EAAE,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC;IAChF,KAAK,MAAM,EAAE,IAAI,SAAS,EAAE,CAAC;QAC5B,IAAI,EAAE,KAAK,OAAO;YAAE,MAAM,CAAC,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC;IAChD,CAAC;IACD,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC9B,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QACpC,KAAK,MAAM,EAAE,IAAI,SAAS,EAAE,CAAC;YAC5B,IAAI,EAAE,KAAK,KAAK;gBAAE,MAAM,CAAC,OAAO,MAAM,EAAE,CAAC,GAAG,KAAK,CAAC,GAAG,IAAI,IAAI,CAAC;YAC9D,IAAI,EAAE,KAAK,KAAK;gBAAE,MAAM,CAAC,OAAO,MAAM,EAAE,CAAC,GAAG,KAAK,CAAC,GAAG,IAAI,IAAI,CAAC;YAC9D,IAAI,EAAE,KAAK,KAAK;gBAAE,MAAM,CAAC,OAAO,MAAM,EAAE,CAAC,GAAG,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC;YAC/E,IAAI,EAAE,KAAK,KAAK;gBAAE,MAAM,CAAC,OAAO,MAAM,EAAE,CAAC,GAAG,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;QAC9F,CAAC;IACF,CAAC;IACD,OAAO,MAAM,CAAC;AACf,CAAC;AAED,SAAgB,uBAAuB,CACtC,OAAsD,EACtD,OAAqC;IAErC,IAAI,CAAC,OAAO;QAAE,OAAO;IACrB,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,KAAK,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACxD,OAAO,CAAC,IAAI,CACX,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CACf,yBAAyB,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,SAAS,CACnF,CAAC;AACH,CAAC;AAED,SAAS,yBAAyB,CACjC,IAAwC,EACxC,KAAyC;IAEzC,IAAI,IAAI,KAAK,KAAK;QAAE,OAAO,CAAC,CAAC;IAC7B,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO,CAAC,CAAC;IAClD,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,CAAC,CAAC,CAAC;IACrD,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,IAAI,GAAG,KAAK,CAAC;IAC/E,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;AAClD,CAAC"}

package/dist/modules/agents/tools/knowledge/csv.operation.d.ts

@@ -0,0 +1,61 @@
+import type { CsvAggregateInput, CsvDistinctInput, CsvProfileInput, CsvQueryInput } from './schemas';
+import type { WorkspaceFiles } from './file-references';
+export declare function queryCsv(workspaceRoot: string, files: WorkspaceFiles, input: CsvQueryInput): Promise<{
+    fileName: string;
+    relativePath: string;
+    columns: string[];
+    rowNumbers: number[];
+    rows: string[][];
+    records: {
+        rowNumber: number;
+        fileLineNumber: number;
+        values: Record<string, string>;
+    }[];
+    rowCount: number;
+    truncated: boolean;
+    rowNumberBase: string;
+    ambiguity: {
+        matchedRows: number;
+        message: string;
+        suggestedColumns: string[];
+        sampleDistinctValues: {
+            [k: string]: string[];
+        } | undefined;
+    } | undefined;
+}>;
+export declare function profileCsv(workspaceRoot: string, files: WorkspaceFiles, input: CsvProfileInput): Promise<{
+    fileName: string;
+    relativePath: string;
+    columns: string[];
+    rowCount: number;
+    sampleRows: Record<string, string>[];
+    columnProfiles: {
+        name: string;
+        inferredType: "string" | "number" | "boolean" | "empty" | "date" | "integer";
+        emptyCount: number;
+        distinctCount: number;
+        distinctCountTruncated: boolean;
+        sampleValues: string[];
+    }[];
+    likelyKeyColumns: string[];
+    likelyDisambiguatingColumns: string[];
+}>;
+export declare function distinctCsv(workspaceRoot: string, files: WorkspaceFiles, input: CsvDistinctInput): Promise<{
+    fileName: string;
+    relativePath: string;
+    column: string;
+    values: string[];
+    distinctCount: number;
+    truncated: boolean;
+}>;
+export declare function aggregateCsv(workspaceRoot: string, files: WorkspaceFiles, input: CsvAggregateInput): Promise<{
+    fileName: string;
+    relativePath: string;
+    rowCount: number;
+    functions: ("max" | "min" | "count" | "sum" | "avg")[];
+    metrics: string[];
+    groupBy: string[] | undefined;
+    results: Record<string, string | number | null>[];
+    truncated: boolean;
+    skippedNonNumeric: Record<string, number>;
+}>;

package/dist/modules/agents/tools/knowledge/csv.operation.js

@@ -0,0 +1,207 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.queryCsv = queryCsv;
+exports.profileCsv = profileCsv;
+exports.distinctCsv = distinctCsv;
+exports.aggregateCsv = aggregateCsv;
+const csv_helpers_1 = require("./csv-helpers");
+async function queryCsv(workspaceRoot, files, input) {
+    const file = (0, csv_helpers_1.resolveCsvFile)(files, input.file);
+    const headers = [];
+    const limit = input.limit ?? 20;
+    const select = input.select;
+    const rows = [];
+    const rowNumbers = [];
+    const records = [];
+    let ambiguityTracker;
+    let matched = 0;
+    if (input.rowNumber === undefined && select === undefined) {
+        throw new Error('csv_query requires select unless rowNumber is provided.');
+    }
+    await (0, csv_helpers_1.streamCsvRecords)(workspaceRoot, file, {
+        onHeaders: (parsedHeaders) => {
+            headers.push(...parsedHeaders);
+            (0, csv_helpers_1.validateCsvColumns)(headers, file.fileName, [
+                ...(select ?? []),
+                ...(input.where ?? []).map((filter) => filter.column),
+            ]);
+            ambiguityTracker = (0, csv_helpers_1.createCsvDistinctTracker)((0, csv_helpers_1.getSuggestedDisambiguatingColumns)(headers, input.where ?? [], select ?? []), csv_helpers_1.CSV_SAMPLE_VALUE_LIMIT);
+        },
+        onRecord: ({ record, fileLineNumber }) => {
+            if (input.rowNumber !== undefined && fileLineNumber !== input.rowNumber)
+                return;
+            if (input.rowNumber === undefined && !(0, csv_helpers_1.matchesFilters)(record, input.where ?? []))
+                return;
+            matched++;
+            ambiguityTracker?.add(record);
+            const columns = select ?? headers;
+            if (rows.length < limit) {
+                const values = (0, csv_helpers_1.toCsvRecordValues)(record, columns);
+                rows.push(columns.map((column) => values[column]));
+                rowNumbers.push(fileLineNumber);
+                records.push({ rowNumber: fileLineNumber, fileLineNumber, values });
+            }
+        },
+    });
+    if (headers.length === 0)
+        (0, csv_helpers_1.validateCsvColumns)(headers, file.fileName, select ?? []);
+    const columns = select ?? headers;
+    const truncated = matched > rows.length;
+    return {
+        fileName: file.fileName,
+        relativePath: file.relativePath,
+        columns,
+        rowNumbers,
+        rows,
+        records,
+        rowCount: matched,
+        truncated,
+        rowNumberBase: 'rowNumber is the CSV file line number; line 1 is the header row.',
+        ambiguity: input.rowNumber === undefined && (matched > 1 || truncated)
+            ? (0, csv_helpers_1.buildCsvAmbiguity)(matched, input.limit ?? 20, ambiguityTracker)
+            : undefined,
+    };
+}
+async function profileCsv(workspaceRoot, files, input) {
+    const file = (0, csv_helpers_1.resolveCsvFile)(files, input.file);
+    const headers = [];
+    const sampleRows = [];
+    const rowCountByColumn = new Map();
+    let rowCount = 0;
+    const distinctLimit = input.distinctLimit ?? csv_helpers_1.CSV_PROFILE_DISTINCT_LIMIT;
+    await (0, csv_helpers_1.streamCsvRecords)(workspaceRoot, file, {
+        onHeaders: (parsedHeaders) => {
+            headers.push(...parsedHeaders);
+            for (const header of headers) {
+                rowCountByColumn.set(header, (0, csv_helpers_1.createCsvColumnProfileState)(distinctLimit));
+            }
+        },
+        onRecord: ({ record }) => {
+            rowCount++;
+            if (sampleRows.length < (input.sampleSize ?? 5)) {
+                sampleRows.push((0, csv_helpers_1.toCsvRecordValues)(record, headers));
+            }
+            for (const header of headers) {
+                rowCountByColumn.get(header)?.add((0, csv_helpers_1.normaliseCsvValue)(record[header]));
+            }
+        },
+    });
+    const columnProfiles = headers.map((header) => {
+        const profile = rowCountByColumn.get(header) ?? (0, csv_helpers_1.createCsvColumnProfileState)(distinctLimit);
+        return profile.toOutput(header);
+    });
+    return {
+        fileName: file.fileName,
+        relativePath: file.relativePath,
+        columns: headers,
+        rowCount,
+        sampleRows,
+        columnProfiles,
+        likelyKeyColumns: columnProfiles
+            .filter((column) => column.distinctCount === rowCount && rowCount > 0)
+            .map((column) => column.name),
+        likelyDisambiguatingColumns: (0, csv_helpers_1.getLikelyDisambiguatingColumns)(columnProfiles, rowCount),
+    };
+}
+async function distinctCsv(workspaceRoot, files, input) {
+    const file = (0, csv_helpers_1.resolveCsvFile)(files, input.file);
+    const values = new Set();
+    let distinctTruncated = false;
+    const outputValues = [];
+    await (0, csv_helpers_1.streamCsvRecords)(workspaceRoot, file, {
+        onHeaders: (headers) => {
+            (0, csv_helpers_1.validateCsvColumns)(headers, file.fileName, [
+                input.column,
+                ...(input.where ?? []).map((filter) => filter.column),
+            ]);
+        },
+        onRecord: ({ record }) => {
+            if (!(0, csv_helpers_1.matchesFilters)(record, input.where ?? []))
+                return;
+            const value = (0, csv_helpers_1.normaliseCsvValue)(record[input.column]);
+            if (!values.has(value)) {
+                if (values.size < csv_helpers_1.CSV_DISTINCT_TRACK_LIMIT) {
+                    values.add(value);
+                }
+                else {
+                    distinctTruncated = true;
+                }
+                if (outputValues.length < (input.limit ?? 50))
+                    outputValues.push(value);
+            }
+        },
+    });
+    return {
+        fileName: file.fileName,
+        relativePath: file.relativePath,
+        column: input.column,
+        values: outputValues,
+        distinctCount: values.size,
+        truncated: distinctTruncated || values.size > outputValues.length,
+    };
+}
+async function aggregateCsv(workspaceRoot, files, input) {
+    const file = (0, csv_helpers_1.resolveCsvFile)(files, input.file);
+    const functions = input.functions ?? ['count'];
+    const metrics = Array.from(new Set([...(input.metric ? [input.metric] : []), ...(input.metrics ?? [])]));
+    const needsMetric = functions.some((fn) => fn !== 'count');
+    if (needsMetric && metrics.length === 0) {
+        throw new Error('csv_aggregate requires metric or metrics for min, max, sum, or avg.');
+    }
+    const groups = new Map();
+    let rowCount = 0;
+    let groupLimitReached = false;
+    await (0, csv_helpers_1.streamCsvRecords)(workspaceRoot, file, {
+        onHeaders: (headers) => {
+            (0, csv_helpers_1.validateCsvColumns)(headers, file.fileName, [
+                ...metrics,
+                ...(input.groupBy ?? []),
+                ...(input.where ?? []).map((filter) => filter.column),
+            ]);
+        },
+        onRecord: ({ record }) => {
+            if (!(0, csv_helpers_1.matchesFilters)(record, input.where ?? []))
+                return;
+            rowCount++;
+            const groupValues = (0, csv_helpers_1.toCsvRecordValues)(record, input.groupBy ?? []);
+            const key = JSON.stringify(groupValues);
+            let group = groups.get(key);
+            if (!group) {
+                if (groups.size >= csv_helpers_1.CSV_MAX_AGGREGATE_GROUPS) {
+                    groupLimitReached = true;
+                    return;
+                }
+                group = (0, csv_helpers_1.createCsvAggregateGroup)(groupValues, metrics);
+                groups.set(key, group);
+            }
+            group.count++;
+            for (const metric of metrics) {
+                group.metrics[metric].add((0, csv_helpers_1.normaliseCsvValue)(record[metric]));
+            }
+        },
+    });
+    if (groups.size === 0 && input.groupBy === undefined) {
+        groups.set(JSON.stringify({}), (0, csv_helpers_1.createCsvAggregateGroup)({}, metrics));
+    }
+    const results = Array.from(groups.values()).map((group) => (0, csv_helpers_1.formatCsvAggregateGroup)(group, functions, metrics));
+    (0, csv_helpers_1.sortCsvAggregateResults)(results, input.orderBy);
+    const limit = input.limit ?? 50;
+    const skippedNonNumeric = {};
+    for (const group of groups.values()) {
+        for (const metric of metrics) {
+            skippedNonNumeric[metric] = (skippedNonNumeric[metric] ?? 0) + group.metrics[metric].skipped;
+        }
+    }
+    return {
+        fileName: file.fileName,
+        relativePath: file.relativePath,
+        rowCount,
+        functions,
+        metrics,
+        groupBy: input.groupBy,
+        results: results.slice(0, limit),
+        truncated: results.length > limit || groupLimitReached,
+        skippedNonNumeric,
+    };
+}
+//# sourceMappingURL=csv.operation.js.map

package/dist/modules/agents/tools/knowledge/csv.operation.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"csv.operation.js","sourceRoot":"","sources":["../../../../../src/modules/agents/tools/knowledge/csv.operation.ts"],"names":[],"mappings":";;AA+BA,4BAiEC;AAED,gCA+CC;AAED,kCAuCC;AAED,oCA2EC;AAhQD,+CAsBuB;AAEhB,KAAK,UAAU,QAAQ,CAAC,aAAqB,EAAE,KAAqB,EAAE,KAAoB;IAChG,MAAM,IAAI,GAAG,IAAA,4BAAc,EAAC,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;IAC/C,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,IAAI,EAAE,CAAC;IAChC,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;IAC5B,MAAM,IAAI,GAAe,EAAE,CAAC;IAC5B,MAAM,UAAU,GAAa,EAAE,CAAC;IAChC,MAAM,OAAO,GAIR,EAAE,CAAC;IACR,IAAI,gBAAgD,CAAC;IACrD,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,KAAK,CAAC,SAAS,KAAK,SAAS,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;QAC3D,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;IAC5E,CAAC;IAED,MAAM,IAAA,8BAAgB,EAAC,aAAa,EAAE,IAAI,EAAE;QAC3C,SAAS,EAAE,CAAC,aAAa,EAAE,EAAE;YAC5B,OAAO,CAAC,IAAI,CAAC,GAAG,aAAa,CAAC,CAAC;YAC/B,IAAA,gCAAkB,EAAC,OAAO,EAAE,IAAI,CAAC,QAAQ,EAAE;gBAC1C,GAAG,CAAC,MAAM,IAAI,EAAE,CAAC;gBACjB,GAAG,CAAC,KAAK,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC;aACrD,CAAC,CAAC;YACH,gBAAgB,GAAG,IAAA,sCAAwB,EAC1C,IAAA,+CAAiC,EAAC,OAAO,EAAE,KAAK,CAAC,KAAK,IAAI,EAAE,EAAE,MAAM,IAAI,EAAE,CAAC,EAC3E,oCAAsB,CACtB,CAAC;QACH,CAAC;QACD,QAAQ,EAAE,CAAC,EAAE,MAAM,EAAE,cAAc,EAAE,EAAE,EAAE;YACxC,IAAI,KAAK,CAAC,SAAS,KAAK,SAAS,IAAI,cAAc,KAAK,KAAK,CAAC,SAAS;gBAAE,OAAO;YAChF,IAAI,KAAK,CAAC,SAAS,KAAK,SAAS,IAAI,CAAC,IAAA,4BAAc,EAAC,MAAM,EAAE,KAAK,CAAC,KAAK,IAAI,EAAE,CAAC;gBAAE,OAAO;YAExF,OAAO,EAAE,CAAC;YACV,gBAAgB,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;YAC9B,MAAM,OAAO,GAAG,MAAM,IAAI,OAAO,CAAC;YAClC,IAAI,IAAI,CAAC,MAAM,GAAG,KAAK,EAAE,CAAC;gBACzB,MAAM,MAAM,GAAG,IAAA,+BAAiB,EAAC,MAAM,EAAE,OAAO,CAAC,CAAC;gBAClD,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;gBACnD,UAAU,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;gBAChC,OAAO,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,EAAE,CAAC,CAAC;YACrE,CAAC;QACF,CAAC;KACD,CAAC,CAAC;IACH,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,IAAA,gCAAkB,EAAC,OAAO,EAAE,IAAI,CAAC,QAAQ,EAAE,MAAM,IAAI,EAAE,CAAC,CAAC;IAEnF,MAAM,OAAO,GAAG,MAAM,IAAI,OAAO,CAAC;IAClC,MAAM,SAAS,GAAG,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC;IAExC,OAAO;QACN,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,YAAY,EAAE,IAAI,CAAC,YAAY;QAC/B,OAAO;QACP,UAAU;QACV,IAAI;QACJ,OAAO;QACP,QAAQ,EAAE,OAAO;QACjB,SAAS;QACT,aAAa,EAAE,kEAAkE;QACjF,SAAS,EACR,KAAK,CAAC,SAAS,KAAK,SAAS,IAAI,CAAC,OAAO,GAAG,CAAC,IAAI,SAAS,CAAC;YAC1D,CAAC,CAAC,IAAA,+BAAiB,EAAC,OAAO,EAAE,KAAK,CAAC,KAAK,IAAI,EAAE,EAAE,gBAAgB,CAAC;YACjE,CAAC,CAAC,SAAS;KACb,CAAC;AACH,CAAC;AAEM,KAAK,UAAU,UAAU,CAC/B,aAAqB,EACrB,KAAqB,EACrB,KAAsB;IAEtB,MAAM,IAAI,GAAG,IAAA,4BAAc,EAAC,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;IAC/C,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,UAAU,GAAkC,EAAE,CAAC;IACrD,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAAiC,CAAC;IAClE,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,MAAM,aAAa,GAAG,KAAK,CAAC,aAAa,IAAI,wCAA0B,CAAC;IAExE,MAAM,IAAA,8BAAgB,EAAC,aAAa,EAAE,IAAI,EAAE;QAC3C,SAAS,EAAE,CAAC,aAAa,EAAE,EAAE;YAC5B,OAAO,CAAC,IAAI,CAAC,GAAG,aAAa,CAAC,CAAC;YAC/B,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC9B,gBAAgB,CAAC,GAAG,CAAC,MAAM,EAAE,IAAA,yCAA2B,EAAC,aAAa,CAAC,CAAC,CAAC;YAC1E,CAAC;QACF,CAAC;QACD,QAAQ,EAAE,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE;YACxB,QAAQ,EAAE,CAAC;YACX,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,KAAK,CAAC,UAAU,IAAI,CAAC,CAAC,EAAE,CAAC;gBACjD,UAAU,CAAC,IAAI,CAAC,IAAA,+BAAiB,EAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;YACrD,CAAC;YACD,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC9B,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,GAAG,CAAC,IAAA,+BAAiB,EAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YACtE,CAAC;QACF,CAAC;KACD,CAAC,CAAC;IAEH,MAAM,cAAc,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;QAC7C,MAAM,OAAO,GAAG,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,IAAA,yCAA2B,EAAC,aAAa,CAAC,CAAC;QAC3F,OAAO,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IACjC,CAAC,CAAC,CAAC;IAEH,OAAO;QACN,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,YAAY,EAAE,IAAI,CAAC,YAAY;QAC/B,OAAO,EAAE,OAAO;QAChB,QAAQ;QACR,UAAU;QACV,cAAc;QACd,gBAAgB,EAAE,cAAc;aAC9B,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,aAAa,KAAK,QAAQ,IAAI,QAAQ,GAAG,CAAC,CAAC;aACrE,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC;QAC9B,2BAA2B,EAAE,IAAA,4CAA8B,EAAC,cAAc,EAAE,QAAQ,CAAC;KACrF,CAAC;AACH,CAAC;AAEM,KAAK,UAAU,WAAW,CAChC,aAAqB,EACrB,KAAqB,EACrB,KAAuB;IAEvB,MAAM,IAAI,GAAG,IAAA,4BAAc,EAAC,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;IAC/C,MAAM,MAAM,GAAG,IAAI,GAAG,EAAU,CAAC;IACjC,IAAI,iBAAiB,GAAG,KAAK,CAAC;IAC9B,MAAM,YAAY,GAAa,EAAE,CAAC;IAElC,MAAM,IAAA,8BAAgB,EAAC,aAAa,EAAE,IAAI,EAAE;QAC3C,SAAS,EAAE,CAAC,OAAO,EAAE,EAAE;YACtB,IAAA,gCAAkB,EAAC,OAAO,EAAE,IAAI,CAAC,QAAQ,EAAE;gBAC1C,KAAK,CAAC,MAAM;gBACZ,GAAG,CAAC,KAAK,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC;aACrD,CAAC,CAAC;QACJ,CAAC;QACD,QAAQ,EAAE,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE;YACxB,IAAI,CAAC,IAAA,4BAAc,EAAC,MAAM,EAAE,KAAK,CAAC,KAAK,IAAI,EAAE,CAAC;gBAAE,OAAO;YACvD,MAAM,KAAK,GAAG,IAAA,+BAAiB,EAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;YACtD,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;gBACxB,IAAI,MAAM,CAAC,IAAI,GAAG,sCAAwB,EAAE,CAAC;oBAC5C,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;gBACnB,CAAC;qBAAM,CAAC;oBACP,iBAAiB,GAAG,IAAI,CAAC;gBAC1B,CAAC;gBACD,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,KAAK,CAAC,KAAK,IAAI,EAAE,CAAC;oBAAE,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACzE,CAAC;QACF,CAAC;KACD,CAAC,CAAC;IAEH,OAAO;QACN,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,YAAY,EAAE,IAAI,CAAC,YAAY;QAC/B,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,MAAM,EAAE,YAAY;QACpB,aAAa,EAAE,MAAM,CAAC,IAAI;QAC1B,SAAS,EAAE,iBAAiB,IAAI,MAAM,CAAC,IAAI,GAAG,YAAY,CAAC,MAAM;KACjE,CAAC;AACH,CAAC;AAEM,KAAK,UAAU,YAAY,CACjC,aAAqB,EACrB,KAAqB,EACrB,KAAwB;IAExB,MAAM,IAAI,GAAG,IAAA,4BAAc,EAAC,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;IAC/C,MAAM,SAAS,GAAG,KAAK,CAAC,SAAS,IAAI,CAAC,OAAO,CAAC,CAAC;IAC/C,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CACzB,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,CAAC,CAC5E,CAAC;IACF,MAAM,WAAW,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,KAAK,OAAO,CAAC,CAAC;IAC3D,IAAI,WAAW,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzC,MAAM,IAAI,KAAK,CAAC,qEAAqE,CAAC,CAAC;IACxF,CAAC;IACD,MAAM,MAAM,GAAG,IAAI,GAAG,EAA6B,CAAC;IACpD,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,iBAAiB,GAAG,KAAK,CAAC;IAE9B,MAAM,IAAA,8BAAgB,EAAC,aAAa,EAAE,IAAI,EAAE;QAC3C,SAAS,EAAE,CAAC,OAAO,EAAE,EAAE;YACtB,IAAA,gCAAkB,EAAC,OAAO,EAAE,IAAI,CAAC,QAAQ,EAAE;gBAC1C,GAAG,OAAO;gBACV,GAAG,CAAC,KAAK,CAAC,OAAO,IAAI,EAAE,CAAC;gBACxB,GAAG,CAAC,KAAK,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC;aACrD,CAAC,CAAC;QACJ,CAAC;QACD,QAAQ,EAAE,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE;YACxB,IAAI,CAAC,IAAA,4BAAc,EAAC,MAAM,EAAE,KAAK,CAAC,KAAK,IAAI,EAAE,CAAC;gBAAE,OAAO;YACvD,QAAQ,EAAE,CAAC;YACX,MAAM,WAAW,GAAG,IAAA,+BAAiB,EAAC,MAAM,EAAE,KAAK,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC;YACnE,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;YACxC,IAAI,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC5B,IAAI,CAAC,KAAK,EAAE,CAAC;gBAGZ,IAAI,MAAM,CAAC,IAAI,IAAI,sCAAwB,EAAE,CAAC;oBAC7C,iBAAiB,GAAG,IAAI,CAAC;oBACzB,OAAO;gBACR,CAAC;gBACD,KAAK,GAAG,IAAA,qCAAuB,EAAC,WAAW,EAAE,OAAO,CAAC,CAAC;gBACtD,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YACxB,CAAC;YACD,KAAK,CAAC,KAAK,EAAE,CAAC;YACd,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC9B,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,IAAA,+BAAiB,EAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YAC9D,CAAC;QACF,CAAC;KACD,CAAC,CAAC;IACH,IAAI,MAAM,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;QACtD,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,EAAE,IAAA,qCAAuB,EAAC,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC;IACtE,CAAC;IAED,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CACzD,IAAA,qCAAuB,EAAC,KAAK,EAAE,SAAS,EAAE,OAAO,CAAC,CAClD,CAAC;IACF,IAAA,qCAAuB,EAAC,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;IAChD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,IAAI,EAAE,CAAC;IAChC,MAAM,iBAAiB,GAA2B,EAAE,CAAC;IACrD,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC;QACrC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC9B,iBAAiB,CAAC,MAAM,CAAC,GAAG,CAAC,iBAAiB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;QAC9F,CAAC;IACF,CAAC;IAED,OAAO;QACN,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,YAAY,EAAE,IAAI,CAAC,YAAY;QAC/B,QAAQ;QACR,SAAS;QACT,OAAO;QACP,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC;QAChC,SAAS,EAAE,OAAO,CAAC,MAAM,GAAG,KAAK,IAAI,iBAAiB;QACtD,iBAAiB;KACjB,CAAC;AACH,CAAC"}

package/dist/modules/agents/tools/knowledge/file-references.d.ts

@@ -0,0 +1,16 @@
+import type { AgentKnowledgeService } from '../../agent-knowledge.service';
+import type { ParsedSearchKnowledgeInput } from './schemas';
+export type WorkspaceFiles = Awaited<ReturnType<AgentKnowledgeService['materializeWorkspace']>>;
+export type FileReferenceResolution = {
+    status: 'found';
+    file: WorkspaceFiles[number];
+} | {
+    status: 'missing';
+    error: string;
+} | {
+    status: 'ambiguous';
+    error: string;
+};
+export declare function resolveFileReference(files: WorkspaceFiles, reference: string): FileReferenceResolution;
+export declare function getRequiredFileReferences(input: ParsedSearchKnowledgeInput): string[] | undefined;
+export declare function mapFileReferences(files: WorkspaceFiles, requestedFiles?: string[]): string[] | undefined;