@xpert-ai/plugin-analytics-files 0.0.0-stage → 0.1.1

package/README.md

@@ -1,3 +1,12 @@
-# Temporary Holding Version
+# File Analytics
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+`@xpert-ai/plugin-analytics-files` provides `file.table`: declarative CSV, JSON and XLSX queries over platform-authorized Workspace files. Standard charts and HTML dashboards consume the same Dataset/Binding results.
+
+Administrators configure a `workspace-files` data source with approved file references, explicit format/read options, column types and business descriptions. Agent discovery returns the exact query contract and a bounded real preview; no new Agent middleware is required. File parsing, aggregation and format dependencies remain in this plugin. The platform owns file permissions, data-source authorization and Artifact lifecycle.
+
+- [Overview](docs/index.mdx) / [中文概览](docs/zh-Hans/index.mdx)
+- [Configuration and query contract](docs/configuration.mdx) / [中文配置](docs/zh-Hans/configuration.mdx)
+- [Verification](docs/verification.mdx) / [验收](docs/zh-Hans/verification.mdx)
+- [Example catalog](examples/sales.json)
+
+Development: `corepack pnpm exec nx run @xpert-ai/plugin-analytics-files:test` from `xpertai/`. Then run the repository plugin lifecycle harness and metadata check. Install at the declared system scope through the platform local deploy command; restart when requested and verify `file.table` discovery before application acceptance.

package/dist/index.d.ts

@@ -0,0 +1,3 @@
+import type { XpertPlugin } from '@xpert-ai/plugin-sdk';
+declare const plugin: XpertPlugin;
+export default plugin;

package/dist/index.js

@@ -0,0 +1,13 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const node_fs_1 = require("node:fs");
+const node_path_1 = require("node:path");
+const plugin_1 = require("./lib/plugin");
+const metadata = JSON.parse((0, node_fs_1.readFileSync)((0, node_path_1.join)(__dirname, '../package.json'), 'utf8'));
+const plugin = {
+    meta: { name: metadata.name, version: metadata.version, category: 'database', displayName: 'File Analytics',
+        description: 'Declarative CSV, JSON and XLSX queries over authorized Workspace files', author: 'XpertAI',
+        level: metadata.xpert.plugin.level, artifactNamespace: metadata.xpert.plugin.artifactNamespace },
+    register: () => ({ module: plugin_1.AnalyticsFilesPlugin })
+};
+exports.default = plugin;

package/dist/lib/cells.d.ts

@@ -0,0 +1,6 @@
+import Decimal from 'decimal.js';
+import type { AnalyticsValue } from '@xpert-ai/contracts';
+import type { Column } from './schema';
+export declare const Exact: typeof Decimal;
+export declare function cell(value: unknown, column: Column, format: 'csv' | 'json' | 'xlsx'): AnalyticsValue;
+export declare function compare(a: AnalyticsValue, b: AnalyticsValue, type: Column['logicalType']): number;

package/dist/lib/cells.js

@@ -0,0 +1,69 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Exact = void 0;
+exports.cell = cell;
+exports.compare = compare;
+const decimal_js_1 = __importDefault(require("decimal.js"));
+exports.Exact = decimal_js_1.default.clone({ precision: 160, toExpNeg: -160, toExpPos: 160 });
+const date = (value) => /^\d{4}-\d{2}-\d{2}$/.test(value) && Number.isFinite(Date.parse(value)) && new Date(value).toISOString().slice(0, 10) === value;
+function cell(value, column, format) {
+    if ((value === null || (typeof value === 'string' && column.nullValues.includes(value))) && column.nullable)
+        return null;
+    if (value instanceof Date && format === 'xlsx' && Number.isFinite(value.getTime()))
+        value = column.logicalType === 'date' ? value.toISOString().slice(0, 10) : value.toISOString();
+    if (typeof value === 'string' && value.length > 4096)
+        throw new Error('FILE_DATASET_CELL_LIMIT');
+    const text = typeof value === 'string' ? value : undefined;
+    switch (column.logicalType) {
+        case 'string':
+            if (text !== undefined)
+                return text;
+            break;
+        case 'boolean':
+            if (typeof value === 'boolean')
+                return value;
+            if (format === 'csv' && (text === 'true' || text === 'false'))
+                return text === 'true';
+            break;
+        case 'int64': {
+            const exact = typeof value === 'number' && Number.isSafeInteger(value) ? String(value) : text;
+            if (exact && /^-?\d{1,19}$/.test(exact) && BigInt(exact) >= -(2n ** 63n) && BigInt(exact) < 2n ** 63n)
+                return BigInt(exact).toString();
+            break;
+        }
+        case 'decimal': {
+            // JSON decimals must be strings. Excel numeric cells carry Excel's stored numeric precision.
+            const exact = format === 'xlsx' && typeof value === 'number' && Number.isFinite(value) ? new exports.Exact(value).toFixed() : text;
+            if (exact && /^-?\d{1,60}(\.\d{1,30})?$/.test(exact))
+                return new exports.Exact(exact).toFixed();
+            break;
+        }
+        case 'float64': {
+            const number = format === 'csv' && text && /^-?\d+(\.\d+)?([eE][+-]?\d+)?$/.test(text) ? Number(text) : value;
+            if (typeof number === 'number' && Number.isFinite(number))
+                return number;
+            break;
+        }
+        case 'date':
+            if (text && date(text))
+                return text;
+            break;
+        case 'timestamp':
+            if (text && /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{1,9})?(Z|[+-]\d{2}:\d{2})$/.test(text) && date(text.slice(0, 10)) && Number.isFinite(Date.parse(text)))
+                return text;
+            break;
+    }
+    throw new Error(`FILE_DATASET_FIELD_TYPE_INVALID: ${column.key}`);
+}
+function compare(a, b, type) {
+    if (a === null || b === null)
+        return a === b ? 0 : a === null ? -1 : 1;
+    if (type === 'decimal' || type === 'int64' || type === 'float64')
+        return new exports.Exact(String(a)).comparedTo(String(b));
+    if (type === 'timestamp')
+        return Math.sign(Date.parse(String(a)) - Date.parse(String(b)));
+    return a === b ? 0 : a < b ? -1 : 1;
+}

package/dist/lib/plugin.d.ts

@@ -0,0 +1,2 @@
+export declare class AnalyticsFilesPlugin {
+}

package/dist/lib/plugin.js

@@ -0,0 +1,17 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AnalyticsFilesPlugin = void 0;
+const plugin_sdk_1 = require("@xpert-ai/plugin-sdk");
+const provider_1 = require("./provider");
+let AnalyticsFilesPlugin = class AnalyticsFilesPlugin {
+};
+exports.AnalyticsFilesPlugin = AnalyticsFilesPlugin;
+exports.AnalyticsFilesPlugin = AnalyticsFilesPlugin = __decorate([
+    (0, plugin_sdk_1.XpertServerPlugin)({ providers: [provider_1.FileAnalyticsProvider] })
+], AnalyticsFilesPlugin);

package/dist/lib/provider.d.ts

@@ -0,0 +1,29 @@
+import type { AnalyticsBinding, AnalyticsDiscoveryResult, AnalyticsProviderDescriptor, AnalyticsValue, JSONValue } from '@xpert-ai/contracts';
+import { type AnalyticsQueryProvider, type AnalyticsProviderContext, type RuntimeCapabilityRegistry } from '@xpert-ai/plugin-sdk';
+import { type FileDataset } from './schema';
+export declare const contractHash: (dataset: FileDataset) => string;
+export declare class FileAnalyticsProvider implements AnalyticsQueryProvider {
+    private readonly runtime;
+    readonly descriptor: AnalyticsProviderDescriptor;
+    constructor(runtime: RuntimeCapabilityRegistry);
+    private check;
+    private file;
+    private read;
+    private guarded;
+    discover(request: JSONValue, context: AnalyticsProviderContext): Promise<AnalyticsDiscoveryResult>;
+    execute(binding: AnalyticsBinding, parameters: Record<string, AnalyticsValue>, context: AnalyticsProviderContext): Promise<{
+        sourceFetchedAt: string;
+        sourceVersion: string;
+        columns: {
+            key: string;
+        }[];
+        rows: Record<string, string | number | boolean | null>[];
+        completeness: {
+            totalRows: number;
+            returnedRows: number;
+            totalAccuracy: "exact";
+            scope: "page" | "full_query_result";
+            truncated: boolean;
+        };
+    }>;
+}

package/dist/lib/provider.js

@@ -0,0 +1,145 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+var __param = (this && this.__param) || function (paramIndex, decorator) {
+    return function (target, key) { decorator(target, key, paramIndex); }
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.FileAnalyticsProvider = exports.contractHash = void 0;
+const common_1 = require("@nestjs/common");
+const node_crypto_1 = require("node:crypto");
+const plugin_sdk_1 = require("@xpert-ai/plugin-sdk");
+const schema_1 = require("./schema");
+const query_1 = require("./query");
+const worker_client_1 = require("./worker-client");
+const hash = (value) => (0, node_crypto_1.createHash)('sha256').update(value).digest('hex');
+const contractHash = (dataset) => hash(JSON.stringify(dataset));
+exports.contractHash = contractHash;
+const instructions = `Use an inspected dataset's bindingPayload and columns. Never supply paths, file references, scope, URLs, SQL, scripts or reader options in Binding.payload. select contains output keys. filters are AND conditions; value is {type:"parameter",name:"region"} or {type:"literal",value:"East"}. Every parameter must appear in filters; bind it through Binding.parameters and controls. For an optional all-values filter use skipWhenNull:true, required:false and defaultValue:null; empty string is a literal value. groupBy requires aggregates. Aggregates are {key:"total",operation:"sum",column:"net_sales"} or {key:"count",operation:"count"}; use a NEW key distinct from source columns. With aggregates select contains group keys and aggregate keys only. Sum of int64 outputs decimal; sum/min/max are nullable; count is non-null int64 and has no unit. Other aggregate types/units follow the source. Output sourceField uses output keys, and logicalType/nullable/unit must match the declared columns exactly. Use the authored business descriptions, never sum incompatible units or duplicate grains. decimal/int64 values and controls use exact strings. Preview every query before saving. Input beyond file/row limits fails instead of producing partial totals. Refresh rereads the current file. Artifact versions pin queries, not source data. XLSX formula caches are not recalculated or guaranteed current. Render with Dataset/Binding; never browser fetch.`;
+const queryContract = { operation: 'query', contractVersion: 1,
+    payloadSchema: (0, schema_1.jsonSchema)(schema_1.payloadSchema), instructions: instructions + ' metadata.binding is a complete canonical Binding; bindingPayload alone is only its payload, not a Binding. Start from metadata.binding and preserve its envelope. Raw rows do not promise uniqueness; keep grain:[] unless verified, or group first and declare the grouping field IDs as grain.' };
+let FileAnalyticsProvider = class FileAnalyticsProvider {
+    constructor(runtime) {
+        this.runtime = runtime;
+        this.descriptor = {
+            id: schema_1.PROVIDER, name: 'Workspace files (CSV / JSON / XLSX)', dataSourceTypes: ['workspace-files'],
+            operations: [{ name: 'query', contractVersion: 1, readOnly: true }],
+            connectionType: { type: 'workspace-files', name: 'Workspace files', configuration: (0, schema_1.jsonSchema)(schema_1.optionsSchema) }
+        };
+    }
+    check(context) {
+        context.signal.throwIfAborted();
+        if (Date.now() >= context.deadlineAt)
+            throw new Error('FILE_DATASET_DEADLINE_EXCEEDED');
+    }
+    async file(definition, context) {
+        this.check(context);
+        const file = definition.file;
+        if ('userId' in file && file.userId !== context.actor.userId)
+            throw new common_1.ForbiddenException('FILE_DATASET_ACCESS_DENIED');
+        if (file.catalog === 'projects') {
+            const visible = await this.runtime.require(plugin_sdk_1.ProjectAccessRuntimeCapability).listReadable({ actor: context.actor, projectIds: [file.projectId] });
+            if (!visible.some(p => p.projectId === file.projectId && !p.archived))
+                throw new common_1.ForbiddenException('FILE_DATASET_ACCESS_DENIED');
+        }
+        const reference = { ...file, ...context.actor };
+        const files = this.runtime.require(plugin_sdk_1.WorkspaceFilesRuntimeCapability);
+        const metadata = await files.resolveFile(reference);
+        this.check(context);
+        if (metadata.catalog !== file.catalog)
+            throw new common_1.ForbiddenException('FILE_DATASET_SCOPE_CHANGED');
+        if (metadata.size === undefined || metadata.size <= 0 || metadata.size > schema_1.MAX_BYTES)
+            throw new Error('FILE_DATASET_SIZE_LIMIT');
+        return { files, reference };
+    }
+    async read(definition, context) {
+        const { files, reference } = await this.file(definition, context);
+        const { buffer } = await files.readBuffer(reference);
+        this.check(context);
+        if (!buffer.length || buffer.length > schema_1.MAX_BYTES)
+            throw new Error('FILE_DATASET_SIZE_LIMIT');
+        return { buffer, fetchedAt: new Date().toISOString() };
+    }
+    async guarded(run) {
+        try {
+            return await run();
+        }
+        catch (error) {
+            if (error instanceof common_1.HttpException)
+                throw error;
+            const code = error instanceof Error && /^FILE_DATASET_[A-Z_]+(?:: [A-Za-z][A-Za-z0-9_]*)?$/.test(error.message) ? error.message : 'FILE_DATASET_INVALID';
+            throw new common_1.BadRequestException({ code: code.split(':')[0], message: code });
+        }
+    }
+    async discover(request, context) {
+        return this.guarded(async () => {
+            const input = schema_1.discoverySchema.parse(request), { datasets } = schema_1.optionsSchema.parse(context.options);
+            const selected = input.datasetId ? datasets.filter(d => d.id === input.datasetId) : datasets.slice((input.page - 1) * 20, input.page * 20);
+            if (input.datasetId && !selected.length)
+                throw new Error('FILE_DATASET_UNAVAILABLE');
+            const items = [];
+            for (const definition of selected) {
+                try {
+                    await this.file(definition, context);
+                }
+                catch (e) {
+                    if (!input.datasetId && e instanceof common_1.ForbiddenException)
+                        continue;
+                    throw e;
+                }
+                const payload = schema_1.payloadSchema.parse({ datasetId: definition.id, revision: definition.revision,
+                    contractHash: (0, exports.contractHash)(definition), select: definition.columns.map(c => c.key) });
+                let metadata = { revision: definition.revision, format: definition.reader.format };
+                if (input.datasetId) {
+                    const read = await this.read(definition, context);
+                    const preview = await (0, worker_client_1.queryWorker)(read.buffer, definition, payload, {}, 5, context.signal, context.deadlineAt);
+                    await this.file(definition, context);
+                    const binding = { schemaVersion: 1, id: definition.id, dataSourceId: context.dataSourceId,
+                        providerId: schema_1.PROVIDER, operation: 'query', contractVersion: 1, payload: JSON.parse(JSON.stringify(payload)), parameters: [],
+                        output: { datasetId: definition.id, grain: [], fields: definition.columns.map(({ key, path, description, nullValues, ...field }) => ({ ...field, id: key, sourceField: key })) } };
+                    const previewFits = Buffer.byteLength(JSON.stringify(preview.rows)) <= 32000;
+                    metadata = { ...metadata, contractHash: payload.contractHash, bindingPayload: JSON.parse(JSON.stringify(payload)),
+                        binding: JSON.parse(JSON.stringify(binding)),
+                        columns: definition.columns.map(({ path, nullValues, ...column }) => column),
+                        ...(definition.reader.format === 'xlsx' ? { sheet: definition.reader.sheet, formulaPolicy: definition.reader.formulaPolicy } : {}),
+                        preview: previewFits ? preview.rows : [], previewOmitted: !previewFits,
+                        ...(previewFits ? { previewCompleteness: preview.completeness } : {}),
+                        limits: { inputBytes: schema_1.MAX_BYTES, inputRows: schema_1.MAX_ROWS }, sourceVersion: hash(read.buffer) };
+                }
+                items.push({ id: definition.id, name: definition.name, description: definition.description, kind: 'dataset', inspectRequest: { datasetId: definition.id }, metadata });
+            }
+            return { schemaVersion: 1, queryContract, items,
+                ...(!input.datasetId && input.page * 20 < datasets.length ? { nextRequest: { page: input.page + 1 } } : {}) };
+        });
+    }
+    async execute(binding, parameters, context) {
+        return this.guarded(async () => {
+            this.check(context);
+            const query = schema_1.payloadSchema.parse(binding.payload), { datasets } = schema_1.optionsSchema.parse(context.options);
+            const definition = datasets.find(d => d.id === query.datasetId);
+            if (!definition)
+                throw new Error('FILE_DATASET_UNAVAILABLE');
+            if (query.revision !== definition.revision || query.contractHash !== (0, exports.contractHash)(definition))
+                throw new Error('FILE_DATASET_CONTRACT_CHANGED');
+            (0, query_1.validateOutput)(binding, (0, query_1.outputColumns)(definition, query));
+            const read = await this.read(definition, context);
+            const result = await (0, worker_client_1.queryWorker)(read.buffer, definition, query, parameters, context.maxRows, context.signal, context.deadlineAt);
+            // No successful result is returned after current file/Assistant/Project access has been revoked.
+            await this.file(definition, context);
+            return { ...result, sourceFetchedAt: read.fetchedAt, sourceVersion: `${query.contractHash}:${hash(read.buffer)}` };
+        });
+    }
+};
+exports.FileAnalyticsProvider = FileAnalyticsProvider;
+exports.FileAnalyticsProvider = FileAnalyticsProvider = __decorate([
+    (0, plugin_sdk_1.AnalyticsProvider)(schema_1.PROVIDER),
+    __param(0, (0, common_1.Inject)(plugin_sdk_1.XPERT_RUNTIME_CAPABILITIES_TOKEN)),
+    __metadata("design:paramtypes", [Object])
+], FileAnalyticsProvider);

package/dist/lib/query.d.ts

@@ -0,0 +1,20 @@
+import type { AnalyticsBinding, AnalyticsValue } from '@xpert-ai/contracts';
+import type { Column, FileDataset, FileQuery } from './schema';
+export type Row = Record<string, AnalyticsValue>;
+export declare function outputColumns(definition: FileDataset, query: FileQuery): Column[];
+export declare function validateOutput(binding: AnalyticsBinding, columns: Column[]): void;
+export declare function executeQuery(rows: Row[], definition: FileDataset, query: FileQuery, parameters: Row, maxRows: number): {
+    columns: {
+        key: string;
+    }[];
+    rows: {
+        [k: string]: AnalyticsValue;
+    }[];
+    completeness: {
+        totalRows: number;
+        returnedRows: number;
+        totalAccuracy: "exact";
+        scope: "page" | "full_query_result";
+        truncated: boolean;
+    };
+};

package/dist/lib/query.js

@@ -0,0 +1,134 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.outputColumns = outputColumns;
+exports.validateOutput = validateOutput;
+exports.executeQuery = executeQuery;
+const cells_1 = require("./cells");
+function requireColumn(columns, name) {
+    const column = columns.get(name);
+    if (!column)
+        throw new Error('FILE_DATASET_COLUMN_UNAVAILABLE');
+    return column;
+}
+function outputColumns(definition, query) {
+    const source = new Map(definition.columns.map(c => [c.key, c])), available = new Map();
+    if (new Set(query.select).size !== query.select.length || new Set(query.groupBy).size !== query.groupBy.length)
+        throw new Error('FILE_DATASET_DUPLICATE_COLUMN');
+    if (query.aggregates.length) {
+        for (const name of query.groupBy)
+            available.set(name, requireColumn(source, name));
+        for (const a of query.aggregates) {
+            if (source.has(a.key) || available.has(a.key))
+                throw new Error('FILE_DATASET_AGGREGATE_ALIAS_INVALID');
+            const column = a.column ? requireColumn(source, a.column) : undefined;
+            if (!column && a.operation !== 'count')
+                throw new Error('FILE_DATASET_AGGREGATE_COLUMN_REQUIRED');
+            if (a.operation === 'sum' && column && !['int64', 'decimal', 'float64'].includes(column.logicalType))
+                throw new Error('FILE_DATASET_SUM_TYPE_INVALID');
+            available.set(a.key, { key: a.key, path: [a.key], label: a.key, description: a.operation,
+                logicalType: a.operation === 'count' ? 'int64' : a.operation === 'sum' && column?.logicalType === 'int64' ? 'decimal' : column.logicalType,
+                nullable: a.operation !== 'count', role: 'measure', nullValues: [], ...(column?.unit && a.operation !== 'count' ? { unit: column.unit } : {}) });
+        }
+    }
+    else {
+        if (query.groupBy.length)
+            throw new Error('FILE_DATASET_GROUP_REQUIRES_AGGREGATE');
+        for (const [name, column] of source)
+            available.set(name, column);
+    }
+    const columns = query.select.map(name => requireColumn(available, name));
+    for (const sort of query.orderBy)
+        if (!query.select.includes(sort.column))
+            throw new Error('FILE_DATASET_SORT_COLUMN_INVALID');
+    for (const filter of query.filters)
+        requireColumn(source, filter.column);
+    return columns;
+}
+function validateOutput(binding, columns) {
+    for (const field of binding.output.fields) {
+        const column = columns.find(c => c.key === field.sourceField);
+        if (!column || field.semanticRef || field.logicalType !== column.logicalType || field.nullable !== column.nullable || field.unit !== column.unit)
+            throw new Error('FILE_DATASET_OUTPUT_CONTRACT_INVALID');
+    }
+}
+function executeQuery(rows, definition, query, parameters, maxRows) {
+    const columns = outputColumns(definition, query), source = new Map(definition.columns.map(c => [c.key, c]));
+    const parameterNames = new Set(query.filters.flatMap(f => f.value.type === 'parameter' ? [f.value.name] : []));
+    if (parameterNames.size !== Object.keys(parameters).length || Object.keys(parameters).some(name => !parameterNames.has(name)))
+        throw new Error('FILE_DATASET_PARAMETERS_INVALID');
+    const filters = query.filters.map(filter => {
+        const column = requireColumn(source, filter.column);
+        const raw = filter.value.type === 'parameter' ? parameters[filter.value.name] : filter.value.value;
+        const value = raw === null ? null : (0, cells_1.cell)(raw, { ...column, nullValues: [] }, 'json');
+        if (filter.operator === 'contains' && column.logicalType !== 'string')
+            throw new Error('FILE_DATASET_FILTER_TYPE_INVALID');
+        if (value === null && !filter.skipWhenNull && !['eq', 'ne'].includes(filter.operator))
+            throw new Error('FILE_DATASET_NULL_COMPARISON_INVALID');
+        return { ...filter, column, value };
+    });
+    let result = rows.filter(row => filters.every(f => {
+        if (f.value === null && f.skipWhenNull)
+            return true;
+        const value = row[f.column.key];
+        if (f.operator === 'contains')
+            return typeof value === 'string' && typeof f.value === 'string' && value.includes(f.value);
+        const order = (0, cells_1.compare)(value, f.value, f.column.logicalType);
+        if (f.operator === 'eq')
+            return order === 0;
+        if (f.operator === 'ne')
+            return order !== 0;
+        if (value === null || f.value === null)
+            return false;
+        return f.operator === 'gt' ? order > 0 : f.operator === 'gte' ? order >= 0 : f.operator === 'lt' ? order < 0 : order <= 0;
+    }));
+    if (query.aggregates.length) {
+        const groups = new Map();
+        if (!query.groupBy.length)
+            groups.set('[]', []);
+        for (const row of result) {
+            const id = JSON.stringify(query.groupBy.map(name => row[name]));
+            const group = groups.get(id) ?? [];
+            group.push(row);
+            groups.set(id, group);
+        }
+        result = [...groups.values()].map(group => {
+            const row = Object.fromEntries(query.groupBy.map(name => [name, group[0][name]]));
+            for (const aggregate of query.aggregates) {
+                const values = aggregate.column ? group.map(r => r[aggregate.column]).filter(v => v !== null) : [];
+                if (aggregate.operation === 'count')
+                    row[aggregate.key] = String(aggregate.column ? values.length : group.length);
+                else if (!values.length)
+                    row[aggregate.key] = null;
+                else if (aggregate.operation === 'sum') {
+                    const sum = values.reduce((sum, value) => sum.plus(String(value)), new cells_1.Exact(0));
+                    const float = source.get(aggregate.column).logicalType === 'float64';
+                    if (float && !Number.isFinite(sum.toNumber()))
+                        throw new Error('FILE_DATASET_SUM_OVERFLOW');
+                    row[aggregate.key] = float ? sum.toNumber() : sum.toFixed();
+                }
+                else
+                    row[aggregate.key] = values.reduce((a, b) => {
+                        const order = (0, cells_1.compare)(a, b, source.get(aggregate.column).logicalType);
+                        return aggregate.operation === 'min' ? order <= 0 ? a : b : order >= 0 ? a : b;
+                    });
+            }
+            return row;
+        });
+    }
+    result.sort((a, b) => {
+        for (const order of query.orderBy) {
+            const diff = (0, cells_1.compare)(a[order.column], b[order.column], columns.find(c => c.key === order.column).logicalType);
+            if (diff)
+                return order.direction === 'asc' ? diff : -diff;
+        }
+        return 0;
+    });
+    const totalRows = result.length, truncated = totalRows > maxRows;
+    const projected = result.slice(0, maxRows).map(row => Object.fromEntries(query.select.map(key => [key, row[key]])));
+    if (Buffer.byteLength(JSON.stringify(projected)) > 2_000_000)
+        throw new Error('FILE_DATASET_RESULT_LIMIT');
+    return { columns: columns.map(c => ({ key: c.key })), rows: projected, completeness: {
+            totalRows, returnedRows: projected.length, totalAccuracy: 'exact',
+            scope: truncated ? 'page' : 'full_query_result', truncated
+        } };
+}

package/dist/lib/readers.d.ts

@@ -0,0 +1,3 @@
+import type { AnalyticsValue } from '@xpert-ai/contracts';
+import { type FileDataset } from './schema';
+export declare function readRows(bytes: Buffer, definition: FileDataset): Promise<Array<Record<string, AnalyticsValue>>>;

package/dist/lib/readers.js

@@ -0,0 +1,97 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.readRows = readRows;
+const sync_1 = require("csv-parse/sync");
+const exceljs_1 = require("exceljs");
+const schema_1 = require("./schema");
+const cells_1 = require("./cells");
+const xlsx_archive_1 = require("./xlsx-archive");
+function matrixRows(matrix, definition) {
+    const headers = matrix.shift();
+    if (!headers || !headers.length || headers.length > 200 || headers.some(h => typeof h !== 'string' || !h || schema_1.forbidden.has(h)) || new Set(headers).size !== headers.length)
+        throw new Error('FILE_DATASET_HEADERS_INVALID');
+    for (const column of definition.columns)
+        if (!headers.includes(column.path[0]))
+            throw new Error('FILE_DATASET_SCHEMA_CHANGED');
+    return matrix.map(row => {
+        if (row.length !== headers.length)
+            throw new Error('FILE_DATASET_ROW_WIDTH_INVALID');
+        return Object.fromEntries(headers.map((header, i) => [String(header), row[i]]));
+    });
+}
+function excelValue(value, policy) {
+    if (value === undefined)
+        return null;
+    if (!value || typeof value !== 'object' || value instanceof Date)
+        return value;
+    if ('formula' in value || 'sharedFormula' in value) {
+        if (policy !== 'cached')
+            throw new Error('FILE_DATASET_FORMULA_DENIED');
+        if (value.result === undefined)
+            throw new Error('FILE_DATASET_FORMULA_CACHE_MISSING');
+        return excelValue(value.result, policy);
+    }
+    if ('richText' in value)
+        return value.richText.map(part => part.text).join('');
+    if ('hyperlink' in value)
+        return value.text; // Never dereference links.
+    throw new Error('FILE_DATASET_EXCEL_CELL_ERROR');
+}
+async function xlsxRows(bytes, definition) {
+    const reader = definition.reader;
+    if (reader.format !== 'xlsx')
+        throw new Error('FILE_DATASET_FORMAT_INVALID');
+    (0, xlsx_archive_1.validateXlsxArchive)(bytes);
+    const workbook = new exceljs_1.Workbook();
+    await workbook.xlsx.load(new Uint8Array(bytes).buffer);
+    const sheet = workbook.getWorksheet(reader.sheet);
+    if (!sheet || sheet.state !== 'visible')
+        throw new Error('FILE_DATASET_SHEET_UNAVAILABLE');
+    if (sheet.rowCount > schema_1.MAX_ROWS + reader.headerRow || sheet.columnCount > 200)
+        throw new Error('FILE_DATASET_INPUT_LIMIT');
+    const header = sheet.getRow(reader.headerRow), width = header.cellCount;
+    const matrix = [Array.from({ length: width }, (_, i) => excelValue(header.getCell(i + 1).value, 'reject'))];
+    for (let i = reader.headerRow + 1; i <= sheet.rowCount; i++) {
+        const row = sheet.getRow(i);
+        if (!row.hasValues)
+            continue;
+        if (row.cellCount > width)
+            throw new Error('FILE_DATASET_ROW_WIDTH_INVALID');
+        matrix.push(Array.from({ length: width }, (_, c) => {
+            const value = row.getCell(c + 1);
+            if (value.isMerged)
+                throw new Error('FILE_DATASET_MERGED_CELL_DENIED');
+            return excelValue(value.value, reader.formulaPolicy);
+        }));
+    }
+    return matrixRows(matrix, definition);
+}
+async function readRows(bytes, definition) {
+    if (!bytes.length || bytes.length > schema_1.MAX_BYTES)
+        throw new Error('FILE_DATASET_SIZE_LIMIT');
+    const reader = definition.reader;
+    let records;
+    if (reader.format === 'xlsx')
+        records = await xlsxRows(bytes, definition);
+    else {
+        const text = new TextDecoder('utf-8', { fatal: true }).decode(bytes);
+        if (reader.format === 'json')
+            records = (0, schema_1.readPath)(JSON.parse(text.replace(/^\uFEFF/, '')), reader.rowsPath);
+        else {
+            const parsed = (0, sync_1.parse)(text, { bom: true, delimiter: reader.delimiter, from_line: reader.headerRow,
+                skip_empty_lines: true, max_record_size: 100_000, to: schema_1.MAX_ROWS + 2 });
+            if (!Array.isArray(parsed) || !parsed.every(Array.isArray))
+                throw new Error('FILE_DATASET_CSV_INVALID');
+            records = matrixRows(parsed, definition);
+        }
+    }
+    if (!Array.isArray(records))
+        throw new Error('FILE_DATASET_ROWS_INVALID');
+    if (records.length > schema_1.MAX_ROWS)
+        throw new Error('FILE_DATASET_INPUT_LIMIT');
+    return records.map(row => {
+        if (!row || typeof row !== 'object' || Array.isArray(row))
+            throw new Error('FILE_DATASET_ROW_INVALID');
+        return Object.fromEntries(definition.columns.map(column => [column.key, (0, cells_1.cell)((0, schema_1.readPath)(row, column.path), column, reader.format)]));
+    });
+}