@qwe8652591/abap-recursive-query 1.1.0

package/dist/producer/RecursiveQueryProducer.js

@@ -0,0 +1,496 @@
+"use strict";
+/**
+ * 递归查询生产者
+ * 负责并发递归查询对象，生成所有下载任务
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RecursiveQueryProducer = void 0;
+const { v4: uuidv4 } = require('uuid');
+const common_1 = require("../common");
+const handlers_1 = require("./handlers");
+/**
+ * 异步任务队列（内部并发控制）
+ * 复用现有的 AsyncTaskQueue 实现
+ */
+class AsyncTaskQueue {
+    queue = [];
+    running = 0;
+    maxConcurrent;
+    taskResults = new Map();
+    processing = false;
+    processingRequests = 0;
+    waiters = [];
+    constructor(maxConcurrent) {
+        this.maxConcurrent = Math.max(1, maxConcurrent);
+    }
+    async add(taskId, task) {
+        if (this.taskResults.has(taskId)) {
+            return this.taskResults.get(taskId);
+        }
+        const taskPromise = new Promise((resolve, reject) => {
+            const wrappedTask = async () => {
+                try {
+                    this.running++;
+                    const result = await task();
+                    resolve(result);
+                    return result;
+                }
+                catch (error) {
+                    reject(error);
+                    throw error;
+                }
+                finally {
+                    this.running--;
+                    this.notifyWaiters();
+                    process.nextTick(() => this.processQueue());
+                }
+            };
+            this.queue.push(wrappedTask);
+        });
+        this.taskResults.set(taskId, taskPromise);
+        taskPromise.finally(() => {
+            setTimeout(() => {
+                this.taskResults.delete(taskId);
+            }, 1000);
+        }).catch(() => { });
+        this.processQueue();
+        return taskPromise;
+    }
+    processQueue() {
+        this.processingRequests++;
+        if (this.processing) {
+            return;
+        }
+        this.processing = true;
+        process.nextTick(() => {
+            try {
+                while (this.running < this.maxConcurrent && this.queue.length > 0) {
+                    const task = this.queue.shift();
+                    if (task) {
+                        task().catch(() => { });
+                    }
+                }
+            }
+            finally {
+                this.processing = false;
+                this.processingRequests--;
+                if (this.processingRequests > 0 || (this.queue.length > 0 && this.running < this.maxConcurrent)) {
+                    this.processingRequests = 0;
+                    this.processQueue();
+                }
+            }
+        });
+    }
+    getQueueSize() {
+        return this.queue.length;
+    }
+    getRunningCount() {
+        return this.running;
+    }
+    notifyWaiters() {
+        const waiters = [...this.waiters];
+        this.waiters = [];
+        waiters.forEach(waiter => waiter());
+    }
+    async waitForAll() {
+        while (this.queue.length > 0 || this.running > 0) {
+            await new Promise((resolve) => {
+                if (this.queue.length === 0 && this.running === 0) {
+                    resolve();
+                    return;
+                }
+                this.waiters.push(resolve);
+                const timeout = setTimeout(() => {
+                    const index = this.waiters.indexOf(resolve);
+                    if (index !== -1) {
+                        this.waiters.splice(index, 1);
+                    }
+                    resolve();
+                }, 100);
+                const originalResolve = resolve;
+                this.waiters[this.waiters.length - 1] = () => {
+                    clearTimeout(timeout);
+                    originalResolve();
+                };
+            });
+            if (this.queue.length > 0 && this.running < this.maxConcurrent) {
+                this.processQueue();
+            }
+        }
+        if (this.taskResults.size > 0) {
+            await Promise.allSettled(Array.from(this.taskResults.values()));
+        }
+    }
+}
+/**
+ * 递归查询生产者
+ */
+class RecursiveQueryProducer {
+    queueManager;
+    taskQueue;
+    jobId;
+    filterCache; // ✅ 优化：缓存过滤规则
+    constructor(queueManager) {
+        this.queueManager = queueManager;
+        this.taskQueue = new AsyncTaskQueue(common_1.queueConfig.producerConcurrency);
+    }
+    /**
+     * 创建 Job 并返回 Job ID（不阻塞）
+     * @param jobId Job ID（必须由外部传入）
+     * @param objectType 对象类型
+     * @param objectName 对象名称
+     * @param config 递归查询配置
+     * @returns Job ID
+     */
+    async createJob(jobId, objectType, objectName, config) {
+        this.jobId = jobId;
+        console.error(`[Producer] 创建 Job: ${jobId}`);
+        console.error(`[Producer] 对象: ${objectType}/${objectName}`);
+        console.error(`[Producer] 配置: 最大深度=${config.maxRecursionDepth}, 并发=${common_1.queueConfig.producerConcurrency}`);
+        // ✅ 优化：初始化过滤规则缓存（预编译正则表达式）
+        this.initializeFilterCache(config);
+        // 创建 Job
+        await this.queueManager.createJob(jobId, objectType, objectName, config);
+        await this.queueManager.updateJobStatus(jobId, 'RUNNING');
+        return jobId;
+    }
+    /**
+     * 提交查询任务（兼容旧版本，阻塞模式）
+     * @param objectType 对象类型
+     * @param objectName 对象名称
+     * @param config 递归查询配置
+     * @returns Job ID
+     */
+    async submitQuery(objectType, objectName, config) {
+        const jobId = uuidv4();
+        await this.createJob(jobId, objectType, objectName, config);
+        // 同步模式: 等待所有任务生成完成
+        await this.generateAllTasks(jobId, objectType, objectName, config);
+        return jobId;
+    }
+    /**
+     * 生成所有任务（可以被外部调用）
+     */
+    async generateAllTasks(jobId, objectType, objectName, config) {
+        try {
+            await this.recursivelyGenerateTasks(jobId, objectType, objectName, 0, config);
+            // 等待所有查询任务完成
+            await this.taskQueue.waitForAll();
+            console.error(`[Producer] Job ${jobId} 所有任务已生成`);
+        }
+        catch (error) {
+            console.error(`[Producer] Job ${jobId} 生成任务失败:`, error);
+            await this.queueManager.updateJobStatus(jobId, 'FAILED');
+            throw error;
+        }
+    }
+    /**
+     * 等待 Job 完成
+     */
+    async waitForCompletion(jobId, timeout) {
+        const startTime = Date.now();
+        const maxTimeout = timeout || common_1.queueConfig.jobTimeout;
+        console.error(`[Producer] 等待 Job ${jobId} 完成...`);
+        while (true) {
+            const job = await this.queueManager.getJob(jobId);
+            if (!job) {
+                throw new Error(`Job ${jobId} 不存在`);
+            }
+            if (job.status === 'COMPLETED' || job.status === 'FAILED' || job.status === 'CANCELLED') {
+                console.error(`[Producer] Job ${jobId} 完成，状态: ${job.status}`);
+                return job;
+            }
+            // 检查超时
+            if (Date.now() - startTime > maxTimeout) {
+                throw new Error(`Job ${jobId} 超时`);
+            }
+            // 等待一段时间后重试
+            await this.sleep(common_1.queueConfig.pollInterval);
+            // 打印进度
+            const progress = await this.queueManager.getProgress(jobId);
+            if (progress) {
+                console.error(`[Producer] Job ${jobId} 进度: ${progress.percentage.toFixed(1)}% (${progress.completedTasks}/${progress.totalTasks})`);
+            }
+        }
+    }
+    /**
+     * 获取进度
+     */
+    async getProgress(jobId) {
+        return await this.queueManager.getProgress(jobId);
+    }
+    /**
+     * 取消 Job
+     */
+    async cancelJob(jobId) {
+        await this.queueManager.updateJobStatus(jobId, 'CANCELLED');
+        console.error(`[Producer] Job ${jobId} 已取消`);
+    }
+    /**
+     * 递归生成任务（并发版本）
+     */
+    async recursivelyGenerateTasks(jobId, objectType, objectName, depth, config) {
+        // 1. 检查深度限制
+        if (depth > config.maxRecursionDepth) {
+            console.error(`[Producer] 达到最大递归深度 ${config.maxRecursionDepth}，停止处理`);
+            return;
+        }
+        const objectKey = `${objectType}/${objectName}`;
+        // 2. 应用过滤规则
+        if (this.shouldFilter(objectType, objectName, config)) {
+            console.error(`[Producer] 对象被过滤规则排除，跳过: ${objectKey}`);
+            return;
+        }
+        // 3. 原子性地检查并标记访问（✅ 优化：从2次Redis操作减少到1次）
+        const isFirstVisit = await this.queueManager.markVisitedIfNotExists(jobId, objectKey);
+        if (!isFirstVisit) {
+            console.error(`[Producer] 对象已访问，跳过: ${objectKey}`);
+            return;
+        }
+        // 5. 背压控制：检查队列大小
+        await this.waitForQueueSpace(jobId, common_1.queueConfig.queueMaxSize);
+        // 6. 使用内部并发队列处理
+        await this.taskQueue.add(objectKey, async () => {
+            console.error(`[Producer] 开始处理对象: ${objectType}/${objectName} (深度: ${depth})`);
+            console.error(`[Producer] 队列状态 - 运行中: ${this.taskQueue.getRunningCount()}, 等待中: ${this.taskQueue.getQueueSize()}`);
+            // 6.1 查询对象
+            const queryResult = await this.queryObject(objectType, objectName);
+            if (!queryResult || !queryResult.success || !queryResult.data) {
+                const errorMsg = `查询对象失败: ${objectType}/${objectName} - ${queryResult?.error || '未知错误'}`;
+                console.error(`[Producer] ${errorMsg}`);
+                // ✅ 记录错误到 Job
+                await this.queueManager.recordJobError(jobId, errorMsg);
+                return;
+            }
+            const objectData = queryResult.data;
+            // 6.2 提取依赖关系
+            const dependencies = this.extractDependencies(objectData);
+            // 6.3 准备缓存数据（移除引用关系）
+            const cacheData = this.prepareCacheData(objectData);
+            // 6.4 压缩并缓存到Redis
+            await this.queueManager.cacheObject(jobId, objectKey, cacheData);
+            // 6.5 创建下载任务
+            const task = {
+                taskId: uuidv4(),
+                jobId,
+                objectKey,
+                taskType: 'DOWNLOAD',
+                objectType,
+                objectName,
+                depth,
+                priority: this.calculatePriority(depth),
+                status: 'PENDING',
+                retryCount: 0,
+                maxRetries: common_1.queueConfig.maxRetries,
+                createdAt: new Date(),
+                metadata: {
+                    cacheKey: `abap:cache:${jobId}:${objectKey}`,
+                    dependencyCount: dependencies.length
+                }
+            };
+            await this.queueManager.createTask(task);
+            console.error(`[Producer] 创建任务: ${objectKey} (优先级: ${task.priority})`);
+            // 6.6 递归处理依赖
+            if (depth < config.maxRecursionDepth && dependencies.length > 0) {
+                console.error(`[Producer] 处理 ${dependencies.length} 个依赖 (深度: ${depth + 1})`);
+                const childPromises = dependencies.map(dep => this.recursivelyGenerateTasks(jobId, dep.objectType, dep.objectName, depth + 1, config));
+                await Promise.allSettled(childPromises);
+            }
+        });
+    }
+    /**
+     * 背压控制：等待队列有空间（✅ 优化：事件驱动替代轮询）
+     */
+    async waitForQueueSpace(jobId, maxSize) {
+        let waitCount = 0;
+        while (true) {
+            const queueSize = await this.queueManager.getQueueSize(jobId);
+            if (queueSize < maxSize) {
+                if (waitCount > 0) {
+                    console.error(`[背压控制] Job ${jobId} 队列恢复 (${queueSize}/${maxSize})，继续生产`);
+                }
+                break;
+            }
+            waitCount++;
+            if (waitCount === 1) {
+                console.error(`[背压控制] Job ${jobId} 队列已满 (${queueSize}/${maxSize})，等待消费者处理...`);
+            }
+            // ✅ 使用事件通知替代轮询
+            await new Promise((resolve) => {
+                let unsubscribe = null;
+                let timeout = null;
+                let resolved = false;
+                const cleanup = async () => {
+                    if (!resolved) {
+                        resolved = true;
+                        if (timeout)
+                            clearTimeout(timeout);
+                        if (unsubscribe)
+                            await unsubscribe();
+                        resolve();
+                    }
+                };
+                // 订阅队列空间事件
+                this.queueManager.subscribeQueueSpace(jobId, () => {
+                    cleanup().catch(() => { });
+                }).then(unsub => {
+                    unsubscribe = unsub;
+                }).catch(() => {
+                    cleanup().catch(() => { });
+                });
+                // 设置超时（防止消息丢失）
+                timeout = setTimeout(() => {
+                    cleanup().catch(() => { });
+                }, common_1.queueConfig.backpressureCheckInterval);
+            });
+        }
+    }
+    /**
+     * 查询对象
+     */
+    async queryObject(objectType, objectName) {
+        try {
+            const result = await (0, handlers_1.handleZObjectQuery)({
+                query_type: objectType,
+                query_name: objectName
+            });
+            if (result && typeof result === 'object' && 'isError' in result && result.isError) {
+                const errorText = Array.isArray(result.content) && result.content.length > 0
+                    ? result.content[0].text || '未知错误'
+                    : '未知错误';
+                return { success: false, error: errorText };
+            }
+            let actualData;
+            if (result && typeof result === 'object' && 'content' in result && Array.isArray(result.content)) {
+                if (result.content.length > 0 && result.content[0].text) {
+                    actualData = result.content[0].text;
+                }
+            }
+            if (actualData) {
+                return { success: true, data: actualData };
+            }
+            else {
+                return { success: false, error: '无有效数据返回' };
+            }
+        }
+        catch (error) {
+            return {
+                success: false,
+                error: error instanceof Error ? error.message : String(error)
+            };
+        }
+    }
+    /**
+     * 提取依赖关系
+     */
+    extractDependencies(objectData) {
+        const dependencies = [];
+        // 提取类依赖
+        if (objectData.ET_REFCLASS && Array.isArray(objectData.ET_REFCLASS)) {
+            for (const cls of objectData.ET_REFCLASS) {
+                if (cls.CLSNAME) {
+                    dependencies.push({
+                        objectType: 'CLASS',
+                        objectName: cls.CLSNAME
+                    });
+                }
+            }
+        }
+        // 提取函数依赖
+        if (objectData.ET_REFFUNCTION && Array.isArray(objectData.ET_REFFUNCTION)) {
+            for (const func of objectData.ET_REFFUNCTION) {
+                // 优先使用 PNAME（程序名），如果不存在则使用 FUNCTIONNAME
+                // PNAME 通常是函数组的主程序名，更准确地表示依赖关系
+                const dependencyName = func.PNAME || func.FUNCTIONNAME;
+                if (dependencyName) {
+                    dependencies.push({
+                        objectType: func.PNAME ? 'PROGRAM' : 'FUNCTION',
+                        objectName: dependencyName
+                    });
+                }
+            }
+        }
+        // 提取程序依赖
+        if (objectData.ET_REFPROGRAM && Array.isArray(objectData.ET_REFPROGRAM)) {
+            for (const prog of objectData.ET_REFPROGRAM) {
+                if (prog.PROGNAME) {
+                    dependencies.push({
+                        objectType: 'PROGRAM',
+                        objectName: prog.PROGNAME
+                    });
+                }
+            }
+        }
+        return dependencies;
+    }
+    /**
+     * 准备缓存数据（包含依赖关系）
+     */
+    prepareCacheData(objectData) {
+        return {
+            ET_FUNCTION: objectData.ET_FUNCTION,
+            ET_PROGRAM: objectData.ET_PROGRAM,
+            ET_CLASSES: objectData.ET_CLASSES,
+            // ✅ 缓存依赖关系（用于生成依赖项XML）
+            ET_REFCLASS: objectData.ET_REFCLASS,
+            ET_REFFUNCTION: objectData.ET_REFFUNCTION,
+            ET_REFPROGRAM: objectData.ET_REFPROGRAM
+        };
+    }
+    /**
+     * 计算任务优先级
+     */
+    calculatePriority(depth) {
+        // 深度越小，优先级越高（数值越小）
+        return depth * 1000 + (Date.now() % 1000);
+    }
+    /**
+     * 初始化过滤规则缓存
+     * ✅ 优化：预编译所有正则表达式，避免每次检查时重复编译
+     * @param config 递归查询配置
+     */
+    initializeFilterCache(config) {
+        // 合并默认过滤规则和用户自定义过滤规则
+        const excludeNames = [
+            ...(common_1.defaultFilters.excludeNames || []),
+            ...(config.recursionFilters?.excludeNames || [])
+        ];
+        const excludeNamePatterns = [
+            ...(common_1.defaultFilters.excludeNamePatterns || []),
+            ...(config.recursionFilters?.excludeNamePatterns || [])
+        ];
+        // 创建过滤缓存（一次性预编译所有规则）
+        this.filterCache = new common_1.FilterCache(excludeNames, excludeNamePatterns);
+        const stats = this.filterCache.getStats();
+        console.error(`[Producer] 过滤规则已加载: 精确匹配=${stats.exactMatchCount}, 模式匹配=${stats.patternCount}`);
+    }
+    /**
+     * 检查对象是否应该被过滤
+     * ✅ 优化：使用预编译的 FilterCache，避免每次检查时重复编译正则表达式
+     * @param objectType 对象类型
+     * @param objectName 对象名称
+     * @param config 递归查询配置
+     * @returns true 表示应该过滤（不递归查询），false 表示不过滤
+     */
+    shouldFilter(objectType, objectName, config) {
+        // 如果没有初始化过滤缓存，先初始化
+        if (!this.filterCache) {
+            this.initializeFilterCache(config);
+        }
+        // 使用预编译的过滤缓存（性能优化）
+        const isFiltered = this.filterCache.matches(objectName);
+        if (isFiltered) {
+            // 获取匹配的模式（用于日志）
+            const matchedPatterns = this.filterCache.getMatchedPatterns(objectName);
+            console.error(`[Producer] 过滤规则: 排除 "${objectName}" (${matchedPatterns.join(', ')})`);
+        }
+        return isFiltered;
+    }
+    /**
+     * 延迟函数
+     */
+    sleep(ms) {
+        return new Promise(resolve => setTimeout(resolve, ms));
+    }
+}
+exports.RecursiveQueryProducer = RecursiveQueryProducer;

package/dist/producer/handlers/handleZObjectQuery.d.ts

@@ -0,0 +1,8 @@
+export declare function handleZObjectQuery(args: any): Promise<{
+    isError: boolean;
+    content: {
+        type: string;
+        text: any;
+    }[];
+}>;
+//# sourceMappingURL=handleZObjectQuery.d.ts.map

package/dist/producer/handlers/handleZObjectQuery.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"handleZObjectQuery.d.ts","sourceRoot":"","sources":["../../../src/producer/handlers/handleZObjectQuery.ts"],"names":[],"mappings":"AAEA,wBAAsB,kBAAkB,CAAC,IAAI,EAAE,GAAG;;;;;;GAmBjD"}

package/dist/producer/handlers/handleZObjectQuery.js

@@ -0,0 +1,24 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.handleZObjectQuery = handleZObjectQuery;
+const utils_1 = require("../../common/utils");
+async function handleZObjectQuery(args) {
+    try {
+        if (!args?.query_type) {
+            console.error('query_type:', args);
+            throw new utils_1.McpError(utils_1.ErrorCode.InvalidParams, 'Query type is required');
+        }
+        if (!args?.query_name) {
+            console.error('query_name:', args);
+            throw new utils_1.McpError(utils_1.ErrorCode.InvalidParams, 'Query name is required');
+        }
+        const encodedQueryType = encodeURIComponent(args.query_type);
+        const encodedQueryName = encodeURIComponent(args.query_name);
+        const url = `${await (0, utils_1.getBaseUrl)()}/sap/bc/z_object_query?type=${encodedQueryType}&name=${encodedQueryName}`;
+        const response = await (0, utils_1.makeAdtRequest)(url, 'GET', 100000);
+        return (0, utils_1.return_response)(response);
+    }
+    catch (error) {
+        return (0, utils_1.return_error)(error);
+    }
+}

package/dist/producer/handlers/index.d.ts

@@ -0,0 +1,6 @@
+/**
+ * 生产者使用的 Handlers
+ * 用于查询对象引用关系
+ */
+export * from './handleZObjectQuery';
+//# sourceMappingURL=index.d.ts.map

package/dist/producer/handlers/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/producer/handlers/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,cAAc,sBAAsB,CAAC"}

package/dist/producer/handlers/index.js

@@ -0,0 +1,21 @@
+"use strict";
+/**
+ * 生产者使用的 Handlers
+ * 用于查询对象引用关系
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./handleZObjectQuery"), exports);

package/dist/producer/index.d.ts

@@ -0,0 +1,7 @@
+/**
+ * 生产者组件导出
+ * 负责递归查询和任务生成
+ */
+export * from './RecursiveQueryProducer';
+export * from './handlers';
+//# sourceMappingURL=index.d.ts.map

package/dist/producer/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/producer/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,cAAc,0BAA0B,CAAC;AACzC,cAAc,YAAY,CAAC"}

package/dist/producer/index.js

@@ -0,0 +1,22 @@
+"use strict";
+/**
+ * 生产者组件导出
+ * 负责递归查询和任务生成
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./RecursiveQueryProducer"), exports);
+__exportStar(require("./handlers"), exports);