lua-cli 2.5.8 → 3.0.0-alpha.10

package/dist/utils/job-management.js

@@ -0,0 +1,264 @@
+/**
+ * Job Management Utilities
+ * Handles job creation via API and YAML configuration management
+ */
+import fs from "fs";
+import path from "path";
+import yaml from "js-yaml";
+import JobApi from '../api/job.api.service.js';
+import { BASE_URLS } from '../config/constants.js';
+import { loadApiKey } from '../services/auth.js';
+import { COMPILE_FILES, SKILL_DEFAULTS, YAML_FORMAT, } from '../config/compile.constants.js';
+/**
+ * Ensures all detected jobs exist in the YAML config with valid job IDs.
+ * If a job doesn't exist or has no ID, creates it via the API.
+ *
+ * @param jobsArray - Array of jobs detected from source code
+ * @param config - The skill configuration from lua.skill.yaml
+ * @returns Updated jobs array with valid job IDs
+ */
+export async function ensureJobsExistInYaml(jobsArray, config) {
+    const updatedJobsArray = [];
+    let yamlUpdated = false;
+    const existingJobs = config?.jobs || [];
+    const existingJobsMap = createJobsMap(existingJobs);
+    // Process each detected job
+    for (const job of jobsArray) {
+        const existingJob = existingJobsMap.get(job.name);
+        if (existingJob && existingJob.jobId && existingJob.jobId !== '') {
+            // Job exists with valid jobId - reuse it
+            updatedJobsArray.push({
+                ...job,
+                jobId: existingJob.jobId
+            });
+        }
+        else {
+            // Job doesn't exist or missing jobId - create via API
+            const createdJob = await createJobViaApi(job, config, existingJobs, existingJob);
+            updatedJobsArray.push(createdJob);
+            yamlUpdated = true;
+        }
+    }
+    // Update YAML file if any changes were made
+    if (yamlUpdated) {
+        await updateYamlWithJobs(config);
+    }
+    return updatedJobsArray;
+}
+/**
+ * Creates a map of existing jobs for quick lookup by name.
+ *
+ * @param existingJobs - Array of jobs from YAML config
+ * @returns Map of job names to job objects
+ */
+function createJobsMap(existingJobs) {
+    const map = new Map();
+    existingJobs.forEach((job) => {
+        map.set(job.name, job);
+    });
+    return map;
+}
+/**
+ * Creates a new job via the Lua API and updates the config.
+ *
+ * @param job - The job to create
+ * @param config - The skill configuration
+ * @param existingJobs - Array of existing jobs (mutated)
+ * @param existingJob - The existing job entry if one exists (may be without ID)
+ * @returns The job with its new ID
+ */
+async function createJobViaApi(job, config, existingJobs, existingJob) {
+    try {
+        // Validate prerequisites
+        const apiKey = await loadApiKey();
+        if (!apiKey) {
+            throw new Error("No API key found. Run 'lua auth configure' first.");
+        }
+        const agentId = config?.agent?.agentId;
+        if (!agentId) {
+            throw new Error("No agent ID found in lua.skill.yaml. Run 'lua init' first.");
+        }
+        // Create job via API
+        const jobPayload = {
+            name: job.name,
+            description: job.description || `A Lua job for ${job.name}`,
+            context: job.context || '',
+            schedule: job.schedule || { type: 'cron', expression: '0 0 * * *' },
+            timeout: job.timeout,
+            retry: job.retry,
+            metadata: job.metadata
+        };
+        const jobApi = new JobApi(BASE_URLS.API, apiKey, agentId);
+        const result = await jobApi.createJob(jobPayload);
+        if (result.success && result.data && result.data.id) {
+            const newJobId = result.data.id;
+            // Update YAML config with new job ID
+            if (!existingJob) {
+                existingJobs.push({
+                    name: job.name || '',
+                    version: job.version || SKILL_DEFAULTS.VERSION,
+                    jobId: newJobId,
+                    schedule: job.schedule
+                });
+            }
+            else {
+                existingJob.jobId = newJobId;
+                existingJob.schedule = job.schedule;
+            }
+            // Add jobs to config if not already present
+            if (!config.jobs) {
+                config.jobs = existingJobs;
+            }
+            return {
+                ...job,
+                jobId: newJobId
+            };
+        }
+        else {
+            console.error(`❌ Failed to create job ${job.name}:`, result.error);
+            throw new Error(result.error?.message || 'Failed to create job - no ID returned from API');
+        }
+    }
+    catch (error) {
+        console.error(`❌ Failed to create job ${job.name}:`, error);
+        throw error;
+    }
+}
+/**
+ * Updates the lua.skill.yaml file with the current jobs array.
+ * Ensures no undefined values are written to the YAML file.
+ *
+ * @param config - Current skill configuration to merge with
+ */
+async function updateYamlWithJobs(config) {
+    const jobs = config?.jobs || [];
+    // Clean jobs array to ensure no undefined values
+    const cleanedJobs = jobs.map(job => ({
+        name: job.name || '',
+        version: job.version || SKILL_DEFAULTS.VERSION,
+        jobId: job.jobId || '',
+        schedule: job.schedule || { type: 'cron', expression: '0 0 * * *' }
+    }));
+    // Update config with cleaned jobs array
+    const updatedConfig = {
+        ...config,
+        jobs: cleanedJobs
+    };
+    // Write updated YAML with consistent formatting
+    const yamlPath = path.join(process.cwd(), COMPILE_FILES.LUA_SKILL_YAML);
+    const yamlContent = yaml.dump(updatedConfig, {
+        indent: YAML_FORMAT.INDENT,
+        lineWidth: YAML_FORMAT.LINE_WIDTH,
+        noRefs: YAML_FORMAT.NO_REFS,
+        replacer: (key, value) => {
+            // Replace undefined values with empty strings
+            return value === undefined ? '' : value;
+        }
+    });
+    fs.writeFileSync(yamlPath, yamlContent);
+}
+/**
+ * Syncs the server jobs with the YAML configuration.
+ * Performs a two-way sync:
+ * 1. Deletes/deactivates jobs from server that aren't in YAML
+ * 2. Updates YAML with active version numbers from server
+ *
+ * @param config - The skill configuration from lua.skill.yaml
+ * @returns Array of messages about sync operations
+ */
+export async function syncServerJobsWithYaml(config) {
+    const messages = [];
+    let yamlNeedsUpdate = false;
+    try {
+        // Validate prerequisites
+        const apiKey = await loadApiKey();
+        if (!apiKey) {
+            console.warn("⚠️  No API key found. Skipping server job sync.");
+            return messages;
+        }
+        const agentId = config?.agent?.agentId;
+        if (!agentId) {
+            console.warn("⚠️  No agent ID found in lua.skill.yaml. Skipping server job sync.");
+            return messages;
+        }
+        // Get jobs from server
+        const jobApi = new JobApi(BASE_URLS.API, apiKey, agentId);
+        const serverJobsResponse = await jobApi.getJobs();
+        if (!serverJobsResponse.success || !serverJobsResponse.data?.jobs) {
+            console.warn("⚠️  Could not retrieve server jobs. Skipping server job sync.");
+            return messages;
+        }
+        const serverJobs = serverJobsResponse.data.jobs;
+        const yamlJobs = config?.jobs || [];
+        // Create maps for efficient lookup
+        const yamlJobsMap = new Map(yamlJobs
+            .filter((job) => job.jobId)
+            .map((job) => [job.jobId, job]));
+        const serverJobsMap = new Map(serverJobs.map(job => [job.id, job]));
+        // Part 1: Delete jobs from server that aren't in YAML
+        const jobsToDelete = serverJobs.filter(serverJob => !yamlJobsMap.has(serverJob.id));
+        for (const job of jobsToDelete) {
+            try {
+                const deleteResponse = await jobApi.deleteJob(job.id);
+                if (deleteResponse.success && deleteResponse.data) {
+                    if (deleteResponse.data.deleted) {
+                        const msg = `✅ Deleted job "${job.name}" from server`;
+                        messages.push(msg);
+                        console.log(msg);
+                    }
+                    else if (deleteResponse.data.deactivated) {
+                        const msg = `⚠️  Job "${job.name}" has versions and cannot be deleted. It has been deactivated instead.`;
+                        messages.push(msg);
+                        console.warn(msg);
+                    }
+                }
+                else {
+                    const msg = `❌ Failed to delete job "${job.name}": ${deleteResponse.error?.message || 'Unknown error'}`;
+                    messages.push(msg);
+                    console.error(msg);
+                }
+            }
+            catch (error) {
+                const msg = `❌ Error deleting job "${job.name}": ${error}`;
+                messages.push(msg);
+                console.error(msg);
+            }
+        }
+        // Part 2: Sync version numbers from server to YAML
+        const updatedYamlJobs = yamlJobs.map((yamlJob) => {
+            const serverJob = serverJobsMap.get(yamlJob.jobId);
+            if (serverJob && serverJob.versions && serverJob.versions.length > 0) {
+                // Find the active version on the server
+                const activeVersion = serverJob.versions.find((v) => v.isActive);
+                if (activeVersion && activeVersion.version !== yamlJob.version) {
+                    const msg = `📝 Updated "${yamlJob.name}" job version in YAML: ${yamlJob.version} → ${activeVersion.version}`;
+                    messages.push(msg);
+                    console.log(msg);
+                    yamlNeedsUpdate = true;
+                    return {
+                        ...yamlJob,
+                        version: activeVersion.version,
+                        schedule: activeVersion.schedule || yamlJob.schedule
+                    };
+                }
+            }
+            return yamlJob;
+        });
+        // Update YAML file if versions changed
+        if (yamlNeedsUpdate) {
+            const updatedConfig = {
+                ...config,
+                jobs: updatedYamlJobs
+            };
+            await updateYamlWithJobs(updatedConfig);
+            console.log("✅ YAML job versions synced with server");
+        }
+        if (jobsToDelete.length === 0 && !yamlNeedsUpdate) {
+            console.log("✅ Server jobs and YAML are fully in sync");
+        }
+    }
+    catch (error) {
+        console.error("❌ Error syncing server jobs:", error);
+    }
+    return messages;
+}

package/dist/utils/postprocessor-management.d.ts

@@ -0,0 +1,9 @@
+/**
+ * PostProcessor Management Utilities
+ */
+import { SkillConfig } from '../types/compile.types.js';
+/**
+ * Ensures all detected postprocessors exist in the YAML config with valid IDs.
+ */
+export declare function ensurePostProcessorsExistInYaml(postprocessorsArray: any[], config: SkillConfig): Promise<any[]>;
+export declare function syncServerPostProcessorsWithYaml(config: SkillConfig): Promise<string[]>;

package/dist/utils/postprocessor-management.js

@@ -0,0 +1,118 @@
+/**
+ * PostProcessor Management Utilities
+ */
+import fs from "fs";
+import path from "path";
+import yaml from "js-yaml";
+import PostProcessorApi from '../api/postprocessor.api.service.js';
+import { BASE_URLS } from '../config/constants.js';
+import { loadApiKey } from '../services/auth.js';
+import { SKILL_DEFAULTS, YAML_FORMAT } from '../config/compile.constants.js';
+/**
+ * Ensures all detected postprocessors exist in the YAML config with valid IDs.
+ */
+export async function ensurePostProcessorsExistInYaml(postprocessorsArray, config) {
+    const updatedPostProcessorsArray = [];
+    let yamlUpdated = false;
+    const existingPostProcessors = config?.postprocessors || [];
+    const existingMap = new Map(existingPostProcessors.map((p) => [p.name, p]));
+    for (const postprocessor of postprocessorsArray) {
+        const existing = existingMap.get(postprocessor.name);
+        if (existing && existing.postprocessorId) {
+            updatedPostProcessorsArray.push({
+                ...postprocessor,
+                postprocessorId: existing.postprocessorId
+            });
+        }
+        else {
+            const created = await createPostProcessorViaApi(postprocessor, config, existingPostProcessors, existing);
+            updatedPostProcessorsArray.push(created);
+            yamlUpdated = true;
+        }
+    }
+    if (yamlUpdated) {
+        await updateYamlWithPostProcessors(config);
+    }
+    return updatedPostProcessorsArray;
+}
+async function createPostProcessorViaApi(postprocessor, config, existingPostProcessors, existingPostProcessor) {
+    try {
+        const apiKey = await loadApiKey();
+        if (!apiKey)
+            throw new Error("No API key found. Run 'lua auth configure' first.");
+        const agentId = config?.agent?.agentId;
+        if (!agentId)
+            throw new Error("No agent ID found in lua.skill.yaml. Run 'lua init' first.");
+        const payload = {
+            name: postprocessor.name,
+            description: postprocessor.description || `A Lua postprocessor for ${postprocessor.name}`,
+            context: postprocessor.context || ''
+        };
+        const api = new PostProcessorApi(BASE_URLS.API, apiKey, agentId);
+        const result = await api.createPostProcessor(payload);
+        if (result.success && result.data && result.data.id) {
+            const newId = result.data.id;
+            if (!existingPostProcessor) {
+                existingPostProcessors.push({
+                    name: postprocessor.name || '',
+                    version: postprocessor.version || SKILL_DEFAULTS.VERSION,
+                    postprocessorId: newId
+                });
+            }
+            else {
+                existingPostProcessor.postprocessorId = newId;
+            }
+            if (!config.postprocessors) {
+                config.postprocessors = existingPostProcessors;
+            }
+            return {
+                ...postprocessor,
+                postprocessorId: newId
+            };
+        }
+        else {
+            throw new Error(result.error?.message || "Failed to create postprocessor");
+        }
+    }
+    catch (error) {
+        console.warn(`Warning: Could not create postprocessor ${postprocessor.name}:`, error);
+        return postprocessor;
+    }
+}
+async function updateYamlWithPostProcessors(config) {
+    const yamlPath = path.join(process.cwd(), 'lua.skill.yaml');
+    const yamlContent = yaml.dump(config, { ...YAML_FORMAT, sortKeys: (a, b) => {
+            const order = ['agent', 'skills', 'webhooks', 'jobs', 'preprocessors', 'postprocessors', 'skill'];
+            const aIndex = order.indexOf(a);
+            const bIndex = order.indexOf(b);
+            if (aIndex !== -1 && bIndex !== -1)
+                return aIndex - bIndex;
+            if (aIndex !== -1)
+                return -1;
+            if (bIndex !== -1)
+                return 1;
+            return a.localeCompare(b);
+        } });
+    fs.writeFileSync(yamlPath, yamlContent);
+}
+export async function syncServerPostProcessorsWithYaml(config) {
+    const messages = [];
+    try {
+        const apiKey = await loadApiKey();
+        if (!apiKey) {
+            console.warn("⚠️  No API key found. Skipping server postprocessor sync.");
+            return messages;
+        }
+        const agentId = config?.agent?.agentId;
+        if (!agentId) {
+            console.warn("⚠️  No agent ID found. Skipping server postprocessor sync.");
+            return messages;
+        }
+        // Server sync will be implemented when backend is ready
+        return messages;
+    }
+    catch (error) {
+        console.warn("⚠️  Error syncing postprocessors:", error);
+        return messages;
+    }
+}

package/dist/utils/pre-bundle-jobs.d.ts

@@ -0,0 +1,26 @@
+/**
+ * Pre-Bundle Jobs - Extract and bundle Jobs.create() before tool bundling
+ *
+ * Flow:
+ * 1. Detect Jobs.create() in source
+ * 2. Extract execute function
+ * 3. Bundle execute with ALL dependencies
+ * 4. Replace execute with placeholder
+ * 5. Bundle tool/webhook/job
+ * 6. Replace placeholder with bundled job code
+ * 7. Compress
+ */
+import { Project } from 'ts-morph';
+/**
+ * Pre-processes source file to extract and bundle Jobs.create() execute functions
+ * Returns modified source code with placeholders
+ */
+export declare function preBundleJobsInSource(sourceFilePath: string, project: Project, distDir: string): Promise<{
+    modifiedSource: string;
+    jobBundles: Map<string, string>;
+}>;
+/**
+ * Replaces placeholders in bundled code with actual job bundles
+ * The bundled job code is NOT compressed - it's JavaScript that will be part of the tool
+ */
+export declare function replaceJobPlaceholders(bundledCode: string, jobBundles: Map<string, string>): string;

package/dist/utils/pre-bundle-jobs.js

@@ -0,0 +1,176 @@
+/**
+ * Pre-Bundle Jobs - Extract and bundle Jobs.create() before tool bundling
+ *
+ * Flow:
+ * 1. Detect Jobs.create() in source
+ * 2. Extract execute function
+ * 3. Bundle execute with ALL dependencies
+ * 4. Replace execute with placeholder
+ * 5. Bundle tool/webhook/job
+ * 6. Replace placeholder with bundled job code
+ * 7. Compress
+ */
+import { Node } from 'ts-morph';
+import { build } from 'esbuild';
+import fs from 'fs';
+import path from 'path';
+import { sandboxGlobalsPlugin } from './bundling.js';
+const JOB_PLACEHOLDER = '__BUNDLED_JOB_EXECUTE__';
+/**
+ * Pre-processes source file to extract and bundle Jobs.create() execute functions
+ * Returns modified source code with placeholders
+ */
+export async function preBundleJobsInSource(sourceFilePath, project, distDir) {
+    const sourceFile = project.getSourceFile(sourceFilePath);
+    if (!sourceFile) {
+        return { modifiedSource: fs.readFileSync(sourceFilePath, 'utf8'), jobBundles: new Map() };
+    }
+    const jobBundles = new Map();
+    const bundlingPromises = [];
+    let jobIndex = 0;
+    let modifiedSource = sourceFile.getFullText();
+    // Find all Jobs.create() calls
+    sourceFile.forEachDescendant((node) => {
+        if (Node.isCallExpression(node)) {
+            const expression = node.getExpression();
+            // Check if this is Jobs.create()
+            if (Node.isPropertyAccessExpression(expression)) {
+                const object = expression.getExpression();
+                const property = expression.getName();
+                if (object.getText() === 'Jobs' && property === 'create') {
+                    jobIndex++;
+                    const placeholder = `${JOB_PLACEHOLDER}_${jobIndex}`;
+                    console.log(`[PreBundleJobs] Found Jobs.create() #${jobIndex}`);
+                    // Get the config object argument
+                    const args = node.getArguments();
+                    if (args.length > 0 && Node.isObjectLiteralExpression(args[0])) {
+                        const configObj = args[0];
+                        // Find the execute property
+                        const executeProperty = configObj.getProperty('execute');
+                        if (executeProperty && Node.isPropertyAssignment(executeProperty)) {
+                            const executeInitializer = executeProperty.getInitializer();
+                            if (executeInitializer) {
+                                const executeCode = executeInitializer.getText();
+                                console.log(`[PreBundleJobs] Extracting execute function for bundling...`);
+                                // Bundle the execute function with dependencies (async)
+                                const bundlePromise = bundleJobExecuteFunction(executeCode, `job_${jobIndex}`, distDir, sourceFilePath).then(bundledCode => {
+                                    if (bundledCode) {
+                                        jobBundles.set(placeholder, bundledCode);
+                                        console.log(`[PreBundleJobs] ✅ Bundled job #${jobIndex} with dependencies`);
+                                    }
+                                });
+                                bundlingPromises.push(bundlePromise);
+                                // Replace execute function with a function that references the bundled code
+                                // Don't use string placeholder - use a reference
+                                const originalText = executeProperty.getText();
+                                const replacementText = `execute: ${placeholder}`;
+                                modifiedSource = modifiedSource.replace(originalText, replacementText);
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    });
+    // Wait for all bundling to complete
+    if (bundlingPromises.length > 0) {
+        console.log(`[PreBundleJobs] Waiting for ${bundlingPromises.length} job(s) to bundle...`);
+        await Promise.all(bundlingPromises);
+        console.log(`[PreBundleJobs] ✅ All job bundles complete`);
+    }
+    return { modifiedSource, jobBundles };
+}
+/**
+ * Bundles a job execute function with all its dependencies
+ */
+async function bundleJobExecuteFunction(executeCode, jobName, distDir, sourceFilePath) {
+    const tempDir = path.join(distDir, '.temp');
+    if (!fs.existsSync(tempDir)) {
+        fs.mkdirSync(tempDir, { recursive: true });
+    }
+    const tempOutput = path.join(tempDir, `${jobName}-bundled.js`);
+    try {
+        // Read the original source file to get its imports
+        const sourceContent = fs.readFileSync(sourceFilePath, 'utf8');
+        // Extract all import statements from the source
+        const importPattern = /^import\s+.+\s+from\s+['"][^'"]+['"];?$/gm;
+        const imports = sourceContent.match(importPattern) || [];
+        // Filter to only imports that might be used (exclude lua-cli internal classes and APIs)
+        const relevantImports = imports.filter(imp => !imp.includes('LuaTool') &&
+            !imp.includes('LuaWebhook') &&
+            !imp.includes('LuaJob') &&
+            !imp.includes('lua-cli') && // Exclude all lua-cli imports (they're available in sandbox)
+            !imp.includes('api-exports') && // Exclude api-exports
+            !imp.includes('../../../dist/api-exports') // Exclude direct api-exports imports
+        );
+        console.log(`[PreBundleJobs] Including ${relevantImports.length} import(s) in job bundle`);
+        // Create a TypeScript module with the execute function and all imports
+        const moduleCode = `
+// Auto-generated job execute bundle
+${relevantImports.join('\n')}
+
+// The execute function with all dependencies available
+const executeFunc = ${executeCode};
+
+export default executeFunc;
+`;
+        // Bundle with esbuild using stdin to preserve import resolution
+        await build({
+            stdin: {
+                contents: moduleCode,
+                resolveDir: path.dirname(sourceFilePath),
+                sourcefile: sourceFilePath,
+                loader: 'ts',
+            },
+            bundle: true,
+            platform: 'node',
+            target: 'node16',
+            format: 'cjs',
+            minify: true,
+            outfile: tempOutput,
+            external: [],
+            plugins: [sandboxGlobalsPlugin],
+        });
+        // Read bundled code
+        const bundledCode = fs.readFileSync(tempOutput, 'utf8');
+        console.log(`[PreBundleJobs] Bundle size: ${(bundledCode.length / 1024).toFixed(2)}KB`);
+        // DON'T compress - just wrap for VM execution
+        // Compression happens later when the entire tool is bundled
+        const wrappedCode = `(async (job) => {
+  ${bundledCode}
+  const executeFunction = module.exports.default || module.exports;
+  return await executeFunction(job);
+})`;
+        // Cleanup temp files
+        try {
+            fs.unlinkSync(tempOutput);
+        }
+        catch { }
+        // Return uncompressed bundled code
+        return wrappedCode;
+    }
+    catch (error) {
+        console.warn(`[PreBundleJobs] Error bundling job ${jobName}:`, error);
+        try {
+            if (fs.existsSync(tempOutput))
+                fs.unlinkSync(tempOutput);
+        }
+        catch { }
+        return '';
+    }
+}
+/**
+ * Replaces placeholders in bundled code with actual job bundles
+ * The bundled job code is NOT compressed - it's JavaScript that will be part of the tool
+ */
+export function replaceJobPlaceholders(bundledCode, jobBundles) {
+    let result = bundledCode;
+    jobBundles.forEach((bundledJobCode, placeholder) => {
+        // The placeholder is used as an identifier (not a string), just replace it directly
+        // It appears as: execute: __BUNDLED_JOB_EXECUTE___1
+        const placeholderPattern = new RegExp(placeholder, 'g');
+        result = result.replace(placeholderPattern, bundledJobCode);
+        console.log(`[PreBundleJobs] ✅ Replaced ${placeholder} with bundled code (${(bundledJobCode.length / 1024).toFixed(1)}KB)`);
+    });
+    return result;
+}

package/dist/utils/preprocessor-management.d.ts

@@ -0,0 +1,9 @@
+/**
+ * PreProcessor Management Utilities
+ */
+import { SkillConfig } from '../types/compile.types.js';
+/**
+ * Ensures all detected preprocessors exist in the YAML config with valid IDs.
+ */
+export declare function ensurePreProcessorsExistInYaml(preprocessorsArray: any[], config: SkillConfig): Promise<any[]>;
+export declare function syncServerPreProcessorsWithYaml(config: SkillConfig): Promise<string[]>;