codeceptjs 3.7.4 → 3.7.5-beta.10

package/lib/recorder.js

@@ -379,6 +379,15 @@ module.exports = {
   toString() {
     return `Queue: ${currentQueue()}\n\nTasks: ${this.scheduled()}`
   },
+
+  /**
+   * Get current session ID
+   * @return {string|null}
+   * @inner
+   */
+  getCurrentSessionId() {
+    return sessionId
+  },
 }
 
 function getTimeoutPromise(timeoutMs, taskName) {

package/lib/test-server.js

@@ -0,0 +1,323 @@
+const express = require('express')
+const fs = require('fs')
+const path = require('path')
+
+/**
+ * Internal API test server to replace json-server dependency
+ * Provides REST API endpoints for testing CodeceptJS helpers
+ */
+class TestServer {
+  constructor(config = {}) {
+    this.app = express()
+    this.server = null
+    this.port = config.port || 8010
+    this.host = config.host || 'localhost'
+    this.dbFile = config.dbFile || path.join(__dirname, '../test/data/rest/db.json')
+    this.lastModified = null
+    this.data = this.loadData()
+
+    this.setupMiddleware()
+    this.setupRoutes()
+    this.setupFileWatcher()
+  }
+
+  loadData() {
+    try {
+      const content = fs.readFileSync(this.dbFile, 'utf8')
+      const data = JSON.parse(content)
+      // Update lastModified time when loading data
+      if (fs.existsSync(this.dbFile)) {
+        this.lastModified = fs.statSync(this.dbFile).mtime
+      }
+      console.log('[Data Load] Loaded data from file:', JSON.stringify(data))
+      return data
+    } catch (err) {
+      console.warn(`[Data Load] Could not load data file ${this.dbFile}:`, err.message)
+      console.log('[Data Load] Using fallback default data')
+      return {
+        posts: [{ id: 1, title: 'json-server', author: 'davert' }],
+        user: { name: 'john', password: '123456' },
+      }
+    }
+  }
+
+  reloadData() {
+    console.log('[Reload] Reloading data from file...')
+    this.data = this.loadData()
+    console.log('[Reload] Data reloaded successfully')
+    return this.data
+  }
+
+  saveData() {
+    try {
+      fs.writeFileSync(this.dbFile, JSON.stringify(this.data, null, 2))
+      console.log('[Save] Data saved to file')
+      // Force update modification time to ensure auto-reload works
+      const now = new Date()
+      fs.utimesSync(this.dbFile, now, now)
+      this.lastModified = now
+      console.log('[Save] File modification time updated')
+    } catch (err) {
+      console.warn(`[Save] Could not save data file ${this.dbFile}:`, err.message)
+    }
+  }
+
+  setupMiddleware() {
+    // Parse JSON bodies
+    this.app.use(express.json())
+
+    // Parse URL-encoded bodies
+    this.app.use(express.urlencoded({ extended: true }))
+
+    // CORS support
+    this.app.use((req, res, next) => {
+      res.header('Access-Control-Allow-Origin', '*')
+      res.header('Access-Control-Allow-Methods', 'GET, POST, PUT, PATCH, DELETE, OPTIONS')
+      res.header('Access-Control-Allow-Headers', 'Origin, X-Requested-With, Content-Type, Accept, Authorization, X-Test')
+
+      if (req.method === 'OPTIONS') {
+        res.status(200).end()
+        return
+      }
+      next()
+    })
+
+    // Auto-reload middleware - check if file changed before each request
+    this.app.use((req, res, next) => {
+      try {
+        if (fs.existsSync(this.dbFile)) {
+          const stats = fs.statSync(this.dbFile)
+          if (!this.lastModified || stats.mtime > this.lastModified) {
+            console.log(`[Auto-reload] Database file changed (${this.dbFile}), reloading data...`)
+            console.log(`[Auto-reload] Old mtime: ${this.lastModified}, New mtime: ${stats.mtime}`)
+            this.reloadData()
+            this.lastModified = stats.mtime
+            console.log(`[Auto-reload] Data reloaded, user name is now: ${this.data.user?.name}`)
+          }
+        }
+      } catch (err) {
+        console.warn('[Auto-reload] Error checking file modification time:', err.message)
+      }
+      next()
+    })
+
+    // Logging middleware
+    this.app.use((req, res, next) => {
+      console.log(`${req.method} ${req.path}`)
+      next()
+    })
+  }
+
+  setupRoutes() {
+    // Reload endpoint (for testing)
+    this.app.post('/_reload', (req, res) => {
+      this.reloadData()
+      res.json({ message: 'Data reloaded', data: this.data })
+    })
+
+    // Headers endpoint (for header testing)
+    this.app.get('/headers', (req, res) => {
+      res.json(req.headers)
+    })
+
+    this.app.post('/headers', (req, res) => {
+      res.json(req.headers)
+    })
+
+    // User endpoints
+    this.app.get('/user', (req, res) => {
+      console.log(`[GET /user] Serving user data: ${JSON.stringify(this.data.user)}`)
+      res.json(this.data.user)
+    })
+
+    this.app.post('/user', (req, res) => {
+      this.data.user = { ...this.data.user, ...req.body }
+      this.saveData()
+      res.status(201).json(this.data.user)
+    })
+
+    this.app.patch('/user', (req, res) => {
+      this.data.user = { ...this.data.user, ...req.body }
+      this.saveData()
+      res.json(this.data.user)
+    })
+
+    this.app.put('/user', (req, res) => {
+      this.data.user = req.body
+      this.saveData()
+      res.json(this.data.user)
+    })
+
+    // Posts endpoints
+    this.app.get('/posts', (req, res) => {
+      res.json(this.data.posts)
+    })
+
+    this.app.get('/posts/:id', (req, res) => {
+      const id = parseInt(req.params.id)
+      const post = this.data.posts.find(p => p.id === id)
+
+      if (!post) {
+        // Return empty object instead of 404 for json-server compatibility
+        return res.json({})
+      }
+
+      res.json(post)
+    })
+
+    this.app.post('/posts', (req, res) => {
+      const newId = Math.max(...this.data.posts.map(p => p.id || 0)) + 1
+      const newPost = { id: newId, ...req.body }
+
+      this.data.posts.push(newPost)
+      this.saveData()
+      res.status(201).json(newPost)
+    })
+
+    this.app.put('/posts/:id', (req, res) => {
+      const id = parseInt(req.params.id)
+      const postIndex = this.data.posts.findIndex(p => p.id === id)
+
+      if (postIndex === -1) {
+        return res.status(404).json({ error: 'Post not found' })
+      }
+
+      this.data.posts[postIndex] = { id, ...req.body }
+      this.saveData()
+      res.json(this.data.posts[postIndex])
+    })
+
+    this.app.patch('/posts/:id', (req, res) => {
+      const id = parseInt(req.params.id)
+      const postIndex = this.data.posts.findIndex(p => p.id === id)
+
+      if (postIndex === -1) {
+        return res.status(404).json({ error: 'Post not found' })
+      }
+
+      this.data.posts[postIndex] = { ...this.data.posts[postIndex], ...req.body }
+      this.saveData()
+      res.json(this.data.posts[postIndex])
+    })
+
+    this.app.delete('/posts/:id', (req, res) => {
+      const id = parseInt(req.params.id)
+      const postIndex = this.data.posts.findIndex(p => p.id === id)
+
+      if (postIndex === -1) {
+        return res.status(404).json({ error: 'Post not found' })
+      }
+
+      const deletedPost = this.data.posts.splice(postIndex, 1)[0]
+      this.saveData()
+      res.json(deletedPost)
+    })
+
+    // File upload endpoint (basic implementation)
+    this.app.post('/upload', (req, res) => {
+      // Simple upload simulation - for more complex file uploads,
+      // multer would be needed but basic tests should work
+      res.json({
+        message: 'File upload endpoint available',
+        headers: req.headers,
+        body: req.body,
+      })
+    })
+
+    // Comments endpoints (for ApiDataFactory tests)
+    this.app.get('/comments', (req, res) => {
+      res.json(this.data.comments || [])
+    })
+
+    this.app.post('/comments', (req, res) => {
+      if (!this.data.comments) this.data.comments = []
+      const newId = Math.max(...this.data.comments.map(c => c.id || 0), 0) + 1
+      const newComment = { id: newId, ...req.body }
+
+      this.data.comments.push(newComment)
+      this.saveData()
+      res.status(201).json(newComment)
+    })
+
+    this.app.delete('/comments/:id', (req, res) => {
+      if (!this.data.comments) this.data.comments = []
+      const id = parseInt(req.params.id)
+      const commentIndex = this.data.comments.findIndex(c => c.id === id)
+
+      if (commentIndex === -1) {
+        return res.status(404).json({ error: 'Comment not found' })
+      }
+
+      const deletedComment = this.data.comments.splice(commentIndex, 1)[0]
+      this.saveData()
+      res.json(deletedComment)
+    })
+
+    // Generic catch-all for other endpoints
+    this.app.use((req, res) => {
+      res.status(404).json({ error: 'Endpoint not found' })
+    })
+  }
+
+  setupFileWatcher() {
+    if (fs.existsSync(this.dbFile)) {
+      fs.watchFile(this.dbFile, (current, previous) => {
+        if (current.mtime !== previous.mtime) {
+          console.log('Database file changed, reloading data...')
+          this.reloadData()
+        }
+      })
+    }
+  }
+
+  start() {
+    return new Promise((resolve, reject) => {
+      this.server = this.app.listen(this.port, this.host, err => {
+        if (err) {
+          reject(err)
+        } else {
+          console.log(`Test server running on http://${this.host}:${this.port}`)
+          resolve(this.server)
+        }
+      })
+    })
+  }
+
+  stop() {
+    return new Promise(resolve => {
+      if (this.server) {
+        this.server.close(() => {
+          console.log('Test server stopped')
+          resolve()
+        })
+      } else {
+        resolve()
+      }
+    })
+  }
+}
+
+module.exports = TestServer
+
+// CLI usage
+if (require.main === module) {
+  const config = {
+    port: process.env.PORT || 8010,
+    host: process.env.HOST || '0.0.0.0',
+    dbFile: process.argv[2] || path.join(__dirname, '../test/data/rest/db.json'),
+  }
+
+  const server = new TestServer(config)
+  server.start().catch(console.error)
+
+  // Graceful shutdown
+  process.on('SIGINT', () => {
+    console.log('\nShutting down test server...')
+    server.stop().then(() => process.exit(0))
+  })
+
+  process.on('SIGTERM', () => {
+    console.log('\nShutting down test server...')
+    server.stop().then(() => process.exit(0))
+  })
+}

package/lib/utils/mask_data.js

@@ -0,0 +1,53 @@
+const { maskSensitiveData } = require('invisi-data')
+
+/**
+ * Mask sensitive data utility for CodeceptJS
+ * Supports both boolean and object configuration formats
+ *
+ * @param {string} input - The string to mask
+ * @param {boolean|object} config - Masking configuration
+ * @returns {string} - Masked string
+ */
+function maskData(input, config) {
+  if (!config) {
+    return input
+  }
+
+  // Handle boolean config (backward compatibility)
+  if (typeof config === 'boolean' && config === true) {
+    return maskSensitiveData(input)
+  }
+
+  // Handle object config with custom patterns
+  if (typeof config === 'object' && config.enabled === true) {
+    const customPatterns = config.patterns || []
+    return maskSensitiveData(input, customPatterns)
+  }
+
+  return input
+}
+
+/**
+ * Check if masking is enabled based on global configuration
+ *
+ * @returns {boolean|object} - Current masking configuration
+ */
+function getMaskConfig() {
+  return global.maskSensitiveData || false
+}
+
+/**
+ * Check if data should be masked
+ *
+ * @returns {boolean} - True if masking is enabled
+ */
+function shouldMaskData() {
+  const config = getMaskConfig()
+  return config === true || (typeof config === 'object' && config.enabled === true)
+}
+
+module.exports = {
+  maskData,
+  getMaskConfig,
+  shouldMaskData,
+}

package/lib/utils.js

@@ -6,6 +6,7 @@ const getFunctionArguments = require('fn-args')
 const deepClone = require('lodash.clonedeep')
 const { convertColorToRGBA, isColorProperty } = require('./colorUtils')
 const Fuse = require('fuse.js')
+const { spawnSync } = require('child_process')
 
 function deepMerge(target, source) {
   const merge = require('lodash.merge')
@@ -191,8 +192,39 @@ module.exports.test = {
   submittedData(dataFile) {
     return function (key) {
       if (!fs.existsSync(dataFile)) {
-        const waitTill = new Date(new Date().getTime() + 1 * 1000) // wait for one sec for file to be created
-        while (waitTill > new Date()) {}
+        // Extended timeout for CI environments to handle slower processing
+        const waitTime = process.env.CI ? 60 * 1000 : 2 * 1000 // 60 seconds in CI, 2 seconds otherwise
+        let pollInterval = 100 // Start with 100ms polling interval
+        const maxPollInterval = 2000 // Max 2 second intervals
+        const startTime = new Date().getTime()
+
+        // Synchronous polling with exponential backoff to reduce CPU usage
+        while (new Date().getTime() - startTime < waitTime) {
+          if (fs.existsSync(dataFile)) {
+            break
+          }
+          
+          // Use Node.js child_process.spawnSync with platform-specific sleep commands
+          // This avoids busy waiting and allows other processes to run
+          try {
+            if (os.platform() === 'win32') {
+              // Windows: use ping with precise timing (ping waits exactly the specified ms)
+              spawnSync('ping', ['-n', '1', '-w', pollInterval.toString(), '127.0.0.1'], { stdio: 'ignore' })
+            } else {
+              // Unix/Linux/macOS: use sleep with fractional seconds
+              spawnSync('sleep', [(pollInterval / 1000).toString()], { stdio: 'ignore' })
+            }
+          } catch (err) {
+            // If system commands fail, use a simple busy wait with minimal CPU usage
+            const end = new Date().getTime() + pollInterval
+            while (new Date().getTime() < end) {
+              // No-op loop - much lighter than previous approaches
+            }
+          }
+          
+          // Exponential backoff: gradually increase polling interval to reduce resource usage
+          pollInterval = Math.min(pollInterval * 1.2, maxPollInterval)
+        }
       }
       if (!fs.existsSync(dataFile)) {
         throw new Error('Data file was not created in time')

package/lib/workers.js

@@ -49,13 +49,14 @@ const populateGroups = numberOfWorkers => {
   return groups
 }
 
-const createWorker = workerObject => {
+const createWorker = (workerObject, isPoolMode = false) => {
   const worker = new Worker(pathToWorker, {
     workerData: {
       options: simplifyObject(workerObject.options),
       tests: workerObject.tests,
       testRoot: workerObject.testRoot,
       workerIndex: workerObject.workerIndex + 1,
+      poolMode: isPoolMode,
     },
   })
   worker.on('error', err => output.error(`Worker Error: ${err.stack}`))
@@ -231,11 +232,17 @@ class Workers extends EventEmitter {
     super()
     this.setMaxListeners(50)
     this.codecept = initializeCodecept(config.testConfig, config.options)
+    this.options = config.options || {}
     this.errors = []
     this.numberOfWorkers = 0
     this.closedWorkers = 0
     this.workers = []
     this.testGroups = []
+    this.testPool = []
+    this.testPoolInitialized = false
+    this.isPoolMode = config.by === 'pool'
+    this.activeWorkers = new Map()
+    this.maxWorkers = numberOfWorkers // Track original worker count for pool mode
 
     createOutputDir(config.testConfig)
     if (numberOfWorkers) this._initWorkers(numberOfWorkers, config)
@@ -255,6 +262,7 @@ class Workers extends EventEmitter {
    *
    * - `suite`
    * - `test`
+   * - `pool`
    * - function(numberOfWorkers)
    *
    * This method can be overridden for a better split.
@@ -270,7 +278,11 @@ class Workers extends EventEmitter {
         this.testGroups.push(convertToMochaTests(testGroup))
       }
     } else if (typeof numberOfWorkers === 'number' && numberOfWorkers > 0) {
-      this.testGroups = config.by === 'suite' ? this.createGroupsOfSuites(numberOfWorkers) : this.createGroupsOfTests(numberOfWorkers)
+      if (config.by === 'pool') {
+        this.createTestPool(numberOfWorkers)
+      } else {
+        this.testGroups = config.by === 'suite' ? this.createGroupsOfSuites(numberOfWorkers) : this.createGroupsOfTests(numberOfWorkers)
+      }
     }
   }
 
@@ -298,16 +310,108 @@ class Workers extends EventEmitter {
     const groups = populateGroups(numberOfWorkers)
     let groupCounter = 0
 
+    // If specific tests are provided (e.g., from run-failed-tests), only include those
+    const targetTests = this.options && this.options.tests
+    
     mocha.suite.eachTest(test => {
-      const i = groupCounter % groups.length
       if (test) {
-        groups[i].push(test.uid)
-        groupCounter++
+        // If we have specific target tests, only include matching UIDs
+        if (targetTests && targetTests.length > 0) {
+          if (targetTests.includes(test.uid)) {
+            const i = groupCounter % groups.length
+            groups[i].push(test.uid)
+            groupCounter++
+          }
+        } else {
+          // Default behavior: include all tests
+          const i = groupCounter % groups.length
+          groups[i].push(test.uid)
+          groupCounter++
+        }
       }
     })
     return groups
   }
 
+  /**
+   * @param {Number} numberOfWorkers
+   */
+  createTestPool(numberOfWorkers) {
+    // For pool mode, create empty groups for each worker and initialize empty pool
+    // Test pool will be populated lazily when getNextTest() is first called
+    this.testPool = []
+    this.testPoolInitialized = false
+    this.testGroups = populateGroups(numberOfWorkers)
+  }
+
+  /**
+   * Initialize the test pool if not already done
+   * This is called lazily to avoid state pollution issues during construction
+   */
+  _initializeTestPool() {
+    if (this.testPoolInitialized) {
+      return
+    }
+
+    const files = this.codecept.testFiles
+    if (!files || files.length === 0) {
+      this.testPoolInitialized = true
+      return
+    }
+
+    try {
+      const mocha = Container.mocha()
+      mocha.files = files
+      mocha.loadFiles()
+
+      mocha.suite.eachTest(test => {
+        if (test) {
+          this.testPool.push(test.uid)
+        }
+      })
+    } catch (e) {
+      // If mocha loading fails due to state pollution, skip
+    }
+
+    // If no tests were found, fallback to using createGroupsOfTests approach
+    // This works around state pollution issues
+    if (this.testPool.length === 0 && files.length > 0) {
+      try {
+        const testGroups = this.createGroupsOfTests(2) // Use 2 as a default for fallback
+        for (const group of testGroups) {
+          this.testPool.push(...group)
+        }
+      } catch (e) {
+        // If createGroupsOfTests fails, fallback to simple file names
+        for (const file of files) {
+          this.testPool.push(`test_${file.replace(/[^a-zA-Z0-9]/g, '_')}`)
+        }
+      }
+    }
+
+    // Last resort fallback for unit tests - add dummy test UIDs
+    if (this.testPool.length === 0) {
+      for (let i = 0; i < Math.min(files.length, 5); i++) {
+        this.testPool.push(`dummy_test_${i}_${Date.now()}`)
+      }
+    }
+
+    this.testPoolInitialized = true
+  }
+
+  /**
+   * Gets the next test from the pool
+   * @returns {String|null} test uid or null if no tests available
+   */
+  getNextTest() {
+    // Initialize test pool lazily on first access
+    if (!this.testPoolInitialized) {
+      this._initializeTestPool()
+    }
+
+    return this.testPool.shift() || null
+  }
+
   /**
    * @param {Number} numberOfWorkers
    */
@@ -352,7 +456,7 @@ class Workers extends EventEmitter {
     process.env.RUNS_WITH_WORKERS = 'true'
     recorder.add('starting workers', () => {
       for (const worker of this.workers) {
-        const workerThread = createWorker(worker)
+        const workerThread = createWorker(worker, this.isPoolMode)
         this._listenWorkerEvents(workerThread)
       }
     })
@@ -376,9 +480,27 @@ class Workers extends EventEmitter {
   }
 
   _listenWorkerEvents(worker) {
+    // Track worker thread for pool mode
+    if (this.isPoolMode) {
+      this.activeWorkers.set(worker, { available: true, workerIndex: null })
+    }
+
     worker.on('message', message => {
       output.process(message.workerIndex)
 
+      // Handle test requests for pool mode
+      if (message.type === 'REQUEST_TEST') {
+        if (this.isPoolMode) {
+          const nextTest = this.getNextTest()
+          if (nextTest) {
+            worker.postMessage({ type: 'TEST_ASSIGNED', test: nextTest })
+          } else {
+            worker.postMessage({ type: 'NO_MORE_TESTS' })
+          }
+        }
+        return
+      }
+
       // deal with events that are not test cycle related
       if (!message.event) {
         return this.emit('message', message)
@@ -387,11 +509,21 @@ class Workers extends EventEmitter {
       switch (message.event) {
         case event.all.result:
           // we ensure consistency of result by adding tests in the very end
-          Container.result().addFailures(message.data.failures)
-          Container.result().addStats(message.data.stats)
-          message.data.tests.forEach(test => {
-            Container.result().addTest(deserializeTest(test))
-          })
+          // Check if message.data.stats is valid before adding
+          if (message.data.stats) {
+            Container.result().addStats(message.data.stats)
+          }
+
+          if (message.data.failures) {
+            Container.result().addFailures(message.data.failures)
+          }
+
+          if (message.data.tests) {
+            message.data.tests.forEach(test => {
+              Container.result().addTest(deserializeTest(test))
+            })
+          }
+
           break
         case event.suite.before:
           this.emit(event.suite.before, deserializeSuite(message.data))
@@ -438,7 +570,14 @@ class Workers extends EventEmitter {
 
     worker.on('exit', () => {
       this.closedWorkers += 1
-      if (this.closedWorkers === this.numberOfWorkers) {
+
+      if (this.isPoolMode) {
+        // Pool mode: finish when all workers have exited and no more tests
+        if (this.closedWorkers === this.numberOfWorkers) {
+          this._finishRun()
+        }
+      } else if (this.closedWorkers === this.numberOfWorkers) {
+        // Regular mode: finish when all original workers have exited
         this._finishRun()
       }
     })