@sap/cds-dk 4.7.3 → 4.8.0

package/lib/init/util/templateUtil.js

@@ -1,5 +1,5 @@
 const path = require('path');
-
+const YAML = require('@sap/cds-foss')('yaml');
 const Mustache = require('mustache');
 
 const TEMPLATE_EXTENSION = '.hbs';
@@ -41,7 +41,7 @@ module.exports = class TemplateUtil {
     }
 
     async _processTemplateFile(src, templateValues) {
-        let content = await this.fsUtil.readFile(src);
+        const content = await this.fsUtil.readFile(src);
         return Mustache.render(content, templateValues);
     }
 
@@ -52,4 +52,268 @@ module.exports = class TemplateUtil {
     _isTemplateFile(filename) {
         return path.extname(filename) === TEMPLATE_EXTENSION;
     }
-}
+
+    /*
+    Will do a syntactic merge when `semantics` is undefined.
+    Syntactic merges are sufficient in some cases,
+    e.g merging dependencies in a package.json, as they are
+    just maps where you can do a key-value replacement.
+    */
+    _deepMerge(target, source) {
+        if (this._isObject(target) && this._isObject(source)) {
+            for (const key in source) {
+                if (this._isObject(source[key])) {
+                    if (!target[key]) Object.assign(target, { [key]: source[key] });
+                    if ('#overwrite' in source[key]) Object.assign(target, { [key]: source[key]["#overwrite"] })
+                    else this._deepMerge(target[key], source[key]);
+                } else if (Array.isArray(source[key]) && Array.isArray(target[key])) {
+                    const unique = array => [...new Set(array.map(JSON.stringify))].map(JSON.parse);
+                    target[key] = unique([...source[key], ...target[key]])
+                } else {
+                    Object.assign(target, { [key]: target[key] || source[key] });
+                }
+            }
+        } else if (Array.isArray(target) && Array.isArray(source)) {
+            const unique = array => [...new Set(array.map(JSON.stringify))].map(JSON.parse);
+            target = unique([...source, ...target])
+        }
+        return target || source;
+    }
+
+    async mergeJSON(targetPath, sourcePath, projectDescriptor, semantics) {
+        const source = await this.fsUtil.readJSON(sourcePath, projectDescriptor)
+
+        if (! await this.fsUtil.pathExists(targetPath)) {
+            await this.fsUtil.writeJSON(targetPath, source);
+        } else if (semantics) {
+            const target = await this.fsUtil.readJSON(targetPath)
+
+            if (!target) return // Has no JSON file
+
+            const targetYAML = await YAML.parseDocument(YAML.stringify(target))
+            const sourceYAML = await YAML.parseDocument(YAML.stringify(source))
+            const resultYAML = await this.mergeYAML(targetYAML, sourceYAML, projectDescriptor, semantics)
+            const result = YAML.parse(YAML.stringify(resultYAML))
+            await this.fsUtil.writeJSON(targetPath, result)
+        } else {
+            const target = await this.fsUtil.readJSON(targetPath);
+            const sourceFile = await this._processTemplateFile(sourcePath, projectDescriptor);
+            const source = JSON.parse(sourceFile);
+            const result = target && source ? this._deepMerge(target, source) : target || source;
+            await this.fsUtil.writeJSON(targetPath, result);
+        }
+    }
+
+    async mergeYAML(into, from, projectDescriptor, semantics) {
+        const target = typeof into === 'string' ? await this.fsUtil.readYAML(into) : into
+        const source = typeof from === 'string' ? await this.fsUtil.readYAML(from, projectDescriptor) : from
+
+        if (!target || !target.contents) return // Has no YAML file
+
+        const existenceMap = new Map()
+        const templateExistenceMap = new Map()
+
+        const existences = semantics.existences && semantics.existences.filter(e => e)
+
+        existences && existences.forEach(existence => {
+            existenceMap.set(existence.ref, undefined)
+            templateExistenceMap.set(existence.ref, undefined)
+        });
+
+        const missingRelationshipsMap = new Map()
+
+        const relationships = semantics.relationships && semantics.relationships.filter(r => r)
+        relationships && relationships.forEach(relationship =>
+            missingRelationshipsMap.set(relationship.in, undefined)
+        )
+
+        let collectionStack = [target.contents]
+
+        const _getProperty = (object, keyPath) => keyPath.reduce((p, n) => p && p[n], object)
+        const _getYAMLProperty = (object, keyPath) => keyPath.reduce((p, n) => p && p.get(n), object)
+
+        const _validateConstraints = (node, dict, index) => {
+            existences.forEach(existence => {
+                const json = JSON.parse(String(node))
+                const constraintsFulfilled = existence.constraints.every(constraint =>
+                    constraint.value === _getProperty(json, constraint.comparisonKeyPath)
+                )
+                if (constraintsFulfilled) dict.set(existence.ref, { json, node, index })
+            })
+        }
+
+        function _traverseYAMLNode(node, index, actions, templateNode) {
+            let shifted = false
+            switch (node.type) {
+                case 'MAP':
+                    actions && actions.visitMap && actions.visitMap(node, index)
+                    if (collectionStack[0] && collectionStack[0].type === 'SEQ') { collectionStack.unshift(node); shifted = true}
+                    break;
+                case 'SEQ':
+                    break;
+                case 'PAIR':
+                    if (node.value && node.value.items) collectionStack.unshift(node.value)
+                    break;
+                case 'PLAIN':
+                    break;
+            }
+
+            if (node.items) {
+                if ((node.type === 'SEQ' || node.type === 'FLOW_SEQ') && templateNode && templateNode.items) {
+                    actions.mergeCollection && actions.mergeCollection(node, templateNode)
+                    node.type === 'SEQ' && _traverseYAMLCollection(node, actions, templateNode)
+                } else {
+                    _traverseYAMLCollection(node, actions, templateNode)
+                }
+            } else if (node.value && node.value.items) {
+                _traverseYAMLNode(node.value, index, actions, templateNode && templateNode.value)
+            }
+
+            if (node.value && node.value.items || shifted) {
+                collectionStack.shift()
+            }
+        }
+        _traverseYAMLNode.bind(this)
+
+        const _traverseYAMLCollection = (collection, actions, templateCollection) => {
+            if (!collection) return
+
+            const keyIndexMap = new Map()
+            const templateIndexMap = new Map()
+
+            // Map collection items to their semantic counterpart(s)
+            collection.items.forEach((node, i) => {
+                if (node.key) keyIndexMap.set(node.key.value, i)
+                const existence = [...existenceMap.entries()].find(([,value]) => value && value.node === node)
+                if (existence) {
+                    const [existenceKey] = existence
+                    const templateExistence = existenceKey && templateExistenceMap.get(existenceKey)
+                    const templateIndex = templateExistence && templateExistence.index
+                    if (templateIndex !== undefined) templateIndexMap.set(templateIndex, i)
+                }
+                if (!templateCollection) _traverseYAMLNode(node, i, actions)
+            });
+
+            if (!templateCollection) return
+
+            templateCollection.items.forEach((templateNode, templateIndex) => {
+                if (templateNode.type === 'PAIR') {
+                    const i = keyIndexMap.get(templateNode.key.value)
+                    const [collection] = collectionStack
+                    const targetNode = collection.items && collection.items[i]
+                    if (targetNode) {
+                        _traverseYAMLNode(targetNode, templateIndex, actions, templateNode)
+                    } else if (actions.mergePair) {
+                        actions.mergePair(collection, templateNode)
+                    }
+                } else if (templateNode.type === 'MAP') {
+                    const targetNode = collection.items[templateIndexMap.get(templateIndex)]
+                    if (actions.mergeCollection) actions.mergeCollection(targetNode, templateNode)
+                    if (targetNode) {
+                        _traverseYAMLNode(targetNode, templateIndex, actions, templateNode);
+                    } else if (!actions.mergeCollection) {
+                        collection.add(YAML.createNode(templateNode))
+                        //this.fsUtil.writeYAML(targetPath, target)
+                    }
+                }
+            })
+        }
+        _traverseYAMLCollection.bind(this)
+
+        // 1. Validate the existences in the template
+        _traverseYAMLNode(source.contents, null, {
+            visitMap: (node, index) => {
+                _validateConstraints(node, templateExistenceMap, index)
+            }
+        })
+
+        // 2. Validate the existences in the project
+        _traverseYAMLNode(target.contents, null, {
+            visitMap: (node, index) => {
+                _validateConstraints(node, existenceMap, index)
+            }
+        })
+
+        // 3. Apply overwrites to already found existences
+        semantics.overwrites && semantics.overwrites.forEach(({ in: ref, keyPath, replacement }) => {
+            const existence = existenceMap.get(ref)
+            if (existence) {
+                _getYAMLProperty(existence.node, keyPath.slice(0, keyPath.length - 1))
+                    .set(keyPath[keyPath.length - 1], replacement)
+            }
+        })
+
+        // 4. Create missing existences and pairs
+        _traverseYAMLNode(target.contents, null, {
+            mergePair: (collection, templateNode) => {
+                const childIsArray = false//templateNode.value.type !== 'MAP' && !!templateNode.value.items
+                if (collection.type === 'MAP' && !childIsArray) {
+                    collection.add(YAML.createNode(templateNode))
+                    //this.fsUtil.writeYAML(targetPath, target)
+                }
+            },
+            mergeCollection: (targetNode, templateNode) => {
+                if (templateNode.type === 'SEQ' || templateNode.type === 'FLOW_SEQ') {
+                    const [,parent] = collectionStack
+                    const missingExistences = semantics.existences
+                        .filter(e => e)
+                        .filter(existence =>
+                            parent.get(existence.keyPath[0]) === targetNode
+                        ).filter(existence =>
+                            !existenceMap.get(existence.ref)
+                        )
+                    missingExistences.forEach(existence => {
+                        const templateExistence = YAML.createNode(templateExistenceMap.get(existence.ref).node)
+                        templateExistence.spaceBefore = existence.spaceBefore
+                        targetNode.add(templateExistence)
+                        existenceMap.set(existence.ref, templateExistence)
+                        //this.fsUtil.writeYAML(targetPath, target)
+                    })
+                }
+            },
+        }, source.contents)
+
+        // 5. Re-validate the existences in the project
+        _traverseYAMLNode(target.contents, null, {
+            visitMap: (node, index) => {
+                _validateConstraints(node, existenceMap, index)
+            }
+        })
+
+        // 6. Create missing relationships
+        _traverseYAMLNode(target.contents, null, {
+            mergeCollection: (targetNode, templateNode) => {
+                if (templateNode.type === 'SEQ') {
+                    const targetJSON = YAML.parse(String(targetNode))
+                    const relationship = semantics.relationships && semantics.relationships
+                        .filter(r => r)
+                        .find(relationship => {
+                            const existingNode = _getYAMLProperty(existenceMap.get(relationship.in).node, relationship.inKeyPath)
+                            return targetNode === existingNode
+                    })
+                    if (!relationship) return
+
+                    const missingPairs = relationship.existences
+                        .filter(existenceKey =>
+                            !targetJSON.some(item =>
+                                _getProperty(existenceMap.get(existenceKey).json, relationship.existenceKeyPath) === item[relationship.into]
+                            )
+                        )
+                        .map(existenceKey => existenceMap.get(existenceKey).node.get(relationship.into))
+                    missingPairs.forEach(pair => {
+                        targetNode.add(YAML.createNode({ [relationship.into]: pair }))
+                        //await this.fsUtil.writeYAML(targetPath, target)
+                    })
+                }
+            }
+        }, source.contents)
+
+        typeof into === 'string' && await this.fsUtil.writeYAML(into, target)
+        return target
+    }
+
+    _isObject(item) {
+        return item && typeof item === 'object' && !Array.isArray(item)
+    }
+};
+

package/lib/util/exec.d.ts

@@ -0,0 +1,54 @@
+/*
+ * Typings only for common use cases
+ */
+
+interface AdditionalExecOptions {
+    acceptStatuses? : Array<Number> | '*',
+    appendEnv?: {[key: string]: string},
+    log?: object
+}
+
+interface SpawnReturns {
+    status : number,
+    stdout? : string,
+    stderr? : string
+}
+
+import { SpawnOptions, SpawnSyncOptions, SpawnSyncReturns } from "child_process";
+
+type execTemplate<T> = (template: TemplateStringsArray, ...args : any[]) => T;
+
+/**
+ * Execute commands with template strings asynchronously.
+ *
+ * @example
+ *
+ * await exec `ls -l .`
+ * await exec `ls -l ${'spaces are safe in expressions'}` => spawn('ls', [ '-l', 'spaces are safe in expressions' ])
+ * await exec `ls -l "dir ${'with spaces'}"
+ * await exec `ls -l ${['db', 'srv']}`
+ * const out = (await exec({stdio: ['inherit', 'pipe', 'inherit']}) `ls -l`).stdout
+ * const out = await exec({result: 'stdout}) `ls -l`
+ */
+export function exec( template : TemplateStringsArray, ...args : any[] ) : Promise<SpawnReturns>
+export function exec( options : { result: 'stdout' | 'stderr' } & SpawnOptions & AdditionalExecOptions) : execTemplate<Promise<string>>;
+export function exec( options : SpawnOptions & AdditionalExecOptions) : execTemplate<Promise<SpawnReturns>>;
+
+/**
+ * Execute commands with template strings synchronously.
+ *
+ * @example
+ *
+ * execSync `ls -l .`
+ * execSync `ls -l ${'spaces are safe in expressions'}` => spawn('ls', [ '-l', 'spaces are safe in expressions' ])
+ * execSync `ls -l "dir ${'with spaces'}"
+ * execSync `ls -l ${['db', 'srv']}`
+ * const out = (execSync({stdio: ['inherit', 'pipe', 'inherit']}) `ls -l`).stdout
+ * const out = execSync({result: 'stdout}) `ls -l`
+ */
+export function execSync( template : TemplateStringsArray, ...args : any[] ) : SpawnSyncReturns<string>;
+export function execSync( options : { result: 'stdout' | 'stderr' } & SpawnSyncOptions & AdditionalExecOptions) : execTemplate<string>;
+export function execSync( options : { encoding : 'buffer' | null } & SpawnOptions & AdditionalExecOptions) : execTemplate<SpawnSyncReturns<Buffer>>;
+export function execSync( options : SpawnOptions & AdditionalExecOptions) : execTemplate<SpawnSyncReturns<string>>;
+
+export { execSync as $ };

package/lib/util/exec.js

@@ -0,0 +1,210 @@
+const { spawn, spawnSync } = require('child_process');
+
+const cds = require('@sap/cds/lib');
+const log = cds.log('exec');
+
+function* commandIterator(strs, ...args) {
+    for (let i=0; i<strs.length; i++) {
+        const str = strs[i];
+        const regex = /((\s+)|(["'])|$)/g;
+        let match;
+        let lastIndex = 0;
+        do {
+            match = regex.exec(str);
+            if (match.index > lastIndex) {
+                yield { type: 'WORD', value: str.substr(lastIndex, match.index - lastIndex) };
+            }
+            if (match[2]) {
+                yield { type: 'SPACE', value: match[2] };
+            }
+            if (match[3]) {
+                yield { type: 'QUOTE', value: match[3] }
+            }
+            lastIndex = match.index + match[0].length;
+        } while (lastIndex < str.length);
+
+        if (args[i]) {
+            if (Array.isArray(args[i])) {
+                let sep = false;
+                for (const value of args[i]) {
+                    if (sep) yield { type: 'SPACE', value: ' '};
+                    yield { type: 'WORD', value };
+                    sep = true;
+                }
+            } else {
+                yield { type: 'WORD', value: args[i] };
+            }
+        }
+    }
+}
+
+function processTemplate(strs, args) {
+    let quote;
+    const result = [];
+    let index = 0;
+
+    const append = str => result[index] = (result[index] || '') + str;
+    const push = () => { if (index < result.length) index++; };
+
+    for (const token of commandIterator(strs, args)) {
+        switch (token.type) {
+            case 'WORD':
+                append(token.value);
+                break;
+            case 'SPACE':
+                if (quote) {
+                    append(token.value);
+                } else {
+                    push();
+                }
+                break;
+            case 'QUOTE':
+                if (quote === token.value) {
+                    quote = undefined;
+                } else if (quote) {
+                    append(token.value);
+                } else {
+                    quote = token.value;
+                }
+                break;
+        }
+    }
+
+    return result;
+}
+
+function fetchOptions(cmd) {
+    const { options, args } = cmd;
+    if (typeof args[0] === 'object' && !Array.isArray(args[0])) {
+        const newOptions = args[0];
+        cmd.options      = {...options, ...newOptions};
+        cmd.args         = args.slice(1);
+        return true;
+    } else {
+        return false;
+    }
+}
+
+function exec_func(...args) {
+    const cmd = { options: {
+            log,
+            acceptStatuses: [ 0 ],
+            stdio: 'inherit',
+            shell: false,
+            encoding: 'utf-8'
+        }, args };
+
+    let hasOptions = false;
+    while (fetchOptions(cmd)) hasOptions = true;
+    if (hasOptions && cmd.args.length === 0) {
+        return (...args) => exec_func(cmd.options, ...args);
+    }
+
+    if (Array.isArray(cmd.args[0])) cmd.args = processTemplate(...cmd.args);
+    return cmd.options.executer(cmd.options, ...cmd.args);
+}
+
+function getExecOptions(options) {
+    const execOptions = {...options};
+    [ 'acceptStatuses', 'appendEnv', 'exec', 'log', 'result' ].forEach( name => delete execOptions[name] );
+
+    const { appendEnv, result } = options;
+    if (appendEnv) {
+        execOptions.env = {...(options.env || process.env),...appendEnv};
+    }
+
+    switch (result) {
+        case 'stdout':
+            execOptions.stdio = [ 'inherit', 'pipe', 'inherit' ];
+            break;
+        case 'stderr':
+            execOptions.stdio = [ 'inherit', 'inherit', 'pipe' ];
+            break;
+        case undefined:
+            break;
+
+        default:
+            throw new Error `Unexpected "result" option: "${result}"`;
+    }
+
+    return execOptions;
+}
+
+function toString(a) {
+    return typeof a === 'string' ? a : Buffer.from(a).toString('utf-8');
+}
+
+function processResult(result, options) {
+    switch (options.result) {
+        case 'stdout':
+            return toString(result.stdout);
+        case 'stderr':
+            return toString(result.stderr);
+        case undefined:
+            return result;
+
+        default:
+            throw new Error(`Unexpected "result" option: "${result}"`);
+    }
+}
+
+
+function executeSync(options = {}, command, ...args) {
+    const { acceptStatuses } = options;
+    const execOptions = getExecOptions(options);
+
+    log.debug(`exec: ${command} ${args.map( v => v.match(/\s/) ? `"${v}"` : v ).join(' ')}`);
+    const result = spawnSync(command, args, execOptions);
+
+    if (acceptStatuses !== '*' && acceptStatuses.indexOf(result.status) === -1) throw new Error(`Command returned with status code ${result.status}`);
+    return processResult(result, options);
+}
+
+async function executeAsync(options = {}, command, ...args) {
+    const { acceptStatuses } = options;
+    const execOptions = getExecOptions(options);
+
+//    ctx.log(`exec: ${command} ${args.map( v => v.match(/\s/) ? `"${v}"` : v ).join(' ')}`);
+
+
+    const result = await new Promise( (resolve, reject) => {
+        const child = spawn(command, args, execOptions);
+        const stdoutChunks = [];
+        const stderrChunks = [];
+
+        if (child.stdout) {
+            child.stdout.on('data', data => {
+                stdoutChunks.push(Buffer.from(data));
+            });
+        }
+
+        if (child.stderr) {
+            child.stderr.on('data', data => {
+                stderrChunks.push(Buffer.from(data));
+            });
+        }
+
+        const exit = status => {
+            if (acceptStatuses !== '*' && acceptStatuses.indexOf(status) === -1) reject(new Error(`Command returned with status code ${status}`))
+            else resolve({
+                status,
+                stdout: child.stdout ? Buffer.concat(stdoutChunks).toString("utf-8") : undefined,
+                stderr: child.stderr ? Buffer.concat(stderrChunks).toString("utf-8") : undefined
+            })
+        };
+
+        child.on('close', exit);
+        child.on('exit', exit);
+    });
+
+    return processResult(result, options);
+}
+
+const exec = exec_func({ executer: executeAsync });
+const execSync = exec_func({ executer: executeSync });
+
+module.exports = {
+    exec,
+    execSync,
+    $ : execSync
+}

package/lib/watch/livereload-server.js

@@ -14,46 +14,58 @@ module.exports = class LiveReload {
     app.use('/livereload.js', express.static(require.resolve('livereload-js')))
 
     return new Promise ((resolve, reject) => {
-      // start server
-      const server = app.listen (port,'localhost')
-      server.on('error', reject)
-      server.on('listening', async () => {
+      let server
+      const listening = async () => {
         const {address,port} = server.address()
+        startWs(server)
         this.url = await livereloadURL(address, port)
         debug && debug(`[cds] - live reload available at ${this.url}`)
         resolve(this.url)
+      }
+        // start server
+      server = app.listen (port,'localhost')
+      server.once('listening', listening)
+      server.on('error', e => {
+        // try once more w/ ephemeral port 0
+        if (e.code === 'EADDRINUSE') {
+          server = app.listen (0,'localhost')
+          server.once('listening', listening)
+          server.on('error', reject)
+        }
+        else reject(e)
       })
 
       // start websocket
-      this.wsServer = new ws.Server({ server })
-      this.wsServer.on('connection', (socket) => {
-        debug && debug('Client connected')
-        socket.on('message', (msg) => {
-          debug && debug('Client message', msg)
-          const request = JSON.parse(msg)
+      const startWs = (server) => {
+        this.wsServer = new ws.Server({ server })
+        this.wsServer.on('connection', (socket) => {
+          debug && debug('Client connected')
+          socket.on('message', (msg) => {
+            debug && debug('Client message', msg)
+            const request = JSON.parse(msg)
 
-          if (request.command === 'hello') {
-            debug && debug('Client handshake')
-            const data = JSON.stringify({
-              command: 'hello',
-              protocols: [
-                'http://livereload.com/protocols/official-7',
-                'http://livereload.com/protocols/official-8',
-                'http://livereload.com/protocols/official-9',
-                'http://livereload.com/protocols/2.x-origin-version-negotiation',
-                'http://livereload.com/protocols/2.x-remote-control',
-              ],
-              serverName: 'cds-watch',
-            })
-            socket.send(data)
-          } else if (request.command === 'info') {
-            debug && debug('Server received client data. Not sending response.')
-          }
+            if (request.command === 'hello') {
+              debug && debug('Client handshake')
+              const data = JSON.stringify({
+                command: 'hello',
+                protocols: [
+                  'http://livereload.com/protocols/official-7',
+                  'http://livereload.com/protocols/official-8',
+                  'http://livereload.com/protocols/official-9',
+                  'http://livereload.com/protocols/2.x-origin-version-negotiation',
+                  'http://livereload.com/protocols/2.x-remote-control',
+                ],
+                serverName: 'cds-watch',
+              })
+              socket.send(data)
+            } else if (request.command === 'info') {
+              debug && debug('Server received client data. Not sending response.')
+            }
+          })
         })
-      })
-      this.wsServer.on('close', () => { debug && debug('Socket closed') })
-      this.wsServer.on('error', (e) => debug && debug(e))
-
+        this.wsServer.on('close', () => { debug && debug('Socket closed') })
+        this.wsServer.on('error', (e) => debug && debug(e))
+      }
     })
   }
 

package/lib/watch/open.js

@@ -19,7 +19,7 @@ function _open(args) {
   // http://stackoverflow.com/q/1480971/3191, but see below for Windows.
   let platform = process.platform
 
-  // Attempt to detect Windows Subystem for Linux (WSL). WSL  itself as Linux (which works in most cases), but in
+  // Attempt to detect Windows Subsystem for Linux (WSL). WSL  itself as Linux (which works in most cases), but in
   // this specific case we need to treat it as actually being Windows. The 'Windows-way' of opening things through
   // cmd.exe works just fine here, whereas using xdg-open does not, since there is no X Windows in WSL.
   if (platform === 'linux' && os.release().indexOf('Microsoft') !== -1) {