@cyklang/cli 0.1.2 → 0.2.1

package/src/AssetCommand.ts

@@ -0,0 +1,307 @@
+import { Command } from "commander"
+import { getLogger, resetLevel } from "loglevel"
+import { Cmd } from "./Cmd"
+import { DBClient } from "./DBClient"
+import * as fs from "fs"
+import path from "path"
+import mime from 'mime-types'
+import readline from 'readline'
+import { DBAsset, DBExecuteRequest, ObjectData, parseXML, PrimitiveData } from "@cyklang/core"
+import FormData from 'form-data'
+
+const logger = getLogger('AssetCommand.ts')
+logger.setLevel('debug')
+
+export class AssetCommand extends Command {
+    constructor(name: string, description: string) {
+        super(name)
+        this.description(description)
+        this.addCommand(new AssetList('list', 'list assets'))
+        this.addCommand(new AssetList('l', '(l)ist assets'))
+        this.addCommand(new AssetUpload('upload', 'upload assets from current directory'))
+        this.addCommand(new AssetUpload('u', '(u)pload assets'))
+    }
+}
+
+class AssetList extends Cmd {
+    constructor(name: string, description: string) {
+        super(name)
+        this.description(description)
+            .option('-s --sort <columns>', 'sort list by column numbers (begins by 0) separated by comma')
+            .action(async (options: any) => { await this.commandList(options) })
+    }
+    async commandList(options: any) {
+        try {
+            await this.prologue(options)
+            if (this.dbManager === undefined) throw 'dbManager undefined'
+
+            const dbClient = new DBClient(this.dbManager)
+            dbClient.selectFromTable('List of Assets', 'cyk_asset',
+                { fields: 'asset_id,asset_route,asset_last_update,asset_mimetype', sort: options.sort || '1' }
+            )
+        }
+        catch (err) {
+            logger.error(err)
+        }
+    }
+}
+
+//----------------------------------------------------------------------------------------------
+// class AssetUpload
+//----------------------------------------------------------------------------------------------
+
+interface FileDescriptor {
+    path: string
+    mtime: Date
+}
+class AssetUpload extends Cmd {
+
+    constructor(name: string, description: string) {
+        super(name)
+        this.description(description)
+            .option('-d --dest <destination>', 'route of destination')
+            .argument('[sources...]', 'local files or directories to upload as assets to the server')
+            .action(async (sources: any, options: any) => {
+                await this.commandUpload(sources, options)
+            })
+    }
+
+    async commandUpload(sources: string[], options: any) {
+
+        try {
+            await this.prologue(options)
+            if (this.dbManager === undefined) throw 'dbManager undefined'
+
+            if (sources.length === 0) throw 'no source to upload'
+
+            let dest = "/"
+            if (options.dest !== undefined) dest = options.dest
+            if (dest.substring(0, 1) !== '/') dest = '/' + dest
+            if (dest.substring(dest.length - 1) !== '/') dest = dest + '/'
+
+            for (let ind = 0; ind < sources.length; ind++) {
+                const source = sources[ind]
+                if (fs.existsSync(source) === false) throw 'source ' + source + ' does not exist'
+                const filestat = fs.lstatSync(source)
+                if (filestat.isDirectory() === true) {
+                    await this.uploadDirectory(source, dest)
+                }
+                else if (filestat.isFile() === true) {
+                    const base = path.parse(source).base
+                    logger.debug('upload file ' + source + ' to ' + dest + base)
+                    await this.uploadAsset({path:source, mtime: filestat.mtime}, dest + base)
+                }
+            }
+        }
+        catch (err) {
+            logger.error(err)
+        }
+    }
+
+    //----------------------------------------------------------------------------------------------
+    // uploadDirectory
+    //----------------------------------------------------------------------------------------------
+
+    async uploadDirectory(source: string, dest: string) {
+
+        const localList: FileDescriptor[] = []
+        let dirName: string
+        if (source.substring(source.length - 1) === '/') {
+            // trailing directory separator
+            dirName = source
+        }
+        else {
+            dirName = source + '/'
+        }
+
+        logger.debug('dirName', dirName)
+
+        this.scanDir(dirName, localList)
+
+        const remoteList: FileDescriptor[] = []
+        await this.scanAssets(remoteList)
+
+        const uploadList = this.buildUploadList(dirName, dest, localList, remoteList)
+
+        // logger.info(fileNames.join('\n'))
+        if (uploadList.length === 0) throw 'None file to upload'
+
+        const rl = readline.createInterface(process.stdin, process.stdout)
+        rl.question('Do you want to upload this(these) ' + uploadList.length + ' file(s) (Y/N/y/n) ?', async (reply) => {
+            rl.close()
+            if (reply !== 'Y' && reply !== 'y') {
+                logger.info('upload cancelled')
+                return
+            }
+
+            await this.uploadFiles(dirName, uploadList, dest)
+        })
+    }
+
+    //----------------------------------------------------------------------------------------------
+    // buildUploadList
+    //----------------------------------------------------------------------------------------------
+
+    buildUploadList(dirName: string, dest: string, localList: FileDescriptor[], remoteList: FileDescriptor[]): FileDescriptor[] {
+        const result: FileDescriptor[] = []
+        const remoteMap = new Map<string, FileDescriptor>()
+        for(let ind = 0; ind < remoteList.length; ind++) {
+            const remote = remoteList[ind]
+            remoteMap.set(remote.path, remote)
+        }
+        for(let ind = 0; ind < localList.length; ind++ ) {
+            const localDesc = localList[ind]
+            const route = dest + localDesc.path.substring(dirName.length)
+            const remoteDesc = remoteMap.get(route)
+
+            if (remoteDesc === undefined || remoteDesc.mtime.getTime() + 1000 < localDesc.mtime.getTime()) {
+                if (remoteDesc === undefined) logger.debug('remoteDesc undefined')
+                else logger.debug('remoteDesc.mtime '+ remoteDesc.mtime.getTime(), 'localDesc.mtime ' + localDesc.mtime.getTime())
+                logger.info(localDesc.path)
+                result.push(localDesc)
+            }
+        }
+        return result
+    }
+    
+    
+    //----------------------------------------------------------------------------------------------
+    // scanAssets
+    //----------------------------------------------------------------------------------------------
+
+    async scanAssets(result: FileDescriptor[]) {
+
+        if (this.dbManager === undefined) throw 'dbManager undefined'
+        const dbReq = new DBExecuteRequest()
+        dbReq.selectFromTable = 'cyk_asset'
+        const pFields = dbReq.parameters.addVariable('fields', this.dbManager.scope.structure.stringDataType)
+        pFields.data = new PrimitiveData(this.dbManager.scope.structure.stringDataType, "asset_id,asset_route,asset_last_update,asset_mimetype")
+        const xmlResult = await this.dbManager.dbExecute(dbReq)
+        if (xmlResult === undefined || xmlResult === null) throw 'xmlResult null or undefined'
+        const tag = parseXML('scanAssets', xmlResult)
+        const objResult = await this.dbManager.scope.structure.objectDataType.parseData(tag, this.dbManager.scope) as ObjectData
+        const rows = (objResult.variables.getData('cyk_asset') as ObjectData).variables
+        for(let ind = 0; ind < rows.list.length; ind++) {
+            const {variable: row} = rows.list[ind]
+            const record = row.data as ObjectData
+            const asset_route = (record.variables.getData('asset_route') as PrimitiveData).value as string
+            const asset_last_update = (record.variables.getData('asset_last_update') as PrimitiveData).value as Date
+            // logger.debug('asset_route ' + asset_route,'asset_last_update ' + asset_last_update)
+            result.push({path: asset_route, mtime: asset_last_update})
+        }
+    }
+
+    //----------------------------------------------------------------------------------------------
+    // scanDir
+    //----------------------------------------------------------------------------------------------
+
+    scanDir(dirName: string, result: FileDescriptor[]) {
+        const files = fs.readdirSync(dirName)
+        for (let ind = 0; ind < files.length; ind++) {
+            const fileName = files[ind]
+            if (this.file2exclude(fileName) === true) continue
+            const fstat = fs.statSync(dirName + fileName)
+            if (fstat.isDirectory() === true) {
+                this.scanDir(dirName + fileName + '/', result)
+            }
+            else {
+                if (mime.lookup(fileName) === false) {
+                    logger.info(fileName + ' file extension is unknown and will not be uploaded')
+                    continue
+                }
+                //const dbAssetExist = this.dbManager?.dbAssetExist()
+                result.push({path: dirName + fileName, mtime: fstat.mtime})
+                // logger.debug(dirName + '/' + fileName)
+            }
+        }
+    }
+
+    //----------------------------------------------------------------------------------------------
+    // file2exclude
+    //----------------------------------------------------------------------------------------------
+
+    ignorePatterns = ['.git', 'node_modules']
+
+    file2exclude(fileName: string): boolean {
+        let result = false
+        this.ignorePatterns.forEach((pattern) => {
+            if (fileName === pattern) result = true
+        })
+        return result
+    }
+
+    //----------------------------------------------------------------------------------------------
+    // uploadFiles
+    //----------------------------------------------------------------------------------------------
+
+    async uploadFiles(dirName: string, uploadList: FileDescriptor[], dest: string) {
+        for (let ind = 0; ind < uploadList.length; ind++) {
+            const upload = uploadList[ind]
+            const route = dest + upload.path.substring(dirName.length)
+            logger.debug('upload ' + upload.path + ' to ' + route)
+            await this.uploadAsset(upload, route)
+        }
+    }
+
+    //----------------------------------------------------------------------------------------------
+    // uploadAsset
+    //----------------------------------------------------------------------------------------------
+
+    async uploadAsset(upload: FileDescriptor, route: string) {
+        logger.debug('uploadAsset ' + upload.path + ' to route ' + route + ' with mimetype ' + mime.lookup(upload.path))
+
+        try {
+            let dbAsset: DBAsset | undefined = await this.dbManager?.dbAssetExist(route)
+            if (dbAsset === undefined) {
+                dbAsset = new DBAsset()
+            }
+            const mimetype = mime.lookup(upload.path)
+            if (mimetype === false) throw 'mimetype false for ' + upload
+            dbAsset.mimetype = mimetype
+            dbAsset.route = route
+
+            // const fstat = fs.statSync(fileName)
+            dbAsset.last_update = upload.mtime
+
+            // const contentBuffer = fs.readFileSync(fileName)
+            // dbAsset.content = contentBuffer
+            // logger.debug('dbAsset.content length ' + dbAsset.content.length)
+
+            if (dbAsset.id === undefined) {
+                dbAsset.id = await this.dbManager?.dbAssetInsert(dbAsset)
+                logger.info('inserted asset ' + dbAsset.id + ' route ' + dbAsset.route)
+            }
+            else {
+                await this.dbManager?.dbAssetUpdate(dbAsset)
+                logger.info('updated asset ' + dbAsset.route)
+            }
+
+            await this.uploadAssetContent(dbAsset, upload.path)
+        }
+        catch (err) {
+            logger.error(err)
+        }
+    }
+
+    //----------------------------------------------------------------------------------------------
+    // uploadAssetContent
+    //----------------------------------------------------------------------------------------------
+
+    async uploadAssetContent(dbAsset: DBAsset, fileName: string) {
+        logger.debug('uploadAssetContent file ' + fileName + ' to path ' + path)
+        try {
+            const file = fs.createReadStream(fileName)
+            const form = new FormData()
+            form.append('uploadFile', file)
+
+            const route = '/upload/cyk_asset/asset_content?asset_id=' + dbAsset.id
+            const resp = await this.dbRemote?.apiServer.post(route, form)
+            if ( resp.status === 200 ) {
+                logger.info(fileName + ' uploaded')
+            }
+        }
+        catch (err) {
+            logger.error(err)
+        }
+    }
+}

package/src/Cmd.ts

@@ -8,6 +8,7 @@ logger.setLevel('debug')
 
 export class Cmd extends Command {
     dbManager: DBManager | undefined
+    dbRemote: DBRemote | undefined
 
     constructor(name: string) {
         super(name);
@@ -23,11 +24,11 @@ export class Cmd extends Command {
         if (process.env.DBREMOTE_URL === undefined) throw 'DBREMOTE_URL undefined'
         logger.debug('DBREMOTE_URL ' + process.env.DBREMOTE_URL)
         const nodeCrypto = new NodeCrypto()
-        const dbRemote = new DBRemote(structure.scope, process.env.DBREMOTE_URL, nodeCrypto)
-        const login = await dbRemote.signin(process.env.USER_NAME, undefined, process.env.USER_PASSWORD)
+        this.dbRemote = new DBRemote(structure.scope, process.env.DBREMOTE_URL, nodeCrypto)
+        const login = await this.dbRemote.signin(process.env.USER_NAME, undefined, process.env.USER_PASSWORD)
         // logger.debug(login)
         
-        this.dbManager = new DBManager(structure.scope, dbRemote)
+        this.dbManager = new DBManager(structure.scope, this.dbRemote)
         await this.dbManager.initialize()
     }
 }

package/src/DBClient.ts

@@ -11,7 +11,7 @@ export class DBClient {
     }
 
     async selectFromTable(title: string, tableName: string, options:
-        { fields: string, sort?: string | undefined, where?: string | undefined }) {
+        { fields: string, sort?: string | undefined, where?: string | undefined, width?: string | undefined }) {
 
         try {
 
@@ -26,13 +26,13 @@ export class DBClient {
                 const colnames = options.fields.split(',')
                 const sortcols = options.sort.split(',')
                 let order_by = ''
-                for (let ind = 0; ind < sortcols.length; ind++ ) {
+                for (let ind = 0; ind < sortcols.length; ind++) {
                     const coli = sortcols[ind]
                     const colname = colnames[Number(coli)]
                     if (order_by !== '') order_by += ','
                     order_by += colname
                 }
-                logger.debug('order_by '+order_by)
+                logger.debug('order_by ' + order_by)
                 request.parameters.addVariable('order_by', stringDataType).data = new PrimitiveData(stringDataType, order_by)
             }
 
@@ -42,37 +42,38 @@ export class DBClient {
             //         request.parameters.addVariable(key, stringDataType).data = data
             //     })
             // }
-    
+
             const xmlResult = await this.dbManager.dbExecute(request)
             if (xmlResult === undefined) throw 'xmlResult undefined'
             const tagXmlResult = parseXML('xmlResult', xmlResult)
-    
+
             const objXmlResult = (await objectDataType.parseData(tagXmlResult, this.dbManager.scope)) as ObjectData
             const objectMeta = (objXmlResult.variables.getData('meta')) as ObjectData
             if (objectMeta === undefined) throw 'objectMeta not found'
             const xmlMeta = objectMeta.variables.getString(tableName)
             if (xmlMeta === undefined) throw 'meta does not have ' + tableName + ' description'
-    
+
             const tagDBTable = parseXML('dbTable', xmlMeta)
             const dbTable = new DBTable(tagDBTable)
-    
+
             const fields = options.fields.split(',')
             const list = new List()
-    
+            if (options.width !== undefined) list.width = options.width
+
             for (let ind = 0; ind < fields.length; ind++) {
                 const field = fields[ind]
                 const dbColumn = dbTable.columns.filter((dbColumn) => dbColumn.name === field)[0]
                 if (dbColumn === undefined) throw 'field ' + field + ' not found'
                 list.addDBColumn(dbColumn)
             }
-    
+
             const objDataset = objXmlResult.variables.getData(tableName) as ObjectData
             for (let ind = 0; ind < objDataset.variables.length(); ind++) {
                 const record = (objDataset.variables.at(ind)?.data as ObjectData)
                 list.addObjectData(record)
             }
             logger.info(list.renderList(title))
-            
+
         }
         catch (err) {
             logger.error(err)
@@ -81,6 +82,7 @@ export class DBClient {
 }
 
 class List {
+    width = "0"
     columns: Column[] = []
     rows: Row[] = []
     lines: string[] = []
@@ -107,20 +109,44 @@ class List {
     }
 
     renderList(title: string): string {
-        // calculate column.width
+
+        // parse width option
+
+        const colws: number[] = []
+        const colws_ = this.width.split(',')
+
+        for (let ind = 0; ind < colws_.length; ind++) {
+            const colw = Number.parseInt(colws_[ind])
+            if (isNaN(colw)) {
+                const msg = '--width incorrect format : ' + this.width
+                throw msg
+            }
+            colws.push(colw)
+        }
+
+        logger.debug('colws', colws.join(','))
+
+        // calculate column.actualWidth
         let total_width = 0
         for (let indi = 0; indi < this.columns.length; indi++) {
             const column = this.columns[indi]
-            column.width = column.label.length
+
+            if (indi < colws.length) column.width = colws[indi]
+            else column.width = colws[colws.length - 1]
+
+            column.actualWidth = column.label.length
             this.rows.forEach((row) => {
                 const cell = row.cells[indi]
                 const renderedCell = this.renderCell(cell)
-                if (renderedCell.length > column.width) {
-                    column.width = renderedCell.length
+                if (renderedCell.length > column.actualWidth) {
+                    if (column.width === 0 || renderedCell.length < column.width)
+                        column.actualWidth = renderedCell.length
+                    else
+                        column.actualWidth = column.width
                 }
             })
             if (total_width > 0) total_width += 3
-            total_width += column.width
+            total_width += column.actualWidth
         }
 
         // title
@@ -131,43 +157,46 @@ class List {
         this.lines.push(this.renderSeparator())
 
         // rows
-        for (let indj = 0; indj < this.rows.length; indj++ ) {
+        for (let indj = 0; indj < this.rows.length; indj++) {
             const row = this.rows[indj]
             this.lines.push(this.renderRow(row))
         }
+        this.lines.push(this.renderSeparator())
+        this.lines.push('Number of lines : ' + this.rows.length)
+        
         return this.lines.join('\n')
     }
 
     renderHeaderRow(): string {
         let result = ''
-        for (let indi = 0; indi < this.columns.length; indi++ ) {
+        for (let indi = 0; indi < this.columns.length; indi++) {
             const column = this.columns[indi]
             if (result !== '') result += ' | '
-            result += this.renderCell(column.label, column.width, 'center')
+            result += this.renderCell(column.label, column.actualWidth, 'center')
         }
         return result
     }
 
     renderRow(row: Row): string {
         let result = ''
-        for (let indi = 0; indi < this.columns.length; indi++ ) {
+        for (let indi = 0; indi < this.columns.length; indi++) {
             const column = this.columns[indi]
             let justify: 'left' | 'right' | 'center' | undefined = 'left'
             if (column.dbColumn?.dataTypeName === 'number' || column?.type === 'number') {
                 justify = 'right'
             }
             if (result !== '') result += ' | '
-            result += this.renderCell(row.cells[indi], column.width, justify)
+            result += this.renderCell(row.cells[indi], column.actualWidth, justify)
         }
         return result
     }
 
     renderSeparator(): string {
         let result = ''
-        for ( let indi = 0; indi < this.columns.length; indi++ ) {
+        for (let indi = 0; indi < this.columns.length; indi++) {
             const column = this.columns[indi]
             if (result !== '') result += '-+-'
-            result += '-'.repeat(column.width)
+            result += '-'.repeat(column.actualWidth)
         }
         return result
     }
@@ -180,20 +209,29 @@ class List {
         else if (value === null) {
             result = 'null'
         }
+        else if (value instanceof Date) {
+            const d = value as Date
+            result = d.toISOString()
+        }
         else {
             result = value.toString()
         }
-        if (width !== undefined && result.length < width) {
-            if (justify === undefined || justify === 'left') {
-                result += ' '.repeat(width - result.length)
-            }
-            else if (justify === 'right') {
-                result = ' '.repeat(width - result.length) + result
+        if (width !== undefined && width !== 0) {
+            if (result.length >= width) {
+                result = result.substring(0, width)
             }
             else {
-                // justify === center
-                const before = Math.floor((width - result.length) / 2)
-                result = ' '.repeat(before) + result + ' '.repeat(width - result.length - before)
+                if (justify === undefined || justify === 'left') {
+                    result += ' '.repeat(width - result.length)
+                }
+                else if (justify === 'right') {
+                    result = ' '.repeat(width - result.length) + result
+                }
+                else {
+                    // justify === center
+                    const before = Math.floor((width - result.length) / 2)
+                    result = ' '.repeat(before) + result + ' '.repeat(width - result.length - before)
+                }
             }
         }
         return result
@@ -207,6 +245,7 @@ class Column {
     type?: string | undefined;
     computed?: string | undefined;
     width: number = 0
+    actualWidth: number = 0
     constructor(name: string) {
         this.name = name
     }

package/src/ModuleCommand.ts

@@ -31,9 +31,11 @@ Otherwise, a new module is inserted in the database. To rename a module, use opt
 }
 
 function getModuleDBName(filePath: string): string {
-    let result
-    if (filePath.endsWith('.xml') === false) throw filePath + ' does not have .xml extension'
-    result = path.basename(filePath, '.xml')
+    // let result
+    // if (filePath.endsWith('.xml') === false) throw filePath + ' does not have .xml extension'
+    // result = path.basename(filePath, '.xml')
+
+    const result = path.parse(filePath).name
     const identifierRegex = /^[_a-zA-Z][_a-zA-Z0-9]*$/
     if (result.match(identifierRegex) === null) throw 'filename ' + result + ' does not match identifier syntax'
     return result
@@ -46,7 +48,7 @@ class ModuleUpload extends Cmd {
     constructor(name: string, description: string) {
         super(name)
         this.description(description)
-            .option('-i --id <id>', 'module ID to download')
+            .option('-i --id <id>', 'module ID to upload')
             .argument('[files...]', 'local module file(s) to upload to the server')
             .action(async (files: any, options: any) => {
                 await this.commandUpload(files, options)
@@ -156,7 +158,7 @@ class ModuleDownload extends Cmd {
         }
         const filePath = dbname + '.xml'
         if (fs.existsSync(filePath) === true) {
-            logger.info('file ' + filePath + ' already exists and will not be overwritten ')
+            logger.info('file ' + filePath + ' already exists. Remove it before launching the command if you want to overwrite.')
         }
         else {
             fs.writeFileSync(filePath, dbModule.source || '')
@@ -191,3 +193,27 @@ class ModuleList extends Cmd {
     }
 }
 
+class ModuleDelete extends Cmd {
+    constructor(name: string, description: string) {
+        super(name)
+        this.description(description)
+        .argument('[files...]', 'module(s) to delete from the database')
+        .action(async(files: any, options: any) => {
+            await this.commandDelete(files, options)
+        })
+    }
+    async commandDelete(files: string[], options: any) {
+        try {
+            await this.prologue(options)
+            if (this.dbManager === undefined) throw 'dbManager undefined'
+
+            if (files.length === 0) throw 'no module to delete'
+
+
+        }
+        catch (err){
+            logger.error(err)
+        } 
+    }
+}
+