@servicenow/sdk-build-plugins 3.0.3 → 4.0.1

package/src/server-module-plugin/index.ts

@@ -0,0 +1,470 @@
+import {
+    Plugin,
+    Shape,
+    type Diagnostics,
+    type Transform,
+    type Result,
+    SourceFileShape,
+    StringShape,
+    type Source,
+    type Project,
+    type RecordContext,
+    type Compiler,
+} from '@servicenow/sdk-build-core'
+import { path as pathModule, ts, type FileSystem, type NowConfig } from '@servicenow/sdk-build-core'
+import { RepackService } from '../repack'
+import { SBOMBuilder } from './sbom-builder'
+import isEqual from 'lodash/isEqual'
+import { INVALID_XML_CHARACTERS, applyPathMappings } from '../utils'
+import zip from 'lodash/zip'
+
+const GLUE_CODE_PREFIX = '// @fluent-module'
+const GLUE_CODE_META_REGEX = new RegExp(`^${GLUE_CODE_PREFIX} (.*);(true|false);(.*)`) // name;isDefault;path
+const GLUE_CODE_WARNING = `
+// WARNING: This code is generated by the ServiceNow SDK in order to provide
+// support for modular JavaScript. Modifications of any kind are likely to
+// result in unintended behavior. In most cases, you should edit the source
+// file of the module imported below. If you are absolutely certain you want
+// to take control of this code, you can remove this comment to prevent the
+// SDK from regenerating it. However, you will then be responsible for the
+// management of this code in your Fluent file.`
+
+type GlueCodeMeta = {
+    name: string
+    path: string
+    isDefault: boolean
+}
+
+function isGlueCode(string: StringShape): boolean {
+    return string.startsWith(GLUE_CODE_PREFIX)
+}
+
+function parseMeta(string: StringShape): GlueCodeMeta {
+    const result = string.getValue().match(GLUE_CODE_META_REGEX)
+    const [, name, isDefault, path] = result ?? []
+
+    if (typeof name !== 'string' || typeof isDefault !== 'string' || typeof path !== 'string') {
+        throw new Error(`Invalid glue code format: ${string.getValue()}`)
+    }
+
+    return { name, path, isDefault: isDefault === 'true' }
+}
+
+export class ModuleFunctionShape extends Shape {
+    constructor(
+        source: Source,
+        private readonly meta: GlueCodeMeta
+    ) {
+        super({ source })
+    }
+
+    override toString(
+        callExpressionProvider: (functionName: string) => string = (name) => `${name}()`,
+        defaultParams?: string[]
+    ): StringShape {
+        const { name, path, isDefault } = this.meta
+        const moduleParamNames = getFunctionParameters(this.getSource())
+        const variables = zip(defaultParams ?? [], moduleParamNames).map(
+            ([defaultParam, moduleParamName]) => defaultParam ?? moduleParamName
+        )
+        const moduleCallExpression = callExpressionProvider(name).replace('{{PARAMS}}', variables.join(', '))
+
+        const code = isDefault
+            ? `const ${name} = require('${path}').default;\n${moduleCallExpression};`
+            : `const { ${name} } = require('${path}');\n${moduleCallExpression};`
+
+        return Shape.from(this, `${GLUE_CODE_PREFIX} ${name};${isDefault};${path}${GLUE_CODE_WARNING}\n${code}`)
+            .asString()
+            .withContentType('cdata')
+    }
+
+    override equals(other: Shape): boolean {
+        if (other.is(ModuleFunctionShape)) {
+            return isEqual(this.meta, other.meta)
+        } else if (other.isString()) {
+            if (!isGlueCode(other)) {
+                return false
+            }
+
+            return isEqual(this.meta, parseMeta(other))
+        } else {
+            return super.equals(other)
+        }
+    }
+}
+
+/**
+ * Removes invalid control characters that would cause XML parsing errors.
+ */
+function sanitizeModuleContent(content: string): string {
+    return content.replace(INVALID_XML_CHARACTERS, '')
+}
+
+function getEmitOutput(
+    file: ts.SourceFile,
+    {
+        fs,
+        diagnostics,
+        config,
+        project,
+        compiler,
+    }: {
+        fs: FileSystem
+        diagnostics: Diagnostics
+        config: NowConfig
+        project: Project
+        compiler: Compiler
+    }
+): string | undefined {
+    const path = file.getFilePath()
+    if (!path.endsWith('.ts')) {
+        return file.getFullText()
+    }
+
+    const tsConfigPath = config.tsconfigPath ? project.resolvePath(config.tsconfigPath) : undefined
+    return compiler.compileModule(file, fs, diagnostics, tsConfigPath)
+}
+
+function getFunctionParameters(source: Source): string[] {
+    if (!(source instanceof ts.VariableDeclaration || source instanceof ts.FunctionDeclaration)) {
+        return []
+    }
+    const node = source as ts.VariableDeclaration | ts.VariableDeclaration
+
+    let functionNode: ts.FunctionDeclaration | ts.FunctionExpression | ts.ArrowFunction | undefined
+    if (ts.Node.isFunctionDeclaration(node)) {
+        functionNode = node
+    } else if (ts.Node.isVariableDeclaration(node) && node.getInitializerIfKind(ts.SyntaxKind.FunctionExpression)) {
+        functionNode = node.getInitializerIfKindOrThrow(ts.SyntaxKind.FunctionExpression)
+    } else if (ts.Node.isVariableDeclaration(node) && node.getInitializerIfKind(ts.SyntaxKind.ArrowFunction)) {
+        functionNode = node.getInitializerIfKindOrThrow(ts.SyntaxKind.ArrowFunction)
+    } else {
+        return []
+    }
+
+    return functionNode.getParameters().map((param) => param.getName()) ?? []
+}
+
+async function parseDeclaration(
+    node: ts.FunctionDeclaration | ts.VariableDeclaration,
+    { transform }: { transform: Transform }
+): Promise<Result<ModuleFunctionShape>> {
+    const result = await transform.toRecord(node.getSourceFile())
+    if (!result.success) {
+        return result
+    }
+
+    const record = result.value
+    if (record.getTable() !== 'sys_module') {
+        return { success: false }
+    }
+
+    return {
+        success: true,
+        value: new ModuleFunctionShape(node, {
+            name: node.getName() ?? 'functionModule',
+            path: record.get('path').asString().getValue(),
+            isDefault: node.isDefaultExport(),
+        }),
+    }
+}
+
+const dependencyIgnoreList = ['@servicenow/glide', '@servicenow/sdk']
+function isIgnoredDependency(name: string): boolean {
+    return dependencyIgnoreList.some((dependency) => name.startsWith(dependency))
+}
+
+function validateAndGetModuleSpecifier(name: string): { name: string; entry?: string } {
+    const scopedRegex = /@[a-z\d][\w\-.]+\/[a-z\d][\w\-.]*/gi
+    scopedRegex.test(name)
+    const idx = scopedRegex.lastIndex
+
+    const isSubPathImport = name.indexOf('/', idx)
+    if (isSubPathImport > 0) {
+        return { name: name.slice(0, isSubPathImport), entry: name.slice(isSubPathImport + 1) }
+    }
+    return { name }
+}
+
+function isValidRequireCall(callExpression: ts.CallExpression, requirePath: ts.StringLiteral): boolean {
+    const expression = callExpression.getExpression()
+    const isRequire = ts.Node.isIdentifier(expression) && expression.getText() === 'require'
+    const isRelativePath =
+        requirePath.getLiteralText().startsWith('../') || requirePath.getLiteralText().startsWith('./')
+    return isRequire && !isRelativePath
+}
+
+class ModuleDependencyShape extends Shape {
+    private readonly moduleName: string
+
+    constructor({
+        node,
+        moduleName,
+    }: {
+        node: ts.ImportDeclaration | ts.ExportDeclaration | ts.CallExpression
+        moduleName: string
+    }) {
+        super({ source: node })
+        this.moduleName = moduleName
+    }
+
+    getModuleName(): string {
+        return this.moduleName
+    }
+}
+
+// TODO: Need to have some invalidation mechanism. Maybe the plugin framework can provide plugins with a managed cache to use for stuff like this?
+const DEPENDENCY_CACHE: globalThis.Record<string, Result<ModuleDependencyShape>> = {}
+
+function parseModuleDependency(
+    node: ts.ImportDeclaration | ts.ExportDeclaration | ts.CallExpression
+): Result<ModuleDependencyShape> {
+    let moduleName: string | undefined
+    if (ts.Node.isImportDeclaration(node) && !node.isModuleSpecifierRelative()) {
+        moduleName = node.getModuleSpecifierValue()
+    } else if (ts.Node.isExportDeclaration(node) && node.hasModuleSpecifier() && !node.isModuleSpecifierRelative()) {
+        moduleName = node.getModuleSpecifierValue()
+    } else if (ts.Node.isCallExpression(node)) {
+        const args = node.getArguments()
+        const requirePath = args[0]
+        if (ts.Node.isStringLiteral(requirePath) && isValidRequireCall(node, requirePath)) {
+            moduleName = requirePath.getLiteralValue()
+        }
+    }
+
+    if (!moduleName || isIgnoredDependency(moduleName)) {
+        return { success: false }
+    }
+
+    const cached = DEPENDENCY_CACHE[moduleName]
+    if (cached && (!cached.success || !cached.value.getOriginalNode().wasForgotten())) {
+        return cached
+    }
+
+    return (DEPENDENCY_CACHE[moduleName] = {
+        success: true,
+        value: new ModuleDependencyShape({ node, moduleName }),
+    })
+}
+
+function generateSBOMContent(context: RecordContext) {
+    const { database } = context
+    const moduleRecords = database.query('sys_module', { external_source: true })
+    const sbomBuilder = new SBOMBuilder()
+    for (const mod of moduleRecords) {
+        const modulePath = mod.get('path').asString().getValue()
+        if (modulePath.endsWith('/package.json')) {
+            const moduleContent = mod.get('content').asString().getValue()
+            sbomBuilder.addPackageJson(modulePath, moduleContent)
+        }
+    }
+    return sbomBuilder.generateSBOM(context)
+}
+
+export const ServerModulePlugin = Plugin.create({
+    name: 'ServerModulePlugin',
+    files: [
+        {
+            entryPoint: true,
+            matcher: /[/\\]package\.json$/,
+        },
+        {
+            entryPoint: true,
+            matcher: (path, { config, project }) =>
+                !path.endsWith('.test.ts') &&
+                !path.endsWith('.test.js') &&
+                project.isInDir(config.serverModulesDir, path),
+        },
+    ],
+    records: {
+        sys_module: {
+            toFile(record, context) {
+                const { config } = context
+                // If the record is not the sbom, we defer to using the record plugin
+                if (!record.get('path').ifString()?.endsWith('/bom.json')) {
+                    return {
+                        success: false,
+                    }
+                }
+                const sbomContent = generateSBOMContent(context)
+
+                return {
+                    success: true,
+                    value: {
+                        name: `sys_module_${record.getId().getValue()}.xml`,
+                        category: record.getInstallCategory(),
+                        content: `<?xml version="1.0"?>
+<record_update>
+  <sys_module action="INSERT_OR_UPDATE">
+    <sys_id>${record.getId().getValue()}</sys_id>
+    <sys_scope display_value="${config.scope}">${config.scopeId}</sys_scope>
+    <path>${record.get('path').getValue()}</path>
+    <external_source>${record.get('external_source').getValue()}</external_source>
+    <content><![CDATA[${StringShape.escapeCdataTags(sanitizeModuleContent(sbomContent))}]]></content>
+  </sys_module>
+</record_update>`,
+                    },
+                }
+            },
+            toShape(record) {
+                return { success: true, value: Shape.noOp(record) }
+            },
+        },
+    },
+    shapes: [
+        {
+            shape: SourceFileShape,
+            fileTypes: ['module'],
+            toRecord(file, { factory, fs, diagnostics, project, config, packageJson, compiler }) {
+                const path = file.getPath()
+                if (!path.startsWith(project.resolvePath(config.serverModulesDir))) {
+                    return { success: false }
+                }
+
+                const mappedPath = applyPathMappings(path, config.modulePaths)
+                const resolvedPath = project.resolvePath(mappedPath)
+                const mappedFile =
+                    compiler.getSourceFile(resolvedPath) ?? compiler.addSourceFileAtPathIfExists(resolvedPath)
+
+                if (!mappedFile) {
+                    diagnostics.error(
+                        file,
+                        `Module path was mapped from '${path}' to '${resolvedPath}' but no file exists at the mapped location.`
+                    )
+                    return { success: false }
+                }
+
+                const content = getEmitOutput(mappedFile, { fs, diagnostics, config, project, compiler })
+                if (!content) {
+                    return { success: false }
+                }
+
+                const relativePath = pathModule.relative(project.getRootDir(), resolvedPath)
+                const sysModulePath = pathModule.join(config.scope, packageJson.name, packageJson.version, relativePath)
+
+                return {
+                    success: true,
+                    value: factory.createRecord({
+                        source: file,
+                        table: 'sys_module',
+                        explicitId: relativePath.replaceAll(/[./\\]/g, '_'),
+                        properties: {
+                            // Module resolution at runtime requires this format
+                            path: sysModulePath,
+                            content: Shape.from(file, sanitizeModuleContent(content))
+                                .asString()
+                                .withContentType('cdata'),
+                            external_source: false,
+                            sys_name: sysModulePath,
+                        },
+                    }),
+                }
+            },
+        },
+        {
+            shape: ModuleDependencyShape,
+            fileTypes: ['module'],
+            // TODO: When managed cache is provided to plugins, cache dependencies that were already handled to avoid reprocessing
+            async toRecord(shape, { packageJson, diagnostics, fs, logger, project, factory, config }) {
+                const dependencies = packageJson.dependencies ?? {}
+                const { name, entry } = validateAndGetModuleSpecifier(shape.getModuleName())
+
+                const version = dependencies[name]
+                if (!version) {
+                    diagnostics.error(shape, `Dependency ${name} is not found in package.json`)
+                    return { success: false }
+                }
+
+                const id = `${name}@${version}`
+                const repack = await RepackService.create(logger, fs, project.getRootDir())
+                const dependencyNodes = await repack.execute({
+                    id,
+                    entry: entry ? [entry] : ['.'],
+                })
+
+                if (!dependencyNodes) {
+                    throw new Error(`Failed to build dependency ${id}`)
+                }
+
+                const modules: { id: string; path: string; content: string }[] = []
+                const { Lint } = await import('../repack/lint/index.js')
+                for (const node of dependencyNodes) {
+                    const { packagePath, files, updatedManifest } = node
+                    const { name, version } = updatedManifest
+
+                    for (const file of files) {
+                        const fileContent = fs.readFileSync(pathModule.join(packagePath, file)).toString('utf-8')
+                        if (/(.js|.cjs|.mjs)$/.test(pathModule.extname(file))) {
+                            const result = await new Lint().check(fileContent)
+                            if (result) {
+                                logger.warn(`Use of unsupported APIs detected in npm dependency ${name}`)
+                                logger.warn(result)
+                            }
+                        }
+
+                        modules.push({
+                            id: `${name}@${version}/${file}`,
+                            path: pathModule.join(config.scope, name, version, file),
+                            content: fileContent,
+                        })
+                    }
+                }
+
+                const [first, ...rest] = modules.map((m) => {
+                    return factory.createRecord({
+                        source: shape,
+                        table: 'sys_module',
+                        explicitId: m.id,
+                        properties: {
+                            path: m.path,
+                            content: Shape.from(shape, sanitizeModuleContent(m.content))
+                                .asString()
+                                .withContentType('cdata'),
+                            external_source: true,
+                            sys_name: m.path,
+                        },
+                    })
+                })
+
+                if (!first) {
+                    return { success: false }
+                }
+
+                return {
+                    success: true,
+                    value: first.with(...rest),
+                }
+            },
+        },
+    ],
+    nodes: [
+        { node: 'CallExpression', fileTypes: ['module'], entryPoint: true, toShape: parseModuleDependency },
+        { node: 'ImportDeclaration', fileTypes: ['module'], entryPoint: true, toShape: parseModuleDependency },
+        { node: 'ExportDeclaration', fileTypes: ['module'], entryPoint: true, toShape: parseModuleDependency },
+        { node: 'FunctionDeclaration', fileTypes: ['module'], toShape: parseDeclaration },
+        {
+            node: 'FunctionExpression',
+            fileTypes: ['module'],
+            toShape(node, context) {
+                const parent = node.getParentIfKind(ts.SyntaxKind.VariableDeclaration)
+                if (!parent) {
+                    return { success: false }
+                }
+
+                return parseDeclaration(parent, context)
+            },
+        },
+        {
+            node: 'ArrowFunction',
+            fileTypes: ['module'],
+            toShape(node, context) {
+                const parent = node.getParentIfKind(ts.SyntaxKind.VariableDeclaration)
+                if (!parent) {
+                    return { success: false }
+                }
+
+                return parseDeclaration(parent, context)
+            },
+        },
+    ],
+})

package/src/server-module-plugin/sbom-builder.ts

@@ -0,0 +1,183 @@
+import { path, FileSystem } from '@servicenow/sdk-build-core'
+import type { RecordContext } from '@servicenow/sdk-build-core'
+import { JsonSerializer, JSON as CycloneDXJSON } from '@cyclonedx/cyclonedx-library/Serialize'
+import { ExternalReference, type Metadata } from '@cyclonedx/cyclonedx-library/Models'
+import { ExternalReferenceType } from '@cyclonedx/cyclonedx-library/Enums'
+import * as CDXUtils from '@cyclonedx/cyclonedx-library/Utils'
+import * as Builders from '@cyclonedx/cyclonedx-library/Builders'
+import * as Enums from '@cyclonedx/cyclonedx-library/Enums'
+import * as Factories from '@cyclonedx/cyclonedx-library/Factories'
+import * as Models from '@cyclonedx/cyclonedx-library/Models'
+import * as Spec from '@cyclonedx/cyclonedx-library/Spec'
+import { PackageURL } from 'packageurl-js'
+
+export class SBOMBuilder {
+    /**
+     * The set of package.json files that are used to generate the SBOM.
+     * @private
+     */
+    private packageJsonFiles: Map<string, string> = new Map()
+
+    /**
+     * The package.json of the SDK build plugins. This is used to generate the tool component in the SBOM.
+     */
+    private toolPkgJsonDefault = {
+        name: '@servicenow/sdk-build-plugins',
+        version: '0.0.1',
+        author: 'ServiceNow',
+    }
+
+    /**
+     * Add a package.json file to the set of package.json files that are used to generate the SBOM for the application.
+     * @param file the path to the package.json file
+     */
+    public addPackageJson(file: string, content: string) {
+        //We don't care about the package.json path but it least keeps the content unique
+        this.packageJsonFiles.set(file, content)
+    }
+
+    /**
+     * Generate a CycloneDX SBOM for the application.
+     * @param context the build context
+     */
+    public generateSBOM(context: RecordContext): string {
+        const cdxExternalReferenceFactory = new Factories.FromNodePackageJson.ExternalReferenceFactory()
+        const cdxLicenseFactory = new Factories.LicenseFactory()
+        const cdxPurlFactory = new Factories.FromNodePackageJson.PackageUrlFactory('npm')
+        const cdxComponentBuilder = new Builders.FromNodePackageJson.ComponentBuilder(
+            cdxExternalReferenceFactory,
+            cdxLicenseFactory
+        )
+
+        const jsonSerializer = new JsonSerializer(new CycloneDXJSON.Normalize.Factory(Spec.Spec1dot6))
+        const sdkJson = this.getSdkInfo(context)
+        const toolComponent = cdxComponentBuilder.makeComponent(sdkJson, Enums.ComponentType.Library)
+
+        const rootComponent = cdxComponentBuilder.makeComponent(context.packageJson, Enums.ComponentType.Application)
+
+        let qualifiers: Record<string, string> | undefined
+        if (context.config.scopeId) {
+            qualifiers = {
+                sys_scope_id: context.config.scopeId,
+            }
+        }
+        const purl = new PackageURL(
+            'servicenow',
+            undefined,
+            context.packageJson.name,
+            context.packageJson.version,
+            qualifiers,
+            undefined
+        )
+        if (rootComponent && purl) {
+            rootComponent.purl = purl
+            rootComponent.bomRef.value = purl.toString()
+        }
+        const metadata = new Models.Metadata({
+            timestamp: new Date(),
+            component: rootComponent,
+        })
+        toolComponent && metadata.tools.components.add(toolComponent)
+        this.addExternalReferences(metadata)
+
+        const bom = new Models.Bom({
+            metadata,
+            serialNumber: CDXUtils.BomUtility.randomSerialNumber(),
+        })
+
+        for (const [, content] of this.packageJsonFiles.entries()) {
+            const data = content
+            const json = JSON.parse(data)
+            if (json) {
+                const { name, version } = json
+                if (!(context.packageJson.name === name && context.packageJson.version === version)) {
+                    const component = cdxComponentBuilder.makeComponent(json, Enums.ComponentType.Application)
+                    if (component) {
+                        this.registerPackageUrlOnComponent(component, cdxPurlFactory)
+                        bom.components.add(component)
+                        rootComponent?.dependencies.add(component.bomRef)
+                    }
+                }
+            }
+        }
+
+        const nowDir: string = path.resolve(context.project.getRootDir(), '.now')
+        FileSystem.ensureDirSync(context.fs, nowDir)
+        const bomPath = path.resolve(nowDir, 'bom.json')
+        const bomContent: string = jsonSerializer.serialize(bom, { space: 2, sortLists: true })
+        context.fs.writeFileSync(bomPath, bomContent)
+
+        return bomContent
+    }
+
+    /**
+     * Helper method to get the JSON needed to build the tool component for the @servicenow/sdk-build-plugins package.
+     * @param context the build context
+     * @private
+     */
+    public getSdkInfo(context: RecordContext): { name: string; version: string; author: string } {
+        try {
+            //TODO can we retrieve the package.json's name and version from the context for the sdk-build-plugins package?
+            const packageJsonPath = path.join(
+                context.project.getRootDir(),
+                'node_modules',
+                '@servicenow/sdk-build-plugins',
+                'package.json'
+            )
+            const packageJson = JSON.parse(context.fs.readFileSync(packageJsonPath, { encoding: 'utf-8' }))
+            return {
+                name: packageJson.name,
+                version: packageJson.version,
+                author: 'ServiceNow',
+            }
+        } catch (error) {
+            // ignore
+        }
+        return this.toolPkgJsonDefault
+    }
+
+    /**
+     * Helper method to generate a package url from a component and register it on the component. No idea why the base
+     * component builder doesn't do this automatically.
+     * @param component the component to register the package url on
+     * @param purlFactory the package url factory
+     */
+    private registerPackageUrlOnComponent(
+        component: Models.Component | undefined,
+        purlFactory: Factories.FromNodePackageJson.PackageUrlFactory
+    ) {
+        if (component) {
+            const purl = purlFactory.makeFromComponent(component)
+            if (purl) {
+                component.purl = purl
+                component.bomRef.value = purl.toString()
+            }
+        }
+    }
+
+    /**
+     * Add external references to the root component that are retrieved from common Jenkins environment variables.
+     * @param metadata the bom metadata reference
+     * @private
+     */
+    private addExternalReferences(metadata: Metadata) {
+        const buildUrl = process.env['BUILD_URL']
+        if (buildUrl) {
+            const buildSystemExRef = new ExternalReference(buildUrl, ExternalReferenceType.BuildSystem, {
+                comment: 'As obtained from the environment variable BUILD_URL',
+            })
+            if (metadata.component && metadata.component.externalReferences) {
+                metadata.component.externalReferences.add(buildSystemExRef)
+            }
+        }
+        const vcsUrl = process.env['GIT_URL']
+        if (vcsUrl) {
+            const vcsExRef = new ExternalReference(vcsUrl, ExternalReferenceType.VCS, {
+                comment: 'As obtained from the environment variable GIT_URL',
+            })
+            if (metadata.component && metadata.component.externalReferences) {
+                metadata.component.externalReferences.add(vcsExRef)
+            }
+        }
+    }
+}