@servicenow/sdk-build-plugins 3.0.3 → 4.0.1

package/src/static-content-plugin.ts

@@ -0,0 +1,254 @@
+import MD5 from 'md5.js'
+import * as mime from 'mime-types'
+import { Shape, Plugin, type Record, unloadBuilder, gzipSync } from '@servicenow/sdk-build-core'
+import { path as pathModule } from '@servicenow/sdk-build-core'
+import { INVALID_XML_CHARACTERS } from './utils'
+
+// based on tectonic code for attachments
+// https://code.devsnc.com/dev/sn-tectonic/blob/b3ab42ce742158cb5a0d00efd540b97eeafcbdd7/core/metadata-transform-san-diego/utils/index.js#L66
+const MAX_INLINE_CONTENT_SIZE = 4194304
+// https://code.devsnc.com/dev/sn-tectonic/blob/b3ab42ce742158cb5a0d00efd540b97eeafcbdd7/core/metadata-transform-san-diego/attachment/index.js#L7
+const CHUNK_SIZE = 933336
+
+const base64 = (data: Uint8Array | Buffer | string): string => Buffer.from(data).toString('base64')
+
+const chunkData = (data: string): string[] => {
+    const numChunks = Math.ceil(data.length / CHUNK_SIZE)
+    const chunks: string[] = new Array(numChunks)
+    for (let chunk = 0, offset = 0; chunk < numChunks; chunk += 1, offset += CHUNK_SIZE) {
+        chunks[chunk] = data.substring(offset, CHUNK_SIZE)
+    }
+
+    return chunks
+}
+
+const generateId = (...parts: Array<string | number>): string => new MD5().update(parts.join(':')).digest('hex')
+
+// based on tectonic code for validating XML content
+// https://code.devsnc.com/dev/sn-tectonic/blob/b3ab42ce742158cb5a0d00efd540b97eeafcbdd7/core/metadata-transform-san-diego/utils/index.js#L51-L63
+function validateCdata(data: string): boolean {
+    if (data && data.includes('<![CDATA[')) {
+        return false
+    }
+    if (data.match(INVALID_XML_CHARACTERS)) {
+        return false
+    }
+    return true
+}
+
+const multipleRecordsToFile = (mainRecord, { config }) => {
+    const recordBuilder = unloadBuilder(config)
+    for (const record of mainRecord.flat()) {
+        const builder = recordBuilder.record(record)
+
+        record
+            .entries()
+            .sort(([a], [b]) => a.localeCompare(b)) // Sort keys to make outputs more deterministic
+            .forEach(([prop, shape]) => builder.field(prop, shape))
+    }
+    return {
+        success: true,
+        value: {
+            name: `${mainRecord.getTable()}_${mainRecord.getId().getValue()}.xml`,
+            category: mainRecord.getInstallCategory(),
+            content: recordBuilder.end(),
+        },
+    }
+}
+
+const attachmentRelationships = {
+    sys_attachment: {
+        via: 'table_sys_id',
+        descendant: true,
+        relationships: {
+            sys_attachment_doc: {
+                via: 'sys_attachment',
+                descendant: true,
+            },
+        },
+    },
+}
+
+const toNoOpShape = (record) => {
+    return { success: true, value: Shape.noOp(record) }
+}
+
+export const StaticContentPlugin = Plugin.create({
+    name: 'StaticContentPlugin',
+    records: {
+        // TODO: remove when we have generic attachment support
+        sys_attachment: {
+            toShape: toNoOpShape,
+        },
+        // TODO: remove when we have generic attachment support
+        sys_attachment_doc: {
+            toShape: toNoOpShape,
+        },
+        sys_ux_lib_asset: {
+            relationships: attachmentRelationships,
+            toShape: toNoOpShape,
+            toFile: multipleRecordsToFile,
+        },
+        db_image: {
+            relationships: attachmentRelationships,
+            toShape: toNoOpShape,
+            toFile: multipleRecordsToFile,
+        },
+        sys_ux_theme_asset: {
+            relationships: attachmentRelationships,
+            toShape: toNoOpShape,
+            toFile: multipleRecordsToFile,
+        },
+    },
+    files: [
+        {
+            entryPoint: true,
+            matcher: (path, { config, project }) => project.isInDir(config.staticContentDir, path),
+            toRecord(file, { project, config, factory, logger, fs }) {
+                const { path } = file
+                const relativePath = pathModule.relative(project.resolvePath(config.staticContentDir), path)
+                const ext = pathModule.extname(relativePath)
+                const normalizedExt = ext.replace('dbx', '')
+                const mimeType = mime.lookup(normalizedExt)
+                // TODO: support binary file content so we don't have to read the contents again here
+                const rawFileContent = fs.readFileSync(path)
+                const stringFileContent = file.content
+                const hash = new MD5().update(rawFileContent).digest('hex')
+                let tableName: string | undefined
+                let assetName: string | undefined
+                const uxAssetProperties = {
+                    sys_package: config.scopeId,
+                    mime_type: mimeType,
+                    checksum: hash,
+                }
+                const recordProperties = {}
+                let useAttachment = false
+                const logSkippedWithReason = (reason: string) => {
+                    // it would be nice to use diagnostics.warn here, but we don't have a compatible source for this file
+                    // so we use logger instead, similar to what the server-module-plugin does
+                    // https://code.devsnc.com/dev/fluent/blob/fbb5dfcf19dd3bf1ad8700de939a12817b16d0ad/packages/build-plugins/src/server-module-plugin.ts#L367-L368
+                    logger.warn(`Skipping packaging of '${relativePath}'. ${reason}`)
+                }
+                if (rawFileContent.length > MAX_INLINE_CONTENT_SIZE || !validateCdata(stringFileContent)) {
+                    useAttachment = true
+                }
+                Object.assign(uxAssetProperties, {
+                    is_attachment: useAttachment,
+                    content: useAttachment
+                        ? undefined
+                        : Shape.from(file, stringFileContent).asString().withContentType('cdata'),
+                })
+                if (mimeType === 'text/html') {
+                    // This content will be handled by the UiPage referencing it
+                } else if (mimeType === 'application/javascript') {
+                    tableName = 'sys_ux_lib_asset'
+                    assetName = pathModule.join(
+                        config.scope,
+                        relativePath.substring(0, relativePath.length - ext.length)
+                    )
+                    Object.assign(recordProperties, {
+                        ...uxAssetProperties,
+                        name: assetName,
+                        category: 'component',
+                        es_module: true,
+                        engine: 21,
+                    })
+                } else if (normalizedExt === '.map') {
+                    tableName = 'sys_ux_lib_asset'
+                    assetName = pathModule.join(config.scope, relativePath.replace('dbx', ''))
+                    Object.assign(recordProperties, {
+                        ...uxAssetProperties,
+                        name: assetName,
+                        category: 'source_map',
+                        es_module: false,
+                    })
+                } else if (!mimeType) {
+                    logSkippedWithReason('Unknown content type.')
+                } else if (mimeType.indexOf('image/') === 0) {
+                    useAttachment = true
+                    tableName = 'db_image'
+                    assetName = pathModule.join(config.scope, relativePath)
+                    Object.assign(recordProperties, {
+                        name: assetName,
+                        size_bytes: rawFileContent.length,
+                        active: true,
+                    })
+                } else if (mimeType.indexOf('font/') === 0 || mimeType === 'text/css') {
+                    useAttachment = true
+                    tableName = 'sys_ux_theme_asset'
+                    assetName = relativePath
+                    Object.assign(recordProperties, {
+                        name: assetName,
+                        // this is a required field, and the only other value allowed is 'image'
+                        // it doesn't affect the content type of the response
+                        category: 'font',
+                    })
+                } else {
+                    logSkippedWithReason(`Unsupported content type ${mimeType}.`)
+                }
+                if (tableName && assetName) {
+                    const mainRecord = factory.createRecord({
+                        source: file,
+                        table: tableName,
+                        explicitId: assetName,
+                        properties: recordProperties,
+                    })
+                    const mainRecordId = mainRecord.getId().getValue()
+                    const overrideMainRecordProperties = {}
+                    const attachmentRecords: Record[] = []
+                    if (useAttachment) {
+                        const attachmentRecordId = generateId(mainRecordId, 'sys_attachment', assetName)
+                        const zipEncodedContent = gzipSync(rawFileContent, { mtime: 0 })
+                        const attachmentRecord = factory.createRecord({
+                            source: file,
+                            table: 'sys_attachment',
+                            properties: {
+                                sys_id: attachmentRecordId,
+                                file_name: assetName,
+                                hash: hash,
+                                table_name: tableName,
+                                table_sys_id: mainRecordId,
+                                size_bytes: rawFileContent.length,
+                                size_compressed: zipEncodedContent.length,
+                                chunk_size_bytes: CHUNK_SIZE,
+                                compressed: true,
+                                content_type: mimeType,
+                                state: 'available',
+                            },
+                        })
+                        attachmentRecords.push(attachmentRecord)
+                        if (tableName === 'db_image') {
+                            Object.assign(overrideMainRecordProperties, { image: attachmentRecordId })
+                        }
+                        const base64Encoding = base64(zipEncodedContent)
+                        const chunks = chunkData(base64Encoding)
+                        for (let i = 0; i < chunks.length; i++) {
+                            const sysId = generateId(mainRecordId, 'sys_attachment_doc', assetName, i)
+                            attachmentRecords.push(
+                                factory.createRecord({
+                                    source: file,
+                                    table: 'sys_attachment_doc',
+                                    properties: {
+                                        sys_id: sysId,
+                                        data: chunks[i],
+                                        position: i,
+                                        length: chunks[i]!.length,
+                                        sys_attachment: attachmentRecordId,
+                                    },
+                                })
+                            )
+                        }
+                    }
+
+                    const mergedRecord = mainRecord.merge(overrideMainRecordProperties)
+
+                    return {
+                        success: true,
+                        value: mergedRecord.with(...attachmentRecords),
+                    }
+                }
+                return { success: false }
+            },
+        },
+    ],
+})